├── .gitignore ├── CClash.Tests ├── ArgumentsTest.cs ├── CClashTests.csproj ├── CClashTestsFixtureSetup.cs ├── CompilerCacheTest.cs ├── CompilerTest.cs ├── FileCacheDatabaseTest.cs ├── FileCacheTest.cs ├── FileCacheTestBase.cs ├── HashingTest.cs ├── Properties │ └── AssemblyInfo.cs ├── app.config ├── packages.config └── test-sources │ ├── another.h │ ├── cl.txt │ ├── compiler1.resp │ ├── compiler2.resp │ ├── exists.c │ ├── hello.c │ ├── hello.cc │ ├── hello.h │ ├── hello2.h │ ├── inc with spaces │ └── another.h │ ├── x.txt │ └── y.txt ├── CClash ├── App.config ├── ArgumentUtils.cs ├── CClash.csproj ├── CClash.csproj.user ├── CClashErrorException.cs ├── CClashMessage.cs ├── CClashServer.cs ├── CClashServerClient.cs ├── CClashServerNotReadyException.cs ├── CClashWarningException.cs ├── CacheInformation.cs ├── CacheLockType.cs ├── CacheManifest.cs ├── CacheMode.cs ├── CacheSessionContext.cs ├── Compiler.cs ├── CompilerCacheBase.cs ├── DirectCompilerCache.cs ├── DirectCompilerCacheServer.cs ├── DirectoryWatcher.cs ├── ExtensionMethods.cs ├── FileCacheBase.cs ├── FileCacheDatabase.cs ├── FileCacheDatabaseWriteStream.cs ├── FileCacheStore.cs ├── FileUtils.cs ├── HashUtil.cs ├── ICacheInfo.cs ├── ICompiler.cs ├── ICompilerCache.cs ├── IFileCacheStore.cs ├── Logging.cs ├── NullCompilerCache.cs ├── ParseTrackerFile.cs ├── ProcessUtils.cs ├── Program.cs ├── Properties │ └── AssemblyInfo.cs ├── Settings.cs ├── StatOutputs.cs └── packages.config ├── README.md ├── benchmark ├── get_vc_envs.bat ├── test_performance.py └── test_performance_openssl.py ├── build_openssl.py ├── cclash.sln ├── cct ├── App.config ├── Program.cs ├── Properties │ └── AssemblyInfo.cs └── cct.csproj └── packages └── repositories.config /.gitignore: -------------------------------------------------------------------------------- 1 | # git ignore file 2 | *.dll 3 | *.pdb 4 | *.exe 5 | *.mdf 6 | *.vsp* 7 | /*.suo 8 | CClash.Tests/bin/* 9 | CClash.Tests/obj/* 10 | cclash/obj/* 11 | cclash/bin/* 12 | UnitTests/* 13 | packages/* 14 | TestResults/* 15 | CClash.Tests/dummy/* 16 | testprogram/bin/* 17 | testprogram/bin/* 18 | testprogram/obj/* 19 | testserver/bin/* 20 | testserver/bin/* 21 | testserver/obj/* 22 | Installer/Installer/Express/DVD-5/LogFiles/*.* 23 | Installer/Installer/Express/DVD-5/Reports/*.* 24 | Installer/Installer/Express/SingleImage/LogFiles/*.* 25 | Installer/Installer/Express/SingleImage/Reports/*.* 26 | -------------------------------------------------------------------------------- /CClash.Tests/ArgumentsTest.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Text; 5 | using System.Threading.Tasks; 6 | using NUnit.Framework; 7 | 8 | namespace CClash.Tests 9 | { 10 | [TestFixture] 11 | public class ArgumentsTest 12 | { 13 | [Test] 14 | public void EscapeSpaces() 15 | { 16 | Assert.AreEqual("\\\"foo\\\" bar \"baz spaces\" \"bar spaces\"", 17 | ArgumentUtils.JoinAguments(new string[] { "\"foo\"", "bar", "baz spaces", "bar spaces" })); 18 | 19 | Assert.AreEqual("\" \" \"x x x\"", 20 | ArgumentUtils.JoinAguments(new string[] { " ", "x x x" })); 21 | } 22 | 23 | [Test] 24 | public void EscapeSlashes() 25 | { 26 | Assert.AreEqual("foo\\bar\\file.c", 27 | ArgumentUtils.JoinAguments(new string[] { "foo\\bar\\file.c" })); 28 | } 29 | 30 | [Test] 31 | public void EscapeQuotes() 32 | { 33 | Assert.AreEqual(new string[] {"/D"}, 34 | ArgumentUtils.FixupArgs(new string[]{"/D"})); 35 | } 36 | 37 | 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /CClash.Tests/CClashTests.csproj: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | 5 | Debug 6 | AnyCPU 7 | {B4A548CB-16A3-4FE9-A0D9-76A66182E0B1} 8 | Library 9 | Properties 10 | CClash.Tests 11 | CClash.Tests 12 | v4.5 13 | 512 14 | 15 | 16 | true 17 | full 18 | false 19 | bin\Debug\ 20 | DEBUG;TRACE 21 | prompt 22 | 4 23 | x86 24 | 25 | 26 | pdbonly 27 | true 28 | bin\Release\ 29 | TRACE 30 | prompt 31 | 4 32 | 33 | 34 | 35 | ..\packages\EntityFramework.6.0.0\lib\net45\EntityFramework.dll 36 | 37 | 38 | ..\packages\EntityFramework.6.0.0\lib\net45\EntityFramework.SqlServer.dll 39 | 40 | 41 | ..\packages\NUnit.2.6.3\lib\nunit.framework.dll 42 | 43 | 44 | 45 | 46 | 47 | False 48 | ..\packages\System.Data.SQLite.Core.1.0.99.0\lib\net45\System.Data.SQLite.dll 49 | 50 | 51 | ..\packages\System.Data.SQLite.EF6.1.0.99.0\lib\net45\System.Data.SQLite.EF6.dll 52 | 53 | 54 | ..\packages\System.Data.SQLite.Linq.1.0.99.0\lib\net45\System.Data.SQLite.Linq.dll 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | 75 | 76 | 77 | 78 | Always 79 | 80 | 81 | Always 82 | 83 | 84 | 85 | 86 | {7bcf13a4-c4c9-493d-898b-e89aefc93c1a} 87 | CClash 88 | 89 | 90 | 91 | 92 | Always 93 | 94 | 95 | Always 96 | 97 | 98 | Always 99 | 100 | 101 | Always 102 | 103 | 104 | Always 105 | 106 | 107 | Always 108 | 109 | 110 | 111 | 112 | 113 | 114 | This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}. 115 | 116 | 117 | 118 | 125 | -------------------------------------------------------------------------------- /CClash.Tests/CClashTestsFixtureSetup.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Text; 5 | using System.Threading.Tasks; 6 | using NUnit.Framework; 7 | 8 | namespace CClash.Tests { 9 | [SetUpFixture] 10 | public class CClashTestsFixtureSetup 11 | { 12 | public static string InitialDir = null; 13 | 14 | [SetUp] 15 | public void Init() { 16 | if (InitialDir == null) 17 | InitialDir = Environment.CurrentDirectory; 18 | Environment.CurrentDirectory = "c:\\"; 19 | } 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /CClash.Tests/CompilerTest.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Text; 5 | using System.Threading.Tasks; 6 | using System.IO; 7 | using NUnit.Framework; 8 | 9 | using CClash; 10 | 11 | namespace CClash.Tests 12 | { 13 | [TestFixture] 14 | public class CompilerTest 15 | { 16 | public const string CompilerPath = "C:\\Program Files (x86)\\Microsoft Visual Studio 11.0\\VC\\bin\\cl.exe"; 17 | static bool setpaths = false; 18 | 19 | public static string InitialDir { 20 | get { 21 | return CClashTestsFixtureSetup.InitialDir; 22 | } 23 | } 24 | 25 | public static void SetEnvs() 26 | { 27 | Environment.SetEnvironmentVariable("CCLASH_DISABLED", null); 28 | Environment.SetEnvironmentVariable("CCLASH_ATTEMPT_PDB_CACHE", null); 29 | Environment.SetEnvironmentVariable("CCLASH_PPMODE", null); 30 | Environment.SetEnvironmentVariable("CCLASH_DISABLE_WHEN_VAR", null); 31 | Environment.SetEnvironmentVariable("CCLASH_ENABLE_WHEN_VAR", null); 32 | Environment.SetEnvironmentVariable("CCLASH_SLOWOBJ_TIMEOUT", "3000"); 33 | Environment.SetEnvironmentVariable("CCLASH_Z7_OBJ", null); 34 | if (!setpaths) 35 | { 36 | Environment.SetEnvironmentVariable("PATH", 37 | @"C:\Program Files (x86)\Microsoft Visual Studio 11.0\VC\bin;" 38 | + Environment.GetEnvironmentVariable("PATH")); 39 | Environment.SetEnvironmentVariable("PATH", 40 | @"C:\Program Files (x86)\Microsoft Visual Studio 11.0\Common7\IDE;" 41 | + Environment.GetEnvironmentVariable("PATH")); 42 | Environment.SetEnvironmentVariable("INCLUDE", 43 | @"C:\Program Files (x86)\Microsoft Visual Studio 11.0\VC\INCLUDE;C:\Program Files (x86)\Microsoft Visual Studio 11.0\VC\ATLMFC\INCLUDE;C:\Program Files (x86)\Windows Kits\8.0\include\shared;C:\Program Files (x86)\Windows Kits\8.0\include\um;C:\Program Files (x86)\Windows Kits\8.0\include\winrt;c:\stuff\nonsense\;c:\comp\1;c:\comp2;c:\comp3;c:\foo"); 44 | setpaths = true; 45 | } 46 | } 47 | 48 | [SetUp] 49 | public void Init() 50 | { 51 | SetEnvs(); 52 | Environment.CurrentDirectory = "c:\\"; 53 | } 54 | 55 | [TearDown] 56 | public void Down() 57 | { 58 | SetEnvs(); 59 | Environment.CurrentDirectory = "c:\\"; 60 | } 61 | 62 | void EnsureDeleted(string file) 63 | { 64 | if ( !string.IsNullOrEmpty(file) && FileUtils.Exists(file)) 65 | File.Delete(file); 66 | } 67 | 68 | [Test] 69 | [TestCase("exists.obj", "/c", "test-sources\\exists.c")] 70 | [TestCase("whatever.obj", "/c", "test-sources\\exists.c", "/Fowhatever.obj")] 71 | [TestCase("test-sources.obj", "/c", "test-sources\\exists.c", "/Fotest-sources")] 72 | [TestCase("test-sources\\exists.obj", "/c", "test-sources\\exists.c", "/Fotest-sources\\")] 73 | public void CompilerArgumentParsing(string objfile, params string[] argv) 74 | { 75 | var c = new Compiler(); 76 | c.SetWorkingDirectory(InitialDir); 77 | c.SetEnvironment(Compiler.GetEnvironmentDictionary()); 78 | c.SetWaitForSlowObject(3000); 79 | var sbo = new StringBuilder(); 80 | var sbe = new StringBuilder(); 81 | Assert.IsTrue(c.ProcessArguments(argv)); 82 | Assert.IsFalse(c.Linking); 83 | Assert.IsTrue(c.SingleSource); 84 | Assert.IsNotNullOrEmpty(c.ObjectTarget); 85 | Assert.IsFalse(c.PrecompiledHeaders); 86 | Assert.AreNotEqual(c.SingleSourceFile, c.ObjectTarget); 87 | Assert.AreEqual(Path.Combine(InitialDir, objfile), c.ObjectTarget); 88 | 89 | EnsureDeleted(c.ObjectTarget); 90 | EnsureDeleted(c.PdbFile); 91 | 92 | c.CompilerExe = CompilerPath; 93 | c.SetWorkingDirectory(InitialDir); 94 | c.SetEnvironment(Compiler.GetEnvironmentDictionary()); 95 | 96 | var stderr = new StringBuilder(); 97 | var stdout = new StringBuilder(); 98 | 99 | var ec = c.InvokeCompiler( 100 | c.CommandLine, 101 | (x) => { stderr.AppendLine(x); }, 102 | (x) => { stdout.AppendLine(x); }, 103 | false, null); 104 | 105 | Assert.AreEqual(0, ec); 106 | 107 | Assert.IsTrue(File.Exists(c.ObjectTarget)); 108 | } 109 | 110 | [Test] 111 | [TestCase("/c", "test-sources\\exists.c", "/Zi")] 112 | [TestCase("/c", "test-sources\\exists.c", "/Zi", "/Fowhatever.obj")] 113 | [TestCase("/c", "test-sources\\exists.c", "/Zi", "/Fotest-sources")] 114 | public void ParseUnSupportedPdbArgs(params string[] argv) 115 | { 116 | var c = new Compiler(); 117 | c.SetWorkingDirectory(InitialDir); 118 | c.CompilerExe = CompilerPath; 119 | Assert.IsFalse(c.ProcessArguments(argv)); 120 | } 121 | 122 | [TestCase("/c", "test-sources\\exists.c", "/Zi", "/Fdtest-sources\\stuff.pdb")] 123 | public void ParseSupportedPdbArgs(params string[] argv) 124 | { 125 | Assert.IsFalse(Settings.AttemptPDBCaching); 126 | var c = new Compiler(); 127 | c.SetWorkingDirectory(InitialDir); 128 | c.CompilerExe = CompilerPath; 129 | Assert.IsFalse(c.ProcessArguments(argv)); 130 | } 131 | 132 | 133 | [Test] 134 | [TestCase("/c", "test-sources\\exists.c", "/Zi","/Z7")] 135 | [TestCase("/c", "test-sources\\exists.c", "/Zi", "/Fowhatever.obj", "/Z7")] 136 | [TestCase("/c", "test-sources\\exists.c", "/Zi", "/Fotest-sources", "/Z7")] 137 | [TestCase("/c", "test-sources\\exists.c", "/Zi", "/Fdtest-sources\\stuff.pdb", "/Z7")] 138 | public void ParseSupportedDebugArgs(params string[] argv) { 139 | var c = new Compiler(); 140 | c.SetWorkingDirectory(InitialDir); 141 | c.CompilerExe = CompilerPath; 142 | Assert.IsTrue(c.ProcessArguments(argv)); 143 | Assert.IsFalse(c.GeneratePdb); 144 | } 145 | 146 | [Test] 147 | [TestCase("/c", "test-sources\\doesnotexist.c")] 148 | [TestCase("/c", "test-sources\\exists.c", "/Yu")] 149 | [TestCase("/c", "test-sources\\exists.c", "/Zi")] 150 | [TestCase("/c", "test-sources\\exists.c", "/Zi", "/Fowhatever.obj")] 151 | public void ParseUnSupportedArgs(params string[] argv) 152 | { 153 | var c = new Compiler(); 154 | c.SetWorkingDirectory(InitialDir); 155 | Assert.IsFalse(c.ProcessArguments(argv)); 156 | } 157 | 158 | [Test] 159 | public void ParseQTArgs() 160 | { 161 | Environment.SetEnvironmentVariable("CCLASH_Z7_OBJ", "yes"); 162 | var argv = "/nologo /TP -DLIBSNORE_PLUGIN_PATH=\"r:/plugins/libsnore-qt5\" -DQT_CORE_LIB -DQT_GUI_LIB -DQT_NETWORK_LIB -DQT_NO_CAST_FROM_ASCII -DQT_NO_CAST_FROM_BYTEARRAY -DQT_NO_CAST_TO_ASCII -DQT_NO_SIGNALS_SLOTS_KEYWORDS -DQT_NO_URL_CAST_FROM_STRING -DQT_STRICT_ITERATORS -DQT_USE_FAST_OPERATOR_PLUS -DQT_USE_QSTRINGBUILDER -DSNORE_SUFFIX=\"-qt5\" -DUNICODE -DWIN32_LEAN_AND_MEAN -DWINVER=0x0600 -D_CRT_NONSTDC_NO_DEPRECATE -D_CRT_SECURE_NO_DEPRECATE -D_CRT_SECURE_NO_WARNINGS -D_SCL_SECURE_NO_WARNINGS -D_UNICODE -D_USE_MATH_DEFINES -D_WIN32_IE=0x0600 -D_WIN32_WINNT=0x0600 -Dlibsnore_EXPORTS -Isrc\\libsnore -IQ:\\snorenotify\\src\\libsnore -IQ:\\snorenotify\\src -Isrc -IR:\\include\\qt5 -IR:\\include\\qt5\\QtCore -IR:\\.\\mkspecs\\win32-msvc2015 -IR:\\include\\qt5\\QtGui -IR:\\include\\qt5\\QtNetwork /DWIN32 /D_WINDOWS /W3 /GR /EHsc /wd4250 /wd4251 /wd4396 /wd4661 /D_DEBUG /MDd /Zi /Ob0 /Od /RTC1 /showIncludes /Fosrc\\libsnore\\CMakeFiles\\libsnore.dir\\plugins\\plugincontainer.cpp.obj /Fdsrc\\libsnore\\CMakeFiles\\libsnore.dir\\ /FS -c Q:\\snorenotify\\src\\libsnore\\plugins\\plugincontainer.cpp"; 163 | var args = argv.Split(new char[] {' '}); 164 | var c = new Compiler(); 165 | c.SetWorkingDirectory(InitialDir); 166 | Assert.IsFalse(c.ProcessArguments(args)); 167 | } 168 | 169 | [Test] 170 | [TestCase("/c", "test-sources\\hello.c", "/Itest-sources\\inc with spaces")] 171 | public void IncludeFileTest(params string[] argv) 172 | { 173 | var c = new Compiler() { CompilerExe = CompilerPath }; 174 | var hv = new List(); 175 | c.SetWorkingDirectory(InitialDir); 176 | c.SetEnvironment(Compiler.GetEnvironmentDictionary()); 177 | Assert.IsTrue(c.ProcessArguments(argv)); 178 | hv.Add(Path.GetFullPath(c.SingleSourceFile)); 179 | List incfiles = new List(); 180 | var rv = c.InvokeCompiler(argv, Console.Error.Write, Console.Out.Write, true, incfiles); 181 | hv.AddRange(incfiles); 182 | Assert.AreEqual(0, rv); 183 | Assert.IsTrue(hv.Count > 0); 184 | } 185 | 186 | [Test] 187 | [TestCase("/c", "test-sources\\hello.c", "/Itest-sources\\inc with spaces")] 188 | [TestCase("/c", "test-sources\\hello.c", "/I", "test-sources\\inc with spaces", "/D", "a_hash_define")] 189 | [TestCase("/c", "test-sources\\hello.c", "/I", "test-sources\\inc with spaces", "/DEXPAND=\"test.123\"")] 190 | public void PreprocessorTest(params string[] argv) 191 | { 192 | var c = new Compiler() { CompilerExe = CompilerPath }; 193 | c.SetWorkingDirectory(InitialDir); 194 | c.SetEnvironment(Compiler.GetEnvironmentDictionary()); 195 | 196 | var supported = c.ProcessArguments(argv); 197 | 198 | Assert.IsTrue(supported); 199 | Assert.AreEqual(1, c.CliIncludePaths.Count); 200 | Assert.AreEqual( Path.Combine(InitialDir, "test-sources\\inc with spaces"), c.CliIncludePaths[0]); 201 | Assert.AreEqual( Path.Combine(InitialDir, "test-sources\\hello.c"), c.SingleSourceFile); 202 | using (var sw = new StreamWriter(new MemoryStream())) 203 | { 204 | var rv = c.InvokePreprocessor(sw); 205 | Assert.AreEqual(0, rv); 206 | } 207 | } 208 | 209 | [Test] 210 | public void PreprocessorRespFileTest() 211 | { 212 | PreprocessorTest("@test-sources\\compiler1.resp"); 213 | } 214 | 215 | [Test] 216 | [TestCase("/c", "test-sources\\hello.c", "/Itest-sources\\inc with spaces")] 217 | [TestCase("/c", "test-sources\\hello.c", "/I","test-sources\\inc with spaces","/D","a_hash_define")] 218 | [TestCase("/c", "test-sources\\hello.c", "/I", "test-sources\\inc with spaces", "/DEXPAND=\"test.123\"")] 219 | public void CompileObjectTest(params string[] argv) 220 | { 221 | var c = new Compiler() { CompilerExe = CompilerPath }; 222 | c.SetWorkingDirectory(InitialDir); 223 | c.SetEnvironment(Compiler.GetEnvironmentDictionary()); 224 | 225 | Assert.IsTrue(c.ProcessArguments(argv)); 226 | Assert.AreEqual(1, c.CliIncludePaths.Count); 227 | Assert.AreEqual( Path.Combine(InitialDir, "test-sources\\inc with spaces"), c.CliIncludePaths[0]); 228 | Assert.AreEqual( Path.Combine(InitialDir, "test-sources\\hello.c"), c.SingleSourceFile); 229 | var stderr = new StringBuilder(); 230 | var stdout = new StringBuilder(); 231 | var rv = c.InvokeCompiler(c.CommandLine, x => stderr.Append(x), x => stdout.Append(x), false, null); 232 | 233 | Assert.AreEqual(0, rv); 234 | } 235 | 236 | [Test] 237 | public void CompileObjectRespFileTest() 238 | { 239 | CompileObjectTest("@test-sources\\compiler1.resp"); 240 | } 241 | 242 | [Test] 243 | [TestCase("/o","foo.exe")] 244 | [TestCase("/c","test-sources\\exists.c","test-sources\\hello.c")] 245 | public void DetectNotSupported(params string[] argv) 246 | { 247 | var c = new Compiler() { 248 | CompilerExe = CompilerPath }; 249 | c.SetWorkingDirectory(InitialDir); 250 | c.SetEnvironment(Compiler.GetEnvironmentDictionary()); 251 | Assert.IsFalse(c.ProcessArguments(argv)); 252 | } 253 | 254 | [Test] 255 | public void DetectLinkNotSupported() 256 | { 257 | DetectNotSupported("/link"); 258 | } 259 | } 260 | } 261 | -------------------------------------------------------------------------------- /CClash.Tests/FileCacheDatabaseTest.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.IO; 3 | using System.Collections.Generic; 4 | using System.Linq; 5 | using System.Text; 6 | using System.Threading.Tasks; 7 | using NUnit.Framework; 8 | using CClash; 9 | 10 | namespace CClash.Tests 11 | { 12 | [TestFixture] 13 | public class FileCacheDatabaseTest : FileCacheTestBase 14 | { 15 | 16 | [Test] 17 | public void TestPopulateCacheSQLite() 18 | { 19 | PopulateTest(); 20 | } 21 | 22 | [Test] 23 | public void TestReadCacheSQLite() 24 | { 25 | ReadTest(); 26 | } 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /CClash.Tests/FileCacheTest.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.IO; 4 | using NUnit.Framework; 5 | using CClash; 6 | 7 | namespace CClash.Tests 8 | { 9 | [TestFixture] 10 | public class FileCacheTest : FileCacheTestBase 11 | { 12 | string thistestdll = typeof(FileCacheTest).Assembly.Location; 13 | 14 | [Test] 15 | public void Test0DeleteCache() 16 | { 17 | if (Directory.Exists("clcachetest_init")) 18 | Directory.Delete("clcachetest_init", true); 19 | } 20 | 21 | 22 | [Test] 23 | [TestCase("clcachetest_init")] 24 | public void Test1CacheSetup(string folder) 25 | { 26 | using (var fc = FileCacheStore.Load(folder)) 27 | { 28 | Assert.IsTrue(Directory.Exists(fc.FolderPath)); 29 | } 30 | } 31 | 32 | [Test] 33 | public void TestTextFileAddRemove() 34 | { 35 | using (var fc = FileCacheStore.Load("clcachetest_text")) 36 | { 37 | fc.WaitOne(); 38 | try 39 | { 40 | fc.AddTextFileContent("aa12345", "test.txt", "hello"); 41 | Assert.IsTrue(fc.ContainsEntry("aa12345", "test.txt")); 42 | fc.Remove("aa12345"); 43 | Assert.IsFalse(fc.ContainsEntry("aa12345", "test.txt")); 44 | } 45 | finally 46 | { 47 | fc.ReleaseMutex(); 48 | } 49 | } 50 | } 51 | 52 | [Test] 53 | [Repeat(1000)] 54 | public void FileMissing_FileDotExists() 55 | { 56 | File.Exists("c:\\nosuchfile.txt"); 57 | } 58 | 59 | [Test] 60 | [Repeat(1000)] 61 | public void FileMissing_FileUtilsExists() 62 | { 63 | FileUtils.Exists("c:\\nosuchfile.txt"); 64 | } 65 | 66 | [Test] 67 | [Repeat(1000)] 68 | public void FileMissing_FileUtilsFileMissing() 69 | { 70 | FileUtils.FileMissing("c:\\nosuchfile.txt"); 71 | } 72 | 73 | [Test] 74 | public void TestPopulateCacheFiles() 75 | { 76 | PopulateTest(); 77 | } 78 | 79 | [Test] 80 | public void TestReadCacheFiles() 81 | { 82 | ReadTest(); 83 | } 84 | } 85 | } 86 | -------------------------------------------------------------------------------- /CClash.Tests/FileCacheTestBase.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Text; 5 | using System.IO; 6 | using NUnit.Framework; 7 | 8 | namespace CClash.Tests 9 | { 10 | public class FileCacheTestBase 11 | { 12 | public string TempFolder { get; protected set; } 13 | 14 | [TestFixtureSetUp] 15 | public void MakeTempFolder() 16 | { 17 | var fname = Guid.NewGuid().ToString(); 18 | var tmp = Path.GetTempPath(); 19 | TempFolder = Path.Combine(tmp, fname); 20 | Directory.CreateDirectory(TempFolder); 21 | } 22 | 23 | [TestFixtureTearDown] 24 | public void CleanTempFolder() 25 | { 26 | if (!String.IsNullOrEmpty(TempFolder)) 27 | { 28 | for (int i = 0; i < 5; i++) 29 | { 30 | try 31 | { 32 | if (Directory.Exists(TempFolder)) 33 | Directory.Delete(TempFolder, true); 34 | return; 35 | } 36 | catch 37 | { 38 | System.Threading.Thread.Sleep(200); 39 | } 40 | } 41 | } 42 | } 43 | 44 | public void PopulateTest() where T : FileCacheBase, IFileCacheStore, new() 45 | { 46 | int nfiles = 100; 47 | int filesize = 1024 * 1024 * 2; 48 | var ms = new MemoryStream(filesize); 49 | 50 | using (var db = FileCacheStore.Load(TempFolder)) 51 | { 52 | for (int i = 0; i < nfiles; i++) 53 | { 54 | string fname = "populate" + i.ToString(); 55 | using (var ws = db.OpenFileStream("cdef", fname, FileMode.CreateNew, FileAccess.Write)) 56 | { 57 | ms.CopyTo(ws); 58 | } 59 | } 60 | } 61 | } 62 | 63 | public void ReadTest() where T : FileCacheBase, IFileCacheStore, new() 64 | { 65 | int nfiles = 100; 66 | int filesize = 1024 * 1024 * 2; 67 | var ms = new MemoryStream(filesize); 68 | 69 | using (var db = FileCacheStore.Load(TempFolder)) 70 | { 71 | for (int i = 0; i < nfiles; i++) 72 | { 73 | string fname = "populate" + i.ToString(); 74 | using (var ws = db.OpenFileStream("cdef", fname, FileMode.Open, FileAccess.Read)) 75 | { 76 | ws.CopyTo(ms); 77 | } 78 | } 79 | } 80 | } 81 | 82 | } 83 | } 84 | -------------------------------------------------------------------------------- /CClash.Tests/HashingTest.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Text; 5 | using System.Threading.Tasks; 6 | using NUnit.Framework; 7 | using System.IO; 8 | using CClash; 9 | 10 | namespace CClash.Tests 11 | { 12 | [TestFixture] 13 | public class HashingTest 14 | { 15 | const string IncludeDir = "C:\\Program Files (x86)\\Microsoft Visual Studio 11.0\\VC\\include"; 16 | 17 | [Test] 18 | [Repeat(2)] 19 | public void TestExists() 20 | { 21 | var files = Directory.GetFiles(IncludeDir); 22 | foreach (var f in files) 23 | Assert.IsTrue(FileUtils.Exists(f)); 24 | } 25 | 26 | [Test] 27 | [Repeat(2)] 28 | [TestCase(1)] 29 | [TestCase(4)] 30 | public void HashIncludeFiles(int threads) 31 | { 32 | using (var ic = FileCacheStore.Load("testincs")) 33 | { 34 | var ht = new HashUtil(ic); 35 | ht.HashingThreadCount = threads; 36 | var files = Directory.GetFiles(IncludeDir); 37 | foreach (var f in files) 38 | { 39 | var hr = ht.DigestSourceFile(f); 40 | Assert.IsNotNull(hr.Hash); 41 | } 42 | } 43 | } 44 | 45 | [Test] 46 | [Repeat(2)] 47 | public void ThreadyHashIncludeFiles() 48 | { 49 | using (var ic = FileCacheStore.Load("testincs")) 50 | { 51 | var ht = new HashUtil(ic); 52 | 53 | var hashes = ht.ThreadyDigestFiles(Directory.GetFiles(IncludeDir), true); 54 | Assert.IsTrue(hashes.Count > 0); 55 | } 56 | } 57 | 58 | [Test] 59 | [Repeat(2)] 60 | public void ThreadyHashIncludeFilesCacheTest() 61 | { 62 | using (var ic = FileCacheStore.Load("testincs")) 63 | { 64 | var ht = new HashUtil(ic); 65 | 66 | var hashes = ht.DigestFiles(Directory.GetFiles(IncludeDir), IncludeDir); 67 | Assert.IsTrue(hashes.Count > 0); 68 | System.Threading.Thread.Sleep(500); 69 | var hashes2 = ht.DigestFiles(Directory.GetFiles(IncludeDir), IncludeDir); 70 | foreach (var h in hashes2) 71 | { 72 | if (hashes.ContainsKey(h.Key)) 73 | { 74 | Assert.IsTrue(h.Value.Cached); 75 | } 76 | } 77 | } 78 | } 79 | 80 | [Test] 81 | [Repeat(2)] 82 | public void HashesMatch() 83 | { 84 | var files = Directory.GetFiles(IncludeDir); 85 | using (var ic = FileCacheStore.Load("testincs")) 86 | { 87 | var ht = new HashUtil(ic); 88 | var hashes = ht.ThreadyDigestFiles(files, true); 89 | foreach (var f in files) 90 | { 91 | var hash = ht.DigestSourceFile(f); 92 | 93 | if (hash.Result == DataHashResult.Ok) 94 | { 95 | Assert.AreEqual(hash.Hash, hashes[f.ToLower()].Hash); 96 | } 97 | } 98 | 99 | Assert.AreEqual(files.Length, hashes.Count); 100 | } 101 | } 102 | } 103 | } 104 | -------------------------------------------------------------------------------- /CClash.Tests/Properties/AssemblyInfo.cs: -------------------------------------------------------------------------------- 1 | using System.Reflection; 2 | using System.Runtime.CompilerServices; 3 | using System.Runtime.InteropServices; 4 | 5 | // General Information about an assembly is controlled through the following 6 | // set of attributes. Change these attribute values to modify the information 7 | // associated with an assembly. 8 | [assembly: AssemblyTitle("CClast.Tests")] 9 | [assembly: AssemblyDescription("")] 10 | [assembly: AssemblyConfiguration("")] 11 | [assembly: AssemblyCompany("")] 12 | [assembly: AssemblyProduct("CClast.Tests")] 13 | [assembly: AssemblyCopyright("Copyright © 2013")] 14 | [assembly: AssemblyTrademark("")] 15 | [assembly: AssemblyCulture("")] 16 | 17 | // Setting ComVisible to false makes the types in this assembly not visible 18 | // to COM components. If you need to access a type in this assembly from 19 | // COM, set the ComVisible attribute to true on that type. 20 | [assembly: ComVisible(false)] 21 | 22 | // The following GUID is for the ID of the typelib if this project is exposed to COM 23 | [assembly: Guid("27b11b95-d777-4f2c-aa4c-965429fe5d19")] 24 | 25 | // Version information for an assembly consists of the following four values: 26 | // 27 | // Major Version 28 | // Minor Version 29 | // Build Number 30 | // Revision 31 | // 32 | // You can specify all the values or you can default the Build and Revision Numbers 33 | // by using the '*' as shown below: 34 | // [assembly: AssemblyVersion("1.0.*")] 35 | [assembly: AssemblyVersion("1.0.0.0")] 36 | [assembly: AssemblyFileVersion("1.0.0.0")] 37 | -------------------------------------------------------------------------------- /CClash.Tests/app.config: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | 5 |
6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | -------------------------------------------------------------------------------- /CClash.Tests/packages.config: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | -------------------------------------------------------------------------------- /CClash.Tests/test-sources/another.h: -------------------------------------------------------------------------------- 1 | #define FROMSPACE "from space" 2 | -------------------------------------------------------------------------------- /CClash.Tests/test-sources/compiler1.resp: -------------------------------------------------------------------------------- 1 | /c test-sources\hello.c /I "test-sources\inc with spaces" /D a_hash_define -------------------------------------------------------------------------------- /CClash.Tests/test-sources/compiler2.resp: -------------------------------------------------------------------------------- 1 | /c test-sources\hello.c /I"test-sources\inc with spaces" /D a_hash_define -------------------------------------------------------------------------------- /CClash.Tests/test-sources/exists.c: -------------------------------------------------------------------------------- 1 | // this is empty -------------------------------------------------------------------------------- /CClash.Tests/test-sources/hello.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | 7 | #include "hello.h" 8 | 9 | #include "hello2.h" 10 | 11 | #include "exists.c" 12 | 13 | #include "another.h" 14 | 15 | int main( int argc, char** argv ) 16 | { 17 | #ifdef FOO 18 | printf(FOO); 19 | #endif 20 | 21 | 22 | #ifdef EXPAND 23 | printf("%s\n", EXPAND); 24 | #endif 25 | 26 | return HELLO_TEST; 27 | } -------------------------------------------------------------------------------- /CClash.Tests/test-sources/hello.cc: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | 5 | using namespace std; 6 | 7 | int main(void) 8 | { 9 | vector SS; 10 | 11 | SS.push_back("The number is 10"); 12 | SS.push_back("The number is 20"); 13 | SS.push_back("The number is 30"); 14 | 15 | cout << "Loop by index:" << endl; 16 | 17 | int ii; 18 | for(ii=0; ii < SS.size(); ii++) 19 | { 20 | cout << SS[ii] << endl; 21 | } 22 | 23 | } -------------------------------------------------------------------------------- /CClash.Tests/test-sources/hello.h: -------------------------------------------------------------------------------- 1 | #define HELLO_TEST 1 -------------------------------------------------------------------------------- /CClash.Tests/test-sources/hello2.h: -------------------------------------------------------------------------------- 1 | #define HELLO_TEST 1 -------------------------------------------------------------------------------- /CClash.Tests/test-sources/inc with spaces/another.h: -------------------------------------------------------------------------------- 1 | #define FROMSPACE "from space" 2 | -------------------------------------------------------------------------------- /CClash/App.config: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | 5 |
6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | -------------------------------------------------------------------------------- /CClash/ArgumentUtils.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Text; 5 | using System.Text.RegularExpressions; 6 | 7 | namespace CClash 8 | { 9 | /// 10 | /// Class for doing things to command line arguments and argument lists 11 | /// 12 | public class ArgumentUtils 13 | { 14 | static char[] force_escape = new char[] { 15 | '\t', ' ' 16 | }; 17 | 18 | /// 19 | /// Join a list of command line args such that it can be passed as a single 20 | /// escaped string for subprocess arguments. 21 | /// 22 | /// 23 | /// 24 | public static string JoinAguments(IEnumerable args) 25 | { 26 | 27 | var sb = new System.Text.StringBuilder(512); 28 | foreach (string arg in args) 29 | { 30 | string a = arg; 31 | if (a.Contains("\"")) 32 | { 33 | sb.Append(a.Replace("\"", "\\\"")); 34 | } 35 | else 36 | { 37 | if (a.IndexOfAny(force_escape) != -1) 38 | { 39 | sb.Append('\"'); 40 | sb.Append(a); 41 | sb.Append('\"'); 42 | } 43 | else 44 | { 45 | sb.Append(a); 46 | } 47 | } 48 | sb.Append(" "); 49 | } 50 | return sb.ToString().TrimEnd(); 51 | } 52 | 53 | public static bool TargetIsFolder(string target) 54 | { 55 | switch (target.Last()) { 56 | case '/': 57 | case '\\': 58 | return true; 59 | 60 | default: 61 | return false; 62 | } 63 | } 64 | 65 | /// 66 | /// Given a whole path or single file, append .obj if the file has no dots in it's name. 67 | /// 68 | /// 69 | /// 70 | public static string TargetObject(string target) 71 | { 72 | var parts = target.Split('/', '\\'); 73 | var last = parts.Last(); 74 | if (last.Contains(".")) return target; 75 | return target + ".obj"; 76 | } 77 | 78 | public static string CanonicalArgument(string arg) 79 | { 80 | if (arg.StartsWith("-")) 81 | arg = "/" + arg.Substring(1); 82 | return arg; 83 | } 84 | 85 | /// 86 | /// Given a string we expect is a disk path, swap "/" to "\". 87 | /// 88 | /// 89 | /// This might be from running under cygwin or mingw, cl is quite lax about this. 90 | /// 91 | /// 92 | /// 93 | public static string MakeWindowsPath(string path) 94 | { 95 | return path.Replace('/', '\\'); 96 | } 97 | 98 | public static IEnumerable FixupArgs(IEnumerable args) 99 | { 100 | var rv = new List(); 101 | var aa = args.ToArray(); 102 | for (int i = 0; i < aa.Length; i++) 103 | { 104 | var a = aa[i]; 105 | if ( CanonicalArgument(a) == "/D" ) 106 | { 107 | string val; 108 | if (a.Length == 2 && (i + 1 < aa.Length)) 109 | { 110 | val = aa[++i]; 111 | } 112 | else 113 | { 114 | val = a.Substring(2); 115 | } 116 | 117 | if (val.Contains("=\"")) 118 | { 119 | val = Regex.Replace(val, "\"", "\"\"\""); 120 | } 121 | 122 | rv.Add("/D" + val); 123 | } 124 | else 125 | { 126 | rv.Add(a); 127 | } 128 | } 129 | 130 | return rv; 131 | } 132 | } 133 | } 134 | -------------------------------------------------------------------------------- /CClash/CClash.csproj: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | 5 | Debug 6 | AnyCPU 7 | {7BCF13A4-C4C9-493D-898B-E89AEFC93C1A} 8 | Exe 9 | Properties 10 | CClash 11 | cl 12 | v4.5 13 | 512 14 | false 15 | publish\ 16 | true 17 | Disk 18 | false 19 | Foreground 20 | 7 21 | Days 22 | false 23 | false 24 | true 25 | 1 26 | 0.3.15.%2a 27 | false 28 | true 29 | 30 | 31 | x86 32 | true 33 | full 34 | false 35 | bin\Debug\ 36 | DEBUG;TRACE 37 | prompt 38 | 4 39 | true 40 | 41 | 42 | AnyCPU 43 | pdbonly 44 | true 45 | bin\Release\ 46 | TRACE 47 | prompt 48 | 4 49 | true 50 | 51 | 52 | CClash.Program 53 | 54 | 55 | false 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | ..\packages\System.Data.SQLite.Core.1.0.99.0\lib\net45\System.Data.SQLite.dll 64 | False 65 | 66 | 67 | ..\packages\System.Data.SQLite.Linq.1.0.99.0\lib\net45\System.Data.SQLite.Linq.dll 68 | False 69 | 70 | 71 | 72 | 73 | 74 | False 75 | True 76 | 77 | 78 | 79 | 80 | 81 | 82 | 83 | 84 | 85 | 86 | 87 | 88 | 89 | 90 | 91 | 92 | 93 | 94 | 95 | 96 | 97 | 98 | 99 | 100 | 101 | 102 | 103 | 104 | 105 | 106 | 107 | 108 | 109 | 110 | 111 | 112 | 113 | 114 | 115 | 116 | 117 | 118 | 119 | 120 | 121 | 122 | 123 | 124 | False 125 | Microsoft .NET Framework 4.5 %28x86 and x64%29 126 | true 127 | 128 | 129 | False 130 | .NET Framework 3.5 SP1 Client Profile 131 | false 132 | 133 | 134 | False 135 | .NET Framework 3.5 SP1 136 | false 137 | 138 | 139 | 140 | 141 | copy $(TargetPath) $(TargetDir)\cclash.exe 142 | 143 | 144 | 145 | 146 | This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}. 147 | 148 | 149 | 150 | 157 | -------------------------------------------------------------------------------- /CClash/CClash.csproj.user: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | --cclash-server --debug --pdb-to-z7 --sqlite --cachedir=..\..\oslcache 5 | 6 | 7 | publish\ 8 | 9 | 10 | 11 | 12 | 13 | en-US 14 | false 15 | 16 | -------------------------------------------------------------------------------- /CClash/CClashErrorException.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Text; 5 | using System.Threading.Tasks; 6 | 7 | namespace CClash 8 | { 9 | public class CClashErrorException : Exception 10 | { 11 | public CClashErrorException(string fmt, params object[] args) 12 | : base(string.Format(fmt, args)) 13 | { 14 | } 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /CClash/CClashMessage.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Text; 5 | using System.Threading.Tasks; 6 | using System.Runtime.Serialization.Formatters.Binary; 7 | using System.IO; 8 | 9 | namespace CClash 10 | { 11 | [Serializable] 12 | public enum Command 13 | { 14 | Run = 0, 15 | GetStats = 1, 16 | Quit = 2, 17 | GetResult = 3, 18 | DisableCache = 4, 19 | EnableCache = 5, 20 | ClearCache = 6, 21 | } 22 | 23 | [Serializable] 24 | public class CClashMessage 25 | { 26 | public byte[] Serialize() 27 | { 28 | using (var ms = new MemoryStream()) 29 | { 30 | Serialize(ms); 31 | return ms.ToArray(); 32 | } 33 | } 34 | 35 | public void Serialize( Stream str ) 36 | { 37 | new BinaryFormatter().Serialize(str, this); 38 | } 39 | 40 | public static T Deserialize(Stream str) 41 | { 42 | return (T)(new BinaryFormatter().Deserialize(str)); 43 | } 44 | 45 | public static T Deserialize(byte[] bb) 46 | where T : CClashMessage, new() 47 | { 48 | var rv = new T(); 49 | using (var ms = new MemoryStream(bb)) 50 | { 51 | rv = Deserialize(ms); 52 | } 53 | return rv; 54 | } 55 | } 56 | 57 | [Serializable] 58 | public class CClashRequest : CClashMessage 59 | { 60 | public Command cmd; 61 | public string workdir; 62 | public IDictionary envs; 63 | public IList argv; 64 | public string compiler; 65 | public int tag; 66 | public int pid; 67 | } 68 | 69 | [Serializable] 70 | public class CClashResponse : CClashMessage 71 | { 72 | public bool supported; 73 | public int exitcode; 74 | public string stderr; 75 | public string stdout; 76 | public int tag; 77 | } 78 | 79 | 80 | } 81 | -------------------------------------------------------------------------------- /CClash/CClashServer.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Text; 5 | using System.IO; 6 | using System.IO.Pipes; 7 | using System.Web.Script.Serialization; 8 | using System.Threading; 9 | 10 | namespace CClash 11 | { 12 | public sealed class CClashServer : IDisposable 13 | { 14 | bool quitnow = false; 15 | DirectCompilerCacheServer cache; 16 | 17 | public const int DefaultQuitAfterIdleMinutes = 90; 18 | 19 | public int MaxServerThreads 20 | { 21 | get 22 | { 23 | var rv = Settings.MaxServerThreads; 24 | if (rv == 0) rv = Environment.ProcessorCount + 1; 25 | return rv; 26 | } 27 | } 28 | 29 | public int QuitAfterIdleMinutes 30 | { 31 | get 32 | { 33 | var rv = Settings.ServerQuitAfterIdleMinutes; 34 | if (rv == 0) rv = DefaultQuitAfterIdleMinutes; 35 | return rv; 36 | } 37 | } 38 | 39 | List serverPipes = new List(); 40 | List serverThreads = new List(); 41 | 42 | string cdto = Path.GetPathRoot( Environment.GetFolderPath(Environment.SpecialFolder.Windows)); 43 | 44 | public CClashServer() 45 | { 46 | Directory.SetCurrentDirectory(cdto); 47 | } 48 | 49 | Mutex serverMutex; 50 | 51 | public bool Preflight(string cachedir) 52 | { 53 | Logging.Emit("cclash server preflight check"); 54 | var mtx = new Mutex(false, "cclash_serv_" + cachedir.ToLower().GetHashCode()); 55 | serverMutex = mtx; 56 | try 57 | { 58 | if (!mtx.WaitOne(1000)) 59 | { 60 | quitnow = true; 61 | Logging.Error("another server is already running"); 62 | return false; // some other process is holding it! 63 | } 64 | else 65 | { 66 | Logging.Emit("cclash server preflight ok"); 67 | } 68 | } 69 | catch (AbandonedMutexException) 70 | { 71 | Logging.Warning("previous instance did not exit cleanly!"); 72 | } 73 | return true; 74 | } 75 | 76 | int busyThreads = 0; 77 | 78 | public bool FirstThreadReady { get; private set; } 79 | 80 | public int BusyThreadCount 81 | { 82 | get 83 | { 84 | return busyThreads; 85 | } 86 | } 87 | 88 | void ThreadIsBusy() 89 | { 90 | lock (serverThreads) 91 | { 92 | busyThreads++; 93 | } 94 | } 95 | 96 | void ThreadIsIdle() 97 | { 98 | lock (serverThreads) 99 | { 100 | busyThreads--; 101 | } 102 | } 103 | 104 | DateTime lastRequest = DateTime.Now; 105 | 106 | void ThreadBeforeProcessRequest() 107 | { 108 | lastRequest = DateTime.Now; 109 | if (BusyThreadCount > Environment.ProcessorCount) 110 | { 111 | System.Threading.Thread.Sleep(60/Environment.ProcessorCount); 112 | } 113 | } 114 | 115 | public void ConnectionThreadFn(object con) 116 | { 117 | using (var nss = con as NamedPipeServerStream) 118 | { 119 | try 120 | { 121 | 122 | while (!quitnow) 123 | { 124 | var w = nss.BeginWaitForConnection(null, null); 125 | FirstThreadReady = true; 126 | Logging.Emit("waiting for client.."); 127 | while (!w.AsyncWaitHandle.WaitOne(1000)) 128 | { 129 | if (quitnow) 130 | { 131 | return; 132 | } 133 | } 134 | nss.EndWaitForConnection(w); 135 | Logging.Emit("got client"); 136 | if (nss.IsConnected) 137 | { 138 | Logging.Emit("server connected"); 139 | ThreadBeforeProcessRequest(); 140 | ThreadIsBusy(); 141 | ServiceRequest(nss); 142 | } 143 | 144 | ThreadIsIdle(); 145 | } 146 | } 147 | catch (IOException ex) 148 | { 149 | Logging.Error("server thread got {0}, {1}", ex.GetType().Name, ex.Message); 150 | Logging.Error(":{0}", ex.ToString()); 151 | } 152 | } 153 | } 154 | 155 | public void ServiceRequest(NamedPipeServerStream nss) 156 | { 157 | var msgbuf = new List(8192); 158 | var rxbuf = new byte[256 * 1024]; 159 | int count = 0; 160 | 161 | 162 | Logging.Emit("reading from client"); 163 | do 164 | { 165 | count = nss.Read(rxbuf, msgbuf.Count, rxbuf.Length); 166 | if (count > 0) 167 | { 168 | msgbuf.AddRange(rxbuf.Take(count)); 169 | } 170 | 171 | } while (!nss.IsMessageComplete); 172 | 173 | Logging.Emit("server read {0} bytes", msgbuf.Count); 174 | 175 | // deserialize message from msgbuf 176 | var req = CClashMessage.Deserialize(msgbuf.ToArray()); 177 | cache.Setup(); // needed? 178 | Logging.Emit("processing request"); 179 | var resp = ProcessRequest(req); 180 | Logging.Emit("request complete: supported={0}, exitcode={1}", resp.supported, resp.exitcode); 181 | var tx = resp.Serialize(); 182 | nss.Write(tx, 0, tx.Length); 183 | nss.Flush(); 184 | Logging.Emit("server written {0} bytes", tx.Length); 185 | 186 | nss.WaitForPipeDrain(); 187 | nss.Disconnect(); 188 | 189 | Logging.Emit("request done"); 190 | } 191 | 192 | void NewServerThread(string cachedir) 193 | { 194 | var t = new Thread(new ParameterizedThreadStart(ConnectionThreadFn)); 195 | t.IsBackground = true; 196 | serverThreads.Add(t); 197 | var nss = new NamedPipeServerStream(MakePipeName(cachedir), PipeDirection.InOut, MaxServerThreads, PipeTransmissionMode.Message, PipeOptions.WriteThrough | PipeOptions.Asynchronous); 198 | if (Settings.PipeSecurityEveryone) { 199 | var npa = new PipeAccessRule("Everyone", PipeAccessRights.ReadWrite, System.Security.AccessControl.AccessControlType.Allow); 200 | var nps = new PipeSecurity(); 201 | nps.AddAccessRule(npa); 202 | nss.SetAccessControl(nps); 203 | } 204 | t.Start(nss); 205 | Logging.Emit("server thread started"); 206 | } 207 | 208 | public void Listen(string cachedir) 209 | { 210 | Environment.CurrentDirectory = cdto; 211 | Logging.Emit("creating direct cache server.."); 212 | cache = new DirectCompilerCacheServer(cachedir); 213 | Logging.Emit("starting server threads.."); 214 | 215 | while (serverThreads.Count < MaxServerThreads) 216 | { 217 | NewServerThread(cachedir); 218 | } 219 | 220 | // maintain the threadpool 221 | while (!quitnow) 222 | { 223 | foreach (var t in serverThreads.ToArray()) 224 | { 225 | if (busyThreads > 0) 226 | Logging.Emit("{0} busy threads", busyThreads); 227 | if (t.Join(1000)) 228 | { 229 | serverThreads.Remove(t); 230 | Logging.Emit("replacing thread"); 231 | NewServerThread(cachedir); 232 | } 233 | } 234 | if (busyThreads < 1) { 235 | Logging.Emit("server is idle.."); 236 | } 237 | if (DateTime.Now.Subtract(lastRequest).TotalMinutes > QuitAfterIdleMinutes) 238 | { 239 | quitnow = true; 240 | } 241 | } 242 | Logging.Emit("waiting for threads to finish"); 243 | foreach (var t in serverThreads) 244 | { 245 | Logging.Emit("joining thread {0}", t.ManagedThreadId); 246 | if (!t.Join(2000)) { 247 | Logging.Emit("thread still running.."); 248 | } 249 | } 250 | 251 | Logging.Emit("commiting stats"); 252 | cache.SetupStats(); 253 | Logging.Emit("server quitting"); 254 | serverMutex.ReleaseMutex(); 255 | } 256 | 257 | public static string MakePipeName(string cachedir) 258 | { 259 | var x = cachedir.Replace('\\', ' '); 260 | x = x.Replace('\"', '_'); 261 | return x.Replace(':', '=') + ".pipe"; 262 | } 263 | 264 | public CClashResponse ProcessRequest(CClashRequest req) 265 | { 266 | var rv = new CClashResponse() { supported = false }; 267 | Logging.Emit("{0}", DateTime.Now.ToString("s")); 268 | Logging.Emit("server req: cmd = {0}, workdir = {1}", 269 | req.cmd, req.workdir); 270 | 271 | switch (req.cmd) 272 | { 273 | 274 | case Command.GetStats: 275 | rv.exitcode = 0; 276 | cache.SetupStats(); // commits stats to disk 277 | rv.stdout = StatOutputs.GetStatsString(req.compiler, cache); 278 | break; 279 | 280 | case Command.DisableCache: 281 | DisableCaching = true; 282 | rv.supported = true; 283 | break; 284 | 285 | case Command.ClearCache: 286 | DisableCaching = true; 287 | cache.SetupStats(); 288 | cache.Lock(CacheLockType.ReadWrite); 289 | cache.OutputCache.ClearLocked(); 290 | cache.IncludeCache.ClearLocked(); 291 | cache.Unlock(CacheLockType.ReadWrite); 292 | rv.supported = true; 293 | break; 294 | 295 | case Command.EnableCache: 296 | DisableCaching = false; 297 | rv.supported = true; 298 | break; 299 | 300 | case Command.Run: 301 | var stdout = new StringBuilder(); 302 | var stderr = new StringBuilder(); 303 | var comp = cache.SetCompilerEx(req.pid, req.compiler, req.workdir, new Dictionary( req.envs )); 304 | cache.SetCaptureCallback(comp, (so) => { stdout.Append(so); }, (se) => { stderr.Append(se); }); 305 | if (DisableCaching) { 306 | rv.exitcode = comp.InvokeCompiler(req.argv, null, null, false, new List()); 307 | } else { 308 | rv.exitcode = cache.CompileOrCache(comp, req.argv, req); 309 | } 310 | rv.supported = true; 311 | rv.stderr = stderr.ToString(); 312 | rv.stdout = stdout.ToString(); 313 | 314 | break; 315 | 316 | case Command.Quit: 317 | cache.SetupStats(); 318 | Stop(); 319 | break; 320 | } 321 | 322 | Logging.Emit("server resp: {0}", rv.exitcode); 323 | 324 | return rv; 325 | } 326 | 327 | public bool DisableCaching { get; private set; } 328 | 329 | public void Stop() 330 | { 331 | quitnow = true; 332 | Dispose(true); 333 | } 334 | 335 | public void Dispose() 336 | { 337 | Dispose(true); 338 | } 339 | 340 | private void Dispose(bool disposing) 341 | { 342 | if (disposing) 343 | { 344 | if ( cache != null ) cache.Dispose(); 345 | } 346 | } 347 | } 348 | } 349 | -------------------------------------------------------------------------------- /CClash/CClashServerClient.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Text; 5 | using System.Threading.Tasks; 6 | using System.IO; 7 | using System.IO.Pipes; 8 | using System.Web.Script.Serialization; 9 | using System.Diagnostics; 10 | 11 | namespace CClash 12 | { 13 | public sealed class CClashServerClient : ICompilerCache 14 | { 15 | NamedPipeClientStream ncs; 16 | string pipename = null; 17 | 18 | public CClashServerClient(string cachedir) 19 | { 20 | pipename = CClashServer.MakePipeName(cachedir); 21 | } 22 | 23 | public IFileCacheStore OutputCache 24 | { 25 | get 26 | { 27 | throw new NotImplementedException(); 28 | } 29 | } 30 | 31 | void Open() 32 | { 33 | ncs = new NamedPipeClientStream(".", pipename, PipeDirection.InOut); 34 | } 35 | 36 | void Connect() 37 | { 38 | var exe = GetType().Assembly.Location; 39 | if (ncs == null) 40 | Open(); 41 | 42 | try 43 | { 44 | ConnectClient(); 45 | return; 46 | } 47 | catch (IOException ex) 48 | { 49 | Logging.Emit("error connecting {0}", ex.Message); 50 | try { ncs.Dispose(); Open(); } 51 | catch { } 52 | } 53 | catch (TimeoutException) 54 | { 55 | Logging.Emit("could not connect to cclash service"); 56 | throw new CClashServerNotReadyException(); 57 | } 58 | } 59 | 60 | public static void StartBackgroundServer() { 61 | using (var ssm = new System.Threading.Mutex(false, "cclash_server_spawn")) { 62 | var can_start_server = ssm.WaitOne(500); 63 | try { 64 | if (can_start_server) { 65 | Logging.Emit("starting new server"); 66 | // start the server 67 | var p = new Process(); 68 | var ours = FileUtils.GetShortPath( typeof(CClashServerClient).Assembly.Location); 69 | var exedir = Path.GetDirectoryName(ours); 70 | var exepath = Path.Combine(exedir, "cclash.exe"); 71 | if (!File.Exists(exepath)) { 72 | exepath = ours; 73 | } 74 | var pargs = new List 75 | { 76 | exepath, 77 | "--cclash-server" 78 | }; 79 | if (Settings.DebugFile != null) { 80 | pargs.Add("--debug"); 81 | } 82 | 83 | var command = "cmd"; 84 | var command_args = "/c " + string.Join(" ", pargs.ToArray()); 85 | 86 | p.StartInfo = new ProcessStartInfo(command); 87 | p.StartInfo.UseShellExecute = false; 88 | p.StartInfo.CreateNoWindow = true; 89 | p.StartInfo.Arguments = command_args; 90 | p.StartInfo.ErrorDialog = false; 91 | p.StartInfo.WorkingDirectory = Path.GetPathRoot(Environment.CurrentDirectory); 92 | p.StartInfo.WindowStyle = ProcessWindowStyle.Hidden; 93 | p.Start(); 94 | } 95 | System.Threading.Thread.Sleep(1000); 96 | } finally { 97 | if (can_start_server) { 98 | ssm.ReleaseMutex(); 99 | } 100 | } 101 | } 102 | } 103 | 104 | private void ConnectClient() 105 | { 106 | if (!ncs.IsConnected) 107 | ncs.Connect(500); 108 | ncs.ReadMode = PipeTransmissionMode.Message; 109 | } 110 | 111 | public ICacheInfo Stats 112 | { 113 | get 114 | { 115 | return null; 116 | } 117 | } 118 | 119 | public bool IsSupported(ICompiler comp, IEnumerable args) 120 | { 121 | return true; 122 | } 123 | 124 | string compilerPath; 125 | string workingdir; 126 | Dictionary environment; 127 | 128 | public bool CheckCache(ICompiler comp, IEnumerable args, DataHash commonkey, out CacheManifest manifest) 129 | { 130 | manifest = null; 131 | return false; 132 | } 133 | 134 | public ICompiler SetCompiler(string compiler, string workdir, System.Collections.Generic.Dictionary envs ) 135 | { 136 | if (string.IsNullOrEmpty(compiler)) throw new ArgumentNullException("compiler"); 137 | if (string.IsNullOrEmpty(workdir)) throw new ArgumentNullException("workdir"); 138 | environment = envs; 139 | workingdir = workdir; 140 | compilerPath = System.IO.Path.GetFullPath(compiler); 141 | try 142 | { 143 | Connect(); 144 | } 145 | catch (CClashServerNotReadyException) 146 | { 147 | var c = new Compiler() { CompilerExe = compiler }; 148 | c.SetWorkingDirectory(workdir); 149 | c.SetEnvironment(envs); 150 | return c; 151 | } 152 | return null; 153 | } 154 | 155 | Action stdOutCallback = null; 156 | Action stdErrCallback = null; 157 | public void SetCaptureCallback(ICompiler comp, Action onOutput, Action onError) 158 | { 159 | stdOutCallback = onOutput; 160 | stdErrCallback = onError; 161 | } 162 | 163 | public int CompileOrCache(ICompiler comp, IEnumerable args, CClashRequest unused) 164 | { 165 | Logging.Emit("client args: {0}", string.Join(" ", args.ToArray())); 166 | 167 | if (comp != null) // compiler is set, server wasnt ready 168 | { 169 | return comp.InvokeCompiler(args, null, null, false, new List()); 170 | } 171 | 172 | try { 173 | var req = new CClashRequest() 174 | { 175 | cmd = Command.Run, 176 | compiler = compilerPath, 177 | envs = environment, 178 | workdir = workingdir, 179 | argv = new List ( args ), 180 | }; 181 | var resp = Transact(req); 182 | if (resp != null) 183 | { 184 | if (stdErrCallback != null) 185 | { 186 | stdErrCallback(resp.stderr); 187 | } 188 | else 189 | { 190 | Console.Error.Write(resp.stderr); 191 | } 192 | if (stdOutCallback != null) 193 | { 194 | stdOutCallback(resp.stdout); 195 | } 196 | else 197 | { 198 | Console.Out.Write(resp.stdout); 199 | } 200 | 201 | return resp.exitcode; 202 | } 203 | else 204 | { 205 | throw new CClashErrorException("server returned no response"); 206 | } 207 | } catch (Exception e) { 208 | Logging.Emit("server error! {0}", e); 209 | throw new CClashWarningException("server error"); 210 | } 211 | } 212 | 213 | public CClashResponse Transact(CClashRequest req) 214 | { 215 | Connect(); 216 | CClashResponse resp = null; 217 | req.pid = System.Diagnostics.Process.GetCurrentProcess().Id; 218 | var txbuf = req.Serialize(); 219 | 220 | ncs.Write(txbuf, 0, txbuf.Length); 221 | ncs.Flush(); 222 | 223 | var rx = new List(); 224 | 225 | var rxbuf = new byte[8192]; 226 | do 227 | { 228 | var rbytes = ncs.Read(rxbuf, 0, rxbuf.Length); 229 | rx.AddRange(rxbuf.Take(rbytes)); 230 | } while (!ncs.IsMessageComplete); 231 | 232 | if (rx.Count > 0) 233 | { 234 | resp = CClashMessage.Deserialize(rx.ToArray()); 235 | ncs.Close(); 236 | } 237 | return resp; 238 | } 239 | 240 | public DataHash DeriveHashKey(ICompiler comp, IEnumerable args) 241 | { 242 | throw new NotSupportedException(); 243 | } 244 | 245 | public string GetStats(string compiler) 246 | { 247 | var req = new CClashRequest() 248 | { 249 | cmd = Command.GetStats, 250 | compiler = compiler, 251 | }; 252 | 253 | var resp = Transact(req); 254 | return resp.stdout; 255 | } 256 | 257 | public void Dispose() 258 | { 259 | Dispose(true); 260 | } 261 | 262 | private void Dispose(bool disposing) 263 | { 264 | if (disposing) 265 | { 266 | ncs.Dispose(); 267 | } 268 | } 269 | } 270 | } 271 | -------------------------------------------------------------------------------- /CClash/CClashServerNotReadyException.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Text; 5 | using System.Threading.Tasks; 6 | 7 | namespace CClash 8 | { 9 | public class CClashServerNotReadyException : Exception 10 | { 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /CClash/CClashWarningException.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Text; 5 | using System.Threading.Tasks; 6 | 7 | namespace CClash 8 | { 9 | public class CClashWarningException : Exception 10 | { 11 | public CClashWarningException(string fmt, params object[] args) : base (string.Format(fmt,args)) 12 | { 13 | } 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /CClash/CacheInformation.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.IO; 3 | using System.Collections.Generic; 4 | using System.Linq; 5 | using System.Text; 6 | using System.Threading; 7 | using System.Threading.Tasks; 8 | 9 | namespace CClash 10 | { 11 | public sealed class FastCacheInfo : IDisposable, ICacheInfo 12 | { 13 | IFileCacheStore statstore; 14 | public FastCacheInfo(IFileCacheStore stats) 15 | { 16 | statstore = stats; 17 | } 18 | 19 | public long CacheHits 20 | { 21 | get; 22 | set; 23 | } 24 | 25 | public long CacheMisses 26 | { 27 | get; 28 | set; 29 | } 30 | 31 | public long CacheObjects 32 | { 33 | get; 34 | set; 35 | } 36 | 37 | public long CacheSize 38 | { 39 | get; 40 | set; 41 | } 42 | 43 | public long CacheUnsupported 44 | { 45 | get; 46 | set; 47 | } 48 | 49 | public void LockStatsCall(Action x) 50 | { 51 | x.Invoke(); 52 | } 53 | 54 | public long SlowHitCount 55 | { 56 | get; 57 | set; 58 | } 59 | 60 | public void Commit() 61 | { 62 | using (var real = new CacheInfo(statstore)) 63 | { 64 | real.LockStatsCall(() => 65 | { 66 | real.SlowHitCount += this.SlowHitCount; 67 | real.CacheUnsupported += this.CacheUnsupported; 68 | real.CacheSize += this.CacheSize; 69 | real.CacheObjects += this.CacheObjects; 70 | real.CacheMisses += this.CacheMisses; 71 | real.CacheHits += this.CacheHits; 72 | }); 73 | } 74 | } 75 | 76 | public void Dispose() 77 | { 78 | Commit(); 79 | } 80 | } 81 | 82 | public class CacheInfo : IDisposable, ICacheInfo 83 | { 84 | 85 | public const string K_Stats = "stats"; 86 | 87 | public const string F_StatObjects = "objects.txt"; 88 | public const string F_StatDiskUsage = "usage.txt"; 89 | public const string F_StatHits = "hits.txt"; 90 | public const string F_StatMiss = "misses.txt"; 91 | public const string F_StatUnsupported = "unsupported.txt"; 92 | 93 | public const string F_StatSlowHits = "slow_hits.txt"; 94 | public const string F_CacheVersion = "version.txt"; 95 | public const string F_CacheType = "type.txt"; 96 | public const string F_CacheSchema = "schema.txt"; 97 | 98 | public const string CacheFormat = "v7a"; 99 | 100 | IFileCacheStore cache; 101 | Mutex statMtx = null; 102 | 103 | public void Commit() 104 | { 105 | 106 | } 107 | 108 | public CacheInfo(IFileCacheStore statCache) 109 | { 110 | cache = statCache; 111 | Logging.Emit("creating cache info mutex"); 112 | statMtx = new Mutex(false, "cclash_stat_" + cache.FolderPath.ToLower().GetHashCode()); 113 | Logging.Emit("created cache info mutex"); 114 | } 115 | 116 | 117 | public void LockStatsCall(Action x) 118 | { 119 | statMtx.WaitOne(); 120 | x.Invoke(); 121 | statMtx.ReleaseMutex(); 122 | } 123 | 124 | public long ReadStat(string statfile) 125 | { 126 | try 127 | { 128 | cache.EnsureKey(K_Stats); 129 | using (var stats = cache.OpenFileStream(K_Stats, statfile, FileMode.Open, FileAccess.Read)) 130 | { 131 | using (var sr = new StreamReader(stats)) 132 | { 133 | var x = sr.ReadToEnd(); 134 | return Int64.Parse(x); 135 | } 136 | } 137 | } 138 | catch 139 | { 140 | return 0; 141 | } 142 | } 143 | 144 | public void WriteStat(string statfile, long value) 145 | { 146 | cache.EnsureKey(K_Stats); 147 | using (var stats = cache.OpenFileStream(K_Stats, statfile, FileMode.OpenOrCreate, FileAccess.Write)) 148 | { 149 | using (var sw = new StreamWriter(stats)) 150 | { 151 | sw.Write(value.ToString()); 152 | } 153 | } 154 | } 155 | 156 | public long SlowHitCount 157 | { 158 | get 159 | { 160 | return ReadStat(F_StatSlowHits); 161 | } 162 | set 163 | { 164 | WriteStat(F_StatSlowHits, value); 165 | } 166 | } 167 | 168 | public long CacheHits 169 | { 170 | get 171 | { 172 | return ReadStat(F_StatHits); 173 | } 174 | set 175 | { 176 | WriteStat(F_StatHits, value); 177 | } 178 | } 179 | 180 | public long CacheSize 181 | { 182 | get 183 | { 184 | return ReadStat(F_StatDiskUsage); 185 | } 186 | set 187 | { 188 | WriteStat(F_StatDiskUsage, value); 189 | } 190 | } 191 | 192 | public long CacheMisses 193 | { 194 | get 195 | { 196 | return ReadStat(F_StatMiss); 197 | } 198 | set 199 | { 200 | WriteStat(F_StatMiss, value); 201 | } 202 | } 203 | 204 | public long CacheUnsupported 205 | { 206 | get 207 | { 208 | return ReadStat(F_StatUnsupported); 209 | } 210 | set 211 | { 212 | WriteStat(F_StatUnsupported, value); 213 | } 214 | } 215 | 216 | public long CacheObjects 217 | { 218 | get 219 | { 220 | return ReadStat(F_StatObjects); 221 | } 222 | set 223 | { 224 | WriteStat(F_StatObjects, value); 225 | } 226 | } 227 | 228 | private void Dispose(bool disposing) 229 | { 230 | if ( disposing ) statMtx.Dispose(); 231 | } 232 | 233 | public void Dispose() 234 | { 235 | Dispose(true); 236 | } 237 | 238 | } 239 | } 240 | -------------------------------------------------------------------------------- /CClash/CacheLockType.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Text; 5 | using System.Threading.Tasks; 6 | 7 | namespace CClash 8 | { 9 | public enum CacheLockType 10 | { 11 | Read, 12 | ReadWrite, 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /CClash/CacheManifest.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.IO; 3 | using System.Collections.Generic; 4 | using System.Linq; 5 | using System.Text; 6 | using System.Threading.Tasks; 7 | 8 | namespace CClash 9 | { 10 | [Serializable] 11 | public class CacheManifest : CClashMessage 12 | { 13 | public static CacheManifest Deserialize(Stream stream) 14 | { 15 | return CClashMessage.Deserialize(stream); 16 | } 17 | 18 | /// 19 | /// The next invocation of the compiler in this session (if any). 20 | /// 21 | public string NextOperation { get; set; } 22 | 23 | /// 24 | /// The request that created this entry. 25 | /// 26 | public CClashRequest Request { get; set; } 27 | 28 | /// 29 | /// Next time this job appears, just run the compiler. 30 | /// 31 | public bool Disable { get; set; } 32 | 33 | /// 34 | /// When this manifest was created. 35 | /// 36 | public string TimeStamp { get; set; } 37 | 38 | /// 39 | /// How long the original build took (msec). 40 | /// 41 | public int Duration { get; set; } 42 | 43 | /// 44 | /// Hash of the compiler, envs, cwd and args. 45 | /// 46 | public string SessionHash { get; set; } 47 | 48 | /// 49 | /// Hash of the pre-existing PDB file before this object was created. 50 | /// 51 | public string EarlierPdbHash { get; set; } 52 | 53 | /// 54 | /// non-null if this entry was made by preprocessing the source 55 | /// 56 | public string PreprocessedSourceHash { get; set; } 57 | 58 | /// 59 | /// The hash of any pdb file produced for this item 60 | /// 61 | public string PdbHash { get; set; } 62 | 63 | /// 64 | /// Hashes and names of each source file (includes and the source) 65 | /// 66 | public Dictionary IncludeFiles { get; set; } 67 | 68 | /// 69 | /// A list of files that did not exist but will require a rebuild if they are added. 70 | /// 71 | public List PotentialNewIncludes { get; set; } 72 | 73 | public int ExitCode { get; set; } 74 | 75 | public bool PPMode 76 | { 77 | get 78 | { 79 | return !String.IsNullOrEmpty(PreprocessedSourceHash); 80 | } 81 | } 82 | } 83 | } -------------------------------------------------------------------------------- /CClash/CacheMode.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | 3 | namespace CClash 4 | { 5 | public enum CacheMode 6 | { 7 | None = 0, 8 | Direct = 1, 9 | Preprocessor = 2, 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /CClash/CacheSessionContext.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.IO; 4 | using System.Linq; 5 | using System.Text; 6 | using System.Threading.Tasks; 7 | 8 | namespace CClash { 9 | public class CacheSessionContext 10 | { 11 | public ICompilerCache Cache { get; set; } 12 | public Compiler Compiler { get; set; } 13 | public StringBuilder StdOutput { get; set; } 14 | public StringBuilder StdError { get; set; } 15 | DateTime OperationStart { get; set; } 16 | 17 | public virtual bool IsSupported(IEnumerable args) { 18 | OperationStart = DateTime.Now; 19 | if (FileUtils.Exists(Compiler.CompilerExe)) 20 | { 21 | var rv = Compiler.ProcessArguments(args.ToArray()); 22 | if (!rv) { 23 | Logging.Emit("args not supported {0}", Cache.GetType().Name); 24 | } 25 | return rv; 26 | } 27 | throw new FileNotFoundException(Compiler.CompilerExe); 28 | } 29 | 30 | 31 | public void CopyOutputFiles(DataHash hc) 32 | { 33 | try { 34 | Cache.CopyFile(Cache.OutputCache.MakePath(hc.Hash, CompilerCacheBase.F_Object), Compiler.ObjectTarget); 35 | if (Compiler.GeneratePdb) 36 | Cache.CopyFile(Cache.OutputCache.MakePath(hc.Hash, CompilerCacheBase.F_Pdb), Compiler.PdbFile); 37 | } catch (Exception e) { 38 | Logging.Error("{0}", e); 39 | throw; 40 | } 41 | } 42 | 43 | public void CopyStdio(DataHash hc) 44 | { 45 | var stderrfile = Cache.OutputCache.MakePath(hc.Hash, CompilerCacheBase.F_Stderr); 46 | var stdoutfile = Cache.OutputCache.MakePath(hc.Hash, CompilerCacheBase.F_Stdout); 47 | 48 | StdOutput.Clear(); 49 | StdError.Clear(); 50 | StdOutput.Append(File.ReadAllText(stdoutfile)); 51 | StdError.Append(File.ReadAllText(stderrfile)); 52 | } 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /CClash/DirectCompilerCache.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.IO; 4 | 5 | namespace CClash 6 | { 7 | [System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Design", "CA1063:ImplementIDisposableCorrectly")] 8 | public class DirectCompilerCache : CompilerCacheBase, ICompilerCache 9 | { 10 | public DirectCompilerCache(string cacheFolder) 11 | : base(cacheFolder) 12 | { 13 | Logging.Emit("direct compiler cache"); 14 | } 15 | 16 | public IFileCacheStore OutputCache 17 | { 18 | get 19 | { 20 | return outputCache; 21 | } 22 | } 23 | 24 | public IFileCacheStore IncludeCache { 25 | get { 26 | return includeCache; 27 | } 28 | } 29 | 30 | public override void Setup() 31 | { 32 | } 33 | 34 | public override void Finished() 35 | { 36 | } 37 | 38 | /// 39 | /// When this returns, we will hold the output cache mutex. 40 | /// 41 | /// 42 | /// 43 | /// 44 | public override bool CheckCache(ICompiler comp, IEnumerable args, DataHash commonkey, out CacheManifest manifest ) 45 | { 46 | manifest = null; 47 | Lock(CacheLockType.Read); 48 | manifest = GetCachedManifestLocked(commonkey); 49 | if (manifest != null) 50 | { 51 | #region build missed before 52 | if (manifest.Disable) 53 | { 54 | Logging.Emit("disabled by manifest"); 55 | return false; 56 | } 57 | #region check includes 58 | foreach (var f in manifest.PotentialNewIncludes) 59 | { 60 | if (!FileUtils.FileMissing(f)) 61 | { 62 | Logging.Emit("detected added include file {0}", f); 63 | Logging.Miss(commonkey.SessionHash, DataHashResult.FileAdded, Directory.GetCurrentDirectory(), comp.SingleSourceFile, f); 64 | return false; 65 | } 66 | } 67 | var files = new List(); 68 | files.AddRange(manifest.IncludeFiles.Keys); 69 | var hashes = GetHashes(files, comp.WorkingDirectory); 70 | 71 | foreach (var h in hashes) 72 | { 73 | if (h.Value.Result == DataHashResult.Ok) 74 | { 75 | string mhash; 76 | if (manifest.IncludeFiles.TryGetValue(h.Key, out mhash)) 77 | { 78 | if (mhash != h.Value.Hash) 79 | { 80 | Logging.Emit("include file hash changed {0}", h.Key); 81 | Logging.Miss(commonkey.SessionHash, DataHashResult.FileChanged, Directory.GetCurrentDirectory(), comp.SingleSourceFile, h.Key); 82 | return false; 83 | } 84 | } 85 | else 86 | { 87 | Logging.Emit("include file added {0}", h.Key); 88 | Logging.Miss(commonkey.SessionHash, DataHashResult.FileAdded, Directory.GetCurrentDirectory(), comp.SingleSourceFile, h.Key); 89 | return false; 90 | } 91 | } 92 | else 93 | { 94 | Logging.Emit("include file hash error {0} {1}", h.Key, h.Value.Result); 95 | Logging.Miss(commonkey.SessionHash, h.Value.Result, Directory.GetCurrentDirectory(), comp.SingleSourceFile, h.Key); 96 | return false; 97 | } 98 | } 99 | #endregion 100 | 101 | #region check pdb 102 | if (comp.AttemptPdb) 103 | { 104 | if (comp.PdbExistsAlready) 105 | { 106 | var pdbhash = hasher.DigestBinaryFile(comp.PdbFile); 107 | if (pdbhash.Hash != manifest.EarlierPdbHash) 108 | { 109 | outputCache.Remove(commonkey.Hash); 110 | Logging.Miss(commonkey.Hash, DataHashResult.FileChanged, commonkey.Hash, comp.PdbFile, ""); 111 | return false; 112 | } 113 | } 114 | } 115 | #endregion 116 | 117 | #region check cached data exists 118 | foreach (var f in new string[] { F_Manifest, F_Object }) 119 | { 120 | if (!outputCache.ContainsEntry(commonkey.SessionHash, f)) 121 | { 122 | outputCache.Remove(commonkey.SessionHash); 123 | Logging.Miss(commonkey.SessionHash, DataHashResult.CacheCorrupt, commonkey.SessionHash, comp.SingleSourceFile, ""); 124 | return false; 125 | } 126 | } 127 | #endregion 128 | if (Settings.MissLogEnabled) 129 | { 130 | Logging.Emit("hit hc={0},dir={1},src={2}", commonkey.Hash, comp.WorkingDirectory, comp.SingleSourceFile); 131 | } 132 | return true; // cache hit, all includes match and no new files added 133 | #endregion 134 | } 135 | 136 | Logging.Miss(commonkey.Hash, DataHashResult.NoPreviousBuild, comp.WorkingDirectory, comp.SingleSourceFile, ""); 137 | return false; 138 | } 139 | 140 | TimeSpan lastCompileDuration = default(TimeSpan); 141 | 142 | string MakeTrackerFolderName() 143 | { 144 | return String.Format("cclash-track-{0}", Guid.NewGuid().ToString().Substring(0, 8)); 145 | } 146 | 147 | protected virtual int Compile(ICompiler comp, IEnumerable args, Stream stderr, Stream stdout, List includes) 148 | { 149 | #region compile 150 | var start = DateTime.Now; 151 | using (var stderrfs = new StreamWriter(stderr)) 152 | { 153 | using (var stdoutfs = new StreamWriter(stdout)) 154 | { 155 | if (Settings.TrackerMode) 156 | { 157 | comp.EnableTracker(MakeTrackerFolderName()); 158 | } 159 | var rv = CompileWithStreams(comp, args, stderrfs, stdoutfs, includes); 160 | lastCompileDuration = DateTime.Now.Subtract(start); 161 | 162 | return rv; 163 | } 164 | } 165 | #endregion 166 | } 167 | 168 | protected virtual void SaveOutputsLocked(CacheManifest m, ICompiler c ) 169 | { 170 | outputCache.AddFile(m.SessionHash, c.ObjectTarget, F_Object); 171 | if (c.GeneratePdb) 172 | { 173 | var pdbhash = hasher.DigestBinaryFile(c.PdbFile); 174 | m.PdbHash = pdbhash.Hash; 175 | outputCache.AddFile(m.SessionHash, c.PdbFile, F_Pdb); 176 | Stats.LockStatsCall(() => Stats.CacheSize += new FileInfo(c.PdbFile).Length); 177 | } 178 | 179 | Stats.LockStatsCall(() => Stats.CacheObjects++); 180 | Stats.LockStatsCall(() => Stats.CacheSize += new FileInfo(c.ObjectTarget).Length); 181 | 182 | // write manifest 183 | var duration = c.Age; 184 | m.Duration = (int)duration.TotalMilliseconds; 185 | 186 | Logging.Emit("cache miss took {0}ms", (int)duration.TotalMilliseconds); 187 | 188 | using (var manifest = outputCache.OpenFileStream(m.SessionHash, F_Manifest, FileMode.OpenOrCreate, FileAccess.Write)) 189 | { 190 | m.Serialize(manifest); 191 | } 192 | } 193 | 194 | protected override int OnCacheMissLocked(ICompiler comp, DataHash hc, IEnumerable args, CClashRequest req) 195 | { 196 | Logging.Emit("cache miss"); 197 | outputCache.EnsureKey(hc.Hash); 198 | var ifiles = new List(); 199 | Stats.LockStatsCall(() => Stats.CacheMisses++); 200 | using (var stderr = outputCache.OpenFileStream(hc.Hash, F_Stderr, FileMode.OpenOrCreate, FileAccess.Write)) 201 | using (var stdout = outputCache.OpenFileStream(hc.Hash, F_Stdout, FileMode.OpenOrCreate, FileAccess.Write)) 202 | { 203 | int rv = Compile(comp, args, stderr, stdout, ifiles); 204 | // we still hold the cache lock, create the manifest asap or give up now! 205 | 206 | if (rv != 0) 207 | { 208 | Unlock(CacheLockType.Read); 209 | } 210 | else 211 | { 212 | // this unlocks for us 213 | try 214 | { 215 | DoCacheMiss(comp, hc, args, req, ifiles); 216 | } 217 | catch (CClashWarningException) 218 | { 219 | return CompileOnly(comp, args); 220 | } 221 | } 222 | return rv; 223 | } 224 | } 225 | 226 | protected virtual void DoCacheMiss(ICompiler c, DataHash hc, IEnumerable args, CClashRequest req, List ifiles) 227 | { 228 | bool good = true; 229 | CacheManifest m = null; 230 | try 231 | { 232 | var idirs = c.GetUsedIncludeDirs(ifiles); 233 | if (idirs.Count < 1) 234 | { 235 | throw new InvalidDataException( 236 | string.Format("could not find any include folders?! [{0}]", 237 | string.Join(" ", args))); 238 | } 239 | #region process includes folders 240 | // save manifest and other things to cache 241 | var others = c.GetPotentialIncludeFiles(idirs, ifiles); 242 | m = new CacheManifest(); 243 | m.Request = req; 244 | m.PotentialNewIncludes = others; 245 | m.IncludeFiles = new Dictionary(); 246 | m.TimeStamp = DateTime.Now.ToString("s"); 247 | m.SessionHash = hc.SessionHash; 248 | 249 | #endregion 250 | 251 | var hashes = GetHashes(ifiles, c.WorkingDirectory); 252 | 253 | #region check include files 254 | 255 | foreach (var x in hashes) 256 | { 257 | if (x.Value.Result == DataHashResult.Ok) 258 | { 259 | m.IncludeFiles[x.Key] = x.Value.Hash; 260 | } 261 | else 262 | { 263 | Logging.Emit("input hash error {0} {1}", x.Key, x.Value.Result); 264 | Logging.Miss(hc.SessionHash, x.Value.Result, c.WorkingDirectory, c.SingleSourceFile, x.Key); 265 | good = false; 266 | m.Disable = true; 267 | break; 268 | } 269 | } 270 | 271 | #endregion 272 | } 273 | finally 274 | { 275 | Unlock(CacheLockType.Read); 276 | if (good) 277 | { 278 | Lock(CacheLockType.ReadWrite); 279 | try 280 | { 281 | if (m != null) 282 | { 283 | SaveOutputsLocked(m, c); 284 | } 285 | } 286 | finally 287 | { 288 | Unlock(CacheLockType.ReadWrite); 289 | } 290 | } 291 | } 292 | } 293 | } 294 | } 295 | -------------------------------------------------------------------------------- /CClash/DirectCompilerCacheServer.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.IO; 4 | using System.Text; 5 | using System.Linq; 6 | using System.Threading; 7 | 8 | namespace CClash 9 | { 10 | public class DirectCompilerCacheServer : DirectCompilerCache 11 | { 12 | Dictionary dwatchers = new Dictionary(); 13 | 14 | ReaderWriterLockSlim slimlock = new ReaderWriterLockSlim(); 15 | 16 | public DirectCompilerCacheServer(string cachedir) 17 | : base(cachedir) 18 | { 19 | SetupStats(); 20 | base.includeCache.CacheEntryChecksInMemory = true; 21 | Logging.Emit("server locking cache data"); 22 | base.Lock(CacheLockType.ReadWrite); // base is a multi-process lock, keep this forever 23 | } 24 | 25 | public override void Setup() 26 | { 27 | } 28 | 29 | public override void Finished() 30 | { 31 | } 32 | 33 | public override void Lock(CacheLockType mode) 34 | { 35 | if (mode == CacheLockType.Read) 36 | { 37 | slimlock.EnterReadLock(); 38 | } 39 | else 40 | { 41 | slimlock.EnterWriteLock(); 42 | } 43 | } 44 | 45 | public override void Unlock(CacheLockType mode) 46 | { 47 | if (mode == CacheLockType.Read) 48 | { 49 | slimlock.ExitReadLock(); 50 | } 51 | else 52 | { 53 | slimlock.ExitWriteLock(); 54 | } 55 | } 56 | 57 | public void WatchFile(string path) 58 | { 59 | if (!path.ToLower().Contains(":\\progra")) return; 60 | var dir = Path.GetDirectoryName(path); 61 | if ( !Path.IsPathRooted(dir) ) 62 | dir = Path.GetFullPath(dir); 63 | 64 | if (!Directory.Exists(dir)) 65 | { 66 | Logging.Error("ignored watch on missing folder {0}", dir); 67 | return; 68 | } 69 | 70 | DirectoryWatcher w; 71 | 72 | if (!dwatchers.TryGetValue(dir, out w)) 73 | { 74 | if (!FileExists(path)) 75 | { 76 | Logging.Error("ignored watch on missing file {0}", path); 77 | return; 78 | } 79 | 80 | Logging.Emit("create new watcher for {0}", dir); 81 | w = new DirectoryWatcher(dir); 82 | dwatchers.Add(dir, w); 83 | w.FileChanged += OnWatchedFileChanged; 84 | w.Enable(); 85 | } 86 | var file = Path.GetFileName(path); 87 | w.Watch(file); 88 | } 89 | 90 | public void UnwatchFile(string path) 91 | { 92 | var dir = Path.GetDirectoryName(path); 93 | DirectoryWatcher w; 94 | 95 | if (dwatchers.TryGetValue(dir, out w)) 96 | { 97 | var file = Path.GetFileName(path); 98 | w.UnWatch(file); 99 | if (w.Files.Count == 0) 100 | { 101 | dwatchers.Remove(dir); 102 | w.Dispose(); 103 | } 104 | } 105 | } 106 | 107 | public void OnWatchedFileChanged( object sender, FileChangedEventArgs args) 108 | { 109 | lock (hashcache) 110 | { 111 | hashcache.Remove(args.FilePath); 112 | } 113 | } 114 | 115 | public override bool FileExists(string path) 116 | { 117 | lock (hashcache) 118 | { 119 | if (hashcache.ContainsKey(path)) 120 | return true; 121 | } 122 | return base.FileExists(path); 123 | } 124 | 125 | public override DataHash DigestCompiler(string compilerPath) 126 | { 127 | lock (hashcache) 128 | { 129 | if (hashcache.ContainsKey(compilerPath)) 130 | return hashcache[compilerPath]; 131 | 132 | var h = base.DigestCompiler(compilerPath); 133 | 134 | WatchFile(compilerPath); 135 | 136 | hashcache.Add(compilerPath, h); 137 | 138 | return h; 139 | } 140 | } 141 | 142 | Dictionary hashcache = new Dictionary(); 143 | 144 | public override Dictionary GetHashes(IEnumerable fnames, string workdir) 145 | { 146 | if (hashcache.Count > 20000) 147 | { 148 | lock (hashcache) 149 | { 150 | hashcache.Clear(); 151 | } 152 | } 153 | 154 | var unknown = new List(); 155 | var rv = new Dictionary(); 156 | foreach (var n in fnames) 157 | { 158 | var x = n.ToLower(); 159 | lock (hashcache) 160 | { 161 | if (hashcache.ContainsKey(x)) 162 | { 163 | rv[x] = hashcache[x]; 164 | } 165 | else 166 | { 167 | unknown.Add(x); 168 | } 169 | } 170 | } 171 | 172 | if (unknown.Count > 0) 173 | { 174 | Logging.Emit("hash {0}/{1} new/changed files", unknown.Count, fnames.Count()); 175 | var tmp = base.GetHashes(fnames, workdir); 176 | lock (hashcache) 177 | { 178 | foreach (var filename in tmp.Keys) 179 | { 180 | var flow = filename.ToLower(); 181 | hashcache[flow] = tmp[filename]; 182 | rv[flow] = tmp[filename]; 183 | WatchFile(flow); 184 | } 185 | } 186 | } 187 | 188 | return rv; 189 | } 190 | 191 | public override int CompileOrCache(ICompiler comp, IEnumerable args, CClashRequest req) 192 | { 193 | return base.CompileOrCache(comp, args, req); 194 | } 195 | 196 | protected override void Dispose( bool disposing) 197 | { 198 | if (disposing) 199 | { 200 | foreach (var x in dwatchers) 201 | { 202 | x.Value.Dispose(); 203 | } 204 | dwatchers.Clear(); 205 | } 206 | base.Dispose(disposing); 207 | } 208 | 209 | public void SetupStats() 210 | { 211 | lock (outputCache) 212 | { 213 | if (Stats != null) 214 | Stats.Dispose(); 215 | Stats = new FastCacheInfo(outputCache); 216 | } 217 | } 218 | } 219 | } 220 | -------------------------------------------------------------------------------- /CClash/DirectoryWatcher.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.IO; 4 | 5 | namespace CClash 6 | { 7 | public class FileChangedEventArgs : EventArgs 8 | { 9 | public string FilePath { get; set; } 10 | } 11 | 12 | public sealed class DirectoryWatcher : IDisposable 13 | { 14 | FileSystemWatcher w; 15 | 16 | public delegate void FileChangedHandler(object sender, FileChangedEventArgs e); 17 | 18 | public DirectoryWatcher(string folder) 19 | { 20 | Files = new List(); 21 | w = new FileSystemWatcher(folder); 22 | w.Changed += (o, a) => { OnChange(a.FullPath); }; 23 | w.Deleted += (o, a) => { OnChange(a.FullPath); }; 24 | w.Renamed += (o, a) => { OnChange(a.FullPath); }; 25 | } 26 | 27 | public event FileChangedHandler FileChanged; 28 | 29 | public void Enable() 30 | { 31 | w.EnableRaisingEvents = true; 32 | } 33 | 34 | public void Watch(string file) 35 | { 36 | 37 | lock (Files) 38 | { 39 | if (!Files.Contains(file)) 40 | { 41 | Files.Add(file); 42 | } 43 | } 44 | } 45 | 46 | public void UnWatch(string file) 47 | { 48 | lock (Files) 49 | { 50 | if (Files.Contains(file)) 51 | Files.Remove(file); 52 | } 53 | } 54 | 55 | public List Files { get; private set; } 56 | 57 | void OnChange(string file) 58 | { 59 | file = Path.GetFileName(file); 60 | if (Files.Contains(file)) 61 | { 62 | if (FileChanged != null) 63 | { 64 | FileChanged(this, new FileChangedEventArgs() { FilePath = file }); 65 | } 66 | } 67 | } 68 | 69 | public void Dispose() 70 | { 71 | if (w.EnableRaisingEvents) 72 | { 73 | w.EnableRaisingEvents = false; 74 | w.Dispose(); 75 | w = null; 76 | } 77 | } 78 | } 79 | } 80 | -------------------------------------------------------------------------------- /CClash/ExtensionMethods.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Text; 3 | 4 | namespace CClash 5 | { 6 | public static class ExtensionMethods 7 | { 8 | 9 | public static void WriteLine(this StringBuilder sb, string fmt, params object[] args) 10 | { 11 | sb.AppendFormat(fmt, args); 12 | sb.AppendLine(); 13 | } 14 | 15 | public static TimeSpan Age(this DateTime dt) 16 | { 17 | return DateTime.Now.Subtract(dt); 18 | } 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /CClash/FileCacheBase.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Text; 5 | using System.Threading; 6 | using System.Threading.Tasks; 7 | 8 | namespace CClash 9 | { 10 | public class FileCacheBase : IDisposable 11 | { 12 | public string FolderPath { get; protected set; } 13 | 14 | Mutex mtx = null; 15 | 16 | protected void SetupLocks() 17 | { 18 | mtx = new Mutex(false, "cclash_mtx_" + FolderPath.ToLower().GetHashCode()); 19 | } 20 | 21 | public void WaitOne() 22 | { 23 | Logging.Emit("WaitOne {0}", FolderPath); 24 | if (!mtx.WaitOne()) throw new InvalidProgramException("mutex lock failed " + mtx.ToString()); 25 | } 26 | 27 | public void ReleaseMutex() 28 | { 29 | Logging.Emit("ReleaseMutex {0}", FolderPath); 30 | mtx.ReleaseMutex(); 31 | } 32 | 33 | public virtual void Dispose() 34 | { 35 | if (mtx != null) 36 | { 37 | mtx.Dispose(); 38 | } 39 | } 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /CClash/FileCacheDatabase.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.IO; 3 | using System.Collections.Generic; 4 | using System.Linq; 5 | using System.Text; 6 | using System.Data.SQLite; 7 | using System.Threading; 8 | 9 | namespace CClash 10 | { 11 | public class FileCacheDatabase : FileCacheBase, IFileCacheStore 12 | { 13 | const string DBFile = "cache.sqlite3"; 14 | const string DBSchema = "v5"; 15 | const int MaxConnections = 8; 16 | 17 | /* sqlite connections are not thread-safe! */ 18 | List connections = new List(); 19 | 20 | void MakeConnections() 21 | { 22 | while (connections.Count < MaxConnections) 23 | { 24 | var conn = OpenConnection(); 25 | connections.Add(conn); 26 | } 27 | } 28 | 29 | SQLiteConnection GetConnection() 30 | { 31 | SQLiteConnection conn; 32 | lock (connections) 33 | { 34 | MakeConnections(); 35 | conn = connections.First(); 36 | connections.RemoveAt(0); 37 | } 38 | return conn; 39 | } 40 | 41 | void ReturnConnection(SQLiteConnection conn) 42 | { 43 | ThreadPool.QueueUserWorkItem( (x) => { 44 | conn.Close(); 45 | } ); 46 | } 47 | 48 | string ReadTextMetafile(string metafile) 49 | { 50 | try 51 | { 52 | return File.ReadAllText(Path.Combine(FolderPath, metafile)); 53 | } 54 | catch { 55 | return String.Empty; 56 | } 57 | } 58 | 59 | void WriteTextMatafile(string metafile, string content) 60 | { 61 | File.WriteAllText(Path.Combine(FolderPath, metafile), content); 62 | } 63 | 64 | public void Open(string folderPath) 65 | { 66 | FolderPath = folderPath; 67 | bool created = false; 68 | base.SetupLocks(); 69 | 70 | while (true) 71 | { 72 | try 73 | { 74 | if (ReadTextMetafile(CacheInfo.F_CacheType) != "sqlite") 75 | { 76 | if (Directory.Exists(FolderPath)) 77 | Directory.Delete(FolderPath, true); 78 | } 79 | if (ReadTextMetafile(CacheInfo.F_CacheSchema) != DBSchema) 80 | { 81 | if (Directory.Exists(FolderPath)) 82 | Directory.Delete(FolderPath, true); 83 | } 84 | } 85 | catch (IOException) 86 | { 87 | System.Threading.Thread.Sleep(100); 88 | } 89 | break; 90 | } 91 | 92 | if (!Directory.Exists(FolderPath)) 93 | { 94 | created = true; 95 | Directory.CreateDirectory(FolderPath); 96 | } 97 | 98 | var conn = OpenConnection(); 99 | 100 | if (created) 101 | { 102 | WriteTextMatafile(CacheInfo.F_CacheType, "sqlite"); 103 | WriteTextMatafile(CacheInfo.F_CacheSchema, DBSchema); 104 | InitTables(conn); 105 | } 106 | } 107 | 108 | private SQLiteConnection OpenConnection() 109 | { 110 | var cs = new SQLiteConnectionStringBuilder(); 111 | cs.DataSource = Path.Combine(FolderPath, DBFile); 112 | cs.BusyTimeout = 1000; 113 | cs.Version = 3; 114 | var conn = new SQLiteConnection(cs.ToString()); 115 | conn.Open(); 116 | new SQLiteCommand("PRAGMA synchronous = OFF", conn).ExecuteNonQuery(); 117 | new SQLiteCommand("PRAGMA journal_mode = WAL", conn).ExecuteNonQuery(); 118 | return conn; 119 | } 120 | 121 | void InitTables(SQLiteConnection conn) 122 | { 123 | using (var txn = conn.BeginTransaction()) 124 | { 125 | var schema = @" 126 | CREATE TABLE IF NOT EXISTS cachedata 127 | ( 128 | hashkey TEXT NOT NULL, 129 | filename TEXT NOT NULL, 130 | filedata BLOB, 131 | CONSTRAINT hashitem PRIMARY KEY ( hashkey, filename ) 132 | )"; 133 | var cmd = new SQLiteCommand(schema, conn, txn); 134 | cmd.ExecuteNonQuery(); 135 | txn.Commit(); 136 | } 137 | } 138 | 139 | public event FileCacheStoreAddedHandler Added; 140 | 141 | public event FileCacheStoreRemovedHandler Removed; 142 | 143 | public bool CacheEntryChecksInMemory 144 | { 145 | get; 146 | set; 147 | } 148 | 149 | public void ClearLocked() 150 | { 151 | var conn = GetConnection(); 152 | try 153 | { 154 | using (var txn = conn.BeginTransaction()) 155 | { 156 | var sql = @"DELETE FROM cachedata"; 157 | var cmd = new SQLiteCommand(sql, conn, txn); 158 | cmd.ExecuteNonQuery(); 159 | txn.Commit(); 160 | } 161 | } 162 | finally 163 | { 164 | ReturnConnection(conn); 165 | } 166 | } 167 | 168 | public void EnsureKey(string key) 169 | { 170 | } 171 | 172 | Stream GetFileData(string key, string filename) 173 | { 174 | var sql = @"SELECT filedata FROM cachedata WHERE hashkey = @hk AND filename = @fn"; 175 | var conn = GetConnection(); 176 | try 177 | { 178 | var cmd = new SQLiteCommand(sql, conn); 179 | AddSQLiteParams(cmd, key, filename); 180 | var reader = cmd.ExecuteReader(); 181 | if (reader.Read()) 182 | { 183 | return reader.GetStream(0); 184 | } 185 | } 186 | finally 187 | { 188 | ReturnConnection(conn); 189 | } 190 | 191 | throw new System.IO.FileNotFoundException( 192 | String.Format("cachedata does not contain {0} in {1}", filename, key), filename); 193 | } 194 | 195 | public Stream OpenFileStream(string key, string filename, System.IO.FileMode mode, System.IO.FileAccess access) 196 | { 197 | switch (access) 198 | { 199 | case FileAccess.Read: 200 | return GetFileData(key, filename); 201 | case FileAccess.Write: 202 | switch (mode) 203 | { 204 | case FileMode.Open: 205 | throw new FileNotFoundException(); 206 | case FileMode.Append: 207 | throw new InvalidOperationException(); 208 | 209 | default: 210 | var ms = new FileCacheDatabaseWriteStream(); 211 | ms.HashKey = key; 212 | ms.Filename = filename; 213 | ms.Cache = this; 214 | return ms; 215 | } 216 | default: 217 | throw new InvalidOperationException(); 218 | } 219 | throw new NotImplementedException(); 220 | } 221 | 222 | void AddSQLiteParams(SQLiteCommand cmd, string key, string filename) 223 | { 224 | cmd.Parameters.Add(new SQLiteParameter("@hk", key)); 225 | if (filename != null) 226 | cmd.Parameters.Add(new SQLiteParameter("@fn", filename)); 227 | } 228 | 229 | 230 | public bool ContainsEntry(string key, string filename) 231 | { 232 | var sql = @"SELECT COUNT(filename) FROM cachedata WHERE hashkey = @hk AND filename = @fn"; 233 | var conn = GetConnection(); 234 | var cmd = new SQLiteCommand(sql, conn); 235 | AddSQLiteParams(cmd, key, filename); 236 | var output = (long) cmd.ExecuteScalar(); 237 | ReturnConnection(conn); 238 | return output == 1; 239 | } 240 | 241 | public void Remove(string key) 242 | { 243 | var conn = GetConnection(); 244 | using (var txn = conn.BeginTransaction()) 245 | { 246 | var sql = @"DELETE FROM cachedata WHERE hashkey = @hk"; 247 | var cmd = new SQLiteCommand(sql, conn, txn); 248 | AddSQLiteParams(cmd, key, null); 249 | cmd.ExecuteNonQuery(); 250 | txn.Commit(); 251 | } 252 | ReturnConnection(conn); 253 | } 254 | 255 | public void AddEntry(string key) 256 | { 257 | } 258 | 259 | public void AddFile(string key, string filePath, string contentName) 260 | { 261 | using (var fs = new FileStream(filePath, FileMode.Open)) 262 | { 263 | if (fs.Length > Int32.MaxValue) 264 | throw new InvalidDataException("file impossibly huge!"); 265 | ReplaceBinaryFileContent(key, filePath, fs); 266 | } 267 | } 268 | 269 | public void AddTextFileContent(string key, string filename, string content) 270 | { 271 | using (var ms = new MemoryStream(content.Length)) 272 | { 273 | var sw = new StreamWriter(ms); 274 | sw.Write(content); 275 | sw.Flush(); 276 | ReplaceBinaryFileContent(key, filename, ms); 277 | } 278 | } 279 | 280 | public void ReplaceBinaryFileContent(string key, string filename, Stream readfrom) 281 | { 282 | var conn = GetConnection(); 283 | using (var txn = conn.BeginTransaction()) 284 | { 285 | var sql = @"REPLACE INTO cachedata (hashkey, filename, filedata) VALUES (@hk, @fn, @dat)"; 286 | var cmd = new SQLiteCommand(sql, conn, txn); 287 | AddSQLiteParams(cmd, key, filename); 288 | var data = new SQLiteParameter("@dat", System.Data.DbType.Binary, (int)readfrom.Length); 289 | readfrom.Seek(0, SeekOrigin.Begin); 290 | var savebuf = new byte[readfrom.Length]; 291 | readfrom.Read(savebuf, 0, savebuf.Length); 292 | data.Value = savebuf; 293 | cmd.Parameters.Add(data); 294 | cmd.ExecuteNonQuery(); 295 | txn.Commit(); 296 | } 297 | ReturnConnection(conn); 298 | } 299 | 300 | public override void Dispose() 301 | { 302 | base.Dispose(); 303 | lock (connections) 304 | { 305 | foreach (var conn in connections) 306 | { 307 | conn.Close(); 308 | } 309 | connections.Clear(); 310 | } 311 | } 312 | } 313 | } 314 | -------------------------------------------------------------------------------- /CClash/FileCacheDatabaseWriteStream.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.IO; 3 | using System.Collections.Generic; 4 | using System.Linq; 5 | using System.Text; 6 | using System.Threading.Tasks; 7 | 8 | namespace CClash 9 | { 10 | public class FileCacheDatabaseWriteStream : Stream 11 | { 12 | public string HashKey { get; set; } 13 | public string Filename { get; set; } 14 | public FileCacheDatabase Cache { get; set; } 15 | 16 | public const int InitialStreamSize = 1024 * 1024; 17 | 18 | MemoryStream mem = new MemoryStream(); 19 | 20 | protected override void Dispose(bool disposing) 21 | { 22 | Close(); 23 | base.Dispose(disposing); 24 | if (mem != null) mem.Dispose(); 25 | mem = null; 26 | } 27 | 28 | public override void Close() 29 | { 30 | if (Cache != null) 31 | { 32 | Cache.ReplaceBinaryFileContent(HashKey, Filename, this); 33 | } 34 | } 35 | 36 | public override bool CanRead 37 | { 38 | get { return mem.CanRead; } 39 | } 40 | 41 | public override bool CanSeek 42 | { 43 | get { return mem.CanSeek; } 44 | } 45 | 46 | public override bool CanWrite 47 | { 48 | get { return mem.CanWrite; } 49 | } 50 | 51 | public override void Flush() 52 | { 53 | mem.Flush(); 54 | } 55 | 56 | public override long Length 57 | { 58 | get { return mem.Length; } 59 | } 60 | 61 | public override long Position 62 | { 63 | get 64 | { 65 | return mem.Position; 66 | } 67 | set 68 | { 69 | mem.Position = value; 70 | } 71 | } 72 | 73 | public override int Read(byte[] buffer, int offset, int count) 74 | { 75 | return mem.Read(buffer, offset, count); 76 | } 77 | 78 | public override long Seek(long offset, SeekOrigin origin) 79 | { 80 | return mem.Seek(offset, origin); 81 | } 82 | 83 | public override void SetLength(long value) 84 | { 85 | mem.SetLength(value); 86 | } 87 | 88 | public override void Write(byte[] buffer, int offset, int count) 89 | { 90 | mem.Write(buffer, offset, count); 91 | } 92 | } 93 | } 94 | -------------------------------------------------------------------------------- /CClash/FileCacheStore.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.IO; 4 | using System.Threading; 5 | 6 | namespace CClash 7 | { 8 | public class FileCacheStore : FileCacheBase, IFileCacheStore 9 | { 10 | public static IFileCacheStore Load(string cachefolder) where T : class, IFileCacheStore, new() 11 | { 12 | var cache = new T(); 13 | cache.Open(cachefolder); 14 | return cache; 15 | } 16 | 17 | public static IFileCacheStore Load(string cacheFolder) 18 | { 19 | var ctype = Settings.CacheType; 20 | switch (ctype) 21 | { 22 | case CacheStoreType.FileCache: 23 | return Load(cacheFolder); 24 | case CacheStoreType.SQLite: 25 | return Load(cacheFolder); 26 | default: 27 | throw new NotImplementedException(ctype.ToString()); 28 | } 29 | } 30 | 31 | public void Open( string folderPath ) 32 | { 33 | FolderPath = Path.GetFullPath(folderPath); 34 | SetupLocks(); 35 | Logging.Emit("locking file store: {0}", FolderPath); 36 | WaitOne(); 37 | 38 | var tlist = new List(); 39 | try 40 | { 41 | if (Directory.Exists(FolderPath)) 42 | { 43 | bool bad_cache_format = false; 44 | if (File.Exists(Path.Combine(FolderPath, CacheInfo.F_CacheVersion))) 45 | { 46 | var cdv = File.ReadAllText(Path.Combine(FolderPath, CacheInfo.F_CacheVersion)); 47 | bad_cache_format = cdv != CacheInfo.CacheFormat; 48 | } 49 | 50 | if (File.Exists(Path.Combine(FolderPath, CacheInfo.F_CacheType))) 51 | { 52 | var ct = File.ReadAllText(Path.Combine(FolderPath, CacheInfo.F_CacheType)); 53 | bad_cache_format = ct != Settings.CacheType.ToString(); 54 | } 55 | 56 | if (bad_cache_format) 57 | { 58 | Logging.Emit("corrupt/new filestore, deleting: {0}", FolderPath); 59 | Directory.Delete(FolderPath, true); 60 | } 61 | } 62 | 63 | if (!Directory.Exists(FolderPath)){ 64 | Logging.Emit("create fresh filestore"); 65 | Directory.CreateDirectory(FolderPath); 66 | File.WriteAllText(Path.Combine(FolderPath, CacheInfo.F_CacheVersion), CacheInfo.CacheFormat); 67 | File.WriteAllText(Path.Combine(FolderPath, CacheInfo.F_CacheType), Settings.CacheType.ToString()); 68 | } 69 | Logging.Emit("filestore ready: {0}", FolderPath); 70 | } 71 | catch (IOException) 72 | { 73 | throw new CClashErrorException("could not clear cache!"); 74 | } 75 | catch (UnauthorizedAccessException uae) 76 | { 77 | throw new CClashWarningException("cache access error: " + uae.Message); 78 | } 79 | finally 80 | { 81 | ReleaseMutex(); 82 | } 83 | } 84 | 85 | string MakePath(string key ) 86 | { 87 | var tlf = key.Substring(0, 2); 88 | return Path.Combine(FolderPath, tlf, key); 89 | } 90 | 91 | string MakePath(string key, string contentFile) 92 | { 93 | var tlf = key.Substring(0, 2); 94 | return Path.Combine(FolderPath, tlf, key, contentFile); 95 | } 96 | 97 | public event FileCacheStoreAddedHandler Added; 98 | 99 | public void EnsureKey(string key) 100 | { 101 | var kp = MakePath(key); 102 | if (!Directory.Exists(kp.Substring(0,2))) 103 | { 104 | Directory.CreateDirectory(kp.Substring(0,2)); 105 | } 106 | if (!Directory.Exists(kp)) 107 | { 108 | Directory.CreateDirectory(kp); 109 | } 110 | } 111 | 112 | public bool CacheEntryChecksInMemory 113 | { 114 | get; 115 | set; 116 | } 117 | 118 | public void ClearLocked() { 119 | entryCache.Clear(); 120 | var files = Directory.GetFiles(FolderPath); 121 | foreach (var f in files) 122 | File.Delete(f); 123 | var contents = Directory.GetDirectories(FolderPath); 124 | foreach (var d in contents) 125 | Directory.Delete(d, true); 126 | } 127 | 128 | HashSet entryCache = new HashSet(); 129 | 130 | public bool ContainsEntry(string key, string filename) 131 | { 132 | var p = MakePath(key, filename); 133 | if (CacheEntryChecksInMemory) 134 | { 135 | if (entryCache.Contains(p)) 136 | { 137 | return true; 138 | } 139 | } 140 | var rv = FileUtils.Exists(p); 141 | if (CacheEntryChecksInMemory && rv) 142 | { 143 | entryCache.Add(p); 144 | } 145 | return rv; 146 | } 147 | 148 | public Stream OpenFileStream(string key, string filename, FileMode mode, FileAccess access) 149 | { 150 | if (access == FileAccess.ReadWrite) throw new InvalidOperationException(); 151 | var fpath = MakePath(key, filename); 152 | switch(mode) { 153 | case FileMode.Create: 154 | case FileMode.CreateNew: 155 | var fdir = Path.GetDirectoryName(fpath); 156 | if (!Directory.Exists(fdir)) 157 | Directory.CreateDirectory(fdir); 158 | break; 159 | } 160 | return File.Open(fpath, mode, access); 161 | } 162 | 163 | public void AddEntry(string key) 164 | { 165 | EnsureKey(key); 166 | } 167 | 168 | public void AddFile(string key, string filePath, string contentName) 169 | { 170 | EnsureKey(key); 171 | var target = MakePath(key, contentName); 172 | 173 | FileUtils.CopyUnlocked(filePath, target); 174 | 175 | if (Added != null) 176 | { 177 | Added(this, new FileCacheStoreAddedEventArgs() { SizeKB = (int)(new FileInfo(filePath).Length / 1024) }); 178 | } 179 | } 180 | 181 | public void AddTextFileContent(string key, string filename, string content) 182 | { 183 | EnsureKey(key); 184 | FileUtils.WriteTextFile(MakePath(key, filename), content); 185 | if (Added != null) 186 | { 187 | Added(this, new FileCacheStoreAddedEventArgs() { SizeKB = content.Length * sizeof(char) }); 188 | } 189 | } 190 | 191 | public event FileCacheStoreRemovedHandler Removed; 192 | 193 | public void Remove(string key) 194 | { 195 | var p = MakePath(key); 196 | if (Directory.Exists(p)) 197 | { 198 | int sz = 0; 199 | var di = new DirectoryInfo(p); 200 | foreach (var f in di.GetFiles()) 201 | { 202 | sz += (int)(f.Length / 1024); 203 | } 204 | Directory.Delete(MakePath(key), true); 205 | if (Removed != null) 206 | { 207 | Removed(this, new FileCacheStoreRemovedEventArgs() { SizeKB = sz }); 208 | } 209 | } 210 | } 211 | } 212 | } 213 | -------------------------------------------------------------------------------- /CClash/FileUtils.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.IO; 4 | using System.Linq; 5 | using System.Runtime.InteropServices; 6 | using System.Text; 7 | using System.Threading.Tasks; 8 | 9 | namespace CClash { 10 | public sealed class FileUtils 11 | { 12 | 13 | public static bool Exists(string path) 14 | { 15 | return new FileInfo(path).Exists; 16 | } 17 | 18 | [DllImport("kernel32.dll", CharSet = CharSet.Auto, ThrowOnUnmappableChar = true, BestFitMapping = false)] 19 | static extern int GetLongPathName( 20 | [MarshalAs(UnmanagedType.LPTStr)] string path, 21 | [MarshalAs(UnmanagedType.LPTStr)] StringBuilder longPath, 22 | int longPathLength 23 | ); 24 | 25 | public static string ToLongPathName(string path) 26 | { 27 | if ( !string.IsNullOrWhiteSpace(path) && Path.IsPathRooted(path) && path.Contains("~")) 28 | { 29 | var sb = new StringBuilder(512); 30 | GetLongPathName(path, sb, sb.Capacity); 31 | return sb.ToString(); 32 | } 33 | return path; 34 | } 35 | 36 | // this should be much faster if a file doesn't exist 37 | [DllImport("Shlwapi.dll", SetLastError = true, CharSet = CharSet.Auto, BestFitMapping = false, ThrowOnUnmappableChar = true)] 38 | private extern static bool PathFileExists(StringBuilder path); 39 | 40 | static Dictionary recent_missing = new Dictionary(); 41 | 42 | public static bool FileMissing(string path) 43 | { 44 | DateTime mt; 45 | DateTime now = DateTime.Now; 46 | if (recent_missing.TryGetValue(path, out mt)) 47 | { 48 | if (now.Subtract(mt).TotalMilliseconds < 200) return true; 49 | } 50 | 51 | var sb = new StringBuilder(path); 52 | var missing = !PathFileExists(sb); 53 | if (missing) 54 | { 55 | lock (recent_missing) 56 | { 57 | if (recent_missing.Count > 5000) recent_missing.Clear(); 58 | recent_missing[path] = DateTime.Now; 59 | } 60 | } 61 | return missing; 62 | } 63 | 64 | static int FileIORetrySleep = 10; 65 | static int FileIORetryCount = 4; 66 | 67 | public static void WriteTextFile(string path, string content) 68 | { 69 | int attempts = FileIORetryCount; 70 | do 71 | { 72 | try 73 | { 74 | File.WriteAllText(path, content); 75 | return; 76 | } 77 | catch (IOException) 78 | { 79 | attempts--; 80 | if (attempts == 0) throw; 81 | System.Threading.Thread.Sleep(FileIORetrySleep); 82 | } 83 | } while (true); 84 | } 85 | 86 | [DllImport("Kernel32.dll", CharSet = CharSet.Unicode)] 87 | static extern bool CreateHardLink( 88 | string lpFileName, 89 | string lpExistingFileName, 90 | IntPtr lpSecurityAttributes 91 | ); 92 | 93 | public static void CopyUnlocked(string from, string to) 94 | { 95 | int attempts = FileIORetryCount; 96 | do 97 | { 98 | try 99 | { 100 | using (var ifs = new FileStream(from, FileMode.Open, FileAccess.Read, FileShare.Read, 4096, FileOptions.SequentialScan)) 101 | { 102 | using (var ofs = new FileStream(to, FileMode.Create, FileAccess.Write, FileShare.Write, 4096)) 103 | { 104 | ifs.CopyTo(ofs); 105 | return; 106 | } 107 | } 108 | } 109 | catch (IOException) 110 | { 111 | attempts--; 112 | if (attempts == 0) throw; 113 | System.Threading.Thread.Sleep(FileIORetrySleep); 114 | } 115 | } while (true); 116 | } 117 | 118 | [DllImport("kernel32.dll", CharSet = CharSet.Auto)] 119 | static extern int GetShortPathName( 120 | [MarshalAs(UnmanagedType.LPTStr)] 121 | string path, 122 | [MarshalAs(UnmanagedType.LPTStr)] 123 | StringBuilder shortPath, 124 | int shortPathLength 125 | ); 126 | 127 | public static string GetShortPath(string path) 128 | { 129 | var sb = new StringBuilder(255); 130 | GetShortPathName(path, sb, 255); 131 | return sb.ToString(); 132 | } 133 | } 134 | } 135 | -------------------------------------------------------------------------------- /CClash/HashUtil.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Linq; 3 | using System.Collections.Generic; 4 | using System.IO; 5 | using System.Runtime.Remoting.Metadata.W3cXsd2001; 6 | using System.Security.Cryptography; 7 | using System.Text.RegularExpressions; 8 | using System.Threading; 9 | 10 | 11 | namespace CClash 12 | { 13 | public enum DataHashResult 14 | { 15 | Ok, 16 | ContainsTimeOrDate, 17 | FileNotFound, 18 | AccessDenied, 19 | FileAdded, 20 | FileChanged, 21 | NoPreviousBuild, 22 | CacheCorrupt, 23 | } 24 | 25 | public sealed class DataHash 26 | { 27 | public string InputName { get; set; } 28 | public DataHashResult Result { get; set; } 29 | 30 | public string Hash { get; set; } 31 | 32 | // hash of everything (compiler, cwd, envs, args) except the source file content 33 | public string SessionHash { get; set; } 34 | // hash of the source file 35 | public string SourceHash { get; set; } 36 | public DateTime TimeStamp { get; set; } 37 | public bool Cached { get; set; } 38 | 39 | public DataHash() 40 | { 41 | TimeStamp = DateTime.Now; 42 | } 43 | 44 | public TimeSpan Age 45 | { 46 | get 47 | { 48 | return DateTime.Now.Subtract(TimeStamp); 49 | } 50 | } 51 | } 52 | 53 | public sealed class HashUtil 54 | { 55 | const string FindDateTimePattern = "__(TIM|DAT)E__"; 56 | const string F_HasDateTime = "hasdatetime"; 57 | const string F_NotDateTime = "notdatetime"; 58 | 59 | const int SavedHashMaxAgeMinutes = 3; 60 | 61 | static Regex FindDateTime = new Regex(FindDateTimePattern); 62 | 63 | IFileCacheStore includeCache; 64 | 65 | public HashUtil(IFileCacheStore includecache) { 66 | if (includecache == null) throw new ArgumentNullException("includecache"); 67 | includeCache = includecache; 68 | } 69 | 70 | private int hashingThreadCount = Settings.HashThreadCount; 71 | 72 | public int HashingThreadCount 73 | { 74 | get { return hashingThreadCount; } 75 | set { hashingThreadCount = value; } 76 | } 77 | 78 | Dictionary recentHashes = new Dictionary(); 79 | ReaderWriterLockSlim recentHashLock = new ReaderWriterLockSlim(); 80 | 81 | public DataHash DigestString(string input) 82 | { 83 | MD5CryptoServiceProvider md5 = new MD5CryptoServiceProvider(); 84 | 85 | var rv = new DataHash() 86 | { 87 | InputName = "string", 88 | Result = DataHashResult.Ok, 89 | Hash = new SoapHexBinary( md5.ComputeHash( System.Text.Encoding.Unicode.GetBytes( input ) ) ).ToString() 90 | }; 91 | return rv; 92 | } 93 | 94 | public DataHash DigestStream(Stream s) 95 | { 96 | MD5CryptoServiceProvider md5 = new MD5CryptoServiceProvider(); 97 | var rv = new DataHash() 98 | { 99 | InputName = "stream", 100 | Result = DataHashResult.Ok, 101 | Hash = new SoapHexBinary( md5.ComputeHash(s) ).ToString() 102 | }; 103 | return rv; 104 | } 105 | 106 | public Dictionary DigestFiles(IEnumerable files, string workdir) 107 | { 108 | var tohash = new List(); 109 | Dictionary rv = new Dictionary(); 110 | 111 | recentHashLock.EnterReadLock(); 112 | 113 | foreach (var f in files.Distinct()) 114 | { 115 | var filepath = f.ToLower(); 116 | if (!Path.IsPathRooted(filepath)) 117 | { 118 | if (!string.IsNullOrEmpty(workdir)) 119 | { 120 | filepath = Path.Combine(workdir, filepath); 121 | } 122 | } 123 | 124 | if (recentHashes.ContainsKey(filepath) && (recentHashes[filepath].Age.TotalMinutes < SavedHashMaxAgeMinutes)) 125 | { 126 | rv[filepath] = recentHashes[filepath]; 127 | rv[filepath].Cached = true; 128 | } 129 | else 130 | { 131 | tohash.Add(filepath); 132 | } 133 | } 134 | 135 | recentHashLock.ExitReadLock(); 136 | var newhashes = ThreadyDigestFiles(tohash, true); 137 | recentHashLock.EnterWriteLock(); 138 | foreach (var nh in newhashes) 139 | { 140 | rv[nh.Key] = nh.Value; 141 | recentHashes[nh.Key] = nh.Value; 142 | } 143 | recentHashLock.ExitWriteLock(); 144 | return rv; 145 | } 146 | 147 | public Dictionary ThreadyDigestFiles(IEnumerable files, bool stopOnUnCachable) 148 | { 149 | lock (includeCache) 150 | { 151 | var fcount = files.Count(); 152 | var rv = new Dictionary(); 153 | var threadcount = HashingThreadCount; 154 | if ((threadcount < 2) || (fcount < threadcount)) 155 | { 156 | Logging.Emit("st hash {0} files", fcount); 157 | foreach (var f in files) 158 | { 159 | var d = DigestSourceFile(f); 160 | rv[f.ToLower()] = d; 161 | if (d.Result != DataHashResult.Ok) break; 162 | } 163 | } 164 | else 165 | { 166 | Logging.Emit("mt hash {0} files on {1} threads", fcount, threadcount); 167 | var fa = files.ToArray(); 168 | var tl = new List(); 169 | var taken = 0; 170 | var chunk = (1 + fcount / (threadcount)); 171 | if (chunk < 1) chunk = 1; 172 | 173 | var inputs = new List(); 174 | 175 | do 176 | { 177 | var input = new ThreadyDigestInput() 178 | { 179 | files = fa, 180 | results = new List(), 181 | provider = new MD5CryptoServiceProvider(), 182 | begin = taken, 183 | chunksize = chunk, 184 | stopOnCachable = stopOnUnCachable, 185 | }; 186 | 187 | var t = new Thread(ThreadyDigestWorker); 188 | taken += chunk; 189 | t.Start(input); 190 | inputs.Add(input); 191 | tl.Add(t); 192 | } while (taken < fcount); 193 | 194 | for (var i = 0; i < tl.Count; i++) 195 | { 196 | var t = tl[i]; 197 | t.Join(); // thread finished, store it's results 198 | foreach (var h in inputs[i].results) 199 | { 200 | rv[h.InputName.ToLower()] = h; 201 | } 202 | } 203 | } 204 | 205 | return rv; 206 | } 207 | } 208 | 209 | struct ThreadyDigestInput 210 | { 211 | public int begin; 212 | public int chunksize; 213 | public string[] files; 214 | public List results; 215 | public MD5 provider; 216 | public bool stopOnCachable; 217 | } 218 | 219 | void ThreadyDigestWorker(object arg) 220 | { 221 | var input = (ThreadyDigestInput)arg; 222 | var files = input.files; 223 | var end = input.begin + input.chunksize; 224 | if (end > files.Length) end = files.Length; 225 | var rx = new Regex(FindDateTimePattern); 226 | var hashed = new List(); 227 | for ( var i = input.begin; i < end; i++ ) 228 | { 229 | var d = DigestFile( input.provider, files[i], rx ); 230 | hashed.Add(d); 231 | input.results.Add(d); 232 | if (input.stopOnCachable && d.Result != DataHashResult.Ok) break; 233 | } 234 | 235 | } 236 | 237 | public DataHash DigestSourceFile(string filepath) 238 | { 239 | return DigestFile(filepath, true); 240 | } 241 | 242 | public DataHash DigestBinaryFile(string filepath) 243 | { 244 | try { 245 | recentHashLock.EnterReadLock(); 246 | if (recentHashes.ContainsKey(filepath) && recentHashes[filepath].Age.TotalMinutes < SavedHashMaxAgeMinutes) 247 | { 248 | return recentHashes[filepath]; 249 | } 250 | } finally { 251 | recentHashLock.ExitReadLock(); 252 | } 253 | 254 | recentHashLock.EnterWriteLock(); 255 | var rv = DigestFile(filepath, false); 256 | recentHashes[filepath] = rv; 257 | recentHashLock.ExitWriteLock(); 258 | return rv; 259 | } 260 | 261 | DataHash DigestFile(string filepath, bool checkDateTime) 262 | { 263 | MD5CryptoServiceProvider md5 = new MD5CryptoServiceProvider(); 264 | return DigestFile(md5, filepath, checkDateTime ? FindDateTime : null); 265 | } 266 | 267 | DataHash DigestFile( MD5 provider, string filepath, Regex findDateTime) 268 | { 269 | var rv = new DataHash() { 270 | Result = DataHashResult.FileNotFound, 271 | InputName = filepath, 272 | }; 273 | 274 | if (!FileUtils.Exists(filepath)) return rv; 275 | provider.Initialize(); 276 | var fs = new FileStream(filepath, FileMode.Open, FileAccess.Read, FileShare.Read, 2048, FileOptions.SequentialScan); 277 | using (var bs = new BufferedStream(fs)) 278 | { 279 | Logging.Emit("digest {0}", filepath); 280 | rv.Hash = new SoapHexBinary(provider.ComputeHash(bs)).ToString(); 281 | rv.Result = DataHashResult.Ok; 282 | 283 | 284 | if (findDateTime != null) 285 | { 286 | bool mark_with_datetime = false; 287 | if (Settings.HonorCPPTimes) 288 | { 289 | // check include cache for this file 290 | if (includeCache.ContainsEntry(rv.Hash, F_NotDateTime)) 291 | { 292 | return rv; 293 | } 294 | if (includeCache.ContainsEntry(rv.Hash, F_HasDateTime)) 295 | { 296 | rv.Result = DataHashResult.ContainsTimeOrDate; 297 | return rv; 298 | } 299 | 300 | 301 | bs.Seek(0, SeekOrigin.Begin); 302 | using (var ts = new StreamReader(bs)) 303 | { 304 | string line = null; 305 | do 306 | { 307 | line = ts.ReadLine(); 308 | if (line != null) 309 | { 310 | if (findDateTime.IsMatch(line)) 311 | { 312 | mark_with_datetime = true; 313 | break; 314 | } 315 | } 316 | 317 | } while (line != null); 318 | } 319 | } 320 | 321 | includeCache.WaitOne(); 322 | if (!mark_with_datetime) 323 | { 324 | includeCache.AddTextFileContent(rv.Hash, F_NotDateTime, ""); 325 | } 326 | else 327 | { 328 | includeCache.AddTextFileContent(rv.Hash, F_HasDateTime, ""); 329 | } 330 | includeCache.ReleaseMutex(); 331 | 332 | } 333 | } 334 | rv.Result = DataHashResult.Ok; 335 | 336 | return rv; 337 | } 338 | } 339 | } 340 | -------------------------------------------------------------------------------- /CClash/ICacheInfo.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | namespace CClash 3 | { 4 | public interface ICacheInfo : IDisposable 5 | { 6 | long CacheHits { get; set; } 7 | long CacheMisses { get; set; } 8 | long CacheObjects { get; set; } 9 | long CacheSize { get; set; } 10 | long CacheUnsupported { get; set; } 11 | void LockStatsCall(Action x); 12 | long SlowHitCount { get; set; } 13 | void Commit(); 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /CClash/ICompiler.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Text; 3 | namespace CClash 4 | { 5 | public interface ICompiler 6 | { 7 | TimeSpan Age { get; } 8 | string AbsoluteSourceFile { get; } 9 | bool AttemptPdb { get; set; } 10 | System.Collections.Generic.List CliIncludePaths { get; } 11 | string[] CommandLine { get; set; } 12 | string[] CompileArgs { get; set; } 13 | string CompilerExe { get; set; } 14 | System.Collections.Generic.Dictionary EnvironmentVariables { get; } 15 | bool GeneratePdb { get; set; } 16 | System.Collections.Generic.List GetPotentialIncludeFiles(System.Collections.Generic.IEnumerable incdirs, System.Collections.Generic.IEnumerable incfiles); 17 | System.Collections.Generic.List GetUsedIncludeDirs(System.Collections.Generic.List files); 18 | bool HasDashC { get; } 19 | int InvokeCompiler(System.Collections.Generic.IEnumerable args, Action onStdErr, Action onStdOut, bool showIncludes, System.Collections.Generic.List foundIncludes); 20 | int InvokePreprocessor(System.IO.StreamWriter stdout); 21 | bool Linking { get; set; } 22 | string ObjectTarget { get; set; } 23 | int ParentPid { get; set; } 24 | bool PdbExistsAlready { get; set; } 25 | string PdbFile { get; set; } 26 | bool PrecompiledHeaders { get; set; } 27 | bool ProcessArguments(string[] args); 28 | string ResponseFile { get; set; } 29 | void SetEnvironment(System.Collections.Generic.Dictionary envs); 30 | void SetWorkingDirectory(string path); 31 | bool SingleSource { get; } 32 | string SingleSourceFile { get; } 33 | string[] SourceFiles { get; } 34 | string WorkingDirectory { get; } 35 | void EnableTracker(string folder); 36 | string TrackerFolder { get; } 37 | int ParallelCompilers { get; } 38 | 39 | string[] OnlyOptions { get; } 40 | string[] SourceFilesOptions { get; } 41 | 42 | Action StdErrorCallback { get; set; } 43 | Action StdOutputCallback { get; set; } 44 | 45 | } 46 | } 47 | 48 | -------------------------------------------------------------------------------- /CClash/ICompilerCache.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Text; 5 | using System.Threading.Tasks; 6 | 7 | namespace CClash 8 | { 9 | public interface ICompilerCache : IDisposable 10 | { 11 | ICacheInfo Stats { get; } 12 | IFileCacheStore OutputCache { get; } 13 | bool CheckCache(ICompiler comp, IEnumerable args, DataHash commonkey, out CacheManifest manifest); 14 | ICompiler SetCompiler(string compiler, string workdir, Dictionary envs); 15 | bool IsSupported( ICompiler comp, IEnumerable args); 16 | int CompileOrCache( ICompiler comp, IEnumerable args, CClashRequest req); 17 | void SetCaptureCallback(ICompiler comp, Action onOutput, Action onError); 18 | DataHash DeriveHashKey(ICompiler comp, IEnumerable args); 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /CClash/IFileCacheStore.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.IO; 3 | using System.Collections.Generic; 4 | using System.Linq; 5 | using System.Text; 6 | using System.Threading.Tasks; 7 | 8 | namespace CClash 9 | { 10 | public delegate void FileCacheStoreAddedHandler(object sender, FileCacheStoreAddedEventArgs e); 11 | public delegate void FileCacheStoreRemovedHandler(object sender, FileCacheStoreRemovedEventArgs e); 12 | 13 | 14 | public interface IFileCacheStore : IDisposable 15 | { 16 | void Open(string folderPath); 17 | 18 | void WaitOne(); 19 | void ReleaseMutex(); 20 | 21 | event FileCacheStoreAddedHandler Added; 22 | event FileCacheStoreRemovedHandler Removed; 23 | 24 | bool CacheEntryChecksInMemory { get; set; } 25 | string FolderPath { get; } 26 | 27 | void ClearLocked(); 28 | 29 | void EnsureKey(string key); 30 | Stream OpenFileStream(string key, string filename, FileMode mode, FileAccess access); 31 | 32 | bool ContainsEntry(string key, string filename); 33 | void Remove(string key); 34 | void AddEntry(string key); 35 | void AddFile(string key, string filePath, string contentName); 36 | void AddTextFileContent(string key, string filename, string content); 37 | } 38 | 39 | public class FileCacheStoreAddedEventArgs : EventArgs 40 | { 41 | public int SizeKB { get; set; } 42 | } 43 | 44 | public class FileCacheStoreRemovedEventArgs : EventArgs 45 | { 46 | public int SizeKB { get; set; } 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /CClash/Logging.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.IO; 4 | using System.Linq; 5 | using System.Text; 6 | using System.Threading; 7 | using System.Threading.Tasks; 8 | 9 | namespace CClash 10 | { 11 | public sealed class Logging 12 | { 13 | static int pid = System.Diagnostics.Process.GetCurrentProcess().Id; 14 | 15 | static object miss_log_sync = new object(); 16 | 17 | public static void Miss( string hc, DataHashResult reason, string dir, string srcfile, string headerfile) 18 | { 19 | switch (reason) 20 | { 21 | case DataHashResult.FileNotFound: 22 | break; 23 | default: 24 | break; 25 | } 26 | 27 | HitMissRecord(reason.ToString() + " hc=" + hc, dir, srcfile, headerfile); 28 | } 29 | 30 | public static void Hit( string hashkey, string dir, string obj) 31 | { 32 | if (Settings.DebugEnabled) 33 | { 34 | AppendMissLog(string.Format("HIT {0},dir={1},obj={2}", hashkey, dir, obj)); 35 | } 36 | } 37 | 38 | static void AppendMissLog(string str) 39 | { 40 | if (Settings.MissLogEnabled) 41 | { 42 | lock (miss_log_sync) 43 | { 44 | File.AppendAllText(Settings.MissLogFile, DateTime.Now.ToString("s") + str + Environment.NewLine); 45 | } 46 | } 47 | } 48 | 49 | static void HitMissRecord(string reason, string dir, string srcfile, string headerfile) { 50 | AppendMissLog(string.Format(" {0},dir={1},src={2},hdr={3}", 51 | reason, dir, srcfile, headerfile)); 52 | } 53 | 54 | public static void Warning(string fmt, params object[] args) 55 | { 56 | Console.Error.WriteLine(fmt, args); 57 | Logging.Emit("warning: {0}", string.Format(fmt, args)); 58 | } 59 | 60 | public static void Input(string dir, string target, IEnumerable args) 61 | { 62 | if (Settings.DebugEnabled) 63 | { 64 | var cfiles = from a in args where a.Contains(".c") select a; 65 | Logging.Emit("invoked: dir={0}, target={1} srcs={2}", dir, target, string.Join(",", cfiles.ToArray())); 66 | } 67 | } 68 | 69 | public static void Error(string fmt, params object[] args) 70 | { 71 | Console.Error.WriteLine(fmt, args); 72 | Logging.Emit("error: {0}", string.Format(fmt, args)); 73 | } 74 | 75 | public static void Emit(string fmt, params object[] args) 76 | { 77 | if (Settings.DebugEnabled) 78 | { 79 | for (int i = 0; i < 4; i++) 80 | { 81 | try 82 | { 83 | if (Settings.DebugFile == "Console") { 84 | Console.Error.WriteLine("p{0} t{1}:{2}", pid, 85 | Thread.CurrentThread.ManagedThreadId, 86 | string.Format(fmt, args)); 87 | } else { 88 | File.AppendAllLines(Settings.DebugFile, new string[] { pid + ":" + string.Format(fmt, args) }); 89 | } 90 | return; 91 | } 92 | catch {} 93 | } 94 | } 95 | } 96 | } 97 | } 98 | -------------------------------------------------------------------------------- /CClash/NullCompilerCache.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Text; 5 | using System.Threading.Tasks; 6 | 7 | namespace CClash { 8 | public class NullCompilerCache : CompilerCacheBase , ICompilerCache { 9 | 10 | public NullCompilerCache(string cachedir) 11 | : base() { 12 | } 13 | 14 | public override void Setup() { 15 | 16 | } 17 | 18 | public override void Finished() { 19 | 20 | } 21 | 22 | public IFileCacheStore OutputCache 23 | { 24 | get 25 | { 26 | return null; 27 | } 28 | } 29 | 30 | public override bool IsSupported(ICompiler comp, IEnumerable args) { 31 | return false; 32 | } 33 | 34 | public override bool CheckCache(ICompiler comp, IEnumerable args, DataHash commonkey, out CacheManifest manifest) 35 | { 36 | throw new NotImplementedException(); 37 | } 38 | 39 | protected override int OnCacheMissLocked(ICompiler comp, DataHash hc, IEnumerable args, CClashRequest req) 40 | { 41 | throw new NotImplementedException(); 42 | } 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /CClash/ParseTrackerFile.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.IO; 3 | using System.Collections.Generic; 4 | using System.Linq; 5 | using System.Text; 6 | using System.Threading.Tasks; 7 | 8 | namespace CClash 9 | { 10 | public class ParseTrackerFile 11 | { 12 | public static List GetFiles(string folder) 13 | { 14 | var files = new List(); 15 | foreach (var fname in System.IO.Directory.EnumerateFiles(folder)) { 16 | files.Add(fname); 17 | } 18 | return files; 19 | } 20 | 21 | public static List Parse(string filename) 22 | { 23 | var lines = System.IO.File.ReadAllLines(filename); 24 | var rv = new List(); 25 | foreach (var line in lines) 26 | { 27 | if (!line.StartsWith("#")) 28 | { 29 | var fname = line.Trim(); 30 | rv.Add(fname); 31 | } 32 | } 33 | return rv; 34 | } 35 | 36 | public static List ParseReads(string folder) 37 | { 38 | var found = new List(); 39 | foreach (var fname in GetFiles(folder)) 40 | { 41 | if (fname.Contains(".read.")) 42 | { 43 | found.AddRange(Parse(Path.Combine(folder, fname))); 44 | } 45 | } 46 | return found; 47 | } 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /CClash/ProcessUtils.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Management; 4 | using System.Diagnostics; 5 | namespace CClash 6 | { 7 | public class ProcessUtils 8 | { 9 | public string GetParentProcessName(int childpid) 10 | { 11 | var query = string.Format("SELECT ParentProcessId FROM Win32_Process WHERE ProcessId = {0}", childpid); 12 | var search = new ManagementObjectSearcher("root\\CIMV2", query); 13 | var results = search.Get().GetEnumerator(); 14 | if (!results.MoveNext()) return null; 15 | var queryObj = results.Current; 16 | uint parentId = (uint)queryObj["ParentProcessId"]; 17 | if (parentId > 0) 18 | { 19 | try 20 | { 21 | var parent = Process.GetProcessById((int)parentId); 22 | return parent.ProcessName; 23 | } 24 | catch (InvalidOperationException) 25 | { 26 | } 27 | } 28 | return null; 29 | } 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /CClash/Program.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Text; 5 | using System.Threading.Tasks; 6 | using System.IO; 7 | 8 | namespace CClash 9 | { 10 | public sealed class Program 11 | { 12 | public static StringBuilder MainStdErr = new StringBuilder(); 13 | public static StringBuilder MainStdOut = new StringBuilder(); 14 | 15 | public static CClashServer Server = null; 16 | 17 | public static bool WasHit { get; private set; } 18 | 19 | public static int Main(string[] args) 20 | { 21 | var start = DateTime.Now; 22 | WasHit = false; 23 | 24 | var dbg = Environment.GetEnvironmentVariable("CCLASH_DEBUG"); 25 | if (!string.IsNullOrEmpty(dbg)) 26 | { 27 | Settings.DebugFile = dbg; 28 | Settings.DebugEnabled = true; 29 | } 30 | 31 | if (Settings.DebugEnabled) 32 | { 33 | Logging.Emit("command line args:"); 34 | foreach (var a in args) 35 | { 36 | Logging.Emit("arg: {0}", a); 37 | } 38 | } 39 | 40 | if (args.Contains("--cclash-server")) 41 | { 42 | foreach (var opt in args) 43 | { 44 | switch (opt) 45 | { 46 | case "--attempt-pdb": 47 | Environment.SetEnvironmentVariable("CCLASH_ATTEMPT_PDB_CACHE", "yes"); 48 | break; 49 | case "--pdb-to-z7": 50 | Environment.SetEnvironmentVariable("CCLASH_Z7_OBJ", "yes"); 51 | break; 52 | case "--sqlite": 53 | Environment.SetEnvironmentVariable("CCLASH_CACHE_TYPE", "sqlite"); 54 | break; 55 | case "--debug": 56 | if (Settings.DebugFile == null) { 57 | Settings.DebugFile = "Console"; 58 | Settings.DebugEnabled = true; 59 | } 60 | break; 61 | default: 62 | if (opt.StartsWith("--cachedir=")) 63 | { 64 | var dir = opt.Substring(1 + opt.IndexOf('=')); 65 | dir = Path.GetFullPath(dir); 66 | Settings.CacheDirectory = dir; 67 | } 68 | break; 69 | } 70 | } 71 | 72 | 73 | if (Settings.DebugEnabled) 74 | { 75 | if (Settings.DebugFile != null && Settings.DebugFile != "Console") 76 | { 77 | Settings.DebugFile += ".serv"; 78 | } 79 | } 80 | 81 | Logging.Emit("starting in server mode"); 82 | Logging.Emit("cache dir is {0}", Settings.CacheDirectory); 83 | Logging.Emit("cache type is {0}", Settings.CacheType); 84 | 85 | if (Settings.DebugFile != "Console") { 86 | Logging.Emit("closing server console"); 87 | Console.Out.Close(); 88 | Console.Error.Close(); 89 | Console.In.Close(); 90 | } 91 | 92 | 93 | Server = new CClashServer(); 94 | if (Server.Preflight(Settings.CacheDirectory)) 95 | { 96 | Logging.Emit("server created"); 97 | Server.Listen(Settings.CacheDirectory); 98 | return 0; 99 | } 100 | else 101 | { 102 | Logging.Emit("another server is running.. quitting"); 103 | return 1; 104 | } 105 | } 106 | 107 | if (args.Contains("--cclash")) 108 | { 109 | Logging.Emit("maint mode"); 110 | Console.Error.WriteLine("cclash {0} (c) Ian Norton, April 2016", 111 | typeof(Program).Assembly.GetName().Version.ToString()); 112 | 113 | var compiler = Compiler.Find(); 114 | if (Settings.ServiceMode) 115 | { 116 | for (int i = 0; i < 3; i++) 117 | { 118 | try 119 | { 120 | var cc = new CClashServerClient(Settings.CacheDirectory); 121 | if (args.Contains("--stop")) 122 | { 123 | Console.Error.WriteLine("stopping server.."); 124 | cc.Transact(new CClashRequest() { cmd = Command.Quit }); 125 | } 126 | else { 127 | #region server commands 128 | if (args.Contains("--clear")) { 129 | cc.Transact(new CClashRequest() { cmd = Command.ClearCache }); 130 | } else if ( args.Contains("--disable") ){ 131 | cc.Transact(new CClashRequest() { cmd = Command.DisableCache }); 132 | } else if (args.Contains("--enable") ){ 133 | cc.Transact(new CClashRequest() { cmd = Command.EnableCache }); 134 | } else if (args.Contains("--start")) { 135 | Console.Out.WriteLine("starting server"); 136 | CClashServerClient.StartBackgroundServer(); 137 | } else { 138 | var stats = cc.Transact(new CClashRequest() { cmd = Command.GetStats }); 139 | Console.Out.WriteLine(stats.stdout); 140 | } 141 | return 0; 142 | 143 | #endregion 144 | } 145 | 146 | } 147 | catch (CClashErrorException ex) 148 | { 149 | Logging.Error(ex.Message); 150 | return -1; 151 | } 152 | catch (CClashWarningException) 153 | { 154 | System.Threading.Thread.Sleep(2000); 155 | } 156 | catch (CClashServerNotReadyException) 157 | { 158 | Logging.Emit("server not ready, try again"); 159 | return -1; 160 | } 161 | catch (IOException ex) 162 | { 163 | Logging.Error(ex.ToString()); 164 | return -1; 165 | } 166 | } 167 | } 168 | else 169 | { 170 | ICompiler comp; 171 | using (ICompilerCache cc = 172 | CompilerCacheFactory.Get(Settings.DirectMode, Settings.CacheDirectory, compiler, Environment.CurrentDirectory, Compiler.GetEnvironmentDictionary(), out comp)) 173 | { 174 | Console.Out.WriteLine(StatOutputs.GetStatsString(compiler, cc)); 175 | } 176 | } 177 | return 0; 178 | } 179 | 180 | var rv = RunBuild(args, start, AppendStdout, AppendStderr); 181 | if (rv != 0) 182 | { 183 | if (!Settings.NoAutoRebuild) 184 | { 185 | for (int i = 1; i < 4; i++) 186 | { 187 | MainStdErr.Clear(); 188 | MainStdOut.Clear(); 189 | rv = RunBuild(args, start, AppendStdout, AppendStderr); 190 | if (rv == 0) break; 191 | System.Threading.Thread.Sleep(100); 192 | } 193 | } 194 | } 195 | Console.Error.Write(MainStdErr.ToString()); 196 | Console.Out.Write(MainStdOut.ToString()); 197 | 198 | if (spawnServer) { 199 | Logging.Emit("server needs to be started"); 200 | } 201 | return rv; 202 | } 203 | 204 | static void AppendStderr(string str) 205 | { 206 | MainStdErr.Append(str); 207 | } 208 | 209 | static void AppendStdout(string str) 210 | { 211 | MainStdOut.Append(str); 212 | } 213 | 214 | static bool spawnServer = false; 215 | 216 | private static int RunBuild(string[] args, DateTime start, Action stdout, Action stderr) 217 | { 218 | Logging.Emit("client mode = {0}", Settings.ServiceMode); 219 | try 220 | { 221 | if (!Settings.Disabled) 222 | { 223 | string compiler = Compiler.Find(); 224 | if (compiler == null) 225 | throw new System.IO.FileNotFoundException("cant find real cl compiler"); 226 | 227 | var cachedir = Settings.CacheDirectory; 228 | Logging.Emit("compiler: {0}", compiler); 229 | ICompiler comp; 230 | using (ICompilerCache cc = 231 | CompilerCacheFactory.Get(Settings.DirectMode, cachedir, compiler, Environment.CurrentDirectory, Compiler.GetEnvironmentDictionary(), out comp)) 232 | { 233 | if (comp != null) spawnServer = true; 234 | cc.SetCaptureCallback(comp, stdout, stderr); 235 | long last_hits = 0; 236 | if (!Settings.ServiceMode) 237 | { 238 | last_hits = cc.Stats.CacheHits; 239 | } 240 | 241 | int res = cc.CompileOrCache(comp, args, null); 242 | 243 | if (!Settings.ServiceMode) 244 | { 245 | if (last_hits < cc.Stats.CacheHits) 246 | { 247 | WasHit = true; 248 | } 249 | } 250 | 251 | return res; 252 | } 253 | } 254 | else 255 | { 256 | Logging.Emit("disabled by environment"); 257 | } 258 | } 259 | catch (CClashWarningException e) 260 | { 261 | Logging.Warning(e.Message); 262 | } 263 | catch (Exception e) 264 | { 265 | Logging.Emit("{0} after {1} ms", e.GetType().Name, DateTime.Now.Subtract(start).TotalMilliseconds); 266 | Logging.Emit("{0} {1}", e.GetType().Name + " message: " + e.Message); 267 | #if DEBUG 268 | Logging.Error("Exception from cacher {0}!!!", e); 269 | #endif 270 | } 271 | 272 | int rv = -1; 273 | 274 | try 275 | { 276 | 277 | var c = new Compiler() 278 | { 279 | CompilerExe = Compiler.Find(), 280 | }; 281 | c.SetEnvironment(Compiler.GetEnvironmentDictionary()); 282 | c.SetWorkingDirectory(Environment.CurrentDirectory); 283 | rv = c.InvokeCompiler(args, stderr, stdout, false, null); 284 | Logging.Emit("exit {0} after {1} ms", rv, DateTime.Now.Subtract(start).TotalMilliseconds); 285 | } 286 | catch (CClashErrorException e) 287 | { 288 | Logging.Error(e.Message); 289 | throw; 290 | } 291 | catch (CClashWarningException e) 292 | { 293 | Logging.Warning(e.Message); 294 | } 295 | return rv; 296 | } 297 | } 298 | } 299 | -------------------------------------------------------------------------------- /CClash/Properties/AssemblyInfo.cs: -------------------------------------------------------------------------------- 1 | using System.Reflection; 2 | using System.Runtime.CompilerServices; 3 | using System.Runtime.InteropServices; 4 | 5 | // General Information about an assembly is controlled through the following 6 | // set of attributes. Change these attribute values to modify the information 7 | // associated with an assembly. 8 | [assembly: AssemblyTitle("cclash")] 9 | [assembly: AssemblyDescription("CClash Compiler Cache")] 10 | [assembly: AssemblyConfiguration("")] 11 | [assembly: AssemblyCompany("Ian Norton")] 12 | [assembly: AssemblyProduct("cclash")] 13 | [assembly: AssemblyCopyright("Copyright Ian Norton ©2016")] 14 | [assembly: AssemblyTrademark("")] 15 | [assembly: AssemblyCulture("")] 16 | 17 | // Setting ComVisible to false makes the types in this assembly not visible 18 | // to COM components. If you need to access a type in this assembly from 19 | // COM, set the ComVisible attribute to true on that type. 20 | [assembly: ComVisible(false)] 21 | 22 | // The following GUID is for the ID of the typelib if this project is exposed to COM 23 | [assembly: Guid("514178f6-3f3d-4dc9-8665-2be25fb92a4d")] 24 | 25 | // Version information for an assembly consists of the following four values: 26 | // 27 | // Major Version 28 | // Minor Version 29 | // Build Number 30 | // Revision 31 | // 32 | // You can specify all the values or you can default the Build and Revision Numbers 33 | // by using the '*' as shown below: 34 | // [assembly: AssemblyVersion("1.0.*")] 35 | [assembly: AssemblyVersion("0.3.17.*")] 36 | [assembly: AssemblyFileVersion("0.3.17.1")] 37 | -------------------------------------------------------------------------------- /CClash/Settings.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Text; 5 | using System.Threading.Tasks; 6 | 7 | namespace CClash 8 | { 9 | public enum CacheStoreType 10 | { 11 | FileCache, 12 | SQLite, 13 | } 14 | 15 | public sealed class Settings 16 | { 17 | public static bool DebugEnabled { get; set; } 18 | public static string DebugFile { get; set; } 19 | 20 | public static string MissLogFile { 21 | get 22 | { 23 | return Environment.GetEnvironmentVariable("CCLASH_MISSES"); 24 | } 25 | } 26 | public static bool MissLogEnabled { 27 | get 28 | { 29 | return !string.IsNullOrEmpty(MissLogFile); 30 | } 31 | } 32 | 33 | static Settings() { } 34 | 35 | static bool ConditionVarsAreTrue(string prefix) 36 | { 37 | var var = Environment.GetEnvironmentVariable(prefix + "_VAR"); 38 | if (!string.IsNullOrEmpty(var)) 39 | { 40 | var values = Environment.GetEnvironmentVariable(prefix + "_VALUES"); 41 | if (!string.IsNullOrEmpty(values)) 42 | { 43 | var check = Environment.GetEnvironmentVariable(var); 44 | var vlist = values.Split(','); 45 | foreach (var v in vlist) 46 | { 47 | if (v == check) return true; 48 | } 49 | } 50 | } 51 | return false; 52 | } 53 | 54 | /// 55 | /// Attempt to cache and restore PDB files. If the target PDB already exists then we will count 56 | /// that towards the common key and cache the file. If not we mark that it doesnt and cache the file. 57 | /// 58 | /// If on a subsequent run, the pdb already exists exactly as it was when we cached it or is missing then 59 | /// we allow a hit. 60 | /// 61 | /// This basically only works for pdb builds that were sequential. 62 | /// 63 | public static bool AttemptPDBCaching 64 | { 65 | get 66 | { 67 | return false; 68 | //TODO - fix other things before enabling this 69 | //return Environment.GetEnvironmentVariable("CCLASH_ATTEMPT_PDB_CACHE") == "yes"; 70 | } 71 | } 72 | 73 | /// 74 | /// When an object compilation with pdb generation (Zi) is requested. Instead 75 | /// generate embedded debug info (Z7). 76 | /// 77 | public static bool ConvertObjPdbToZ7 78 | { 79 | get 80 | { 81 | return Environment.GetEnvironmentVariable("CCLASH_Z7_OBJ") == "yes"; 82 | } 83 | } 84 | 85 | public static bool PipeSecurityEveryone { 86 | get { 87 | return Environment.GetEnvironmentVariable("CCLASH_LAX_PIPE") == "yes"; 88 | } 89 | } 90 | 91 | static bool EnabledByConditions() 92 | { 93 | return ConditionVarsAreTrue("CCLASH_ENABLE_WHEN"); 94 | } 95 | 96 | static bool DisabledByConditions() 97 | { 98 | return ConditionVarsAreTrue("CCLASH_DISABLE_WHEN"); 99 | } 100 | 101 | public static bool Disabled 102 | { 103 | get 104 | { 105 | var dis = !string.IsNullOrEmpty(Environment.GetEnvironmentVariable("CCLASH_DISABLED")); 106 | return (dis || DisabledByConditions()) && (!EnabledByConditions()); 107 | } 108 | } 109 | 110 | static string cachedir = null; 111 | public static string CacheDirectory 112 | { 113 | get 114 | { 115 | if (cachedir == null) 116 | { 117 | cachedir = Environment.GetEnvironmentVariable("CCLASH_DIR"); 118 | if (string.IsNullOrEmpty(cachedir)) 119 | { 120 | var appdata = Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData); 121 | cachedir = System.IO.Path.Combine(appdata, "cclash"); 122 | } 123 | if (cachedir.Contains('"')) 124 | cachedir = cachedir.Replace("\"", ""); 125 | 126 | cachedir.TrimEnd(new[] { '\\' }); 127 | if (ServiceMode) 128 | { 129 | cachedir = System.IO.Path.Combine(cachedir, "server"); 130 | } 131 | } 132 | return cachedir; 133 | } 134 | set 135 | { 136 | cachedir = value; 137 | } 138 | } 139 | 140 | public static bool PreprocessorMode 141 | { 142 | get 143 | { 144 | var dm = Environment.GetEnvironmentVariable("CCLASH_PPMODE"); 145 | if (dm != null) 146 | { 147 | return true; 148 | } 149 | return ConditionVarsAreTrue("CCLASH_PPMODE_WHEN"); 150 | } 151 | } 152 | 153 | public static bool IsCygwin { 154 | get { 155 | if (Environment.GetEnvironmentVariable("NO_CCLASH_CYGWIN_FIX") == null) 156 | return true; 157 | 158 | if (Environment.GetEnvironmentVariable("OSTYPE") == "cygwin") 159 | return true; 160 | return false; 161 | } 162 | } 163 | 164 | public static bool TrackerMode 165 | { 166 | get 167 | { 168 | string tmode = Environment.GetEnvironmentVariable("CCLASH_TRACKER_MODE"); 169 | return tmode == "yes"; 170 | } 171 | } 172 | 173 | public static bool DirectMode 174 | { 175 | get 176 | { 177 | return !PreprocessorMode && !TrackerMode; 178 | } 179 | } 180 | 181 | public static bool ServiceMode 182 | { 183 | get 184 | { 185 | return Environment.GetEnvironmentVariable("CCLASH_SERVER") != null; 186 | } 187 | } 188 | 189 | private static int hashThreadCount; 190 | public static int HashThreadCount 191 | { 192 | get 193 | { 194 | if (hashThreadCount == 0) hashThreadCount = Environment.ProcessorCount; 195 | return hashThreadCount; 196 | } 197 | set 198 | { 199 | hashThreadCount = value; 200 | } 201 | } 202 | 203 | public static bool NoAutoRebuild 204 | { 205 | get 206 | { 207 | return Environment.GetEnvironmentVariable("CCLASH_AUTOREBUILD") == "no"; 208 | } 209 | } 210 | 211 | static string GetString(string envvar, string defaultValue) 212 | { 213 | var env = Environment.GetEnvironmentVariable(envvar); 214 | if (env == null) return defaultValue; 215 | return env; 216 | } 217 | 218 | static int GetInteger(string envvar, int defaultValue) 219 | { 220 | int rv = defaultValue; 221 | var env = GetString(envvar, null); 222 | if (env != null) 223 | { 224 | Int32.TryParse(env, out rv); 225 | if (rv < 0) rv = 0; 226 | } 227 | return (int)rv; 228 | } 229 | 230 | public static int ServerQuitAfterIdleMinutes 231 | { 232 | get 233 | { 234 | return GetInteger("CCLASH_EXIT_IDLETIME", 0); 235 | } 236 | } 237 | 238 | public static int SlowObjectTimeout 239 | { 240 | get 241 | { 242 | return GetInteger("CCLASH_SLOWOBJ_TIMEOUT", 0); 243 | } 244 | } 245 | 246 | public static int MaxServerThreads 247 | { 248 | get 249 | { 250 | return GetInteger("CCLASH_MAX_SERVER_THREADS", 0); 251 | } 252 | } 253 | 254 | 255 | const CacheStoreType DefaultCacheType = CacheStoreType.FileCache; 256 | 257 | public static CacheStoreType CacheType 258 | { 259 | get 260 | { 261 | string st = GetString("CCLASH_CACHE_TYPE", "files"); 262 | switch (st) 263 | { 264 | case "sqlite": 265 | return CacheStoreType.SQLite; 266 | 267 | case "files": 268 | return CacheStoreType.FileCache; 269 | } 270 | return DefaultCacheType; 271 | } 272 | } 273 | 274 | public static bool HonorCPPTimes 275 | { 276 | get 277 | { 278 | return GetString("CCLASH_HONOR_CPP_TIMES", "yes") == "yes"; 279 | } 280 | } 281 | 282 | } 283 | } 284 | -------------------------------------------------------------------------------- /CClash/StatOutputs.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Text; 5 | using System.Threading.Tasks; 6 | 7 | namespace CClash 8 | { 9 | public class StatOutputs 10 | { 11 | public static string GetStatsString(string compiler, ICompilerCache cache) 12 | { 13 | var sb = new StringBuilder(); 14 | sb.WriteLine("compiler: {0}", compiler); 15 | sb.WriteLine("cachedir: {0}", Settings.CacheDirectory); 16 | sb.WriteLine("cachetype: {0}", Settings.CacheType); 17 | if (Settings.DebugEnabled) 18 | { 19 | sb.WriteLine("debug file: {0}", Settings.DebugFile); 20 | } 21 | if (Settings.Disabled) 22 | { 23 | sb.WriteLine("disabled: yes"); 24 | } 25 | else 26 | { 27 | sb.WriteLine("disabled: no"); 28 | } 29 | if (cache != null) 30 | { 31 | using (var stats = new CacheInfo(cache.OutputCache)) 32 | { 33 | sb.WriteLine("outputCache usage: {0} kb", (int)(stats.CacheSize / 1024)); 34 | sb.WriteLine("cached files: {0}", stats.CacheObjects); 35 | sb.WriteLine("hits: {0}", stats.CacheHits); 36 | sb.WriteLine("misses: {0}", stats.CacheMisses); 37 | sb.WriteLine("unsupported: {0}", stats.CacheUnsupported); 38 | sb.WriteLine("slow hits: {0}", stats.SlowHitCount); 39 | } 40 | } 41 | return sb.ToString(); 42 | } 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /CClash/packages.config: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | 5 | 6 | 7 | 8 | -------------------------------------------------------------------------------- /benchmark/get_vc_envs.bat: -------------------------------------------------------------------------------- 1 | call %1 2 | set -------------------------------------------------------------------------------- /benchmark/test_performance.py: -------------------------------------------------------------------------------- 1 | """ 2 | Test cclash speed 3 | """ 4 | import sys 5 | import os 6 | import pytest 7 | import subprocess 8 | import threading 9 | import test_performance_openssl as tpo 10 | 11 | THISDIR = os.path.dirname(os.path.abspath(__file__)) 12 | CCLASH_BIN = os.path.join(os.path.dirname(THISDIR), "cclash", "bin", "debug") 13 | CCLASH_EXE = os.path.join(CCLASH_BIN, "cl.exe") 14 | if not os.path.exists(CCLASH_EXE): 15 | CCLASH_BIN = os.path.join(os.path.dirname(THISDIR), "cclash", "bin", "release") 16 | CCLASH_EXE = os.path.join(CCLASH_BIN, "cl.exe") 17 | 18 | 19 | def run_server(): 20 | """ 21 | Run the cclash server 22 | :return: 23 | """ 24 | envs = setup_cclache_envs() 25 | try: 26 | print subprocess.check_output([CCLASH_EXE, "--cclash-server"], env=envs) 27 | except subprocess.CalledProcessError as cpe: 28 | print cpe.output 29 | raise 30 | 31 | 32 | def setup_module(): 33 | """ 34 | Before all tests 35 | :return: 36 | """ 37 | assert os.path.isfile(CCLASH_EXE), "you need to build a Debug cclash first" 38 | print "cclash is at {}".format(CCLASH_EXE) 39 | 40 | tpo.get_vc_envs() 41 | tpo.download_openssl() 42 | setup_module.server = threading.Thread(target=run_server) 43 | setup_module.server.start() 44 | setup_module.server = None 45 | 46 | 47 | def teardown_module(): 48 | """ 49 | Clean up the server 50 | :return: 51 | """ 52 | envs = setup_cclache_envs() 53 | subprocess.check_call([CCLASH_EXE, "--cclash", "--stop"], env=envs) 54 | 55 | 56 | def setup_function(request): 57 | """ 58 | Before each test 59 | :param request: 60 | :return: 61 | """ 62 | envs = setup_cclache_envs() 63 | tpo.setup_function(request) 64 | print "cachedir {}".format(envs["CCLASH_DIR"]) 65 | print subprocess.check_output([CCLASH_EXE, "--cclash"], env=envs) 66 | 67 | 68 | def setup_cclache_envs(): 69 | """ 70 | return a dict of envs suitable for cclache to work with 71 | :return: 72 | """ 73 | envs = dict(tpo.ENVS) 74 | cachedir = os.path.join(os.getcwd(), "cclache_cachedir") 75 | envs["CCLASH_DIR"] = cachedir 76 | envs["CCLASH_Z7_OBJ"] = "yes" 77 | envs["CCLASH_SERVER"] = "1" 78 | return envs 79 | 80 | 81 | def test_build_nocache(): 82 | """ 83 | Time an openssl build with no caching involved at all 84 | :return: 85 | """ 86 | tpo.build_openssl(None) 87 | 88 | 89 | def build_withcclache_cold(): 90 | """ 91 | Time an openssl build with a cold cache 92 | :return: 93 | """ 94 | envs = setup_cclache_envs() 95 | tpo.retry_delete(envs["CCLASH_DIR"]) 96 | tpo.build_openssl(CCLASH_BIN, envs) 97 | 98 | 99 | def test_build_withcclache_01_warm(): 100 | """ 101 | Time an openssl build with a warm cache 102 | :return: 103 | """ 104 | # 105 | # Benchmarking on my win10 AMD A6-3500 (3 core). 106 | # On a good run this is 12.5 mins total, 107 | # 108 | # approx 450 sec cold 109 | # approx 120 sec warm 110 | # 111 | # overhead is non-compiler configure or clean time 112 | # 113 | envs = setup_cclache_envs() 114 | print "-" * 80 115 | print "Start cold cache" 116 | print "-" * 80 117 | build_withcclache_cold() 118 | 119 | tpo.setup_function(None) 120 | print "-" * 80 121 | print "Start warm cache" 122 | print "-" * 80 123 | tpo.build_openssl(CCLASH_BIN, envs) 124 | 125 | 126 | if __name__ == "__main__": 127 | pytest.main(sys.argv[1:]) 128 | -------------------------------------------------------------------------------- /benchmark/test_performance_openssl.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | """ 3 | A py.test test that attempts to build openssl and benchmark the effect of clcache 4 | """ 5 | import sys 6 | import os 7 | import shutil 8 | import pytest 9 | import urllib 10 | import zipfile 11 | import subprocess 12 | import time 13 | 14 | 15 | OPENSSL_ZIP = "OpenSSL_1_0_2-stable.zip" 16 | OPENSSL_URL = "https://codeload.github.com/openssl/openssl/zip/OpenSSL_1_0_2-stable" 17 | 18 | THISDIR = os.path.dirname(os.path.abspath(__file__)) 19 | DISTDIR = os.path.join(os.path.dirname(THISDIR), "dist") 20 | CLCACHE = os.path.join(DISTDIR, "cl.exe") 21 | SOURCES = [] 22 | ENVS = dict(os.environ) 23 | os.chdir(THISDIR) 24 | 25 | 26 | def retry_delete(path): 27 | """ 28 | Repeatedly attempt to delete path 29 | :param path: 30 | :return: 31 | """ 32 | for _ in range(30): 33 | # antivirus might be busy in here.. 34 | try: 35 | shutil.rmtree(path) 36 | return 37 | except WindowsError: 38 | time.sleep(5) 39 | if os.path.exists(path): 40 | raise Exception("could not delete {}".format(path)) 41 | 42 | 43 | class BlockMessage(object): 44 | """ 45 | Little class to emit "begin .. end" messages for a block of code 46 | """ 47 | 48 | def __init__(self, message): 49 | self.started = 0 50 | self.message = message 51 | 52 | def __enter__(self): 53 | self.started = time.time() 54 | print "\n..begin {} .. ".format(self.message) 55 | 56 | def __exit__(self, exc_type, exc_val, exc_tb): 57 | result = "OK" 58 | if exc_val is not None: 59 | result = "ERROR" 60 | print "\n..end {} {}.. ({}sec)".format(self.message, result, 61 | time.time() - self.started) 62 | 63 | 64 | def download_openssl(): 65 | """ 66 | Get the openssl zip and unpack it 67 | """ 68 | if not os.path.exists(OPENSSL_ZIP): 69 | getoslsrc = urllib.URLopener() 70 | with BlockMessage("download openssl"): 71 | getoslsrc.retrieve(OPENSSL_URL, OPENSSL_ZIP + ".part") 72 | os.rename(OPENSSL_ZIP + ".part", OPENSSL_ZIP) 73 | 74 | 75 | def clean_openssl_build(): 76 | """ 77 | Unpack the openssl source, possibly deleting the previous one 78 | :return: 79 | """ 80 | with zipfile.ZipFile(OPENSSL_ZIP, "r") as unzip: 81 | folder = unzip.namelist()[0] 82 | if os.path.exists(folder): 83 | with BlockMessage("delete old openssl folder"): 84 | retry_delete(folder) 85 | 86 | with BlockMessage("unzip openssl"): 87 | unzip.extractall() 88 | 89 | if len(SOURCES) == 0: 90 | SOURCES.append(folder.rstrip("/")) 91 | 92 | 93 | def find_visual_studio(): 94 | """ 95 | Attempt to find vs 11 or vs 12 96 | :return: 97 | """ 98 | vcvers = ["13.0", "12.0", "11.0"] 99 | for vc in vcvers: 100 | vcdir = os.path.join("c:\\", "Program Files (x86)", 101 | "Microsoft Visual Studio {}".format(vc), 102 | "VC", "bin") 103 | vcvars = os.path.join(vcdir, "vcvars32.bat") 104 | if os.path.exists(vcvars): 105 | return vcdir, vcvars 106 | 107 | raise Exception("cannot find visual studio!") 108 | 109 | 110 | def configure_openssl(): 111 | """ 112 | Run the configure steps (requires perl) 113 | :return: 114 | """ 115 | with BlockMessage("configure openssl"): 116 | subprocess.check_call(["perl", 117 | "Configure", "VC-WIN32", "no-asm", "--prefix=c:\openssl"], 118 | env=ENVS, 119 | cwd=SOURCES[0]) 120 | 121 | with BlockMessage("generate makefiles"): 122 | subprocess.check_call([os.path.join("ms", "do_ms.bat")], 123 | shell=True, 124 | env=ENVS, 125 | cwd=SOURCES[0]) 126 | 127 | 128 | def setup_function(request): 129 | """ 130 | Ensure a clean build tree before each test 131 | :return: 132 | """ 133 | clean_openssl_build() 134 | configure_openssl() 135 | 136 | 137 | def get_vc_envs(): 138 | """ 139 | Get the visual studio dev env vars 140 | :return: 141 | """ 142 | _, vcvars = find_visual_studio() 143 | with BlockMessage("getting vc envs"): 144 | getenvs = subprocess.check_output([os.path.join(THISDIR, "get_vc_envs.bat"), vcvars]) 145 | for line in getenvs.splitlines(): 146 | if "=" in line: 147 | name, val = line.split("=", 1) 148 | ENVS[name.upper()] = val 149 | 150 | 151 | def setup_module(): 152 | """ 153 | Check that our exe has been built. 154 | :return: 155 | """ 156 | if not os.path.isfile(CLCACHE): 157 | pytest.fail("please build the exe first") 158 | get_vc_envs() 159 | download_openssl() 160 | 161 | 162 | def replace_wipe_cflags(filename): 163 | """ 164 | Open the nmake file given and turn off PDB generation for .obj files 165 | :param filename: 166 | :return: 167 | """ 168 | lines = [] 169 | with open(filename, "rb") as makefile: 170 | for line in makefile.readlines(): 171 | if line.startswith("APP_CFLAG="): 172 | lines.append("APP_CFLAG=") 173 | elif line.startswith("LIB_CFLAG="): 174 | lines.append("LIB_CFLAG=/Zl") 175 | else: 176 | lines.append(line.rstrip()) 177 | 178 | with open(filename, "wb") as makefile: 179 | for line in lines: 180 | makefile.write(line + "\r\n") 181 | 182 | 183 | def build_openssl(addpath=None, envs=ENVS, pdbs=False): 184 | """ 185 | Build openssl, optionally prefixing addpath to $PATH 186 | :param addpath: 187 | :param envs: env var dict to use 188 | :param pdbs: if False, turn off pdb generation in the makefile 189 | :return: 190 | """ 191 | nmakefile = os.path.join("ms", "nt.mak") 192 | if not pdbs: 193 | replace_wipe_cflags(os.path.join(SOURCES[0], nmakefile)) 194 | 195 | if addpath is not None: 196 | envs["PATH"] = addpath + os.pathsep + envs["PATH"] 197 | 198 | try: 199 | with BlockMessage("running nmake"): 200 | subprocess.check_output(["nmake", "-f", nmakefile], 201 | shell=True, 202 | env=envs, 203 | cwd=SOURCES[0]) 204 | except subprocess.CalledProcessError as cpe: 205 | print cpe.output 206 | raise 207 | 208 | 209 | def setup_clcache_envs(): 210 | """ 211 | return a dict of envs suitable for clcache to work with 212 | :return: 213 | """ 214 | envs = dict(ENVS) 215 | vcdir, _ = find_visual_studio() 216 | cachedir = os.path.join("clcache_cachedir") 217 | envs["CLCACHE_DIR"] = cachedir 218 | envs["CLCACHE_CL"] = os.path.join(vcdir, "cl.exe") 219 | return envs 220 | 221 | 222 | def test_build_nocache(): 223 | """ 224 | Time an openssl build with no caching involved at all 225 | :return: 226 | """ 227 | build_openssl(None) 228 | 229 | 230 | def test_build_withclcache_00_cold(): 231 | """ 232 | Time an openssl build with a cold cache 233 | :return: 234 | """ 235 | envs = setup_clcache_envs() 236 | retry_delete(envs["CLCACHE_DIR"]) 237 | build_openssl(DISTDIR, envs) 238 | test_build_withclcache_00_cold.success = True 239 | test_build_withclcache_00_cold.success = False 240 | 241 | 242 | def test_build_withclcache_01_warm(): 243 | """ 244 | Time an openssl build with a warm cache 245 | :return: 246 | """ 247 | assert test_build_withclcache_00_cold.success, "must run test_build_withclcache_00_cold first" 248 | envs = setup_clcache_envs() 249 | build_openssl(DISTDIR, envs) 250 | 251 | 252 | if __name__ == "__main__": 253 | pytest.main(sys.argv[1:]) 254 | -------------------------------------------------------------------------------- /build_openssl.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | """ 3 | Build openssl 4 | """ 5 | import os 6 | import sys 7 | import shutil 8 | import time 9 | import subprocess 10 | 11 | OSLVERSION = "1.0.2g" 12 | THISDIR = os.path.dirname(os.path.abspath(__file__)) 13 | DIST = "Release" 14 | STARTSERVER = True 15 | STATS = True 16 | TRACKER = "no" 17 | 18 | 19 | def envcheck(bindir=None): 20 | """ 21 | Check vcvars 22 | """ 23 | try: 24 | assert "LIB" in os.environ 25 | assert "VCINSTALLDIR" in os.environ 26 | except AssertionError: 27 | print "Run this from a visual studio command prompt" 28 | sys.exit(1) 29 | if bindir is None: 30 | bindir = os.path.join(THISDIR, "CClash", "bin", DIST) 31 | pathvar = os.getenv("PATH") 32 | os.environ["PATH"] = bindir + os.pathsep + pathvar 33 | os.environ["CCLASH_Z7_OBJ"] = "yes" 34 | os.environ["CCLASH_SERVER"] = "1" 35 | os.environ["CCLASH_TRACKER_MODE"] = TRACKER 36 | 37 | cachedir = os.path.join(THISDIR, "oslcache") 38 | os.environ["CCLASH_DIR"] = cachedir 39 | 40 | if STARTSERVER: 41 | try: 42 | subprocess.check_call(["cl", "--cclash", "--stop"]) 43 | except subprocess.CalledProcessError: 44 | pass 45 | subprocess.check_call(["cl", "--cclash", "--start"]) 46 | 47 | 48 | def build(): 49 | """ 50 | Build openssl using cclash 51 | """ 52 | if STATS: 53 | print subprocess.check_output(["cl", "--cclash"]) 54 | oslsrc = "openssl-" + OSLVERSION 55 | os.chdir(THISDIR) 56 | clean_build() 57 | 58 | sys.stdout.write(".. copying openssl source tree ..") 59 | shutil.copytree(os.path.join(THISDIR, oslsrc), 60 | os.path.join(THISDIR, "buildtemp")) 61 | print "done." 62 | 63 | os.chdir("buildtemp") 64 | 65 | sys.stdout.write(".. running Configure ..") 66 | subprocess.check_output(["perl", "Configure", "VC-WIN32", "no-asm", 67 | "--prefix=c:\openssl"]) 68 | print "done." 69 | 70 | sys.stdout.write(".. create makefiles ..") 71 | subprocess.check_output(["ms\\do_ms.bat"]) 72 | print "done." 73 | 74 | sys.stdout.write(".. starting build ..") 75 | started = time.time() 76 | subprocess.check_output(["nmake", "-f", "ms\\nt.mak"]) 77 | ended = time.time() 78 | print "done." 79 | print "total time = {}sec".format(int(ended - started)) 80 | 81 | 82 | def clean_build(): 83 | if os.path.exists("buildtemp"): 84 | print ".. move earlier build.." 85 | repeat = 4 86 | while repeat > 0: 87 | try: 88 | time.sleep(20) # antivirus might still be in here.. 89 | os.rename("buildtemp", "buildtemp." + str(time.time())) 90 | repeat = 0 91 | except Exception as err: 92 | print "cant move! " + str(err) 93 | repeat -= 1 94 | if repeat == 0: 95 | raise 96 | print ".. moved" 97 | 98 | 99 | def try_build(): 100 | """ 101 | Print errors when it goes wrong 102 | """ 103 | try: 104 | build() 105 | except subprocess.CalledProcessError as cpe: 106 | print cpe.output 107 | sys.exit(1) 108 | 109 | 110 | if __name__ == "__main__": 111 | if "--debug" in sys.argv: 112 | DIST = "Debug" 113 | if "--no-start" in sys.argv: 114 | STARTSERVER = False 115 | if "--tracker" in sys.argv: 116 | TRACKER = "yes" 117 | bindir = None 118 | 119 | for item in sys.argv[1:]: 120 | if os.path.isdir(item): 121 | bindir = item 122 | STATS = False 123 | break 124 | 125 | envcheck(bindir) 126 | try_build() 127 | try_build() 128 | try_build() 129 | -------------------------------------------------------------------------------- /cclash.sln: -------------------------------------------------------------------------------- 1 |  2 | Microsoft Visual Studio Solution File, Format Version 12.00 3 | # Visual Studio 2012 4 | Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "CClash", "cclash\CClash.csproj", "{7BCF13A4-C4C9-493D-898B-E89AEFC93C1A}" 5 | EndProject 6 | Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "CClashTests", "CClash.Tests\CClashTests.csproj", "{B4A548CB-16A3-4FE9-A0D9-76A66182E0B1}" 7 | EndProject 8 | Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution Items", "{88D43731-8971-4CCE-999F-B8FE7060AA8E}" 9 | ProjectSection(SolutionItems) = preProject 10 | build_openssl.py = build_openssl.py 11 | README.md = README.md 12 | benchmark\test_performance.py = benchmark\test_performance.py 13 | EndProjectSection 14 | EndProject 15 | Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "cct", "cct\cct.csproj", "{847712E4-AC50-4C76-8979-56880470CF67}" 16 | EndProject 17 | Global 18 | GlobalSection(SolutionConfigurationPlatforms) = preSolution 19 | CD_ROM|Any CPU = CD_ROM|Any CPU 20 | Debug|Any CPU = Debug|Any CPU 21 | DVD-5|Any CPU = DVD-5|Any CPU 22 | Release|Any CPU = Release|Any CPU 23 | SingleImage|Any CPU = SingleImage|Any CPU 24 | EndGlobalSection 25 | GlobalSection(ProjectConfigurationPlatforms) = postSolution 26 | {7BCF13A4-C4C9-493D-898B-E89AEFC93C1A}.CD_ROM|Any CPU.ActiveCfg = Release|Any CPU 27 | {7BCF13A4-C4C9-493D-898B-E89AEFC93C1A}.CD_ROM|Any CPU.Build.0 = Release|Any CPU 28 | {7BCF13A4-C4C9-493D-898B-E89AEFC93C1A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 29 | {7BCF13A4-C4C9-493D-898B-E89AEFC93C1A}.Debug|Any CPU.Build.0 = Debug|Any CPU 30 | {7BCF13A4-C4C9-493D-898B-E89AEFC93C1A}.DVD-5|Any CPU.ActiveCfg = Debug|Any CPU 31 | {7BCF13A4-C4C9-493D-898B-E89AEFC93C1A}.DVD-5|Any CPU.Build.0 = Debug|Any CPU 32 | {7BCF13A4-C4C9-493D-898B-E89AEFC93C1A}.Release|Any CPU.ActiveCfg = Release|Any CPU 33 | {7BCF13A4-C4C9-493D-898B-E89AEFC93C1A}.Release|Any CPU.Build.0 = Release|Any CPU 34 | {7BCF13A4-C4C9-493D-898B-E89AEFC93C1A}.SingleImage|Any CPU.ActiveCfg = Release|Any CPU 35 | {7BCF13A4-C4C9-493D-898B-E89AEFC93C1A}.SingleImage|Any CPU.Build.0 = Release|Any CPU 36 | {B4A548CB-16A3-4FE9-A0D9-76A66182E0B1}.CD_ROM|Any CPU.ActiveCfg = Release|Any CPU 37 | {B4A548CB-16A3-4FE9-A0D9-76A66182E0B1}.CD_ROM|Any CPU.Build.0 = Release|Any CPU 38 | {B4A548CB-16A3-4FE9-A0D9-76A66182E0B1}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 39 | {B4A548CB-16A3-4FE9-A0D9-76A66182E0B1}.Debug|Any CPU.Build.0 = Debug|Any CPU 40 | {B4A548CB-16A3-4FE9-A0D9-76A66182E0B1}.DVD-5|Any CPU.ActiveCfg = Debug|Any CPU 41 | {B4A548CB-16A3-4FE9-A0D9-76A66182E0B1}.DVD-5|Any CPU.Build.0 = Debug|Any CPU 42 | {B4A548CB-16A3-4FE9-A0D9-76A66182E0B1}.Release|Any CPU.ActiveCfg = Release|Any CPU 43 | {B4A548CB-16A3-4FE9-A0D9-76A66182E0B1}.Release|Any CPU.Build.0 = Release|Any CPU 44 | {B4A548CB-16A3-4FE9-A0D9-76A66182E0B1}.SingleImage|Any CPU.ActiveCfg = Release|Any CPU 45 | {B4A548CB-16A3-4FE9-A0D9-76A66182E0B1}.SingleImage|Any CPU.Build.0 = Release|Any CPU 46 | {847712E4-AC50-4C76-8979-56880470CF67}.CD_ROM|Any CPU.ActiveCfg = Release|Any CPU 47 | {847712E4-AC50-4C76-8979-56880470CF67}.CD_ROM|Any CPU.Build.0 = Release|Any CPU 48 | {847712E4-AC50-4C76-8979-56880470CF67}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 49 | {847712E4-AC50-4C76-8979-56880470CF67}.Debug|Any CPU.Build.0 = Debug|Any CPU 50 | {847712E4-AC50-4C76-8979-56880470CF67}.DVD-5|Any CPU.ActiveCfg = Debug|Any CPU 51 | {847712E4-AC50-4C76-8979-56880470CF67}.DVD-5|Any CPU.Build.0 = Debug|Any CPU 52 | {847712E4-AC50-4C76-8979-56880470CF67}.Release|Any CPU.ActiveCfg = Release|Any CPU 53 | {847712E4-AC50-4C76-8979-56880470CF67}.Release|Any CPU.Build.0 = Release|Any CPU 54 | {847712E4-AC50-4C76-8979-56880470CF67}.SingleImage|Any CPU.ActiveCfg = Release|Any CPU 55 | {847712E4-AC50-4C76-8979-56880470CF67}.SingleImage|Any CPU.Build.0 = Release|Any CPU 56 | EndGlobalSection 57 | GlobalSection(SolutionProperties) = preSolution 58 | HideSolutionNode = FALSE 59 | EndGlobalSection 60 | GlobalSection(Performance) = preSolution 61 | HasPerformanceSessions = true 62 | EndGlobalSection 63 | EndGlobal 64 | -------------------------------------------------------------------------------- /cct/App.config: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | 5 | 6 | -------------------------------------------------------------------------------- /cct/Program.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Text; 5 | using System.IO; 6 | using System.Threading; 7 | 8 | using CClash; 9 | 10 | namespace CClash.Tool 11 | { 12 | class Program 13 | { 14 | static int Main(string[] args) 15 | { 16 | if (args.Length > 0) 17 | Settings.CacheDirectory = Path.GetFullPath(args[0]); 18 | 19 | while (true) 20 | { 21 | CClashResponse resp = null; 22 | try 23 | { 24 | var cc = new CClashServerClient(Settings.CacheDirectory); 25 | resp = cc.Transact(new CClashRequest() { cmd = Command.GetStats }); 26 | } 27 | catch (CClashServerNotReadyException) 28 | { 29 | Console.Error.Write("."); 30 | } 31 | catch (Exception) 32 | { 33 | Console.Error.Write("e"); 34 | } 35 | 36 | if (resp != null) 37 | { 38 | Console.Clear(); 39 | Console.Out.WriteLine(DateTime.Now.ToString("s")); 40 | Console.Out.WriteLine(resp.stdout); 41 | } 42 | 43 | 44 | Thread.Sleep(1000); 45 | } 46 | } 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /cct/Properties/AssemblyInfo.cs: -------------------------------------------------------------------------------- 1 | using System.Reflection; 2 | using System.Runtime.CompilerServices; 3 | using System.Runtime.InteropServices; 4 | 5 | // General Information about an assembly is controlled through the following 6 | // set of attributes. Change these attribute values to modify the information 7 | // associated with an assembly. 8 | [assembly: AssemblyTitle("cct")] 9 | [assembly: AssemblyDescription("")] 10 | [assembly: AssemblyConfiguration("")] 11 | [assembly: AssemblyCompany("")] 12 | [assembly: AssemblyProduct("cct")] 13 | [assembly: AssemblyCopyright("Copyright © 2016")] 14 | [assembly: AssemblyTrademark("")] 15 | [assembly: AssemblyCulture("")] 16 | 17 | // Setting ComVisible to false makes the types in this assembly not visible 18 | // to COM components. If you need to access a type in this assembly from 19 | // COM, set the ComVisible attribute to true on that type. 20 | [assembly: ComVisible(false)] 21 | 22 | // The following GUID is for the ID of the typelib if this project is exposed to COM 23 | [assembly: Guid("c4ef3c91-6507-4d0c-940b-357a6eff0521")] 24 | 25 | // Version information for an assembly consists of the following four values: 26 | // 27 | // Major Version 28 | // Minor Version 29 | // Build Number 30 | // Revision 31 | // 32 | // You can specify all the values or you can default the Build and Revision Numbers 33 | // by using the '*' as shown below: 34 | // [assembly: AssemblyVersion("1.0.*")] 35 | [assembly: AssemblyVersion("1.0.0.0")] 36 | [assembly: AssemblyFileVersion("1.0.0.0")] 37 | -------------------------------------------------------------------------------- /cct/cct.csproj: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | 5 | Debug 6 | AnyCPU 7 | {847712E4-AC50-4C76-8979-56880470CF67} 8 | Exe 9 | Properties 10 | cct 11 | cct 12 | v4.5 13 | 512 14 | 15 | 16 | AnyCPU 17 | true 18 | full 19 | false 20 | bin\Debug\ 21 | DEBUG;TRACE 22 | prompt 23 | 4 24 | 25 | 26 | AnyCPU 27 | pdbonly 28 | true 29 | bin\Release\ 30 | TRACE 31 | prompt 32 | 4 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | {7bcf13a4-c4c9-493d-898b-e89aefc93c1a} 53 | CClash 54 | 55 | 56 | 57 | 64 | -------------------------------------------------------------------------------- /packages/repositories.config: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | 5 | 6 | --------------------------------------------------------------------------------