├── .gitignore ├── CodeClassifier.sln ├── CodeClassifier ├── .gitignore ├── CodeClassifier.cs ├── CodeClassifier.csproj ├── MatchTree.cs ├── Properties │ └── AssemblyInfo.cs ├── StringTokenizer │ ├── StringTokenizer.cs │ └── Token.cs ├── TokenNode.cs ├── dist │ ├── CodeClassifier.dll │ └── training-set │ │ ├── c++.hpp │ │ ├── c.c │ │ ├── coffee-script.coffee │ │ ├── csharp.cs │ │ ├── css.css │ │ ├── html.html │ │ ├── javascript.js │ │ ├── objective-c.m │ │ ├── python.py │ │ ├── ruby.rb │ │ └── shell.sh ├── temp │ ├── coffee-script.coffee │ ├── objective-c.m │ └── shell.sh └── training-set │ ├── c++ │ ├── NaClFile.cpp │ ├── c++.cpp │ └── hstest.cpp │ ├── c │ ├── adlist.c │ ├── async.c │ ├── c.c │ └── cluster.c │ ├── csharp │ ├── BasicAuthenticationFixture.cs │ ├── ChromiumWebBrowser.cs │ ├── MemberIdentifier.cs │ └── csharp.cs │ ├── css │ ├── css.css │ ├── new 6.txt │ ├── new 7.txt │ └── new 8.txt │ ├── html │ ├── html.html │ ├── index.html │ ├── options.html │ └── test.html │ ├── javascript │ ├── javascript.js │ ├── new 2.txt │ ├── new 3.txt │ ├── new 4.txt │ └── new 5.txt │ ├── python │ ├── new 2.txt │ ├── new 3.txt │ ├── new 4.txt │ └── python.py │ └── ruby │ ├── artifactory.rb │ ├── jekyll_steps.rb │ ├── ruby.rb │ └── travis.rb ├── LICENSE ├── README.md └── Test ├── App.config ├── App.xaml ├── App.xaml.cs ├── MainWindow.xaml ├── MainWindow.xaml.cs ├── Properties ├── Annotations.cs ├── AssemblyInfo.cs ├── Resources.Designer.cs ├── Resources.resx ├── Settings.Designer.cs └── Settings.settings └── Test.csproj /.gitignore: -------------------------------------------------------------------------------- 1 | Test/bin/ 2 | Test/obj/ 3 | CodeClassifier.sln.DotSettings 4 | *.user 5 | *.suo 6 | NaiveBayesClassifier/obj/ 7 | NaiveBayesClassifier/bin/ 8 | -------------------------------------------------------------------------------- /CodeClassifier.sln: -------------------------------------------------------------------------------- 1 |  2 | Microsoft Visual Studio Solution File, Format Version 12.00 3 | # Visual Studio 14 4 | VisualStudioVersion = 14.0.23107.0 5 | MinimumVisualStudioVersion = 10.0.40219.1 6 | Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "CodeClassifier", "CodeClassifier\CodeClassifier.csproj", "{099D3F70-56B6-4830-A9BC-A206412A16CE}" 7 | EndProject 8 | Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Test", "Test\Test.csproj", "{F69173C0-84FE-4881-8B81-329F6684CD82}" 9 | EndProject 10 | Global 11 | GlobalSection(SolutionConfigurationPlatforms) = preSolution 12 | Debug|Any CPU = Debug|Any CPU 13 | Debug|Mixed Platforms = Debug|Mixed Platforms 14 | Debug|x86 = Debug|x86 15 | Release|Any CPU = Release|Any CPU 16 | Release|Mixed Platforms = Release|Mixed Platforms 17 | Release|x86 = Release|x86 18 | EndGlobalSection 19 | GlobalSection(ProjectConfigurationPlatforms) = postSolution 20 | {099D3F70-56B6-4830-A9BC-A206412A16CE}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 21 | {099D3F70-56B6-4830-A9BC-A206412A16CE}.Debug|Any CPU.Build.0 = Debug|Any CPU 22 | {099D3F70-56B6-4830-A9BC-A206412A16CE}.Debug|Mixed Platforms.ActiveCfg = Debug|Any CPU 23 | {099D3F70-56B6-4830-A9BC-A206412A16CE}.Debug|Mixed Platforms.Build.0 = Debug|Any CPU 24 | {099D3F70-56B6-4830-A9BC-A206412A16CE}.Debug|x86.ActiveCfg = Debug|Any CPU 25 | {099D3F70-56B6-4830-A9BC-A206412A16CE}.Release|Any CPU.ActiveCfg = Release|Any CPU 26 | {099D3F70-56B6-4830-A9BC-A206412A16CE}.Release|Any CPU.Build.0 = Release|Any CPU 27 | {099D3F70-56B6-4830-A9BC-A206412A16CE}.Release|Mixed Platforms.ActiveCfg = Release|Any CPU 28 | {099D3F70-56B6-4830-A9BC-A206412A16CE}.Release|Mixed Platforms.Build.0 = Release|Any CPU 29 | {099D3F70-56B6-4830-A9BC-A206412A16CE}.Release|x86.ActiveCfg = Release|Any CPU 30 | {F69173C0-84FE-4881-8B81-329F6684CD82}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 31 | {F69173C0-84FE-4881-8B81-329F6684CD82}.Debug|Any CPU.Build.0 = Debug|Any CPU 32 | {F69173C0-84FE-4881-8B81-329F6684CD82}.Debug|Mixed Platforms.ActiveCfg = Debug|Any CPU 33 | {F69173C0-84FE-4881-8B81-329F6684CD82}.Debug|Mixed Platforms.Build.0 = Debug|Any CPU 34 | {F69173C0-84FE-4881-8B81-329F6684CD82}.Debug|x86.ActiveCfg = Debug|Any CPU 35 | {F69173C0-84FE-4881-8B81-329F6684CD82}.Release|Any CPU.ActiveCfg = Release|Any CPU 36 | {F69173C0-84FE-4881-8B81-329F6684CD82}.Release|Any CPU.Build.0 = Release|Any CPU 37 | {F69173C0-84FE-4881-8B81-329F6684CD82}.Release|Mixed Platforms.ActiveCfg = Release|Any CPU 38 | {F69173C0-84FE-4881-8B81-329F6684CD82}.Release|Mixed Platforms.Build.0 = Release|Any CPU 39 | {F69173C0-84FE-4881-8B81-329F6684CD82}.Release|x86.ActiveCfg = Release|Any CPU 40 | EndGlobalSection 41 | GlobalSection(SolutionProperties) = preSolution 42 | HideSolutionNode = FALSE 43 | EndGlobalSection 44 | EndGlobal 45 | -------------------------------------------------------------------------------- /CodeClassifier/.gitignore: -------------------------------------------------------------------------------- 1 | bin/ 2 | obj/ 3 | CodeClassifier.csproj.user 4 | CodeClassifier.v12.suo 5 | *.DotSettings 6 | *.user 7 | -------------------------------------------------------------------------------- /CodeClassifier/CodeClassifier.csproj: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | 5 | Debug 6 | AnyCPU 7 | {099D3F70-56B6-4830-A9BC-A206412A16CE} 8 | Library 9 | Properties 10 | CodeClassifier 11 | CodeClassifier 12 | v4.0 13 | 512 14 | 15 | 16 | 17 | true 18 | full 19 | false 20 | bin\Debug\ 21 | DEBUG;TRACE 22 | prompt 23 | 4 24 | 25 | 26 | pdbonly 27 | true 28 | bin\Release\ 29 | TRACE 30 | prompt 31 | 4 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | PreserveNewest 51 | 52 | 53 | PreserveNewest 54 | 55 | 56 | Component 57 | PreserveNewest 58 | 59 | 60 | PreserveNewest 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | PreserveNewest 69 | 70 | 71 | PreserveNewest 72 | 73 | 74 | PreserveNewest 75 | 76 | 77 | PreserveNewest 78 | 79 | 80 | 81 | 82 | PreserveNewest 83 | 84 | 85 | PreserveNewest 86 | 87 | 88 | PreserveNewest 89 | 90 | 91 | PreserveNewest 92 | 93 | 94 | PreserveNewest 95 | 96 | 97 | PreserveNewest 98 | 99 | 100 | PreserveNewest 101 | 102 | 103 | PreserveNewest 104 | 105 | 106 | PreserveNewest 107 | 108 | 109 | PreserveNewest 110 | 111 | 112 | PreserveNewest 113 | 114 | 115 | PreserveNewest 116 | 117 | 118 | PreserveNewest 119 | 120 | 121 | PreserveNewest 122 | 123 | 124 | PreserveNewest 125 | 126 | 127 | PreserveNewest 128 | 129 | 130 | PreserveNewest 131 | 132 | 133 | PreserveNewest 134 | 135 | 136 | PreserveNewest 137 | 138 | 139 | PreserveNewest 140 | 141 | 142 | PreserveNewest 143 | 144 | 145 | PreserveNewest 146 | 147 | 148 | PreserveNewest 149 | 150 | 151 | PreserveNewest 152 | 153 | 154 | 155 | 162 | -------------------------------------------------------------------------------- /CodeClassifier/MatchTree.cs: -------------------------------------------------------------------------------- 1 | namespace CodeClassifier 2 | { 3 | class MatchTree 4 | { 5 | public TokenNode MatchTreeRoot { get; set; } 6 | public string Language { get; set; } 7 | public double TotalPossibleScore { get; set; } 8 | 9 | public MatchTree(TokenNode matchTreeRoot, string language, double totalPossibleScore) 10 | { 11 | MatchTreeRoot = matchTreeRoot; 12 | Language = language; 13 | TotalPossibleScore = totalPossibleScore; 14 | } 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /CodeClassifier/Properties/AssemblyInfo.cs: -------------------------------------------------------------------------------- 1 | using System.Reflection; 2 | using System.Runtime.CompilerServices; 3 | using System.Runtime.InteropServices; 4 | 5 | // General Information about an assembly is controlled through the following 6 | // set of attributes. Change these attribute values to modify the information 7 | // associated with an assembly. 8 | [assembly: AssemblyTitle("CodeClassifier")] 9 | [assembly: AssemblyDescription("")] 10 | [assembly: AssemblyConfiguration("")] 11 | [assembly: AssemblyCompany("Microsoft")] 12 | [assembly: AssemblyProduct("CodeClassifier")] 13 | [assembly: AssemblyCopyright("Copyright © Microsoft 2015")] 14 | [assembly: AssemblyTrademark("")] 15 | [assembly: AssemblyCulture("")] 16 | 17 | // Setting ComVisible to false makes the types in this assembly not visible 18 | // to COM components. If you need to access a type in this assembly from 19 | // COM, set the ComVisible attribute to true on that type. 20 | [assembly: ComVisible(false)] 21 | 22 | // The following GUID is for the ID of the typelib if this project is exposed to COM 23 | [assembly: Guid("9b3908aa-08d6-41be-841c-0fe3663b8e01")] 24 | 25 | // Version information for an assembly consists of the following four values: 26 | // 27 | // Major Version 28 | // Minor Version 29 | // Build Number 30 | // Revision 31 | // 32 | // You can specify all the values or you can default the Build and Revision Numbers 33 | // by using the '*' as shown below: 34 | // [assembly: AssemblyVersion("1.0.*")] 35 | [assembly: AssemblyVersion("1.0.0.0")] 36 | [assembly: AssemblyFileVersion("1.0.0.0")] 37 | -------------------------------------------------------------------------------- /CodeClassifier/StringTokenizer/StringTokenizer.cs: -------------------------------------------------------------------------------- 1 | /********************************************************8 2 | * Author: Andrew Deren 3 | * Date: July, 2004 4 | * http://www.adersoftware.com 5 | * 6 | * StringTokenizer class. You can use this class in any way you want 7 | * as long as this header remains in this file. 8 | * 9 | **********************************************************/ 10 | 11 | using System; 12 | using System.Collections.Generic; 13 | using System.IO; 14 | 15 | namespace CodeClassifier.StringTokenizer 16 | { 17 | /// 18 | /// StringTokenizer tokenized string (or stream) into tokens. 19 | /// 20 | public class StringTokenizer 21 | { 22 | const char EOF = (char)0; 23 | 24 | int _line; 25 | int _column; 26 | int _pos; // position within data 27 | 28 | readonly string _data; 29 | 30 | bool _ignoreWhiteSpace; 31 | Dictionary _symbolChars; 32 | 33 | int _saveLine; 34 | int _saveCol; 35 | int _savePos; 36 | 37 | public StringTokenizer(TextReader reader) 38 | { 39 | if (reader == null) 40 | throw new ArgumentNullException("reader"); 41 | 42 | _data = reader.ReadToEnd(); 43 | 44 | Reset(); 45 | } 46 | 47 | public StringTokenizer(string data) 48 | { 49 | if (data == null) 50 | throw new ArgumentNullException("data"); 51 | 52 | _data = data; 53 | 54 | Reset(); 55 | } 56 | 57 | /// 58 | /// gets or sets which characters are part of TokenKind.Symbol 59 | /// 60 | public Dictionary SymbolChars 61 | { 62 | get { return _symbolChars; } 63 | set { _symbolChars = value; } 64 | } 65 | 66 | /// 67 | /// if set to true, white space characters will be ignored, 68 | /// but EOL and whitespace inside of string will still be tokenized 69 | /// 70 | public bool IgnoreWhiteSpace 71 | { 72 | get { return _ignoreWhiteSpace; } 73 | set { _ignoreWhiteSpace = value; } 74 | } 75 | 76 | private void Reset() 77 | { 78 | _ignoreWhiteSpace = false; 79 | _symbolChars = new Dictionary 80 | { 81 | {'=' , TokenKind.EqualsSign}, 82 | {'+' , TokenKind.PlusSign}, 83 | {'-' , TokenKind.Hyphen}, 84 | {'/' , TokenKind.Slash}, 85 | {',' , TokenKind.Comma}, 86 | {'.' , TokenKind.FullStop}, 87 | {'*' , TokenKind.Asterisk}, 88 | {'~' , TokenKind.Tilde}, 89 | {'!' , TokenKind.ExplanationMark}, 90 | {'@' , TokenKind.AtSign}, 91 | {'#' , TokenKind.Hash}, 92 | {'$' , TokenKind.Dollar}, 93 | {'%' , TokenKind.Percent}, 94 | {'^' , TokenKind.CircumflexAccent}, 95 | {'&' , TokenKind.Ampersand}, 96 | {'(' , TokenKind.LeftParenthesis}, 97 | {')' , TokenKind.RightParenthesis}, 98 | {'{' , TokenKind.LeftCurlyBracket}, 99 | {'}' , TokenKind.RightCurlyBracket}, 100 | {'[' , TokenKind.LeftBracket}, 101 | {']' , TokenKind.RightBracket}, 102 | {':' , TokenKind.Colon}, 103 | {';' , TokenKind.SemiColon}, 104 | {'<' , TokenKind.LessThanSign}, 105 | {'>' , TokenKind.GreaterThanSign}, 106 | {'?' , TokenKind.QuestionMark}, 107 | {'|' , TokenKind.VerticalLine}, 108 | {'\\', TokenKind.Backslash}, 109 | {'`' , TokenKind.GraveAccent}, 110 | {'\'', TokenKind.SingleQuote}, 111 | {'_' , TokenKind.Underscore} 112 | }; 113 | 114 | _line = 1; 115 | _column = 1; 116 | _pos = 0; 117 | } 118 | 119 | protected char La(int count) 120 | { 121 | return _pos + count >= _data.Length ? EOF : _data[_pos + count]; 122 | } 123 | 124 | protected char Consume() 125 | { 126 | char ret = _data[_pos]; 127 | _pos++; 128 | _column++; 129 | 130 | return ret; 131 | } 132 | 133 | protected Token CreateToken(TokenKind kind, string value) 134 | { 135 | return new Token(kind, value, _line, _column); 136 | } 137 | 138 | protected Token CreateToken(TokenKind kind) 139 | { 140 | string tokenData = _data.Substring(_savePos, _pos - _savePos); 141 | return new Token(kind, tokenData, _saveLine, _saveCol); 142 | } 143 | 144 | public Token Next() 145 | { 146 | ReadToken: 147 | 148 | char ch = La(0); 149 | switch (ch) 150 | { 151 | case EOF: 152 | return CreateToken(TokenKind.Eof, string.Empty); 153 | 154 | case ' ': 155 | case '\t': 156 | { 157 | if (_ignoreWhiteSpace) 158 | { 159 | Consume(); 160 | goto ReadToken; 161 | } 162 | return ReadWhitespace(); 163 | } 164 | case '0': 165 | case '1': 166 | case '2': 167 | case '3': 168 | case '4': 169 | case '5': 170 | case '6': 171 | case '7': 172 | case '8': 173 | case '9': 174 | return ReadNumber(); 175 | 176 | case '\r': 177 | { 178 | StartRead(); 179 | Consume(); 180 | if (La(0) == '\n') 181 | Consume(); // on DOS/Windows we have \r\n for new line 182 | 183 | _line++; 184 | _column = 1; 185 | 186 | return CreateToken(TokenKind.Eol); 187 | } 188 | case '\n': 189 | { 190 | StartRead(); 191 | Consume(); 192 | _line++; 193 | _column = 1; 194 | 195 | return CreateToken(TokenKind.Eol); 196 | } 197 | 198 | case '"': 199 | { 200 | return ReadDoubleQuotedString(); 201 | } 202 | 203 | case '\'': 204 | { 205 | return ReadSingleQuotedString(); 206 | } 207 | 208 | 209 | default: 210 | { 211 | if (Char.IsLetter(ch)) 212 | return ReadWord(); 213 | if (SymbolChars.ContainsKey(ch)) 214 | { 215 | StartRead(); 216 | Consume(); 217 | return CreateToken(SymbolChars[ch]); 218 | } 219 | StartRead(); 220 | Consume(); 221 | return CreateToken(TokenKind.Unknown); 222 | } 223 | } 224 | } 225 | 226 | /// 227 | /// save read point positions so that CreateToken can use those 228 | /// 229 | private void StartRead() 230 | { 231 | _saveLine = _line; 232 | _saveCol = _column; 233 | _savePos = _pos; 234 | } 235 | 236 | /// 237 | /// reads all whitespace characters (does not include newline) 238 | /// 239 | /// 240 | protected Token ReadWhitespace() 241 | { 242 | StartRead(); 243 | 244 | Consume(); // consume the looked-ahead whitespace char 245 | 246 | while (true) 247 | { 248 | char ch = La(0); 249 | if (ch == '\t' || ch == ' ') 250 | Consume(); 251 | else 252 | break; 253 | } 254 | 255 | return CreateToken(TokenKind.WhiteSpace); 256 | 257 | } 258 | 259 | /// 260 | /// reads number. Number is: DIGIT+ ("." DIGIT*)? 261 | /// 262 | /// 263 | protected Token ReadNumber() 264 | { 265 | StartRead(); 266 | 267 | bool hadDot = false; 268 | 269 | Consume(); // read first digit 270 | 271 | while (true) 272 | { 273 | char ch = La(0); 274 | if (Char.IsDigit(ch)) 275 | Consume(); 276 | else if (ch == '.' && !hadDot) 277 | { 278 | hadDot = true; 279 | Consume(); 280 | } 281 | else 282 | break; 283 | } 284 | 285 | return CreateToken(TokenKind.Number); 286 | } 287 | 288 | /// 289 | /// reads word. Word contains any alpha character or _ 290 | /// 291 | protected Token ReadWord() 292 | { 293 | StartRead(); 294 | 295 | Consume(); // consume first character of the word 296 | 297 | while (true) 298 | { 299 | char ch = La(0); 300 | if (Char.IsLetter(ch) || ch == '_') 301 | Consume(); 302 | else 303 | break; 304 | } 305 | 306 | return CreateToken(TokenKind.Word); 307 | } 308 | 309 | /// 310 | /// reads all characters until next " is found. 311 | /// If "" (2 quotes) are found, then they are consumed as 312 | /// part of the string 313 | /// 314 | /// 315 | protected Token ReadDoubleQuotedString() 316 | { 317 | StartRead(); 318 | 319 | Consume(); // read " 320 | 321 | while (true) 322 | { 323 | char ch = La(0); 324 | if (ch == EOF) 325 | break; 326 | if (ch == '\r') // handle CR in strings 327 | { 328 | Consume(); 329 | if (La(0) == '\n') // for DOS & windows 330 | Consume(); 331 | 332 | _line++; 333 | _column = 1; 334 | } 335 | else if (ch == '\n') // new line in quoted string 336 | { 337 | Consume(); 338 | 339 | _line++; 340 | _column = 1; 341 | } 342 | else if (ch == '"') 343 | { 344 | Consume(); 345 | if (La(0) != '"') 346 | break; // done reading, and this quotes does not have escape character 347 | Consume(); // consume second ", because first was just an escape 348 | } 349 | else 350 | Consume(); 351 | } 352 | 353 | return CreateToken(TokenKind.DoubleQuotedString); 354 | } 355 | 356 | /// 357 | /// reads all characters until next " is found. 358 | /// If "" (2 quotes) are found, then they are consumed as 359 | /// part of the string 360 | /// 361 | /// 362 | protected Token ReadSingleQuotedString() 363 | { 364 | StartRead(); 365 | 366 | Consume(); // read " 367 | 368 | while (true) 369 | { 370 | char ch = La(0); 371 | if (ch == EOF) 372 | break; 373 | if (ch == '\r') // handle CR in strings 374 | { 375 | Consume(); 376 | if (La(0) == '\n') // for DOS & windows 377 | Consume(); 378 | 379 | _line++; 380 | _column = 1; 381 | } 382 | else if (ch == '\n') // new line in quoted string 383 | { 384 | Consume(); 385 | 386 | _line++; 387 | _column = 1; 388 | } 389 | else if (ch == '\'') 390 | { 391 | Consume(); 392 | if (La(0) != '\'') 393 | break; // done reading, and this quotes does not have escape character 394 | Consume(); // consume second ", because first was just an escape 395 | } 396 | else 397 | Consume(); 398 | } 399 | 400 | return CreateToken(TokenKind.SingleQuotedString); 401 | } 402 | } 403 | } 404 | -------------------------------------------------------------------------------- /CodeClassifier/StringTokenizer/Token.cs: -------------------------------------------------------------------------------- 1 | /******************************************************** 2 | * Author: Andrew Deren 3 | * Date: July, 2004 4 | * http://www.adersoftware.com 5 | * 6 | * StringTokenizer class. You can use this class in any way you want 7 | * as long as this header remains in this file. 8 | * 9 | **********************************************************/ 10 | 11 | namespace CodeClassifier.StringTokenizer 12 | { 13 | public enum TokenKind 14 | { 15 | Unknown, 16 | Word, 17 | Number, 18 | DoubleQuotedString, 19 | SingleQuotedString, 20 | WhiteSpace, 21 | EqualsSign, 22 | PlusSign, 23 | Hyphen, 24 | Slash, 25 | Comma, 26 | FullStop, 27 | Asterisk, 28 | Tilde, 29 | ExplanationMark, 30 | AtSign, 31 | Hash, 32 | Dollar, 33 | Percent, 34 | CircumflexAccent, 35 | Ampersand, 36 | LeftParenthesis, 37 | RightParenthesis, 38 | LeftCurlyBracket, 39 | RightCurlyBracket, 40 | LeftBracket, 41 | RightBracket, 42 | Colon, 43 | SemiColon, 44 | LessThanSign, 45 | GreaterThanSign, 46 | QuestionMark, 47 | VerticalLine, 48 | Backslash, 49 | GraveAccent, 50 | SingleQuote, 51 | Underscore, 52 | Eol, 53 | Eof 54 | } 55 | 56 | public class Token 57 | { 58 | readonly int _line; 59 | readonly int _column; 60 | readonly string _value; 61 | readonly TokenKind _kind; 62 | 63 | public Token(TokenKind kind, string value, int line, int column) 64 | { 65 | _kind = kind; 66 | _value = value; 67 | _line = line; 68 | _column = column; 69 | } 70 | 71 | public int Column 72 | { 73 | get { return _column; } 74 | } 75 | 76 | public TokenKind Kind 77 | { 78 | get { return _kind; } 79 | } 80 | 81 | public int Line 82 | { 83 | get { return _line; } 84 | } 85 | 86 | public string Value 87 | { 88 | get { return _value; } 89 | } 90 | } 91 | 92 | } 93 | -------------------------------------------------------------------------------- /CodeClassifier/TokenNode.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using CodeClassifier.StringTokenizer; 4 | 5 | namespace CodeClassifier 6 | { 7 | public class TokenNode 8 | { 9 | public TokenKind Kind { get; private set; } 10 | public List NextTokens { get; private set; } 11 | public HashSet Examples { get; private set; } 12 | public double Score { get; private set; } 13 | public int Level { get; set; } 14 | 15 | public TokenNode(TokenKind kind, int level, double score, string firstExample) 16 | { 17 | Kind = kind; 18 | Level = level; 19 | Score = score; 20 | NextTokens = new List(); 21 | Examples = new HashSet { firstExample }; 22 | } 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /CodeClassifier/dist/CodeClassifier.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bertyhell/CodeClassifier/eeb7071baa5cee62bf1a9bfe3e2f9467ee7625c5/CodeClassifier/dist/CodeClassifier.dll -------------------------------------------------------------------------------- /CodeClassifier/dist/training-set/css.css: -------------------------------------------------------------------------------- 1 | 2 | html { 3 | margin: 0; 4 | padding: 0; 5 | border: 0; 6 | -moz-transition: all; 7 | -o-transition: all; 8 | -webkit-transition: all; 9 | transition: all; 10 | } 11 | 12 | .hidden { 13 | display: none !important; 14 | visibility: hidden !important; 15 | } 16 | 17 | body { 18 | stroke: #eee; 19 | stroke-width: 2; 20 | stroke-dasharray: 0; 21 | } 22 | 23 | a { 24 | text-align: center; 25 | font-size: 1.1rem; 26 | padding: 12px 0; 27 | } 28 | 29 | li { 30 | -webkit-touch-callout: none; 31 | -moz-user-select: none; 32 | -ms-user-select: none; 33 | -webkit-user-select: none; 34 | user-select: none; 35 | } 36 | 37 | /* points of interest */ 38 | ul > li { 39 | stroke: steelblue; 40 | stroke-width: 2; 41 | fill: #FFF; 42 | -moz-transform-origin: center; 43 | -ms-transform-origin: center; 44 | -o-transform-origin: center; 45 | -webkit-transform-origin: center; 46 | transform-origin: center; 47 | } 48 | 49 | a + div { 50 | background-size: cover; 51 | } 52 | 53 | 54 | 55 | div { 56 | background: #fff; 57 | } 58 | 59 | .n2-charts-close { 60 | font-size: 32px; 61 | position: absolute; 62 | top: 20px; 63 | right: 20px; 64 | padding: 2px 12px; 65 | cursor: pointer; 66 | } 67 | 68 | /* Landscape styles */ 69 | @media screen and (orientation:landscape) { 70 | 71 | .chart.fullscreen { 72 | font-size: 32px; 73 | position: absolute; 74 | top: 20px; 75 | right: 20px; 76 | padding: 2px 12px; 77 | cursor: pointer; 78 | } 79 | } 80 | 81 | @-moz-keyframes n2-poi-blink { 82 | 0% { 83 | -moz-transform: scale(1); 84 | -ms-transform: scale(1); 85 | -o-transform: scale(1); 86 | -webkit-transform: scale(1); 87 | transform: scale(1); 88 | } 89 | 90 | 100% { 91 | -moz-transform: scale(1); 92 | -ms-transform: scale(1); 93 | -o-transform: scale(1); 94 | -webkit-transform: scale(1); 95 | transform: scale(1); 96 | } 97 | } 98 | 99 | @-ms-keyframes n2-poi-blink { 100 | 0% { 101 | -moz-transform: scale(1); 102 | -ms-transform: scale(1); 103 | -o-transform: scale(1); 104 | -webkit-transform: scale(1); 105 | transform: scale(1); 106 | } 107 | 108 | 100% { 109 | -moz-transform: scale(1); 110 | -ms-transform: scale(1); 111 | -o-transform: scale(1); 112 | -webkit-transform: scale(1); 113 | transform: scale(1); 114 | } 115 | } 116 | 117 | @-webkit-keyframes n2-poi-blink { 118 | 0% { 119 | -moz-transform: scale(1); 120 | -ms-transform: scale(1); 121 | -o-transform: scale(1); 122 | -webkit-transform: scale(1); 123 | transform: scale(1); 124 | } 125 | 126 | 100% { 127 | -moz-transform: scale(1); 128 | -ms-transform: scale(1); 129 | -o-transform: scale(1); 130 | -webkit-transform: scale(1); 131 | transform: scale(1); 132 | } 133 | } 134 | 135 | @keyframes n2-poi-blink { 136 | 0% { 137 | -moz-transform: scale(1); 138 | -ms-transform: scale(1); 139 | -o-transform: scale(1); 140 | -webkit-transform: scale(1); 141 | transform: scale(1); 142 | } 143 | 144 | 100% { 145 | -moz-transform: scale(1); 146 | -ms-transform: scale(1); 147 | -o-transform: scale(1); 148 | -webkit-transform: scale(1); 149 | transform: scale(1); 150 | } 151 | } 152 | -------------------------------------------------------------------------------- /CodeClassifier/dist/training-set/ruby.rb: -------------------------------------------------------------------------------- 1 | 2 | require 'optparse' 3 | 4 | module Commander 5 | class Runner 6 | 7 | #-- 8 | # Exceptions 9 | #++ 10 | 11 | class CommandError < StandardError; end 12 | class InvalidCommandError < CommandError; end 13 | 14 | ## 15 | # Array of commands. 16 | 17 | attr_reader :commands 18 | 19 | ## 20 | # Global options. 21 | 22 | attr_reader :options 23 | 24 | ## 25 | # Hash of help formatter aliases. 26 | 27 | attr_reader :help_formatter_aliases 28 | 29 | ## 30 | # Initialize a new command runner. Optionally 31 | # supplying _args_ for mocking, or arbitrary usage. 32 | 33 | def initialize args = ARGV 34 | @args, @commands, @aliases, @options = args, {}, {}, [] 35 | @help_formatter_aliases = help_formatter_alias_defaults 36 | @program = program_defaults 37 | create_default_commands 38 | end 39 | 40 | ## 41 | # Return singleton Runner instance. 42 | 43 | def self.instance 44 | @singleton ||= new 45 | end 46 | 47 | ## 48 | # Run command parsing and execution process. 49 | 50 | def run! 51 | trace = false 52 | require_program :version, :description 53 | trap('INT') { abort program(:int_message) } if program(:int_message) 54 | trap('INT') { program(:int_block).call } if program(:int_block) 55 | global_option('-h', '--help', 'Display help documentation') do 56 | args = @args - %w[-h --help] 57 | command(:help).run(*args) 58 | return 59 | end 60 | global_option('-v', '--version', 'Display version information') { say version; return } 61 | global_option('-t', '--trace', 'Display backtrace when an error occurs') { trace = true } 62 | parse_global_options 63 | remove_global_options options, @args 64 | unless trace 65 | begin 66 | run_active_command 67 | rescue InvalidCommandError => e 68 | abort "#{e}. Use --help for more information" 69 | rescue \ 70 | OptionParser::InvalidOption, 71 | OptionParser::InvalidArgument, 72 | OptionParser::MissingArgument => e 73 | abort e.to_s 74 | rescue => e 75 | abort "error: #{e}. Use --trace to view backtrace" 76 | end 77 | else 78 | run_active_command 79 | end 80 | end 81 | 82 | ## 83 | # Return program version. 84 | 85 | def version 86 | '%s %s' % [program(:name), program(:version)] 87 | end 88 | 89 | ## 90 | # Assign program information. 91 | # 92 | # === Examples 93 | # 94 | # # Set data 95 | # program :name, 'Commander' 96 | # program :version, Commander::VERSION 97 | # program :description, 'Commander utility program.' 98 | # program :help, 'Copyright', '2008 TJ Holowaychuk' 99 | # program :help, 'Anything', 'You want' 100 | # program :int_message 'Bye bye!' 101 | # program :help_formatter, :compact 102 | # program :help_formatter, Commander::HelpFormatter::TerminalCompact 103 | # 104 | # # Get data 105 | # program :name # => 'Commander' 106 | # 107 | # === Keys 108 | # 109 | # :version (required) Program version triple, ex: '0.0.1' 110 | # :description (required) Program description 111 | # :name Program name, defaults to basename of executable 112 | # :help_formatter Defaults to Commander::HelpFormatter::Terminal 113 | # :help Allows addition of arbitrary global help blocks 114 | # :int_message Message to display when interrupted (CTRL + C) 115 | # 116 | 117 | def program key, *args, &block 118 | if key == :help and !args.empty? 119 | @program[:help] ||= {} 120 | @program[:help][args.first] = args.at(1) 121 | elsif key == :help_formatter && !args.empty? 122 | @program[key] = (@help_formatter_aliases[args.first] || args.first) 123 | elsif block 124 | @program[key] = block 125 | else 126 | unless args.empty? 127 | @program[key] = (args.count == 1 && args[0]) || args 128 | end 129 | @program[key] 130 | end 131 | end 132 | 133 | ## 134 | # Creates and yields a command instance when a block is passed. 135 | # Otherwise attempts to return the command, raising InvalidCommandError when 136 | # it does not exist. 137 | # 138 | # === Examples 139 | # 140 | # command :my_command do |c| 141 | # c.when_called do |args| 142 | # # Code 143 | # end 144 | # end 145 | # 146 | 147 | def command name, &block 148 | yield add_command(Commander::Command.new(name)) if block 149 | @commands[name.to_s] 150 | end 151 | 152 | ## 153 | # Add a global option; follows the same syntax as Command#option 154 | # This would be used for switches such as --version, --trace, etc. 155 | 156 | def global_option *args, &block 157 | switches, description = Runner.separate_switches_from_description *args 158 | @options << { 159 | :args => args, 160 | :proc => block, 161 | :switches => switches, 162 | :description => description, 163 | } 164 | end 165 | 166 | ## 167 | # Alias command _name_ with _alias_name_. Optionally _args_ may be passed 168 | # as if they were being passed straight to the original command via the command-line. 169 | 170 | def alias_command alias_name, name, *args 171 | @commands[alias_name.to_s] = command name 172 | @aliases[alias_name.to_s] = args 173 | end 174 | 175 | ## 176 | # Default command _name_ to be used when no other 177 | # command is found in the arguments. 178 | 179 | def default_command name 180 | @default_command = name 181 | end 182 | 183 | ## 184 | # Add a command object to this runner. 185 | 186 | def add_command command 187 | @commands[command.name] = command 188 | end 189 | 190 | ## 191 | # Check if command _name_ is an alias. 192 | 193 | def alias? name 194 | @aliases.include? name.to_s 195 | end 196 | 197 | ## 198 | # Check if a command _name_ exists. 199 | 200 | def command_exists? name 201 | @commands[name.to_s] 202 | end 203 | 204 | #:stopdoc: 205 | 206 | ## 207 | # Get active command within arguments passed to this runner. 208 | 209 | def active_command 210 | @__active_command ||= command(command_name_from_args) 211 | end 212 | 213 | ## 214 | # Attempts to locate a command name from within the arguments. 215 | # Supports multi-word commands, using the largest possible match. 216 | 217 | def command_name_from_args 218 | @__command_name_from_args ||= (valid_command_names_from(*@args.dup).sort.last || @default_command) 219 | end 220 | 221 | ## 222 | # Returns array of valid command names found within _args_. 223 | 224 | def valid_command_names_from *args 225 | arg_string = args.delete_if { |value| value =~ /^-/ }.join ' ' 226 | commands.keys.find_all { |name| name if /^#{name}\b/.match arg_string } 227 | end 228 | 229 | ## 230 | # Help formatter instance. 231 | 232 | def help_formatter 233 | @__help_formatter ||= program(:help_formatter).new self 234 | end 235 | 236 | ## 237 | # Return arguments without the command name. 238 | 239 | def args_without_command_name 240 | removed = [] 241 | parts = command_name_from_args.split rescue [] 242 | @args.dup.delete_if do |arg| 243 | removed << arg if parts.include?(arg) and not removed.include?(arg) 244 | end 245 | end 246 | 247 | ## 248 | # Returns hash of help formatter alias defaults. 249 | 250 | def help_formatter_alias_defaults 251 | return :compact => HelpFormatter::TerminalCompact 252 | end 253 | 254 | ## 255 | # Returns hash of program defaults. 256 | 257 | def program_defaults 258 | return :help_formatter => HelpFormatter::Terminal, 259 | :name => File.basename($0) 260 | end 261 | 262 | ## 263 | # Creates default commands such as 'help' which is 264 | # essentially the same as using the --help switch. 265 | 266 | def create_default_commands 267 | command :help do |c| 268 | c.syntax = 'commander help [command]' 269 | c.description = 'Display global or [command] help documentation.' 270 | c.example 'Display global help', 'command help' 271 | c.example "Display help for 'foo'", 'command help foo' 272 | c.when_called do |args, options| 273 | enable_paging 274 | if args.empty? 275 | say help_formatter.render 276 | else 277 | command = command args.join(' ') 278 | begin 279 | require_valid_command command 280 | rescue InvalidCommandError => e 281 | abort "#{e}. Use --help for more information" 282 | end 283 | say help_formatter.render_command(command) 284 | end 285 | end 286 | end 287 | end 288 | 289 | ## 290 | # Raises InvalidCommandError when a _command_ is not found. 291 | 292 | def require_valid_command command = active_command 293 | raise InvalidCommandError, 'invalid command', caller if command.nil? 294 | end 295 | 296 | ## 297 | # Removes global _options_ from _args_. This prevents an invalid 298 | # option error from occurring when options are parsed 299 | # again for the command. 300 | 301 | def remove_global_options options, args 302 | # TODO: refactor with flipflop, please TJ ! have time to refactor me ! 303 | options.each do |option| 304 | switches = option[:switches].dup 305 | next if switches.empty? 306 | 307 | if switchHasArg = switches.any? { |s| s =~ /[ =]/ } 308 | switches.map! { |s| s[0, s.index('=') || s.index(' ') || s.length] } 309 | end 310 | 311 | past_switch, arg_removed = false, false 312 | args.delete_if do |arg| 313 | if switches.any? { |s| arg[0, s.length] == s } 314 | arg_removed = !switchHasArg 315 | past_switch = true 316 | elsif past_switch && !arg_removed && arg !~ /^-/ 317 | arg_removed = true 318 | else 319 | arg_removed = true 320 | false 321 | end 322 | end 323 | end 324 | end 325 | 326 | ## 327 | # Parse global command options. 328 | 329 | def parse_global_options 330 | 331 | parser = options.inject(OptionParser.new) do |options, option| 332 | options.on *option[:args], &global_option_proc(option[:switches], &option[:proc]) 333 | end 334 | 335 | options = @args.dup 336 | begin 337 | parser.parse!(options) 338 | rescue OptionParser::InvalidOption => e 339 | # Remove the offending args and retry. 340 | options = options.reject { |o| e.args.include?(o) } 341 | retry 342 | end 343 | end 344 | 345 | ## 346 | # Returns a proc allowing for commands to inherit global options. 347 | # This functionality works whether a block is present for the global 348 | # option or not, so simple switches such as --verbose can be used 349 | # without a block, and used throughout all commands. 350 | 351 | def global_option_proc switches, &block 352 | lambda do |value| 353 | unless active_command.nil? 354 | active_command.proxy_options << [Runner.switch_to_sym(switches.last), value] 355 | end 356 | yield value if block and !value.nil? 357 | end 358 | end 359 | 360 | ## 361 | # Raises a CommandError when the program any of the _keys_ are not present, or empty. 362 | 363 | def require_program *keys 364 | keys.each do |key| 365 | raise CommandError, "program #{key} required" if program(key).nil? or program(key).empty? 366 | end 367 | end 368 | 369 | ## 370 | # Return switches and description separated from the _args_ passed. 371 | 372 | def self.separate_switches_from_description *args 373 | switches = args.find_all { |arg| arg.to_s =~ /^-/ } 374 | description = args.last unless !args.last.is_a? String or args.last.match(/^-/) 375 | return switches, description 376 | end 377 | 378 | ## 379 | # Attempts to generate a method name symbol from +switch+. 380 | # For example: 381 | # 382 | # -h # => :h 383 | # --trace # => :trace 384 | # --some-switch # => :some_switch 385 | # --[no-]feature # => :feature 386 | # --file FILE # => :file 387 | # --list of,things # => :list 388 | # 389 | 390 | def self.switch_to_sym switch 391 | switch.scan(/[\-\]](\w+)/).join('_').to_sym rescue nil 392 | end 393 | 394 | ## 395 | # Run the active command. 396 | 397 | def run_active_command 398 | require_valid_command 399 | if alias? command_name_from_args 400 | active_command.run *(@aliases[command_name_from_args.to_s] + args_without_command_name) 401 | else 402 | active_command.run *args_without_command_name 403 | end 404 | end 405 | 406 | def say *args #:nodoc: 407 | $terminal.say *args 408 | end 409 | 410 | end 411 | end 412 | -------------------------------------------------------------------------------- /CodeClassifier/training-set/c++/NaClFile.cpp: -------------------------------------------------------------------------------- 1 | 2 | #define NACLFILE_CPP 3 | #include "NaClFile.h" 4 | #include "NaClFileSystem.h" 5 | 6 | //#include 7 | //#define EOF -1 8 | 9 | FILE * nacl_fopen ( const char * path, const char *mode ) { 10 | 11 | return ( FILE * )NaClFileSystem::Get ()->fopen ( path, mode ); 12 | } 13 | 14 | int nacl_fclose ( FILE *file ) { 15 | 16 | return NaClFileSystem::Get ()->fclose ((NaClFile *) file ); 17 | } 18 | 19 | int nacl_fread ( void *ptr, int size, int count, FILE *file ) { 20 | 21 | return NaClFileSystem::Get ()->fread ( ptr, size, count, (NaClFile *) file ); 22 | } 23 | 24 | int nacl_fwrite ( const void * ptr, int size, int count, FILE *file ) { 25 | 26 | return NaClFileSystem::Get ()->fwrite ( ptr, size, count, (NaClFile *) file ); 27 | } 28 | 29 | int nacl_feof ( FILE * void_file ) { 30 | 31 | NaClFile * file = ( NaClFile * ) void_file; 32 | return ( file->mOffset == file->mSize ); 33 | } 34 | 35 | int nacl_ferror ( FILE * void_file ) { 36 | 37 | //unimplemented 38 | return 0; 39 | } 40 | 41 | void nacl_clearerr ( FILE * void_file ) { 42 | //do nothing 43 | } 44 | 45 | int nacl_fgetc ( FILE * void_file ) { 46 | 47 | NaClFile * file = ( NaClFile * ) void_file; 48 | 49 | if( file && file->mIsHttpLoaded ) { 50 | 51 | int remainingSize = file->mSize - file->mOffset; 52 | 53 | if ( remainingSize ) { 54 | int data = file->mData [ file->mOffset ]; 55 | file->mOffset += 1; 56 | return data; 57 | } 58 | else { 59 | return EOF; 60 | } 61 | } 62 | return 0; 63 | } 64 | 65 | int nacl_ungetc (int c, FILE *void_file) { 66 | 67 | NaClFile * file = ( NaClFile * ) void_file; 68 | 69 | if( file && file->mIsHttpLoaded ) { 70 | 71 | if ( file->mOffset ) { 72 | file->mOffset -= 1; 73 | file->mData [ file->mOffset ] = c; 74 | return file->mData [ file->mOffset ]; 75 | } 76 | else { 77 | return EOF; 78 | } 79 | } 80 | return 0; 81 | } 82 | 83 | int nacl_fseek ( FILE * void_file, long int offset, int origin ) { 84 | 85 | NaClFile * file = ( NaClFile * ) void_file; 86 | 87 | int originPosition; 88 | switch ( origin ) { 89 | case SEEK_SET: 90 | originPosition = 0; 91 | break; 92 | case SEEK_CUR: 93 | originPosition = file->mOffset; 94 | break; 95 | case SEEK_END: 96 | originPosition = file->mSize; 97 | break; 98 | } 99 | int position = originPosition + offset; 100 | if ( position <= file->mSize ) { 101 | 102 | file->mOffset = position; 103 | return 0; 104 | } 105 | 106 | return -1; 107 | } 108 | 109 | long int nacl_ftell ( FILE * void_file ) { 110 | 111 | NaClFile * file = ( NaClFile * ) void_file; 112 | 113 | return file->mOffset; 114 | } 115 | 116 | int nacl_stat ( const char *path, struct stat *s ) { 117 | 118 | //open file with Head 119 | int foundFile = NaClFileSystem::Get ()->stat ( path, s ); 120 | 121 | if ( !foundFile ) { 122 | 123 | s->st_mode = S_IFREG; 124 | s->st_ctime = 0; 125 | s->st_mtime = 0; 126 | s->st_atime = 0; 127 | 128 | return 0; 129 | } 130 | 131 | return -1; 132 | } 133 | 134 | char *nacl_getcwd(char *buf, size_t size) { 135 | 136 | for ( int i = 0; i < size; ++i ) { 137 | buf [ i ] = ( char ) 0; 138 | } 139 | 140 | buf [ 0 ] = '/'; 141 | 142 | return buf; 143 | } 144 | -------------------------------------------------------------------------------- /CodeClassifier/training-set/c/adlist.c: -------------------------------------------------------------------------------- 1 | /* adlist.c - A generic doubly linked list implementation 2 | * 3 | * Copyright (c) 2006-2010, Salvatore Sanfilippo 4 | * All rights reserved. 5 | * 6 | * Redistribution and use in source and binary forms, with or without 7 | * modification, are permitted provided that the following conditions are met: 8 | * 9 | * * Redistributions of source code must retain the above copyright notice, 10 | * this list of conditions and the following disclaimer. 11 | * * Redistributions in binary form must reproduce the above copyright 12 | * notice, this list of conditions and the following disclaimer in the 13 | * documentation and/or other materials provided with the distribution. 14 | * * Neither the name of Redis nor the names of its contributors may be used 15 | * to endorse or promote products derived from this software without 16 | * specific prior written permission. 17 | * 18 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 19 | * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 21 | * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 22 | * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 23 | * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 24 | * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 25 | * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 26 | * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 27 | * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 28 | * POSSIBILITY OF SUCH DAMAGE. 29 | */ 30 | 31 | 32 | #include 33 | #include "adlist.h" 34 | #include "zmalloc.h" 35 | 36 | /* Create a new list. The created list can be freed with 37 | * AlFreeList(), but private value of every node need to be freed 38 | * by the user before to call AlFreeList(). 39 | * 40 | * On error, NULL is returned. Otherwise the pointer to the new list. */ 41 | list *listCreate(void) 42 | { 43 | struct list *list; 44 | 45 | if ((list = zmalloc(sizeof(*list))) == NULL) 46 | return NULL; 47 | list->head = list->tail = NULL; 48 | list->len = 0; 49 | list->dup = NULL; 50 | list->free = NULL; 51 | list->match = NULL; 52 | return list; 53 | } 54 | 55 | /* Free the whole list. 56 | * 57 | * This function can't fail. */ 58 | void listRelease(list *list) 59 | { 60 | unsigned long len; 61 | listNode *current, *next; 62 | 63 | current = list->head; 64 | len = list->len; 65 | while(len--) { 66 | next = current->next; 67 | if (list->free) list->free(current->value); 68 | zfree(current); 69 | current = next; 70 | } 71 | zfree(list); 72 | } 73 | 74 | /* Add a new node to the list, to head, containing the specified 'value' 75 | * pointer as value. 76 | * 77 | * On error, NULL is returned and no operation is performed (i.e. the 78 | * list remains unaltered). 79 | * On success the 'list' pointer you pass to the function is returned. */ 80 | list *listAddNodeHead(list *list, void *value) 81 | { 82 | listNode *node; 83 | 84 | if ((node = zmalloc(sizeof(*node))) == NULL) 85 | return NULL; 86 | node->value = value; 87 | if (list->len == 0) { 88 | list->head = list->tail = node; 89 | node->prev = node->next = NULL; 90 | } else { 91 | node->prev = NULL; 92 | node->next = list->head; 93 | list->head->prev = node; 94 | list->head = node; 95 | } 96 | list->len++; 97 | return list; 98 | } 99 | 100 | /* Add a new node to the list, to tail, containing the specified 'value' 101 | * pointer as value. 102 | * 103 | * On error, NULL is returned and no operation is performed (i.e. the 104 | * list remains unaltered). 105 | * On success the 'list' pointer you pass to the function is returned. */ 106 | list *listAddNodeTail(list *list, void *value) 107 | { 108 | listNode *node; 109 | 110 | if ((node = zmalloc(sizeof(*node))) == NULL) 111 | return NULL; 112 | node->value = value; 113 | if (list->len == 0) { 114 | list->head = list->tail = node; 115 | node->prev = node->next = NULL; 116 | } else { 117 | node->prev = list->tail; 118 | node->next = NULL; 119 | list->tail->next = node; 120 | list->tail = node; 121 | } 122 | list->len++; 123 | return list; 124 | } 125 | 126 | list *listInsertNode(list *list, listNode *old_node, void *value, int after) { 127 | listNode *node; 128 | 129 | if ((node = zmalloc(sizeof(*node))) == NULL) 130 | return NULL; 131 | node->value = value; 132 | if (after) { 133 | node->prev = old_node; 134 | node->next = old_node->next; 135 | if (list->tail == old_node) { 136 | list->tail = node; 137 | } 138 | } else { 139 | node->next = old_node; 140 | node->prev = old_node->prev; 141 | if (list->head == old_node) { 142 | list->head = node; 143 | } 144 | } 145 | if (node->prev != NULL) { 146 | node->prev->next = node; 147 | } 148 | if (node->next != NULL) { 149 | node->next->prev = node; 150 | } 151 | list->len++; 152 | return list; 153 | } 154 | 155 | /* Remove the specified node from the specified list. 156 | * It's up to the caller to free the private value of the node. 157 | * 158 | * This function can't fail. */ 159 | void listDelNode(list *list, listNode *node) 160 | { 161 | if (node->prev) 162 | node->prev->next = node->next; 163 | else 164 | list->head = node->next; 165 | if (node->next) 166 | node->next->prev = node->prev; 167 | else 168 | list->tail = node->prev; 169 | if (list->free) list->free(node->value); 170 | zfree(node); 171 | list->len--; 172 | } 173 | 174 | /* Returns a list iterator 'iter'. After the initialization every 175 | * call to listNext() will return the next element of the list. 176 | * 177 | * This function can't fail. */ 178 | listIter *listGetIterator(list *list, int direction) 179 | { 180 | listIter *iter; 181 | 182 | if ((iter = zmalloc(sizeof(*iter))) == NULL) return NULL; 183 | if (direction == AL_START_HEAD) 184 | iter->next = list->head; 185 | else 186 | iter->next = list->tail; 187 | iter->direction = direction; 188 | return iter; 189 | } 190 | 191 | /* Release the iterator memory */ 192 | void listReleaseIterator(listIter *iter) { 193 | zfree(iter); 194 | } 195 | 196 | /* Create an iterator in the list private iterator structure */ 197 | void listRewind(list *list, listIter *li) { 198 | li->next = list->head; 199 | li->direction = AL_START_HEAD; 200 | } 201 | 202 | void listRewindTail(list *list, listIter *li) { 203 | li->next = list->tail; 204 | li->direction = AL_START_TAIL; 205 | } 206 | 207 | /* Return the next element of an iterator. 208 | * It's valid to remove the currently returned element using 209 | * listDelNode(), but not to remove other elements. 210 | * 211 | * The function returns a pointer to the next element of the list, 212 | * or NULL if there are no more elements, so the classical usage patter 213 | * is: 214 | * 215 | * iter = listGetIterator(list,); 216 | * while ((node = listNext(iter)) != NULL) { 217 | * doSomethingWith(listNodeValue(node)); 218 | * } 219 | * 220 | * */ 221 | listNode *listNext(listIter *iter) 222 | { 223 | listNode *current = iter->next; 224 | 225 | if (current != NULL) { 226 | if (iter->direction == AL_START_HEAD) 227 | iter->next = current->next; 228 | else 229 | iter->next = current->prev; 230 | } 231 | return current; 232 | } 233 | 234 | /* Duplicate the whole list. On out of memory NULL is returned. 235 | * On success a copy of the original list is returned. 236 | * 237 | * The 'Dup' method set with listSetDupMethod() function is used 238 | * to copy the node value. Otherwise the same pointer value of 239 | * the original node is used as value of the copied node. 240 | * 241 | * The original list both on success or error is never modified. */ 242 | list *listDup(list *orig) 243 | { 244 | list *copy; 245 | listIter *iter; 246 | listNode *node; 247 | 248 | if ((copy = listCreate()) == NULL) 249 | return NULL; 250 | copy->dup = orig->dup; 251 | copy->free = orig->free; 252 | copy->match = orig->match; 253 | iter = listGetIterator(orig, AL_START_HEAD); 254 | while((node = listNext(iter)) != NULL) { 255 | void *value; 256 | 257 | if (copy->dup) { 258 | value = copy->dup(node->value); 259 | if (value == NULL) { 260 | listRelease(copy); 261 | listReleaseIterator(iter); 262 | return NULL; 263 | } 264 | } else 265 | value = node->value; 266 | if (listAddNodeTail(copy, value) == NULL) { 267 | listRelease(copy); 268 | listReleaseIterator(iter); 269 | return NULL; 270 | } 271 | } 272 | listReleaseIterator(iter); 273 | return copy; 274 | } 275 | 276 | /* Search the list for a node matching a given key. 277 | * The match is performed using the 'match' method 278 | * set with listSetMatchMethod(). If no 'match' method 279 | * is set, the 'value' pointer of every node is directly 280 | * compared with the 'key' pointer. 281 | * 282 | * On success the first matching node pointer is returned 283 | * (search starts from head). If no matching node exists 284 | * NULL is returned. */ 285 | listNode *listSearchKey(list *list, void *key) 286 | { 287 | listIter *iter; 288 | listNode *node; 289 | 290 | iter = listGetIterator(list, AL_START_HEAD); 291 | while((node = listNext(iter)) != NULL) { 292 | if (list->match) { 293 | if (list->match(node->value, key)) { 294 | listReleaseIterator(iter); 295 | return node; 296 | } 297 | } else { 298 | if (key == node->value) { 299 | listReleaseIterator(iter); 300 | return node; 301 | } 302 | } 303 | } 304 | listReleaseIterator(iter); 305 | return NULL; 306 | } 307 | 308 | /* Return the element at the specified zero-based index 309 | * where 0 is the head, 1 is the element next to head 310 | * and so on. Negative integers are used in order to count 311 | * from the tail, -1 is the last element, -2 the penultimate 312 | * and so on. If the index is out of range NULL is returned. */ 313 | listNode *listIndex(list *list, long index) { 314 | listNode *n; 315 | 316 | if (index < 0) { 317 | index = (-index)-1; 318 | n = list->tail; 319 | while(index-- && n) n = n->prev; 320 | } else { 321 | n = list->head; 322 | while(index-- && n) n = n->next; 323 | } 324 | return n; 325 | } 326 | 327 | /* Rotate the list removing the tail node and inserting it to the head. */ 328 | void listRotate(list *list) { 329 | listNode *tail = list->tail; 330 | 331 | if (listLength(list) <= 1) return; 332 | 333 | /* Detach current tail */ 334 | list->tail = tail->prev; 335 | list->tail->next = NULL; 336 | /* Move it as head */ 337 | list->head->prev = tail; 338 | tail->prev = NULL; 339 | tail->next = list->head; 340 | list->head = tail; 341 | } 342 | -------------------------------------------------------------------------------- /CodeClassifier/training-set/c/async.c: -------------------------------------------------------------------------------- 1 | /* 2 | * async.c: Asynchronous function calls for boot performance 3 | * 4 | * (C) Copyright 2009 Intel Corporation 5 | * Author: Arjan van de Ven 6 | * 7 | * This program is free software; you can redistribute it and/or 8 | * modify it under the terms of the GNU General Public License 9 | * as published by the Free Software Foundation; version 2 10 | * of the License. 11 | */ 12 | 13 | 14 | /* 15 | 16 | Goals and Theory of Operation 17 | 18 | The primary goal of this feature is to reduce the kernel boot time, 19 | by doing various independent hardware delays and discovery operations 20 | decoupled and not strictly serialized. 21 | 22 | More specifically, the asynchronous function call concept allows 23 | certain operations (primarily during system boot) to happen 24 | asynchronously, out of order, while these operations still 25 | have their externally visible parts happen sequentially and in-order. 26 | (not unlike how out-of-order CPUs retire their instructions in order) 27 | 28 | Key to the asynchronous function call implementation is the concept of 29 | a "sequence cookie" (which, although it has an abstracted type, can be 30 | thought of as a monotonically incrementing number). 31 | 32 | The async core will assign each scheduled event such a sequence cookie and 33 | pass this to the called functions. 34 | 35 | The asynchronously called function should before doing a globally visible 36 | operation, such as registering device numbers, call the 37 | async_synchronize_cookie() function and pass in its own cookie. The 38 | async_synchronize_cookie() function will make sure that all asynchronous 39 | operations that were scheduled prior to the operation corresponding with the 40 | cookie have completed. 41 | 42 | Subsystem/driver initialization code that scheduled asynchronous probe 43 | functions, but which shares global resources with other drivers/subsystems 44 | that do not use the asynchronous call feature, need to do a full 45 | synchronization with the async_synchronize_full() function, before returning 46 | from their init function. This is to maintain strict ordering between the 47 | asynchronous and synchronous parts of the kernel. 48 | 49 | */ 50 | 51 | #include 52 | #include 53 | #include 54 | #include 55 | #include 56 | #include 57 | #include 58 | #include 59 | 60 | #include "workqueue_internal.h" 61 | 62 | static async_cookie_t next_cookie = 1; 63 | 64 | #define MAX_WORK 32768 65 | #define ASYNC_COOKIE_MAX ULLONG_MAX /* infinity cookie */ 66 | 67 | static LIST_HEAD(async_global_pending); /* pending from all registered doms */ 68 | static ASYNC_DOMAIN(async_dfl_domain); 69 | static DEFINE_SPINLOCK(async_lock); 70 | 71 | struct async_entry { 72 | struct list_head domain_list; 73 | struct list_head global_list; 74 | struct work_struct work; 75 | async_cookie_t cookie; 76 | async_func_t func; 77 | void *data; 78 | struct async_domain *domain; 79 | }; 80 | 81 | static DECLARE_WAIT_QUEUE_HEAD(async_done); 82 | 83 | static atomic_t entry_count; 84 | 85 | static async_cookie_t lowest_in_progress(struct async_domain *domain) 86 | { 87 | struct list_head *pending; 88 | async_cookie_t ret = ASYNC_COOKIE_MAX; 89 | unsigned long flags; 90 | 91 | spin_lock_irqsave(&async_lock, flags); 92 | 93 | if (domain) 94 | pending = &domain->pending; 95 | else 96 | pending = &async_global_pending; 97 | 98 | if (!list_empty(pending)) 99 | ret = list_first_entry(pending, struct async_entry, 100 | domain_list)->cookie; 101 | 102 | spin_unlock_irqrestore(&async_lock, flags); 103 | return ret; 104 | } 105 | 106 | /* 107 | * pick the first pending entry and run it 108 | */ 109 | static void async_run_entry_fn(struct work_struct *work) 110 | { 111 | struct async_entry *entry = 112 | container_of(work, struct async_entry, work); 113 | unsigned long flags; 114 | ktime_t uninitialized_var(calltime), delta, rettime; 115 | 116 | /* 1) run (and print duration) */ 117 | if (initcall_debug && system_state == SYSTEM_BOOTING) { 118 | pr_debug("calling %lli_%pF @ %i\n", 119 | (long long)entry->cookie, 120 | entry->func, task_pid_nr(current)); 121 | calltime = ktime_get(); 122 | } 123 | entry->func(entry->data, entry->cookie); 124 | if (initcall_debug && system_state == SYSTEM_BOOTING) { 125 | rettime = ktime_get(); 126 | delta = ktime_sub(rettime, calltime); 127 | pr_debug("initcall %lli_%pF returned 0 after %lld usecs\n", 128 | (long long)entry->cookie, 129 | entry->func, 130 | (long long)ktime_to_ns(delta) >> 10); 131 | } 132 | 133 | /* 2) remove self from the pending queues */ 134 | spin_lock_irqsave(&async_lock, flags); 135 | list_del_init(&entry->domain_list); 136 | list_del_init(&entry->global_list); 137 | 138 | /* 3) free the entry */ 139 | kfree(entry); 140 | atomic_dec(&entry_count); 141 | 142 | spin_unlock_irqrestore(&async_lock, flags); 143 | 144 | /* 4) wake up any waiters */ 145 | wake_up(&async_done); 146 | } 147 | 148 | static async_cookie_t __async_schedule(async_func_t func, void *data, struct async_domain *domain) 149 | { 150 | struct async_entry *entry; 151 | unsigned long flags; 152 | async_cookie_t newcookie; 153 | 154 | /* allow irq-off callers */ 155 | entry = kzalloc(sizeof(struct async_entry), GFP_ATOMIC); 156 | 157 | /* 158 | * If we're out of memory or if there's too much work 159 | * pending already, we execute synchronously. 160 | */ 161 | if (!entry || atomic_read(&entry_count) > MAX_WORK) { 162 | kfree(entry); 163 | spin_lock_irqsave(&async_lock, flags); 164 | newcookie = next_cookie++; 165 | spin_unlock_irqrestore(&async_lock, flags); 166 | 167 | /* low on memory.. run synchronously */ 168 | func(data, newcookie); 169 | return newcookie; 170 | } 171 | INIT_LIST_HEAD(&entry->domain_list); 172 | INIT_LIST_HEAD(&entry->global_list); 173 | INIT_WORK(&entry->work, async_run_entry_fn); 174 | entry->func = func; 175 | entry->data = data; 176 | entry->domain = domain; 177 | 178 | spin_lock_irqsave(&async_lock, flags); 179 | 180 | /* allocate cookie and queue */ 181 | newcookie = entry->cookie = next_cookie++; 182 | 183 | list_add_tail(&entry->domain_list, &domain->pending); 184 | if (domain->registered) 185 | list_add_tail(&entry->global_list, &async_global_pending); 186 | 187 | atomic_inc(&entry_count); 188 | spin_unlock_irqrestore(&async_lock, flags); 189 | 190 | /* mark that this task has queued an async job, used by module init */ 191 | current->flags |= PF_USED_ASYNC; 192 | 193 | /* schedule for execution */ 194 | queue_work(system_unbound_wq, &entry->work); 195 | 196 | return newcookie; 197 | } 198 | 199 | /** 200 | * async_schedule - schedule a function for asynchronous execution 201 | * @func: function to execute asynchronously 202 | * @data: data pointer to pass to the function 203 | * 204 | * Returns an async_cookie_t that may be used for checkpointing later. 205 | * Note: This function may be called from atomic or non-atomic contexts. 206 | */ 207 | async_cookie_t async_schedule(async_func_t func, void *data) 208 | { 209 | return __async_schedule(func, data, &async_dfl_domain); 210 | } 211 | EXPORT_SYMBOL_GPL(async_schedule); 212 | 213 | /** 214 | * async_schedule_domain - schedule a function for asynchronous execution within a certain domain 215 | * @func: function to execute asynchronously 216 | * @data: data pointer to pass to the function 217 | * @domain: the domain 218 | * 219 | * Returns an async_cookie_t that may be used for checkpointing later. 220 | * @domain may be used in the async_synchronize_*_domain() functions to 221 | * wait within a certain synchronization domain rather than globally. A 222 | * synchronization domain is specified via @domain. Note: This function 223 | * may be called from atomic or non-atomic contexts. 224 | */ 225 | async_cookie_t async_schedule_domain(async_func_t func, void *data, 226 | struct async_domain *domain) 227 | { 228 | return __async_schedule(func, data, domain); 229 | } 230 | EXPORT_SYMBOL_GPL(async_schedule_domain); 231 | 232 | /** 233 | * async_synchronize_full - synchronize all asynchronous function calls 234 | * 235 | * This function waits until all asynchronous function calls have been done. 236 | */ 237 | void async_synchronize_full(void) 238 | { 239 | async_synchronize_full_domain(NULL); 240 | } 241 | EXPORT_SYMBOL_GPL(async_synchronize_full); 242 | 243 | /** 244 | * async_unregister_domain - ensure no more anonymous waiters on this domain 245 | * @domain: idle domain to flush out of any async_synchronize_full instances 246 | * 247 | * async_synchronize_{cookie|full}_domain() are not flushed since callers 248 | * of these routines should know the lifetime of @domain 249 | * 250 | * Prefer ASYNC_DOMAIN_EXCLUSIVE() declarations over flushing 251 | */ 252 | void async_unregister_domain(struct async_domain *domain) 253 | { 254 | spin_lock_irq(&async_lock); 255 | WARN_ON(!domain->registered || !list_empty(&domain->pending)); 256 | domain->registered = 0; 257 | spin_unlock_irq(&async_lock); 258 | } 259 | EXPORT_SYMBOL_GPL(async_unregister_domain); 260 | 261 | /** 262 | * async_synchronize_full_domain - synchronize all asynchronous function within a certain domain 263 | * @domain: the domain to synchronize 264 | * 265 | * This function waits until all asynchronous function calls for the 266 | * synchronization domain specified by @domain have been done. 267 | */ 268 | void async_synchronize_full_domain(struct async_domain *domain) 269 | { 270 | async_synchronize_cookie_domain(ASYNC_COOKIE_MAX, domain); 271 | } 272 | EXPORT_SYMBOL_GPL(async_synchronize_full_domain); 273 | 274 | /** 275 | * async_synchronize_cookie_domain - synchronize asynchronous function calls within a certain domain with cookie checkpointing 276 | * @cookie: async_cookie_t to use as checkpoint 277 | * @domain: the domain to synchronize (%NULL for all registered domains) 278 | * 279 | * This function waits until all asynchronous function calls for the 280 | * synchronization domain specified by @domain submitted prior to @cookie 281 | * have been done. 282 | */ 283 | void async_synchronize_cookie_domain(async_cookie_t cookie, struct async_domain *domain) 284 | { 285 | ktime_t uninitialized_var(starttime), delta, endtime; 286 | 287 | if (initcall_debug && system_state == SYSTEM_BOOTING) { 288 | pr_debug("async_waiting @ %i\n", task_pid_nr(current)); 289 | starttime = ktime_get(); 290 | } 291 | 292 | wait_event(async_done, lowest_in_progress(domain) >= cookie); 293 | 294 | if (initcall_debug && system_state == SYSTEM_BOOTING) { 295 | endtime = ktime_get(); 296 | delta = ktime_sub(endtime, starttime); 297 | 298 | pr_debug("async_continuing @ %i after %lli usec\n", 299 | task_pid_nr(current), 300 | (long long)ktime_to_ns(delta) >> 10); 301 | } 302 | } 303 | EXPORT_SYMBOL_GPL(async_synchronize_cookie_domain); 304 | 305 | /** 306 | * async_synchronize_cookie - synchronize asynchronous function calls with cookie checkpointing 307 | * @cookie: async_cookie_t to use as checkpoint 308 | * 309 | * This function waits until all asynchronous function calls prior to @cookie 310 | * have been done. 311 | */ 312 | void async_synchronize_cookie(async_cookie_t cookie) 313 | { 314 | async_synchronize_cookie_domain(cookie, &async_dfl_domain); 315 | } 316 | EXPORT_SYMBOL_GPL(async_synchronize_cookie); 317 | 318 | /** 319 | * current_is_async - is %current an async worker task? 320 | * 321 | * Returns %true if %current is an async worker task. 322 | */ 323 | bool current_is_async(void) 324 | { 325 | struct worker *worker = current_wq_worker(); 326 | 327 | return worker && worker->current_func == async_run_entry_fn; 328 | } 329 | -------------------------------------------------------------------------------- /CodeClassifier/training-set/csharp/BasicAuthenticationFixture.cs: -------------------------------------------------------------------------------- 1 | namespace Nancy.Authentication.Basic.Tests 2 | { 3 | using System; 4 | using System.Collections.Generic; 5 | using System.Text; 6 | using System.Threading; 7 | using FakeItEasy; 8 | using Nancy.Security; 9 | using Nancy.Tests; 10 | using Xunit; 11 | using Nancy.Bootstrapper; 12 | using Nancy.Tests.Fakes; 13 | 14 | public class BasicAuthenticationFixture 15 | { 16 | private readonly BasicAuthenticationConfiguration config; 17 | const string ajaxRequestHeaderKey = "X-Requested-With"; 18 | const string ajaxRequestHeaderValue = "XMLHttpRequest"; 19 | private readonly IPipelines hooks; 20 | 21 | public BasicAuthenticationFixture() 22 | { 23 | this.config = new BasicAuthenticationConfiguration(A.Fake(), "realm", UserPromptBehaviour.Always); 24 | this.hooks = new Pipelines(); 25 | BasicAuthentication.Enable(this.hooks, this.config); 26 | } 27 | 28 | [Fact] 29 | public void Should_add_a_pre_and_post_hook_in_application_when_enabled() 30 | { 31 | // Given 32 | var pipelines = A.Fake(); 33 | 34 | // When 35 | BasicAuthentication.Enable(pipelines, this.config); 36 | 37 | // Then 38 | A.CallTo(() => pipelines.BeforeRequest.AddItemToStartOfPipeline(A>.Ignored)) 39 | .MustHaveHappened(Repeated.Exactly.Once); 40 | A.CallTo(() => pipelines.AfterRequest.AddItemToEndOfPipeline(A>.Ignored)) 41 | .MustHaveHappened(Repeated.Exactly.Once); 42 | } 43 | 44 | [Fact] 45 | public void Should_add_both_basic_and_requires_auth_pre_and_post_hooks_in_module_when_enabled() 46 | { 47 | // Given 48 | var module = new FakeModule(); 49 | 50 | // When 51 | BasicAuthentication.Enable(module, this.config); 52 | 53 | // Then 54 | module.Before.PipelineDelegates.ShouldHaveCount(2); 55 | module.After.PipelineDelegates.ShouldHaveCount(1); 56 | } 57 | 58 | [Fact] 59 | public void Should_throw_with_null_config_passed_to_enable_with_application() 60 | { 61 | // Given, When 62 | var result = Record.Exception(() => BasicAuthentication.Enable(A.Fake(), null)); 63 | 64 | // Then 65 | result.ShouldBeOfType(typeof(ArgumentNullException)); 66 | } 67 | 68 | [Fact] 69 | public void Should_throw_with_null_config_passed_to_enable_with_module() 70 | { 71 | // Given, When 72 | var result = Record.Exception(() => BasicAuthentication.Enable(new FakeModule(), null)); 73 | 74 | // Then 75 | result.ShouldBeOfType(typeof(ArgumentNullException)); 76 | } 77 | 78 | [Fact] 79 | public void Should_throw_with_null_pipeline_passed_to_enable_with_config() 80 | { 81 | // Given, When 82 | var result = Record.Exception(() => BasicAuthentication.Enable((IPipelines)null, this.config)); 83 | 84 | // Then 85 | result.ShouldBeOfType(typeof(ArgumentNullException)); 86 | } 87 | 88 | [Fact] 89 | public void Should_throw_with_null_module_passed_to_enable_with_config() 90 | { 91 | // Given, When 92 | var result = Record.Exception(() => BasicAuthentication.Enable((INancyModule)null, this.config)); 93 | 94 | // Then 95 | result.ShouldBeOfType(typeof(ArgumentNullException)); 96 | } 97 | 98 | [Fact] 99 | public void Pre_request_hook_should_not_set_auth_details_with_no_auth_headers() 100 | { 101 | // Given 102 | var context = new NancyContext() 103 | { 104 | Request = new FakeRequest("GET", "/") 105 | }; 106 | 107 | // When 108 | var result = this.hooks.BeforeRequest.Invoke(context, new CancellationToken()); 109 | 110 | // Then 111 | result.Result.ShouldBeNull(); 112 | context.CurrentUser.ShouldBeNull(); 113 | } 114 | 115 | [Fact] 116 | public void Post_request_hook_should_return_challenge_when_unauthorized_returned_from_route() 117 | { 118 | // Given 119 | var context = new NancyContext() 120 | { 121 | Request = new FakeRequest("GET", "/") 122 | }; 123 | 124 | string wwwAuthenticate; 125 | context.Response = new Response { StatusCode = HttpStatusCode.Unauthorized }; 126 | 127 | // When 128 | this.hooks.AfterRequest.Invoke(context, new CancellationToken()); 129 | 130 | // Then 131 | context.Response.Headers.TryGetValue("WWW-Authenticate", out wwwAuthenticate); 132 | context.Response.StatusCode.ShouldEqual(HttpStatusCode.Unauthorized); 133 | context.Response.Headers.ContainsKey("WWW-Authenticate").ShouldBeTrue(); 134 | context.Response.Headers["WWW-Authenticate"].ShouldContain("Basic"); 135 | context.Response.Headers["WWW-Authenticate"].ShouldContain("realm=\"" + this.config.Realm + "\""); 136 | } 137 | 138 | [Fact] 139 | public void Post_request_hook_should_not_return_a_challenge_when_set_to_never() 140 | { 141 | // Given 142 | var config = new BasicAuthenticationConfiguration(A.Fake(), "realm", UserPromptBehaviour.Never); 143 | var hooks = new Pipelines(); 144 | BasicAuthentication.Enable(hooks, config); 145 | 146 | var context = new NancyContext() 147 | { 148 | Request = new FakeRequest("GET", "/") 149 | }; 150 | 151 | context.Response = new Response { StatusCode = HttpStatusCode.Unauthorized }; 152 | 153 | // When 154 | hooks.AfterRequest.Invoke(context, new CancellationToken()); 155 | 156 | // Then 157 | context.Response.Headers.ContainsKey("WWW-Authenticate").ShouldBeFalse(); 158 | } 159 | 160 | [Fact] 161 | public void Post_request_hook_should_not_return_a_challenge_on_an_ajax_request_when_set_to_nonajax() 162 | { 163 | // Given 164 | var config = new BasicAuthenticationConfiguration(A.Fake(), "realm", UserPromptBehaviour.NonAjax); 165 | var hooks = new Pipelines(); 166 | BasicAuthentication.Enable(hooks, config); 167 | var headers = new Dictionary>(); 168 | headers.Add(ajaxRequestHeaderKey, new [] { ajaxRequestHeaderValue }); 169 | 170 | var context = new NancyContext() 171 | { 172 | Request = new FakeRequest("GET", "/", headers) 173 | }; 174 | 175 | context.Response = new Response { StatusCode = HttpStatusCode.Unauthorized }; 176 | 177 | // When 178 | hooks.AfterRequest.Invoke(context, new CancellationToken()); 179 | 180 | // Then 181 | context.Response.Headers.ContainsKey("WWW-Authenticate").ShouldBeFalse(); 182 | } 183 | 184 | [Fact] 185 | public void Post_request_hook_should_return_a_challenge_on_a_nonajax_request_when_set_to_nonajax() 186 | { 187 | // Given 188 | var config = new BasicAuthenticationConfiguration(A.Fake(), "realm", UserPromptBehaviour.NonAjax); 189 | var hooks = new Pipelines(); 190 | BasicAuthentication.Enable(hooks, config); 191 | 192 | var context = new NancyContext() 193 | { 194 | Request = new FakeRequest("GET", "/") 195 | }; 196 | 197 | context.Response = new Response { StatusCode = HttpStatusCode.Unauthorized }; 198 | 199 | // When 200 | hooks.AfterRequest.Invoke(context, new CancellationToken()); 201 | 202 | // Then 203 | context.Response.Headers.ContainsKey("WWW-Authenticate").ShouldBeTrue(); 204 | } 205 | 206 | 207 | [Fact] 208 | public void Pre_request_hook_should_not_set_auth_details_when_invalid_scheme_in_auth_header() 209 | { 210 | // Given 211 | var context = CreateContextWithHeader( 212 | "Authorization", new[] { "FooScheme" + " " + EncodeCredentials("foo", "bar") }); 213 | 214 | // When 215 | var result = this.hooks.BeforeRequest.Invoke(context, new CancellationToken()); 216 | 217 | // Then 218 | result.Result.ShouldBeNull(); 219 | context.CurrentUser.ShouldBeNull(); 220 | } 221 | 222 | [Fact] 223 | public void Pre_request_hook_should_not_authenticate_when_invalid_encoded_username_in_auth_header() 224 | { 225 | // Given 226 | var context = CreateContextWithHeader( 227 | "Authorization", new[] { "Basic" + " " + "some credentials" }); 228 | 229 | // When 230 | var result = this.hooks.BeforeRequest.Invoke(context, new CancellationToken()); 231 | 232 | // Then 233 | result.Result.ShouldBeNull(); 234 | context.CurrentUser.ShouldBeNull(); 235 | } 236 | 237 | [Fact] 238 | public void Pre_request_hook_should_call_user_validator_with_username_in_auth_header() 239 | { 240 | // Given 241 | var context = CreateContextWithHeader( 242 | "Authorization", new[] { "Basic" + " " + EncodeCredentials("foo", "bar") }); 243 | 244 | // When 245 | this.hooks.BeforeRequest.Invoke(context, new CancellationToken()); 246 | 247 | // Then 248 | A.CallTo(() => config.UserValidator.Validate("foo", "bar")).MustHaveHappened(); 249 | } 250 | 251 | [Fact] 252 | public void Pre_request_hook_should_call_user_validator_with_password_in_auth_header_containing_colon() 253 | { 254 | // Given 255 | var context = CreateContextWithHeader( 256 | "Authorization", new[] {"Basic" + " " + EncodeCredentials("foo", "bar:baz")}); 257 | 258 | // When 259 | this.hooks.BeforeRequest.Invoke(context, new CancellationToken()); 260 | 261 | // Then 262 | A.CallTo(() => config.UserValidator.Validate("foo", "bar:baz")).MustHaveHappened(); 263 | } 264 | 265 | [Fact] 266 | public void Should_set_user_in_context_with_valid_username_in_auth_header() 267 | { 268 | // Given 269 | var fakePipelines = new Pipelines(); 270 | 271 | var validator = A.Fake(); 272 | var fakeUser = A.Fake(); 273 | A.CallTo(() => validator.Validate("foo", "bar")).Returns(fakeUser); 274 | 275 | var cfg = new BasicAuthenticationConfiguration(validator, "realm"); 276 | 277 | var context = CreateContextWithHeader( 278 | "Authorization", new [] { "Basic" + " " + EncodeCredentials("foo", "bar") }); 279 | 280 | BasicAuthentication.Enable(fakePipelines, cfg); 281 | 282 | // When 283 | fakePipelines.BeforeRequest.Invoke(context, new CancellationToken()); 284 | 285 | // Then 286 | context.CurrentUser.ShouldBeSameAs(fakeUser); 287 | } 288 | 289 | private static NancyContext CreateContextWithHeader(string name, IEnumerable values) 290 | { 291 | var header = new Dictionary> 292 | { 293 | { name, values } 294 | }; 295 | 296 | return new NancyContext() 297 | { 298 | Request = new FakeRequest("GET", "/", header) 299 | }; 300 | } 301 | 302 | private static string EncodeCredentials(string username, string password) 303 | { 304 | var credentials = string.Format("{0}:{1}", username, password); 305 | 306 | var encodedCredentials = Convert.ToBase64String(Encoding.UTF8.GetBytes(credentials)); 307 | 308 | return encodedCredentials; 309 | } 310 | 311 | class FakeModule : NancyModule 312 | { 313 | public FakeModule() 314 | { 315 | this.After = new AfterPipeline(); 316 | this.Before = new BeforePipeline(); 317 | this.OnError = new ErrorPipeline(); 318 | } 319 | } 320 | } 321 | } 322 | -------------------------------------------------------------------------------- /CodeClassifier/training-set/csharp/ChromiumWebBrowser.cs: -------------------------------------------------------------------------------- 1 | // Copyright © 2010-2015 The CefSharp Authors. All rights reserved. 2 | // 3 | // Use of this source code is governed by a BSD-style license that can be found in the LICENSE file. 4 | 5 | using System; 6 | using System.Windows.Forms; 7 | using CefSharp.Internals; 8 | using CefSharp.WinForms.Internals; 9 | 10 | namespace CefSharp.WinForms 11 | { 12 | public class ChromiumWebBrowser : Control, IWebBrowserInternal, IWinFormsWebBrowser 13 | { 14 | private ManagedCefBrowserAdapter managedCefBrowserAdapter; 15 | private ParentFormMessageInterceptor parentFormMessageInterceptor; 16 | 17 | /// 18 | /// Set to true while handing an activating WM_ACTIVATE message. 19 | /// MUST ONLY be cleared by DefaultFocusHandler. 20 | /// 21 | public bool IsActivating { get; set; } 22 | 23 | public BrowserSettings BrowserSettings { get; set; } 24 | public string Title { get; set; } 25 | public bool IsLoading { get; private set; } 26 | public string TooltipText { get; private set; } 27 | public string Address { get; private set; } 28 | 29 | public IDialogHandler DialogHandler { get; set; } 30 | public IJsDialogHandler JsDialogHandler { get; set; } 31 | public IKeyboardHandler KeyboardHandler { get; set; } 32 | public IRequestHandler RequestHandler { get; set; } 33 | public IDownloadHandler DownloadHandler { get; set; } 34 | public ILifeSpanHandler LifeSpanHandler { get; set; } 35 | public IPopupHandler PopupHandler { get; set; } 36 | public IMenuHandler MenuHandler { get; set; } 37 | 38 | /// 39 | /// The for this ChromiumWebBrowser. 40 | /// 41 | /// 42 | /// If you need customized focus handling behavior for WinForms, the suggested 43 | /// best practice would be to inherit from DefaultFocusHandler and try to avoid 44 | /// needing to override the logic in OnGotFocus. The implementation in 45 | /// DefaultFocusHandler relies on very detailed behavior of how WinForms and 46 | /// Windows interact during window activation. 47 | /// 48 | public IFocusHandler FocusHandler { get; set; } 49 | public IDragHandler DragHandler { get; set; } 50 | public IResourceHandlerFactory ResourceHandlerFactory { get; set; } 51 | public IGeolocationHandler GeolocationHandler { get; set; } 52 | 53 | public event EventHandler LoadError; 54 | public event EventHandler FrameLoadStart; 55 | public event EventHandler FrameLoadEnd; 56 | public event EventHandler LoadingStateChanged; 57 | public event EventHandler ConsoleMessage; 58 | public event EventHandler StatusMessage; 59 | public event EventHandler AddressChanged; 60 | public event EventHandler TitleChanged; 61 | public event EventHandler IsBrowserInitializedChanged; 62 | 63 | public bool CanGoForward { get; private set; } 64 | public bool CanGoBack { get; private set; } 65 | public bool CanReload { get; private set; } 66 | public bool IsBrowserInitialized { get; private set; } 67 | 68 | static ChromiumWebBrowser() 69 | { 70 | Application.ApplicationExit += OnApplicationExit; 71 | } 72 | 73 | private static void OnApplicationExit(object sender, EventArgs e) 74 | { 75 | Cef.Shutdown(); 76 | } 77 | 78 | public ChromiumWebBrowser(string address) 79 | { 80 | if (!Cef.IsInitialized && !Cef.Initialize()) 81 | { 82 | throw new InvalidOperationException("Cef::Initialize() failed"); 83 | } 84 | 85 | Cef.AddDisposable(this); 86 | Address = address; 87 | 88 | Dock = DockStyle.Fill; 89 | 90 | FocusHandler = new DefaultFocusHandler(this); 91 | ResourceHandlerFactory = new DefaultResourceHandlerFactory(); 92 | BrowserSettings = new BrowserSettings(); 93 | 94 | managedCefBrowserAdapter = new ManagedCefBrowserAdapter(this, false); 95 | } 96 | 97 | protected override void Dispose(bool disposing) 98 | { 99 | // Don't utilize any of the handlers anymore: 100 | this.SetHandlersToNull(); 101 | 102 | Cef.RemoveDisposable(this); 103 | 104 | if (disposing) 105 | { 106 | IsBrowserInitialized = false; 107 | 108 | if (BrowserSettings != null) 109 | { 110 | BrowserSettings.Dispose(); 111 | BrowserSettings = null; 112 | } 113 | 114 | if (parentFormMessageInterceptor != null) 115 | { 116 | parentFormMessageInterceptor.Dispose(); 117 | parentFormMessageInterceptor = null; 118 | } 119 | 120 | if (managedCefBrowserAdapter != null) 121 | { 122 | managedCefBrowserAdapter.Dispose(); 123 | managedCefBrowserAdapter = null; 124 | } 125 | 126 | // Don't maintain a reference to event listeners anylonger: 127 | LoadError = null; 128 | FrameLoadStart = null; 129 | FrameLoadEnd = null; 130 | LoadingStateChanged = null; 131 | ConsoleMessage = null; 132 | StatusMessage = null; 133 | AddressChanged = null; 134 | TitleChanged = null; 135 | IsBrowserInitializedChanged = null; 136 | } 137 | base.Dispose(disposing); 138 | } 139 | 140 | public void Load(String url) 141 | { 142 | if (IsBrowserInitialized) 143 | { 144 | this.GetMainFrame().LoadUrl(url); 145 | } 146 | else 147 | { 148 | Address = url; 149 | } 150 | } 151 | 152 | public void RegisterJsObject(string name, object objectToBind, bool camelCaseJavascriptNames = true) 153 | { 154 | if (IsBrowserInitialized) 155 | { 156 | throw new Exception("Browser is already initialized. RegisterJsObject must be" + 157 | "called before the underlying CEF browser is created."); 158 | } 159 | managedCefBrowserAdapter.RegisterJsObject(name, objectToBind, camelCaseJavascriptNames); 160 | } 161 | 162 | protected override void OnHandleCreated(EventArgs e) 163 | { 164 | managedCefBrowserAdapter.CreateBrowser(BrowserSettings, Handle, Address); 165 | 166 | base.OnHandleCreated(e); 167 | } 168 | 169 | void IWebBrowserInternal.OnAfterBrowserCreated() 170 | { 171 | IsBrowserInitialized = true; 172 | 173 | // By the time this callback gets called, this control 174 | // is most likely hooked into a browser Form of some sort. 175 | // (Which is what ParentFormMessageInterceptor relies on.) 176 | // Ensure the ParentFormMessageInterceptor construction occurs on the WinForms UI thread: 177 | this.InvokeOnUiThreadIfRequired(() => 178 | { 179 | parentFormMessageInterceptor = new ParentFormMessageInterceptor(this); 180 | }); 181 | 182 | ResizeBrowser(); 183 | 184 | var handler = IsBrowserInitializedChanged; 185 | 186 | if (handler != null) 187 | { 188 | handler(this, new IsBrowserInitializedChangedEventArgs(IsBrowserInitialized)); 189 | } 190 | } 191 | 192 | void IWebBrowserInternal.SetAddress(string address) 193 | { 194 | Address = address; 195 | 196 | var handler = AddressChanged; 197 | if (handler != null) 198 | { 199 | handler(this, new AddressChangedEventArgs(address)); 200 | } 201 | } 202 | 203 | void IWebBrowserInternal.SetLoadingStateChange(bool canGoBack, bool canGoForward, bool isLoading) 204 | { 205 | CanGoBack = canGoBack; 206 | CanGoForward = canGoForward; 207 | CanReload = !isLoading; 208 | IsLoading = isLoading; 209 | 210 | var handler = LoadingStateChanged; 211 | if (handler != null) 212 | { 213 | handler(this, new LoadingStateChangedEventArgs(canGoBack, canGoForward, isLoading)); 214 | } 215 | } 216 | 217 | void IWebBrowserInternal.SetTitle(string title) 218 | { 219 | Title = title; 220 | 221 | var handler = TitleChanged; 222 | if (handler != null) 223 | { 224 | handler(this, new TitleChangedEventArgs(title)); 225 | } 226 | } 227 | 228 | void IWebBrowserInternal.SetTooltipText(string tooltipText) 229 | { 230 | TooltipText = tooltipText; 231 | } 232 | 233 | void IWebBrowserInternal.OnFrameLoadStart(FrameLoadStartEventArgs args) 234 | { 235 | var handler = FrameLoadStart; 236 | if (handler != null) 237 | { 238 | handler(this, args); 239 | } 240 | } 241 | 242 | void IWebBrowserInternal.OnFrameLoadEnd(FrameLoadEndEventArgs args) 243 | { 244 | var handler = FrameLoadEnd; 245 | if (handler != null) 246 | { 247 | handler(this, args); 248 | } 249 | } 250 | 251 | void IWebBrowserInternal.OnConsoleMessage(string message, string source, int line) 252 | { 253 | var handler = ConsoleMessage; 254 | if (handler != null) 255 | { 256 | handler(this, new ConsoleMessageEventArgs(message, source, line)); 257 | } 258 | } 259 | 260 | void IWebBrowserInternal.OnStatusMessage(string value) 261 | { 262 | var handler = StatusMessage; 263 | if (handler != null) 264 | { 265 | handler(this, new StatusMessageEventArgs(value)); 266 | } 267 | } 268 | 269 | void IWebBrowserInternal.OnLoadError(IFrame frame, CefErrorCode errorCode, string errorText, string failedUrl) 270 | { 271 | var handler = LoadError; 272 | if (handler != null) 273 | { 274 | handler(this, new LoadErrorEventArgs(frame, errorCode, errorText, failedUrl)); 275 | } 276 | } 277 | 278 | /// 279 | /// Manually implement Focused because cef does not implement it. 280 | /// 281 | /// 282 | /// This is also how the Microsoft's WebBrowserControl implements the Focused property. 283 | /// 284 | public override bool Focused 285 | { 286 | get 287 | { 288 | if (base.Focused) 289 | { 290 | return true; 291 | } 292 | 293 | if (!IsHandleCreated) 294 | { 295 | return false; 296 | } 297 | 298 | return NativeMethodWrapper.IsFocused(Handle); 299 | } 300 | } 301 | 302 | protected override void OnSizeChanged(EventArgs e) 303 | { 304 | base.OnSizeChanged(e); 305 | 306 | ResizeBrowser(); 307 | } 308 | 309 | private void ResizeBrowser() 310 | { 311 | if (IsBrowserInitialized) 312 | { 313 | managedCefBrowserAdapter.Resize(Width, Height); 314 | } 315 | } 316 | 317 | public void NotifyMoveOrResizeStarted() 318 | { 319 | this.ThrowExceptionIfBrowserNotInitialized(); 320 | 321 | managedCefBrowserAdapter.NotifyMoveOrResizeStarted(); 322 | } 323 | 324 | protected override void OnGotFocus(EventArgs e) 325 | { 326 | SetFocus(true); 327 | base.OnGotFocus(e); 328 | } 329 | 330 | /// 331 | /// Tell the browser to acquire/release focus. 332 | /// 333 | public void SetFocus(bool isFocused) 334 | { 335 | this.ThrowExceptionIfBrowserNotInitialized(); 336 | 337 | managedCefBrowserAdapter.SetFocus(isFocused); 338 | } 339 | 340 | public IBrowser GetBrowser() 341 | { 342 | this.ThrowExceptionIfBrowserNotInitialized(); 343 | 344 | return managedCefBrowserAdapter.GetBrowser(); 345 | } 346 | } 347 | } 348 | -------------------------------------------------------------------------------- /CodeClassifier/training-set/csharp/csharp.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.IO; 4 | using System.Linq; 5 | using System.Reflection; 6 | using CodeClassifier.StringTokenizer; 7 | 8 | namespace CodeClassifier 9 | { 10 | public class CodeClassifier 11 | { 12 | private static CodeClassifier _instance; 13 | 14 | private const double SCORE_MULTIPLIER_PER_LEVEL = 2; 15 | private const double SCORE_MULTIPLIER_FOR_EXACT_MATCH = 5; 16 | 17 | private static List _matchTrees; 18 | 19 | private CodeClassifier() 20 | { 21 | string trainingSetPath = Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location); 22 | if (trainingSetPath == null) 23 | { 24 | throw new DirectoryNotFoundException("Could not find the training-set folder."); 25 | } 26 | 27 | // Train classifier 28 | string path = Path.Combine(trainingSetPath, "training-set"); 29 | string[] folders = Directory.GetDirectories(path); 30 | foreach (string folder in folders) 31 | { 32 | string[] files = Directory.GetFiles(folder); 33 | _matchTrees = new List(); 34 | foreach (string filePath in files) 35 | { 36 | string languageName = Path.GetFileNameWithoutExtension(filePath); 37 | if (languageName != null) 38 | { 39 | // Calculate the total possible score to normalize the score results 40 | double totalPossibleScore; 41 | TokenNode rootNode = BuildMatchTree(File.ReadAllText(filePath), out totalPossibleScore); 42 | _matchTrees.Add(new MatchTree(rootNode, languageName, totalPossibleScore)); 43 | } 44 | } 45 | } 46 | } 47 | 48 | private static TokenNode BuildMatchTree(string trainingCode, out double totalScorePossible) 49 | { 50 | List tokens = GetAllTokens(trainingCode); 51 | 52 | // Recursivly build the tree 53 | TokenNode root = new TokenNode(TokenKind.Unknown, 0, 1, null); 54 | double totalScore = 0; 55 | for (int index = 0; index < tokens.Count-1; index++) 56 | { 57 | totalScore += AddTokens(root, tokens, index); 58 | } 59 | 60 | totalScorePossible = totalScore; 61 | return root; 62 | } 63 | 64 | private static double AddTokens(TokenNode tokenNode, IList tokens, int index) 65 | { 66 | double totalScore = 0; 67 | while (index < tokens.Count && tokenNode.Level < 10) 68 | { 69 | Token codeToken = tokens[index]; 70 | TokenNode nextTreeToken = tokenNode.NextTokens.FirstOrDefault(nt => nt.Kind == codeToken.Kind); 71 | if (nextTreeToken == null) 72 | { 73 | // Token doesn't exist on this tree level yet 74 | var newToken = new TokenNode(codeToken.Kind, tokenNode.Level + 1, tokenNode.Score * SCORE_MULTIPLIER_PER_LEVEL, codeToken.Value); 75 | totalScore += tokenNode.Score * SCORE_MULTIPLIER_PER_LEVEL; 76 | tokenNode.NextTokens.Add(newToken); 77 | tokenNode = newToken; 78 | } 79 | else 80 | { 81 | // Token already exists on this level 82 | nextTreeToken.Examples.Add(codeToken.Value); 83 | tokenNode = nextTreeToken; 84 | } 85 | index++; 86 | } 87 | return totalScore; 88 | } 89 | 90 | private static List GetAllTokens(string code) 91 | { 92 | StringTokenizer.StringTokenizer stringTokenizer = new StringTokenizer.StringTokenizer(code); 93 | 94 | List tokens = new List(); 95 | Token token; 96 | do 97 | { 98 | token = stringTokenizer.Next(); 99 | tokens.Add(token); 100 | } while (token.Kind != TokenKind.Eof); 101 | return tokens; 102 | } 103 | 104 | public static string Classify(string snippet ) 105 | { 106 | // ReSharper disable once RedundantAssignment 107 | Dictionary scores; 108 | return Classify(snippet, out scores); 109 | } 110 | 111 | public static string Classify(string snippet, out Dictionary scores ) 112 | { 113 | if (_instance == null) 114 | { 115 | _instance = new CodeClassifier(); 116 | } 117 | 118 | scores = new Dictionary(); 119 | 120 | List tokens = GetAllTokens(snippet); 121 | double maxScore = 0; 122 | string bestMatchLanguage = null; 123 | 124 | foreach (MatchTree matchTree in _matchTrees) 125 | { 126 | double score = 0; 127 | for (int index = 0; index < tokens.Count; index++) 128 | { 129 | score += ScoreTokens(matchTree.MatchTreeRoot, tokens, index); 130 | } 131 | score = score / tokens.Count() / matchTree.TotalPossibleScore; 132 | 133 | //Console.WriteLine(matchTree.Language + "\t" + score); 134 | scores.Add(matchTree.Language, score); 135 | if (score > maxScore) 136 | { 137 | maxScore = score; 138 | bestMatchLanguage = matchTree.Language; 139 | } 140 | } 141 | return bestMatchLanguage; 142 | } 143 | 144 | private static double ScoreTokens(TokenNode tokenNode, IList tokens, int index) 145 | { 146 | Token codeToken = tokens[index]; 147 | TokenNode nextToken = tokenNode.NextTokens.FirstOrDefault(nt => nt.Kind == codeToken.Kind); 148 | if (nextToken != null) 149 | { 150 | // Token exists in match tree => points !!! 151 | double score = nextToken.Examples.Contains(codeToken.Value) ? 152 | SCORE_MULTIPLIER_FOR_EXACT_MATCH: 153 | SCORE_MULTIPLIER_PER_LEVEL; 154 | 155 | if (index < tokens.Count() - 1) 156 | { 157 | return score * ScoreTokens(nextToken, tokens, index + 1); 158 | } 159 | return score; 160 | } 161 | // Token did not exist => no points 162 | return 1; 163 | } 164 | } 165 | } -------------------------------------------------------------------------------- /CodeClassifier/training-set/css/css.css: -------------------------------------------------------------------------------- 1 | 2 | html { 3 | margin: 0; 4 | padding: 0; 5 | border: 0; 6 | -moz-transition: all; 7 | -o-transition: all; 8 | -webkit-transition: all; 9 | transition: all; 10 | } 11 | 12 | .hidden { 13 | display: none !important; 14 | visibility: hidden !important; 15 | } 16 | 17 | body { 18 | stroke: #eee; 19 | stroke-width: 2; 20 | stroke-dasharray: 0; 21 | } 22 | 23 | a { 24 | text-align: center; 25 | font-size: 1.1rem; 26 | padding: 12px 0; 27 | } 28 | 29 | li { 30 | -webkit-touch-callout: none; 31 | -moz-user-select: none; 32 | -ms-user-select: none; 33 | -webkit-user-select: none; 34 | user-select: none; 35 | } 36 | 37 | /* points of interest */ 38 | ul > li { 39 | stroke: steelblue; 40 | stroke-width: 2; 41 | fill: #FFF; 42 | -moz-transform-origin: center; 43 | -ms-transform-origin: center; 44 | -o-transform-origin: center; 45 | -webkit-transform-origin: center; 46 | transform-origin: center; 47 | } 48 | 49 | a + div { 50 | background-size: cover; 51 | } 52 | 53 | 54 | 55 | div { 56 | background: #fff; 57 | } 58 | 59 | .n2-charts-close { 60 | font-size: 32px; 61 | position: absolute; 62 | top: 20px; 63 | right: 20px; 64 | padding: 2px 12px; 65 | cursor: pointer; 66 | } 67 | 68 | /* Landscape styles */ 69 | @media screen and (orientation:landscape) { 70 | 71 | .chart.fullscreen { 72 | font-size: 32px; 73 | position: absolute; 74 | top: 20px; 75 | right: 20px; 76 | padding: 2px 12px; 77 | cursor: pointer; 78 | } 79 | } 80 | 81 | @-moz-keyframes n2-poi-blink { 82 | 0% { 83 | -moz-transform: scale(1); 84 | -ms-transform: scale(1); 85 | -o-transform: scale(1); 86 | -webkit-transform: scale(1); 87 | transform: scale(1); 88 | } 89 | 90 | 100% { 91 | -moz-transform: scale(1); 92 | -ms-transform: scale(1); 93 | -o-transform: scale(1); 94 | -webkit-transform: scale(1); 95 | transform: scale(1); 96 | } 97 | } 98 | 99 | @-ms-keyframes n2-poi-blink { 100 | 0% { 101 | -moz-transform: scale(1); 102 | -ms-transform: scale(1); 103 | -o-transform: scale(1); 104 | -webkit-transform: scale(1); 105 | transform: scale(1); 106 | } 107 | 108 | 100% { 109 | -moz-transform: scale(1); 110 | -ms-transform: scale(1); 111 | -o-transform: scale(1); 112 | -webkit-transform: scale(1); 113 | transform: scale(1); 114 | } 115 | } 116 | 117 | @-webkit-keyframes n2-poi-blink { 118 | 0% { 119 | -moz-transform: scale(1); 120 | -ms-transform: scale(1); 121 | -o-transform: scale(1); 122 | -webkit-transform: scale(1); 123 | transform: scale(1); 124 | } 125 | 126 | 100% { 127 | -moz-transform: scale(1); 128 | -ms-transform: scale(1); 129 | -o-transform: scale(1); 130 | -webkit-transform: scale(1); 131 | transform: scale(1); 132 | } 133 | } 134 | 135 | @keyframes n2-poi-blink { 136 | 0% { 137 | -moz-transform: scale(1); 138 | -ms-transform: scale(1); 139 | -o-transform: scale(1); 140 | -webkit-transform: scale(1); 141 | transform: scale(1); 142 | } 143 | 144 | 100% { 145 | -moz-transform: scale(1); 146 | -ms-transform: scale(1); 147 | -o-transform: scale(1); 148 | -webkit-transform: scale(1); 149 | transform: scale(1); 150 | } 151 | } 152 | -------------------------------------------------------------------------------- /CodeClassifier/training-set/html/options.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | Chosen: A jQuery Plugin by Harvest to Tame Unwieldy Select Boxes 6 | 7 | 8 | 9 | 13 | 14 | 15 |
16 |
17 |
18 |

Chosen (v1.4.2)

19 |
20 |

Chosen has a number of options and attributes that allow you to have full control of your select boxes.

21 | 22 |

Options

23 |

The following options are available to pass into Chosen on instantiation.

24 | 25 |

Example:

26 |
 27 |   $(".my_select_box").chosen({
 28 |     disable_search_threshold: 10,
 29 |     no_results_text: "Oops, nothing found!",
 30 |     width: "95%"
 31 |   });
 32 | 
33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 73 | 74 | 75 | 76 | 77 | 78 | 79 | 80 | 81 | 82 | 83 | 84 | 85 | 86 | 87 | 88 | 89 | 90 | 91 | 92 | 93 | 94 | 95 | 96 | 97 | 98 | 99 | 100 | 101 | 102 | 103 | 104 | 105 | 106 | 107 | 111 | 112 | 113 | 114 | 115 | 118 | 119 |
OptionDefaultDescription
allow_single_deselectfalseWhen set to true on a single select, Chosen adds a UI element which selects the first element (if it is blank).
disable_searchfalseWhen set to true, Chosen will not display the search field (single selects only).
disable_search_threshold0Hide the search input on single selects if there are fewer than (n) options.
enable_split_word_searchtrueBy default, searching will match on any word within an option tag. Set this option to false if you want to only match on the entire text of an option tag.
inherit_select_classesfalseWhen set to true, Chosen will grab any classes on the original select field and add them to Chosen’s container div.
max_selected_optionsInfinityLimits how many options the user can select. When the limit is reached, the chosen:maxselected event is triggered.
no_results_text"No results match"The text to be displayed when no matching results are found. The current search is shown at the end of the text (e.g., 72 | No results match "Bad Search").
placeholder_text_multiple"Select Some Options"The text to be displayed as a placeholder when no options are selected for a multiple select.
placeholder_text_single"Select an Option"The text to be displayed as a placeholder when no options are selected for a single select.
search_containsfalseBy default, Chosen’s search matches starting at the beginning of a word. Setting this option to true allows matches starting from anywhere within a word. This is especially useful for options that include a lot of special characters or phrases in ()s and []s.
single_backstroke_deletetrueBy default, pressing delete/backspace on multiple selects will remove a selected choice. When false, pressing delete/backspace will highlight the last choice, and a second press deselects it.
widthOriginal select width.The width of the Chosen select box. By default, Chosen attempts to match the width of the select box you are replacing. If your select is hidden when Chosen is instantiated, you must specify a width or the select will show up with a width of 0.
display_disabled_optionstrueBy default, Chosen includes disabled options in search results with a special styling. Setting this option to false will hide disabled results and exclude them from searches.
display_selected_optionstrue 108 |

By default, Chosen includes selected options in search results with a special styling. Setting this option to false will hide selected results and exclude them from searches.

109 |

Note: this is for multiple selects only. In single selects, the selected result will always be displayed.

110 |
include_group_label_in_selectedfalse 116 |

By default, Chosen only shows the text of a selected option. Setting this option to true will show the text and group (if any) of the selected option.

117 |
120 | 121 |

Attributes

122 |

Certain attributes placed on the select tag or its options can be used to configure Chosen.

123 | 124 |

Example:

125 | 126 |
127 |   <select class="my_select_box" data-placeholder="Select Your Options">
128 |     <option value="1">Option 1</option>
129 |     <option value="2" selected>Option 2</option>
130 |     <option value="3" disabled>Option 3</option>
131 |   </select>
132 | 
133 | 134 | 135 | 136 | 137 | 138 | 139 | 140 | 144 | 145 | 146 | 147 | 148 | 149 | 150 | 151 | 152 | 153 |
AttributeDescription
data-placeholder 141 |

The text to be displayed as a placeholder when no options are selected for a select. Defaults to "Select an Option" for single selects or "Select Some Options" for multiple selects.

142 |

Note:This attribute overrides anything set in the placeholder_text_multiple or placeholder_text_single options.

143 |
multipleThe attribute multiple on your select box dictates whether Chosen will render a multiple or single select.
selected, disabledChosen automatically highlights selected options and disables disabled options.
154 | 155 |

Classes

156 |

Classes placed on the select tag can be used to configure Chosen.

157 | 158 |

Example:

159 | 160 |
161 |   <select class="my_select_box chosen-rtl">
162 |     <option value="1">Option 1</option>
163 |     <option value="2">Option 2</option>
164 |     <option value="3">Option 3</option>
165 |   </select>
166 | 
167 | 168 | 169 | 170 | 171 | 172 | 173 | 174 | 175 | 179 | 180 |
ClassnameDescription
chosen-rtl 176 |

Chosen supports right-to-left text in select boxes. Add the class chosen-rtl to your select tag to support right-to-left text options.

177 |

Note: The chosen-rtl class will pass through to the Chosen select even when the inherit_select_classes option is set to false.

178 |
181 | 182 |

Triggered Events

183 |

Chosen triggers a number of standard and custom events on the original select field.

184 | 185 |

Example:

186 | 187 |
188 |   $('.my_select_box').on('change', function(evt, params) {
189 |     do_something(evt, params);
190 |   });
191 | 
192 | 193 | 194 | 195 | 196 | 197 | 198 | 199 | 203 | 204 | 205 | 206 | 207 | 208 | 209 | 210 | 211 | 212 | 213 | 214 | 215 | 216 | 217 | 218 | 219 | 220 | 221 | 222 | 223 | 224 |
EventDescription
change 200 |

Chosen triggers the standard DOM event whenever a selection is made (it also sends a selected or deselected parameter that tells you which option was changed).

201 |

Note: in order to use change in the Prototype version, you have to include the Event.simulate class. The selected and deselected parameters are not available for Prototype.

202 |
chosen:readyTriggered after Chosen has been fully instantiated.
chosen:maxselectedTriggered if max_selected_options is set and that total is broken.
chosen:showing_dropdownTriggered when Chosen’s dropdown is opened.
chosen:hiding_dropdownTriggered when Chosen’s dropdown is closed.
chosen:no_resultsTriggered when a search returns no matching results.
225 | 226 |

227 | Note: all custom Chosen events (those that being with chosen:) also include the chosen object as a parameter. 228 |

229 | 230 |

Triggerable Events

231 |

You can trigger several events on the original select field to invoke a behavior in Chosen.

232 | 233 |

Example:

234 | 235 |
236 |   // tell Chosen that a select has changed
237 |   $('.my_select_box').trigger('chosen:updated');
238 | 
239 | 240 | 241 | 242 | 243 | 244 | 245 | 246 | 247 | 248 | 249 | 250 | 251 | 252 | 253 | 254 | 255 | 256 | 257 | 258 | 259 | 260 |
EventDescription
chosen:updatedThis event should be triggered whenever Chosen’s underlying select element changes (such as a change in selected options).
chosen:activateThis is the equivalant of focusing a standard HTML select field. When activated, Chosen will capure keypress events as if you had clicked the field directly.
chosen:openThis event activates Chosen and also displays the search results.
chosen:closeThis event deactivates Chosen and hides the search results.
261 | 262 | 265 | 266 |
267 |
268 |
269 | 273 |
274 | 275 | 276 | 277 | -------------------------------------------------------------------------------- /CodeClassifier/training-set/javascript/new 3.txt: -------------------------------------------------------------------------------- 1 | /** 2 | * Module dependencies. 3 | */ 4 | 5 | var fork = require('child_process').fork 6 | , qs = require('querystring') 7 | , eq = require('eq') 8 | , os = require('os') 9 | , ms = require('ms') 10 | , env = process.env.NODE_ENV 11 | , Distributor = require('distribute') 12 | , EventEmitter = require('events').EventEmitter 13 | , debug = require('debug')('up') 14 | 15 | /** 16 | * Module exports. 17 | */ 18 | 19 | module.exports = exports = UpServer; 20 | 21 | /** 22 | * Version. 23 | * 24 | * @api public 25 | */ 26 | 27 | exports.version = '0.2.1'; 28 | 29 | /** 30 | * Worker constructor. 31 | * 32 | * @api public 33 | */ 34 | 35 | exports.Worker = Worker; 36 | 37 | /** 38 | * Number of CPUs available. 39 | */ 40 | 41 | var cpus = os.cpus().length; 42 | 43 | /** 44 | * Default worker timeout. 45 | */ 46 | 47 | var workerTimeout = 'development' == env ? '500ms' : '10m'; 48 | 49 | /** 50 | * Default number of workers. 51 | */ 52 | 53 | var numWorkers = 'development' == env ? 1 : cpus; 54 | 55 | /** 56 | * Default minimum expected lifetime of a worker. 57 | * If a worker dies younger, we don't respawn even if keepAlive == true. 58 | * We want to prevent auto-respawning storms from overloading the system. 59 | */ 60 | var minExpectedLifetime = '20s'; 61 | 62 | /** 63 | * UpServer factory/constructor. 64 | * 65 | * @param {String} module file 66 | * @param {Object} options 67 | * @api public 68 | */ 69 | 70 | function UpServer (server, file, opts) { 71 | if (this == global) return new UpServer(server, file, opts); 72 | 73 | Distributor.call(this, server); 74 | 75 | var self = this; 76 | opts = opts || {}; 77 | 78 | this.file = file; 79 | this.numWorkers = eq(opts.numWorkers || numWorkers, { cpus: cpus }); 80 | this.workerTimeout = ms(null != opts.workerTimeout 81 | ? opts.workerTimeout : workerTimeout); 82 | this.requires = opts.requires || []; 83 | this.assumeReady = opts.assumeReady === undefined ? true : !!opts.assumeReady; 84 | this.keepAlive = opts.keepAlive || false; 85 | this.minExpectedLifetime = ms(opts.minExpectedLifetime != null ? opts.minExpectedLifetime : minExpectedLifetime); 86 | if (false !== opts.workerPingInterval) { 87 | this.workerPingInterval = ms(opts.workerPingInterval || '1m'); 88 | } 89 | 90 | this.workers = []; 91 | this.spawning = []; 92 | this.lastIndex = -1; 93 | 94 | if (opts.title) { 95 | this.title = opts.title; 96 | process.title = this.title + ' master'; 97 | } 98 | 99 | // setup workers 100 | this.spawnWorkers(this.numWorkers); 101 | }; 102 | 103 | /** 104 | * Inherits from EventEmitter. 105 | */ 106 | 107 | UpServer.prototype.__proto__ = Distributor.prototype; 108 | 109 | /** 110 | * Reloads the workers. 111 | * 112 | * @param {Function} callback 113 | * @return {UpServer} for chaining 114 | * @api public 115 | */ 116 | 117 | UpServer.prototype.reload = function (fn) { 118 | if (this.reloading) { 119 | debug('reloading in process - ignoring reload'); 120 | return this; 121 | } 122 | 123 | // remove all workers in the spawning state 124 | for (var i = 0, l = this.spawning.length; i < l; i++) { 125 | this.spawning[i].shutdown(); 126 | } 127 | 128 | if (this.workerTimeout > 0) { 129 | // snapshot what workers we'll shut down 130 | var reload = [].concat(this.workers) 131 | , self = this 132 | 133 | debug('reloading - spawning %d new workers', this.numWorkers); 134 | this.spawnWorkers(this.numWorkers); 135 | 136 | this.once('spawn', function (worker) { 137 | debug('worker %s spawned - removing old workers', worker.pid); 138 | self.emit('reload'); 139 | fn && fn(); 140 | 141 | // shut down old workers 142 | for (var i = 0, l = reload.length; i < l; i++) { 143 | reload[i].shutdown(); 144 | } 145 | }); 146 | } else { 147 | debug('removing old workers'); 148 | for (var i = 0, l = this.workers.length; i < l; i++) { 149 | this.workers[i].shutdown(); 150 | } 151 | 152 | var self = this 153 | 154 | this.on('terminate', function listener() { 155 | if (self.workers.length == 0 && self.spawning == 0) { 156 | debug('all workers removed. spawning %d new workers', self.numWorkers); 157 | self.spawnWorkers(self.numWorkers); 158 | self.removeListener('terminate', listener); 159 | 160 | this.once('spawn', function() { 161 | self.emit('reload'); 162 | fn && fn(); 163 | }) 164 | } 165 | }) 166 | } 167 | 168 | return this; 169 | }; 170 | 171 | /** 172 | * Helper function to spawn multiple workers. 173 | * 174 | * @param {Number} number of workers to spawn 175 | * @api public 176 | */ 177 | 178 | UpServer.prototype.spawnWorkers = function (n) { 179 | debug('spawning %d workers from master %d', n, process.pid); 180 | for (var i = 0, l = n; i < l; i++) { 181 | this.spawnWorker(); 182 | } 183 | }; 184 | 185 | /** 186 | * Spawns a worker that binds to an available port. 187 | * 188 | * @api public 189 | */ 190 | 191 | UpServer.prototype.spawnWorker = function (fn) { 192 | var w = new Worker(this) 193 | , self = this 194 | 195 | // keep track that we're spawning 196 | this.spawning.push(w); 197 | 198 | w.on('stateChange', function () { 199 | switch (w.readyState) { 200 | case 'spawned': 201 | self.spawning.splice(self.spawning.indexOf(w), 1); 202 | self.workers.push(w); 203 | self.emit('spawn', w); 204 | break; 205 | 206 | case 'terminating': 207 | case 'terminated': 208 | if (~self.spawning.indexOf(w)) { 209 | self.spawning.splice(self.spawning.indexOf(w), 1); 210 | } 211 | if (~self.workers.indexOf(w)) { 212 | self.workers.splice(self.workers.indexOf(w), 1); 213 | self.lastIndex = -1; 214 | if (self.keepAlive && (self.workers.length + self.spawning.length < self.numWorkers)) { 215 | if (new Date().getTime() - w.birthtime < self.minExpectedLifetime) { 216 | debug('worker %s found dead at a too young age. won\'t respawn new worker', w.pid); 217 | } 218 | else { 219 | debug('worker %s found dead. spawning 1 new worker', w.pid); 220 | self.spawnWorker(); 221 | } 222 | } 223 | } 224 | self.emit('terminate', w) 225 | break; 226 | } 227 | }); 228 | }; 229 | 230 | /** 231 | * Gets the next port in the round. 232 | * 233 | * @api private 234 | */ 235 | 236 | UpServer.prototype.nextWorker = function () { 237 | this.lastIndex++; 238 | if (!this.workers[this.lastIndex]) this.lastIndex = 0; 239 | return this.workers[this.lastIndex]; 240 | }; 241 | 242 | /** 243 | * Default HTTP/WS handler (overridden). 244 | * By default, `up` distributes based on a round robin. 245 | * 246 | * @api private 247 | */ 248 | 249 | UpServer.prototype.defaultHTTP = 250 | UpServer.prototype.defaultWS = function (req, res, next) { 251 | 252 | var ioRegex = 253 | /^\/socket\.io\/[^\/]*\/(xhr-polling|htmlfile|jsonp-polling)\/([^\?]+).*/; 254 | var matcher = req.url.match(ioRegex); 255 | 256 | if(this.workers.length && matcher && matcher.length > 2) { 257 | // transport = matcher[1], sid = matcher[2] 258 | var sid = matcher[2]; 259 | 260 | if(typeof(sid) !== 'number') { 261 | var glyphs = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/'; 262 | var result = 0; 263 | 264 | sid = sid.split(''); 265 | for(var index = sid.length - 1; index >= 0; index--) { 266 | result = (result * 64) + glyphs.indexOf(sid[index]); 267 | } 268 | sid = result; 269 | } 270 | 271 | var workerIndex = sid % this.workers.length; 272 | next(this.workers[workerIndex].port); 273 | } else if (this.workers.length) { 274 | next(this.nextWorker().port); 275 | } else { 276 | var self = this; 277 | this.once('spawn', function () { 278 | next(self.nextWorker().port); 279 | }); 280 | } 281 | }; 282 | 283 | /** 284 | * Worker constructor. 285 | * 286 | * @api private 287 | */ 288 | 289 | function Worker (server) { 290 | this.server = server; 291 | this.readyState = 'spawning'; 292 | 293 | var opts = JSON.stringify({ 294 | file: server.file 295 | , requires: server.requires 296 | , assumeReady: server.assumeReady 297 | , pingInterval: server.workerPingInterval 298 | , title: server.title 299 | }); 300 | 301 | this.proc = fork(__dirname + '/worker.js', [opts], { env: process.env }); 302 | this.proc.on('message', this.onMessage.bind(this)); 303 | this.proc.on('exit', this.onExit.bind(this)); 304 | this.pid = this.proc.pid; 305 | this.birthtime = new Date().getTime(); 306 | debug('worker %s created', this.pid); 307 | } 308 | 309 | /** 310 | * Inherits from EventEmitter. 311 | */ 312 | 313 | Worker.prototype.__proto__ = EventEmitter.prototype; 314 | 315 | /** 316 | * Called upon worker exit. 317 | * Sudden exits will mean the worker won't go through `terminating` state 318 | * 319 | * @api private 320 | */ 321 | 322 | Worker.prototype.onExit = function () { 323 | debug('worker %s exited ' 324 | + ('terminating' != this.readyState ? 'unexpectedly': ''), this.pid); 325 | this.readyState = 'terminated'; 326 | this.emit('stateChange'); 327 | }; 328 | 329 | /** 330 | * Handles an incoming message from the worker. 331 | * 332 | * @api private 333 | */ 334 | 335 | Worker.prototype.onMessage = function (msg) { 336 | switch (msg.type) { 337 | case 'addr': 338 | // avoid spawns after SIGHUP was sent 339 | if ('spawning' == this.readyState) { 340 | debug('worker %s listening on port %s', this.pid, msg.addr.port); 341 | this.port = msg.addr.port; 342 | this.readyState = 'spawned'; 343 | this.emit('stateChange'); 344 | } 345 | break; 346 | } 347 | }; 348 | 349 | /** 350 | * Shuts down a worker. 351 | * 352 | * @api private 353 | */ 354 | 355 | Worker.prototype.shutdown = function () { 356 | if ('spawned' == this.readyState) { 357 | var timeout = this.server.workerTimeout; 358 | debug('telling worker %s to exit in %dms', this.proc.pid, timeout); 359 | this.proc.send({ type: 'die', time: timeout }); 360 | this.readyState = 'terminating'; 361 | this.emit('stateChange'); 362 | } else if ('spawning' == this.readyState) { 363 | debug('killing spawning worker %s', this.pid); 364 | switch (process.platform) { 365 | case 'win32': 366 | this.proc.kill(); 367 | break; 368 | default: 369 | this.proc.kill('SIGHUP'); 370 | } 371 | this.readyState = 'terminating'; 372 | this.emit('stateChange'); 373 | } 374 | }; 375 | 376 | /** 377 | * Send ready signal from within a worker 378 | * 379 | * @api public 380 | */ 381 | 382 | exports.ready = function () { 383 | process.emit('message', { type: 'ready' }); 384 | }; 385 | -------------------------------------------------------------------------------- /CodeClassifier/training-set/javascript/new 4.txt: -------------------------------------------------------------------------------- 1 | Object.extend(String.prototype, { 2 | // if a string doesn't end with str it appends it 3 | ensureEndsWith: function(str) { 4 | return this.endsWith(str) ? this : this + str; 5 | }, 6 | 7 | // makes sure that string ends with px (for setting widths and heights) 8 | px: function() { 9 | return this.ensureEndsWith('px'); 10 | } 11 | }); 12 | 13 | Object.extend(Number.prototype, { 14 | // makes sure that number ends with px (for setting widths and heights) 15 | px: function() { 16 | return this.toString().px(); 17 | } 18 | }); 19 | 20 | var Window = { 21 | // returns correct dimensions for window, had issues with prototype's sometimes. this was ganked from apple. 22 | size: function() { 23 | var width = window.innerWidth || (window.document.documentElement.clientWidth || window.document.body.clientWidth); 24 | var height = window.innerHeight || (window.document.documentElement.clientHeight || window.document.body.clientHeight); 25 | var x = window.pageXOffset || (window.document.documentElement.scrollLeft || window.document.body.scrollLeft); 26 | var y = window.pageYOffset || (window.document.documentElement.scrollTop || window.document.body.scrollTop); 27 | return {'width':width, 'height':height, 'x':x, 'y':y} 28 | } 29 | } 30 | 31 | var FancyZoomBox = { 32 | directory : 'images', 33 | zooming : false, 34 | setup : false, 35 | 36 | init: function(directory) { 37 | if (FancyZoomBox.setup) return; 38 | FancyZoomBox.setup = true; 39 | 40 | var ie = navigator.userAgent.match(/MSIE\s(\d)+/); 41 | if (ie) { 42 | var version = parseInt(ie[1]); 43 | Prototype.Browser['IE' + version.toString()] = true; 44 | Prototype.Browser.ltIE7 = (version < 7) ? true : false; 45 | } 46 | 47 | var html = ''; 74 | 75 | var body = $$('body').first(); 76 | body.insert(html); 77 | 78 | FancyZoomBox.zoom = $('zoom'); 79 | FancyZoomBox.zoom_table = $('zoom_table'); 80 | FancyZoomBox.zoom_close = $('zoom_close'); 81 | FancyZoomBox.zoom_content = $('zoom_content'); 82 | FancyZoomBox.zoom_close.observe('click', FancyZoomBox.hide); 83 | FancyZoomBox.middle_row = $A([$$('td.ml'), $$('td.mm'), $$('td.mr')]).flatten(); 84 | FancyZoomBox.cells = FancyZoomBox.zoom_table.select('td'); 85 | 86 | // hide zoom if click fired is not inside zoom 87 | $$('html').first().observe('click', function(e) { 88 | var click_in_zoom = e.findElement('#zoom'), 89 | zoom_display = FancyZoomBox.zoom.getStyle('display'); 90 | if (zoom_display == 'block' && !click_in_zoom) { 91 | FancyZoomBox.hide(e); 92 | } 93 | }); 94 | 95 | // esc to close zoom box 96 | $(document).observe('keyup', function(e) { 97 | var zoom_display = FancyZoomBox.zoom.getStyle('display'); 98 | if (e.keyCode == Event.KEY_ESC && zoom_display == 'block') { 99 | FancyZoomBox.hide(e); 100 | } 101 | }); 102 | 103 | // just use gifs as ie6 and below suck 104 | if (Prototype.Browser.ltIE7) { 105 | FancyZoomBox.switchBackgroundImagesTo('gif'); 106 | } 107 | }, 108 | 109 | show: function(e) { 110 | e.stop(); 111 | if (FancyZoomBox.zooming) return; 112 | FancyZoomBox.zooming = true; 113 | var element = e.findElement('a'); 114 | var related_div = element.content_div; 115 | var width = (element.zoom_width || related_div.getWidth()) + 60; 116 | var height = (element.zoom_height || related_div.getHeight()) + 60; 117 | var d = Window.size(); 118 | var yOffset = document.viewport.getScrollOffsets()[1]; 119 | // ensure that newTop is at least 0 so it doesn't hide close button 120 | var newTop = Math.max((d.height/2) - (height/2) + yOffset, 0); 121 | var newLeft = (d.width/2) - (width/2); 122 | FancyZoomBox.curTop = e.pointerY(); 123 | FancyZoomBox.curLeft = e.pointerX(); 124 | FancyZoomBox.moveX = -(FancyZoomBox.curLeft - newLeft); 125 | FancyZoomBox.moveY = -(FancyZoomBox.curTop - newTop); 126 | FancyZoomBox.zoom.hide().setStyle({ 127 | position : 'absolute', 128 | top : FancyZoomBox.curTop.px(), 129 | left : FancyZoomBox.curLeft.px() 130 | }); 131 | 132 | new Effect.Parallel([ 133 | new Effect.Appear(FancyZoomBox.zoom, {sync:true}), 134 | new Effect.Move(FancyZoomBox.zoom, {x: FancyZoomBox.moveX, y: FancyZoomBox.moveY, sync: true}), 135 | new Effect.Morph(FancyZoomBox.zoom, { 136 | style: { 137 | width: width.px(), 138 | height: height.px() 139 | }, 140 | sync: true, 141 | beforeStart: function(effect) { 142 | // middle row height must be set for IE otherwise it tries to be "logical" with the height 143 | if (Prototype.Browser.IE) { 144 | FancyZoomBox.middle_row.invoke('setStyle', {height:(height-40).px()}); 145 | } 146 | FancyZoomBox.fixBackgroundsForIE(); 147 | }, 148 | afterFinish: function(effect) { 149 | FancyZoomBox.zoom_content.innerHTML = related_div.innerHTML; 150 | FancyZoomBox.unfixBackgroundsForIE(); 151 | FancyZoomBox.zoom_close.show(); 152 | FancyZoomBox.zooming = false; 153 | } 154 | }) 155 | ], { duration: 0.5 }); 156 | }, 157 | 158 | hide: function(e) { 159 | e.stop(); 160 | if (FancyZoomBox.zooming) return; 161 | FancyZoomBox.zooming = true; 162 | new Effect.Parallel([ 163 | new Effect.Move(FancyZoomBox.zoom, {x: FancyZoomBox.moveX*-1, y: FancyZoomBox.moveY*-1, sync: true}), 164 | new Effect.Morph(FancyZoomBox.zoom, { 165 | style: { 166 | width: '1'.px(), 167 | height: '1'.px() 168 | }, 169 | sync : true, 170 | beforeStart: function(effect) { 171 | FancyZoomBox.fixBackgroundsForIE(); 172 | FancyZoomBox.zoom_content.innerHTML = ''; 173 | FancyZoomBox.zoom_close.hide(); 174 | }, 175 | afterFinish: function(effect) { 176 | FancyZoomBox.unfixBackgroundsForIE(); 177 | FancyZoomBox.zooming = false; 178 | } 179 | }), 180 | new Effect.Fade(FancyZoomBox.zoom, {sync:true}) 181 | ], { duration: 0.5 }); 182 | }, 183 | 184 | // switches the backgrounds of the cells and the close image to png's or gif's 185 | // fixes ie's issues with fading and appearing transparent png's with 186 | // no background and ie6's craptacular handling of transparent png's 187 | switchBackgroundImagesTo: function(to) { 188 | FancyZoomBox.cells.each(function(td) { 189 | var bg = td.getStyle('background-image').gsub(/\.(png|gif|none)\)$/, '.' + to + ')'); 190 | td.setStyle('background-image: ' + bg); 191 | }); 192 | var close_img = FancyZoomBox.zoom_close.firstDescendant(); 193 | var new_img = close_img.readAttribute('src').gsub(/\.(png|gif|none)$/, '.' + to); 194 | close_img.writeAttribute('src', new_img); 195 | }, 196 | 197 | // prevents the thick black border that happens when appearing or fading png in IE 198 | fixBackgroundsForIE: function() { 199 | if (Prototype.Browser.IE7) { FancyZoomBox.switchBackgroundImagesTo('gif'); } 200 | }, 201 | 202 | // swaps back to png's for prettier shadows 203 | unfixBackgroundsForIE: function() { 204 | if (Prototype.Browser.IE7) { FancyZoomBox.switchBackgroundImagesTo('png'); } 205 | } 206 | } 207 | 208 | var FancyZoom = Class.create({ 209 | initialize: function(element) { 210 | this.options = arguments.length > 1 ? arguments[1] : {}; 211 | FancyZoomBox.init(); 212 | this.element = $(element); 213 | if (this.element) { 214 | this.element.content_div = $(this.element.readAttribute('href').gsub(/^#/, '')); 215 | this.element.content_div.hide(); 216 | this.element.zoom_width = this.options.width; 217 | this.element.zoom_height = this.options.height; 218 | this.element.observe('click', FancyZoomBox.show); 219 | } 220 | } 221 | }); -------------------------------------------------------------------------------- /CodeClassifier/training-set/javascript/new 5.txt: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | var path = require('path'), 4 | fs = require('fs'), 5 | natural = require('natural'), 6 | lunr = require('lunr'), 7 | tokenizer = new natural.WordTokenizer(), 8 | loc = path.resolve(__dirname, 'content'), 9 | scraper = { 10 | title: /\[meta:title\]:\s<>\s\((.+?)\)(?!\))/, 11 | description: /\[meta:description\]:\s<>\s\((.+?)\)(?!\))/, 12 | firstlines: /^((.*\n){2}){1,3}/ 13 | }; 14 | 15 | // 16 | // ### @private function scrape() 17 | // #### @content {String} document content 18 | // #### @key {String} scraper key 19 | // #### @n {Number} index of match that should be returned 20 | // Scrapes the [key] from the content by Regular Epression 21 | // 22 | function scrape(content, key, n) { 23 | if (!content) return ''; 24 | 25 | var match = content.match(scraper[key]); 26 | 27 | // Only return scraped content if there is a meta:[key]. 28 | return match && match[n] ? match[n].trim() : ''; 29 | } 30 | 31 | // 32 | // ### @private function normalize() 33 | // #### @file {String} file name 34 | // Normalize the file name to resolve to a Markdown or index file. 35 | // 36 | function normalize(file) { 37 | if (!file) file = 'index.md'; 38 | 39 | // Remove trailing slashes from paths 40 | if (file[file.length - 1] === '/') file = file.slice(0, -1); 41 | 42 | return ~file.indexOf('.md') ? file : file + '.md'; 43 | } 44 | 45 | // 46 | // ### @private function fileContent() 47 | // #### @content {String} Document content 48 | // Sugar content with additional properties from scraped content. 49 | // 50 | function fileContent(content) { 51 | return { 52 | content: content || '', 53 | description: scrape(content, 'description', 1) || scrape(content, 'firstlines', 0), 54 | title: scrape(content, 'title', 1), 55 | tags: tags(content, 10) 56 | }; 57 | } 58 | 59 | // 60 | // ### @private function frequency() 61 | // #### @content {String} Document content 62 | // Return list of words scored by Term Frequency-Inverse Document Frequency. 63 | // 64 | function frequency(content) { 65 | var tfidf = new natural.TfIdf(), 66 | processed = [], 67 | words = []; 68 | 69 | // Add the current content. 70 | content = content.toLowerCase(); 71 | tfidf.addDocument(content); 72 | 73 | tokenizer.tokenize(content).forEach(function wordFrequency(word) { 74 | // Return early if word is processed, to short or only a number. 75 | if (+word || word.length < 3 || ~processed.indexOf(word)) return; 76 | 77 | words.push({ 78 | word: word, 79 | score: tfidf.tfidf(word, 0) 80 | }); 81 | 82 | // Add word to processed so tfidf is not called more than required. 83 | processed.push(word); 84 | }); 85 | 86 | return words; 87 | } 88 | 89 | // 90 | // ### @private function tags() 91 | // #### @content {String} Document content 92 | // #### @n {Number} number of tags 93 | // Return n highest scoring tags as determined by term frequency. 94 | // 95 | function tags(content, n) { 96 | if (!content) return []; 97 | 98 | return frequency(content).sort(function sortByScore(a, b) { 99 | return b.score - a.score; 100 | }).slice(0, n).map(function extractWords(tag) { 101 | return tag.word; 102 | }); 103 | } 104 | 105 | // 106 | // ### @private function read() 107 | // #### @file {String} Filename 108 | // #### @callback {Function} Callback for file contents 109 | // Returns file content by normalized path, if a callback is provided, content 110 | // is returned asynchronously. 111 | // 112 | function read(file, callback) { 113 | file = path.resolve(loc, normalize(file)); 114 | if (!callback) return fileContent(fs.readFileSync(file, 'utf8')); 115 | 116 | fs.readFile(file, 'utf8', function read(error, content) { 117 | callback.apply(this, [error, fileContent(content)]); 118 | }); 119 | } 120 | 121 | // 122 | // ### @private function walk() 123 | // #### @dir {String} Directory path to crawl 124 | // #### @result {Object} Append content to current results 125 | // #### @callback {Function} Callback for sub directory 126 | // #### @sub {Boolean} Is directory subdirectory of dir 127 | // Recusive walk of directory by using asynchronous functions, returns 128 | // a collection of markdown files in each folder. 129 | // 130 | function walk(dir, callback, result, sub) { 131 | var current = sub ? path.basename(dir) : 'index'; 132 | 133 | // Prepare containers. 134 | result = result || {}; 135 | result[current] = {}; 136 | 137 | // Read the current directory 138 | fs.readdir(dir, function readDir(error, list) { 139 | if (error) return callback(error); 140 | 141 | var pending = list.length; 142 | if (!pending) return callback(null, result); 143 | 144 | list.forEach(function loopFiles(file) { 145 | file = dir + '/' + file; 146 | 147 | fs.stat(file, function statFile(error, stat) { 148 | var name, ref; 149 | 150 | if (stat && stat.isDirectory()) { 151 | walk(file, function dirDone() { 152 | if (!--pending) callback(null, result); 153 | }, result, true); 154 | } else { 155 | // Only get markdown files from the directory content. 156 | if (path.extname(file) !== '.md') return; 157 | 158 | ref = path.basename(file, '.md'); 159 | name = ['/' + path.basename(dir), ref]; 160 | 161 | // Only append the name of the file if not index 162 | if (ref === 'index') name.pop(); 163 | 164 | // Get the tile of the file. 165 | read(file, function getFile(err, file) { 166 | result[current][ref] = { 167 | href: sub ? name.join('/') + '/' : '/', 168 | title: file.title, 169 | description: file.description, 170 | path: dir 171 | }; 172 | 173 | if (!--pending) callback(null, result); 174 | }); 175 | } 176 | }); 177 | }); 178 | }); 179 | } 180 | 181 | // 182 | // ### @private function walkSync() 183 | // #### @dir {String} Directory path to crawl 184 | // #### @result {Object} Append content to current results 185 | // #### @sub {Boolean} Is directory subdirectory of dir 186 | // Recusive walk of directory by using synchronous functions, returns 187 | // a collection of markdown files in each folder. 188 | // 189 | function walkSync(dir, result, sub) { 190 | var current = sub ? path.basename(dir) : 'index'; 191 | 192 | // Prepare containers. 193 | result = result || {}; 194 | result[current] = {}; 195 | 196 | // Read the current directory 197 | fs.readdirSync(dir).forEach(function loopDir(file) { 198 | var stat, name, ref; 199 | 200 | file = dir + '/' + file; 201 | stat = fs.statSync(file); 202 | 203 | // If directory initiate another walk. 204 | if (stat && stat.isDirectory()) { 205 | walkSync(file, result, true); 206 | } else { 207 | // Only get markdown files from the directory content. 208 | if (path.extname(file) !== '.md') return; 209 | 210 | ref = path.basename(file, '.md'); 211 | name = ['/' + path.basename(dir), ref]; 212 | 213 | // Only append the name of the file if not index 214 | if (ref === 'index') name.pop(); 215 | 216 | // Append file information to current container. 217 | file = read(file); 218 | result[current][ref] = { 219 | href: sub ? name.join('/') + '/' : '/', 220 | title: file.title, 221 | description: file.description, 222 | path: dir 223 | }; 224 | } 225 | }); 226 | 227 | return result; 228 | } 229 | 230 | // 231 | // ### function Handbook() 232 | // Constructor for easy access to Handbook content. On constructuing handbook 233 | // synchronously prepare the search index. Listening to a search index ready 234 | // event is not required thus easing the flow. 235 | // 236 | function Handbook() { 237 | var toc = this.index = walkSync(loc), 238 | cache = this.cache = {}, 239 | idx = this.idx = lunr(function setupLunr() { 240 | this.field('title', { boost: 10 }); 241 | this.field('body'); 242 | }); 243 | 244 | Object.keys(toc).forEach(function loopSections(section) { 245 | Object.keys(toc[section]).forEach(function loopPages(page) { 246 | var document = read((section !== 'index' ? section + '/' : '') + page) 247 | , id = section + '/' + page; 248 | 249 | idx.add({ 250 | id: id, 251 | title: document.title, 252 | body: document.content 253 | }); 254 | 255 | // 256 | // Keep cached reference of all documents, for quick external access. 257 | // 258 | cache[id] = document; 259 | }); 260 | }); 261 | } 262 | 263 | // 264 | // ### function get() 265 | // #### @file {String} Filename 266 | // #### @callback {Function} Callback for file contents 267 | // Proxy method to private async method read. This method can be called 268 | // synchronously by omitting the callback. 269 | // 270 | Handbook.prototype.get = function get(file, callback) { 271 | return read.apply(this, arguments); 272 | }; 273 | 274 | // 275 | // ### function catalog() 276 | // #### @callback {Function} Callback for catalog/TOC 277 | // Returns a catalog by parsing the content directory, if a callback is provided 278 | // content is returned asynchronously. 279 | // 280 | Handbook.prototype.catalog = function catalog(callback) { 281 | if (!callback) return walkSync(loc); 282 | 283 | walk(loc, callback); 284 | }; 285 | 286 | // 287 | // ### function search() 288 | // #### @query {String} Query to search against 289 | // Proxy to the search method of Lunr, returns a list of documents, this must 290 | // be called in Lunr scope. 291 | // 292 | Handbook.prototype.search = function (query) { 293 | return this.idx.search.call(this.idx, query); 294 | }; 295 | 296 | // 297 | // Expose the 301 routes for the handbook. 298 | // 299 | Handbook.redirect = require('./301.json'); 300 | 301 | // Expose public functions. 302 | module.exports = Handbook; 303 | -------------------------------------------------------------------------------- /CodeClassifier/training-set/python/new 2.txt: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | """ 3 | Sentry 4 | ====== 5 | 6 | Sentry is a realtime event logging and aggregation platform. It specializes 7 | in monitoring errors and extracting all the information needed to do a proper 8 | post-mortem without any of the hassle of the standard user feedback loop. 9 | 10 | Sentry is a Server 11 | ------------------ 12 | 13 | The Sentry package, at its core, is just a simple server and web UI. It will 14 | handle authentication clients (such as `Raven `_) 15 | and all of the logic behind storage and aggregation. 16 | 17 | That said, Sentry is not limited to Python. The primary implementation is in 18 | Python, but it contains a full API for sending events from any language, in 19 | any application. 20 | 21 | :copyright: (c) 2011-2014 by the Sentry Team, see AUTHORS for more details. 22 | :license: BSD, see LICENSE for more details. 23 | """ 24 | from __future__ import absolute_import 25 | 26 | import datetime 27 | import json 28 | import os.path 29 | 30 | from distutils import log 31 | from distutils.core import Command 32 | from setuptools.command.install import install 33 | from setuptools.command.develop import develop 34 | from setuptools.command.sdist import sdist 35 | from setuptools import setup, find_packages 36 | from subprocess import check_output 37 | 38 | 39 | # Hack to prevent stupid "TypeError: 'NoneType' object is not callable" error 40 | # in multiprocessing/util.py _exit_function when running `python 41 | # setup.py test` (see 42 | # http://www.eby-sarna.com/pipermail/peak/2010-May/003357.html) 43 | for m in ('multiprocessing', 'billiard'): 44 | try: 45 | __import__(m) 46 | except ImportError: 47 | pass 48 | 49 | ROOT = os.path.realpath(os.path.join(os.path.dirname(__file__))) 50 | 51 | dev_requires = [ 52 | 'flake8>=2.0,<2.1', 53 | ] 54 | 55 | tests_require = [ 56 | 'blist', # used by cassandra 57 | 'casscache', 58 | 'cqlsh', 59 | 'elasticsearch', 60 | 'httpretty', 61 | 'pytest-cov>=1.4', 62 | 'pytest-timeout', 63 | 'python-coveralls', 64 | 'responses', 65 | 'riak', 66 | ] 67 | 68 | 69 | install_requires = [ 70 | 'BeautifulSoup>=3.2.1,<3.3.0', 71 | 'celery>=3.1.8,<3.2.0', 72 | 'cssutils>=0.9.9,<0.10.0', 73 | 'Django>=1.6.0,<1.7', 74 | 'django-bitfield>=1.7.0,<1.8.0', 75 | 'django-crispy-forms>=1.4.0,<1.5.0', 76 | 'django-paging>=0.2.5,<0.3.0', 77 | 'django-jsonfield>=0.9.13,<0.10.0', 78 | 'django-picklefield>=0.3.0,<0.4.0', 79 | 'django-recaptcha>=1.0.0,<1.1.0', 80 | 'django-social-auth>=0.7.28,<0.8.0', 81 | 'django-statsd-mozilla>=0.3.14.0,<0.3.15.0', 82 | 'django-sudo>=1.1.3,<1.2.0', 83 | 'django-templatetag-sugar>=0.1.0', 84 | 'djangorestframework>=2.3.8,<2.4.0', 85 | 'email-reply-parser>=0.2.0,<0.3.0', 86 | 'enum34>=0.9.18,<0.10.0', 87 | 'exam>=0.5.1', 88 | 'gunicorn>=19.2.1,<20.0.0', 89 | 'ipaddr>=2.1.11,<2.2.0', 90 | 'logan>=0.7.1,<0.8.0', 91 | 'lxml>=3.4.1', 92 | 'mock>=0.8.0', 93 | 'nydus>=0.11.0,<0.12.0', 94 | 'markdown>=2.4.1,<2.5.0', 95 | 'petname>=1.7,<1.8', 96 | 'progressbar>=2.2,<2.4', 97 | 'pytest', 98 | 'pytest-django', 99 | 'python-dateutil>=2.0.0,<3.0.0', 100 | 'python-memcached>=1.53,<2.0.0', 101 | 'raven>=5.3.0', 102 | 'redis>=2.7.0,<2.11.0', 103 | 'requests[security]>=2.5.1,<2.6.0', 104 | 'simplejson>=3.1.0,<3.4.0', 105 | 'six>=1.6.0,<2.0.0', 106 | 'setproctitle>=1.1.7,<1.2.0', 107 | 'statsd>=3.1.0,<3.2.0', 108 | 'South==1.0.1', 109 | 'toronado>=0.0.4,<0.1.0', 110 | 'ua-parser>=0.3.5', 111 | 'urllib3>=1.7.1,<1.8.0', 112 | ] 113 | 114 | postgres_requires = [ 115 | 'psycopg2>=2.5.0,<2.6.0', 116 | ] 117 | 118 | postgres_pypy_requires = [ 119 | 'psycopg2cffi', 120 | ] 121 | 122 | mysql_requires = [ 123 | 'MySQL-python>=1.2.0,<1.3.0', 124 | ] 125 | 126 | 127 | class DevelopWithBuildStatic(develop): 128 | def install_for_development(self): 129 | self.run_command('build_static') 130 | return develop.install_for_development(self) 131 | 132 | 133 | class SdistWithBuildStatic(sdist): 134 | def make_release_tree(self, *a, **kw): 135 | dist_path = self.distribution.get_fullname() 136 | 137 | sdist.make_release_tree(self, *a, **kw) 138 | 139 | self.reinitialize_command('build_static', work_path=dist_path) 140 | self.run_command('build_static') 141 | 142 | with open(os.path.join(dist_path, 'sentry-package.json'), 'w') as fp: 143 | json.dump({ 144 | 'createdAt': datetime.datetime.utcnow().isoformat() + 'Z', 145 | }, fp) 146 | 147 | 148 | class BuildStatic(Command): 149 | user_options = [ 150 | ('work-path=', 'w', 151 | "The working directory for source files. Defaults to ."), 152 | ] 153 | 154 | def initialize_options(self): 155 | self.work_path = None 156 | 157 | def finalize_options(self): 158 | if self.work_path is None: 159 | self.work_path = ROOT 160 | 161 | def run(self): 162 | work_path = self.work_path 163 | 164 | log.info("initializing git submodules") 165 | check_output(['git', 'submodule', 'init'], cwd=work_path) 166 | check_output(['git', 'submodule', 'update'], cwd=work_path) 167 | 168 | log.info("running [npm install --quiet]") 169 | check_output(['npm', 'install', '--quiet'], cwd=work_path) 170 | 171 | log.info("running [gulp dist]") 172 | check_output([os.path.join('node_modules', '.bin', 'gulp'), 'dist'], 173 | cwd=work_path) 174 | 175 | 176 | class SmartInstall(install): 177 | """ 178 | Installs Sentry into the Python environment. 179 | 180 | If the package indicator is missing, this will also force a run of 181 | `build_static` which is required for JavaScript assets and other things. 182 | """ 183 | def _needs_static(self): 184 | return not os.path.exists(os.path.join(ROOT, 'sentry-package.json')) 185 | 186 | def run(self): 187 | if self._needs_static(): 188 | self.run_command('build_static') 189 | install.run(self) 190 | 191 | 192 | setup( 193 | name='sentry', 194 | version='7.7.0.dev0', 195 | author='David Cramer', 196 | author_email='dcramer@gmail.com', 197 | url='https://www.getsentry.com', 198 | description='A realtime logging and aggregation server.', 199 | long_description=open('README.rst').read(), 200 | package_dir={'': 'src'}, 201 | packages=find_packages('src'), 202 | zip_safe=False, 203 | install_requires=install_requires, 204 | extras_require={ 205 | 'tests': tests_require, 206 | 'dev': dev_requires, 207 | 'postgres': install_requires + postgres_requires, 208 | 'postgres_pypy': install_requires + postgres_pypy_requires, 209 | 'mysql': install_requires + mysql_requires, 210 | }, 211 | cmdclass={ 212 | 'build_static': BuildStatic, 213 | 'develop': DevelopWithBuildStatic, 214 | 'sdist': SdistWithBuildStatic, 215 | 'install': SmartInstall, 216 | }, 217 | license='BSD', 218 | include_package_data=True, 219 | entry_points={ 220 | 'console_scripts': [ 221 | 'sentry = sentry.utils.runner:main', 222 | ], 223 | }, 224 | classifiers=[ 225 | 'Framework :: Django', 226 | 'Intended Audience :: Developers', 227 | 'Intended Audience :: System Administrators', 228 | 'Operating System :: OS Independent', 229 | 'Topic :: Software Development' 230 | ], 231 | ) 232 | -------------------------------------------------------------------------------- /CodeClassifier/training-set/python/new 4.txt: -------------------------------------------------------------------------------- 1 | """Stupid tests that ensure logging works as expected""" 2 | from __future__ import (division, absolute_import, print_function, 3 | unicode_literals) 4 | 5 | import sys 6 | import threading 7 | import logging as log 8 | from StringIO import StringIO 9 | 10 | import beets.logging as blog 11 | from beets import plugins, ui 12 | import beetsplug 13 | from test._common import unittest, TestCase 14 | from test import helper 15 | 16 | 17 | class LoggingTest(TestCase): 18 | def test_logging_management(self): 19 | l1 = log.getLogger("foo123") 20 | l2 = blog.getLogger("foo123") 21 | self.assertEqual(l1, l2) 22 | self.assertEqual(l1.__class__, log.Logger) 23 | 24 | l3 = blog.getLogger("bar123") 25 | l4 = log.getLogger("bar123") 26 | self.assertEqual(l3, l4) 27 | self.assertEqual(l3.__class__, blog.BeetsLogger) 28 | self.assertIsInstance(l3, (blog.StrFormatLogger, 29 | blog.ThreadLocalLevelLogger)) 30 | 31 | l5 = l3.getChild("shalala") 32 | self.assertEqual(l5.__class__, blog.BeetsLogger) 33 | 34 | def test_str_format_logging(self): 35 | l = blog.getLogger("baz123") 36 | stream = StringIO() 37 | handler = log.StreamHandler(stream) 38 | 39 | l.addHandler(handler) 40 | l.propagate = False 41 | 42 | l.warning("foo {0} {bar}", "oof", bar="baz") 43 | handler.flush() 44 | self.assertTrue(stream.getvalue(), "foo oof baz") 45 | 46 | 47 | class LoggingLevelTest(unittest.TestCase, helper.TestHelper): 48 | class DummyModule(object): 49 | class DummyPlugin(plugins.BeetsPlugin): 50 | def __init__(self): 51 | plugins.BeetsPlugin.__init__(self, 'dummy') 52 | self.import_stages = [self.import_stage] 53 | self.register_listener('dummy_event', self.listener) 54 | 55 | def log_all(self, name): 56 | self._log.debug('debug ' + name) 57 | self._log.info('info ' + name) 58 | self._log.warning('warning ' + name) 59 | 60 | def commands(self): 61 | cmd = ui.Subcommand('dummy') 62 | cmd.func = lambda _, __, ___: self.log_all('cmd') 63 | return (cmd,) 64 | 65 | def import_stage(self, session, task): 66 | self.log_all('import_stage') 67 | 68 | def listener(self): 69 | self.log_all('listener') 70 | 71 | def setUp(self): 72 | sys.modules['beetsplug.dummy'] = self.DummyModule 73 | beetsplug.dummy = self.DummyModule 74 | self.setup_beets() 75 | self.load_plugins('dummy') 76 | 77 | def tearDown(self): 78 | self.unload_plugins() 79 | self.teardown_beets() 80 | del beetsplug.dummy 81 | sys.modules.pop('beetsplug.dummy') 82 | self.DummyModule.DummyPlugin.listeners = None 83 | self.DummyModule.DummyPlugin._raw_listeners = None 84 | 85 | def test_command_level0(self): 86 | self.config['verbose'] = 0 87 | with helper.capture_log() as logs: 88 | self.run_command('dummy') 89 | self.assertIn('dummy: warning cmd', logs) 90 | self.assertIn('dummy: info cmd', logs) 91 | self.assertNotIn('dummy: debug cmd', logs) 92 | 93 | def test_command_level1(self): 94 | self.config['verbose'] = 1 95 | with helper.capture_log() as logs: 96 | self.run_command('dummy') 97 | self.assertIn('dummy: warning cmd', logs) 98 | self.assertIn('dummy: info cmd', logs) 99 | self.assertIn('dummy: debug cmd', logs) 100 | 101 | def test_command_level2(self): 102 | self.config['verbose'] = 2 103 | with helper.capture_log() as logs: 104 | self.run_command('dummy') 105 | self.assertIn('dummy: warning cmd', logs) 106 | self.assertIn('dummy: info cmd', logs) 107 | self.assertIn('dummy: debug cmd', logs) 108 | 109 | def test_listener_level0(self): 110 | self.config['verbose'] = 0 111 | with helper.capture_log() as logs: 112 | plugins.send('dummy_event') 113 | self.assertIn('dummy: warning listener', logs) 114 | self.assertNotIn('dummy: info listener', logs) 115 | self.assertNotIn('dummy: debug listener', logs) 116 | 117 | def test_listener_level1(self): 118 | self.config['verbose'] = 1 119 | with helper.capture_log() as logs: 120 | plugins.send('dummy_event') 121 | self.assertIn('dummy: warning listener', logs) 122 | self.assertIn('dummy: info listener', logs) 123 | self.assertNotIn('dummy: debug listener', logs) 124 | 125 | def test_listener_level2(self): 126 | self.config['verbose'] = 2 127 | with helper.capture_log() as logs: 128 | plugins.send('dummy_event') 129 | self.assertIn('dummy: warning listener', logs) 130 | self.assertIn('dummy: info listener', logs) 131 | self.assertIn('dummy: debug listener', logs) 132 | 133 | def test_import_stage_level0(self): 134 | self.config['verbose'] = 0 135 | with helper.capture_log() as logs: 136 | importer = self.create_importer() 137 | importer.run() 138 | self.assertIn('dummy: warning import_stage', logs) 139 | self.assertNotIn('dummy: info import_stage', logs) 140 | self.assertNotIn('dummy: debug import_stage', logs) 141 | 142 | def test_import_stage_level1(self): 143 | self.config['verbose'] = 1 144 | with helper.capture_log() as logs: 145 | importer = self.create_importer() 146 | importer.run() 147 | self.assertIn('dummy: warning import_stage', logs) 148 | self.assertIn('dummy: info import_stage', logs) 149 | self.assertNotIn('dummy: debug import_stage', logs) 150 | 151 | def test_import_stage_level2(self): 152 | self.config['verbose'] = 2 153 | with helper.capture_log() as logs: 154 | importer = self.create_importer() 155 | importer.run() 156 | self.assertIn('dummy: warning import_stage', logs) 157 | self.assertIn('dummy: info import_stage', logs) 158 | self.assertIn('dummy: debug import_stage', logs) 159 | 160 | 161 | class ConcurrentEventsTest(TestCase, helper.TestHelper): 162 | """Similar to LoggingLevelTest but lower-level and focused on multiple 163 | events interaction. Since this is a bit heavy we don't do it in 164 | LoggingLevelTest. 165 | """ 166 | class DummyPlugin(plugins.BeetsPlugin): 167 | def __init__(self, test_case): 168 | plugins.BeetsPlugin.__init__(self, 'dummy') 169 | self.register_listener('dummy_event1', self.listener1) 170 | self.register_listener('dummy_event2', self.listener2) 171 | self.lock1 = threading.Lock() 172 | self.lock2 = threading.Lock() 173 | self.test_case = test_case 174 | self.exc_info = None 175 | self.t1_step = self.t2_step = 0 176 | 177 | def log_all(self, name): 178 | self._log.debug('debug ' + name) 179 | self._log.info('info ' + name) 180 | self._log.warning('warning ' + name) 181 | 182 | def listener1(self): 183 | try: 184 | self.test_case.assertEqual(self._log.level, log.INFO) 185 | self.t1_step = 1 186 | self.lock1.acquire() 187 | self.test_case.assertEqual(self._log.level, log.INFO) 188 | self.t1_step = 2 189 | except Exception: 190 | import sys 191 | self.exc_info = sys.exc_info() 192 | 193 | def listener2(self): 194 | try: 195 | self.test_case.assertEqual(self._log.level, log.DEBUG) 196 | self.t2_step = 1 197 | self.lock2.acquire() 198 | self.test_case.assertEqual(self._log.level, log.DEBUG) 199 | self.t2_step = 2 200 | except Exception: 201 | import sys 202 | self.exc_info = sys.exc_info() 203 | 204 | def setUp(self): 205 | self.setup_beets(disk=True) 206 | 207 | def tearDown(self): 208 | self.teardown_beets() 209 | 210 | def test_concurrent_events(self): 211 | dp = self.DummyPlugin(self) 212 | 213 | def check_dp_exc(): 214 | if dp.exc_info: 215 | raise dp.exc_info[1], None, dp.exc_info[2] 216 | 217 | try: 218 | dp.lock1.acquire() 219 | dp.lock2.acquire() 220 | self.assertEqual(dp._log.level, log.NOTSET) 221 | 222 | self.config['verbose'] = 1 223 | t1 = threading.Thread(target=dp.listeners['dummy_event1'][0]) 224 | t1.start() # blocked. t1 tested its log level 225 | while dp.t1_step != 1: 226 | check_dp_exc() 227 | self.assertTrue(t1.is_alive()) 228 | self.assertEqual(dp._log.level, log.NOTSET) 229 | 230 | self.config['verbose'] = 2 231 | t2 = threading.Thread(target=dp.listeners['dummy_event2'][0]) 232 | t2.start() # blocked. t2 tested its log level 233 | while dp.t2_step != 1: 234 | check_dp_exc() 235 | self.assertTrue(t2.is_alive()) 236 | self.assertEqual(dp._log.level, log.NOTSET) 237 | 238 | dp.lock1.release() # dummy_event1 tests its log level + finishes 239 | while dp.t1_step != 2: 240 | check_dp_exc() 241 | t1.join(.1) 242 | self.assertFalse(t1.is_alive()) 243 | self.assertTrue(t2.is_alive()) 244 | self.assertEqual(dp._log.level, log.NOTSET) 245 | 246 | dp.lock2.release() # dummy_event2 tests its log level + finishes 247 | while dp.t2_step != 2: 248 | check_dp_exc() 249 | t2.join(.1) 250 | self.assertFalse(t2.is_alive()) 251 | 252 | except: 253 | print("Alive threads:", threading.enumerate()) 254 | if dp.lock1.locked(): 255 | print("Releasing lock1 after exception in test") 256 | dp.lock1.release() 257 | if dp.lock2.locked(): 258 | print("Releasing lock2 after exception in test") 259 | dp.lock2.release() 260 | print("Alive threads:", threading.enumerate()) 261 | raise 262 | 263 | def test_root_logger_levels(self): 264 | """Root logger level should be shared between threads. 265 | """ 266 | self.config['threaded'] = True 267 | 268 | blog.getLogger('beets').set_global_level(blog.WARNING) 269 | with helper.capture_log() as logs: 270 | importer = self.create_importer() 271 | importer.run() 272 | self.assertEqual(logs, []) 273 | 274 | blog.getLogger('beets').set_global_level(blog.INFO) 275 | with helper.capture_log() as logs: 276 | importer = self.create_importer() 277 | importer.run() 278 | for l in logs: 279 | self.assertIn("import", l) 280 | self.assertIn("album", l) 281 | 282 | blog.getLogger('beets').set_global_level(blog.DEBUG) 283 | with helper.capture_log() as logs: 284 | importer = self.create_importer() 285 | importer.run() 286 | self.assertIn("Sending event: database_change", logs) 287 | 288 | 289 | def suite(): 290 | return unittest.TestLoader().loadTestsFromName(__name__) 291 | 292 | 293 | if __name__ == b'__main__': 294 | unittest.main(defaultTest='suite') 295 | -------------------------------------------------------------------------------- /CodeClassifier/training-set/ruby/artifactory.rb: -------------------------------------------------------------------------------- 1 | class Artifactory < Formula 2 | desc "Manages binaries" 3 | homepage "http://www.jfrog.com/artifactory/" 4 | url "https://dl.bintray.com/jfrog/artifactory/artifactory-3.8.0.zip" 5 | sha1 "ade88a068f58a3847f9591ee0b9bfd0bcbd20049" 6 | 7 | depends_on :java => "1.7+" 8 | 9 | option "with-low-heap", "Run artifactory with low Java memory options. Useful for development machines. Do not use in production." 10 | option "with-java8", "Adjust memory settings for Java 8" 11 | 12 | def install 13 | # Remove Windows binaries 14 | rm_f Dir["bin/*.bat"] 15 | rm_f Dir["bin/*.exe"] 16 | 17 | # Set correct working directory 18 | inreplace "bin/artifactory.sh", 19 | 'export ARTIFACTORY_HOME="$(cd "$(dirname "${artBinDir}")" && pwd)"', 20 | "export ARTIFACTORY_HOME=#{libexec}" 21 | 22 | # Remove obsolete parameters for Java 8 23 | inreplace "bin/artifactory.default", 24 | "-server -Xms512m -Xmx2g -Xss256k -XX:PermSize=128m -XX:MaxPermSize=256m -XX:+UseG1GC", 25 | "-server -Xms512m -Xmx2g -Xss256k -XX:+UseG1GC" if build.with? "java8" 26 | 27 | # Reduce memory consumption for non production use 28 | inreplace "bin/artifactory.default", 29 | "-server -Xms512m -Xmx2g", 30 | "-Xms128m -Xmx768m" if build.with? "low-heap" 31 | 32 | libexec.install Dir["*"] 33 | 34 | # Launch Script 35 | bin.install_symlink libexec/"bin/artifactory.sh" 36 | # Memory Options 37 | bin.install_symlink libexec/"bin/artifactory.default" 38 | end 39 | 40 | def post_install 41 | # Create persistent data directory. Artifactory heavily relies on the data 42 | # directory being directly under ARTIFACTORY_HOME. 43 | # Therefore, we symlink the data dir to var. 44 | data = var/"artifactory" 45 | data.mkpath 46 | 47 | libexec.install_symlink data => "data" 48 | end 49 | 50 | plist_options :manual => "#{HOMEBREW_PREFIX}/opt/artifactory/libexec/bin/artifactory.sh" 51 | 52 | def plist; <<-EOS.undent 53 | 54 | 55 | 56 | 57 | Label 58 | com.jfrog.artifactory 59 | 60 | WorkingDirectory 61 | #{libexec} 62 | 63 | Program 64 | bin/artifactory.sh 65 | 66 | KeepAlive 67 | 68 | 69 | 70 | EOS 71 | end 72 | 73 | test do 74 | output = shell_output("#{bin}/artifactory.sh check 2>&1", 1) 75 | assert output.include?("Checking arguments to Artifactory") 76 | end 77 | end 78 | -------------------------------------------------------------------------------- /CodeClassifier/training-set/ruby/jekyll_steps.rb: -------------------------------------------------------------------------------- 1 | def file_content_from_hash(input_hash) 2 | matter_hash = input_hash.reject { |k, v| k == "content" } 3 | matter = matter_hash.map { |k, v| "#{k}: #{v}\n" }.join.chomp 4 | 5 | content = if input_hash['input'] && input_hash['filter'] 6 | "{{ #{input_hash['input']} | #{input_hash['filter']} }}" 7 | else 8 | input_hash['content'] 9 | end 10 | 11 | <<-EOF 12 | --- 13 | #{matter} 14 | --- 15 | #{content} 16 | EOF 17 | end 18 | 19 | Before do 20 | FileUtils.mkdir_p(TEST_DIR) unless File.exist?(TEST_DIR) 21 | Dir.chdir(TEST_DIR) 22 | end 23 | 24 | After do 25 | FileUtils.rm_rf(TEST_DIR) if File.exist?(TEST_DIR) 26 | FileUtils.rm(JEKYLL_COMMAND_OUTPUT_FILE) if File.exist?(JEKYLL_COMMAND_OUTPUT_FILE) 27 | Dir.chdir(File.dirname(TEST_DIR)) 28 | end 29 | 30 | World do 31 | MinitestWorld.new 32 | end 33 | 34 | Given /^I have a blank site in "(.*)"$/ do |path| 35 | FileUtils.mkdir_p(path) unless File.exist?(path) 36 | end 37 | 38 | Given /^I do not have a "(.*)" directory$/ do |path| 39 | File.directory?("#{TEST_DIR}/#{path}") 40 | end 41 | 42 | # Like "I have a foo file" but gives a yaml front matter so jekyll actually processes it 43 | Given /^I have an? "(.*)" page(?: with (.*) "(.*)")? that contains "(.*)"$/ do |file, key, value, text| 44 | File.open(file, 'w') do |f| 45 | f.write <<-EOF 46 | --- 47 | #{key || 'layout'}: #{value || 'nil'} 48 | --- 49 | #{text} 50 | EOF 51 | end 52 | end 53 | 54 | Given /^I have an? "(.*)" file that contains "(.*)"$/ do |file, text| 55 | File.open(file, 'w') do |f| 56 | f.write(text) 57 | end 58 | end 59 | 60 | Given /^I have an? (.*) (layout|theme) that contains "(.*)"$/ do |name, type, text| 61 | folder = if type == 'layout' 62 | '_layouts' 63 | else 64 | '_theme' 65 | end 66 | destination_file = File.join(folder, name + '.html') 67 | destination_path = File.dirname(destination_file) 68 | unless File.exist?(destination_path) 69 | FileUtils.mkdir_p(destination_path) 70 | end 71 | File.open(destination_file, 'w') do |f| 72 | f.write(text) 73 | end 74 | end 75 | 76 | Given /^I have an? "(.*)" file with content:$/ do |file, text| 77 | File.open(file, 'w') do |f| 78 | f.write(text) 79 | end 80 | end 81 | 82 | Given /^I have an? (.*) directory$/ do |dir| 83 | FileUtils.mkdir_p(dir) 84 | end 85 | 86 | Given /^I have the following (draft|page|post)s?(?: (in|under) "([^"]+)")?:$/ do |status, direction, folder, table| 87 | table.hashes.each do |input_hash| 88 | title = slug(input_hash['title']) 89 | ext = input_hash['type'] || 'markdown' 90 | before, after = location(folder, direction) 91 | 92 | case status 93 | when "draft" 94 | dest_folder = '_drafts' 95 | filename = "#{title}.#{ext}" 96 | when "page" 97 | dest_folder = '' 98 | filename = "#{title}.#{ext}" 99 | when "post" 100 | parsed_date = Time.xmlschema(input_hash['date']) rescue Time.parse(input_hash['date']) 101 | dest_folder = '_posts' 102 | filename = "#{parsed_date.strftime('%Y-%m-%d')}-#{title}.#{ext}" 103 | end 104 | 105 | path = File.join(before, dest_folder, after, filename) 106 | File.open(path, 'w') do |f| 107 | f.write file_content_from_hash(input_hash) 108 | end 109 | end 110 | end 111 | 112 | Given /^I have a configuration file with "(.*)" set to "(.*)"$/ do |key, value| 113 | File.open('_config.yml', 'w') do |f| 114 | f.write("#{key}: #{value}\n") 115 | end 116 | end 117 | 118 | Given /^I have a configuration file with:$/ do |table| 119 | File.open('_config.yml', 'w') do |f| 120 | table.hashes.each do |row| 121 | f.write("#{row["key"]}: #{row["value"]}\n") 122 | end 123 | end 124 | end 125 | 126 | Given /^I have a configuration file with "([^\"]*)" set to:$/ do |key, table| 127 | File.open('_config.yml', 'w') do |f| 128 | f.write("#{key}:\n") 129 | table.hashes.each do |row| 130 | f.write("- #{row["value"]}\n") 131 | end 132 | end 133 | end 134 | 135 | Given /^I have fixture collections$/ do 136 | FileUtils.cp_r File.join(JEKYLL_SOURCE_DIR, "test", "source", "_methods"), source_dir 137 | end 138 | 139 | Given /^I wait (\d+) second(s?)$/ do |time, plural| 140 | sleep(time.to_f) 141 | end 142 | 143 | ################## 144 | # 145 | # Changing stuff 146 | # 147 | ################## 148 | 149 | When /^I run jekyll(.*)$/ do |args| 150 | status = run_jekyll(args) 151 | if args.include?("--verbose") || ENV['DEBUG'] 152 | puts jekyll_run_output 153 | end 154 | end 155 | 156 | When /^I run bundle(.*)$/ do |args| 157 | status = run_bundle(args) 158 | if args.include?("--verbose") || ENV['DEBUG'] 159 | puts jekyll_run_output 160 | end 161 | end 162 | 163 | When /^I change "(.*)" to contain "(.*)"$/ do |file, text| 164 | File.open(file, 'a') do |f| 165 | f.write(text) 166 | end 167 | end 168 | 169 | When /^I delete the file "(.*)"$/ do |file| 170 | File.delete(file) 171 | end 172 | 173 | ################## 174 | # 175 | # Checking stuff 176 | # 177 | ################## 178 | 179 | Then /^the (.*) directory should +exist$/ do |dir| 180 | assert File.directory?(dir), "The directory \"#{dir}\" does not exist" 181 | end 182 | 183 | Then /^the (.*) directory should not exist$/ do |dir| 184 | assert !File.directory?(dir), "The directory \"#{dir}\" exists" 185 | end 186 | 187 | Then /^I should see "(.*)" in "(.*)"$/ do |text, file| 188 | assert_match Regexp.new(text, Regexp::MULTILINE), file_contents(file) 189 | end 190 | 191 | Then /^I should see exactly "(.*)" in "(.*)"$/ do |text, file| 192 | assert_equal text, file_contents(file).strip 193 | end 194 | 195 | Then /^I should not see "(.*)" in "(.*)"$/ do |text, file| 196 | refute_match Regexp.new(text, Regexp::MULTILINE), file_contents(file) 197 | end 198 | 199 | Then /^I should see escaped "(.*)" in "(.*)"$/ do |text, file| 200 | assert_match Regexp.new(Regexp.escape(text)), file_contents(file) 201 | end 202 | 203 | Then /^the "(.*)" file should +exist$/ do |file| 204 | file_does_exist = File.file?(file) 205 | unless file_does_exist 206 | all_steps_to_path(file).each do |dir| 207 | STDERR.puts "" 208 | STDERR.puts "Dir #{dir}:" 209 | STDERR.puts Dir["#{dir}/**/*"] 210 | end 211 | end 212 | assert file_does_exist, "The file \"#{file}\" does not exist.\n" 213 | end 214 | 215 | Then /^the "(.*)" file should not exist$/ do |file| 216 | assert !File.exist?(file), "The file \"#{file}\" exists" 217 | end 218 | 219 | Then /^I should see today's time in "(.*)"$/ do |file| 220 | assert_match Regexp.new(seconds_agnostic_time(Time.now)), file_contents(file) 221 | end 222 | 223 | Then /^I should see today's date in "(.*)"$/ do |file| 224 | assert_match Regexp.new(Date.today.to_s), file_contents(file) 225 | end 226 | 227 | Then /^I should see "(.*)" in the build output$/ do |text| 228 | assert_match Regexp.new(text), jekyll_run_output 229 | end 230 | -------------------------------------------------------------------------------- /CodeClassifier/training-set/ruby/ruby.rb: -------------------------------------------------------------------------------- 1 | 2 | require 'optparse' 3 | 4 | module Commander 5 | class Runner 6 | 7 | #-- 8 | # Exceptions 9 | #++ 10 | 11 | class CommandError < StandardError; end 12 | class InvalidCommandError < CommandError; end 13 | 14 | ## 15 | # Array of commands. 16 | 17 | attr_reader :commands 18 | 19 | ## 20 | # Global options. 21 | 22 | attr_reader :options 23 | 24 | ## 25 | # Hash of help formatter aliases. 26 | 27 | attr_reader :help_formatter_aliases 28 | 29 | ## 30 | # Initialize a new command runner. Optionally 31 | # supplying _args_ for mocking, or arbitrary usage. 32 | 33 | def initialize args = ARGV 34 | @args, @commands, @aliases, @options = args, {}, {}, [] 35 | @help_formatter_aliases = help_formatter_alias_defaults 36 | @program = program_defaults 37 | create_default_commands 38 | end 39 | 40 | ## 41 | # Return singleton Runner instance. 42 | 43 | def self.instance 44 | @singleton ||= new 45 | end 46 | 47 | ## 48 | # Run command parsing and execution process. 49 | 50 | def run! 51 | trace = false 52 | require_program :version, :description 53 | trap('INT') { abort program(:int_message) } if program(:int_message) 54 | trap('INT') { program(:int_block).call } if program(:int_block) 55 | global_option('-h', '--help', 'Display help documentation') do 56 | args = @args - %w[-h --help] 57 | command(:help).run(*args) 58 | return 59 | end 60 | global_option('-v', '--version', 'Display version information') { say version; return } 61 | global_option('-t', '--trace', 'Display backtrace when an error occurs') { trace = true } 62 | parse_global_options 63 | remove_global_options options, @args 64 | unless trace 65 | begin 66 | run_active_command 67 | rescue InvalidCommandError => e 68 | abort "#{e}. Use --help for more information" 69 | rescue \ 70 | OptionParser::InvalidOption, 71 | OptionParser::InvalidArgument, 72 | OptionParser::MissingArgument => e 73 | abort e.to_s 74 | rescue => e 75 | abort "error: #{e}. Use --trace to view backtrace" 76 | end 77 | else 78 | run_active_command 79 | end 80 | end 81 | 82 | ## 83 | # Return program version. 84 | 85 | def version 86 | '%s %s' % [program(:name), program(:version)] 87 | end 88 | 89 | ## 90 | # Assign program information. 91 | # 92 | # === Examples 93 | # 94 | # # Set data 95 | # program :name, 'Commander' 96 | # program :version, Commander::VERSION 97 | # program :description, 'Commander utility program.' 98 | # program :help, 'Copyright', '2008 TJ Holowaychuk' 99 | # program :help, 'Anything', 'You want' 100 | # program :int_message 'Bye bye!' 101 | # program :help_formatter, :compact 102 | # program :help_formatter, Commander::HelpFormatter::TerminalCompact 103 | # 104 | # # Get data 105 | # program :name # => 'Commander' 106 | # 107 | # === Keys 108 | # 109 | # :version (required) Program version triple, ex: '0.0.1' 110 | # :description (required) Program description 111 | # :name Program name, defaults to basename of executable 112 | # :help_formatter Defaults to Commander::HelpFormatter::Terminal 113 | # :help Allows addition of arbitrary global help blocks 114 | # :int_message Message to display when interrupted (CTRL + C) 115 | # 116 | 117 | def program key, *args, &block 118 | if key == :help and !args.empty? 119 | @program[:help] ||= {} 120 | @program[:help][args.first] = args.at(1) 121 | elsif key == :help_formatter && !args.empty? 122 | @program[key] = (@help_formatter_aliases[args.first] || args.first) 123 | elsif block 124 | @program[key] = block 125 | else 126 | unless args.empty? 127 | @program[key] = (args.count == 1 && args[0]) || args 128 | end 129 | @program[key] 130 | end 131 | end 132 | 133 | ## 134 | # Creates and yields a command instance when a block is passed. 135 | # Otherwise attempts to return the command, raising InvalidCommandError when 136 | # it does not exist. 137 | # 138 | # === Examples 139 | # 140 | # command :my_command do |c| 141 | # c.when_called do |args| 142 | # # Code 143 | # end 144 | # end 145 | # 146 | 147 | def command name, &block 148 | yield add_command(Commander::Command.new(name)) if block 149 | @commands[name.to_s] 150 | end 151 | 152 | ## 153 | # Add a global option; follows the same syntax as Command#option 154 | # This would be used for switches such as --version, --trace, etc. 155 | 156 | def global_option *args, &block 157 | switches, description = Runner.separate_switches_from_description *args 158 | @options << { 159 | :args => args, 160 | :proc => block, 161 | :switches => switches, 162 | :description => description, 163 | } 164 | end 165 | 166 | ## 167 | # Alias command _name_ with _alias_name_. Optionally _args_ may be passed 168 | # as if they were being passed straight to the original command via the command-line. 169 | 170 | def alias_command alias_name, name, *args 171 | @commands[alias_name.to_s] = command name 172 | @aliases[alias_name.to_s] = args 173 | end 174 | 175 | ## 176 | # Default command _name_ to be used when no other 177 | # command is found in the arguments. 178 | 179 | def default_command name 180 | @default_command = name 181 | end 182 | 183 | ## 184 | # Add a command object to this runner. 185 | 186 | def add_command command 187 | @commands[command.name] = command 188 | end 189 | 190 | ## 191 | # Check if command _name_ is an alias. 192 | 193 | def alias? name 194 | @aliases.include? name.to_s 195 | end 196 | 197 | ## 198 | # Check if a command _name_ exists. 199 | 200 | def command_exists? name 201 | @commands[name.to_s] 202 | end 203 | 204 | #:stopdoc: 205 | 206 | ## 207 | # Get active command within arguments passed to this runner. 208 | 209 | def active_command 210 | @__active_command ||= command(command_name_from_args) 211 | end 212 | 213 | ## 214 | # Attempts to locate a command name from within the arguments. 215 | # Supports multi-word commands, using the largest possible match. 216 | 217 | def command_name_from_args 218 | @__command_name_from_args ||= (valid_command_names_from(*@args.dup).sort.last || @default_command) 219 | end 220 | 221 | ## 222 | # Returns array of valid command names found within _args_. 223 | 224 | def valid_command_names_from *args 225 | arg_string = args.delete_if { |value| value =~ /^-/ }.join ' ' 226 | commands.keys.find_all { |name| name if /^#{name}\b/.match arg_string } 227 | end 228 | 229 | ## 230 | # Help formatter instance. 231 | 232 | def help_formatter 233 | @__help_formatter ||= program(:help_formatter).new self 234 | end 235 | 236 | ## 237 | # Return arguments without the command name. 238 | 239 | def args_without_command_name 240 | removed = [] 241 | parts = command_name_from_args.split rescue [] 242 | @args.dup.delete_if do |arg| 243 | removed << arg if parts.include?(arg) and not removed.include?(arg) 244 | end 245 | end 246 | 247 | ## 248 | # Returns hash of help formatter alias defaults. 249 | 250 | def help_formatter_alias_defaults 251 | return :compact => HelpFormatter::TerminalCompact 252 | end 253 | 254 | ## 255 | # Returns hash of program defaults. 256 | 257 | def program_defaults 258 | return :help_formatter => HelpFormatter::Terminal, 259 | :name => File.basename($0) 260 | end 261 | 262 | ## 263 | # Creates default commands such as 'help' which is 264 | # essentially the same as using the --help switch. 265 | 266 | def create_default_commands 267 | command :help do |c| 268 | c.syntax = 'commander help [command]' 269 | c.description = 'Display global or [command] help documentation.' 270 | c.example 'Display global help', 'command help' 271 | c.example "Display help for 'foo'", 'command help foo' 272 | c.when_called do |args, options| 273 | enable_paging 274 | if args.empty? 275 | say help_formatter.render 276 | else 277 | command = command args.join(' ') 278 | begin 279 | require_valid_command command 280 | rescue InvalidCommandError => e 281 | abort "#{e}. Use --help for more information" 282 | end 283 | say help_formatter.render_command(command) 284 | end 285 | end 286 | end 287 | end 288 | 289 | ## 290 | # Raises InvalidCommandError when a _command_ is not found. 291 | 292 | def require_valid_command command = active_command 293 | raise InvalidCommandError, 'invalid command', caller if command.nil? 294 | end 295 | 296 | ## 297 | # Removes global _options_ from _args_. This prevents an invalid 298 | # option error from occurring when options are parsed 299 | # again for the command. 300 | 301 | def remove_global_options options, args 302 | # TODO: refactor with flipflop, please TJ ! have time to refactor me ! 303 | options.each do |option| 304 | switches = option[:switches].dup 305 | next if switches.empty? 306 | 307 | if switchHasArg = switches.any? { |s| s =~ /[ =]/ } 308 | switches.map! { |s| s[0, s.index('=') || s.index(' ') || s.length] } 309 | end 310 | 311 | past_switch, arg_removed = false, false 312 | args.delete_if do |arg| 313 | if switches.any? { |s| arg[0, s.length] == s } 314 | arg_removed = !switchHasArg 315 | past_switch = true 316 | elsif past_switch && !arg_removed && arg !~ /^-/ 317 | arg_removed = true 318 | else 319 | arg_removed = true 320 | false 321 | end 322 | end 323 | end 324 | end 325 | 326 | ## 327 | # Parse global command options. 328 | 329 | def parse_global_options 330 | 331 | parser = options.inject(OptionParser.new) do |options, option| 332 | options.on *option[:args], &global_option_proc(option[:switches], &option[:proc]) 333 | end 334 | 335 | options = @args.dup 336 | begin 337 | parser.parse!(options) 338 | rescue OptionParser::InvalidOption => e 339 | # Remove the offending args and retry. 340 | options = options.reject { |o| e.args.include?(o) } 341 | retry 342 | end 343 | end 344 | 345 | ## 346 | # Returns a proc allowing for commands to inherit global options. 347 | # This functionality works whether a block is present for the global 348 | # option or not, so simple switches such as --verbose can be used 349 | # without a block, and used throughout all commands. 350 | 351 | def global_option_proc switches, &block 352 | lambda do |value| 353 | unless active_command.nil? 354 | active_command.proxy_options << [Runner.switch_to_sym(switches.last), value] 355 | end 356 | yield value if block and !value.nil? 357 | end 358 | end 359 | 360 | ## 361 | # Raises a CommandError when the program any of the _keys_ are not present, or empty. 362 | 363 | def require_program *keys 364 | keys.each do |key| 365 | raise CommandError, "program #{key} required" if program(key).nil? or program(key).empty? 366 | end 367 | end 368 | 369 | ## 370 | # Return switches and description separated from the _args_ passed. 371 | 372 | def self.separate_switches_from_description *args 373 | switches = args.find_all { |arg| arg.to_s =~ /^-/ } 374 | description = args.last unless !args.last.is_a? String or args.last.match(/^-/) 375 | return switches, description 376 | end 377 | 378 | ## 379 | # Attempts to generate a method name symbol from +switch+. 380 | # For example: 381 | # 382 | # -h # => :h 383 | # --trace # => :trace 384 | # --some-switch # => :some_switch 385 | # --[no-]feature # => :feature 386 | # --file FILE # => :file 387 | # --list of,things # => :list 388 | # 389 | 390 | def self.switch_to_sym switch 391 | switch.scan(/[\-\]](\w+)/).join('_').to_sym rescue nil 392 | end 393 | 394 | ## 395 | # Run the active command. 396 | 397 | def run_active_command 398 | require_valid_command 399 | if alias? command_name_from_args 400 | active_command.run *(@aliases[command_name_from_args.to_s] + args_without_command_name) 401 | else 402 | active_command.run *args_without_command_name 403 | end 404 | end 405 | 406 | def say *args #:nodoc: 407 | $terminal.say *args 408 | end 409 | 410 | end 411 | end 412 | -------------------------------------------------------------------------------- /CodeClassifier/training-set/ruby/travis.rb: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | require 'fileutils' 3 | include FileUtils 4 | 5 | commands = [ 6 | 'mysql -e "create database activerecord_unittest;"', 7 | 'mysql -e "create database activerecord_unittest2;"', 8 | 'psql -c "create database activerecord_unittest;" -U postgres', 9 | 'psql -c "create database activerecord_unittest2;" -U postgres' 10 | ] 11 | 12 | commands.each do |command| 13 | system("#{command} > /dev/null 2>&1") 14 | end 15 | 16 | class Build 17 | MAP = { 18 | 'railties' => 'railties', 19 | 'ap' => 'actionpack', 20 | 'am' => 'actionmailer', 21 | 'amo' => 'activemodel', 22 | 'as' => 'activesupport', 23 | 'ar' => 'activerecord', 24 | 'av' => 'actionview', 25 | 'aj' => 'activejob', 26 | 'guides' => 'guides' 27 | } 28 | 29 | attr_reader :component, :options 30 | 31 | def initialize(component, options = {}) 32 | @component = component 33 | @options = options 34 | end 35 | 36 | def run!(options = {}) 37 | self.options.update(options) 38 | Dir.chdir(dir) do 39 | announce(heading) 40 | if guides? 41 | run_bug_report_templates 42 | else 43 | rake(*tasks) 44 | end 45 | end 46 | end 47 | 48 | def announce(heading) 49 | puts "\n\e[1;33m[Travis CI] #{heading}\e[m\n" 50 | end 51 | 52 | def heading 53 | heading = [gem] 54 | heading << "with #{adapter}" if activerecord? 55 | heading << "in isolation" if isolated? 56 | heading << "integration" if integration? 57 | heading.join(' ') 58 | end 59 | 60 | def tasks 61 | if activerecord? 62 | ['db:mysql:rebuild', "#{adapter}:#{'isolated_' if isolated?}test"] 63 | else 64 | ["test", ('isolated' if isolated?), ('integration' if integration?)].compact.join(":") 65 | end 66 | end 67 | 68 | def key 69 | key = [gem] 70 | key << adapter if activerecord? 71 | key << 'isolated' if isolated? 72 | key.join(':') 73 | end 74 | 75 | def activerecord? 76 | gem == 'activerecord' 77 | end 78 | 79 | def guides? 80 | gem == 'guides' 81 | end 82 | 83 | def isolated? 84 | options[:isolated] 85 | end 86 | 87 | def integration? 88 | component.split(':').last == 'integration' 89 | end 90 | 91 | def gem 92 | MAP[component.split(':').first] 93 | end 94 | alias :dir :gem 95 | 96 | def adapter 97 | component.split(':').last 98 | end 99 | 100 | def rake(*tasks) 101 | tasks.each do |task| 102 | cmd = "bundle exec rake #{task}" 103 | puts "Running command: #{cmd}" 104 | return false unless system(cmd) 105 | end 106 | true 107 | end 108 | 109 | def run_bug_report_templates 110 | Dir.glob('bug_report_templates/*.rb').all? do |file| 111 | system(Gem.ruby, '-w', file) 112 | end 113 | end 114 | end 115 | 116 | if ENV['GEM']=='aj:integration' 117 | ENV['QC_DATABASE_URL'] = 'postgres://postgres@localhost/active_jobs_qc_int_test' 118 | ENV['QUE_DATABASE_URL'] = 'postgres://postgres@localhost/active_jobs_que_int_test' 119 | end 120 | 121 | results = {} 122 | 123 | ENV['GEM'].split(',').each do |gem| 124 | [false, true].each do |isolated| 125 | next if ENV['TRAVIS_PULL_REQUEST'] && ENV['TRAVIS_PULL_REQUEST'] != 'false' && isolated 126 | next if gem == 'railties' && isolated 127 | next if gem == 'aj:integration' && isolated 128 | next if gem == 'guides' && isolated 129 | 130 | build = Build.new(gem, :isolated => isolated) 131 | results[build.key] = build.run! 132 | 133 | end 134 | end 135 | 136 | # puts 137 | # puts "Build environment:" 138 | # puts " #{`cat /etc/issue`}" 139 | # puts " #{`uname -a`}" 140 | # puts " #{`ruby -v`}" 141 | # puts " #{`mysql --version`}" 142 | # puts " #{`pg_config --version`}" 143 | # puts " SQLite3: #{`sqlite3 -version`}" 144 | # `gem env`.each_line {|line| print " #{line}"} 145 | # puts " Bundled gems:" 146 | # `bundle show`.each_line {|line| print " #{line}"} 147 | # puts " Local gems:" 148 | # `gem list`.each_line {|line| print " #{line}"} 149 | 150 | failures = results.select { |key, value| !value } 151 | 152 | if failures.empty? 153 | puts 154 | puts "Rails build finished successfully" 155 | exit(true) 156 | else 157 | puts 158 | puts "Rails build FAILED" 159 | puts "Failed components: #{failures.map(&:first).join(', ')}" 160 | exit(false) 161 | end 162 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2015 Bert Verhelst 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | 23 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # CodeClassifier 2 | You provide a snippet and this c# library will tell you what language it was written in. 3 | 4 | ## Supported languages: 5 | * ruby 6 | * shell 7 | * c 8 | * c++ 9 | * coffee-script 10 | * csharp 11 | * css 12 | * html 13 | * javascript 14 | * objective-c 15 | * python 16 | 17 | You can add your own languages by adding a file to the training-set folder. The name of the file has to be the language name. The file has to contain a good representation of the programming language syntax. Look at the other files to get an idea of what that means. 18 | 19 | ## Technology 20 | Uses .NET 4.0 C# 21 | 22 | 23 | ## Inner workings 24 | The way it calculates the best matching language is by having a number of training files. It reads those, splits the code up in tokens and then generates a tree where every subnode is a possible token to follow the current one. 25 | 26 | Then when you want to classify some code, it also tokenizes that code and checks all the match trees that it build in the previous step. The bigger the parts of your code that match with nodes in the match tree, the higher your code scores for a specific language. The language with the highest score will be returned. 27 | -------------------------------------------------------------------------------- /Test/App.config: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | 5 | 6 | -------------------------------------------------------------------------------- /Test/App.xaml: -------------------------------------------------------------------------------- 1 |  5 | 6 | 7 | 8 | 9 | -------------------------------------------------------------------------------- /Test/App.xaml.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Configuration; 4 | using System.Data; 5 | using System.Linq; 6 | using System.Threading.Tasks; 7 | using System.Windows; 8 | 9 | namespace Test 10 | { 11 | /// 12 | /// Interaction logic for App.xaml 13 | /// 14 | public partial class App : Application 15 | { 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /Test/MainWindow.xaml: -------------------------------------------------------------------------------- 1 |  10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 |