├── .editorconfig ├── .gitignore ├── .gitmodules ├── .travis.yml ├── LICENSE ├── README.md ├── appveyor.yml ├── dub.json ├── makefile ├── src └── dfix.d └── test ├── testfile_expected.d └── testfile_master.d /.editorconfig: -------------------------------------------------------------------------------- 1 | [*] 2 | end_of_line = lf 3 | insert_final_newline = true 4 | indent_size = 4 5 | tab_width = 4 6 | trim_trailing_whitespace = true 7 | indent_style = tab 8 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .dub 2 | docs.json 3 | __dummy.html 4 | *.o 5 | *.obj 6 | bin 7 | test/testfile.d 8 | dfix 9 | dub.selections.json 10 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "libdparse"] 2 | path = libdparse 3 | url = https://github.com/Hackerpilot/libdparse.git 4 | [submodule "stdx-allocator"] 5 | path = stdx-allocator 6 | url = https://github.com/dlang-community/stdx-allocator 7 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | sudo: false 2 | language: d 3 | d: 4 | #- dmd-nightly 5 | #- dmd-beta 6 | - dmd 7 | #- ldc-beta 8 | - ldc 9 | script: 10 | - git submodule update --init --recursive && make 11 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Boost Software License - Version 1.0 - August 17th, 2003 2 | 3 | Permission is hereby granted, free of charge, to any person or organization 4 | obtaining a copy of the software and accompanying documentation covered by 5 | this license (the "Software") to use, reproduce, display, distribute, 6 | execute, and transmit the Software, and to prepare derivative works of the 7 | Software, and to permit third-parties to whom the Software is furnished to 8 | do so, all subject to the following: 9 | 10 | The copyright notices in the Software and this entire statement, including 11 | the above license grant, this restriction and the following disclaimer, 12 | must be included in all copies of the Software, in whole or in part, and 13 | all derivative works of the Software, unless such copies or derivative 14 | works are solely in the form of machine-executable object code generated by 15 | a source language processor. 16 | 17 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 18 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 19 | FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT 20 | SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE 21 | FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, 22 | ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 23 | DEALINGS IN THE SOFTWARE. 24 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # dfix [![CI status](https://travis-ci.org/dlang-community/dfix.svg?branch=master)](https://travis-ci.org/dlang-community/dfix/) 2 | 3 | Tool for automatically upgrading D source code 4 | 5 | ## Features 6 | 7 | * Updates old-style alias syntax to new-style 8 | * Fixes implicit concatenation of string literals 9 | * Automatic conversion of C-style array declarations and parameters to D-style. 10 | * Upgrades code to comply with [DIP64](https://wiki.dlang.org/DIP64) when the `--dip64` switch is specified. (Not recommended) 11 | * Upgrades code to comply with [DIP65](https://wiki.dlang.org/DIP65) unless the `--dip65=false` switch is specified. 12 | * Upgrades code to comply with [DIP1003](https://github.com/dlang/DIPs/blob/master/DIPs/accepted/DIP1003.md) unless the `--dip1003=false` switch is specified. 13 | * Rewrites functions declared `const`, `immutable` and `inout` to be more clear by moving these keywords from the left side of the return type to the right side of the parameter list. 14 | 15 | ## Notes 16 | 17 | dfix will edit your files in-place. Do not use dfix on files that have no 18 | backup copies. Source control solves this problem for you. Double-check the 19 | results before checking in the modified code. 20 | 21 | ## Installation 22 | 23 | OS X users with homebrew should be able to install via `brew install dfix` for the latest stable release or `brew install dfix --HEAD` for the latest git master branch. 24 | 25 | Other users should manually install, e.g. on \*nix systems: 26 | 27 | * `git clone https://github.com/dlang-community/dfix && git submodule update --init` 28 | * `cd dfix` 29 | * `git checkout v0.3.5` if you want the stable release 30 | * `make` to build 31 | * `make test` to test 32 | * either add the `bin` directory to your path or copy to another directory that is on your path. 33 | 34 | ### Installing with DUB 35 | 36 | ```sh 37 | > dub fetch dfix && dub run dfix 38 | ``` 39 | -------------------------------------------------------------------------------- /appveyor.yml: -------------------------------------------------------------------------------- 1 | platform: x64 2 | environment: 3 | matrix: 4 | #- DC: dmd 5 | #DVersion: nightly 6 | #arch: x64 7 | #- DC: dmd 8 | #DVersion: nightly 9 | #arch: x86 10 | #- DC: dmd 11 | #DVersion: beta 12 | #arch: x64 13 | #- DC: dmd 14 | #DVersion: beta 15 | #arch: x86 16 | - DC: dmd 17 | DVersion: stable 18 | arch: x64 19 | - DC: dmd 20 | DVersion: stable 21 | arch: x86 22 | #- DC: ldc 23 | # DVersion: beta 24 | # arch: x86 25 | #- DC: ldc 26 | # DVersion: beta 27 | # arch: x64 28 | #- DC: ldc 29 | # DVersion: stable 30 | # arch: x86 31 | #- DC: ldc 32 | # DVersion: stable 33 | # arch: x64 34 | 35 | skip_tags: false 36 | branches: 37 | only: 38 | - master 39 | 40 | install: 41 | - ps: function ResolveLatestDMD 42 | { 43 | $version = $env:DVersion; 44 | if($version -eq "stable") { 45 | $latest = (Invoke-WebRequest "http://downloads.dlang.org/releases/LATEST").toString(); 46 | $url = "http://downloads.dlang.org/releases/2.x/$($latest)/dmd.$($latest).windows.7z"; 47 | }elseif($version -eq "beta") { 48 | $latest = (Invoke-WebRequest "http://downloads.dlang.org/pre-releases/LATEST").toString(); 49 | $latestVersion = $latest.split("-")[0].split("~")[0]; 50 | $url = "http://downloads.dlang.org/pre-releases/2.x/$($latestVersion)/dmd.$($latest).windows.7z"; 51 | }elseif($version -eq "nightly") { 52 | $url = "http://nightlies.dlang.org/dmd-master-2017-05-20/dmd.master.windows.7z" 53 | }else { 54 | $url = "http://downloads.dlang.org/releases/2.x/$($version)/dmd.$($version).windows.7z"; 55 | } 56 | $env:PATH += ";C:\dmd2\windows\bin;"; 57 | return $url; 58 | } 59 | - ps: function ResolveLatestLDC 60 | { 61 | $version = $env:DVersion; 62 | if($version -eq "stable") { 63 | $latest = (Invoke-WebRequest "https://ldc-developers.github.io/LATEST").toString().replace("`n","").replace("`r",""); 64 | $url = "https://github.com/ldc-developers/ldc/releases/download/v$($latest)/ldc2-$($latest)-windows-x64.7z"; 65 | }elseif($version -eq "beta") { 66 | $latest = (Invoke-WebRequest "https://ldc-developers.github.io/LATEST_BETA").toString().replace("`n","").replace("`r",""); 67 | $url = "https://github.com/ldc-developers/ldc/releases/download/v$($latest)/ldc2-$($latest)-windows-x64.7z"; 68 | } else { 69 | $latest = $version; 70 | $url = "https://github.com/ldc-developers/ldc/releases/download/v$($version)/ldc2-$($version)-win64-msvc.zip"; 71 | } 72 | $env:PATH += ";C:\ldc2-$($latest)-windows-x64\bin"; 73 | $env:DC = "ldc2"; 74 | return $url; 75 | } 76 | - ps: function SetUpDCompiler 77 | { 78 | $env:toolchain = "msvc"; 79 | if($env:DC -eq "dmd"){ 80 | echo "downloading ..."; 81 | $url = ResolveLatestDMD; 82 | echo $url; 83 | Invoke-WebRequest $url -OutFile "c:\dmd.7z"; 84 | echo "finished."; 85 | pushd c:\\; 86 | 7z x dmd.7z > $null; 87 | popd; 88 | } 89 | elseif($env:DC -eq "ldc"){ 90 | echo "downloading ..."; 91 | $url = ResolveLatestLDC; 92 | echo $url; 93 | Invoke-WebRequest $url -OutFile "c:\ldc.zip"; 94 | echo "finished."; 95 | pushd c:\\; 96 | 7z x ldc.7z > $null; 97 | popd; 98 | } 99 | } 100 | - ps: SetUpDCompiler 101 | 102 | build_script: 103 | - ps: if($env:arch -eq "x86"){ 104 | $env:compilersetupargs = "x86"; 105 | $env:Darch = "x86"; 106 | $env:DConf = "m32"; 107 | }elseif($env:arch -eq "x64"){ 108 | $env:compilersetupargs = "amd64"; 109 | $env:Darch = "x86_64"; 110 | $env:DConf = "m64"; 111 | } 112 | - ps: $env:compilersetup = "C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\vcvarsall"; 113 | - '"%compilersetup%" %compilersetupargs%' 114 | 115 | test_script: 116 | - echo %PLATFORM% 117 | - echo %Darch% 118 | - echo %DC% 119 | - echo %PATH% 120 | - '%DC% --version' 121 | - dub build --arch=%Darch% --compiler=%DC% 122 | -------------------------------------------------------------------------------- /dub.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "dfix", 3 | "description": "Tool for automatically upgrading D source code", 4 | "copyright": "© Brian Schott", 5 | "authors": ["Brian Schott"], 6 | "license" : "BSL-1.0", 7 | "targetType": "executable", 8 | "dependencies": { 9 | "libdparse": "~>0.8.7" 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /makefile: -------------------------------------------------------------------------------- 1 | DMD ?= dmd 2 | 3 | FLAGS := -Ilibdparse/src/\ 4 | -Istdx-allocator/source/\ 5 | -wi\ 6 | -g\ 7 | -dip25\ 8 | -ofbin/dfix 9 | 10 | FILES := src/dfix.d \ 11 | $(shell find libdparse/src/ -name "*.d") \ 12 | $(shell find stdx-allocator/source/ -name "*.d") 13 | 14 | dfix_binary: 15 | rm -rf bin 16 | mkdir -p bin 17 | ${DMD} ${FILES} ${FLAGS} 18 | rm -f bin/dfix.o 19 | 20 | clean: 21 | rm -rf bin 22 | rm -rf test/testfile.d 23 | 24 | test: dfix_binary 25 | cp test/testfile_master.d test/testfile.d 26 | ./bin/dfix test/testfile.d 27 | diff test/testfile.d test/testfile_expected.d 28 | # Make sure that running dfix on the output of dfix changes nothing. 29 | ./bin/dfix test/testfile.d 30 | diff test/testfile.d test/testfile_expected.d 31 | -------------------------------------------------------------------------------- /src/dfix.d: -------------------------------------------------------------------------------- 1 | module dfix; 2 | 3 | import std.experimental.lexer; 4 | import dparse.lexer; 5 | import dparse.parser; 6 | import dparse.ast; 7 | import std.stdio; 8 | import std.format; 9 | import std.file; 10 | 11 | int main(string[] args) 12 | { 13 | import std.getopt : getopt; 14 | import std.parallelism : parallel; 15 | 16 | // http://wiki.dlang.org/DIP64 17 | bool dip64; 18 | // http://wiki.dlang.org/DIP65 19 | bool dip65 = true; 20 | //https://github.com/dlang/DIPs/blob/master/DIPs/DIP1003.md 21 | bool dip1003 = true; 22 | 23 | bool help; 24 | 25 | try 26 | { 27 | getopt(args, 28 | "dip64", &dip64, 29 | "dip65", &dip65, 30 | "dip1003", &dip1003, 31 | "help|h", &help, 32 | ); 33 | } 34 | catch (Exception e) 35 | { 36 | stderr.writeln(e.msg); 37 | return 1; 38 | } 39 | 40 | if (help) 41 | { 42 | printHelp(); 43 | return 0; 44 | } 45 | 46 | if (args.length < 2) 47 | { 48 | stderr.writeln("File path is a required argument"); 49 | return 1; 50 | } 51 | 52 | string[] files; 53 | 54 | foreach (arg; args[1 .. $]) 55 | { 56 | if (isDir(arg)) 57 | { 58 | foreach (f; dirEntries(arg, "*.{d,di}", SpanMode.depth)) 59 | files ~= f; 60 | } 61 | else 62 | files ~= arg; 63 | } 64 | 65 | foreach (f; parallel(files)) 66 | { 67 | try 68 | upgradeFile(f, dip64, dip65, dip1003); 69 | catch (Exception e) 70 | stderr.writeln("Failed to upgrade ", f, ":(", e.file, ":", e.line, ") ", e.msg); 71 | } 72 | 73 | return 0; 74 | } 75 | 76 | /** 77 | * Prints help message 78 | */ 79 | void printHelp() 80 | { 81 | stdout.writeln(` 82 | Dfix automatically upgrades D source code to comply with new language changes. 83 | Files are modified in place, so have backup copies ready or use a source 84 | control system. 85 | 86 | Usage: 87 | 88 | dfix [Options] FILES DIRECTORIES 89 | 90 | Options: 91 | 92 | --dip64 93 | Rewrites attributes to be compliant with DIP64. This defaults to 94 | "false". Do not use this feature if you want your code to compile. 95 | It exists as a proof-of-concept for enabling DIP64. 96 | --dip65 97 | Rewrites catch blocks to be compliant with DIP65. This defaults to 98 | "true". Use --dip65=false to disable this fix. 99 | --dip1003 100 | Rewrites body blocks to be compliant with DIP1003. This defaults to 101 | "true". Use --dip1003=false to disable this fix. 102 | --help -h 103 | Prints this help message 104 | `); 105 | } 106 | 107 | /** 108 | * Fixes the given file. 109 | */ 110 | void upgradeFile(string fileName, bool dip64, bool dip65, bool dip1003) 111 | { 112 | import std.algorithm : filter, canFind; 113 | import std.range : retro; 114 | import std.array : array, uninitializedArray; 115 | import dparse.formatter : Formatter; 116 | import std.exception : enforce; 117 | import dparse.rollback_allocator : RollbackAllocator; 118 | import std.functional : toDelegate; 119 | 120 | File input = File(fileName, "rb"); 121 | ubyte[] inputBytes = uninitializedArray!(ubyte[])(cast(size_t) input.size); 122 | input.rawRead(inputBytes); 123 | input.close(); 124 | StringCache cache = StringCache(StringCache.defaultBucketCount); 125 | LexerConfig config; 126 | config.fileName = fileName; 127 | config.stringBehavior = StringBehavior.source; 128 | auto tokens = byToken(inputBytes, config, &cache).array; 129 | auto parseTokens = tokens.filter!(a => a != tok!"whitespace" 130 | && a != tok!"comment" && a != tok!"specialTokenSequence").array; 131 | 132 | RollbackAllocator allocator; 133 | uint errorCount; 134 | auto mod = parseModule(parseTokens, fileName, &allocator, toDelegate(&reportErrors), &errorCount); 135 | if (errorCount > 0) 136 | { 137 | stderr.writefln("%d parse errors encountered. Aborting upgrade of %s", 138 | errorCount, fileName); 139 | return; 140 | } 141 | 142 | File output = File(fileName, "wb"); 143 | auto visitor = new DFixVisitor; 144 | visitor.visit(mod); 145 | relocateMarkers(visitor.markers, tokens); 146 | 147 | SpecialMarker[] markers = visitor.markers; 148 | 149 | auto formatter = new Formatter!(File.LockingTextWriter)(File.LockingTextWriter.init); 150 | 151 | void writeType(T)(File output, T tokens, ref size_t i) 152 | { 153 | if (isBasicType(tokens[i].type)) 154 | { 155 | writeToken(output, tokens[i]); 156 | i++; 157 | } 158 | else if ((tokens[i] == tok!"const" || tokens[i] == tok!"immutable" 159 | || tokens[i] == tok!"shared" || tokens[i] == tok!"inout") 160 | && tokens[i + 1] == tok!"(") 161 | { 162 | writeToken(output, tokens[i]); 163 | i++; 164 | skipAndWrite!("(", ")")(output, tokens, i); 165 | } 166 | else 167 | { 168 | skipIdentifierChain(output, tokens, i, true); 169 | if (i < tokens.length && tokens[i] == tok!"!") 170 | { 171 | writeToken(output, tokens[i]); 172 | i++; 173 | if (i + 1 < tokens.length && tokens[i + 1] == tok!"(") 174 | skipAndWrite!("(", ")")(output, tokens, i); 175 | else if (tokens[i].type == tok!"identifier") 176 | skipIdentifierChain(output, tokens, i, true); 177 | else 178 | { 179 | writeToken(output, tokens[i]); 180 | i++; 181 | } 182 | } 183 | } 184 | skipWhitespace(output, tokens, i); 185 | // print out suffixes 186 | while (i < tokens.length && (tokens[i] == tok!"*" || tokens[i] == tok!"[")) 187 | { 188 | if (tokens[i] == tok!"*") 189 | { 190 | writeToken(output, tokens[i]); 191 | i++; 192 | } 193 | else if (tokens[i] == tok!"[") 194 | skipAndWrite!("[", "]")(output, tokens, i); 195 | } 196 | } 197 | 198 | for (size_t i = 0; i < tokens.length; i++) 199 | { 200 | markerLoop: foreach (marker; markers) 201 | { 202 | with (SpecialMarkerType) final switch (marker.type) 203 | { 204 | case bodyEnd: 205 | if (tokens[i].index != marker.index) 206 | break; 207 | assert (tokens[i].type == tok!"}", format("%d %s", tokens[i].line, str(tokens[i].type))); 208 | writeToken(output, tokens[i]); 209 | i++; 210 | if (i < tokens.length && tokens[i] == tok!";") 211 | i++; 212 | markers = markers[1 .. $]; 213 | break markerLoop; 214 | case functionAttributePrefix: 215 | if (tokens[i].index != marker.index) 216 | break; 217 | // skip over token to be moved 218 | i++; 219 | skipWhitespace(output, tokens, i, false); 220 | 221 | // skip over function return type 222 | writeType(output, tokens, i); 223 | skipWhitespace(output, tokens, i); 224 | 225 | // skip over function name 226 | skipIdentifierChain(output, tokens, i, true); 227 | skipWhitespace(output, tokens, i, false); 228 | 229 | // skip first paramters 230 | skipAndWrite!("(", ")")(output, tokens, i); 231 | 232 | immutable bookmark = i; 233 | skipWhitespace(output, tokens, i, false); 234 | 235 | // If there is a second set of parameters, go back to the bookmark 236 | // and print out the whitespace 237 | if (i < tokens.length && tokens[i] == tok!"(") 238 | { 239 | i = bookmark; 240 | skipWhitespace(output, tokens, i); 241 | skipAndWrite!("(", ")")(output, tokens, i); 242 | skipWhitespace(output, tokens, i, false); 243 | } 244 | else 245 | i = bookmark; 246 | 247 | // write out the attribute being moved 248 | output.write(" ", marker.functionAttribute); 249 | 250 | // if there was no whitespace, add it after the moved attribute 251 | if (i < tokens.length && tokens[i] != tok!"whitespace" && tokens[i] != tok!";") 252 | output.write(" "); 253 | 254 | markers = markers[1 .. $]; 255 | break markerLoop; 256 | case cStyleArray: 257 | if (i != marker.index) 258 | break; 259 | formatter.sink = output.lockingTextWriter(); 260 | foreach (node; retro(marker.nodes)) 261 | formatter.format(node); 262 | formatter.sink = File.LockingTextWriter.init; 263 | skipWhitespace(output, tokens, i); 264 | writeToken(output, tokens[i]); 265 | i++; 266 | suffixLoop: while (i < tokens.length) switch (tokens[i].type) 267 | { 268 | case tok!"(": skipAndWrite!("(", ")")(output, tokens, i); break; 269 | case tok!"[": skip!("[", "]")(tokens, i); break; 270 | case tok!"*": i++; break; 271 | default: break suffixLoop; 272 | } 273 | markers = markers[1 .. $]; 274 | break markerLoop; 275 | } 276 | } 277 | 278 | if (i >= tokens.length) 279 | break; 280 | 281 | switch (tokens[i].type) 282 | { 283 | case tok!"asm": 284 | skipAsmBlock(output, tokens, i); 285 | goto default; 286 | case tok!"catch": 287 | if (!dip65) 288 | goto default; 289 | size_t j = i + 1; 290 | while (j < tokens.length && (tokens[j] == tok!"whitespace" || tokens[j] == tok!"comment")) 291 | j++; 292 | if (j < tokens.length && tokens[j].type != tok!"(") 293 | { 294 | output.write("catch (Throwable)"); 295 | break; 296 | } 297 | else 298 | goto default; 299 | case tok!"deprecated": 300 | if (dip64) 301 | output.write("@"); 302 | output.writeToken(tokens[i]); 303 | i++; 304 | if (i < tokens.length && tokens[i] == tok!"(") 305 | skipAndWrite!("(", ")")(output, tokens, i); 306 | if (i < tokens.length) 307 | goto default; 308 | else 309 | break; 310 | case tok!"stringLiteral": 311 | immutable size_t stringBookmark = i; 312 | while (tokens[i] == tok!"stringLiteral") 313 | { 314 | i++; 315 | skipWhitespace(output, tokens, i, false); 316 | } 317 | immutable bool parensNeeded = stringBookmark + 1 != i && tokens[i] == tok!"."; 318 | i = stringBookmark; 319 | if (parensNeeded) 320 | output.write("("); 321 | output.writeToken(tokens[i]); 322 | i++; 323 | skipWhitespace(output, tokens, i); 324 | while (tokens[i] == tok!"stringLiteral") 325 | { 326 | output.write("~ "); 327 | output.writeToken(tokens[i]); 328 | i++; 329 | skipWhitespace(output, tokens, i); 330 | } 331 | if (parensNeeded) 332 | output.write(")"); 333 | if (i < tokens.length) 334 | goto default; 335 | else 336 | break; 337 | case tok!"override": 338 | case tok!"final": 339 | case tok!"abstract": 340 | case tok!"align": 341 | case tok!"pure": 342 | case tok!"nothrow": 343 | if (!dip64) 344 | goto default; 345 | output.write("@"); 346 | output.write(str(tokens[i].type)); 347 | break; 348 | case tok!"alias": 349 | bool multipleAliases = false; 350 | bool oldStyle = true; 351 | output.writeToken(tokens[i]); // alias 352 | i++; 353 | size_t j = i + 1; 354 | 355 | int depth; 356 | loop: while (j < tokens.length) switch (tokens[j].type) 357 | { 358 | case tok!"(": 359 | depth++; 360 | j++; 361 | break; 362 | case tok!")": 363 | depth--; 364 | if (depth < 0) 365 | { 366 | oldStyle = false; 367 | break loop; 368 | } 369 | j++; 370 | break; 371 | case tok!"=": 372 | case tok!"this": 373 | j++; 374 | oldStyle = false; 375 | break; 376 | case tok!",": 377 | j++; 378 | if (depth == 0) 379 | multipleAliases = true; 380 | break; 381 | case tok!";": 382 | break loop; 383 | default: 384 | j++; 385 | break; 386 | } 387 | 388 | if (!oldStyle) foreach (k; i .. j + 1) 389 | { 390 | output.writeToken(tokens[k]); 391 | i = k; 392 | } 393 | else 394 | { 395 | skipWhitespace(output, tokens, i); 396 | 397 | size_t beforeStart = i; 398 | size_t beforeEnd = beforeStart; 399 | 400 | loop2: while (beforeEnd < tokens.length) switch (tokens[beforeEnd].type) 401 | { 402 | case tok!"bool": 403 | case tok!"byte": 404 | case tok!"ubyte": 405 | case tok!"short": 406 | case tok!"ushort": 407 | case tok!"int": 408 | case tok!"uint": 409 | case tok!"long": 410 | case tok!"ulong": 411 | case tok!"char": 412 | case tok!"wchar": 413 | case tok!"dchar": 414 | case tok!"float": 415 | case tok!"double": 416 | case tok!"real": 417 | case tok!"ifloat": 418 | case tok!"idouble": 419 | case tok!"ireal": 420 | case tok!"cfloat": 421 | case tok!"cdouble": 422 | case tok!"creal": 423 | case tok!"void": 424 | beforeEnd++; 425 | break loop2; 426 | case tok!".": 427 | beforeEnd++; 428 | goto case; 429 | case tok!"identifier": 430 | skipIdentifierChain(output, tokens, beforeEnd); 431 | break loop2; 432 | case tok!"typeof": 433 | beforeEnd++; 434 | skip!("(", ")")(tokens, beforeEnd); 435 | skipWhitespace(output, tokens, beforeEnd, false); 436 | if (tokens[beforeEnd] == tok!".") 437 | skipIdentifierChain(output, tokens, beforeEnd); 438 | break loop2; 439 | case tok!"@": 440 | beforeEnd++; 441 | if (tokens[beforeEnd] == tok!"identifier") 442 | beforeEnd++; 443 | if (tokens[beforeEnd] == tok!"(") 444 | skip!("(", ")")(tokens, beforeEnd); 445 | skipWhitespace(output, tokens, beforeEnd, false); 446 | break; 447 | case tok!"static": 448 | case tok!"const": 449 | case tok!"immutable": 450 | case tok!"inout": 451 | case tok!"shared": 452 | case tok!"extern": 453 | case tok!"nothrow": 454 | case tok!"pure": 455 | case tok!"__vector": 456 | beforeEnd++; 457 | skipWhitespace(output, tokens, beforeEnd, false); 458 | if (tokens[beforeEnd] == tok!"(") 459 | skip!("(", ")")(tokens, beforeEnd); 460 | if (beforeEnd >= tokens.length) 461 | break loop2; 462 | size_t k = beforeEnd; 463 | skipWhitespace(output, tokens, k, false); 464 | if (k + 1 < tokens.length && tokens[k + 1].type == tok!";") 465 | break loop2; 466 | else 467 | beforeEnd = k; 468 | break; 469 | default: 470 | break loop2; 471 | } 472 | 473 | i = beforeEnd; 474 | 475 | skipWhitespace(output, tokens, i, false); 476 | 477 | if (tokens[i] == tok!"*" || tokens[i] == tok!"[" 478 | || tokens[i] == tok!"function" || tokens[i] == tok!"delegate") 479 | { 480 | beforeEnd = i; 481 | } 482 | 483 | loop3: while (beforeEnd < tokens.length) switch (tokens[beforeEnd].type) 484 | { 485 | case tok!"*": 486 | beforeEnd++; 487 | size_t m = beforeEnd; 488 | skipWhitespace(output, tokens, m, false); 489 | if (m < tokens.length && (tokens[m] == tok!"*" 490 | || tokens[m] == tok!"[" || tokens[m] == tok!"function" 491 | || tokens[m] == tok!"delegate")) 492 | { 493 | beforeEnd = m; 494 | } 495 | break; 496 | case tok!"[": 497 | skip!("[", "]")(tokens, beforeEnd); 498 | size_t m = beforeEnd; 499 | skipWhitespace(output, tokens, m, false); 500 | if (m < tokens.length && (tokens[m] == tok!"*" 501 | || tokens[m] == tok!"[" || tokens[m] == tok!"function" 502 | || tokens[m] == tok!"delegate")) 503 | { 504 | beforeEnd = m; 505 | } 506 | break; 507 | case tok!"function": 508 | case tok!"delegate": 509 | beforeEnd++; 510 | skipWhitespace(output, tokens, beforeEnd, false); 511 | skip!("(", ")")(tokens, beforeEnd); 512 | size_t l = beforeEnd; 513 | skipWhitespace(output, tokens, l, false); 514 | loop4: while (l < tokens.length) switch (tokens[l].type) 515 | { 516 | case tok!"const": 517 | case tok!"nothrow": 518 | case tok!"pure": 519 | case tok!"immutable": 520 | case tok!"inout": 521 | case tok!"shared": 522 | beforeEnd = l + 1; 523 | l = beforeEnd; 524 | skipWhitespace(output, tokens, l, false); 525 | if (l < tokens.length && tokens[l].type == tok!"identifier") 526 | { 527 | beforeEnd = l - 1; 528 | break loop4; 529 | } 530 | break; 531 | case tok!"@": 532 | beforeEnd = l + 1; 533 | skipWhitespace(output, tokens, beforeEnd, false); 534 | if (tokens[beforeEnd] == tok!"(") 535 | skip!("(", ")")(tokens, beforeEnd); 536 | else 537 | { 538 | beforeEnd++; // identifier 539 | skipWhitespace(output, tokens, beforeEnd, false); 540 | if (tokens[beforeEnd] == tok!"(") 541 | skip!("(", ")")(tokens, beforeEnd); 542 | } 543 | l = beforeEnd; 544 | skipWhitespace(output, tokens, l, false); 545 | if (l < tokens.length && tokens[l].type == tok!"identifier") 546 | { 547 | beforeEnd = l - 1; 548 | break loop4; 549 | } 550 | break; 551 | default: 552 | break loop4; 553 | } 554 | break; 555 | default: 556 | break loop3; 557 | } 558 | 559 | i = beforeEnd; 560 | skipWhitespace(output, tokens, i, false); 561 | 562 | output.writeToken(tokens[i]); 563 | output.write(" = "); 564 | foreach (l; beforeStart .. beforeEnd) 565 | output.writeToken(tokens[l]); 566 | 567 | if (multipleAliases) 568 | { 569 | i++; 570 | skipWhitespace(output, tokens, i, false); 571 | while (tokens[i] == tok!",") 572 | { 573 | i++; // , 574 | output.write(", "); 575 | skipWhitespace(output, tokens, i, false); 576 | output.writeToken(tokens[i]); 577 | output.write(" = "); 578 | foreach (l; beforeStart .. beforeEnd) 579 | output.writeToken(tokens[l]); 580 | } 581 | } 582 | } 583 | break; 584 | case tok!"identifier": 585 | if (tokens[i].text == "body") 586 | (dip1003 && tokens.isBodyKw(i)) ? output.write("do") : output.write("body"); 587 | else 588 | goto default; 589 | break; 590 | default: 591 | output.writeToken(tokens[i]); 592 | break; 593 | } 594 | } 595 | } 596 | 597 | /** 598 | * The types of special token ranges identified by the parsing pass 599 | */ 600 | enum SpecialMarkerType 601 | { 602 | /// Function declarations such as "const int foo();" 603 | functionAttributePrefix, 604 | /// Variable and parameter declarations such as "int bar[]" 605 | cStyleArray, 606 | /// The location of a closing brace for an interface, class, struct, union, 607 | /// or enum. 608 | bodyEnd 609 | } 610 | 611 | /** 612 | * Identifies ranges of tokens in the source tokens that need to be rewritten 613 | */ 614 | struct SpecialMarker 615 | { 616 | /// Range type 617 | SpecialMarkerType type; 618 | 619 | /// Begin byte position (before relocateMarkers) or token index 620 | /// (after relocateMarkers) 621 | size_t index; 622 | 623 | /// The type suffix AST nodes that should be moved 624 | const(TypeSuffix[]) nodes; 625 | 626 | /// The function attribute such as const, immutable, or inout to move 627 | string functionAttribute; 628 | } 629 | 630 | /** 631 | * Scans a module's parsed AST and looks for C-style array variables and 632 | * parameters, storing the locations in the markers array. 633 | */ 634 | class DFixVisitor : ASTVisitor 635 | { 636 | // C-style arrays variables 637 | override void visit(const VariableDeclaration varDec) 638 | { 639 | if (varDec.declarators.length == 0) 640 | return; 641 | markers ~= SpecialMarker(SpecialMarkerType.cStyleArray, 642 | varDec.declarators[0].name.index, varDec.declarators[0].cstyle); 643 | } 644 | 645 | // C-style array parameters 646 | override void visit(const Parameter param) 647 | { 648 | if (param.cstyle.length > 0) 649 | markers ~= SpecialMarker(SpecialMarkerType.cStyleArray, param.name.index, 650 | param.cstyle); 651 | param.accept(this); 652 | } 653 | 654 | // interface, union, class, struct body closing braces 655 | override void visit(const StructBody structBody) 656 | { 657 | structBody.accept(this); 658 | markers ~= SpecialMarker(SpecialMarkerType.bodyEnd, structBody.endLocation); 659 | } 660 | 661 | // enum body closing braces 662 | override void visit(const EnumBody enumBody) 663 | { 664 | enumBody.accept(this); 665 | // skip over enums whose body is a single semicolon 666 | if (enumBody.endLocation == 0 && enumBody.startLocation == 0) 667 | return; 668 | markers ~= SpecialMarker(SpecialMarkerType.bodyEnd, enumBody.endLocation); 669 | } 670 | 671 | // Confusing placement of function attributes 672 | override void visit(const Declaration dec) 673 | { 674 | if (dec.functionDeclaration is null) 675 | goto end; 676 | if (dec.attributes.length == 0) 677 | goto end; 678 | foreach (attr; dec.attributes) 679 | { 680 | if (attr.attribute == tok!"") 681 | continue; 682 | if (attr.attribute == tok!"const" 683 | || attr.attribute == tok!"inout" 684 | || attr.attribute == tok!"immutable") 685 | { 686 | markers ~= SpecialMarker(SpecialMarkerType.functionAttributePrefix, 687 | attr.attribute.index, null, str(attr.attribute.type)); 688 | } 689 | } 690 | end: 691 | dec.accept(this); 692 | } 693 | 694 | alias visit = ASTVisitor.visit; 695 | 696 | /// Parts of the source file identified as needing a rewrite 697 | SpecialMarker[] markers; 698 | } 699 | 700 | /** 701 | * Converts the marker index from a byte index into the source code to an index 702 | * into the tokens array. 703 | */ 704 | void relocateMarkers(SpecialMarker[] markers, const(Token)[] tokens) pure nothrow @nogc 705 | { 706 | foreach (ref marker; markers) 707 | { 708 | if (marker.type != SpecialMarkerType.cStyleArray) 709 | continue; 710 | size_t index = 0; 711 | while (tokens[index].index != marker.index) 712 | index++; 713 | marker.index = index - 1; 714 | } 715 | } 716 | 717 | /** 718 | * Writes a token to the output file. 719 | */ 720 | void writeToken(File output, ref const(Token) token) 721 | { 722 | output.write(token.text is null ? str(token.type) : token.text); 723 | } 724 | 725 | void skipAndWrite(alias Open, alias Close)(File output, const(Token)[] tokens, ref size_t index) 726 | { 727 | int depth = 1; 728 | writeToken(output, tokens[index]); 729 | index++; 730 | while (index < tokens.length && depth > 0) switch (tokens[index].type) 731 | { 732 | case tok!Open: 733 | depth++; 734 | writeToken(output, tokens[index]); 735 | index++; 736 | break; 737 | case tok!Close: 738 | depth--; 739 | writeToken(output, tokens[index]); 740 | index++; 741 | break; 742 | default: 743 | writeToken(output, tokens[index]); 744 | index++; 745 | break; 746 | } 747 | } 748 | 749 | /** 750 | * Returns true if `body` is a keyword and false if it's an identifier. 751 | */ 752 | bool isBodyKw(const(Token)[] tokens, size_t index) 753 | { 754 | assert(index); 755 | index -= 1; 756 | L0: while (index--) switch (tokens[index].type) 757 | { 758 | // `in {} body {}` 759 | case tok!"}": 760 | return true; 761 | case tok!"comment": 762 | continue; 763 | // `void foo () return {}` or `return body;` 764 | case tok!"return": 765 | continue; 766 | // `void foo () @safe pure body {}` 767 | case tok!")": 768 | case tok!"const": 769 | case tok!"immutable": 770 | case tok!"inout": 771 | case tok!"shared": 772 | case tok!"@": 773 | case tok!"pure": 774 | case tok!"nothrow": 775 | case tok!"scope": 776 | return true; 777 | default: 778 | break L0; 779 | } 780 | return false; 781 | } 782 | 783 | /** 784 | * Skips balanced parens, braces, or brackets. index will be incremented to 785 | * index tokens just after the balanced closing token. 786 | */ 787 | void skip(alias Open, alias Close)(const(Token)[] tokens, ref size_t index) 788 | { 789 | int depth = 1; 790 | index++; 791 | while (index < tokens.length && depth > 0) switch (tokens[index].type) 792 | { 793 | case tok!Open: depth++; index++; break; 794 | case tok!Close: depth--; index++; break; 795 | default: index++; break; 796 | } 797 | } 798 | 799 | /** 800 | * Skips whitespace tokens, incrementing index until it indexes tokens at a 801 | * non-whitespace token. 802 | */ 803 | void skipWhitespace(File output, const(Token)[] tokens, ref size_t index, bool print = true) 804 | { 805 | while (index < tokens.length && (tokens[index] == tok!"whitespace" || tokens[index] == tok!"comment")) 806 | { 807 | if (print) output.writeToken(tokens[index]); 808 | index++; 809 | } 810 | } 811 | 812 | /** 813 | * Advances index until it indexs the token just after an identifier or template 814 | * chain. 815 | */ 816 | void skipIdentifierChain(File output, const(Token)[] tokens, ref size_t index, bool print = false) 817 | { 818 | loop: while (index < tokens.length) switch (tokens[index].type) 819 | { 820 | case tok!".": 821 | if (print) 822 | writeToken(output, tokens[index]); 823 | index++; 824 | skipWhitespace(output, tokens, index, false); 825 | break; 826 | case tok!"identifier": 827 | if (print) 828 | writeToken(output, tokens[index]); 829 | index++; 830 | size_t i = index; 831 | skipWhitespace(output, tokens, i, false); 832 | if (tokens[i] == tok!"!") 833 | { 834 | i++; 835 | if (print) 836 | writeToken(output, tokens[index]); 837 | index++; 838 | skipWhitespace(output, tokens, i, false); 839 | if (tokens[i] == tok!"(") 840 | { 841 | if (print) 842 | skipAndWrite!("(", ")")(output, tokens, i); 843 | else 844 | skip!("(", ")")(tokens, i); 845 | index = i; 846 | } 847 | else 848 | { 849 | i++; 850 | if (print) 851 | writeToken(output, tokens[index]); 852 | index++; 853 | } 854 | } 855 | if (tokens[i] != tok!".") 856 | break loop; 857 | break; 858 | case tok!"whitespace": 859 | index++; 860 | break; 861 | default: 862 | break loop; 863 | } 864 | } 865 | 866 | /** 867 | * Skips over an attribute 868 | */ 869 | void skipAttribute(File output, const(Token)[] tokens, ref size_t i) 870 | { 871 | switch (tokens[i].type) 872 | { 873 | case tok!"@": 874 | output.writeToken(tokens[i]); 875 | i++; // @ 876 | skipWhitespace(output, tokens, i, true); 877 | switch (tokens[i].type) 878 | { 879 | case tok!"identifier": 880 | output.writeToken(tokens[i]); 881 | i++; // identifier 882 | skipWhitespace(output, tokens, i, true); 883 | if (tokens[i].type == tok!"(") 884 | goto case tok!"("; 885 | break; 886 | case tok!"(": 887 | int depth = 1; 888 | output.writeToken(tokens[i]); 889 | i++; 890 | while (i < tokens.length && depth > 0) switch (tokens[i].type) 891 | { 892 | case tok!"(": depth++; output.writeToken(tokens[i]); i++; break; 893 | case tok!")": depth--; output.writeToken(tokens[i]); i++; break; 894 | default: output.writeToken(tokens[i]); i++; break; 895 | } 896 | break; 897 | default: 898 | break; 899 | } 900 | break; 901 | case tok!"nothrow": 902 | case tok!"pure": 903 | output.writeToken(tokens[i]); 904 | i++; 905 | break; 906 | default: 907 | break; 908 | } 909 | } 910 | 911 | /** 912 | * Skips over (and prints) an asm block 913 | */ 914 | void skipAsmBlock(File output, const(Token)[] tokens, ref size_t i) 915 | { 916 | import std.exception : enforce; 917 | 918 | output.write("asm"); 919 | i++; // asm 920 | skipWhitespace(output, tokens, i); 921 | loop: while (true) switch (tokens[i].type) 922 | { 923 | case tok!"@": 924 | case tok!"nothrow": 925 | case tok!"pure": 926 | skipAttribute(output, tokens, i); 927 | skipWhitespace(output, tokens, i); 928 | break; 929 | case tok!"{": 930 | break loop; 931 | default: 932 | break loop; 933 | } 934 | enforce(tokens[i].type == tok!"{"); 935 | output.write("{"); 936 | i++; // { 937 | int depth = 1; 938 | while (depth > 0 && i < tokens.length) switch (tokens[i].type) 939 | { 940 | case tok!"{": depth++; goto default; 941 | case tok!"}": depth--; goto default; 942 | default: writeToken(output, tokens[i]); i++; break; 943 | } 944 | } 945 | 946 | /** 947 | * Dummy message output function for the lexer/parser 948 | */ 949 | void reportErrors(string fileName, size_t lineNumber, size_t columnNumber, 950 | string message, bool isError) 951 | { 952 | import std.stdio : stderr; 953 | 954 | if (!isError) 955 | return; 956 | stderr.writefln("%s(%d:%d)[error]: %s", fileName, lineNumber, columnNumber, message); 957 | } 958 | -------------------------------------------------------------------------------- /test/testfile_expected.d: -------------------------------------------------------------------------------- 1 | pure nothrow void doStuff(int[] x) { 2 | try 3 | whatever(); 4 | catch (Throwable) 5 | somethingElse(); 6 | asm pure 7 | { 8 | mov sp[EBP], ESP; 9 | inc _iSemlockCtrs[EDX * 2]; 10 | } 11 | } 12 | 13 | int[string] someMapping; 14 | 15 | void*[] pointers; 16 | int[][] multiDim; 17 | int[b][a] multiDim; 18 | 19 | enum a = "a"; 20 | enum someString = "123" 21 | ~ "456"; 22 | 23 | alias y = x; 24 | 25 | template Tst(string s) { enum Tst = s; } 26 | alias Mod = Tst!"Test"; 27 | alias Mod2 = Tst!"Test2"; 28 | 29 | alias IInt = immutable(int); 30 | 31 | alias WNDPROC = LRESULT function (HWND, UINT, WPARAM, LPARAM); 32 | alias LPOFNHOOKPROC = UINT function (HWND, UINT, WPARAM, LPARAM); 33 | alias LanguageSpecificHandler = EXCEPTION_DISPOSITION function ( 34 | EXCEPTION_RECORD *exceptionRecord, 35 | DEstablisherFrame *frame, 36 | CONTEXT *context, 37 | void *dispatcherContext); 38 | private extern (D) alias fp_t = void function (Object); 39 | 40 | alias Opcode = static immutable(OpInfo); 41 | alias EntryFn = extern (C) void function(); 42 | 43 | alias Vector = __vector(T); 44 | 45 | 46 | alias __externC = extern(C) RT function(P) nothrow @nogc; 47 | 48 | private alias __dl_iterate_hdr_callback = extern(C) int function(dl_phdr_info*, size_t, void*); 49 | alias externCVoidFunc = extern(C) void function(); 50 | alias ScanAllThreadsFn = void delegate(void*, void*) nothrow; /// The scanning function. 51 | alias ScanAllThreadsTypeFn = void delegate(ScanType, void*, void*) nothrow; /// ditto 52 | alias gcGetFn = void* function(); 53 | 54 | alias PCONTEXT = CONTEXT*, LPCONTEXT = CONTEXT*; 55 | alias PEXCEPTION_RECORD = EXCEPTION_RECORD*, LPEXCEPTION_RECORD = EXCEPTION_RECORD*; 56 | alias PEXCEPTION_POINTERS = EXCEPTION_POINTERS*, LPEXCEPTION_POINTERS = EXCEPTION_POINTERS*; 57 | 58 | void foo() { ("abc%s" ~ "def%s").format("123", "456"); } 59 | void bar() { "ghi".writeln(); } 60 | 61 | enum SomeEnum { a, b } 62 | struct SomeStruct { int a; } 63 | 64 | struct MisplacedAttribute 65 | { 66 | int aFunction() const nothrow { return 1; } 67 | int bFunction(T)() const { return 1; } 68 | int cFunction() const { return 1; } 69 | int dFunction() const 70 | { 71 | return 1; 72 | } 73 | } 74 | 75 | deprecated("string" "concat") int x; 76 | 77 | enum bool isSome(T) = is(T == int); 78 | 79 | immutable(char) toString() const; 80 | string[] doStuff() const; 81 | string* doStuff() const; 82 | 83 | alias c = a .b; 84 | 85 | size_t replicateBits(size_t , )() {} 86 | 87 | void bodyToDo() 88 | in {} 89 | do {} 90 | -------------------------------------------------------------------------------- /test/testfile_master.d: -------------------------------------------------------------------------------- 1 | pure nothrow void doStuff(int x[]) { 2 | try 3 | whatever(); 4 | catch 5 | somethingElse(); 6 | asm pure 7 | { 8 | mov sp[EBP], ESP; 9 | inc _iSemlockCtrs[EDX * 2]; 10 | } 11 | } 12 | 13 | int someMapping[string]; 14 | 15 | void* pointers[]; 16 | int multiDim[][]; 17 | int multiDim[a][b]; 18 | 19 | enum a = "a"; 20 | enum someString = "123" 21 | "456"; 22 | 23 | alias x y; 24 | 25 | template Tst(string s) { enum Tst = s; } 26 | alias Tst!"Test" Mod; 27 | alias Tst!"Test2" Mod2; 28 | 29 | alias immutable(int) IInt; 30 | 31 | alias LRESULT function (HWND, UINT, WPARAM, LPARAM) WNDPROC; 32 | alias UINT function (HWND, UINT, WPARAM, LPARAM) LPOFNHOOKPROC; 33 | alias EXCEPTION_DISPOSITION function ( 34 | EXCEPTION_RECORD *exceptionRecord, 35 | DEstablisherFrame *frame, 36 | CONTEXT *context, 37 | void *dispatcherContext) LanguageSpecificHandler; 38 | private extern (D) alias void function (Object) fp_t; 39 | 40 | alias static immutable(OpInfo) Opcode; 41 | alias extern (C) void function() EntryFn; 42 | 43 | alias __vector(T) Vector; 44 | 45 | 46 | alias extern(C) RT function(P) nothrow @nogc __externC; 47 | 48 | private alias extern(C) int function(dl_phdr_info*, size_t, void*) __dl_iterate_hdr_callback; 49 | alias extern(C) void function() externCVoidFunc; 50 | alias void delegate(void*, void*) nothrow ScanAllThreadsFn; /// The scanning function. 51 | alias void delegate(ScanType, void*, void*) nothrow ScanAllThreadsTypeFn; /// ditto 52 | alias void* function() gcGetFn; 53 | 54 | alias CONTEXT* PCONTEXT, LPCONTEXT; 55 | alias EXCEPTION_RECORD* PEXCEPTION_RECORD, LPEXCEPTION_RECORD; 56 | alias EXCEPTION_POINTERS* PEXCEPTION_POINTERS, LPEXCEPTION_POINTERS; 57 | 58 | void foo() { "abc%s" "def%s".format("123", "456"); } 59 | void bar() { "ghi".writeln(); } 60 | 61 | enum SomeEnum { a, b }; 62 | struct SomeStruct { int a; }; 63 | 64 | struct MisplacedAttribute 65 | { 66 | const int aFunction() nothrow { return 1; } 67 | const int bFunction(T)() { return 1; } 68 | const int cFunction(){ return 1; } 69 | const int dFunction() 70 | { 71 | return 1; 72 | } 73 | } 74 | 75 | deprecated("string" "concat") int x; 76 | 77 | enum bool isSome(T) = is(T == int); 78 | 79 | const immutable(char) toString(); 80 | const string[] doStuff(); 81 | const string* doStuff(); 82 | 83 | alias a .b c; 84 | 85 | size_t replicateBits(size_t , )() {} 86 | 87 | void bodyToDo() 88 | in {} 89 | body {} 90 | --------------------------------------------------------------------------------