├── FBuild
    ├── BuildComponents
    │   ├── BuildComponent.cs
    │   ├── Compiler.cs
    │   ├── CompilerTypes.cs
    │   ├── ExecutableComponent.cs
    │   ├── Linker.cs
    │   ├── LinkerComponent.cs
    │   ├── ObjectGroupComponent.cs
    │   └── PchOptions.cs
    ├── FASTBuildCommon.cs
    └── FASTBuildConfiguration.cs
├── FastBuild.cs
└── README.md


/FBuild/BuildComponents/BuildComponent.cs:
--------------------------------------------------------------------------------
 1 | using System;
 2 | using System.Collections.Generic;
 3 | using System.Linq;
 4 | using System.Text;
 5 | using System.Threading.Tasks;
 6 | 
 7 | namespace UnrealBuildTool.FBuild.BuildComponents
 8 | {
 9 |     public abstract class BuildComponent
10 |     {        
11 |         public Action Action { get; set; }
12 | 
13 |         public string NodeType { get; set; }
14 | 
15 |         public int AliasIndex { get; set; }
16 | 
17 |         public List<BuildComponent> Dependencies { get; set; }
18 | 
19 |         public string Alias => NodeType + "-" + AliasIndex;
20 | 
21 |         protected BuildComponent()
22 |         {
23 |             Dependencies = new List<BuildComponent>();
24 |         }
25 |     }
26 | 
27 | }
28 | 


--------------------------------------------------------------------------------
/FBuild/BuildComponents/Compiler.cs:
--------------------------------------------------------------------------------
  1 | using System;
  2 | using System.Collections.Generic;
  3 | using System.IO;
  4 | using System.Linq;
  5 | using System.Text;
  6 | using System.Threading.Tasks;
  7 | 
  8 | namespace UnrealBuildTool.FBuild.BuildComponents
  9 | {
 10 |     public class Compiler : Linker
 11 |     {
 12 |         public Compiler(string exePath)
 13 |             : base(exePath)
 14 |         {
 15 |             LocaliseCompilerPath();
 16 |         }
 17 |         public override string InputFileRegex
 18 |         {
 19 |             get
 20 |             {                
 21 |                 return "(?<=( \")|(@\"))(.*?)(?=\")";
 22 |             }
 23 |         }
 24 |         public override string OutputFileRegex
 25 |         {
 26 |             get
 27 |             {
 28 |                 if (Type == CompilerTypes.Msvc || Type == CompilerTypes.Rc)
 29 |                 {
 30 |                     return "(?<=(/Fo \"|/Fo\"))(.*?)(?=\")";
 31 |                 }
 32 |                 else
 33 |                 {
 34 |                     return "(?<=(-o \"|-o\"))(.*?)(?=\")";
 35 |                 }
 36 |             }
 37 |         }
 38 |         public override string PchOutputRegex
 39 |         {
 40 |             get
 41 |             {
 42 |                 if (Type == CompilerTypes.Msvc || Type == CompilerTypes.Rc)
 43 |                 {
 44 |                     return "(?<=(/Fp \"|/Fp\"))(.*?)(?=\")";
 45 |                 }
 46 |                 return "(?<=(/Fp \"|/Fp\"))(.*?)(?=\")";
 47 |             }
 48 |         }
 49 |         public string Alias;
 50 | 
 51 |         public string GetBffArguments(string arguments)
 52 |         {
 53 |             var output = new StringBuilder();
 54 |             output.AppendFormat(" .CompilerOptions\t = '{0}'\r\n", arguments);
 55 |             return output.ToString();
 56 |         }
 57 |         public override string ToString()
 58 |         {
 59 |             var sb = new StringBuilder();
 60 |             sb.AppendFormat("Compiler('{0}')\n{{\n", Alias);
 61 |             sb.AppendFormat("\t.Executable\t\t            = '{0}' \n " +
 62 |                             "\t.ExtraFiles\t\t  = {{ {1} }}\n",
 63 |                             ExecPath,
 64 |                             string.Join("\n\t\t\t", GetExtraFiles().Select(e => "\t\t'" + e + "'")));
 65 | 
 66 |             sb.Append("}\n");
 67 |             return sb.ToString();
 68 |         }
 69 |         private IEnumerable<string> GetExtraFiles()
 70 |         {
 71 |             if (Type != CompilerTypes.Msvc) return Enumerable.Empty<string>();
 72 | 
 73 |             var output = new List<string>();
 74 | 
 75 |             var msvcVer = "";
 76 |             switch (WindowsPlatform.Compiler)
 77 |             {
 78 |                 case WindowsCompiler.VisualStudio2013:
 79 |                     msvcVer = "120";
 80 |                     break;
 81 |                 case WindowsCompiler.VisualStudio2015:
 82 |                     msvcVer = "140";
 83 |                     break;
 84 |             }
 85 | 
 86 |             var compilerDir = Path.GetDirectoryName(ExecPath);
 87 | 
 88 | 
 89 |             output.Add(compilerDir + "\\1033\\clui.dll");
 90 |             if (WindowsPlatform.Compiler == WindowsCompiler.VisualStudio2013)
 91 |             {
 92 |                 output.Add(compilerDir + "\\c1ast.dll");
 93 |                 output.Add(compilerDir + "\\c1xxast.dll");
 94 |             }
 95 |             output.Add(compilerDir + "\\c1xx.dll");
 96 |             output.Add(compilerDir + "\\c2.dll");
 97 |             output.Add(compilerDir + "\\c1.dll");
 98 |             
 99 |             output.Add(compilerDir + "\\msobj" + msvcVer + ".dll");
100 |             output.Add(compilerDir + "\\mspdb" + msvcVer + ".dll");
101 |             output.Add(compilerDir + "\\mspdbsrv.exe");
102 |             output.Add(compilerDir + "\\mspdbcore.dll");
103 |             output.Add(compilerDir + "\\mspft" + msvcVer + ".dll");
104 |             return output;
105 |         }
106 |         private void LocaliseCompilerPath()
107 |         {
108 |             var compilerPath = "";
109 |             if (ExecPath.Contains("cl.exe"))
110 |             {
111 |                 var compilerPathComponents = ExecPath.Replace('\\', '/').Split('/');
112 |                 var startIndex = Array.FindIndex(compilerPathComponents, row => row == "VC");
113 |                 if (startIndex > 0)
114 |                 {
115 |                     Type = CompilerTypes.Msvc;
116 |                     compilerPath = "$VSBasePath$";
117 |                     for (int i = startIndex + 1; i < compilerPathComponents.Length; ++i)
118 |                     {
119 |                         compilerPath += "/" + compilerPathComponents[i];
120 |                     }
121 |                 }
122 |                 ExecPath = compilerPath;
123 |             }
124 |             else if (ExecPath.Contains("rc.exe"))
125 |             {
126 |                 Type = CompilerTypes.Rc;
127 |                 var compilerPathComponents = ExecPath.Replace('\\', '/').Split('/');
128 |                 compilerPath = "$WindowsSDKBasePath$";
129 |                 var startIndex = Array.FindIndex(compilerPathComponents, row => row == "8.1");
130 |                 if (startIndex > 0)
131 |                 {
132 |                     for (var i = startIndex + 1; i < compilerPathComponents.Length; ++i)
133 |                     {
134 |                         compilerPath += "/" + compilerPathComponents[i];
135 |                     }
136 |                 }
137 |                 ExecPath = compilerPath;
138 |             }
139 |             else if (ExecPath.Contains("orbis-clang.exe"))
140 |             {
141 |                 Type = CompilerTypes.OrbisClang;
142 |             }
143 |         }
144 |     }
145 | }
146 | 


--------------------------------------------------------------------------------
/FBuild/BuildComponents/CompilerTypes.cs:
--------------------------------------------------------------------------------
 1 | using System;
 2 | using System.Collections.Generic;
 3 | using System.Linq;
 4 | using System.Text;
 5 | using System.Threading.Tasks;
 6 | 
 7 | namespace UnrealBuildTool.FBuild.BuildComponents
 8 | {
 9 |     public enum CompilerTypes
10 |     {
11 |         Msvc,
12 |         Rc,
13 |         Clang,
14 |         OrbisClang,
15 |         OrbisSnarl
16 |     }
17 | }
18 | 


--------------------------------------------------------------------------------
/FBuild/BuildComponents/ExecutableComponent.cs:
--------------------------------------------------------------------------------
 1 | using System;
 2 | using System.Collections.Generic;
 3 | using System.Linq;
 4 | using System.Text;
 5 | using System.Threading.Tasks;
 6 | 
 7 | namespace UnrealBuildTool.FBuild.BuildComponents
 8 | {
 9 |     public class ExecutableComponent : BuildComponent
10 |     {        
11 |         public Linker Linker;
12 | 
13 |         public string Arguments { get; set; }
14 | 
15 |         public ExecutableComponent()
16 |         {
17 |             NodeType = "Exec";
18 |         }
19 | 
20 |         public override string ToString()
21 |         {
22 |             //Carry on here. Need to strip input/output out, or change to not require in/out
23 |             var sb = new StringBuilder();
24 |             sb.AppendFormat("Exec('{0}')\n{{\n", Alias);
25 |             sb.AppendFormat("\t.ExecExecutable\t\t  = '{0}' \n " +
26 |                             "\t.ExecArguments\t\t  = '{1}'\n" +
27 |                             "\t.DoNotUseOutput = true\n" +
28 |                             "\t.ExecOutput\t\t = '{2}-{0}'\n",
29 |                             Linker.ExecPath,
30 |                             Arguments, Alias);
31 | 
32 |             if (Dependencies.Any())
33 |             {
34 |                 sb.AppendFormat("\t.PreBuildDependencies\t\t= {{ {0} }} \n ",
35 |                     string.Join("\n\t\t\t", Dependencies.Select(d => "'" + d.Alias + "'").Distinct()));
36 |             }
37 |             sb.Append("}\n");
38 | 
39 |             return sb.ToString();
40 |         }
41 |     }
42 | 
43 | }
44 | 


--------------------------------------------------------------------------------
/FBuild/BuildComponents/Linker.cs:
--------------------------------------------------------------------------------
  1 | using System;
  2 | using System.Collections.Generic;
  3 | using System.Linq;
  4 | using System.Text;
  5 | using System.Threading.Tasks;
  6 | 
  7 | namespace UnrealBuildTool.FBuild.BuildComponents
  8 | {
  9 |     public class Linker
 10 |     {
 11 |         public string ExecPath { get; set; }
 12 |         public CompilerTypes Type;
 13 | 
 14 |         private List<string> _allowedInputTypes;
 15 |         public virtual List<string> AllowedInputTypes
 16 |         {
 17 |             get
 18 |             {
 19 |                 if (_allowedInputTypes != null) return _allowedInputTypes;
 20 | 
 21 |                 switch (Type)
 22 |                 {
 23 |                     case CompilerTypes.Msvc:
 24 |                         _allowedInputTypes = new List<string>() { ".response", ".lib", ".obj" };
 25 |                         break;
 26 |                     case CompilerTypes.OrbisClang:
 27 |                     case CompilerTypes.OrbisSnarl:
 28 |                         _allowedInputTypes = new List<string>() { ".response", ".a" };
 29 |                         break;
 30 |                     case CompilerTypes.Rc:
 31 |                     case CompilerTypes.Clang:
 32 |                     default:
 33 |                         break;
 34 |                 }
 35 |                 return _allowedInputTypes;
 36 |             }
 37 |         }
 38 | 
 39 |         private List<string> _allowedOutputTypes;
 40 |         public virtual List<string> AllowedOutputTypes
 41 |         {
 42 |             get
 43 |             {
 44 |                 if (_allowedOutputTypes != null) return _allowedOutputTypes;
 45 | 
 46 |                 switch (Type)
 47 |                 {
 48 |                     case CompilerTypes.Msvc:
 49 |                         _allowedOutputTypes = new List<string>() { ".dll", ".lib", ".exe" };
 50 |                         break;
 51 |                     case CompilerTypes.OrbisClang:
 52 |                     case CompilerTypes.OrbisSnarl:
 53 |                         _allowedOutputTypes = new List<string>() { ".self", ".a", ".so" };
 54 |                         break;
 55 |                     case CompilerTypes.Rc:
 56 |                     case CompilerTypes.Clang:
 57 |                     default:
 58 |                         break;
 59 |                 }
 60 |                 return _allowedOutputTypes;
 61 |             }
 62 |         }
 63 | 
 64 |         public virtual string InputFileRegex
 65 |         {
 66 |             get
 67 |             {
 68 |                 return Type == CompilerTypes.OrbisClang ? "(?<=\")(.*?)(?=\")" : "(?<=@\")(.*?)(?=\")";
 69 |             }
 70 |         }
 71 |         public virtual string OutputFileRegex
 72 |         {
 73 |             get
 74 |             {
 75 |                 switch (Type)
 76 |                 {
 77 |                     case CompilerTypes.Msvc:
 78 |                     case CompilerTypes.Rc:
 79 |                         return "(?<=(/OUT: \"|/OUT:\"))(.*?)(?=\")";
 80 |                     case CompilerTypes.Clang:
 81 |                     case CompilerTypes.OrbisClang:
 82 |                         return "(?<=(-o \"|-o\"))(.*?)(?=\")";
 83 |                     case CompilerTypes.OrbisSnarl:
 84 |                         return "(?<=\")(.*?.a)(?=\")";
 85 |                     default:
 86 |                         break;
 87 |                 }
 88 |                 return "";
 89 |             }
 90 |         }
 91 |         public virtual string ImportLibraryRegex
 92 |         {
 93 |             get
 94 |             {
 95 |                 if (Type == CompilerTypes.Msvc || Type == CompilerTypes.Rc)
 96 |                 {
 97 |                     return "(?<=(/IMPLIB: \"|/IMPLIB:\"))(.*?)(?=\")";
 98 |                 }
 99 |                 return "";
100 |             }
101 |         }
102 |         public virtual string PchOutputRegex
103 |         {
104 |             get
105 |             {
106 |                 if (Type == CompilerTypes.Msvc || Type == CompilerTypes.Rc)
107 |                 {
108 |                     return "(?<=(/Fp \"|/Fp\"))(.*?)(?=\")";
109 |                 }
110 |                 else
111 |                 {
112 |                     return "(?<=(/Fp \"|/Fp\"))(.*?)(?=\")";
113 |                 }
114 |             }
115 |         }
116 | 
117 |         public Linker(string execPath)
118 |         {
119 |             ExecPath = execPath;
120 |             LocaliseLinkerPath();
121 |         }
122 | 
123 |         private void LocaliseLinkerPath()
124 |         {
125 |             var compilerPath = "";
126 |             if (ExecPath.Contains("link.exe") || ExecPath.Contains("lib.exe"))
127 |             {
128 |                 var compilerPathComponents = ExecPath.Replace('\\', '/').Split('/');
129 |                 var startIndex = Array.FindIndex(compilerPathComponents, row => row == "VC");
130 |                 if (startIndex > 0)
131 |                 {
132 |                     Type = CompilerTypes.Msvc;
133 |                     compilerPath = "$VSBasePath$";
134 |                     for (var i = startIndex + 1; i < compilerPathComponents.Length; ++i)
135 |                     {
136 |                         compilerPath += "/" + compilerPathComponents[i];
137 |                     }
138 |                 }
139 |                 ExecPath = compilerPath;
140 |             }
141 |             else if (ExecPath.Contains("orbis-snarl.exe"))
142 |             {
143 |                 Type = CompilerTypes.OrbisSnarl;
144 |             }
145 |             else if (ExecPath.Contains("orbis-clang.exe"))
146 |             {
147 |                 Type = CompilerTypes.OrbisClang;
148 |             }
149 |         }
150 | 
151 |         public static bool IsKnownLinker(string args)
152 |         {
153 |             return args.Contains("lib.exe") || args.Contains("link.exe") || args.Contains("orbis-clang.exe") || args.Contains("orbis-snarl.exe");
154 |         }
155 |     }
156 | 
157 | }
158 | 


--------------------------------------------------------------------------------
/FBuild/BuildComponents/LinkerComponent.cs:
--------------------------------------------------------------------------------
  1 | using System;
  2 | using System.Collections.Generic;
  3 | using System.IO;
  4 | using System.Linq;
  5 | using System.Text;
  6 | using System.Threading.Tasks;
  7 | 
  8 | namespace UnrealBuildTool.FBuild.BuildComponents
  9 | {
 10 |     public enum LinkerType
 11 |     {
 12 |         Static,
 13 |         Dynamic
 14 |     }
 15 | 
 16 |     public class LinkerComponent : ExecutableComponent
 17 |     {
 18 |         public string OutputLibrary { get; set; }
 19 | 
 20 |         public string ImportLibrary { get; set; }
 21 | 
 22 |         public List<string> Inputs { get; set; }
 23 | 
 24 |         public bool LocalOnly { get; set; }
 25 | 
 26 |         public LinkerType LinkerType
 27 |         {
 28 |             get
 29 |             {
 30 |                 return OutputLibrary.EndsWith(".lib") ? LinkerType.Static : LinkerType.Dynamic;
 31 |             }
 32 |         }
 33 | 
 34 |         public LinkerComponent()
 35 |         {
 36 |             NodeType = "DLL";
 37 |         }
 38 | 
 39 | 
 40 |         public override string ToString()
 41 |         {
 42 |             return LinkerType == LinkerType.Static ? ToLibString() : ToDllString();
 43 |         }
 44 | 
 45 |         private string ToLibString()
 46 |         {
 47 |             var linkerArgs = Arguments;
 48 |             
 49 |             List<string> libraryInputs = null;
 50 |             List<string> systemInputs = null;
 51 |             
 52 |             if (Inputs.Count == 1)
 53 |             {
 54 |                 var responseFile = Inputs[0];
 55 |                 linkerArgs = linkerArgs.Replace(Inputs[0], "%1");
 56 |                 linkerArgs = linkerArgs.Replace(OutputLibrary, "%2");
 57 |                 GetResponseFileInputs(ref responseFile, FASTBuildConfiguration.UseSinglePassCompilation ? FASTBuild.AllActions : FASTBuild.LinkerActions, out libraryInputs, out systemInputs);
 58 |                                 
 59 |                 //If we found resolvable references in the response file, use those as the inputs and re-add the response file.
 60 |                 //If not, use the response file itself just to stop fbuild erroring.
 61 |                 if (libraryInputs.Any())
 62 |                 {
 63 |                     linkerArgs = linkerArgs.Replace("%1", responseFile + "\" \"%1");
 64 |                 }
 65 |                 else
 66 |                 {
 67 |                     libraryInputs.Add(responseFile);
 68 |                 }
 69 |             }
 70 |             else
 71 |             {
 72 |                 throw new Exception("Don't know what to do here yet");
 73 |             }
 74 | 
 75 | 
 76 |             var sb = new StringBuilder();
 77 |             sb.AppendFormat("Library('{0}')\n{{\n", Alias);
 78 | 
 79 |             //We do not provide file inputs (we use separate ObjectLists) for simplicity, so we do not need to specify an actually defined compiler.
 80 |             sb.AppendFormat("\t.Compiler\t\t = 'Null'\n"); 
 81 |             sb.AppendFormat("\t.CompilerOptions\t\t = '%1 %2 %3'\n");
 82 |             sb.AppendFormat("\t.CompilerOutputPath\t\t = ' '\n");
 83 |             sb.AppendFormat("\t.Librarian\t\t            = '{0}' \n " +
 84 |                             "\t.LibrarianOutput\t\t  =  '{1}' \n",
 85 |                             Linker.ExecPath,
 86 |                             OutputLibrary);
 87 |             sb.AppendFormat("\t.LibrarianOptions\t\t = '{0}'\n", linkerArgs);
 88 |             if (systemInputs.Any())
 89 |             {
 90 |                 sb.AppendFormat("\t.LibrarianOptions\t\t   {0} \n ",
 91 |                 string.Join("\n\t\t\t", systemInputs.Select(d => "+ ' " + d + "'")));
 92 |             }
 93 |             if(libraryInputs.Any())
 94 |             {
 95 |                 sb.AppendFormat("\t.LibrarianAdditionalInputs\t\t   = {{ {0} }} \n ",
 96 |                                     string.Join("\n\t\t\t", libraryInputs.Select(d => "'" + d + "'")));
 97 |             }
 98 | 
 99 |             sb.Append("}\n");
100 | 
101 |             return sb.ToString();
102 |         }
103 |         
104 |         private string ToDllString()
105 |         {
106 |             var linkerArgs = Arguments;
107 | 
108 |             List<string> libraryInputs = null;
109 |             List<string> systemInputs = null;
110 |             
111 |             if (Inputs.Count == 1)
112 |             {
113 |                 var responseFile = Inputs[0];
114 |                 linkerArgs = linkerArgs.Replace(Inputs[0], "%1");
115 |                 linkerArgs = linkerArgs.Replace(OutputLibrary, "%2");
116 |                 GetResponseFileInputs(ref responseFile, FASTBuildConfiguration.UseSinglePassCompilation ? FASTBuild.AllActions : FASTBuild.LinkerActions, out libraryInputs, out systemInputs);
117 | 
118 |                 //If we found resolvable references in the response file, use those as the inputs and re-add the response file.
119 |                 //If not, use the response file itself just to stop fbuild erroring.
120 |                 if (libraryInputs.Any())
121 |                 {
122 |                     linkerArgs = linkerArgs.Replace("%1", responseFile + "\" \"%1");
123 |                 }
124 |                 else
125 |                 {
126 |                     libraryInputs.Add(responseFile);
127 |                 }
128 |             }
129 |             else
130 |             {
131 |                 throw new Exception("Don't know what to do here yet");
132 |             }
133 |             
134 |             linkerArgs = linkerArgs.Replace(OutputLibrary, "%2");
135 |             
136 |             var sb = new StringBuilder();
137 |             sb.AppendFormat("DLL('{0}')\n{{\n", Alias);
138 |             sb.AppendFormat("\t.LinkerLinkObjects\t\t = false\n");
139 |             sb.AppendFormat("\t.Linker\t\t            = '{0}' \n " +
140 |                             "\t.LinkerOutput\t\t  =  '{1}' \n",
141 |                             Linker.ExecPath,
142 |                             OutputLibrary);
143 |             sb.AppendFormat("\t.LinkerOptions\t\t = '{0}'\n", linkerArgs);
144 |             sb.AppendFormat("\t.Libraries\t\t   = {{ {0} }} \n ",
145 |                                     string.Join("\n\t\t\t", libraryInputs.Select(d => "'" + d + "'")));
146 | 
147 | 
148 |             sb.Append("}\n");
149 | 
150 |             return sb.ToString();
151 |         }
152 | 
153 |         private void GetResponseFileInputs(ref string file, List<BuildComponent> resolvableDependencies, out List<string> inputs, out List<string> systemInputs)
154 |         {
155 |             var lines = File.ReadAllLines(file);
156 | 
157 |             inputs = new List<string>();
158 |             systemInputs = new List<string>();
159 |             var unresolvedLines = lines.Select(n => n).ToList();
160 |             
161 |             foreach (var line in lines)
162 |             {
163 |                 // Strip the additional quotes from the response file
164 |                 var resolvedLine = line.Replace("\"", "");                
165 | 
166 |                 // UE4 Provides response outputs to project files as absolute paths.
167 |                 // A quick way to check if something is a system include is whether it is a rooted path or not
168 |                 if (!Path.IsPathRooted(resolvedLine)) continue;
169 | 
170 |                 // We should resolve project includes to see if we're building the node for that this pass as well
171 |                 BuildComponent matchingDependency = null;
172 | 
173 |                 foreach(var dependency in resolvableDependencies)
174 |                 {
175 |                     if(dependency is LinkerComponent)
176 |                     {
177 |                         var linkDependency = (LinkerComponent)dependency;
178 |                         if(linkDependency.ImportLibrary == resolvedLine || linkDependency.OutputLibrary == resolvedLine)
179 |                         {
180 |                             matchingDependency = dependency;
181 |                             break;
182 |                         }
183 |                     }
184 |                     else if(dependency is ObjectGroupComponent)
185 |                     {
186 |                         var objectDependency = (ObjectGroupComponent)dependency;
187 |                         if (objectDependency.ActionOutputs.Contains(resolvedLine) ||
188 |                             (objectDependency.PchOptions != null && 
189 |                              Path.ChangeExtension(objectDependency.PchOptions.Output, objectDependency.OutputExt) == resolvedLine))
190 |                         {
191 |                             matchingDependency = dependency;
192 |                             break;
193 |                         }
194 |                     }
195 |                 }
196 | 
197 |                 //if (FASTBuildConfiguration.UseSinglePassCompilation || matchingDependency != null)
198 |                 {
199 |                     unresolvedLines.Remove(line);
200 |                     if (matchingDependency != null)
201 |                     {
202 |                         resolvedLine = matchingDependency.Alias;
203 |                     }
204 |                     inputs.Add(resolvedLine);
205 |                 }
206 |             }
207 | 
208 |             file += ".fbuild";
209 |             File.WriteAllLines(file, unresolvedLines);
210 |         }
211 |     }
212 | }
213 | 


--------------------------------------------------------------------------------
/FBuild/BuildComponents/ObjectGroupComponent.cs:
--------------------------------------------------------------------------------
 1 | using System;
 2 | using System.Collections.Generic;
 3 | using System.IO;
 4 | using System.Linq;
 5 | using System.Text;
 6 | using System.Threading.Tasks;
 7 | 
 8 | namespace UnrealBuildTool.FBuild.BuildComponents
 9 | {
10 |     public class ObjectGroupComponent : BuildComponent
11 |     {        
12 |         public Dictionary<Action, string> ActionInputs { get; set; }
13 |         public IEnumerable<string> ActionOutputs
14 |         {
15 |             get
16 |             {
17 |                 return ActionInputs.Select(n => OutputPath + Path.GetFileNameWithoutExtension(n.Value) + OutputExt);
18 |             }
19 |         }
20 | 
21 |         public string MatchHash { get; set; }
22 | 
23 |         public string CompilerArguments { get; set; }
24 | 
25 |         public string OutputPath { get; set; }
26 | 
27 |         public string OutputExt { get; set; }
28 | 
29 |         public Compiler ObjectCompiler;
30 | 
31 |         public PchOptions PchOptions { get; set; }
32 | 
33 |         public bool LocalOnly { get; set; }
34 |         
35 |         public ObjectGroupComponent()
36 |         {
37 |             ActionInputs = new Dictionary<Action, string>();
38 |             NodeType = "ObjG";
39 |         }
40 | 
41 |         public override string ToString()
42 |         {
43 |             var sb = new StringBuilder();
44 | 
45 |             sb.AppendFormat("ObjectList('{0}')\n{{\n", Alias);
46 | 
47 |             sb.AppendFormat("\t.Compiler\t\t            = '{0}' \n " +
48 |                             "\t.CompilerInputFiles\t\t  = {{ {1} }}\n" +
49 |                             "\t.CompilerOutputPath\t\t  = '{2}'\n" +
50 |                             "\t.CompilerOutputExtension\t\t  = '{3}'\n" +
51 |                             "\t.CompilerOptions\t = '{4}'\n",
52 |                             ObjectCompiler.Alias,
53 |                             string.Join("\n\t\t\t", ActionInputs.Select(a => "'" + a.Value + "'")),
54 |                             OutputPath,
55 |                             OutputExt,
56 |                             CompilerArguments);
57 |             if (PchOptions != null)
58 |             {
59 |                 sb.AppendFormat("\t.PCHInputFile\t\t          = '{0}' \n " +
60 |                                 "\t.PCHOutputFile\t\t         = '{1}' \n " +
61 |                                 "\t.PCHOptions\t\t            = '{2}' \n ",
62 |                                 PchOptions.Input,
63 |                                 PchOptions.Output,
64 |                                 PchOptions.Options);
65 |             }
66 | 
67 |             var dependencyString = "";
68 |             if (Dependencies.Any())
69 |             {                
70 |                 foreach(var dependency in Dependencies)
71 |                 {
72 |                     dependencyString += "\n\t\t\t '" + dependency.Alias + "'";
73 |                     //if(dependency is ObjectGroupComponent)
74 |                     //{
75 |                     //    if(((ObjectGroupComponent)dependency).PchOptions != null)
76 |                     //    {
77 |                     //        //dependencyString += "\n\t\t\t '" + dependency.Alias + "-PCH'";
78 |                     //    }
79 |                     //}
80 |                 }
81 |             }
82 | 
83 |             if(!string.IsNullOrEmpty(dependencyString))
84 |             {
85 |                 sb.AppendFormat("\t.PreBuildDependencies\t\t   = {{ {0} }} \n ",
86 |                     dependencyString);
87 |             }
88 | 
89 |             sb.Append("}\n");
90 |             
91 |             return sb.ToString();
92 |         }
93 |     }
94 | 
95 | }
96 | 


--------------------------------------------------------------------------------
/FBuild/BuildComponents/PchOptions.cs:
--------------------------------------------------------------------------------
 1 | using System;
 2 | using System.Collections.Generic;
 3 | using System.Linq;
 4 | using System.Text;
 5 | using System.Threading.Tasks;
 6 | 
 7 | namespace UnrealBuildTool.FBuild.BuildComponents
 8 | {
 9 |     public class PchOptions
10 |     {
11 |         public string Options;
12 |         public string Input;
13 |         public string Output;
14 |     }
15 | 
16 | }
17 | 


--------------------------------------------------------------------------------
/FBuild/FASTBuildCommon.cs:
--------------------------------------------------------------------------------
 1 | using System;
 2 | using System.Collections.Generic;
 3 | using System.Linq;
 4 | using System.Text;
 5 | using System.Threading.Tasks;
 6 | 
 7 | namespace UnrealBuildTool.FBuild
 8 | {
 9 |     public static class FASTBuildCommon
10 |     {
11 |         public static string EntryArguments
12 |         {
13 |             get
14 |             {
15 |                 var entryArguments = "";
16 |                 switch (WindowsPlatform.Compiler)
17 |                 {
18 |                     case WindowsCompiler.VisualStudio2013:
19 |                         entryArguments += ".VSBasePath     	        = '../Extras/FASTBuild/External/VS13.4/VC'\r\n";
20 |                         break;
21 |                     case WindowsCompiler.VisualStudio2015:
22 |                         entryArguments += ".VSBasePath     	        = '../Extras/FASTBuild/External/VS15.0/VC'\r\n";
23 |                         break;
24 |                 }
25 |                 entryArguments += ".ClangBasePath			= '../Extras/FASTBuild/External/ClangForWindows/3.6'\r\n";
26 |                 entryArguments += ".WindowsSDKBasePath      = 'C:/Program Files (x86)/Windows Kits/8.1'\r\n";
27 |                 
28 |                 entryArguments += ";-------------------------------------------------------------------------------\r\n";
29 |                 entryArguments += "; Settings\r\n";
30 |                 entryArguments += ";-------------------------------------------------------------------------------\r\n";
31 |                 entryArguments += "Settings\r\n";
32 |                 entryArguments += "{\r\n";
33 |                 entryArguments += "    .CachePath = '" + FASTBuildConfiguration.CachePath + "'\r\n";
34 |                 entryArguments += "}\r\n";
35 | 
36 |                 return entryArguments;
37 |             }
38 |         }
39 |         public static string GetAliasTag(string alias, List<string> targets)
40 |         {
41 |             var output = "";
42 |             var targetCount = targets.Count;
43 |             output += "Alias( '" + alias + "' )\r\n";
44 |             output += "{\r\n";
45 |             output += " .Targets = {";
46 |             for (var i = 0; i < targetCount; ++i)
47 |             {
48 |                 output += "'" + targets[i] + "'";
49 |                 if (i < targetCount - 1)
50 |                 {
51 |                     output += ",";
52 |                 }
53 |             }
54 |             output += " }\r\n";
55 |             output += "}\r\n";
56 |             return output;
57 | 
58 |         }
59 |     }
60 | }
61 | 


--------------------------------------------------------------------------------
/FBuild/FASTBuildConfiguration.cs:
--------------------------------------------------------------------------------
 1 | using System;
 2 | using System.Collections.Generic;
 3 | using System.Linq;
 4 | using System.Text;
 5 | using System.Threading.Tasks;
 6 | 
 7 | namespace UnrealBuildTool.FBuild
 8 | {
 9 |     public static class FASTBuildConfiguration
10 |     {
11 |         public static bool UseCache => true;
12 | 
13 |         public static bool IsDistrbuted => true;
14 | 
15 |         public static bool UseSinglePassCompilation => false;
16 | 
17 |         public static string AdditionalArguments => "";
18 | 
19 |         public static string CachePath => "";
20 | 
21 |         public static bool EnableCacheGenerationMode
22 |         {
23 |             get
24 |             {
25 |                 var writeFastBuildString = Environment.GetEnvironmentVariable("UE-FB-CACHE-WRITE");
26 |                 if (writeFastBuildString == null) return false;
27 | 
28 |                 return string.Compare("TRUE", writeFastBuildString, StringComparison.OrdinalIgnoreCase) == 0;
29 |             }
30 |         }
31 |     }
32 | }
33 | 


--------------------------------------------------------------------------------
/FastBuild.cs:
--------------------------------------------------------------------------------
  1 | using System;
  2 | using System.Collections.Generic;
  3 | using System.IO;
  4 | using System.Diagnostics;
  5 | using System.Xml;
  6 | using System.Text.RegularExpressions;
  7 | using System.Linq;
  8 | using System.Text;
  9 | using UnrealBuildTool.FBuild.BuildComponents;
 10 | using UnrealBuildTool.FBuild;
 11 | 
 12 | namespace UnrealBuildTool
 13 | {
 14 |     public class FASTBuild
 15 |     {     
 16 |         private static int MaxActionsToExecuteInParallel
 17 |         {
 18 |             get
 19 |             {
 20 |                 return 0;
 21 |             }
 22 |         }
 23 |         private static int JobNumber { get; set; }
 24 |         private static int AliasBase { get; set; } = 1;
 25 | 
 26 |         /** The possible result of executing tasks with SN-DBS. */
 27 |         public enum ExecutionResult
 28 |         {
 29 |             Unavailable,
 30 |             TasksFailed,
 31 |             TasksSucceeded,
 32 |         }
 33 |         private enum BuildStep
 34 |         {
 35 |             CompileObjects,
 36 |             Link,
 37 |             CompileAndLink
 38 |         }
 39 |         
 40 |                 
 41 |         private static readonly Dictionary<string, Compiler> Compilers = new Dictionary<string, Compiler>();
 42 |         private static readonly Dictionary<string, Linker> Linkers = new Dictionary<string, Linker>();
 43 | 
 44 |         /**
 45 |          * Used when debugging Actions outputs all action return values to debug out
 46 |          * 
 47 |          * @param	sender		Sending object
 48 |          * @param	e			Event arguments (In this case, the line of string output)
 49 |          */
 50 |         protected static void ActionDebugOutput(object sender, DataReceivedEventArgs e)
 51 |         {
 52 |             var Output = e.Data;
 53 |             if (Output == null)
 54 |             {
 55 |                 return;
 56 |             }
 57 | 
 58 |             Log.TraceInformation(Output);
 59 |         }
 60 | 
 61 |         internal static ExecutionResult ExecuteLocalActions(List<Action> InLocalActions, Dictionary<Action, ActionThread> InActionThreadDictionary, int TotalNumJobs)
 62 |         {
 63 |             // Time to sleep after each iteration of the loop in order to not busy wait.
 64 |             const float loopSleepTime = 0.1f;
 65 | 
 66 |             ExecutionResult localActionsResult = ExecutionResult.TasksSucceeded;
 67 | 
 68 |             while (true)
 69 |             {
 70 |                 // Count the number of pending and still executing actions.
 71 |                 int NumUnexecutedActions = 0;
 72 |                 int NumExecutingActions = 0;
 73 |                 foreach (Action Action in InLocalActions)
 74 |                 {
 75 |                     ActionThread ActionThread = null;
 76 |                     bool bFoundActionProcess = InActionThreadDictionary.TryGetValue(Action, out ActionThread);
 77 |                     if (bFoundActionProcess == false)
 78 |                     {
 79 |                         NumUnexecutedActions++;
 80 |                     }
 81 |                     else if (ActionThread != null)
 82 |                     {
 83 |                         if (ActionThread.bComplete == false)
 84 |                         {
 85 |                             NumUnexecutedActions++;
 86 |                             NumExecutingActions++;
 87 |                         }
 88 |                     }
 89 |                 }
 90 | 
 91 |                 // If there aren't any pending actions left, we're done executing.
 92 |                 if (NumUnexecutedActions == 0)
 93 |                 {
 94 |                     break;
 95 |                 }
 96 | 
 97 |                 // If there are fewer actions executing than the maximum, look for pending actions that don't have any outdated
 98 |                 // prerequisites.
 99 |                 foreach (Action Action in InLocalActions)
100 |                 {
101 |                     ActionThread ActionProcess = null;
102 |                     bool bFoundActionProcess = InActionThreadDictionary.TryGetValue(Action, out ActionProcess);
103 |                     if (bFoundActionProcess == false)
104 |                     {
105 |                         if (NumExecutingActions < Math.Max(1, MaxActionsToExecuteInParallel))
106 |                         {
107 |                             // Determine whether there are any prerequisites of the action that are outdated.
108 |                             bool bHasOutdatedPrerequisites = false;
109 |                             bool bHasFailedPrerequisites = false;
110 |                             foreach (FileItem PrerequisiteItem in Action.PrerequisiteItems)
111 |                             {
112 |                                 if (PrerequisiteItem.ProducingAction != null && InLocalActions.Contains(PrerequisiteItem.ProducingAction))
113 |                                 {
114 |                                     ActionThread PrerequisiteProcess = null;
115 |                                     bool bFoundPrerequisiteProcess = InActionThreadDictionary.TryGetValue(PrerequisiteItem.ProducingAction, out PrerequisiteProcess);
116 |                                     if (bFoundPrerequisiteProcess == true)
117 |                                     {
118 |                                         if (PrerequisiteProcess == null)
119 |                                         {
120 |                                             bHasFailedPrerequisites = true;
121 |                                         }
122 |                                         else if (PrerequisiteProcess.bComplete == false)
123 |                                         {
124 |                                             bHasOutdatedPrerequisites = true;
125 |                                         }
126 |                                         else if (PrerequisiteProcess.ExitCode != 0)
127 |                                         {
128 |                                             bHasFailedPrerequisites = true;
129 |                                         }
130 |                                     }
131 |                                     else
132 |                                     {
133 |                                         bHasOutdatedPrerequisites = true;
134 |                                     }
135 |                                 }
136 |                             }
137 | 
138 |                             // If there are any failed prerequisites of this action, don't execute it.
139 |                             if (bHasFailedPrerequisites)
140 |                             {
141 |                                 // Add a null entry in the dictionary for this action.
142 |                                 InActionThreadDictionary.Add(Action, null);
143 |                             }
144 |                             // If there aren't any outdated prerequisites of this action, execute it.
145 |                             else if (!bHasOutdatedPrerequisites)
146 |                             {
147 |                                 ActionThread ActionThread = new ActionThread(Action, JobNumber, TotalNumJobs);
148 |                                 ActionThread.Run();
149 | 
150 |                                 InActionThreadDictionary.Add(Action, ActionThread);
151 | 
152 |                                 NumExecutingActions++;
153 |                                 JobNumber++;
154 |                             }
155 |                         }
156 |                     }
157 |                 }
158 | 
159 |                 System.Threading.Thread.Sleep(TimeSpan.FromSeconds(loopSleepTime));
160 |             }
161 | 
162 |             return localActionsResult;
163 |         }
164 | 
165 |         public static List<BuildComponent> CompilationActions { get; set; }
166 |         public static List<BuildComponent> LinkerActions { get; set; }
167 |         public static List<BuildComponent> AllActions => CompilationActions.Concat(LinkerActions).ToList();
168 | 
169 |         private static void DeresponsifyActions(List<Action> actions)
170 |         {
171 |             // UE4.13 started to shove the entire argument into response files. This does not work
172 |             // well with FASTBuild so we'll have to undo it.
173 |             foreach(var action in actions)
174 |             {
175 |                 if (!action.CommandArguments.StartsWith(" @\"") || !action.CommandArguments.EndsWith("\"") ||
176 |                     action.CommandArguments.Count(f => f == '"') != 2) continue;
177 | 
178 |                 var file = Regex.Match(action.CommandArguments, "(?<=@\")(.*?)(?=\")");
179 |                 if (!file.Success) continue;
180 | 
181 |                 var arg = File.ReadAllText(file.Value);
182 |                 action.CommandArguments = arg;
183 |             }
184 |         }
185 | 
186 |         public static ExecutionResult ExecuteActions(List<Action> actions)
187 |         {
188 |             ExecutionResult fastBuildResult = ExecutionResult.TasksSucceeded;
189 |             CompilationActions = null;
190 |             LinkerActions = null;
191 | 
192 |             if (actions.Count <= 0)
193 |             {
194 |                 return fastBuildResult;
195 |             }
196 | 
197 |             if (IsAvailable() == false)
198 |             {
199 |                 return ExecutionResult.Unavailable;
200 |             }
201 | 
202 |             List<Action> unassignedObjects = new List<Action>();
203 |             List<Action> unassignedLinks = new List<Action>();
204 |             List<Action> miscActions = actions.Where(a => a.ActionType != ActionType.Compile && a.ActionType != ActionType.Link).ToList();
205 |             Dictionary<Action, BuildComponent> fastbuildActions = new Dictionary<Action, BuildComponent>();
206 | 
207 |             DeresponsifyActions(actions);
208 | 
209 |             CompilationActions = GatherActionsObjects(actions.Where(a => a.ActionType == ActionType.Compile), ref unassignedObjects, ref fastbuildActions);
210 |             LinkerActions = GatherActionsLink(actions.Where(a => a.ActionType == ActionType.Link), ref unassignedLinks, ref fastbuildActions);
211 |             ResolveDependencies(CompilationActions, LinkerActions, fastbuildActions);
212 | 
213 |             Log.TraceInformation("Actions: "+actions.Count+" - Unassigned: "+unassignedObjects.Count);
214 |             Log.TraceInformation("Misc Actions - FBuild: "+miscActions.Count);
215 |             Log.TraceInformation("Objects - FBuild: "+ CompilationActions.Count+" -- Local: "+unassignedObjects.Count);
216 |             var lg = 0;
217 |             if (LinkerActions != null) lg = LinkerActions.Count;
218 |             Log.TraceInformation("Link - FBuild: " + lg + " -- Local: " + unassignedLinks.Count);
219 | 
220 |             if (unassignedLinks.Count > 0)
221 |             {
222 |                 throw new Exception("Error, unaccounted for lib! Cannot guarantee there will be no prerequisite issues. Fix it");
223 |             }
224 |             if (unassignedObjects.Count > 0)
225 |             {
226 |                 var actionThreadDictionary = new Dictionary<Action, ActionThread>();
227 |                 ExecuteLocalActions(unassignedObjects, actionThreadDictionary, unassignedObjects.Count);
228 |             }
229 | 
230 |             if (FASTBuildConfiguration.UseSinglePassCompilation)
231 |             {
232 |                 RunFBuild(BuildStep.CompileAndLink, GenerateFBuildFileString(BuildStep.CompileAndLink, CompilationActions.Concat(LinkerActions).ToList()));
233 |             }
234 |             else
235 |             {
236 |                 if (CompilationActions.Any())
237 |                 {
238 |                     fastBuildResult = RunFBuild(BuildStep.CompileObjects, GenerateFBuildFileString(BuildStep.CompileObjects, CompilationActions));
239 |                 }
240 |                 if (fastBuildResult == ExecutionResult.TasksSucceeded)
241 |                 {
242 |                     if (LinkerActions != null && LinkerActions.Any())
243 |                     {
244 |                         if (!BuildConfiguration.bFastbuildNoLinking)
245 |                         {
246 |                             fastBuildResult = RunFBuild(BuildStep.Link, GenerateFBuildFileString(BuildStep.Link, LinkerActions));
247 |                         }
248 |                     }
249 |                 }
250 |             }
251 |             return fastBuildResult;
252 |         }
253 | 
254 |         private static void ResolveDependencies(List<BuildComponent> objectGroups, List<BuildComponent> linkGroups, Dictionary<Action, BuildComponent> fastbuildActions)
255 |         {
256 |             var actions = objectGroups.Concat(linkGroups);
257 | 
258 |             foreach (var action in fastbuildActions)
259 |             {
260 |                 var prerequisites = action.Key.PrerequisiteItems
261 |                     .Where(n => n.ProducingAction != null && (n.ProducingAction.ActionType == action.Key.ActionType));
262 | 
263 |                 foreach (var prerequisite in prerequisites)
264 |                 {
265 |                     if (fastbuildActions.ContainsKey(prerequisite.ProducingAction))
266 |                     {
267 |                         if (!action.Value.Dependencies.Contains(fastbuildActions[prerequisite.ProducingAction]))
268 |                         {
269 |                             action.Value.Dependencies.Add(fastbuildActions[prerequisite.ProducingAction]);
270 |                         }
271 |                     }
272 |                 }
273 |             }
274 |         }
275 | 
276 |         private static string GenerateFBuildFileString(BuildStep step, List<BuildComponent> actions)
277 |         {
278 |             var sb = new StringBuilder();
279 |             sb.Append(FASTBuildCommon.EntryArguments);
280 |             if (step == BuildStep.CompileObjects || step == BuildStep.CompileAndLink)
281 |             {
282 |                 sb.Append(GenerateFBuildCompilers());
283 |             }
284 | 
285 |             if (step == BuildStep.CompileAndLink)
286 |             {
287 |                 sb.Append(GenerateFBuildNodeList("DLLListAlias", actions));
288 |                 sb.Append(GenerateFBuildTargets(false, actions.Any()));
289 |             }
290 |             else
291 |             {
292 |                 if (actions.Count > 0)
293 |                 {
294 |                     sb.Append(GenerateFBuildNodeList(step == BuildStep.CompileObjects ? "ObjectsListsAlias" : "DLLListAlias", actions));
295 |                 }
296 |                 sb.Append(GenerateFBuildTargets(step == BuildStep.CompileObjects && actions.Any(),
297 |                                                         step == BuildStep.Link && actions.Any()));
298 |             }
299 |             return sb.ToString();
300 |         }
301 | 
302 |         private static string GenerateFBuildCompilers()
303 |         {
304 |             StringBuilder sb = new StringBuilder();
305 | 
306 |             int objectCount = 0;
307 |             foreach (var compiler in Compilers)
308 |             {
309 |                 compiler.Value.Alias = "Compiler-" + objectCount;
310 |                 sb.Append(compiler.Value.ToString());
311 |                 objectCount++;
312 |             }
313 | 
314 |             return sb.ToString();
315 |         }
316 | 
317 |         private static string GenerateFBuildNodeList(string aliasName, IList<BuildComponent> objectGroups)
318 |         {
319 |             var sb = new StringBuilder();
320 |             var aliases = new List<string>();
321 | 
322 |             // Ensure nodes are placed before nodes which depend on them.
323 |             bool changed;
324 |             do
325 |             {
326 |                 changed = false;
327 |                 for (var i = 0; i < objectGroups.Count; ++i)
328 |                 {
329 |                     if (!objectGroups[i].Dependencies.Any()) continue;
330 | 
331 |                     var highest = objectGroups[i].Dependencies.Select(objectGroups.IndexOf).OrderBy(n => n).Last();
332 |                     if (highest > i)
333 |                     {
334 |                         var thisObject = objectGroups[i];
335 |                         objectGroups.RemoveAt(i);
336 |                         objectGroups.Insert(highest, thisObject);
337 |                         changed = true;
338 |                     }
339 |                 }
340 |             } while (changed);
341 | 
342 |             foreach (var objectGroup in objectGroups)
343 |             {
344 |                 aliases.Add(objectGroup.Alias);
345 |                 sb.Append(objectGroup.ToString());
346 |             }
347 | 
348 |             if (aliases.Any())
349 |             {
350 |                 sb.Append(FASTBuildCommon.GetAliasTag(aliasName, aliases));
351 |             }
352 |             return sb.ToString();
353 |         }
354 | 
355 |         private static string GenerateFBuildTargets(bool anyObjects, bool anyLibs)
356 |         {
357 |             var aliases = new List<string>();
358 |             if (anyObjects)
359 |             {
360 |                 aliases.Add("ObjectsListsAlias");
361 |             }
362 |             if (anyLibs)
363 |             {
364 |                 aliases.Add("DLLListAlias");
365 |             }
366 |             if (anyObjects || anyLibs)
367 |             {
368 |                 return FASTBuildCommon.GetAliasTag("all", aliases);
369 |             }
370 |             else
371 |             {
372 |                 return "";
373 |             }
374 |         }
375 |         
376 |         private static List<BuildComponent> GatherActionsObjects(IEnumerable<Action> compileActions, ref List<Action> unassignedActions, ref Dictionary<Action, BuildComponent> actionLinks)
377 |         {
378 |             var objectGroup = new List<ObjectGroupComponent>();
379 |             
380 |             foreach (var action in compileActions)
381 |             {
382 |                 var obj = ParseCompilerAction(action);
383 |                 if (obj != null)
384 |                 {
385 |                     var group = objectGroup.FirstOrDefault(n => n.MatchHash == obj.MatchHash);
386 |                     if (group != null && !FASTBuildConfiguration.UseSinglePassCompilation)
387 |                     {
388 |                         group.ActionInputs[action] = obj.ActionInputs.FirstOrDefault().Value;
389 |                     }
390 |                     else
391 |                     {
392 |                         obj.AliasIndex = AliasBase++;
393 |                         objectGroup.Add(obj);
394 |                     }
395 |                     actionLinks[action] = obj;
396 |                 }
397 |                 else
398 |                 {
399 |                     Log.TraceInformation("Local Action - \n-Path:"+action.CommandPath+"\n-Args:"+action.CommandArguments);
400 |                     unassignedActions.Add(action);
401 |                 }
402 |             }
403 |             return objectGroup.Cast<BuildComponent>().ToList();
404 |         }
405 | 
406 |         private static List<BuildComponent> GatherActionsLink(IEnumerable<Action> localLinkActions, ref List<Action> unassignedActions, ref Dictionary<Action, BuildComponent> actionLinks)
407 |         {
408 |             var linkActions = new List<ExecutableComponent>();
409 | 
410 |             foreach (var action in localLinkActions)
411 |             {
412 |                 var linkAction = ParseLinkAction(action);
413 |                 if (linkAction != null)
414 |                 {
415 |                     linkActions.Add(linkAction);
416 |                     linkAction.AliasIndex = AliasBase++;
417 |                     actionLinks[action] = linkAction;
418 |                 }
419 |                 else
420 |                 {
421 |                     Log.TraceInformation("Local Action - \n-Path:"+action.CommandPath+"\n-Args:"+action.CommandArguments);
422 |                     unassignedActions.Add(action);
423 |                 }
424 |             }
425 |             return linkActions.Cast<BuildComponent>().ToList();
426 |         }
427 | 
428 |         private static string LocaliseFilePath(string filepath)
429 |         {
430 |             Uri inputUri = new Uri(filepath);
431 |             Uri currentUri = new Uri(Directory.GetCurrentDirectory());
432 |             Uri relativeInputPath = currentUri.MakeRelativeUri(inputUri);
433 |             return "../" + relativeInputPath.ToString();
434 |         }
435 | 
436 |         private static ObjectGroupComponent ParseCompilerAction(Action action)
437 |         {
438 |             string[] compatableExtensions = { ".c", ".cpp", ".rc", ".inl" };
439 |             ObjectGroupComponent outputAction = null;
440 |             Compiler compiler;
441 | 
442 |             if (Compilers.ContainsKey(action.CommandPath))
443 |             {
444 |                 compiler = Compilers[action.CommandPath];
445 |             }
446 |             else
447 |             {
448 |                 compiler = new Compiler(action.CommandPath);
449 |                 Compilers[action.CommandPath] = compiler;
450 |             }
451 | 
452 |             var inputMatches = Regex.Matches(action.CommandArguments, compiler.InputFileRegex, RegexOptions.IgnoreCase);
453 |             var outputMatch = Regex.Match(action.CommandArguments, compiler.OutputFileRegex, RegexOptions.IgnoreCase);
454 |             var usingPch = action.CommandArguments.Contains("/Yc");
455 | 
456 |             if (inputMatches.Count > 0)
457 |             {
458 |                 var input = "";
459 |                 var outputPath = "";
460 |                 var outputExt = "";
461 |                 var matchHash = "";
462 |                 var args = action.CommandArguments;
463 |                 PchOptions pchOptions = null;
464 | 
465 |                 foreach (Match inputMatch in inputMatches)
466 |                 {
467 |                     if (compatableExtensions.Any(ext => inputMatch.Value.EndsWith(ext)))
468 |                     {
469 |                         input = inputMatch.Value;
470 |                     }
471 |                     if (!string.IsNullOrWhiteSpace(input))
472 |                         break;
473 |                 }
474 | 
475 |                 var output = outputMatch.Value;
476 | 
477 |                 if (usingPch)
478 |                 {
479 |                     pchOptions = new PchOptions();
480 |                     outputMatch = Regex.Match(args, compiler.PchOutputRegex, RegexOptions.IgnoreCase);
481 |                     pchOptions.Input = input; 
482 |                     pchOptions.Output = outputMatch.Value;
483 |                     pchOptions.Options = args;
484 | 
485 |                     pchOptions.Options = pchOptions.Options.Replace(input, "%1");
486 |                     pchOptions.Options = pchOptions.Options.Replace(outputMatch.Value, "%2");
487 | 
488 |                     args = args.Replace("/Yc", "/Yu");
489 |                 }
490 |                 args = args.Replace(" c ", "");
491 |                 args = args.Replace(output, "%2");
492 |                 args = args.Replace(input, "%1");
493 | 
494 |                 var pathExtSplit = output.Split('.');
495 |                 outputPath = Path.GetDirectoryName(output) + Path.DirectorySeparatorChar;
496 |                 outputExt = "." + pathExtSplit[pathExtSplit.Length - 2] + "." + pathExtSplit[pathExtSplit.Length - 1];
497 | 
498 |                 if (outputExt == ".h.obj")
499 |                 {
500 |                     outputExt = ".obj";
501 |                 }
502 |                 
503 |                 var dependencies = action.PrerequisiteItems.OrderBy(n => n.AbsolutePath).Select(n => n.AbsolutePath);
504 | 
505 |                 if (pchOptions != null)
506 |                     matchHash = args + outputPath + outputExt + action.CommandPath + pchOptions.Options + pchOptions.Input + pchOptions.Output + dependencies;
507 |                 else
508 |                     matchHash = args + outputPath + outputExt + action.CommandPath + dependencies;
509 | 
510 |                 outputAction = new ObjectGroupComponent()
511 |                 {
512 |                     MatchHash = matchHash,
513 |                     CompilerArguments = args,
514 |                     OutputPath = outputPath,
515 |                     OutputExt = outputExt,
516 |                     LocalOnly = !action.bCanExecuteRemotely,
517 |                     PchOptions = pchOptions,
518 |                     ObjectCompiler = compiler
519 |                 };
520 |                 outputAction.ActionInputs[action] = input;
521 |             }
522 |             return outputAction;
523 |         }
524 | 
525 |         private static ExecutableComponent ParseLinkAction(Action action)
526 |         {
527 |             ExecutableComponent output = null;
528 |             Linker linker;
529 | 
530 |             if (Linkers.ContainsKey(action.CommandPath))
531 |             {
532 |                 linker = Linkers[action.CommandPath];
533 |             }
534 |             else
535 |             {
536 |                 linker = new Linker(action.CommandPath);
537 |                 Linkers[action.CommandPath] = linker;
538 |             }
539 | 
540 |             var linkerFound = false;
541 |             if (Linker.IsKnownLinker(action.CommandPath))
542 |             {
543 |                 var inputMatchesRegex = Regex.Matches(action.CommandArguments, linker.InputFileRegex, RegexOptions.IgnoreCase);
544 |                 var importLibMatchesRegex = Regex.Matches(action.CommandArguments, linker.ImportLibraryRegex, RegexOptions.IgnoreCase);                 
545 |                 var outputMatchesRegex = Regex.Matches(action.CommandArguments, linker.OutputFileRegex, RegexOptions.IgnoreCase);
546 |                 var inputMatches = inputMatchesRegex.Cast<Match>().Where(n => linker.AllowedInputTypes.Any(a => n.Value.EndsWith(a))).ToList();
547 |                 var outputMatches = outputMatchesRegex.Cast<Match>().Where(n => linker.AllowedOutputTypes.Any(a => n.Value.EndsWith(a))).ToList();
548 |                 var importMatches = importLibMatchesRegex.Cast<Match>().ToList();
549 | 
550 |                 if (inputMatches.Count > 0 && outputMatches.Count == 1)
551 |                 {
552 |                     linkerFound = true;
553 |                     output = new LinkerComponent()
554 |                     {
555 |                         Action = action,
556 |                         Linker = linker,
557 |                         Arguments = action.CommandArguments,
558 |                         OutputLibrary = outputMatches[0].Value,
559 |                         ImportLibrary = importMatches.Count > 0 ? importMatches[0].Value : null,
560 |                         Inputs = inputMatches.Select(n => n.Value).ToList(),
561 |                         LocalOnly = !action.bCanExecuteRemotely
562 |                     };
563 |                 }
564 |             }
565 | 
566 |             if (!linkerFound)
567 |             {
568 |                 output = new ExecutableComponent()
569 |                 {
570 |                     Action = action,
571 |                     Arguments = action.CommandArguments,
572 |                     Linker = linker
573 |                 };
574 |             }
575 |             return output;
576 |         }
577 | 
578 |         private static ExecutionResult RunFBuild(BuildStep step, string bffString)
579 |         {
580 |             ExecutionResult result;
581 |             try
582 |             {
583 |                 var watch = Stopwatch.StartNew();
584 |                 Log.TraceInformation(step == BuildStep.CompileObjects ? "Building Objects" : "Linking Objects");
585 | 
586 |                 var distScriptFilename = Path.Combine(BuildConfiguration.BaseIntermediatePath, "fbuild.bff");
587 |                 var distScriptFileStream = new FileStream(distScriptFilename, FileMode.Create, FileAccess.ReadWrite, FileShare.Read);
588 | 
589 |                 var scriptFile = new StreamWriter(distScriptFileStream)
590 |                 {
591 |                     AutoFlush = true
592 |                 };
593 |                 scriptFile.WriteLine(ActionThread.ExpandEnvironmentVariables(bffString));
594 |                 scriptFile.Flush();
595 |                 scriptFile.Close();
596 |                 scriptFile.Dispose();
597 |                 scriptFile = null;
598 | 
599 |                 result = DispatchFBuild();
600 |                 watch.Stop();
601 |                 var elapsedMs = watch.ElapsedMilliseconds;
602 |                 Log.TraceInformation((step == BuildStep.CompileObjects ? "Object Compilation" : "Object Linking") + " Finished. Execution Time: "+elapsedMs);                
603 |             }
604 |             catch (Exception)
605 |             {
606 |                 result = ExecutionResult.TasksFailed;
607 |             }
608 | 
609 |             return result;
610 |         }
611 | 
612 |         private static ExecutionResult DispatchFBuild()
613 |         {
614 |             ProcessStartInfo PSI = new ProcessStartInfo(Path.GetFullPath("../Extras/FASTBuild/") + "FBuild.exe", ""
615 |                 + (FASTBuildConfiguration.IsDistrbuted ? "-dist " : "")
616 |                 + (FASTBuildConfiguration.UseCache ? (FASTBuildConfiguration.EnableCacheGenerationMode ? "-cache " : "-cacheread ") : "")
617 |                 + (BuildConfiguration.bFastbuildContinueOnError ? "-nostoponerror " : "")
618 |                 + FASTBuildConfiguration.AdditionalArguments
619 |                 + " -clean"
620 |                 + " -config " + BuildConfiguration.BaseIntermediatePath + "/fbuild.bff");
621 |             Log.TraceInformation("Arguments: "+PSI.Arguments);
622 |             Log.TraceInformation("Cache Path: "+FASTBuildConfiguration.CachePath);
623 |             Log.TraceInformation("Use Cache: " + (FASTBuildConfiguration.UseCache ? "Yes" : "No"));
624 |             Log.TraceInformation("Is Distributed: " + (FASTBuildConfiguration.IsDistrbuted ? "Yes" : "No"));
625 | 
626 |             PSI.RedirectStandardOutput = true;
627 |             PSI.RedirectStandardError = true;
628 |             PSI.UseShellExecute = false;
629 |             PSI.CreateNoWindow = true;
630 |             PSI.WorkingDirectory = Path.GetFullPath(".");
631 |             var newProcess = new Process
632 |             {
633 |                 StartInfo = PSI
634 |             };
635 |             var output = new DataReceivedEventHandler(ActionDebugOutput);
636 |             newProcess.OutputDataReceived += output;
637 |             newProcess.ErrorDataReceived += output;
638 |             newProcess.Start();
639 |             newProcess.BeginOutputReadLine();
640 |             newProcess.BeginErrorReadLine();
641 |             newProcess.WaitForExit();
642 |             newProcess.OutputDataReceived -= output;
643 |             newProcess.ErrorDataReceived -= output;
644 | 
645 |             return newProcess.ExitCode == 0 ? ExecutionResult.TasksSucceeded : ExecutionResult.TasksFailed;
646 |         }
647 | 
648 |         public static bool IsAvailable()
649 |         {
650 |             var fbRoot = Path.GetFullPath("../Extras/FASTBuild/");
651 | 
652 |             var fbExists = false;
653 | 
654 |             if (fbRoot == null) return false;
655 | 
656 |             var fbExecutable = Path.Combine(fbRoot, "FBuild.exe");
657 |             var compilerDir = Path.Combine(fbRoot, "External/VS15.0");
658 |             var sdkDir = Path.Combine(fbRoot, "External/Windows8.1");
659 | 
660 |             // Check that FASTBuild is available
661 |             fbExists = File.Exists(fbExecutable);
662 |             if (fbExists)
663 |             {
664 |                 if (!Directory.Exists(compilerDir))
665 |                 {
666 |                     fbExists = false;
667 |                 }
668 |                 if (!Directory.Exists(sdkDir))
669 |                 {
670 |                     fbExists = false;
671 |                 }
672 |             }
673 |             return fbExists;
674 |         }
675 | 
676 |     }
677 | }
678 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
  1 | ## Still in progress so beware of potential issues. I also recommend checking out an alternative here: https://github.com/liamkf/Unreal_FASTBuild
  2 | 
  3 | There's been a bit of interest in all the steps required to get fastbuild to compile Unreal Engine 4, so I've repository together with all the steps required. For reference I'm using the unmodified 4.13 branch from GitHub, and an unmodified v0.91 fastbuild. This has only currently been tested on Windows builds with caching (without distribution) enabled. Full console support will come in the future.
  4 | 
  5 | # Setting up FASTBuild
  6 | 
  7 | Due to how fastbuild works you can pretty much place it where you want. I keep mine within the engine, with the following file structure.
  8 | * UE4
  9 | * * Engine
 10 | * * * Extras
 11 | * * * * FASTBuild
 12 | * * * * * FBuild.exe
 13 | * * * * * SDK 
 14 | 
 15 | Inside your SDK, setup junction links to your Windows SDK folder, as well as your Visual Studio 2015 folder. These junction links make sure that multiple developers will have the same path for their Visual Studio folder - which removes a reason for cache hash mismatches.
 16 | 
 17 | # Modifying the Engine
 18 | 
 19 | There are a few files in the UnrealBuildTool project we need to modify, these are:
 20 | - Configuration/BuildConfiguration.cs
 21 | - Configuration/UEBuildPlatform.cs
 22 | - System/ActionGraph.cs
 23 | - Windows/UEBuildWindows.cs
 24 | 
 25 | We also need to add a new file to generate a BFF file from the provided actions list. For this, I'm only going to focus on the Windows platform.
 26 | 
 27 | ### Adding FASTBuild classes
 28 | 
 29 | First step is to add our FASTBuild classes. Place the files in this respository in <UnrealBuildTool>/System
 30 | 
 31 | ### Configuration/BuildConfiguration.cs
 32 | 
 33 | - Add the following properties to the top of the file.
 34 | 
 35 | ```
 36 |  // --> FASTBuild
 37 | 
 38 | /// <summary>
 39 | /// Whether FASTBuild may be used.
 40 | /// </summary>
 41 | [XmlConfig]
 42 | public static bool bAllowFastbuild;
 43 | 
 44 | /// <summary>
 45 | /// Whether linking should be disabled. Useful for cache 
 46 | /// generation builds
 47 | /// </summary>
 48 | [XmlConfig]
 49 | public static bool bFastbuildNoLinking;
 50 | 
 51 | /// <summary>
 52 | /// Whether the build should continue despite errors. 
 53 | /// Useful for cache generation builds
 54 | /// </summary>
 55 | [XmlConfig]
 56 | public static bool bFastbuildContinueOnError;
 57 | 
 58 | // <-- FASTBuild
 59 | ```
 60 | 
 61 | - Add the following to the very bottom of the LoadDefaults method.
 62 | 
 63 | ```
 64 | // --> FASTBuild
 65 | 
 66 | bAllowFastbuild = true;
 67 | bUsePDBFiles = false; //Only required if you're using MSVC
 68 | 
 69 | // <-- FASTBuild
 70 | ```
 71 | 
 72 | Finally, add this to the ValidateConfiguration method near similar lines for XGE/Distcc/SNDBS
 73 | 
 74 | ```
 75 | // --> FASTBuild
 76 | if(!BuildPlatform.CanUseFastbuild())
 77 | {
 78 |     bAllowFastbuild = false;
 79 | }
 80 | // <-- FASTBuild
 81 | ```
 82 | 
 83 | ### Configuration/UEBuildPlatform.cs
 84 | 
 85 | Near similar lines for XGE/Distcc/SNDBS, add the following method.
 86 | 
 87 | ```
 88 | // --> FASTBuild
 89 | public virtual bool CanUseFastbuild()
 90 | {
 91 |     return false;
 92 | }
 93 | // <-- FASTBuild
 94 | ```
 95 | 
 96 | ### System/ActionGraph.cs
 97 | 
 98 | Alter the ExecuteActions method to run this check prior to the XGE one:
 99 | ```
100 |  // --> FASTBuild
101 | if (BuildConfiguration.bAllowFastbuild)
102 | {
103 |     ExecutorName = "Fastbuild";
104 | 
105 |     FASTBuild.ExecutionResult FastBuildResult = FASTBuild.ExecuteActions(ActionsToExecute);
106 |     if (FastBuildResult != FASTBuild.ExecutionResult.Unavailable)
107 |     {
108 |         ExecutorName = "FASTBuild";
109 |         Result = (FastBuildResult == FASTBuild.ExecutionResult.TasksSucceeded);
110 |         bUsedXGE = true;
111 |     }
112 | }
113 | // <-- FASTBuild
114 | ```
115 | 
116 | And alter the XGE if to check for !bUseXGE, like this:
117 | ```
118 | From:
119 | if (BuildConfiguration.bAllowXGE || BuildConfiguration.bXGEExport)
120 | To:
121 | if (!bUsedXGE && (BuildConfiguration.bAllowXGE || BuildConfiguration.bXGEExport))
122 | ```
123 | 
124 | ### Windows/UEBuildWindows
125 | 
126 | Near similar lines for SNDBS, add the following method:
127 | 
128 | ```
129 | // --> FASTBuild
130 | public override bool CanUseFastbuild()
131 | {
132 |     return true;
133 | }
134 | // <-- FASTBuild
135 | ```
136 | 
137 | # Enabling Cache Generation
138 | 
139 | FASTBuildCOnfiguration.cs contains the EnableCacheGenerationMode property. By default this checks for whether an environment variable ("UE-FB-CACHE-WRITE") contains the value TRUE.
140 | 
141 | # Warnings as errors
142 | 
143 | Enabling warnings as errors will cause issues with certain things being falsely detected as digraphs, and will need to be fixed manually by adding a space to separate the "<::X"
144 | 


--------------------------------------------------------------------------------