├── README.md └── mmake.erl /README.md: -------------------------------------------------------------------------------- 1 | #usage 2 | this file based on otp_src/lib/tools/src/make.erl, support the multiple processes compile. 3 | 4 | the exported functions are like make.erl, except for add the Worker Number as the first 5 | argument for every functions. 6 | 7 | e.g. mmake:all(5), means you compile the codes defined in Emakefile by 5 workers. 8 | 9 | in your Makefile or script : 10 | 11 | `erl -eval "case make:files([\"mmake.erl\"], [{outdir, \"ebin\"}]) of error -> halt(1); _ -> ok end" 12 | -eval "case mmake:all(8,[$(MAKE_OPTS)]) of up_to_date -> halt(0); error -> halt(1) end."` 13 | 14 | -------------------------------------------------------------------------------- /mmake.erl: -------------------------------------------------------------------------------- 1 | %% 多进程编译,修改自otp/lib/tools/src/make.erl 2 | %% 解析Emakefile,根据获取{mods, options}列表, 3 | %% 按照次序编译每项(解决编译顺序的问题) 4 | %% 其中mods也可以包含多个模块,当大于1个时, 5 | %% 可以启动多个process进行编译,从而提高编译速度. 6 | -module(mmake). 7 | -export([all/1, all/2, files/2, files/3]). 8 | 9 | -include_lib("kernel/include/file.hrl"). 10 | 11 | -define(MakeOpts,[noexec,load,netload,noload]). 12 | 13 | all(Worker) when is_integer(Worker) -> 14 | all(Worker, []). 15 | 16 | all(Worker, Options) when is_integer(Worker) -> 17 | {MakeOpts, CompileOpts} = sort_options(Options,[],[]), 18 | case read_emakefile('Emakefile', CompileOpts) of 19 | Files when is_list(Files) -> 20 | do_make_files(Worker, Files, MakeOpts); 21 | error -> 22 | error 23 | end. 24 | 25 | files(Worker, Fs) -> 26 | files(Worker, Fs, []). 27 | 28 | files(Worker, Fs0, Options) -> 29 | Fs = [filename:rootname(F,".erl") || F <- Fs0], 30 | {MakeOpts,CompileOpts} = sort_options(Options,[],[]), 31 | case get_opts_from_emakefile(Fs,'Emakefile',CompileOpts) of 32 | Files when is_list(Files) -> 33 | do_make_files(Worker, Files,MakeOpts); 34 | error -> error 35 | end. 36 | 37 | do_make_files(Worker, Fs, Opts) -> 38 | %io:format("worker:~p~nfs:~p~nopts:~p~n", [Worker, Fs, Opts]), 39 | process(Fs, Worker, lists:member(noexec, Opts), load_opt(Opts)). 40 | 41 | sort_options([H|T],Make,Comp) -> 42 | case lists:member(H,?MakeOpts) of 43 | true -> 44 | sort_options(T,[H|Make],Comp); 45 | false -> 46 | sort_options(T,Make,[H|Comp]) 47 | end; 48 | sort_options([],Make,Comp) -> 49 | {Make,lists:reverse(Comp)}. 50 | 51 | %%% Reads the given Emakefile and returns a list of tuples: {Mods,Opts} 52 | %%% Mods is a list of module names (strings) 53 | %%% Opts is a list of options to be used when compiling Mods 54 | %%% 55 | %%% Emakefile can contain elements like this: 56 | %%% Mod. 57 | %%% {Mod,Opts}. 58 | %%% Mod is a module name which might include '*' as wildcard 59 | %%% or a list of such module names 60 | %%% 61 | %%% These elements are converted to [{ModList,OptList},...] 62 | %%% ModList is a list of modulenames (strings) 63 | read_emakefile(Emakefile,Opts) -> 64 | case file:consult(Emakefile) of 65 | {ok, Emake} -> 66 | transform(Emake,Opts,[],[]); 67 | {error,enoent} -> 68 | %% No Emakefile found - return all modules in current 69 | %% directory and the options given at command line 70 | Mods = [filename:rootname(F) || F <- filelib:wildcard("*.erl")], 71 | [{Mods, Opts}]; 72 | {error,Other} -> 73 | io:format("make: Trouble reading 'Emakefile':~n~p~n",[Other]), 74 | error 75 | end. 76 | 77 | transform([{Mod,ModOpts}|Emake],Opts,Files,Already) -> 78 | case expand(Mod,Already) of 79 | [] -> 80 | transform(Emake,Opts,Files,Already); 81 | Mods -> 82 | transform(Emake,Opts,[{Mods,ModOpts++Opts}|Files],Mods++Already) 83 | end; 84 | transform([Mod|Emake],Opts,Files,Already) -> 85 | case expand(Mod,Already) of 86 | [] -> 87 | transform(Emake,Opts,Files,Already); 88 | Mods -> 89 | transform(Emake,Opts,[{Mods,Opts}|Files],Mods++Already) 90 | end; 91 | transform([],_Opts,Files,_Already) -> 92 | lists:reverse(Files). 93 | 94 | expand(Mod,Already) when is_atom(Mod) -> 95 | expand(atom_to_list(Mod),Already); 96 | expand(Mods,Already) when is_list(Mods), not is_integer(hd(Mods)) -> 97 | lists:concat([expand(Mod,Already) || Mod <- Mods]); 98 | expand(Mod,Already) -> 99 | case lists:member($*,Mod) of 100 | true -> 101 | Fun = fun(F,Acc) -> 102 | M = filename:rootname(F), 103 | case lists:member(M,Already) of 104 | true -> Acc; 105 | false -> [M|Acc] 106 | end 107 | end, 108 | lists:foldl(Fun, [], filelib:wildcard(Mod++".erl")); 109 | false -> 110 | Mod2 = filename:rootname(Mod, ".erl"), 111 | case lists:member(Mod2,Already) of 112 | true -> []; 113 | false -> [Mod2] 114 | end 115 | end. 116 | 117 | %%% Reads the given Emakefile to see if there are any specific compile 118 | %%% options given for the modules. 119 | get_opts_from_emakefile(Mods,Emakefile,Opts) -> 120 | case file:consult(Emakefile) of 121 | {ok,Emake} -> 122 | Modsandopts = transform(Emake,Opts,[],[]), 123 | ModStrings = [coerce_2_list(M) || M <- Mods], 124 | get_opts_from_emakefile2(Modsandopts,ModStrings,Opts,[]); 125 | {error,enoent} -> 126 | [{Mods, Opts}]; 127 | {error,Other} -> 128 | io:format("make: Trouble reading 'Emakefile':~n~p~n",[Other]), 129 | error 130 | end. 131 | 132 | get_opts_from_emakefile2([{MakefileMods,O}|Rest],Mods,Opts,Result) -> 133 | case members(Mods,MakefileMods,[],Mods) of 134 | {[],_} -> 135 | get_opts_from_emakefile2(Rest,Mods,Opts,Result); 136 | {I,RestOfMods} -> 137 | get_opts_from_emakefile2(Rest,RestOfMods,Opts,[{I,O}|Result]) 138 | end; 139 | get_opts_from_emakefile2([],[],_Opts,Result) -> 140 | Result; 141 | get_opts_from_emakefile2([],RestOfMods,Opts,Result) -> 142 | [{RestOfMods,Opts}|Result]. 143 | 144 | members([H|T],MakefileMods,I,Rest) -> 145 | case lists:member(H,MakefileMods) of 146 | true -> 147 | members(T,MakefileMods,[H|I],lists:delete(H,Rest)); 148 | false -> 149 | members(T,MakefileMods,I,Rest) 150 | end; 151 | members([],_MakefileMods,I,Rest) -> 152 | {I,Rest}. 153 | 154 | 155 | %% Any flags that are not recognixed as make flags are passed directly 156 | %% to the compiler. 157 | %% So for example make:all([load,debug_info]) will make everything 158 | %% with the debug_info flag and load it. 159 | load_opt(Opts) -> 160 | case lists:member(netload,Opts) of 161 | true -> 162 | netload; 163 | false -> 164 | case lists:member(load,Opts) of 165 | true -> 166 | load; 167 | _ -> 168 | noload 169 | end 170 | end. 171 | 172 | %% 处理 173 | process([{[], _Opts}|Rest], Worker, NoExec, Load) -> 174 | process(Rest, Worker, NoExec, Load); 175 | process([{L, Opts}|Rest], Worker, NoExec, Load) -> 176 | Len = length(L), 177 | Worker2 = erlang:min(Len, Worker), 178 | case catch do_worker(L, Opts, NoExec, Load, Worker2) of 179 | error -> 180 | error; 181 | ok -> 182 | process(Rest, Worker, NoExec, Load) 183 | end; 184 | process([], _Worker, _NoExec, _Load) -> 185 | up_to_date. 186 | 187 | %% worker进行编译 188 | do_worker(L, Opts, NoExec, Load, Worker) -> 189 | WorkerList = do_split_list(L, Worker), 190 | %io:format("worker:~p worker list(~p)~n", [Worker, length(WorkerList)]), 191 | % 启动进程 192 | Ref = make_ref(), 193 | Pids = 194 | [begin 195 | start_worker(E, Opts, NoExec, Load, self(), Ref) 196 | end || E <- WorkerList], 197 | do_wait_worker(length(Pids), Ref). 198 | 199 | %% 等待结果 200 | do_wait_worker(0, _Ref) -> 201 | ok; 202 | do_wait_worker(N, Ref) -> 203 | receive 204 | {ack, Ref} -> 205 | do_wait_worker(N - 1, Ref); 206 | {error, Ref} -> 207 | throw(error); 208 | {'EXIT', _P, _Reason} -> 209 | do_wait_worker(N, Ref); 210 | _Other -> 211 | io:format("receive unknown msg:~p~n", [_Other]), 212 | do_wait_worker(N, Ref) 213 | end. 214 | 215 | %% 将L分割成最多包含N个子列表的列表 216 | do_split_list(L, N) -> 217 | Len = length(L), 218 | % 每个列表的元素数 219 | LLen = (Len + N - 1) div N, 220 | do_split_list(L, LLen, []). 221 | 222 | do_split_list([], _N, Acc) -> 223 | lists:reverse(Acc); 224 | do_split_list(L, N, Acc) -> 225 | {L2, L3} = lists:split(erlang:min(length(L), N), L), 226 | do_split_list(L3, N, [L2 | Acc]). 227 | 228 | %% 启动worker进程 229 | start_worker(L, Opts, NoExec, Load, Parent, Ref) -> 230 | Fun = 231 | fun() -> 232 | [begin 233 | case recompilep(coerce_2_list(F), NoExec, Load, Opts) of 234 | error -> 235 | Parent ! {error, Ref}, 236 | exit(error); 237 | _ -> 238 | ok 239 | end 240 | end || F <- L], 241 | Parent ! {ack, Ref} 242 | end, 243 | spawn_link(Fun). 244 | 245 | recompilep(File, NoExec, Load, Opts) -> 246 | ObjName = lists:append(filename:basename(File), 247 | code:objfile_extension()), 248 | ObjFile = case lists:keysearch(outdir,1,Opts) of 249 | {value,{outdir,OutDir}} -> 250 | filename:join(coerce_2_list(OutDir),ObjName); 251 | false -> 252 | ObjName 253 | end, 254 | case exists(ObjFile) of 255 | true -> 256 | recompilep1(File, NoExec, Load, Opts, ObjFile); 257 | false -> 258 | recompile(File, NoExec, Load, Opts) 259 | end. 260 | 261 | recompilep1(File, NoExec, Load, Opts, ObjFile) -> 262 | {ok, Erl} = file:read_file_info(lists:append(File, ".erl")), 263 | {ok, Obj} = file:read_file_info(ObjFile), 264 | recompilep1(Erl, Obj, File, NoExec, Load, Opts). 265 | 266 | recompilep1(#file_info{mtime=Te}, 267 | #file_info{mtime=To}, File, NoExec, Load, Opts) when Te>To -> 268 | recompile(File, NoExec, Load, Opts); 269 | recompilep1(_Erl, #file_info{mtime=To}, File, NoExec, Load, Opts) -> 270 | recompile2(To, File, NoExec, Load, Opts). 271 | 272 | %% recompile2(ObjMTime, File, NoExec, Load, Opts) 273 | %% Check if file is of a later date than include files. 274 | recompile2(ObjMTime, File, NoExec, Load, Opts) -> 275 | IncludePath = include_opt(Opts), 276 | case check_includes(lists:append(File, ".erl"), IncludePath, ObjMTime) of 277 | true -> 278 | recompile(File, NoExec, Load, Opts); 279 | false -> 280 | false 281 | end. 282 | 283 | include_opt([{i,Path}|Rest]) -> 284 | [Path|include_opt(Rest)]; 285 | include_opt([_First|Rest]) -> 286 | include_opt(Rest); 287 | include_opt([]) -> 288 | []. 289 | 290 | %% recompile(File, NoExec, Load, Opts) 291 | %% Actually recompile and load the file, depending on the flags. 292 | %% Where load can be netload | load | noload 293 | 294 | recompile(File, true, _Load, _Opts) -> 295 | io:format("Out of date: ~s\n",[File]); 296 | recompile(File, false, noload, Opts) -> 297 | io:format("Recompile: ~s\n",[File]), 298 | compile:file(File, [report_errors, report_warnings, error_summary |Opts]); 299 | recompile(File, false, load, Opts) -> 300 | io:format("Recompile: ~s\n",[File]), 301 | c:c(File, Opts); 302 | recompile(File, false, netload, Opts) -> 303 | io:format("Recompile: ~s\n",[File]), 304 | c:nc(File, Opts). 305 | 306 | exists(File) -> 307 | case file:read_file_info(File) of 308 | {ok, _} -> 309 | true; 310 | _ -> 311 | false 312 | end. 313 | 314 | coerce_2_list(X) when is_atom(X) -> 315 | atom_to_list(X); 316 | coerce_2_list(X) -> 317 | X. 318 | 319 | %%% If you an include file is found with a modification 320 | %%% time larger than the modification time of the object 321 | %%% file, return true. Otherwise return false. 322 | check_includes(File, IncludePath, ObjMTime) -> 323 | Path = [filename:dirname(File)|IncludePath], 324 | case epp:open(File, Path, []) of 325 | {ok, Epp} -> 326 | check_includes2(Epp, File, ObjMTime); 327 | _Error -> 328 | false 329 | end. 330 | 331 | check_includes2(Epp, File, ObjMTime) -> 332 | case epp:parse_erl_form(Epp) of 333 | {ok, {attribute, 1, file, {File, 1}}} -> 334 | check_includes2(Epp, File, ObjMTime); 335 | {ok, {attribute, 1, file, {IncFile, 1}}} -> 336 | case file:read_file_info(IncFile) of 337 | {ok, #file_info{mtime=MTime}} when MTime>ObjMTime -> 338 | epp:close(Epp), 339 | true; 340 | _ -> 341 | check_includes2(Epp, File, ObjMTime) 342 | end; 343 | {ok, _} -> 344 | check_includes2(Epp, File, ObjMTime); 345 | {eof, _} -> 346 | epp:close(Epp), 347 | false; 348 | {error, _Error} -> 349 | check_includes2(Epp, File, ObjMTime) 350 | end. 351 | --------------------------------------------------------------------------------