├── .gitignore ├── README.md ├── apps ├── feeds │ └── src │ │ ├── feeds.app.src │ │ ├── feeds_app.erl │ │ ├── feeds_fetch.erl │ │ ├── feeds_parse.erl │ │ ├── feeds_sup.erl │ │ └── feeds_update.erl ├── hasher │ └── src │ │ ├── hasher.app.src │ │ ├── hasher_app.erl │ │ ├── hasher_hash.erl │ │ ├── hasher_recheck.erl │ │ ├── hasher_sup.erl │ │ └── hasher_worker.erl ├── model │ ├── include │ │ └── model.hrl │ └── src │ │ ├── model.app.src │ │ ├── model_app.erl │ │ ├── model_enclosures.erl │ │ ├── model_feeds.erl │ │ ├── model_scrape_queue.erl │ │ ├── model_session.erl │ │ ├── model_stats.erl │ │ ├── model_stats_cache.erl │ │ ├── model_sup.erl │ │ ├── model_token.erl │ │ ├── model_torrents.erl │ │ ├── model_users.erl │ │ └── model_worker.erl ├── seeder │ └── src │ │ ├── seeder.app.src │ │ ├── seeder_app.erl │ │ ├── seeder_listener.erl │ │ ├── seeder_sup.erl │ │ └── seeder_wire_protocol.erl └── shared │ └── src │ ├── benc.erl │ ├── peer_id.erl │ ├── shared.app.src │ ├── storage.erl │ ├── url.erl │ └── util.erl ├── build.sh ├── config └── vm.args ├── dev.erl ├── pg_downloads.sql ├── pg_install.sql ├── pg_meta.sql ├── pg_migrate_enclosures_recheck.sql ├── pg_search.sql ├── pg_stats.sql ├── pg_var.sql ├── rebar.config └── tracker_load.erl /.gitignore: -------------------------------------------------------------------------------- 1 | erl_crash.dump 2 | rel/feeds/feeds/ 3 | rel/hasher/hasher 4 | rel/ui/ui 5 | rel/seeder/seeder 6 | *.beam 7 | ebin 8 | deps/* -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | PritTorrent 2 | =========== 3 | 4 | Build 5 | ----- 6 | 7 | ``` 8 | rebar get-deps compile generate 9 | ``` 10 | 11 | 12 | TODO 13 | ---- 14 | 15 | * URL longener? 16 | * Fix empty downloads.type 17 | * filter.js: 18 | * Fix button style breakage in Mozilla 19 | * Fix z-index (Android?) 20 | * New {feeds,downloads}.{lang,summary,type} in: 21 | * Downloads Feeds 22 | * HTML 23 | 24 | * enforce https for log in 25 | * clickable stats hint 26 | 27 | * 28 | * 29 | 30 | * stop seeding 31 | 32 | * Check U-A & replace RSS links with Miro subscribe URLs 33 | 34 | * Edit user: 35 | * About field 36 | * Edit feeds: 37 | * Add & fetch immediately 38 | 39 | * feeds_parse: http://video.search.yahoo.com/mrss 40 | 41 | * more configurability 42 | 43 | * Fetch & display feed summaries 44 | 45 | * Feed summaries: X items, Y torrents 46 | 47 | * Storage app 48 | - Avoid dup connections to 1 HTTP server (IP) 49 | - Fair queueing 50 | - Caching 51 | * OEmbed 52 | 53 | * Stats: 54 | - DLs by country/client? 55 | 56 | * Rehash on detected enclosure data change 57 | * Multiple sources per feed 58 | 59 | Future features: 60 | 61 | * UDP tracker 62 | * UTP wire protocol 63 | * UI: Wholesome OPML export 64 | * Super-seeding 65 | * Slot queues 66 | * PEX 67 | * DHT support 68 | -------------------------------------------------------------------------------- /apps/feeds/src/feeds.app.src: -------------------------------------------------------------------------------- 1 | {application, feeds, 2 | [ 3 | {description, ""}, 4 | {vsn, "1"}, 5 | {registered, []}, 6 | {applications, [ 7 | kernel, 8 | stdlib, 9 | sasl, 10 | lhttpc, 11 | exmpp, 12 | model, 13 | shared 14 | ]}, 15 | {mod, { feeds_app, []}}, 16 | {env, []} 17 | ]}. 18 | -------------------------------------------------------------------------------- /apps/feeds/src/feeds_app.erl: -------------------------------------------------------------------------------- 1 | -module(feeds_app). 2 | 3 | -behaviour(application). 4 | 5 | %% Application callbacks 6 | -export([start/2, stop/1]). 7 | 8 | %% =================================================================== 9 | %% Application callbacks 10 | %% =================================================================== 11 | 12 | start(_StartType, _StartArgs) -> 13 | feeds_sup:start_link(). 14 | 15 | stop(_State) -> 16 | ok. 17 | -------------------------------------------------------------------------------- /apps/feeds/src/feeds_fetch.erl: -------------------------------------------------------------------------------- 1 | %% TODO: etags/last-modified support 2 | -module(feeds_fetch). 3 | 4 | -export([fetch/3]). 5 | 6 | -include_lib("exmpp/include/exmpp_xml.hrl"). 7 | 8 | -define(TIMEOUT, 30 * 1000). 9 | -define(MAX_REDIRECTS, 3). 10 | 11 | 12 | -spec fetch(string(), string() | undefined, string() | undefined) 13 | -> {ok, {string(), string()}, xmlel()}. 14 | fetch(Url, Etag1, LastModified1) -> 15 | Headers = 16 | if 17 | is_binary(Etag1), size(Etag1) > 0 -> 18 | [{"If-None-Match", binary_to_list(Etag1)}]; 19 | true -> 20 | [] 21 | end ++ 22 | if 23 | is_binary(LastModified1), size(LastModified1) > 0 -> 24 | [{"If-Modified-Since", binary_to_list(LastModified1)}]; 25 | true -> 26 | [] 27 | end, 28 | 29 | Parser = exmpp_xml:start_parser([{max_size, 30 * 1024 * 1024}, 30 | {names_as_atom, false}]), 31 | HttpRes = (catch 32 | http_fold( 33 | Url, Headers, 34 | fun(Els, Chunk) -> 35 | case exmpp_xml:parse(Parser, Chunk) of 36 | continue -> 37 | Els; 38 | Els1 when is_list(Els1) -> 39 | Els1 ++ Els 40 | end 41 | end, [])), 42 | 43 | Result = 44 | case HttpRes of 45 | {ok, {Etag2, LastModified2}, Els1} -> 46 | {Els2, Error} = 47 | case (catch exmpp_xml:parse_final(Parser, <<"">>)) of 48 | done -> 49 | {[], <<"Empty document">>}; 50 | Els3 when is_list(Els3) -> 51 | {Els3, <<"Empty document">>}; 52 | {xml_parser, _, _Reason, _Details} = Error1 -> 53 | {[], Error1} 54 | end, 55 | %% At least one: 56 | case Els1 ++ Els2 of 57 | [#xmlel{} = RootEl | _] -> 58 | {ok, {Etag2, LastModified2}, RootEl}; 59 | _ -> 60 | {error, Error} 61 | end; 62 | not_modified -> 63 | not_modified; 64 | {'EXIT', Reason} -> 65 | {error, Reason}; 66 | {xml_parser, _, _Reason, _Details} = Error1 -> 67 | {error, Error1} 68 | end, 69 | 70 | ok = exmpp_xml:stop_parser(Parser), 71 | Result. 72 | 73 | %% TODO: handle pcast:// 74 | http_fold(URL, ReqHeaders1, F, AccIn) -> 75 | http_fold(URL, ReqHeaders1, F, AccIn, 0). 76 | 77 | http_fold(_, _, _, _, Redirects) when Redirects >= ?MAX_REDIRECTS -> 78 | exit({http, too_many_redirects}); 79 | http_fold(URL, ReqHeaders1, F, AccIn, Redirects) -> 80 | %% Compose request 81 | ReqHeaders2 = 82 | [{"User-Agent", "PritTorrent/0.1"} 83 | | ReqHeaders1], 84 | ReqOptions = 85 | [{partial_download, 86 | [ 87 | %% specifies how many part will be sent to the calling 88 | %% process before waiting for an acknowledgement 89 | {window_size, 4}, 90 | %% specifies the size the body parts should come in 91 | {part_size, 4096} 92 | ]} 93 | ], 94 | case lhttpc:request(URL, get, ReqHeaders2, 95 | [], ?TIMEOUT, ReqOptions) of 96 | %% Ok 97 | {ok, {{200, _}, Headers, Pid}} -> 98 | {ok, Etag, LastModified} = 99 | get_etag_last_modified_from_headers(Headers), 100 | %% Strrream: 101 | {ok, AccOut } = http_fold1(Pid, F, AccIn), 102 | {ok, {Etag, LastModified}, AccOut}; 103 | {ok, {{Status, _}, Headers, Pid}} -> 104 | %% Finalize this response: 105 | http_fold1(Pid, fun(_, _) -> 106 | ok 107 | end, undefined), 108 | 109 | case get_header("location", Headers) of 110 | undefined -> 111 | exit({http, Status}); 112 | Location -> 113 | io:format("HTTP ~B: ~s redirects to ~s~n", [Status, URL, Location]), 114 | http_fold(Location, ReqHeaders1, F, AccIn, Redirects + 1) 115 | end; 116 | 117 | {error, Reason} -> 118 | exit(Reason) 119 | end. 120 | 121 | http_fold1(undefined, _, AccIn) -> 122 | %% No body, no fold. 123 | AccIn; 124 | http_fold1(Pid, F, AccIn) -> 125 | case lhttpc:get_body_part(Pid, ?TIMEOUT) of 126 | {ok, Data} when is_binary(Data) -> 127 | AccOut = F(AccIn, Data), 128 | http_fold1(Pid, F, AccOut); 129 | {ok, {http_eob, _Trailers}} -> 130 | {ok, AccIn} 131 | end. 132 | 133 | 134 | get_etag_last_modified_from_headers(Headers) -> 135 | {ok, 136 | get_header("etag", Headers), 137 | get_header("last-modified", Headers)}. 138 | 139 | %% expects lower-case Name 140 | get_header(_Name, []) -> 141 | undefined; 142 | get_header(Name, [{HName, HValue} | Headers]) -> 143 | case string:to_lower(HName) == Name of 144 | true -> 145 | HValue; 146 | false -> 147 | get_header(Name, Headers) 148 | end. 149 | -------------------------------------------------------------------------------- /apps/feeds/src/feeds_parse.erl: -------------------------------------------------------------------------------- 1 | -module(feeds_parse). 2 | 3 | -export([serialize/1, 4 | get_type/1, get_channel/1, 5 | title/1, lang/1, summary/1, link/1, image/1, 6 | pick_items/1, 7 | item_id/1, item_title/1, item_lang/1, item_summary/1, item_enclosures/1, 8 | item_published/1, item_link/1, item_payment/1, item_image/1]). 9 | 10 | -include_lib("exmpp/include/exmpp_xml.hrl"). 11 | 12 | -define(NS_ATOM, "http://www.w3.org/2005/Atom"). 13 | -define(NS_BITLOVE, "http://bitlove.org"). 14 | 15 | -define(DEFAULT_XMLNS, [{"http://www.w3.org/XML/1998/namespace", "xml"}]). 16 | 17 | serialize(Xml) -> 18 | exmpp_xml:node_to_binary(Xml, [], ?DEFAULT_XMLNS). 19 | 20 | get_type(Xml) -> 21 | case exmpp_xml:get_ns_as_list(Xml) of 22 | ?NS_ATOM -> 23 | atom; 24 | _ -> 25 | rss 26 | end. 27 | 28 | get_channel(Xml) -> 29 | case get_type(Xml) of 30 | atom -> 31 | Xml; 32 | _ -> 33 | exmpp_xml:get_element(Xml, "channel") 34 | end. 35 | 36 | %% Just look for 1st title element 37 | -spec title(xmlel()) -> binary() | undefined. 38 | title(Xml) -> 39 | case exmpp_xml:get_name_as_list(Xml) of 40 | "title" -> 41 | exmpp_xml:get_cdata(Xml); 42 | _ -> 43 | lists:foldl(fun(Child, undefined) -> 44 | title(Child); 45 | (_, Title) -> 46 | Title 47 | end, undefined, exmpp_xml:get_child_elements(Xml)) 48 | end. 49 | 50 | lang(Xml) -> 51 | case exmpp_xml:get_name_as_list(Xml) of 52 | "language" -> 53 | normalize_language(exmpp_xml:get_cdata(Xml)); 54 | _ -> 55 | case normalize_language( 56 | exmpp_xml:get_attribute_as_binary(Xml, <<"lang">>, undefined)) of 57 | Lang when is_binary(Lang) -> 58 | Lang; 59 | _ -> 60 | lists:foldl(fun(Child, undefined) -> 61 | lang(Child); 62 | (_, Lang) -> 63 | Lang 64 | end, undefined, exmpp_xml:get_child_elements(Xml)) 65 | end 66 | end. 67 | 68 | normalize_language(Lang1) when is_binary(Lang1) -> 69 | Lang2 = list_to_binary( 70 | string:to_lower( 71 | binary_to_list(Lang1))), 72 | normalize_language1(Lang2); 73 | normalize_language(_) -> 74 | undefined. 75 | 76 | normalize_language1(<>) -> 77 | Lang; 78 | normalize_language1(<>) -> 79 | Lang; 80 | normalize_language1(<>) -> 81 | Lang; 82 | normalize_language1(<<"english">>) -> 83 | <<"en">>; 84 | normalize_language1(<<"german">>) -> 85 | <<"de">>; 86 | normalize_language1(<<"deutsch">>) -> 87 | <<"de">>; 88 | normalize_language1(<<"russian">>) -> 89 | <<"ru">>; 90 | normalize_language1(<<"espa", _/binary>>) -> 91 | <<"es">>; 92 | normalize_language1(<<"spanish">>) -> 93 | <<"es">>; 94 | normalize_language1(<<"ital", _/binary>>) -> 95 | <<"it">>; 96 | normalize_language1(<<"fran", _/binary>>) -> 97 | <<"fr">>; 98 | normalize_language1(<<"french">>) -> 99 | <<"fr">>; 100 | normalize_language1(_) -> 101 | undefined. 102 | 103 | 104 | %% We name it `summary', but we really want just the briefest description. 105 | summary(Xml) -> 106 | case exmpp_xml:get_elements(Xml, "summary") of 107 | [SummaryEl | _] -> 108 | exmpp_xml:get_cdata(SummaryEl); 109 | _ -> 110 | case exmpp_xml:get_element(Xml, "subtitle") of 111 | [SubtitleEl | _] -> 112 | exmpp_xml:get_cdata(SubtitleEl); 113 | _ -> 114 | undefined 115 | end 116 | end. 117 | 118 | -spec image(xmlel()) -> binary() | undefined. 119 | image(Xml) -> 120 | image1(Xml, ["image", "logo", "icon", "itunes:image"]). 121 | %% TODO: "itunes:"? What happened to XML namespaces? 122 | 123 | image1(_, []) -> 124 | undefined; 125 | image1(Xml, [ChildName | ChildNames]) -> 126 | R = 127 | lists:foldl( 128 | fun(Child, undefined) -> 129 | %% Default: @url 130 | URL1 = 131 | case exmpp_xml:get_elements(Child, "url") of 132 | [UrlEl | _] -> 133 | exmpp_xml:get_cdata(UrlEl); 134 | _ -> 135 | undefined 136 | end, 137 | if 138 | is_binary(URL1), 139 | size(URL1) > 0 -> 140 | URL1; 141 | true -> 142 | %% Fallback 1: text() 143 | case exmpp_xml:get_cdata(Child) of 144 | Cdata 145 | when is_binary(Cdata), 146 | size(Cdata) > 0 -> 147 | Cdata; 148 | _ -> 149 | %% Fallback 2: @href 150 | exmpp_xml:get_attribute_as_binary(Child, <<"href">>, undefined) 151 | end 152 | end; 153 | (_, R) -> 154 | R 155 | end, undefined, exmpp_xml:get_elements(Xml, ChildName)), 156 | case R of 157 | undefined -> 158 | image1(Xml, ChildNames); 159 | _ -> 160 | R 161 | end. 162 | 163 | 164 | 165 | %% Separates items from the feed metadata 166 | -spec pick_items(xmlel()) -> {ok, xmlel(), [xmlel()]}. 167 | pick_items(#xmlel{} = RootEl) -> 168 | case exmpp_xml:get_name_as_list(RootEl) of 169 | %% Handle ATOM 170 | "feed" -> 171 | Entries = 172 | lists:filter( 173 | fun(#xmlel{name="entry"}) -> 174 | true; 175 | (_) -> 176 | false 177 | end, RootEl#xmlel.children), 178 | {ok, Entries}; 179 | 180 | %% Assume RSS 181 | _ -> 182 | Items = 183 | lists:map( 184 | fun(#xmlel{name="channel"}=Channel) -> 185 | lists:filter( 186 | fun(#xmlel{name="item"}) -> 187 | true; 188 | (_) -> 189 | false 190 | end, Channel#xmlel.children); 191 | (_) -> 192 | [] 193 | end, RootEl#xmlel.children), 194 | {ok, lists:append(Items)} 195 | end. 196 | 197 | 198 | item_title(ItemEl) -> 199 | %% Works the same as for the feed in RSS & ATOM 200 | title(ItemEl). 201 | 202 | item_lang(ItemEl) -> 203 | lang(ItemEl). 204 | 205 | item_summary(ItemEl) -> 206 | summary(ItemEl). 207 | 208 | item_id(ItemEl) -> 209 | item_id1(ItemEl, ["id", "guid", "link"]). 210 | 211 | item_id1(_ItemEl, []) -> 212 | undefined; 213 | item_id1(ItemEl, [ChildName | ChildNames]) -> 214 | R = 215 | lists:foldl( 216 | fun(Child, undefined) -> 217 | case exmpp_xml:get_cdata(Child) of 218 | Cdata 219 | when is_binary(Cdata), 220 | size(Cdata) > 0 -> 221 | Cdata; 222 | _ -> 223 | case {exmpp_xml:get_attribute_as_binary( 224 | Child, <<"rel">>, undefined), 225 | exmpp_xml:get_attribute_as_binary( 226 | Child, <<"href">>, undefined)} of 227 | {<<"alternate">>, Href} -> 228 | Href; 229 | _ -> 230 | undefined 231 | end 232 | end; 233 | (_, R) -> 234 | R 235 | end, undefined, exmpp_xml:get_elements(ItemEl, ChildName)), 236 | case R of 237 | undefined -> 238 | item_id1(ItemEl, ChildNames); 239 | _ -> 240 | R 241 | end. 242 | 243 | link(Xml) -> 244 | %% Handle ATOM 245 | case exmpp_xml:get_ns_as_list(Xml) of 246 | ?NS_ATOM -> 247 | lists:foldl( 248 | fun(LinkEl, undefined) -> 249 | case {exmpp_xml:get_attribute_as_binary( 250 | LinkEl, <<"rel">>, undefined), 251 | exmpp_xml:get_attribute_as_binary( 252 | LinkEl, <<"href">>, undefined)} of 253 | {<<"alternate">>, Href} -> 254 | Href; 255 | _ -> 256 | undefined 257 | end; 258 | (_, Href) -> 259 | Href 260 | end, undefined, exmpp_xml:get_elements(Xml, "link")); 261 | _ -> 262 | link1(Xml, ["link", "url"]) 263 | end. 264 | 265 | link1(_Xml, []) -> 266 | undefined; 267 | link1(Xml, [ChildName | ChildNames]) -> 268 | R = 269 | lists:foldl( 270 | fun(Child, undefined) -> 271 | case exmpp_xml:get_cdata(Child) of 272 | Cdata 273 | when is_binary(Cdata), 274 | size(Cdata) > 0 -> 275 | Cdata; 276 | _ -> 277 | undefined 278 | end; 279 | (_, R) -> 280 | R 281 | end, undefined, exmpp_xml:get_elements(Xml, ChildName)), 282 | case R of 283 | undefined -> 284 | link1(Xml, ChildNames); 285 | _ -> 286 | R 287 | end. 288 | 289 | item_published(ItemEl) -> 290 | Published = 291 | item_published1(ItemEl, ["pubDate", "published", "updated"]), 292 | try fix_timestamp(Published) of 293 | <> -> 294 | Result 295 | catch exit:_Reason -> 296 | Published 297 | end. 298 | 299 | item_published1(_ItemEl, []) -> 300 | undefined; 301 | item_published1(ItemEl, [ChildName | ChildNames]) -> 302 | R = 303 | lists:foldl( 304 | fun(Child, undefined) -> 305 | case exmpp_xml:get_cdata(Child) of 306 | Cdata 307 | when is_binary(Cdata), 308 | size(Cdata) > 0 -> 309 | Cdata; 310 | _ -> 311 | undefined 312 | end; 313 | (_, R) -> 314 | R 315 | end, undefined, exmpp_xml:get_elements(ItemEl, ChildName)), 316 | case R of 317 | undefined -> 318 | item_published1(ItemEl, ChildNames); 319 | _ -> 320 | R 321 | end. 322 | 323 | %% Some date parsing for item_published1/2 324 | fix_timestamp(undefined) -> 325 | util:iso8601(calendar:local_time(), local); 326 | 327 | fix_timestamp(S) -> 328 | %% Tue, 12 Jun 2012 22:43:48 +0200 329 | case run_re(S, "(\\d+) (\\S+) (\\d+) (\\d+):(\\d+):(\\d+)\\s*([0-9:+\\-]*)") of 330 | {match, [Day, <>, Year, Hour, Min, Sec, Tz]} -> 331 | Date = {bin_to_int(Year), month_to_int(Mon), bin_to_int(Day)}, 332 | Time = {bin_to_int(Hour), bin_to_int(Min), bin_to_int(Sec)}, 333 | case run_re(Tz, "([+\\-])(\\d{1,2}):?(\\d*)") of 334 | {match, [Sig, TzH1, TzM1]} -> 335 | TzH = 336 | case Sig of 337 | <<"+">> -> 1; 338 | <<"-">> -> -1 339 | end * 340 | bin_to_int(TzH1), 341 | TzM = 342 | case TzM1 of 343 | <<>> -> 344 | 0; 345 | _ -> 346 | bin_to_int(TzM1) 347 | end; 348 | nomatch -> 349 | TzH = 0, 350 | TzM = 0 351 | end, 352 | util:iso8601({Date, Time}, {TzH, TzM}); 353 | nomatch -> 354 | S 355 | end. 356 | 357 | run_re(S, RE) -> 358 | case get({?MODULE, cached_re, RE}) of 359 | undefined -> 360 | {ok, CompiledRE} = re:compile(RE), 361 | put({?MODULE, cached_re, RE}, CompiledRE); 362 | CompiledRE -> 363 | ok 364 | end, 365 | 366 | case re:run(S, CompiledRE) of 367 | {match, [_All | Captures]} -> 368 | {match, 369 | lists:map(fun({CapStart, CapLength}) -> 370 | {_, S1} = split_binary(S, CapStart), 371 | {S2, _} = split_binary(S1, CapLength), 372 | S2 373 | end, Captures)}; 374 | nomatch -> 375 | nomatch 376 | end. 377 | 378 | bin_to_int(B) -> 379 | list_to_integer( 380 | binary_to_list(B)). 381 | 382 | month_to_int(M) -> 383 | month_to_int1(string:to_lower(binary_to_list(M))). 384 | 385 | month_to_int1("jan") -> 1; 386 | month_to_int1("feb") -> 2; 387 | month_to_int1("mar") -> 3; 388 | month_to_int1("apr") -> 4; 389 | month_to_int1("may") -> 5; 390 | month_to_int1("jun") -> 6; 391 | month_to_int1("jul") -> 7; 392 | month_to_int1("aug") -> 8; 393 | month_to_int1("sep") -> 9; 394 | month_to_int1("oct") -> 10; 395 | month_to_int1("nov") -> 11; 396 | month_to_int1("dec") -> 12. 397 | 398 | 399 | item_link(ItemEl) -> 400 | ?MODULE:link(ItemEl). 401 | 402 | item_payment(ItemEl) -> 403 | lists:foldl( 404 | fun(LinkEl, undefined) -> 405 | case {exmpp_xml:get_attribute_as_binary( 406 | LinkEl, <<"rel">>, undefined), 407 | exmpp_xml:get_attribute_as_binary( 408 | LinkEl, <<"href">>, undefined)} of 409 | {<<"payment">>, Href} -> 410 | Href; 411 | _ -> 412 | undefined 413 | end; 414 | (_, Href) -> 415 | Href 416 | end, undefined, exmpp_xml:get_elements(ItemEl, "link")). 417 | 418 | 419 | item_image(ItemEl) -> 420 | image(ItemEl). 421 | 422 | -spec item_enclosures(xmlel()) 423 | -> [{binary(), 424 | (binary() | undefined), 425 | (binary() | undefined) 426 | }]. 427 | item_enclosures(ItemEl) -> 428 | lists:reverse( 429 | lists:foldl(fun(#xmlel{name="enclosure"}=El, URLs) -> 430 | Title = exmpp_xml:get_attribute_as_binary(El, <<"title">>, undefined), 431 | Type = normalize_type( 432 | exmpp_xml:get_attribute_as_binary(El, <<"type">>, undefined)), 433 | GUID = exmpp_xml:get_attribute_as_binary(El, ?NS_BITLOVE, <<"guid">>, undefined), 434 | case exmpp_xml:get_attribute_as_binary(El, <<"url">>, undefined) of 435 | URL when is_binary(URL) -> 436 | [{URL, Type, Title, GUID} | URLs]; 437 | _ -> 438 | case exmpp_xml:get_cdata(El) of 439 | URL when is_binary(URL), size(URL) > 6 -> 440 | [{URL, Type, Title, GUID} | URLs]; 441 | _ -> 442 | URLs 443 | end 444 | end; 445 | (#xmlel{name="link"}=El, URLs) -> 446 | case exmpp_xml:get_attribute_as_binary(El, <<"rel">>, undefined) of 447 | <<"enclosure">> -> 448 | case exmpp_xml:get_attribute_as_binary(El, <<"href">>, undefined) of 449 | URL when is_binary(URL), size(URL) > 6 -> 450 | Title = exmpp_xml:get_attribute_as_binary(El, <<"title">>, undefined), 451 | Type = normalize_type( 452 | exmpp_xml:get_attribute_as_binary(El, <<"type">>, undefined)), 453 | GUID = exmpp_xml:get_attribute_as_binary(El, ?NS_BITLOVE, <<"guid">>, undefined), 454 | [{URL, Type, Title, GUID} | URLs]; 455 | _ -> 456 | URLs 457 | end; 458 | _Rel -> 459 | URLs 460 | end; 461 | (_, URLs) -> 462 | URLs 463 | end, [], exmpp_xml:get_child_elements(ItemEl))). 464 | 465 | 466 | %% Drop MIME Type parameters ";..." 467 | normalize_type(B) when is_binary(B) -> 468 | L1 = binary_to_list(B), 469 | case lists:takewhile(fun(C) -> 470 | C =/= $; 471 | end, L1) of 472 | L2 when L1 == L2 -> 473 | B; 474 | L2 -> 475 | list_to_binary(L2) 476 | end; 477 | normalize_type(A) -> 478 | A. 479 | -------------------------------------------------------------------------------- /apps/feeds/src/feeds_sup.erl: -------------------------------------------------------------------------------- 1 | 2 | -module(feeds_sup). 3 | 4 | -behaviour(supervisor). 5 | 6 | %% API 7 | -export([start_link/0]). 8 | 9 | %% Supervisor callbacks 10 | -export([init/1]). 11 | 12 | %% Helper macro for declaring children of supervisor 13 | -define(CHILD(I, Type), {I, {I, start_link, []}, permanent, 5000, Type, [I]}). 14 | 15 | %% =================================================================== 16 | %% API functions 17 | %% =================================================================== 18 | 19 | start_link() -> 20 | supervisor:start_link({local, ?MODULE}, ?MODULE, []). 21 | 22 | %% =================================================================== 23 | %% Supervisor callbacks 24 | %% =================================================================== 25 | 26 | init([]) -> 27 | UpdateLoop = {update_loop, {feeds_update, start_link, []}, 28 | permanent, 1000, worker, [feeds_update]}, 29 | {ok, { {one_for_one, 5, 10}, [UpdateLoop]} }. 30 | 31 | -------------------------------------------------------------------------------- /apps/feeds/src/feeds_update.erl: -------------------------------------------------------------------------------- 1 | -module(feeds_update). 2 | 3 | -export([start_link/0, update_loop/0, update/1]). 4 | 5 | -include_lib("model/include/model.hrl"). 6 | 7 | -define(INTERVAL, 600). 8 | 9 | start_link() -> 10 | {ok, spawn_link(fun update_loop/0)}. 11 | 12 | update_loop() -> 13 | case model_feeds:to_update(?INTERVAL) of 14 | {ok, {URL, Delay}} 15 | when Delay =< 0 -> 16 | io:format("Update ~s (scheduled in ~.2fs)~n", [URL, Delay]), 17 | case (catch update(URL)) of 18 | {'EXIT', Reason} -> 19 | error_logger:error_msg("Error updating ~s:~n~p~n", [URL, Reason]); 20 | _ -> 21 | ok 22 | end; 23 | {ok, {_, Delay}} -> 24 | io:format("Nothing to update, sleeping ~.2f...~n", [Delay]), 25 | receive 26 | after trunc(Delay * 1000) -> 27 | ok 28 | end 29 | end, 30 | 31 | ?MODULE:update_loop(). 32 | 33 | 34 | update(URL) when is_binary(URL) -> 35 | update(binary_to_list(URL)); 36 | update(URL) -> 37 | {ok, Etag1, LastModified1} = model_feeds:prepare_update(URL), 38 | spawn(fun() -> 39 | case (catch update1(URL, Etag1, LastModified1)) of 40 | {'EXIT', Reason} -> 41 | error_logger:error_msg("feeds update failed for ~s~n~p~n", [URL, Reason]); 42 | _ -> 43 | ok 44 | end 45 | end), 46 | ok. 47 | 48 | update1(URL, Etag1, LastModified1) -> 49 | NormalizeURL = fun(undefined) -> 50 | undefined; 51 | (URL1) -> 52 | url:join(URL, URL1) 53 | end, 54 | R1 = 55 | try feeds_fetch:fetch(URL, Etag1, LastModified1) of 56 | {ok, {Etag, LastModified}, FeedEl} -> 57 | try 58 | {ok, Items1} = 59 | feeds_parse:pick_items(FeedEl), 60 | io:format("Picked ~b items from feed ~s~n", 61 | [length(Items1), URL]), 62 | FeedXml1 = iolist_to_binary(feeds_parse:serialize(FeedEl)), 63 | ChannelEl = case feeds_parse:get_channel(FeedEl) of 64 | undefined -> 65 | exit(invalid_feed); 66 | ChannelEl1 -> 67 | ChannelEl1 68 | end, 69 | Title1 = feeds_parse:title(ChannelEl), 70 | Lang1 = feeds_parse:lang(ChannelEl), 71 | Summary1 = feeds_parse:summary(ChannelEl), 72 | Homepage1 = NormalizeURL(feeds_parse:link(ChannelEl)), 73 | Image1 = NormalizeURL(feeds_parse:image(ChannelEl)), 74 | Items2 = 75 | lists:foldl( 76 | fun(ItemXml, Items2) -> 77 | try xml_to_feed_item(URL, NormalizeURL, ItemXml) of 78 | #feed_item{} = Item1 -> 79 | %% Fill in image 80 | Item2 = case Item1 of 81 | #feed_item{image = <<_:8, _/binary>>} -> 82 | Item1; 83 | _ -> 84 | Item1#feed_item{image = Image1} 85 | end, 86 | %% Fill in lang 87 | Item3 = case Item2 of 88 | #feed_item{lang = <<_:8, _/binary>>} -> 89 | Item2; 90 | _ -> 91 | Item2#feed_item{lang = Lang1} 92 | end, 93 | [Item3 | Items2]; 94 | _ -> 95 | %%io:format("Malformed item: ~s~n", [exmpp_xml:document_to_binary(ItemXml)]), 96 | Items2 97 | catch exit:Reason -> 98 | error_logger:warning_msg("Cannot extract from feed item: ~s~n~p~n~s~n", [URL, Reason, ItemXml]), 99 | Items2 100 | end 101 | end, [], Items1), 102 | if 103 | length(Items2) < length(Items1) -> 104 | error_logger:warning_msg("Lost ~B of ~B items of ~s~n", 105 | [length(Items1) - length(Items2), length(Items1), URL]); 106 | true -> 107 | ok 108 | end, 109 | {ok, {Etag, LastModified}, 110 | FeedXml1, 111 | Title1, Lang1, Summary1, Homepage1, Image1, 112 | lists:reverse(Items2)} 113 | catch exit:Reason1 -> 114 | {error, {Etag, LastModified}, Reason1} 115 | end; 116 | not_modified -> 117 | %% Well, show it to the user... (in green :) 118 | {error, {Etag1, LastModified1}, not_modified}; 119 | {error, Reason1} -> 120 | {error, {Etag1, LastModified1}, Reason1} 121 | catch exit:Reason1 -> 122 | error_logger:error_msg("fetching ~s failed:~n~p~n", [URL, Reason1]), 123 | {error, {Etag1, LastModified1}, Reason1} 124 | end, 125 | 126 | case R1 of 127 | {ok, {Etag2, LastModified2}, 128 | FeedXml, 129 | Title2, Lang2, Summary2, Homepage2, Image2, 130 | Items3} -> 131 | model_feeds:write_update(URL, {Etag2, LastModified2}, 132 | null, FeedXml, 133 | Title2, Lang2, Summary2, Homepage2, Image2, 134 | Items3), 135 | ok; 136 | %% HTTP 304 Not Modified: 137 | {error, {Etag2, LastModified2}, 138 | {http, 304}} -> 139 | model_feeds:write_update(URL, {Etag2, LastModified2}, 140 | not_modified, null, 141 | null, null, null, null, null, 142 | []), 143 | ok; 144 | {error, {Etag2, LastModified2}, Reason} -> 145 | Error = case Reason of 146 | undefined -> null; 147 | _ -> list_to_binary(io_lib:format("~p",[Reason])) 148 | end, 149 | model_feeds:write_update(URL, {Etag2, LastModified2}, 150 | Error, null, 151 | null, null, null, null, null, 152 | []), 153 | {error, Reason} 154 | end. 155 | 156 | xml_to_feed_item(Feed, NormalizeURL, Xml) -> 157 | Id = feeds_parse:item_id(Xml), 158 | Title = feeds_parse:item_title(Xml), 159 | Lang = feeds_parse:item_lang(Xml), 160 | Summary = feeds_parse:item_summary(Xml), 161 | Published = feeds_parse:item_published(Xml), 162 | Homepage = NormalizeURL(feeds_parse:item_link(Xml)), 163 | Payment = NormalizeURL(feeds_parse:item_payment(Xml)), 164 | Image = NormalizeURL(feeds_parse:item_image(Xml)), 165 | Enclosures = lists:map( 166 | fun({Enclosure, EnclosureType, EnclosureTitle, EnclosureGUID1}) -> 167 | EnclosureGUID2 = 168 | if 169 | is_binary(EnclosureGUID1) -> EnclosureGUID1; 170 | true -> Id %% Fallback to item id 171 | end, 172 | {NormalizeURL(Enclosure), EnclosureType, EnclosureTitle, EnclosureGUID2} 173 | end, 174 | feeds_parse:item_enclosures(Xml)), 175 | if 176 | is_binary(Id), 177 | is_binary(Title), 178 | is_binary(Published) -> 179 | #feed_item{feed = Feed, 180 | id = Id, 181 | title = Title, 182 | lang = Lang, 183 | summary = Summary, 184 | published = Published, 185 | homepage = Homepage, 186 | payment = Payment, 187 | image = Image, 188 | enclosures = Enclosures}; 189 | true -> 190 | %% Drop this 191 | null 192 | end. 193 | 194 | 195 | -------------------------------------------------------------------------------- /apps/hasher/src/hasher.app.src: -------------------------------------------------------------------------------- 1 | {application, hasher, 2 | [ 3 | {description, ""}, 4 | {vsn, "1"}, 5 | {registered, []}, 6 | {applications, [ 7 | kernel, 8 | stdlib, 9 | sasl, 10 | lhttpc, 11 | cowlib, 12 | model, 13 | shared 14 | ]}, 15 | {mod, { hasher_app, []}}, 16 | {env, [{announce_url, <<"http://t.bitlove.org/announce">>}]} 17 | ]}. 18 | -------------------------------------------------------------------------------- /apps/hasher/src/hasher_app.erl: -------------------------------------------------------------------------------- 1 | -module(hasher_app). 2 | 3 | -behaviour(application). 4 | 5 | %% Application callbacks 6 | -export([start/2, stop/1]). 7 | 8 | %% =================================================================== 9 | %% Application callbacks 10 | %% =================================================================== 11 | 12 | start(_StartType, _StartArgs) -> 13 | hasher_sup:start_link(). 14 | 15 | stop(_State) -> 16 | ok. 17 | -------------------------------------------------------------------------------- /apps/hasher/src/hasher_hash.erl: -------------------------------------------------------------------------------- 1 | -module(hasher_hash). 2 | 3 | -export([update_torrent/2, make_torrent/1]). 4 | 5 | -define(DEFAULT_PIECE_LENGTH, trunc(math:pow(2, 20))). 6 | 7 | update_torrent(URL, OldTorrent) -> 8 | OldInfo = benc:parse(OldTorrent), 9 | {value, {<<"url-list">>, OldURLList}} = lists:keysearch(<<"url-list">>, 1, OldInfo), 10 | case lists:member(URL, OldURLList) of 11 | true -> 12 | io:format("update_torrent ~s didn't add a new URL~n", [URL]), 13 | OldTorrent; 14 | false -> 15 | io:format("update_torrent: new url ~s~n", [URL]), 16 | NewInfo = lists:keystore(<<"url-list">>, 1, OldInfo, {<<"url-list">>, [URL | OldURLList]}), 17 | benc:to_binary(NewInfo) 18 | end. 19 | 20 | make_torrent(URL) -> 21 | {ok, Size, Pieces, ETag, LastModified} = hash_torrent(URL), 22 | io:format("hash_torrent ~p - Size: ~p, Pieces: ~p~n", [URL, Size, length(Pieces)]), 23 | Name = extract_name_from_url(URL), 24 | InfoValue = 25 | [{<<"name">>, Name}, 26 | {<<"piece length">>, ?DEFAULT_PIECE_LENGTH}, 27 | {<<"pieces">>, list_to_binary(Pieces)}, 28 | {<<"length">>, Size} 29 | ], 30 | AnnounceURL = 31 | case application:get_env(hasher, announce_url) of 32 | {ok, AnnounceURL1} when is_list(AnnounceURL1) -> 33 | list_to_binary(AnnounceURL1); 34 | {ok, AnnounceURL1} when is_binary(AnnounceURL1) -> 35 | AnnounceURL1 36 | end, 37 | Torrent = 38 | [{<<"announce">>, AnnounceURL}, 39 | {<<"url-list">>, [URL]}, 40 | {<<"info">>, InfoValue} 41 | ], 42 | InfoHash = benc:hash(InfoValue), 43 | {ok, InfoHash, Name, Size, benc:to_binary(Torrent), ETag, LastModified}. 44 | 45 | extract_name_from_url(URL) -> 46 | Parts = split_path(URL, []), 47 | lists:foldl(fun(Part, R) -> 48 | if 49 | is_binary(Part), 50 | size(Part) > 0 -> 51 | Part; 52 | true -> 53 | R 54 | end 55 | end, undefined, Parts). 56 | 57 | %% split_path has been bluntly copied from cowboy_router. The function 58 | %% was previously exported. :-( Its license applies. 59 | %% 60 | %% Following RFC2396, this function may return path segments containing any 61 | %% character, including / if, and only if, a / was escaped 62 | %% and part of a path segment. 63 | -spec split_path(binary()) -> [binary()] | badrequest. 64 | split_path(<< $/, Path/bits >>) -> 65 | split_path(Path, []); 66 | split_path(_) -> 67 | badrequest. 68 | 69 | split_path(Path, Acc) -> 70 | try 71 | case binary:match(Path, <<"/">>) of 72 | nomatch when Path =:= <<>> -> 73 | lists:reverse([cow_qs:urldecode(S) || S <- Acc]); 74 | nomatch -> 75 | lists:reverse([cow_qs:urldecode(S) || S <- [Path|Acc]]); 76 | {Pos, _} -> 77 | << Segment:Pos/binary, _:8, Rest/bits >> = Path, 78 | split_path(Rest, [Segment|Acc]) 79 | end 80 | catch 81 | error:badarg -> 82 | badrequest 83 | end. 84 | 85 | 86 | -spec hash_torrent(binary()) 87 | -> {ok, integer(), binary(), binary(), binary()}. 88 | hash_torrent(URL) -> 89 | case storage:resource_info(URL) of 90 | {ok, _, Size, ETag, LastModified} 91 | when is_integer(Size) -> 92 | Storage = {storage, [{URL, Size}]}, 93 | Pieces = 94 | map_pieces( 95 | Size, fun(Offset, Length) -> 96 | io:format("Hash ~s: ~B%~n", [URL, trunc(100 * (Offset + Length) / Size)]), 97 | hash_piece(Storage, Offset, Length) 98 | end), 99 | {ok, Size, Pieces, ETag, LastModified}; 100 | 101 | {ok, _, _, _, _} -> 102 | exit(no_content_length); 103 | 104 | {error, E} -> 105 | {error, E} 106 | end. 107 | 108 | map_pieces(TotalLength, F) -> 109 | map_pieces(TotalLength, 0, F). 110 | 111 | map_pieces(0, _, _) -> 112 | []; 113 | map_pieces(Remain, N, F) -> 114 | Offset = N * ?DEFAULT_PIECE_LENGTH, 115 | Length = min(?DEFAULT_PIECE_LENGTH, Remain), 116 | [F(Offset, Length) | map_pieces(Remain - Length, N + 1, F)]. 117 | 118 | hash_piece(Storage, Offset, Length) -> 119 | {Sha, ActualLength} = 120 | storage:fold(Storage, Offset, Length, 121 | fun({Sha, ActualLength}, Data) -> 122 | {crypto:hash_update(Sha, Data), 123 | ActualLength + size(Data)} 124 | end, {crypto:hash_init(sha), 0}), 125 | Digest = crypto:hash_final(Sha), 126 | if 127 | ActualLength == Length -> 128 | Digest; 129 | true -> 130 | exit({expected_but_received, Length, ActualLength}) 131 | end. 132 | -------------------------------------------------------------------------------- /apps/hasher/src/hasher_recheck.erl: -------------------------------------------------------------------------------- 1 | -module(hasher_recheck). 2 | 3 | -export([start_link/0, loop/0]). 4 | 5 | 6 | start_link() -> 7 | {ok, spawn_link(fun loop/0)}. 8 | 9 | %% Perform just HEAD requests, and if needed recheck async. 10 | loop() -> 11 | case model_enclosures:to_recheck() of 12 | nothing -> 13 | %% Idle 5..10s 14 | Delay = 5000 + random:uniform(5000), 15 | receive 16 | after Delay -> 17 | ok 18 | end; 19 | {ok, URL, Length, ETag, LastModified} -> 20 | io:format("Recheck ~s~n", [URL]), 21 | ETagList = if 22 | is_binary(ETag) -> binary_to_list(ETag); 23 | true -> ETag 24 | end, 25 | LastModifiedList = if 26 | is_binary(LastModified) -> binary_to_list(LastModified); 27 | true -> LastModified 28 | end, 29 | case storage:resource_info(URL) of 30 | {ok, _Type, ContentLength, ContentETag, ContentLastModified} 31 | when (Length == ContentLength andalso 32 | (ContentETag == ETagList orelse 33 | ContentETag == undefined orelse 34 | ETagList == null) andalso 35 | (ContentLastModified == LastModifiedList orelse 36 | ContentLastModified == undefined orelse 37 | LastModifiedList == null) 38 | ) -> 39 | nothing_changed; 40 | 41 | {ok, _Type, ContentLength, ContentETag, ContentLastModified} -> 42 | io:format("Need recheck: ~s~n", [URL]), 43 | io:format("\tLength: ~p /= ~p~n", [Length, ContentLength]), 44 | io:format("\tETag: ~p /= ~p~n", [ETagList, ContentETag]), 45 | io:format("\tLast-Modified: ~p /= ~p~n", [LastModifiedList, ContentLastModified]), 46 | hasher_sup:recheck(URL); 47 | 48 | {error, {http, HttpStatus}} -> 49 | io:format("Recheck ~s resulted in HTTP ~p~n", [URL, HttpStatus]), 50 | 51 | %% We've got an HTTP code, meaning the network is 52 | %% up but the URL has disappeared! 53 | model_enclosures:set_torrent( 54 | URL, list_to_binary(io_lib:format("HTTP ~B", [HttpStatus])), <<"">>, null, null, null); 55 | 56 | {error, {nxdomain, _Reason}} -> 57 | io:format("Recheck ~s failed on DNS~n", [URL]), 58 | 59 | %% We've got an HTTP code, meaning the network is 60 | %% up but the URL has disappeared! 61 | model_enclosures:set_torrent( 62 | URL, <<"NXDOMAIN">>, <<"">>, null, null, null); 63 | 64 | {error, E} -> 65 | io:format("Recheck ~s failed:~n~p~n", [URL, E]) 66 | 67 | end 68 | end, 69 | 70 | ?MODULE:loop(). 71 | 72 | -------------------------------------------------------------------------------- /apps/hasher/src/hasher_sup.erl: -------------------------------------------------------------------------------- 1 | 2 | -module(hasher_sup). 3 | 4 | -behaviour(supervisor). 5 | 6 | %% API 7 | -export([start_link/0, recheck/1]). 8 | 9 | %% Supervisor callbacks 10 | -export([init/1]). 11 | 12 | %% Helper macro for declaring children of supervisor 13 | -define(CHILD(I, Type), {I, {I, start_link, []}, permanent, 5000, Type, [I]}). 14 | 15 | %% =================================================================== 16 | %% API functions 17 | %% =================================================================== 18 | 19 | start_link() -> 20 | supervisor:start_link({local, ?MODULE}, ?MODULE, []). 21 | 22 | pick_any_worker() -> 23 | Pids = [Pid 24 | || {{worker, _}, Pid, _, _} <- supervisor:which_children(?MODULE)], 25 | random:seed(erlang:now()), 26 | lists:nth(random:uniform(length(Pids)), Pids). 27 | 28 | %% To be used interactively: 29 | recheck(URL) -> 30 | Pid = pick_any_worker(), 31 | link(Pid), 32 | Pid ! {recheck, URL, self()}, 33 | receive 34 | recheck_done -> 35 | unlink(Pid), 36 | ok 37 | end. 38 | 39 | 40 | %% =================================================================== 41 | %% Supervisor callbacks 42 | %% =================================================================== 43 | 44 | init([]) -> 45 | NWorkers = 16, 46 | Workers = [{{worker, N}, {hasher_worker, start_link, []}, 47 | permanent, 5000, worker, [hasher_worker, hasher_hash]} 48 | || N <- lists:seq(1, NWorkers)], 49 | NRecheckers = 8, 50 | Recheckers = [{{checker, N}, {hasher_recheck, start_link, []}, 51 | permanent, 5000, worker, [hasher_recheck]} 52 | || N <- lists:seq(1, NRecheckers)], 53 | {ok, { {one_for_one, 100, 1}, Workers ++ Recheckers} }. 54 | 55 | -------------------------------------------------------------------------------- /apps/hasher/src/hasher_worker.erl: -------------------------------------------------------------------------------- 1 | -module(hasher_worker). 2 | 3 | -export([start_link/0, loop/0, hash/1]). 4 | 5 | start_link() -> 6 | {ok, spawn_link(fun init/0)}. 7 | 8 | 9 | init() -> 10 | util:seed_random(), 11 | process_flag(trap_exit, true), 12 | loop(). 13 | 14 | loop() -> 15 | case model_enclosures:to_hash() of 16 | {ok, URL} -> 17 | io:format("To hash: ~p~n", [URL]), 18 | hash(URL); 19 | nothing -> 20 | SleepTime = 30 + random:uniform(30), 21 | io:format("Nothing to hash, sleeping ~Bs~n", [SleepTime]), 22 | receive 23 | {'EXIT', Pid, Reason} -> 24 | error_logger:error_msg("Hasher ~p exited:~n~p~n", [Pid, Reason]); 25 | {recheck, URL, From} -> 26 | io:format("To recheck: ~p~n", [URL]), 27 | hash(URL), 28 | From ! recheck_done 29 | after SleepTime * 1000 -> 30 | ok 31 | end 32 | end, 33 | 34 | ?MODULE:loop(). 35 | 36 | hash(URL) -> 37 | case (catch hash1(URL)) of 38 | {'EXIT', Reason} -> 39 | error_logger:error_msg("Failed hashing ~s~n~p~n", [URL, Reason]), 40 | model_enclosures:set_torrent( 41 | URL, list_to_binary(io_lib:format("~p", [Reason])), <<"">>, null, null, null); 42 | _ -> 43 | ok 44 | end. 45 | 46 | hash1(URL) -> 47 | {ok, InfoHash, Name, Size, TorrentFile, ETag, LastModified} = 48 | hasher_hash:make_torrent(URL), 49 | Updater = fun(OldTorrentFile) -> 50 | hasher_hash:update_torrent(URL, OldTorrentFile) 51 | end, 52 | model_torrents:add_torrent(InfoHash, Name, Size, TorrentFile, Updater), 53 | model_enclosures:set_torrent(URL, <<"">>, InfoHash, Size, ETag, LastModified). 54 | -------------------------------------------------------------------------------- /apps/model/include/model.hrl: -------------------------------------------------------------------------------- 1 | -record(download, { 2 | %% User key: 3 | user :: binary(), 4 | slug :: binary(), 5 | %% Enclosure key: 6 | feed :: binary(), 7 | item :: binary(), 8 | enclosure :: (binary() | null), 9 | %% Torrent data: 10 | info_hash :: binary(), 11 | name :: binary(), 12 | size :: integer(), 13 | type :: binary(), 14 | %% Item data: 15 | feed_title :: (binary() | null), 16 | title :: binary(), 17 | lang :: (binary() | null), 18 | summary :: (binary() | null), 19 | published :: calendar:datetime(), 20 | homepage :: binary(), 21 | payment :: binary(), 22 | %% Enclosure info: 23 | image :: binary(), 24 | %% Scrape data: 25 | seeders :: integer(), 26 | leechers :: integer(), 27 | upspeed :: integer(), 28 | downspeed :: integer(), 29 | downloaded :: integer() 30 | }). 31 | 32 | -record(feed_item, { 33 | user :: binary(), 34 | slug :: binary(), 35 | feed :: binary(), 36 | id :: binary(), 37 | feed_title :: (binary() | null), 38 | title :: (binary() | null), 39 | lang :: (binary() | null), 40 | summary :: (binary() | null), 41 | published :: calendar:datetime(), 42 | homepage :: binary(), 43 | payment :: binary(), 44 | image :: binary(), 45 | enclosures :: [binary()], 46 | downloads :: [#download{}] 47 | }). 48 | -------------------------------------------------------------------------------- /apps/model/src/model.app.src: -------------------------------------------------------------------------------- 1 | {application, model, 2 | [ 3 | {description, ""}, 4 | {vsn, "1"}, 5 | {registered, [pool_torrents, pool_tracker, pool_stats]}, 6 | {applications, [ 7 | kernel, 8 | stdlib, 9 | poolboy, 10 | epgsql 11 | ]}, 12 | {mod, {model_app, []}}, 13 | {env, [{pools, 14 | [{pool_torrents, [{size, 4}, 15 | {max_overflow, 16}, 16 | {host, "localhost"}, 17 | {database, "prittorrent"}, 18 | {user, "prittorrent"}, 19 | {password, "1234"}]}, 20 | {pool_users, [{size, 4}, 21 | {max_overflow, 16}, 22 | {host, "localhost"}, 23 | {database, "prittorrent"}, 24 | {user, "prittorrent"}, 25 | {password, "1234"}]}, 26 | {pool_tracker, [{size, 8}, 27 | {max_overflow, 16}, 28 | {host, "localhost"}, 29 | {database, "prittorrent"}, 30 | {user, "prittorrent"}, 31 | {password, "1234"}]}, 32 | {pool_stats, [{size, 1}, 33 | {max_overflow, 8}, 34 | {host, "localhost"}, 35 | {database, "prittorrent"}, 36 | {user, "prittorrent"}, 37 | {password, "1234"}]}, 38 | {pool_scrape_queue, [{size, 1}, 39 | {max_overflow, 8}, 40 | {host, "localhost"}, 41 | {database, "prittorrent"}, 42 | {user, "prittorrent"}, 43 | {password, "1234"}]}, 44 | {pool_graphs, [{size, 2}, 45 | {max_overflow, 2}, 46 | {host, "localhost"}, 47 | {database, "prittorrent"}, 48 | {user, "prittorrent"}, 49 | {password, "1234"}]} 50 | ]} 51 | ]} 52 | ]}. 53 | -------------------------------------------------------------------------------- /apps/model/src/model_app.erl: -------------------------------------------------------------------------------- 1 | -module(model_app). 2 | 3 | -behaviour(application). 4 | 5 | %% Application callbacks 6 | -export([start/2, stop/1]). 7 | 8 | %% =================================================================== 9 | %% Application callbacks 10 | %% =================================================================== 11 | 12 | start(_StartType, _StartArgs) -> 13 | io:format("start~n"), 14 | {ok, Pools} = application:get_env(pools), 15 | {ok, _Sup} = model_sup:start_link(Pools). 16 | 17 | stop(_State) -> 18 | ok. 19 | -------------------------------------------------------------------------------- /apps/model/src/model_enclosures.erl: -------------------------------------------------------------------------------- 1 | -module(model_enclosures). 2 | 3 | -export([to_hash/0, to_recheck/0, set_torrent/6, 4 | get_info_hash_by_name/3, get_torrent_by_name/3, 5 | purge/3, 6 | recent_downloads/1, popular_downloads/2, 7 | user_downloads/2, feed_downloads/2, 8 | enclosure_downloads/1]). 9 | 10 | -include("../include/model.hrl"). 11 | 12 | -define(POOL, pool_users). 13 | -define(Q(Stmt, Params), model_sup:equery(?POOL, Stmt, Params)). 14 | -define(T(Fun), model_sup:transaction(?POOL, Fun)). 15 | 16 | to_hash() -> 17 | ?T(fun(Q) -> 18 | LockResult = Q("LOCK TABLE \"enclosure_torrents\" IN SHARE ROW EXCLUSIVE MODE", []), 19 | case LockResult of 20 | {ok, [], []} -> 21 | case Q("SELECT \"enclosure_url\" FROM enclosure_to_hash()", []) of 22 | {ok, _, [{URL}]} 23 | when is_binary(URL), 24 | size(URL) > 0 -> 25 | {ok, URL}; 26 | {ok, _, [{null}]} -> 27 | nothing 28 | end; 29 | {error, timeout} -> 30 | nothing 31 | end 32 | end). 33 | 34 | to_recheck() -> 35 | ?T(fun(Q) -> 36 | LockResult = Q("LOCK TABLE \"enclosure_torrents\" IN SHARE ROW EXCLUSIVE MODE", []), 37 | case LockResult of 38 | {ok, [], []} -> 39 | case Q("SELECT e_url, e_length, e_etag, e_last_modified FROM enclosure_to_recheck()", []) of 40 | {ok, _, [{URL, Length, ETag, LastModified}]} 41 | when is_binary(URL), 42 | size(URL) > 0 -> 43 | {ok, URL, Length, ETag, LastModified}; 44 | {ok, _, [{null, null, null, null}]} -> 45 | nothing 46 | end; 47 | {error, timeout} -> 48 | nothing 49 | end 50 | end). 51 | 52 | 53 | set_torrent(URL, Error, InfoHash, Length, undefined, LastModified) -> 54 | set_torrent(URL, Error, InfoHash, Length, null, LastModified); 55 | set_torrent(URL, Error, InfoHash, Length, ETag, undefined) -> 56 | set_torrent(URL, Error, InfoHash, Length, ETag, null); 57 | set_torrent(URL, Error, InfoHash, Length, ETag, LastModified) -> 58 | ?T(fun(Q) -> 59 | case Q("SELECT count(\"url\") FROM enclosure_torrents WHERE \"url\"=$1", [URL]) of 60 | {ok, _, [{0}]} -> 61 | Q("INSERT INTO enclosure_torrents (\"url\", \"last_update\", \"length\", \"etag\", \"last_modified\", \"info_hash\", \"error\") VALUES ($1, CURRENT_TIMESTAMP, $2, $3, $4, $5, $6)", [URL, Length, ETag, LastModified, InfoHash, Error]); 62 | {ok, _, [{1}]} -> 63 | Q("UPDATE enclosure_torrents SET \"last_update\"=CURRENT_TIMESTAMP, \"length\"=$2, \"etag\"=$3, \"last_modified\"=$4, \"info_hash\"=$5, \"error\"=$6 WHERE \"url\"=$1", [URL, Length, ETag, LastModified, InfoHash, Error]) 64 | end 65 | end). 66 | 67 | get_torrent_by_name(UserName, Slug, Name) -> 68 | case ?Q("SELECT torrents.\"torrent\" FROM user_feeds JOIN enclosures USING (feed) JOIN enclosure_torrents USING (url) JOIN torrents USING (info_hash) WHERE user_feeds.\"user\"=$1 AND user_feeds.\"slug\"=$2 AND torrents.\"name\"=$3", 69 | [UserName, Slug, Name]) of 70 | {ok, _, [{Torrent} | _]} -> 71 | {ok, Torrent}; 72 | {ok, _, []} -> 73 | {error, not_found} 74 | end. 75 | 76 | get_info_hash_by_name(UserName, Slug, Name) -> 77 | case ?Q("SELECT torrents.\"info_hash\" FROM user_feeds JOIN enclosures USING (feed) JOIN enclosure_torrents USING (url) JOIN torrents USING (info_hash) WHERE user_feeds.\"user\"=$1 AND user_feeds.\"slug\"=$2 AND torrents.\"name\"=$3", 78 | [UserName, Slug, Name]) of 79 | {ok, _, [{InfoHash} | _]} -> 80 | {ok, InfoHash}; 81 | {ok, _, []} -> 82 | {error, not_found} 83 | end. 84 | 85 | purge(UserName, Slug, Name) -> 86 | ?Q("SELECT * FROM purge_download($1, $2, $3)", [UserName, Slug, Name]). 87 | 88 | recent_downloads(Limit) -> 89 | query_downloads("get_recent_downloads($1)", [Limit]). 90 | 91 | popular_downloads(Limit, peers) -> 92 | query_downloads("get_popular_downloads($1)", [Limit]); 93 | 94 | popular_downloads(Limit, Period) when is_integer(Period) -> 95 | query_downloads("get_most_downloaded($1, $2)", [Limit, Period]); 96 | popular_downloads(Limit, all) -> 97 | %% get_popular_downloads() will select "downloaded" 98 | popular_downloads(Limit, 10000). 99 | 100 | 101 | user_downloads(UserName, Limit) -> 102 | query_downloads("get_user_recent_downloads($2, $1)", [UserName, Limit]). 103 | 104 | feed_downloads(Feed, Limit) -> 105 | query_downloads("get_recent_downloads($2, $1)", [Feed, Limit]). 106 | 107 | enclosure_downloads(Enclosure) -> 108 | {ok, _, Rows} = 109 | ?Q("SELECT * FROM get_enclosure_downloads($1)", [Enclosure]), 110 | rows_to_downloads(Rows). 111 | 112 | query_downloads(View, Params) -> 113 | case ?Q("SELECT * FROM " ++ View, Params) of 114 | {ok, _, Rows} -> 115 | Downloads = 116 | rows_to_downloads(Rows), 117 | FeedItems = group_downloads(Downloads), 118 | {ok, FeedItems}; 119 | {error, Reason} -> 120 | {error, Reason} 121 | end. 122 | 123 | rows_to_downloads(Rows) -> 124 | [#download{user = User, 125 | slug = Slug, 126 | feed = Feed, 127 | item = Item, 128 | enclosure = Enclosure, 129 | info_hash = InfoHash, 130 | name = Name, 131 | size = Size, 132 | type = Type, 133 | feed_title = FeedTitle, 134 | title = Title, 135 | lang = Lang, 136 | summary = Summary, 137 | published = Published, 138 | homepage = Homepage, 139 | payment = Payment, 140 | image = Image, 141 | seeders = Seeders, 142 | leechers = Leechers, 143 | upspeed = Upspeed, 144 | downspeed = Downspeed, 145 | downloaded = Downloaded} 146 | || {User, Slug, Feed, Item, Enclosure, 147 | FeedTitle, _FeedPublic, 148 | InfoHash, Name, Size, Type, 149 | Title, Lang, Summary, Published, Homepage, Payment, Image, 150 | Seeders, Leechers, Upspeed, Downspeed, 151 | Downloaded %% ordered like in Postgres TYPE 'download' 152 | } <- Rows]. 153 | 154 | %% By homepage 155 | group_downloads([]) -> 156 | []; 157 | group_downloads([Download | Downloads]) -> 158 | #download{user = User, 159 | slug = Slug, 160 | feed = Feed, 161 | item = Item, 162 | feed_title = FeedTitle, 163 | title = Title, 164 | lang = Lang, 165 | summary = Summary, 166 | published = Published, 167 | homepage = Homepage, 168 | payment = Payment, 169 | image = Image} = Download, 170 | {SiblingDownloads, OtherDownloads} = 171 | lists:splitwith( 172 | fun(#download{user = User1, 173 | item = Item1, 174 | title = Title1, 175 | homepage = Homepage1}) when User == User1 -> 176 | if 177 | %% Items from the very same homepage 178 | is_binary(Homepage1), 179 | size(Homepage1) > 0, 180 | Homepage == Homepage1 -> 181 | true; 182 | %% Probably same homepage, but masked by 183 | %% Feedburner's redirecting URLs to track users 184 | is_binary(Title1), 185 | size(Title1) > 0, 186 | Title == Title1 -> 187 | true; 188 | %% Equal item id 189 | true -> 190 | Item == Item1 191 | end; 192 | (_) -> 193 | false 194 | end, Downloads), 195 | FeedItem = 196 | #feed_item{user = User, 197 | slug = Slug, 198 | feed = Feed, 199 | id = Item, 200 | feed_title = FeedTitle, 201 | title = Title, 202 | lang = Lang, 203 | summary = Summary, 204 | published = Published, 205 | homepage = Homepage, 206 | payment = Payment, 207 | image = Image, 208 | %% Duplicate downloads may occur for merged 209 | %% feed_items (by homepage): 210 | downloads = unique_downloads([Download | SiblingDownloads]) 211 | }, 212 | [FeedItem | group_downloads(OtherDownloads)]. 213 | 214 | %% Also sorts by name 215 | unique_downloads(Downloads) -> 216 | ByName = 217 | lists:foldl(fun(#download{name = Name} = Download, ByName) -> 218 | case gb_trees:is_defined(Name, ByName) of 219 | false -> 220 | gb_trees:insert(Name, Download, ByName); 221 | true -> 222 | %% Drop duplicate 223 | ByName 224 | end 225 | end, gb_trees:empty(), Downloads), 226 | [Download 227 | || {_Name, Download} <- gb_trees:to_list(ByName)]. 228 | -------------------------------------------------------------------------------- /apps/model/src/model_feeds.erl: -------------------------------------------------------------------------------- 1 | -module(model_feeds). 2 | 3 | -export([to_update/1, prepare_update/1, write_update/10, 4 | hint_enclosure_type/2, 5 | user_feeds_details/2, user_feed_details/2, 6 | feed_data/2, get_directory/0, enclosure_errors/1]). 7 | 8 | -include("../include/model.hrl"). 9 | 10 | -define(POOL, pool_users). 11 | -define(Q(Stmt, Params), model_sup:equery(?POOL, Stmt, Params)). 12 | -define(T(Fun), model_sup:transaction(?POOL, Fun)). 13 | 14 | to_update(MaxAge1) -> 15 | MaxAge2 = {{0,0,MaxAge1},0,0}, 16 | case ?Q("SELECT next_url, wait FROM feed_to_update($1)", 17 | [MaxAge2]) of 18 | {ok, _, [{NextURL, {{H, M, S}, Days, Months}}]} -> 19 | Wait = S + 60 * (M + (60 * (H + 24 * (Days + 30 * Months)))), 20 | {ok, {NextURL, Wait}}; 21 | {ok, _, _} -> 22 | %% Nothing in database? Wait like 10s... 23 | {<<"">>, 10} 24 | end. 25 | 26 | prepare_update(FeedURL) -> 27 | case ?Q("UPDATE \"feeds\" SET \"last_update\"=CURRENT_TIMESTAMP WHERE \"url\"=$1", [FeedURL]) of 28 | {ok, 1} -> 29 | ok; 30 | {ok, N} -> 31 | exit({n_feeds, N}) 32 | end, 33 | 34 | case ?Q("SELECT \"etag\", \"last_modified\" FROM \"feeds\" WHERE \"url\"=$1", [FeedURL]) of 35 | {ok, _, [{Etag, LastModified}]} -> 36 | {ok, Etag, LastModified}; 37 | {ok, _, _} -> 38 | {ok, undefined, undefined} 39 | end. 40 | 41 | -spec write_update(string(), 42 | {binary() | null, binary() | null}, 43 | binary() | null, 44 | binary() | null, 45 | binary() | null, 46 | binary() | null, 47 | binary() | null, 48 | binary() | null, 49 | binary() | null, 50 | [#feed_item{}] 51 | ) -> ok. 52 | write_update(FeedURL, {Etag, LastModified}, 53 | Error, Xml, 54 | Title, Lang, Summary, Homepage, Image, Items) 55 | when is_list(Etag) -> 56 | write_update(FeedURL, {list_to_binary(Etag), LastModified}, 57 | Error, Xml, 58 | Title, Lang, Summary, Homepage, Image, Items); 59 | write_update(FeedURL, {Etag, LastModified}, 60 | Error, Xml, 61 | Title, Lang, Summary, Homepage, Image, Items) 62 | when is_list(LastModified) -> 63 | write_update(FeedURL, {Etag, list_to_binary(LastModified)}, 64 | Error, Xml, 65 | Title, Lang, Summary, Homepage, Image, Items); 66 | %% TODO: don't drop xml on error! 67 | write_update(FeedURL, {Etag, LastModified}, 68 | Error, Xml, 69 | Title, Lang, Summary, Homepage, Image, Items) -> 70 | T1 = util:get_now_us(), 71 | ?T(fun(Q) -> 72 | %% Update feed entry 73 | case Error of 74 | null -> 75 | Stmt = "UPDATE \"feeds\" SET \"last_update\"=CURRENT_TIMESTAMP, \"etag\"=$2, \"last_modified\"=$3, \"error\"=null, \"xml\"=$4, \"title\"=$5, \"lang\"=$6, \"summary\"=$7, \"homepage\"=$8, \"image\"=$9 WHERE \"url\"=$1", 76 | Params = [FeedURL, 77 | enforce_string(Etag), enforce_string(LastModified), 78 | enforce_string(Xml), 79 | enforce_string(Title), 80 | enforce_string(Lang), enforce_string(Summary), 81 | enforce_string(Homepage), enforce_string(Image)]; 82 | not_modified -> 83 | Stmt = "UPDATE \"feeds\" SET \"last_update\"=CURRENT_TIMESTAMP, \"etag\"=$2, \"last_modified\"=$3 WHERE \"url\"=$1", 84 | Params = [FeedURL, 85 | enforce_string(Etag), enforce_string(LastModified)]; 86 | _ when is_binary(Error) -> 87 | Stmt = "UPDATE \"feeds\" SET \"last_update\"=CURRENT_TIMESTAMP, \"etag\"=$2, \"last_modified\"=$3, \"error\"=$4 WHERE \"url\"=$1", 88 | Params = [FeedURL, 89 | enforce_string(Etag), enforce_string(LastModified), 90 | enforce_string(Error)] 91 | end, 92 | case Q(Stmt, Params) of 93 | {ok, 1} -> 94 | ok; 95 | {ok, N} -> 96 | exit({n_feeds, N}) 97 | end, 98 | 99 | %% Update items 100 | lists:foreach( 101 | fun(#feed_item{} = Item) -> 102 | case Q("SELECT count(\"id\") FROM \"feed_items\" WHERE \"feed\"=$1 AND \"id\"=$2", 103 | [FeedURL, Item#feed_item.id]) of 104 | {ok, _, [{0}]} -> 105 | io:format("New feed item:~n~p~n", [Item#feed_item.title]), 106 | lists:foreach(fun({Enclosure, EnclosureType, _, _}) -> 107 | io:format(" e (~s) ~s~n", [EnclosureType, Enclosure]) 108 | end, Item#feed_item.enclosures), 109 | {ok, 1} = 110 | Q("INSERT INTO \"feed_items\" (\"feed\", \"id\", \"title\", \"published\", \"homepage\", \"payment\", \"image\", \"lang\", \"summary\", \"updated\") VALUES ($1, $2, $3, no_future(($4 :: TEXT) :: TIMESTAMP WITH TIME ZONE), $5, $6, $7, $8, $9, CURRENT_TIMESTAMP)", 111 | [FeedURL, Item#feed_item.id, 112 | Item#feed_item.title, Item#feed_item.published, 113 | enforce_string(Item#feed_item.homepage), 114 | enforce_string(Item#feed_item.payment), 115 | enforce_string(Item#feed_item.image), 116 | enforce_string(Item#feed_item.lang), 117 | enforce_string(Item#feed_item.summary) 118 | ]); 119 | {ok, _, [{1}]} -> 120 | {ok, 1} = 121 | Q("UPDATE \"feed_items\" SET \"title\"=$3, \"homepage\"=$4, \"payment\"=$5, \"image\"=$6, \"lang\"=$7, \"summary\"=$8, \"updated\"=CURRENT_TIMESTAMP WHERE \"feed\"=$1 AND \"id\"=$2", 122 | [FeedURL, Item#feed_item.id, 123 | Item#feed_item.title, 124 | enforce_string(Item#feed_item.homepage), 125 | enforce_string(Item#feed_item.payment), 126 | enforce_string(Item#feed_item.image), 127 | enforce_string(Item#feed_item.lang), 128 | enforce_string(Item#feed_item.summary) 129 | ]) 130 | end, 131 | %% Update enclosures 132 | {ok, _, OldRows} = 133 | Q("SELECT \"url\" FROM \"enclosures\" WHERE \"feed\"=$1 AND \"item\"=$2", 134 | [FeedURL, Item#feed_item.id]), 135 | lists:foldl( 136 | fun({Enclosure, EnclosureType, EnclosureTitle, EnclosureGUID}, Old) -> 137 | case sets:is_element(Enclosure, Old) of 138 | true -> 139 | Q("UPDATE \"enclosures\" SET \"type\"=COALESCE($4, \"type\"), \"title\"=$5, \"guid\"=$6 WHERE \"feed\"=$1 AND \"item\"=$2 AND \"url\"=$3", 140 | [FeedURL, Item#feed_item.id, Enclosure, 141 | if 142 | is_binary(EnclosureType), 143 | size(EnclosureType) > 0 -> 144 | EnclosureType; 145 | true -> 146 | null 147 | end, 148 | enforce_string(EnclosureTitle), 149 | enforce_string(EnclosureGUID) 150 | ]), 151 | sets:del_element(Enclosure, Old); 152 | false -> 153 | Q("INSERT INTO \"enclosures\" (\"feed\", \"item\", \"url\", \"type\", \"title\", \"guid\") VALUES ($1, $2, $3, $4, $5, $6)", 154 | [FeedURL, Item#feed_item.id, Enclosure, 155 | enforce_string(EnclosureType), enforce_string(EnclosureTitle), enforce_string(EnclosureGUID)]), 156 | Old 157 | end 158 | end, 159 | sets:from_list([Enclosure || {Enclosure} <- OldRows]), 160 | list_uniq_by(Item#feed_item.enclosures, 161 | fun({E1, _, _, _}, {E2, _, _, _}) -> 162 | E1 == E2 163 | end)) 164 | end, Items), 165 | 166 | ok 167 | end), 168 | 169 | T2 = util:get_now_us(), 170 | io:format("[~.1fms] write_update ~s - ~B items~n", [(T2 - T1) / 1000, FeedURL, length(Items)]). 171 | 172 | 173 | %% Currently used by storage from the hasher 174 | hint_enclosure_type(Enclosure, Type) -> 175 | ?Q("UPDATE \"enclosures\" SET \"type\"=COALESCE($2, \"type\") WHERE \"url\"=$1", 176 | [Enclosure, Type]). 177 | 178 | 179 | user_feeds_details(UserName, CanEdit) -> 180 | Q = case CanEdit of 181 | %% Owner view 182 | true -> 183 | "SELECT user_feeds.\"slug\", feeds.\"url\", COALESCE(user_feeds.\"title\", feeds.\"title\"), feeds.\"homepage\", feeds.\"image\", COALESCE(user_feeds.\"public\", FALSE), feed_errors.\"error\" FROM user_feeds INNER JOIN feeds ON user_feeds.feed=feeds.url LEFT JOIN feed_errors USING (url) WHERE user_feeds.\"user\"=$1 ORDER BY LOWER(feeds.\"title\") ASC"; 184 | %% Public view 185 | false -> 186 | "SELECT user_feeds.\"slug\", feeds.\"url\", COALESCE(user_feeds.\"title\", feeds.\"title\"), feeds.\"homepage\", feeds.\"image\", COALESCE(user_feeds.\"public\", FALSE), NULL FROM user_feeds INNER JOIN feeds ON user_feeds.feed=feeds.url WHERE user_feeds.\"user\"=$1 AND user_feeds.\"public\" ORDER BY LOWER(feeds.\"title\") ASC" 187 | end, 188 | case ?Q(Q, [UserName]) of 189 | {ok, _, Rows} -> 190 | {ok, Rows}; 191 | {error, Reason} -> 192 | {error, Reason} 193 | end. 194 | 195 | user_feed_details(UserName, Slug) -> 196 | case ?Q("SELECT feeds.\"url\", COALESCE(user_feeds.\"title\", feeds.\"title\"), feeds.\"homepage\", feeds.\"image\", COALESCE(user_feeds.\"public\", FALSE), feeds.\"torrentify\", feeds.\"error\" FROM user_feeds INNER JOIN feeds ON user_feeds.feed=feeds.url WHERE user_feeds.\"user\"=$1 AND user_feeds.\"slug\"=$2", 197 | [UserName, Slug]) of 198 | {ok, _, [{URL, Title, Homepage, Image, Public, Torrentify, Error}]} -> 199 | {ok, URL, Title, Homepage, Image, Public, Torrentify, Error}; 200 | {ok, _, []} -> 201 | {error, not_found}; 202 | {error, Reason} -> 203 | {error, Reason} 204 | end. 205 | 206 | -spec feed_data(binary(), integer() 207 | ) -> {ok, binary(), [binary()], [{binary(), binary()}]} 208 | | {error, not_found}. 209 | feed_data(FeedURL, MaxEnclosures) -> 210 | case ?Q("SELECT \"xml\" FROM feeds WHERE \"url\"=$1", 211 | [FeedURL]) of 212 | {ok, _, [{FeedXml}]} -> 213 | {ok, _, EnclosureMap} = 214 | ?Q("SELECT enclosures.url, torrents.name FROM enclosures JOIN enclosure_torrents USING (url) JOIN torrents USING (info_hash) WHERE enclosures.feed=$1 LIMIT $2", 215 | [FeedURL, MaxEnclosures]), 216 | {ok, FeedXml, dict:from_list(EnclosureMap)}; 217 | {ok, _, []} -> 218 | {error, not_found} 219 | end. 220 | 221 | get_directory() -> 222 | {ok, _, Rows} = 223 | ?Q("SELECT \"user\", \"title\", \"image\", \"slug\", \"feed_title\", \"lang\", \"types\" FROM directory", []), 224 | group_directory_feeds(Rows). 225 | 226 | group_directory_feeds([]) -> 227 | []; 228 | group_directory_feeds([{User1, Title1, Image1, _, _, _, _} | _] = Directory) -> 229 | {Directory1, Directory2} = 230 | lists:splitwith( 231 | fun({User2, _, _, _, _, _, _}) -> 232 | User1 == User2 233 | end, Directory), 234 | [{User1, Title1, Image1, 235 | [{Slug, FeedTitle, Lang, Types} 236 | || {_, _, _, Slug, FeedTitle, Lang, Types} <- Directory1] 237 | } | group_directory_feeds(Directory2)]. 238 | 239 | 240 | enclosure_errors(FeedURL) -> 241 | {ok, _, Rows} = 242 | ?Q("SELECT enclosures.\"url\", enclosure_torrents.\"error\" FROM enclosures JOIN enclosure_torrents USING (url) WHERE enclosures.feed=$1 AND enclosure_torrents.\"error\" IS NOT NULL AND enclosure_torrents.\"error\" != ''", 243 | [FeedURL]), 244 | Rows. 245 | 246 | %% 247 | %% Helpers 248 | %% 249 | 250 | enforce_string(S) when is_binary(S); 251 | is_list(S) -> 252 | S; 253 | enforce_string(_) -> 254 | <<"">>. 255 | 256 | list_uniq_by([], _) -> 257 | []; 258 | list_uniq_by([E | L], F) -> 259 | [E | 260 | list_uniq_by([E1 261 | || E1 <- L, 262 | not F(E1, E)], F)]. 263 | -------------------------------------------------------------------------------- /apps/model/src/model_scrape_queue.erl: -------------------------------------------------------------------------------- 1 | -module(model_scrape_queue). 2 | 3 | -behaviour(gen_server). 4 | 5 | %% API 6 | -export([update_scraped/1, start_link/0]). 7 | 8 | %% gen_server callbacks 9 | -export([init/1, handle_call/3, handle_cast/2, handle_info/2, 10 | terminate/2, code_change/3]). 11 | 12 | -define(SERVER, ?MODULE). 13 | 14 | -record(state, {queue = queue:new(), 15 | queued = sets:new(), 16 | worker 17 | }). 18 | 19 | -define(POOL, pool_scrape_queue). 20 | -define(Q(Stmt, Params), model_sup:equery(?POOL, Stmt, Params)). 21 | 22 | %%%=================================================================== 23 | %%% API 24 | %%%=================================================================== 25 | update_scraped(InfoHash) -> 26 | gen_server:cast(?SERVER, {update_scraped, InfoHash}). 27 | 28 | start_link() -> 29 | gen_server:start_link({local, ?SERVER}, ?MODULE, [], []). 30 | 31 | %%%=================================================================== 32 | %%% gen_server callbacks 33 | %%%=================================================================== 34 | init([]) -> 35 | {ok, #state{}}. 36 | 37 | handle_call(_Request, _From, State) -> 38 | {reply, invalid, State}. 39 | 40 | handle_cast({worker_done, Pid}, #state{worker = Pid} = State1) -> 41 | State2 = may_work(State1#state{worker = undefined}), 42 | {noreply, State2}; 43 | 44 | handle_cast({update_scraped, InfoHash}, 45 | #state{queue = Queue1, 46 | queued = Queued1} = State1) -> 47 | case sets:is_element(InfoHash, Queued1) of 48 | true -> 49 | %% Already queued 50 | Queue2 = Queue1, 51 | Queued2 = Queued1; 52 | false -> 53 | %% Enqueue 54 | Queue2 = queue:in(InfoHash, Queue1), 55 | Queued2 = sets:add_element(InfoHash, Queued1) 56 | end, 57 | State2 = State1#state{queue = Queue2, 58 | queued = Queued2}, 59 | 60 | State3 = may_work(State2), 61 | {noreply, State3}. 62 | 63 | handle_info(_Info, State) -> 64 | {noreply, State}. 65 | 66 | terminate(Reason, #state{queue = Queue1} = State) -> 67 | case queue:out(Queue1) of 68 | {empty, _} -> 69 | ok; 70 | {{value, InfoHash}, Queue2} -> 71 | catch do_work(InfoHash), 72 | terminate(Reason, State#state{queue = Queue2}) 73 | end. 74 | 75 | code_change(_OldVsn, State, _Extra) -> 76 | {ok, State}. 77 | 78 | %%%=================================================================== 79 | %%% Internal functions 80 | %%%=================================================================== 81 | 82 | may_work(#state{worker = Worker} = State) when is_pid(Worker) -> 83 | %% A worker is still running 84 | State; 85 | 86 | may_work(#state{queue = Queue1, 87 | queued = Queued1} = State) -> 88 | case queue:out(Queue1) of 89 | {empty, _} -> 90 | %% Nothing to do 91 | State; 92 | {{value, InfoHash}, Queue2} -> 93 | Queued2 = sets:del_element(InfoHash, Queued1), 94 | I = self(), 95 | Worker = spawn_link( 96 | fun() -> 97 | do_work(InfoHash), 98 | gen_server:cast(I, {worker_done, self()}) 99 | end), 100 | State#state{queue = Queue2, 101 | queued = Queued2, 102 | worker = Worker} 103 | end. 104 | 105 | do_work(InfoHash) -> 106 | {ok, _, _} = 107 | ?Q("SELECT * FROM update_scraped($1)", [InfoHash]). 108 | 109 | -------------------------------------------------------------------------------- /apps/model/src/model_session.erl: -------------------------------------------------------------------------------- 1 | -module(model_session). 2 | 3 | -export([generate/1, 4 | validate/1, 5 | invalidate/1 6 | ]). 7 | 8 | -define(POOL, pool_users). 9 | -define(Q(Stmt, Params), model_sup:equery(?POOL, Stmt, Params)). 10 | -define(T(Fun), model_sup:transaction(?POOL, Fun)). 11 | 12 | 13 | generate(UserName) -> 14 | Sid = generate_sid(), 15 | {ok, 1} = 16 | ?Q("INSERT INTO user_sessions (\"user\", \"sid\", \"updated\") VALUES ($1, $2, NOW())", 17 | [UserName, Sid]), 18 | {ok, Sid}. 19 | 20 | generate_sid() -> 21 | util:seed_random(), 22 | 23 | << <<(random:uniform(256) - 1):8>> 24 | || _ <- lists:seq(1, 16) >>. 25 | 26 | validate(Sid) -> 27 | case ?Q("UPDATE user_sessions SET \"updated\"=NOW() WHERE \"sid\"=$1 RETURNING \"user\"", 28 | [Sid]) of 29 | {ok, 1, _, [{UserName}]} -> 30 | {ok, UserName}; 31 | _ -> 32 | {error, invalid_session} 33 | end. 34 | 35 | invalidate(Sid) -> 36 | ?Q("DELETE FROM user_sessions WHERE \"sid\"=$1", [Sid]). 37 | -------------------------------------------------------------------------------- /apps/model/src/model_stats.erl: -------------------------------------------------------------------------------- 1 | -module(model_stats). 2 | 3 | -export([set_gauge/3, 4 | add_counter/3, do_add_counter/3]). 5 | 6 | -define(POOL, pool_stats). 7 | -define(Q(Stmt, Params), model_sup:equery(?POOL, Stmt, Params)). 8 | -define(C(Stmt, Params), 9 | (case (catch ?Q(Stmt, Params)) of 10 | {ok, _, _} -> 11 | ok; 12 | E -> 13 | io:format("model_stats query: ~p~n", [E]), 14 | error 15 | end)). 16 | 17 | %% 18 | %% Gauge w/o caching for now 19 | %% 20 | 21 | set_gauge(Kind, InfoHash, Value) when is_atom(Kind) -> 22 | set_gauge( 23 | list_to_binary(atom_to_list(Kind)), 24 | InfoHash, Value); 25 | 26 | set_gauge(Kind, InfoHash, Value) 27 | when is_binary(Kind), 28 | is_binary(InfoHash), 29 | is_integer(Value) -> 30 | ?C("SELECT * FROM set_gauge($1, $2, $3);", 31 | [Kind, InfoHash, Value]); 32 | 33 | set_gauge(Kind, _InfoHash, Value) 34 | when is_binary(Kind), 35 | is_integer(Value) -> 36 | ?C("SELECT * FROM set_gauge($1, NULL, $3);", 37 | [Kind, Value]). 38 | 39 | %% 40 | %% Caching proxy 41 | %% 42 | 43 | -define(STATS_CACHE, model_stats_cache). 44 | 45 | add_counter(_Kind, _InfoHash, 0) -> 46 | %% Nothing to do 47 | ok; 48 | 49 | add_counter(Kind, InfoHash, Increment) when is_atom(Kind) -> 50 | add_counter( 51 | list_to_binary(atom_to_list(Kind)), 52 | InfoHash, Increment); 53 | 54 | add_counter(Kind, InfoHash, Increment) -> 55 | gen_server:cast(?STATS_CACHE, {add_counter, Kind, InfoHash, Increment}). 56 | 57 | 58 | %% 59 | %% Actual functionality 60 | %% 61 | 62 | 63 | do_add_counter(Kind, InfoHash, Increment) 64 | when is_binary(Kind), 65 | is_binary(InfoHash), 66 | is_integer(Increment) -> 67 | ?C("SELECT * FROM add_counter($1, $2, $3);", 68 | [Kind, InfoHash, Increment]); 69 | 70 | do_add_counter(Kind, _InfoHash, Increment) 71 | when is_binary(Kind), 72 | is_integer(Increment) -> 73 | ?C("SELECT * FROM add_counter($1, NULL, $3);", 74 | [Kind, Increment]). 75 | -------------------------------------------------------------------------------- /apps/model/src/model_stats_cache.erl: -------------------------------------------------------------------------------- 1 | -module(model_stats_cache). 2 | 3 | -behaviour(gen_server). 4 | 5 | %% API 6 | -export([start_link/0]). 7 | 8 | %% gen_server callbacks 9 | -export([init/1, handle_call/3, handle_cast/2, handle_info/2, 10 | terminate/2, code_change/3]). 11 | 12 | -define(SERVER, ?MODULE). 13 | 14 | -record(state, {counters = gb_trees:empty()}). 15 | 16 | -define(CACHE_FLUSH, 10000). 17 | 18 | %%%=================================================================== 19 | %%% API 20 | %%%=================================================================== 21 | start_link() -> 22 | gen_server:start_link({local, ?SERVER}, ?MODULE, [], []). 23 | 24 | %%%=================================================================== 25 | %%% gen_server callbacks 26 | %%%=================================================================== 27 | init([]) -> 28 | {ok, #state{}}. 29 | 30 | handle_call(_Request, _From, State) -> 31 | Reply = ok, 32 | {reply, Reply, State}. 33 | 34 | handle_cast({add_counter, Kind, InfoHash, Increment}, 35 | #state{counters = Counters1} = State) -> 36 | Counters2 = counters_add_counter( 37 | Counters1, {Kind, InfoHash}, Increment), 38 | {noreply, State#state{counters = Counters2}}. 39 | 40 | handle_info({flush, {Kind, InfoHash} = Key}, 41 | #state{counters = Counters1} = State) -> 42 | {ok, Value, Counters2} = 43 | counters_flush_counter(Counters1, Key), 44 | model_stats:do_add_counter(Kind, InfoHash, Value), 45 | {noreply, State#state{counters = Counters2}}; 46 | 47 | handle_info(_Info, State) -> 48 | {noreply, State}. 49 | 50 | terminate(_Reason, #state{counters = Counters}) -> 51 | lists:foreach( 52 | fun({{Kind, InfoHash}, Value}) -> 53 | model_stats:do_add_counter(Kind, InfoHash, Value) 54 | end, gb_trees:to_list(Counters)), 55 | ok. 56 | 57 | code_change(_OldVsn, State, _Extra) -> 58 | {ok, State}. 59 | 60 | %%%=================================================================== 61 | %%% Internal functions 62 | %%%=================================================================== 63 | 64 | counters_add_counter(Counters, Key, Increment) -> 65 | Value2 = 66 | case gb_trees:lookup(Key, Counters) of 67 | none -> 68 | timer:send_after(?CACHE_FLUSH, {flush, Key}), 69 | 0; 70 | {value, Value1} -> 71 | Value1 72 | end, 73 | Value3 = Value2 + Increment, 74 | gb_trees:enter(Key, Value3, Counters). 75 | 76 | counters_flush_counter(Counters1, Key) -> 77 | Value = gb_trees:get(Key, Counters1), 78 | Counters2 = gb_trees:delete(Key, Counters1), 79 | {ok, Value, Counters2}. 80 | -------------------------------------------------------------------------------- /apps/model/src/model_sup.erl: -------------------------------------------------------------------------------- 1 | -module(model_sup). 2 | 3 | -behaviour(supervisor). 4 | 5 | -define(TIMEOUT, 120000). 6 | 7 | %% API 8 | -export([start_link/1, equery/3, transaction/2]). 9 | 10 | %% Supervisor callbacks 11 | -export([init/1]). 12 | 13 | %% Helper macro for declaring children of supervisor 14 | -define(CHILD(I, Type), {I, {I, start_link, []}, permanent, 5000, Type, [I]}). 15 | 16 | %% =================================================================== 17 | %% API functions 18 | %% =================================================================== 19 | 20 | start_link(Pools) -> 21 | supervisor:start_link(?MODULE, [Pools]). 22 | 23 | equery(PoolId, Stmt, Params) -> 24 | Worker = poolboy:checkout(PoolId), 25 | Reply = gen_server:call(Worker, {equery, Stmt, Params}, ?TIMEOUT), 26 | poolboy:checkin(PoolId, Worker), 27 | Reply. 28 | 29 | transaction(PoolId, Fun) -> 30 | Worker = poolboy:checkout(PoolId), 31 | Reply = gen_server:call(Worker, {transaction, Fun}, ?TIMEOUT), 32 | poolboy:checkin(PoolId, Worker), 33 | case Reply of 34 | {rollback, Why} -> exit(Why); 35 | _ -> Reply 36 | end. 37 | 38 | %% =================================================================== 39 | %% Supervisor callbacks 40 | %% =================================================================== 41 | 42 | init([Pools]) -> 43 | ChildSpecs = 44 | [{model_stats_cache, {model_stats_cache, start_link, []}, 45 | permanent, 2000, worker, [model_stats_cache, model_stats]}, 46 | {model_scrape_queue, {model_scrape_queue, start_link, []}, 47 | permanent, 2000, worker, [model_scrape_queue]} | 48 | [{Id, {poolboy, start_link, [[{name, {local, Id}}, 49 | {worker_module, model_worker}] 50 | ++ Options]}, 51 | permanent, 2000, worker, [model_pool]} 52 | || {Id, Options} <- Pools]], 53 | {ok, { {one_for_one, 5, 10}, ChildSpecs} }. 54 | 55 | -------------------------------------------------------------------------------- /apps/model/src/model_token.erl: -------------------------------------------------------------------------------- 1 | -module(model_token). 2 | 3 | -export([generate/2, 4 | validate/2, 5 | peek/2 6 | ]). 7 | 8 | -define(POOL, pool_users). 9 | -define(Q(Stmt, Params), model_sup:equery(?POOL, Stmt, Params)). 10 | -define(T(Fun), model_sup:transaction(?POOL, Fun)). 11 | 12 | 13 | generate(Kind, UserName) when is_atom(Kind) -> 14 | generate(atom_to_list(Kind), UserName); 15 | generate(Kind, UserName) when is_list(Kind) -> 16 | generate(list_to_binary(Kind), UserName); 17 | generate(Kind, UserName) -> 18 | Token = generate_token(), 19 | {ok, 1} = 20 | ?Q("INSERT INTO user_tokens (\"kind\", \"user\", \"token\", \"created\") VALUES ($1, $2, $3, NOW())", 21 | [Kind, UserName, Token]), 22 | {ok, Token}. 23 | 24 | generate_token() -> 25 | util:seed_random(), 26 | 27 | << <<(random:uniform(256) - 1):8>> 28 | || _ <- lists:seq(1, 16) >>. 29 | 30 | validate(Kind, Token) when is_atom(Kind) -> 31 | validate(atom_to_list(Kind), Token); 32 | validate(Kind, Token) when is_list(Kind) -> 33 | validate(list_to_binary(Kind), Token); 34 | validate(Kind, Token) -> 35 | case ?Q("DELETE FROM user_tokens WHERE \"kind\"=$1 AND \"token\"=$2 RETURNING \"user\"", 36 | [Kind, Token]) of 37 | {ok, 1, _, [{UserName}]} -> 38 | {ok, UserName}; 39 | _ -> 40 | {error, invalid_token} 41 | end. 42 | 43 | %% validate w/o removing 44 | peek(Kind, Token) when is_atom(Kind) -> 45 | peek(atom_to_list(Kind), Token); 46 | peek(Kind, Token) when is_list(Kind) -> 47 | peek(list_to_binary(Kind), Token); 48 | peek(Kind, Token) -> 49 | case ?Q("SELECT \"user\" FROM user_tokens WHERE \"kind\"=$1 AND \"token\"=$2", 50 | [Kind, Token]) of 51 | {ok, _, [{UserName}]} -> 52 | {ok, UserName}; 53 | _ -> 54 | {error, invalid_token} 55 | end. 56 | 57 | -------------------------------------------------------------------------------- /apps/model/src/model_torrents.erl: -------------------------------------------------------------------------------- 1 | -module(model_torrents). 2 | 3 | -export([add_torrent/5, get_torrent/1, get_info/1, exists/1, calculate_names_sizes/0]). 4 | 5 | -include("../include/model.hrl"). 6 | 7 | -define(POOL, pool_torrents). 8 | -define(Q(Stmt, Params), model_sup:equery(?POOL, Stmt, Params)). 9 | -define(T(Fun), model_sup:transaction(?POOL, Fun)). 10 | 11 | add_torrent(InfoHash, Name, Size, Torrent, Updater) -> 12 | case ?Q("INSERT INTO torrents (\"info_hash\", \"name\", \"size\", \"torrent\") VALUES ($1, $2, $3, $4)", 13 | [InfoHash, Name, Size, Torrent]) of 14 | {error, {error, _, _, <<"duplicate key", _/binary>>, _}} -> 15 | ?T(fun(Q) -> 16 | {ok, _, [{Torrent1}]} = 17 | Q("SELECT torrent FROM torrents WHERE info_hash=$1", [InfoHash]), 18 | Torrent2 = Updater(Torrent1), 19 | Q("UPDATE torrents SET torrent=$2 WHERE info_hash=$1", [InfoHash, Torrent2]) 20 | end); 21 | {ok, 1} -> 22 | ok 23 | end. 24 | 25 | get_torrent(InfoHash) -> 26 | case ?Q("SELECT \"name\", \"torrent\" FROM torrents WHERE \"info_hash\"=$1", 27 | [InfoHash]) of 28 | {ok, _, [{Name, Torrent}]} -> 29 | {ok, Name, Torrent}; 30 | {ok, _, []} -> 31 | {error, not_found} 32 | end. 33 | 34 | get_info(InfoHash) -> 35 | case ?Q("SELECT \"name\", \"size\" FROM torrents WHERE \"info_hash\"=$1", 36 | [InfoHash]) of 37 | {ok, _, [{Name, Size}]} -> 38 | {ok, Name, Size}; 39 | _ -> 40 | {error, not_found} 41 | end. 42 | 43 | exists(<>) -> 44 | case ?Q("SELECT TRUE FROM torrents WHERE \"info_hash\"=$1", 45 | [InfoHash]) of 46 | {ok, _, [{true}]} -> 47 | true; 48 | {ok, _, _} -> 49 | false 50 | end; 51 | exists(_) -> 52 | false. 53 | 54 | 55 | %% Maintenance 56 | calculate_names_sizes() -> 57 | ?T(fun(Q) -> 58 | {ok, _, Torrents} = 59 | Q("SELECT \"info_hash\", \"torrent\" FROM torrents WHERE \"name\" IS NULL OR \"size\" IS NULL", []), 60 | lists:foreach( 61 | fun({InfoHash, TorrentFile}) -> 62 | Torrent = benc:parse(TorrentFile), 63 | Info = proplists:get_value(<<"info">>, Torrent), 64 | Name = proplists:get_value(<<"name">>, Info), 65 | Size = proplists:get_value(<<"length">>, Info), 66 | Q("UPDATE torrents SET \"name\"=$2, \"size\"=$3 WHERE \"info_hash\"=$1", 67 | [InfoHash, Name, Size]) 68 | end, Torrents) 69 | end). 70 | -------------------------------------------------------------------------------- /apps/model/src/model_users.erl: -------------------------------------------------------------------------------- 1 | -module(model_users). 2 | 3 | -export([register/2, get_salted/1, set_salted/2, 4 | get_details/1, set_details/4, 5 | get_by_email/1, 6 | get_feeds/1, get_feed/2, 7 | add_feed/3, rm_feed/2, 8 | get_user_feed/2, set_user_feed/4]). 9 | 10 | 11 | -define(POOL, pool_users). 12 | -define(Q(Stmt, Params), model_sup:equery(?POOL, Stmt, Params)). 13 | -define(T(Fun), model_sup:transaction(?POOL, Fun)). 14 | 15 | %% TODO: report user exists 16 | register(Name, Email) -> 17 | ?T(fun(Q) -> 18 | case Q("SELECT COUNT(\"name\") FROM users WHERE \"name\"=$1", 19 | [Name]) of 20 | {ok, _, [{0}]} -> 21 | {ok, 1} = 22 | Q("INSERT INTO users (\"name\", \"email\", \"salt\") VALUES ($1, $2, $3)", 23 | [Name, Email, generate_salt()]), 24 | ok; 25 | {ok, _, _} -> 26 | {error, exists} 27 | end 28 | end). 29 | 30 | generate_salt() -> 31 | util:seed_random(), 32 | 33 | << <<(random:uniform(256) - 1):8>> 34 | || _ <- lists:seq(1, 8) >>. 35 | 36 | get_salted(Name) -> 37 | case ?Q("SELECT \"salted\", \"salt\" FROM users WHERE \"name\"=$1", [Name]) of 38 | {ok, _, [{Salted, Salt}]} -> 39 | {ok, Salted, Salt}; 40 | {ok, _, []} -> 41 | {error, not_found} 42 | end. 43 | 44 | set_salted(Name, Salted) -> 45 | {ok, 1} = 46 | ?Q("UPDATE users SET \"salted\"=$2 WHERE \"name\"=$1", [Name, Salted]). 47 | 48 | get_details(Name) -> 49 | case ?Q("SELECT \"title\", \"image\", \"homepage\" FROM users WHERE \"name\"=$1", 50 | [Name]) of 51 | {ok, _, [{Title1, Image, Homepage}]} -> 52 | Title2 = 53 | if 54 | is_binary(Title1), 55 | size(Title1) > 0 -> 56 | Title1; 57 | true -> 58 | Name 59 | end, 60 | {ok, Title2, Image, Homepage}; 61 | {ok, _, []} -> 62 | {error, not_found} 63 | end. 64 | 65 | set_details(UserName, Title, Image, Homepage) -> 66 | {ok, 1} = 67 | ?Q("UPDATE users SET \"title\"=$2, \"image\"=$3, \"homepage\"=$4 WHERE \"name\"=$1", 68 | [UserName, Title, Image, Homepage]). 69 | 70 | get_by_email(Email) -> 71 | {ok, _, Rows} = 72 | ?Q("SELECT \"name\" FROM users WHERE \"email\"=$1", [Email]), 73 | [Name || {Name} <- Rows]. 74 | 75 | 76 | get_feeds(Name) -> 77 | {ok, _, Rows} = ?Q("SELECT \"slug\", \"feed\" FROM user_feeds WHERE \"user\"=$1 ORDER BY LOWER(\"slug\")", 78 | [Name]), 79 | Rows. 80 | 81 | get_feed(Name, Slug) -> 82 | case ?Q("SELECT \"feed\" FROM user_feeds WHERE \"user\"=$1 AND \"slug\"=$2 LIMIT 1", 83 | [Name, Slug]) of 84 | {ok, _, [{Feed}]} -> 85 | {ok, Feed}; 86 | {ok, _, []} -> 87 | {error, not_found} 88 | end. 89 | 90 | add_feed(Name, Slug, Url) -> 91 | case ?Q("SELECT * FROM add_user_feed($1, $2, $3)", 92 | [Name, Slug, Url]) of 93 | {ok, _, [{IsNew}]} -> 94 | {ok, IsNew}; 95 | {error, Reason} -> 96 | {error, Reason} 97 | end. 98 | 99 | rm_feed(Name, Slug) -> 100 | {ok, _} = 101 | ?Q("DELETE FROM user_feeds WHERE \"user\"=$1 AND \"slug\"=$2", 102 | [Name, Slug]), 103 | %% TODO: remove unused from feeds 104 | ok. 105 | 106 | get_user_feed(UserName, Slug) -> 107 | case ?Q("SELECT \"feed\", \"public\", \"title\" FROM user_feeds WHERE \"user\"=$1 AND \"slug\"=$2 LIMIT 1", 108 | [UserName, Slug]) of 109 | {ok, _, [{Feed, Public, Title}]} -> 110 | {ok, Feed, Public, Title}; 111 | {ok, _, []} -> 112 | {error, not_found} 113 | end. 114 | 115 | set_user_feed(UserName, Slug, Public, Title) -> 116 | {ok, 1} = 117 | ?Q("UPDATE user_feeds SET \"public\"=$3, \"title\"=$4 WHERE \"user\"=$1 AND \"slug\"=$2", 118 | [UserName, Slug, Public, Title]). 119 | -------------------------------------------------------------------------------- /apps/model/src/model_worker.erl: -------------------------------------------------------------------------------- 1 | -module(model_worker). 2 | -behaviour(gen_server). 3 | 4 | -export([start_link/1, stop/0, init/1, handle_call/3, handle_cast/2, 5 | handle_info/2, terminate/2, code_change/3]). 6 | 7 | -record(state, {conn}). 8 | 9 | start_link(Args) -> gen_server:start_link(?MODULE, Args, []). 10 | stop() -> gen_server:cast(?MODULE, stop). 11 | 12 | init(Args) -> 13 | Host = proplists:get_value(host, Args), 14 | Database = proplists:get_value(database, Args), 15 | User = proplists:get_value(user, Args), 16 | Password = proplists:get_value(password, Args), 17 | Opts = [{database, Database}], 18 | PortOpts = case proplists:get_value(port, Args) of 19 | undefined -> 20 | []; 21 | Port when is_integer(Port) -> 22 | [{port, Port}] 23 | end, 24 | {ok, Conn} = pgsql:connect(Host, User, Password, Opts ++ PortOpts), 25 | {ok, #state{conn=Conn}}. 26 | 27 | handle_call({equery, Stmt, Params}, _From, #state{conn=Conn}=State) -> 28 | {reply, pgsql:equery(Conn, Stmt, Params), State}; 29 | handle_call({transaction, Fun}, _From, #state{conn=Conn}=State) -> 30 | {reply, 31 | pgsql:with_transaction(Conn, 32 | fun(Conn2) -> 33 | Result = (catch Fun(fun(Stmt, Params) -> 34 | pgsql:equery(Conn2, Stmt, Params) 35 | end)), 36 | case Result of 37 | %% Manually rethrow before the 38 | %% wrapping 39 | %% pgsql:with_transaction/2 40 | %% swallows the stack trace. 41 | {'EXIT', Reason} -> exit(Reason); 42 | _ -> Result 43 | end 44 | end), State}; 45 | handle_call(_Request, _From, State) -> 46 | {reply, ok, State}. 47 | 48 | handle_cast(_Msg, State) -> 49 | {noreply, State}. 50 | 51 | handle_info(stop, State) -> 52 | {stop, shutdown, State}; 53 | handle_info(_Info, State) -> 54 | {noreply, State}. 55 | 56 | terminate(_Reason, #state{conn=Conn}) -> 57 | pgsql:close(Conn), 58 | ok. 59 | 60 | code_change(_OldVsn, State, _Extra) -> 61 | {ok, State}. 62 | -------------------------------------------------------------------------------- /apps/seeder/src/seeder.app.src: -------------------------------------------------------------------------------- 1 | {application, seeder, 2 | [ 3 | {description, ""}, 4 | {vsn, "1"}, 5 | {registered, []}, 6 | {applications, [ 7 | kernel, 8 | stdlib, 9 | sasl, 10 | lhttpc, 11 | model, 12 | ranch, 13 | shared 14 | ]}, 15 | {mod, { seeder_app, []}}, 16 | {env, []} 17 | ]}. 18 | -------------------------------------------------------------------------------- /apps/seeder/src/seeder_app.erl: -------------------------------------------------------------------------------- 1 | -module(seeder_app). 2 | 3 | -behaviour(application). 4 | 5 | %% Application callbacks 6 | -export([start/2, stop/1]). 7 | 8 | %% =================================================================== 9 | %% Application callbacks 10 | %% =================================================================== 11 | 12 | start(_StartType, _StartArgs) -> 13 | seeder_sup:start_link(). 14 | 15 | stop(_State) -> 16 | seeder_listener:stop(), 17 | ok. 18 | -------------------------------------------------------------------------------- /apps/seeder/src/seeder_listener.erl: -------------------------------------------------------------------------------- 1 | -module(seeder_listener). 2 | 3 | -export([start_link/0, stop/0]). 4 | 5 | start_link() -> 6 | IP = case os:getenv("BIND_IP") of 7 | false -> 8 | {0, 0, 0, 0, 0, 0, 0, 0}; 9 | IP1 -> 10 | {ok, IP2} = inet_parse:address(IP1), 11 | IP2 12 | end, 13 | ranch:start_listener( 14 | seeder_wire_listener, 32, 15 | ranch_tcp, [{ip, IP}, 16 | {port, 6881}, 17 | {max_connections, 768}, 18 | %% Not so many but long-lived connections 19 | {backlog, 16} 20 | ], 21 | seeder_wire_protocol, [] 22 | ). 23 | 24 | stop() -> 25 | ranch:stop_listener(seeder_wire_listener). 26 | -------------------------------------------------------------------------------- /apps/seeder/src/seeder_sup.erl: -------------------------------------------------------------------------------- 1 | 2 | -module(seeder_sup). 3 | 4 | -behaviour(supervisor). 5 | 6 | %% API 7 | -export([start_link/0]). 8 | 9 | %% Supervisor callbacks 10 | -export([init/1]). 11 | 12 | %% Helper macro for declaring children of supervisor 13 | -define(CHILD(I, Type), {I, {I, start_link, []}, permanent, 5000, Type, [I]}). 14 | 15 | %% =================================================================== 16 | %% API functions 17 | %% =================================================================== 18 | 19 | start_link() -> 20 | supervisor:start_link({local, ?MODULE}, ?MODULE, []). 21 | 22 | %% =================================================================== 23 | %% Supervisor callbacks 24 | %% =================================================================== 25 | 26 | init([]) -> 27 | SeederListener = 28 | {seeder_listener, {seeder_listener, start_link, []}, 29 | permanent, 2000, worker, [seeder_listener]}, 30 | {ok, { {one_for_one, 5, 10}, [SeederListener]} }. 31 | 32 | -------------------------------------------------------------------------------- /apps/seeder/src/seeder_wire_protocol.erl: -------------------------------------------------------------------------------- 1 | -module(seeder_wire_protocol). 2 | 3 | -behaviour(ranch_protocol). 4 | -behaviour(gen_server). 5 | 6 | %% API 7 | -export([start_link/4]). 8 | 9 | -define(HANDSHAKE_TIMEOUT, 10000). 10 | -define(ACTIVITY_TIMEOUT, 30000). 11 | %% Wait for subsequent chunks of a piece to be requested so they can 12 | %% be merged into bigger HTTP requests. 13 | -define(PIECE_DELAY, 50). 14 | 15 | %% For testing: 16 | -export([make_bitfield/2, make_empty_bitfield/2]). 17 | 18 | %% gen_server callbacks 19 | -export([init/1, handle_call/3, handle_cast/2, handle_info/2, 20 | terminate/2, code_change/3]). 21 | 22 | -record(state, {socket, 23 | transport, 24 | info_hash, 25 | data_length, 26 | piece_length, 27 | has, 28 | request_queue = queue:new(), 29 | timer_armed = false, 30 | storage 31 | }). 32 | -record(request, {offset, 33 | length, 34 | cb}). 35 | 36 | -define(PACKET_OPTS, [{packet, 4}, 37 | {packet_size, 8192}]). 38 | 39 | -define(CHOKE, 0). 40 | -define(UNCHOKE, 1). 41 | -define(INTERESTED, 2). 42 | -define(NOT_INTERESTED, 3). 43 | -define(HAVE, 4). 44 | -define(BITFIELD, 5). 45 | -define(REQUEST, 6). 46 | -define(PIECE, 7). 47 | -define(CANCEL, 8). 48 | 49 | %%%=================================================================== 50 | %%% API 51 | %%%=================================================================== 52 | %% Don't block when invoking start_link/4, it waits for peer handshake 53 | %% to finish. 54 | -spec start_link(pid(), inet:socket(), module(), any()) -> {ok, pid()}. 55 | start_link(Ref, Socket, Transport, Opts) -> 56 | proc_lib:start_link(?MODULE, init, [[Ref, Socket, Transport, Opts]], ?HANDSHAKE_TIMEOUT). 57 | 58 | %%%=================================================================== 59 | %%% gen_server callbacks 60 | %%%=================================================================== 61 | 62 | init([Ref, Socket, Transport, _Opts]) -> 63 | proc_lib:init_ack({ok, self()}), 64 | ranch:accept_ack(Ref), 65 | gen_server:cast(self(), handshake), 66 | gen_server:enter_loop(?MODULE, [], #state{socket = Socket, 67 | transport = Transport}, ?HANDSHAKE_TIMEOUT). 68 | 69 | handle_call(_Request, _From, State) -> 70 | Reply = ok, 71 | {reply, Reply, State, ?ACTIVITY_TIMEOUT}. 72 | 73 | handle_cast(handshake, #state{socket = Socket, 74 | transport = Transport} = State) -> 75 | {ok, Peername} = inet:peername(Socket), 76 | io:format("Peer ~p connected~n", [Peername]), 77 | 78 | %% Handshake 79 | {ok, <<19, "BitTorrent protocol">>} = gen_tcp:recv(Socket, 20), 80 | ok = gen_tcp:send(Socket, <<19, "BitTorrent protocol">>), 81 | 82 | %% Extensions 83 | {ok, _Extensions} = gen_tcp:recv(Socket, 8), 84 | ok = gen_tcp:send(Socket, <<0, 0, 0, 0, 0, 0, 0, 0>>), 85 | 86 | %% Info Hash 87 | {ok, InfoHash} = gen_tcp:recv(Socket, 20), 88 | {ok, _FileName, Length, PieceLength, Storage} = 89 | lookup_torrent(InfoHash), 90 | ok = gen_tcp:send(Socket, InfoHash), 91 | 92 | %% Peer ID 93 | {ok, PeerID} = gen_tcp:recv(Socket, 20), 94 | io:format("Peer ~p~nconnected from ~p~n", [PeerID, Peername]), 95 | ok = gen_tcp:send(Socket, peer_id:generate()), 96 | 97 | %% All following messages will be length-prefixed. 98 | %% Also, we don't download at all: 99 | Transport:setopts(Socket, [{active, once} | ?PACKET_OPTS]), 100 | 101 | %% Initial Bitfield 102 | send_bitfield(Socket, Length, PieceLength), 103 | Has = make_empty_bitfield(Length, PieceLength), 104 | 105 | {noreply, State#state{info_hash = InfoHash, 106 | data_length = Length, 107 | piece_length = PieceLength, 108 | has = Has, 109 | storage = Storage}, ?ACTIVITY_TIMEOUT}. 110 | 111 | handle_info({tcp, Socket, Data}, 112 | #state{socket = Socket, 113 | transport = Transport} = State1) -> 114 | case handle_message(Data, State1) of 115 | {ok, State2} -> 116 | Transport:setopts(Socket, [{active, once} | ?PACKET_OPTS]), 117 | {noreply, State2, ?ACTIVITY_TIMEOUT}; 118 | {close, State2} -> 119 | {stop, normal, State2} 120 | end; 121 | 122 | handle_info({tcp_closed, Socket}, 123 | #state{socket = Socket} = State) -> 124 | {stop, normal, State}; 125 | 126 | handle_info(request_pieces, 127 | #state{} = State1) -> 128 | State2 = State1#state{timer_armed = false}, 129 | case (catch request_pieces(State2)) of 130 | {ok, State3} -> 131 | State4 = may_arm_timer(State3), 132 | {noreply, State4, ?ACTIVITY_TIMEOUT}; 133 | tcp_closed -> 134 | {stop, normal, State2}; 135 | {'EXIT', Reason} -> 136 | error_logger:error_msg("Wire cannot request_pieces:~n~p~n", [Reason]), 137 | {stop, Reason, State2} 138 | end; 139 | 140 | handle_info(timeout, State) -> 141 | error_logger:warning_msg("Activity timeout in ~p~n", [self()]), 142 | {stop, normal, State}; 143 | 144 | handle_info({shoot, _Pid}, State) -> 145 | {noreply, State, ?ACTIVITY_TIMEOUT}; 146 | 147 | handle_info(_Info, State) -> 148 | error_logger:warning_msg("Unhandled wire info in ~p: ~p~n", [self(), _Info]), 149 | {noreply, State, ?ACTIVITY_TIMEOUT}. 150 | 151 | 152 | terminate(_Reason, #state{socket = Socket}) -> 153 | catch gen_tcp:close(Socket), 154 | ok. 155 | 156 | code_change(_OldVsn, State, _Extra) -> 157 | {ok, State}. 158 | 159 | %%%=================================================================== 160 | %%% Internal functions 161 | %%%=================================================================== 162 | 163 | lookup_torrent(InfoHash) -> 164 | case model_torrents:get_torrent(InfoHash) of 165 | {ok, Name, TorrentFile} -> 166 | Torrent = benc:parse(TorrentFile), 167 | 168 | Info = proplists:get_value(<<"info">>, Torrent), 169 | Length = 170 | proplists:get_value(<<"length">>, Info), 171 | PieceLength = 172 | proplists:get_value(<<"piece length">>, Info), 173 | 174 | [URL | _] = proplists:get_value(<<"url-list">>, Torrent), 175 | Storage = storage:make([{URL, Length}]), 176 | if 177 | is_binary(Name), 178 | is_integer(Length), 179 | is_integer(PieceLength), 180 | is_tuple(Storage) -> 181 | {ok, Name, Length, PieceLength, Storage}; 182 | true -> 183 | {error, invalid_torrent} 184 | end; 185 | {error, Reason} -> 186 | {error, Reason} 187 | end. 188 | 189 | 190 | send_message(Sock, Msg) -> 191 | ok = gen_tcp:send(Sock, Msg). 192 | 193 | send_bitfield(Sock, Length, PieceLength) -> 194 | Map = make_bitfield(Length, PieceLength), 195 | send_message(Sock, <>). 196 | 197 | make_bitfield(Length, PieceLength) -> 198 | list_to_binary( 199 | lists:map( 200 | fun(I) -> 201 | if 202 | %% Last piece? 203 | %% Keep spare bits empty 204 | I >= Length - PieceLength * 8 -> 205 | lists:foldl( 206 | fun(I1, R) when I1 >= Length -> 207 | R; 208 | (_, R) -> 209 | 16#80 bor (R bsr 1) 210 | end, 0, lists:seq(I, I + 8 * PieceLength, PieceLength)); 211 | true -> 212 | 255 213 | end 214 | end, lists:seq(0, Length - 1, PieceLength * 8))). 215 | 216 | make_empty_bitfield(Length, PieceLength) -> 217 | << <<0>> 218 | || _ <- lists:seq(0, Length - 1, PieceLength * 8) >>. 219 | 220 | may_arm_timer(#state{timer_armed = false, 221 | request_queue = RequestQueue} = State) -> 222 | case queue:is_empty(RequestQueue) of 223 | false -> 224 | timer:send_after(?PIECE_DELAY, request_pieces), 225 | State#state{timer_armed = true}; 226 | true -> 227 | State 228 | end; 229 | may_arm_timer(State) -> 230 | State. 231 | 232 | %% 233 | %% Incoming message 234 | %% 235 | 236 | handle_message(<<>>, State) -> 237 | %% Keep-alive 238 | {ok, State}; 239 | 240 | handle_message(<>, 241 | #state{socket = Socket} = State) -> 242 | send_message(Socket, <>), 243 | {ok, State}; 244 | 245 | handle_message(<>, #state{has = Has} = State1) -> 246 | %% update piece map 247 | I = trunc(Piece / 8), 248 | <> = Has, 249 | State2 = 250 | State1#state{has = <>}, 253 | 254 | %% disconnect new seeders: 255 | check_bitfield(State2); 256 | 257 | handle_message(<>, State) -> 258 | if 259 | size(Bits) == size(State#state.has) -> 260 | check_bitfield(State#state{has = Bits}); 261 | true -> 262 | exit(bitfield_size_mismatch) 263 | end; 264 | 265 | handle_message(<>, 266 | #state{piece_length = PieceLength, 267 | request_queue = RequestQueue1, 268 | socket = Socket} = State) -> 269 | %% Add 270 | RequestQueue2 = 271 | queue:in( 272 | #request{offset = Piece * PieceLength + Offset, 273 | length = Length, 274 | cb = fun(Data) -> 275 | send_piece(Piece, Offset, 276 | Socket, Data) 277 | end}, 278 | RequestQueue1), 279 | 280 | {ok, may_arm_timer(State#state{request_queue = RequestQueue2})}; 281 | 282 | handle_message(<>, 283 | #state{piece_length = PieceLength, 284 | request_queue = RequestQueue1} = State) -> 285 | Offset1 = Piece * PieceLength + Offset, 286 | %% Remove from queue (though we could be requesting them already) 287 | RequestQueue2 = 288 | queue:filter(fun(#request{offset = Offset2, 289 | length = Length1}) 290 | when Offset1 == Offset2, Length == Length1 -> 291 | false; 292 | (_) -> 293 | true 294 | end, RequestQueue1), 295 | {ok, State#state{request_queue = RequestQueue2}}; 296 | 297 | handle_message(Data, State) -> 298 | error_logger:warning_msg("Unhandled wire message: ~p~n", [Data]), 299 | {ok, State}. 300 | 301 | check_bitfield(#state{has = Has, 302 | data_length = Length, 303 | piece_length = PieceLength} = State) -> 304 | %% TODO: optimize 305 | Pieces = trunc((Length - 1) / PieceLength) + 1, 306 | case is_bitfield_seeder(Has, Pieces) of 307 | true -> 308 | %% Nothing left to seed 309 | {close, State}; 310 | false -> 311 | {ok, State} 312 | end. 313 | 314 | is_bitfield_seeder(Bitfield, Pieces) -> 315 | is_bitfield_seeder(Bitfield, 0, Pieces). 316 | 317 | is_bitfield_seeder(_, Pieces, Pieces) -> 318 | %% All pieces present 319 | true; 320 | is_bitfield_seeder(Bitfield, Piece, Pieces) -> 321 | Remain = 8 * size(Bitfield) - Piece - 1, 322 | <<_:Piece, Bit:1, _:Remain>> = Bitfield, 323 | case Bit of 324 | 1 -> 325 | %% Peer has piece, look further 326 | is_bitfield_seeder(Bitfield, Piece + 1, Pieces); 327 | 0 -> 328 | %% Peer doesn't have piece, terminate 329 | false 330 | end. 331 | 332 | %% 333 | %% Data Transfer 334 | %% 335 | 336 | 337 | request_pieces(#state{request_queue = RequestQueue1, 338 | storage = Storage, 339 | info_hash = InfoHash, 340 | socket = Socket} = State) -> 341 | case queue:out(RequestQueue1) of 342 | {{value, #request{offset = Offset} = Request}, RequestQueue2} -> 343 | {SubsequentRequests, RequestQueue3} = 344 | collect_contiguous_requests(Request#request.offset + Request#request.length, 345 | RequestQueue2), 346 | Requests = [Request | SubsequentRequests], 347 | %% TODO: could be returned by collect_contiguous_requests/2 for 348 | %% performance reasons 349 | Length = lists:foldl(fun(#request{length = Length1}, Length) -> 350 | Length + Length1 351 | end, 0, Requests), 352 | {ok, Peername} = inet:peername(Socket), 353 | io:format("Processing ~B requests for ~p at ~B+~B~n", 354 | [length(Requests), Peername, Request#request.offset, Length]), 355 | 356 | %% Transfer request by request 357 | RemainRequests = 358 | case (catch storage:fold(Storage, 359 | Offset, Length, 360 | fun collect_data/2, {Requests, <<>>})) of 361 | {'EXIT', Reason} -> 362 | error_logger:error_msg("storage request failed:~n~p~n", [Reason]), 363 | %% Just throttle: 364 | receive 365 | after 5000 -> 366 | go_retry 367 | end, 368 | %% Throwing an error means nothing has been processed 369 | Requests; 370 | tcp_closed -> 371 | %% Re-throw to terminate gen_server. The HTTP 372 | %% client hopefully doesn't receive too much 373 | %% excess data... 374 | throw(tcp_closed); 375 | {RemainRequests1, _} -> 376 | RemainRequests1 377 | end, 378 | 379 | %% Calculate stats 380 | NotTransferred = 381 | lists:foldl(fun(#request{length = Length1}, NotTransferred) -> 382 | NotTransferred + Length1 383 | end, 0, RemainRequests), 384 | model_stats:add_counter(up_seeder, InfoHash, Length - NotTransferred), 385 | model_stats:add_counter(storage_fail, InfoHash, NotTransferred), 386 | 387 | %% Retry later 388 | RequestQueue4 = 389 | lists:foldl(fun(RemainRequest, RequestQueue) -> 390 | queue:in(RemainRequest, RequestQueue) 391 | end, RequestQueue3, RemainRequests), 392 | 393 | {ok, may_arm_timer(State#state{request_queue = RequestQueue4})}; 394 | {empty, RequestQueue1} -> 395 | {ok, State} 396 | end. 397 | 398 | collect_contiguous_requests(Offset, RequestQueue1) -> 399 | case queue:peek(RequestQueue1) of 400 | {value, #request{offset = Offset1, 401 | length = Length1}} 402 | when Offset == Offset1 -> 403 | {{value, Request}, RequestQueue2} = 404 | queue:out(RequestQueue1), 405 | {Requests, RequestQueue3} = 406 | collect_contiguous_requests(Offset + Length1, RequestQueue2), 407 | {[Request | Requests], RequestQueue3}; 408 | _ -> 409 | {[], RequestQueue1} 410 | end. 411 | 412 | %% Used with storage:fold/5 413 | collect_data({[], Data}, <<>>) -> 414 | {[], Data}; 415 | 416 | collect_data({Requests, Data}, <<>>) -> 417 | %% No input left, look for next hunk to serve 418 | serve_requests(Requests, Data); 419 | 420 | collect_data({Requests, <<>>}, Data) -> 421 | %% Fast path 422 | collect_data({Requests, Data}, <<>>); 423 | 424 | collect_data({Requests, Data1}, Data2) -> 425 | %% Merge input 426 | collect_data({Requests, <>}, <<>>). 427 | 428 | serve_requests([], Data) -> 429 | {[], Data}; 430 | 431 | serve_requests([#request{length = ReqLength, 432 | cb = ReqCb} | Requests2] = Requests1, Data1) -> 433 | if 434 | size(Data1) >= ReqLength -> 435 | %% Next request satisfied 436 | {Piece, Data2} = split_binary(Data1, ReqLength), 437 | ReqCb(Piece), 438 | serve_requests(Requests2, Data2); 439 | true -> 440 | %% Not enough data 441 | {Requests1, Data1} 442 | end. 443 | 444 | 445 | send_piece(Piece, Offset, Socket, Data) -> 446 | %% We switch to manual packetization for streaming 447 | inet:setopts(Socket, [{active, false}, 448 | {packet, raw}]), 449 | 450 | %% Length prefixed header 451 | PieceHeader = <>, 452 | case gen_tcp:send(Socket, 453 | <<(size(PieceHeader) + size(Data)):32/big, 454 | PieceHeader/binary, Data/binary>>) of 455 | ok -> ok; 456 | {error, closed} -> throw(tcp_closed) 457 | end, 458 | 459 | %% Continue receiving & sending in len-prefixed packets 460 | inet:setopts(Socket, [{active, once} | ?PACKET_OPTS]). 461 | -------------------------------------------------------------------------------- /apps/shared/src/benc.erl: -------------------------------------------------------------------------------- 1 | %%%------------------------------------------------------------------- 2 | %%% Description: Parse bencoded files (.torrent) 3 | %%%------------------------------------------------------------------- 4 | -module(benc). 5 | -export([parse_file/1, parse/1]). 6 | -export([to_binary/1, to_iolist/1, hash/1]). 7 | 8 | %%% API %%% 9 | 10 | parse_file(Filename) -> 11 | {ok, Data} = file:read_file(Filename), 12 | parse(Data). 13 | 14 | parse(Data) -> 15 | {Value, <<>>} = parse_value(Data), 16 | Value. 17 | 18 | %%% parse_value - where data is being typed %%% 19 | 20 | parse_value(<<$d, Remaining/binary>>) -> 21 | parse_dict([], Remaining); 22 | 23 | parse_value(<<$l, Remaining/binary>>) -> 24 | parse_list([], Remaining); 25 | 26 | parse_value(<<$i, Remaining/binary>>) -> 27 | parse_int("", Remaining); 28 | 29 | parse_value(<>) when Z >= $0, Z =< $9 -> 30 | parse_str_len([Z], Remaining). 31 | 32 | %%% parse_dict %%% 33 | 34 | parse_dict(Dict, <<$e, Remaining/binary>>) -> 35 | {lists:reverse(Dict), Remaining}; 36 | 37 | parse_dict(Dict, Remaining) -> 38 | {Key, Remaining2} = parse_value(Remaining), 39 | {Value, Remaining3} = parse_value(Remaining2), 40 | parse_dict([{Key, Value} | Dict], Remaining3). 41 | 42 | %%% parse_list %%% 43 | 44 | parse_list(List, <<$e, Remaining/binary>>) -> 45 | {lists:reverse(List), Remaining}; 46 | 47 | parse_list(List, Remaining) -> 48 | {Value, Remaining2} = parse_value(Remaining), 49 | parse_list([Value | List], Remaining2). 50 | 51 | %%% parse_int %%% 52 | 53 | parse_int(IntS, <<$e, Remaining/binary>>) -> 54 | {Int, []} = string:to_integer(lists:reverse(IntS)), 55 | {Int, Remaining}; 56 | 57 | parse_int("", <<$-, Remaining/binary>>) -> 58 | parse_int([$-], Remaining); 59 | 60 | parse_int(IntS, <>) when Z >= $0, Z =< $9 -> 61 | parse_int([Z | IntS], Remaining). 62 | 63 | %%% parse_str* %%% 64 | 65 | parse_str_len(LenS, <>) when Z >= $0, Z =< $9 -> 66 | parse_str_len([Z | LenS], Remaining); 67 | 68 | parse_str_len(LenS, <<$:, Remaining/binary>>) -> 69 | {Len, []} = string:to_integer(lists:reverse(LenS)), 70 | parse_str(Len, "", Remaining). 71 | 72 | parse_str(0, Str, Remaining) -> 73 | {list_to_binary(lists:reverse(Str)), Remaining}; 74 | 75 | parse_str(Len, Str, <>) -> 76 | parse_str(Len - 1, [C | Str], Remaining). 77 | 78 | %%%%%%%%%%%%%%%%%%% 79 | %% Serialization 80 | %%%%%%%%%%%%%%%%%%% 81 | 82 | to_binary([{_, _} | _] = Dict) -> 83 | Bin = << <<(to_binary(K))/binary, (to_binary(V))/binary>> 84 | || {K, V} <- lists:sort(Dict) >>, 85 | <<"d", Bin/binary, "e">>; 86 | 87 | to_binary(List) when is_list(List) -> 88 | Bin = << <<(to_binary(E))/binary>> || E <- List >>, 89 | <<"l", Bin/binary, "e">>; 90 | 91 | to_binary(I) when is_integer(I) -> 92 | Bin = list_to_binary(integer_to_list(I)), 93 | <<"i", Bin/binary, "e">>; 94 | 95 | to_binary(<>) -> 96 | Length = list_to_binary(integer_to_list(size(String))), 97 | <>. 98 | 99 | 100 | to_iolist([{_, _} | _] = Dict) -> 101 | B = [[to_iolist(K), to_iolist(V)] 102 | || {K, V} <- lists:sort(Dict)], 103 | [$d, B, $e]; 104 | 105 | to_iolist(List) when is_list(List) -> 106 | [$l, lists:map(fun to_iolist/1, List), $e]; 107 | 108 | to_iolist(I) when is_integer(I) -> 109 | [$i, integer_to_list(I), $e]; 110 | 111 | to_iolist(String) when is_binary(String) -> 112 | [integer_to_list(size(String)), $:, String]. 113 | 114 | 115 | hash(Dict) -> 116 | crypto:start(), 117 | crypto:hash(sha, to_binary(Dict)). 118 | 119 | 120 | -------------------------------------------------------------------------------- /apps/shared/src/peer_id.erl: -------------------------------------------------------------------------------- 1 | -module(peer_id). 2 | 3 | -export([generate/0]). 4 | 5 | generate() -> 6 | <<"-<30000-bitlove.org/">>. 7 | -------------------------------------------------------------------------------- /apps/shared/src/shared.app.src: -------------------------------------------------------------------------------- 1 | %% A few shared modules to depend on 2 | {application, shared, 3 | [ 4 | {description, ""}, 5 | {vsn, "1"}, 6 | {registered, []}, 7 | {applications, [ 8 | ]} 9 | ]}. 10 | -------------------------------------------------------------------------------- /apps/shared/src/storage.erl: -------------------------------------------------------------------------------- 1 | -module(storage). 2 | 3 | %% TODO: relative redirects 4 | 5 | -export([make/1, size/1, fold/5, resource_info/1]). 6 | 7 | -define(USER_AGENT, "PritTorrent/1.0"). 8 | -define(TIMEOUT, 30 * 1000). 9 | -define(PART_SIZE, 32768). 10 | -define(MAX_REDIRECTS, 9). 11 | 12 | -record(storage, {urls :: [{binary(), integer()}]}). 13 | 14 | %% URLs for multi-file torrents, not fallback 15 | make(URLs) -> 16 | set_urls(#storage{urls = []}, URLs). 17 | 18 | set_urls(Storage, URLs1) -> 19 | URLs2 = [case URL of 20 | {_, Size} when is_integer(Size) -> 21 | URL; 22 | _ when is_binary(URL) -> 23 | case resource_info(URL) of 24 | {ok, _, Size, _, _} -> 25 | {URL, Size}; 26 | undefined -> 27 | exit(no_content_length) 28 | end 29 | end || URL <- URLs1], 30 | Storage#storage{urls = URLs2}. 31 | 32 | size(#storage{urls = URLs}) -> 33 | lists:foldl(fun({_, Size}, Total) -> 34 | Total + Size 35 | end, 0, URLs). 36 | 37 | resource_info(URL) -> 38 | resource_info(URL, 0). 39 | 40 | resource_info(URL, Redirects) when is_binary(URL) -> 41 | resource_info(binary_to_list(URL), Redirects); 42 | resource_info(_, Redirects) when Redirects > ?MAX_REDIRECTS -> 43 | {error, too_many_redirects}; 44 | resource_info(URL, Redirects) -> 45 | case lhttpc:request(URL, head, [{"User-Agent", ?USER_AGENT}], [], ?TIMEOUT) of 46 | {ok, {{200, _}, Headers, _}} -> 47 | ContentType = extract_header("content-type", Headers), 48 | ContentLength1 = extract_header("content-length", Headers), 49 | ContentLength2 = if 50 | is_list(ContentLength1) -> 51 | list_to_integer(ContentLength1); 52 | true -> 53 | undefined 54 | end, 55 | 56 | ETag = extract_header("etag", Headers), 57 | LastModified = extract_header("last-modified", Headers), 58 | {ok, ContentType, ContentLength2, ETag, LastModified}; 59 | {ok, {{Status, _}, Headers, _}} 60 | when Status >= 300, Status < 400 -> 61 | case extract_header("location", Headers) of 62 | undefined -> 63 | {error, {http, Status}}; 64 | Location -> 65 | io:format("HTTP ~B: ~s redirects to ~s~n", [Status, URL, Location]), 66 | resource_info(Location, Redirects + 1) 67 | end; 68 | {ok, {{Status, _}, _, _}} -> 69 | {error, {http, Status}}; 70 | {error, Reason} -> 71 | {error, Reason} 72 | end. 73 | 74 | 75 | fold(_, _, Length, _, AccOut) when Length =< 0 -> 76 | AccOut; 77 | fold(#storage{urls = URLs} = Storage, 78 | Offset, Length, F, AccIn) -> 79 | {URL, Offset1, Length1} = 80 | lists:foldl( 81 | fun({URL, Size}, {look, Offset1}) -> 82 | if 83 | Offset1 < Size -> 84 | {URL, Offset1, min(Length, Size)}; 85 | true -> 86 | {look, Offset1 - Size} 87 | end; 88 | (_, {URL, Offset1, Length1}) -> 89 | {URL, Offset1, Length1} 90 | end, {look, Offset}, URLs), 91 | 92 | AccOut = fold_resource(URL, Offset1, Length1, F, AccIn), 93 | 94 | fold(Storage, Offset + Length1, Length - Length1, F, AccOut). 95 | 96 | %% FIXME: what if response chunk is smaller than requested? retry in 97 | %% case it's still uploading? 98 | fold_resource(URL, Offset, Length, F, AccIn) when is_binary(URL) -> 99 | fold_resource(binary_to_list(URL), Offset, Length, F, AccIn); 100 | fold_resource(URL, Offset, Length, F, AccIn) -> 101 | fold_resource(URL, Offset, Length, F, AccIn, 0). 102 | 103 | fold_resource(_URL, _Offset, _Length, _F, _AccIn, Redirects) 104 | when Redirects > ?MAX_REDIRECTS -> 105 | exit(too_many_redirects); 106 | fold_resource(URL, Offset, Length, F, AccIn, Redirects) -> 107 | %% Compose request 108 | ReqHeaders = 109 | if 110 | is_integer(Offset), 111 | is_integer(Length) -> 112 | [{"Range", 113 | io_lib:format("bytes=~B-~B", 114 | [Offset, 115 | Offset + Length - 1]) 116 | }]; 117 | true -> 118 | [] 119 | end ++ 120 | [{"User-Agent", ?USER_AGENT}], 121 | ReqOptions = 122 | [{partial_download, 123 | [ 124 | %% specifies how many part will be sent to the calling 125 | %% process before waiting for an acknowledgement 126 | {window_size, 4}, 127 | %% specifies the size the body parts should come in 128 | {part_size, ?PART_SIZE} 129 | ]} 130 | ], 131 | case lhttpc:request(URL, get, ReqHeaders, 132 | [], ?TIMEOUT, ReqOptions) of 133 | %% Partial Content 134 | {ok, {{206, _}, _Headers, Pid}} -> 135 | %% Strrream: 136 | {Transferred, AccOut} = fold_resource1(Pid, F, {0, AccIn}), 137 | if 138 | Transferred == Length -> 139 | %% Ok 140 | AccOut; 141 | Transferred < Length -> 142 | %% Continue, using Redirects as limiter 143 | fold_resource(URL, Offset + Transferred, Length - Transferred, F, AccOut, Redirects - 1); 144 | Transferred > Length -> 145 | %% Data corruption :-( 146 | exit(excess_data) 147 | end; 148 | {ok, {{Status, _}, Headers, Pid}} 149 | when Status >= 300, Status < 400 -> 150 | %% Finalize this response: 151 | fold_resource1(Pid, fun(_, _) -> 152 | ok 153 | end, {0, undefined}), 154 | 155 | case extract_header("location", Headers) of 156 | undefined -> 157 | exit({http, Status}); 158 | Location -> 159 | io:format("HTTP ~B: ~s redirects to ~s~n", [Status, URL, Location]), 160 | %% FIXME: this breaks Offset & Length for multi-file torrents 161 | fold_resource(Location, Offset, Length, F, AccIn, Redirects + 1) 162 | end; 163 | {ok, {{Status, _}, _Headers, Pid}} -> 164 | %% Finalize this response: 165 | exit(Pid, kill), 166 | 167 | exit({http, Status}); 168 | {error, Reason} -> 169 | exit(Reason) 170 | end. 171 | 172 | fold_resource1(undefined, _, A) -> 173 | %% No body, no fold. 174 | A; 175 | fold_resource1(Pid, F, {Size, AccIn} = A) -> 176 | case (catch lhttpc:get_body_part(Pid, ?TIMEOUT)) of 177 | {ok, Data} when is_binary(Data) -> 178 | AccOut = F(AccIn, Data), 179 | fold_resource1(Pid, F, {Size + byte_size(Data), AccOut}); 180 | {ok, {http_eob, _Trailers}} -> 181 | A; 182 | {'EXIT', Reason} -> 183 | error_logger:error_msg("storage fold interrupted: ~p~n", [Reason]), 184 | exit(Reason); 185 | {error, Reason} -> 186 | error_logger:error_msg("storage fold interrupted: ~p~n", [Reason]), 187 | exit(Reason) 188 | end. 189 | 190 | extract_header(Name1, Headers) -> 191 | Name2 = string:to_lower(Name1), 192 | lists:foldl( 193 | fun({Header, Value}, undefined) -> 194 | case string:to_lower(Header) of 195 | Name3 when Name2 == Name3 -> 196 | Value; 197 | _ -> 198 | undefined 199 | end; 200 | (_, Value) -> 201 | Value 202 | end, undefined, Headers). 203 | 204 | -------------------------------------------------------------------------------- /apps/shared/src/url.erl: -------------------------------------------------------------------------------- 1 | -module(url). 2 | 3 | -export([join/2]). 4 | 5 | join(Base, Target) 6 | when is_binary(Base) -> 7 | join(binary_to_list(Base), 8 | Target 9 | ); 10 | 11 | join(Base, Target) 12 | when is_binary(Target) -> 13 | list_to_binary( 14 | join(Base, 15 | binary_to_list(Target) 16 | )); 17 | 18 | join(Base, Target) -> 19 | ColonIndex = 20 | length(lists:takewhile(fun($:) -> false; 21 | (_) -> true 22 | end, Target)), 23 | if 24 | ColonIndex < 16 andalso 25 | ColonIndex < length(Target) -> 26 | %% Target is full link w/ "proto:" 27 | Target; 28 | true -> 29 | %% Target is relative 30 | {Host, Port, Path, Ssl} = 31 | lhttpc_lib:parse_url(Base), 32 | NewPath = filename:join(Path, Target), 33 | io_lib:format("~s://~s:~B~s", 34 | [if 35 | Ssl -> 36 | "https"; 37 | true -> 38 | "http" 39 | end, 40 | Host, 41 | Port, 42 | NewPath 43 | ]) 44 | end. 45 | -------------------------------------------------------------------------------- /apps/shared/src/util.erl: -------------------------------------------------------------------------------- 1 | -module(util). 2 | 3 | -export([get_now/0, get_now_us/0, measure/2, 4 | pmap/2, binary_to_hex/1, hex_to_binary/1, 5 | seed_random/0, 6 | iso8601/1, iso8601/2]). 7 | 8 | get_now() -> 9 | {MS, S, SS} = erlang:now(), 10 | MS * 1000000 + S + SS / 1000000. 11 | 12 | 13 | get_now_us() -> 14 | {MS, S, SS} = erlang:now(), 15 | (MS * 1000000 + S) * 1000000 + SS. 16 | 17 | 18 | measure(Label, F) 19 | when is_list(Label) -> 20 | T1 = get_now_us(), 21 | R = F(), 22 | T2 = get_now_us(), 23 | io:format("[~.1fms] ~s~n", [(T2 - T1) / 1000, Label]), 24 | R; 25 | measure(Label, F) -> 26 | measure(io_lib:format("~p", [Label]), F). 27 | 28 | pmap(F, L) -> 29 | I = self(), 30 | Pids = 31 | [spawn(fun() -> 32 | I ! {ok, self(), F(E)} 33 | end) || E <- L], 34 | [receive 35 | {ok, Pid, E2} -> 36 | E2 37 | end || Pid <- Pids]. 38 | 39 | binary_to_hex(<<>>) -> 40 | []; 41 | binary_to_hex(<>) -> 42 | iolist_to_binary( 43 | [io_lib:format("~2.16.0b", [C]) | binary_to_hex(Bin)] 44 | ). 45 | 46 | hex_to_binary(<<>>) -> 47 | <<>>; 48 | hex_to_binary(<>) -> 49 | <<((hex_to_binary1(A) bsl 4) bor hex_to_binary1(B)):8, 50 | (hex_to_binary(Rest))/binary>>. 51 | 52 | hex_to_binary1(C) 53 | when C >= $0, 54 | C =< $9 -> 55 | C - $0; 56 | hex_to_binary1(C) 57 | when C >= $a, 58 | C =< $f -> 59 | C + 10 - $a; 60 | hex_to_binary1(C) 61 | when C >= $A, 62 | C =< $F -> 63 | C + 10 - $A. 64 | 65 | seed_random() -> 66 | {MS, S, SS} = erlang:now(), 67 | PS = lists:sum(pid_to_list(self())), 68 | random:seed(MS + PS, S, SS). 69 | 70 | %% ISO8601 Date Formatting 71 | 72 | 73 | iso8601({{Y, Mo, D}, {H, M, S}}) -> 74 | list_to_binary( 75 | io_lib:format("~4..0B-~2..0B-~2..0BT~2..0B:~2..0B:~2..0BZ", 76 | [Y, Mo, D, H, M, trunc(S)])). 77 | 78 | iso8601(Local, local) -> 79 | [Universal | _] = calendar:local_time_to_universal_time_dst(Local), 80 | if 81 | Universal < Local -> 82 | {0, {TzH, TzM, _}} = 83 | calendar:time_difference(Universal, Local); 84 | true -> 85 | {0, {TzH1, TzM}} = 86 | calendar:time_difference(Local, Universal), 87 | TzH = -TzH1 88 | end, 89 | iso8601(Local, {TzH, TzM}); 90 | 91 | iso8601(Universal, universal) -> 92 | iso8601(Universal, {0, 0}); 93 | 94 | iso8601({{Y, Mo, D}, {H, M, S}}, {TzH, TzM}) -> 95 | list_to_binary( 96 | io_lib:format("~4..0B-~2..0B-~2..0BT~2..0B:~2..0B:~2..0B~c~2..0B:~2..0B", 97 | [Y, Mo, D, H, M, trunc(S), 98 | if 99 | TzH < 0 -> $-; 100 | true -> $+ 101 | end, abs(TzH), TzM])). 102 | 103 | -------------------------------------------------------------------------------- /build.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash -e 2 | 3 | for app in feeds hasher seeder ; do 4 | rebar3 generate -n $app 5 | mkdir -p _build/default/rel/$app/log 6 | done 7 | -------------------------------------------------------------------------------- /config/vm.args: -------------------------------------------------------------------------------- 1 | # Heartbeat management 2 | -heart 3 | 4 | # Kernel poll 5 | +K true 6 | # Async threads 7 | +A 32 8 | 9 | -env ERL_MAX_PORTS 4096 10 | -------------------------------------------------------------------------------- /dev.erl: -------------------------------------------------------------------------------- 1 | -module(dev). 2 | 3 | -export([init/0, start_deps/0]). 4 | 5 | init() -> 6 | R1 = 7 | add_app_paths("deps") ++ 8 | add_app_paths("apps"), 9 | R2 = 10 | start_deps(), 11 | R3 = 12 | lists:map(fun application:start/1, [shared, model]), 13 | R1 ++ R2 ++ R3. 14 | 15 | add_app_paths(BaseDir) -> 16 | {ok, Filenames} = file:list_dir(BaseDir), 17 | lists:map(fun("." ++ _) -> 18 | ignore; 19 | (Filename) -> 20 | code:add_patha(BaseDir ++ "/" ++ Filename ++ "/ebin") 21 | end, Filenames) ++ 22 | [code:add_patha("ebin")]. 23 | 24 | start_deps() -> 25 | R1 = 26 | lists:map( 27 | fun application:start/1, 28 | [sasl, xmerl, crypto, public_key, ssl, inets, compiler]), 29 | 30 | {ok, Filenames} = file:list_dir("deps"), 31 | R2 = 32 | lists:map( 33 | fun(Filename) -> 34 | application:start(list_to_atom(Filename)) 35 | end, Filenames), 36 | R1 ++ R2. 37 | -------------------------------------------------------------------------------- /pg_downloads.sql: -------------------------------------------------------------------------------- 1 | -- model_enclosures:purge/3 2 | CREATE OR REPLACE FUNCTION purge_download( 3 | "d_user" TEXT, 4 | "d_slug" TEXT, 5 | "d_name" TEXT 6 | ) RETURNS void AS $$ 7 | DECLARE 8 | "d_enclosure" TEXT; 9 | BEGIN 10 | SELECT "url" INTO d_enclosure 11 | FROM enclosure_torrents 12 | JOIN torrents USING (info_hash) 13 | JOIN enclosures USING (url) 14 | JOIN user_feeds USING (feed) 15 | WHERE "user"=d_user AND "slug"=d_slug AND "name"=d_name; 16 | DELETE FROM enclosure_torrents WHERE "url"=d_enclosure; 17 | END; 18 | $$ LANGUAGE plpgsql; 19 | 20 | 21 | CREATE TYPE download AS ( 22 | "user" TEXT, 23 | "slug" TEXT, 24 | "feed" TEXT, 25 | "item" TEXT, 26 | "enclosure" TEXT, 27 | "feed_title" TEXT, 28 | "feed_public" BOOL, 29 | "info_hash" BYTEA, 30 | "name" TEXT, 31 | "size" BIGINT, 32 | "type" TEXT, 33 | "title" TEXT, 34 | "lang" TEXT, 35 | "summary" TEXT, 36 | "published" TIMESTAMP, 37 | "homepage" TEXT, 38 | "payment" TEXT, 39 | "image" TEXT, 40 | "downloaded" BIGINT 41 | ); 42 | 43 | 44 | -- TODO: rm dups 45 | CREATE OR REPLACE FUNCTION get_most_downloaded( 46 | INT, INT, INT 47 | ) RETURNS SETOF download AS $$ 48 | SELECT user_feeds."user", user_feeds."slug", user_feeds."feed", 49 | enclosures.item, enclosures.url AS enclosure, 50 | COALESCE(user_feeds.title, feeds.title) AS feed_title, user_feeds."public" AS feed_public, 51 | torrents.info_hash, torrents.name, torrents.size, enclosures.type, 52 | feed_items.title, feed_items.lang, feed_items.summary, feed_items.published, feed_items.homepage, feed_items.payment, feed_items.image, 53 | COALESCE(downloaded_stats.downloaded, 0) AS "downloaded" 54 | FROM (SELECT info_hash, downloaded 55 | FROM downloaded_stats 56 | ORDER BY (CASE WHEN $3 <= 1 THEN downloaded1 57 | WHEN $3 <= 7 THEN downloaded7 58 | WHEN $3 <= 30 THEN downloaded30 59 | ELSE downloaded 60 | END) DESC 61 | LIMIT $1 OFFSET $2 62 | ) AS downloaded_stats 63 | JOIN torrents USING (info_hash) 64 | JOIN enclosure_torrents USING (info_hash) 65 | JOIN enclosures USING (url) 66 | JOIN feed_items ON (enclosures.feed=feed_items.feed AND enclosures.item=feed_items.id) 67 | JOIN feeds ON (feed_items.feed=feeds.url) 68 | JOIN user_feeds ON (feed_items.feed=user_feeds.feed) 69 | WHERE user_feeds."public"; 70 | $$ LANGUAGE SQL; 71 | 72 | 73 | CREATE OR REPLACE FUNCTION get_torrent_download( 74 | BYTEA 75 | ) RETURNS SETOF download AS $$ 76 | SELECT * 77 | FROM (SELECT user_feeds."user", user_feeds."slug", user_feeds."feed", 78 | enclosures.item, enclosures.url AS enclosure, 79 | COALESCE(user_feeds.title, feeds.title) AS feed_title, user_feeds."public" AS feed_public, 80 | torrents.info_hash, torrents.name, torrents.size, enclosures.type, 81 | feed_items.title, feed_items.lang, feed_items.summary, feed_items.published, feed_items.homepage, feed_items.payment, feed_items.image, 82 | COALESCE(downloaded_stats.downloaded, 0) AS "downloaded" 83 | FROM torrents 84 | LEFT JOIN downloaded_stats USING (info_hash) 85 | JOIN enclosure_torrents USING (info_hash) 86 | JOIN enclosures USING (url) 87 | JOIN feed_items ON (enclosures.feed=feed_items.feed AND enclosures.item=feed_items.id) 88 | JOIN feeds ON (feed_items.feed=feeds.url) 89 | JOIN user_feeds ON (feed_items.feed=user_feeds.feed) 90 | WHERE torrents.info_hash=$1 91 | AND user_feeds."public" 92 | ) AS s 93 | ORDER BY downloaded DESC; 94 | $$ LANGUAGE SQL; 95 | 96 | 97 | CREATE OR REPLACE FUNCTION get_recent_downloads( 98 | INT, INT 99 | ) RETURNS SETOF download AS $$ 100 | SELECT * 101 | FROM (SELECT user_feeds."user", user_feeds."slug", user_feeds."feed", 102 | enclosures.item, enclosures.url AS enclosure, 103 | COALESCE(user_feeds.title, feeds.title) AS feed_title, user_feeds."public" AS feed_public, 104 | torrents.info_hash, torrents.name, torrents.size, enclosures.type, 105 | feed_items.title, feed_items.lang, feed_items.summary, feed_items.published, feed_items.homepage, feed_items.payment, feed_items.image, 106 | COALESCE(downloaded_stats.downloaded, 0) AS "downloaded" 107 | FROM (SELECT feed, id, title, lang, summary, published, homepage, payment, image 108 | FROM feed_items 109 | ORDER BY published DESC 110 | LIMIT $1 OFFSET $2 111 | ) AS feed_items 112 | JOIN enclosures ON (feed_items.feed=enclosures.feed AND feed_items.id=enclosures.item) 113 | JOIN enclosure_torrents ON (enclosures.url=enclosure_torrents.url) 114 | JOIN torrents ON (enclosure_torrents.info_hash=torrents.info_hash) 115 | JOIN feeds ON (feed_items.feed=feeds.url) 116 | JOIN user_feeds ON (feed_items.feed=user_feeds.feed) 117 | LEFT JOIN downloaded_stats ON (enclosure_torrents.info_hash=downloaded_stats.info_hash) 118 | WHERE user_feeds."public" 119 | ) AS s 120 | ORDER BY published DESC; 121 | $$ LANGUAGE SQL; 122 | 123 | CREATE OR REPLACE FUNCTION get_recent_downloads( 124 | INT, INT, TEXT 125 | ) RETURNS SETOF download AS $$ 126 | SELECT * 127 | FROM (SELECT user_feeds."user", user_feeds."slug", user_feeds."feed", 128 | enclosures.item, enclosures.url AS enclosure, 129 | COALESCE(user_feeds.title, feeds.title) AS feed_title, user_feeds."public" AS feed_public, 130 | torrents.info_hash, torrents.name, torrents.size, enclosures.type, 131 | feed_items.title, feed_items.lang, feed_items.summary, feed_items.published, feed_items.homepage, feed_items.payment, feed_items.image, 132 | COALESCE(downloaded_stats.downloaded, 0) AS "downloaded" 133 | FROM (SELECT feed, id, title, lang, summary, published, homepage, payment, image 134 | FROM feed_items 135 | WHERE feed=$3 136 | ORDER BY published DESC 137 | LIMIT $1 OFFSET $2 138 | ) AS feed_items 139 | JOIN enclosures ON (feed_items.feed=enclosures.feed AND feed_items.id=enclosures.item) 140 | JOIN enclosure_torrents ON (enclosures.url=enclosure_torrents.url) 141 | JOIN torrents ON (enclosure_torrents.info_hash=torrents.info_hash) 142 | JOIN feeds ON (feed_items.feed=feeds.url) 143 | JOIN user_feeds ON (feed_items.feed=user_feeds.feed) 144 | LEFT JOIN downloaded_stats ON (enclosure_torrents.info_hash=downloaded_stats.info_hash) 145 | ) AS s 146 | ORDER BY published DESC; 147 | $$ LANGUAGE SQL; 148 | 149 | CREATE OR REPLACE FUNCTION get_user_recent_downloads( 150 | INT, INT, TEXT 151 | ) RETURNS SETOF download AS $$ 152 | SELECT * 153 | FROM (SELECT user_feeds."user", user_feeds."slug", user_feeds."feed", 154 | enclosures.item, enclosures.url AS enclosure, 155 | COALESCE(user_feeds.title, feeds.title) AS feed_title, user_feeds."public" AS feed_public, 156 | torrents.info_hash, torrents.name, torrents.size, enclosures.type, 157 | feed_items.title, feed_items.lang, feed_items.summary, feed_items.published, feed_items.homepage, feed_items.payment, feed_items.image, 158 | COALESCE(downloaded_stats.downloaded, 0) AS "downloaded" 159 | FROM (SELECT feed, id, title, lang, summary, published, homepage, payment, image 160 | FROM feed_items 161 | WHERE feed IN (SELECT feed FROM user_feeds WHERE "user"=$3) 162 | ORDER BY published DESC 163 | LIMIT $1 OFFSET $2 164 | ) AS feed_items 165 | JOIN enclosures ON (feed_items.feed=enclosures.feed AND feed_items.id=enclosures.item) 166 | JOIN enclosure_torrents ON (enclosures.url=enclosure_torrents.url) 167 | JOIN torrents ON (enclosure_torrents.info_hash=torrents.info_hash) 168 | JOIN feeds ON (feed_items.feed=feeds.url) 169 | JOIN user_feeds ON (feed_items.feed=user_feeds.feed) 170 | LEFT JOIN downloaded_stats ON (enclosure_torrents.info_hash=downloaded_stats.info_hash) 171 | WHERE user_feeds."public" 172 | ) AS s 173 | ORDER BY published DESC; 174 | $$ LANGUAGE SQL; 175 | 176 | CREATE OR REPLACE FUNCTION get_enclosure_downloads( 177 | TEXT 178 | ) RETURNS SETOF download AS $$ 179 | SELECT user_feeds."user", user_feeds."slug", user_feeds."feed", 180 | enclosures.item, enclosures.url AS enclosure, 181 | COALESCE(user_feeds.title, feeds.title) AS feed_title, user_feeds."public" AS feed_public, 182 | torrents.info_hash, torrents.name, torrents.size, enclosures.type, 183 | feed_items.title, feed_items.lang, feed_items.summary, feed_items.published, feed_items.homepage, feed_items.payment, feed_items.image, 184 | COALESCE(downloaded_stats.downloaded, 0) AS "downloaded" 185 | FROM (SELECT url, info_hash FROM enclosure_torrents 186 | WHERE url=$1 AND LENGTH(info_hash)=20 187 | LIMIT 100) AS enclosure_torrents 188 | JOIN torrents USING (info_hash) 189 | JOIN enclosures USING (url) 190 | JOIN feed_items ON (enclosures.feed=feed_items.feed AND enclosures.item=feed_items.id) 191 | JOIN feeds ON (feed_items.feed=feeds.url) 192 | JOIN user_feeds ON (feed_items.feed=user_feeds.feed) 193 | LEFT JOIN downloaded_stats ON (enclosure_torrents.info_hash=downloaded_stats.info_hash); 194 | $$ LANGUAGE SQL; 195 | 196 | CREATE OR REPLACE FUNCTION get_guid_downloads( 197 | TEXT 198 | ) RETURNS SETOF download AS $$ 199 | SELECT user_feeds."user", user_feeds."slug", user_feeds."feed", 200 | enclosures.item, enclosures.url AS enclosure, 201 | COALESCE(user_feeds.title, feeds.title) AS feed_title, user_feeds."public" AS feed_public, 202 | torrents.info_hash, torrents.name, torrents.size, enclosures.type, 203 | feed_items.title, feed_items.lang, feed_items.summary, feed_items.published, feed_items.homepage, feed_items.payment, feed_items.image, 204 | COALESCE(downloaded_stats.downloaded, 0) AS "downloaded" 205 | FROM (SELECT feed, item, url, type FROM enclosures 206 | WHERE guid=$1 207 | LIMIT 100) AS enclosures 208 | JOIN enclosure_torrents USING (url) 209 | JOIN torrents USING (info_hash) 210 | JOIN feed_items ON (enclosures.feed=feed_items.feed AND enclosures.item=feed_items.id) 211 | JOIN feeds ON (feed_items.feed=feeds.url) 212 | JOIN user_feeds ON (feed_items.feed=user_feeds.feed) 213 | LEFT JOIN downloaded_stats ON (enclosure_torrents.info_hash=downloaded_stats.info_hash) 214 | WHERE LENGTH(enclosure_torrents.info_hash)=20; 215 | $$ LANGUAGE SQL; 216 | 217 | CREATE OR REPLACE FUNCTION get_torrent_guids( 218 | BYTEA 219 | ) RETURNS SETOF TEXT AS $$ 220 | SELECT DISTINCT enclosures.guid AS "guid" 221 | FROM enclosure_torrents 222 | JOIN enclosures USING (url) 223 | WHERE enclosure_torrents.info_hash=$1 224 | AND enclosures.guid IS NOT NULL 225 | $$ LANGUAGE SQL; 226 | -------------------------------------------------------------------------------- /pg_install.sql: -------------------------------------------------------------------------------- 1 | SET default_tablespace = safe; 2 | \i pg_meta.sql 3 | SET default_tablespace = fast; 4 | \i pg_var.sql 5 | \i pg_downloads.sql 6 | \i pg_stats.sql 7 | \i pg_search.sql 8 | -------------------------------------------------------------------------------- /pg_meta.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE users ("name" TEXT NOT NULL, 2 | "email" TEXT NOT NULL, 3 | "salt" BYTEA, 4 | "salted" BYTEA, 5 | "title" TEXT, 6 | "image" TEXT, 7 | "homepage" TEXT, 8 | PRIMARY KEY ("name")); 9 | 10 | CREATE TABLE feeds ("url" TEXT NOT NULL, 11 | "last_update" TIMESTAMP, 12 | "etag" TEXT, 13 | "last_modified" TEXT, 14 | "error" TEXT, 15 | "title" TEXT, 16 | "lang" TEXT, 17 | "summary" TEXT, 18 | "homepage" TEXT, 19 | "image" TEXT, 20 | "xml" TEXT, 21 | "torrentify" BOOL DEFAULT TRUE, 22 | PRIMARY KEY ("url")); 23 | 24 | CREATE TABLE user_feeds ("user" TEXT NOT NULL REFERENCES "users" ("name"), 25 | "slug" TEXT NOT NULL, 26 | "feed" TEXT NOT NULL REFERENCES "feeds" ("url"), 27 | "public" BOOL, 28 | "title" TEXT, 29 | PRIMARY KEY ("user", "slug")); 30 | 31 | CREATE OR REPLACE FUNCTION add_user_feed( 32 | "f_user" TEXT, 33 | "f_slug" TEXT, 34 | "f_url" TEXT 35 | ) RETURNS BOOL AS $$ 36 | DECLARE 37 | is_new BOOL := TRUE; 38 | BEGIN 39 | SELECT COUNT(url) < 1 INTO is_new 40 | FROM feeds WHERE url=f_url; 41 | IF is_new THEN 42 | INSERT INTO feeds (url) VALUES (f_url); 43 | END IF; 44 | INSERT INTO user_feeds 45 | ("user", "slug", "feed") 46 | VALUES (f_user, f_slug, f_url); 47 | RETURN is_new; 48 | END; 49 | $$ LANGUAGE plpgsql; 50 | 51 | CREATE OR REPLACE FUNCTION feed_to_update( 52 | update_interval INTERVAL, 53 | OUT next_url TEXT, OUT wait INTERVAL 54 | ) RETURNS RECORD AS $$ 55 | DECLARE 56 | next_feed RECORD; 57 | BEGIN 58 | SELECT "url", "last_update" 59 | INTO next_feed 60 | FROM "feeds" 61 | ORDER BY "last_update" ASC NULLS FIRST 62 | LIMIT 1 63 | FOR UPDATE; 64 | 65 | next_url := next_feed.url; 66 | IF next_feed.last_update IS NULL THEN 67 | next_feed.last_update = '1970-01-01 00:00:00'; 68 | END IF; 69 | wait := next_feed.last_update + update_interval - CURRENT_TIMESTAMP; 70 | 71 | IF wait <= '0'::INTERVAL THEN 72 | UPDATE "feeds" 73 | SET "last_update"=CURRENT_TIMESTAMP 74 | WHERE "url"=next_url; 75 | END IF; 76 | END; 77 | $$ LANGUAGE plpgsql; 78 | 79 | 80 | -- Check this with: select * from enclosure_torrents where info_hash not in (select info_hash from torrents); 81 | -- Or add a constraint on info_hash with either NULL or FOREIGN KEY torrents (info_hash) 82 | CREATE TABLE enclosure_torrents ("url" TEXT NOT NULL PRIMARY KEY, 83 | last_update TIMESTAMP, 84 | error TEXT, 85 | info_hash BYTEA); 86 | 87 | CREATE TABLE torrents ("info_hash" BYTEA PRIMARY KEY, 88 | "name" TEXT, 89 | "size" BIGINT, 90 | "torrent" BYTEA); 91 | 92 | CREATE OR REPLACE VIEW active_users AS 93 | SELECT "user", 94 | COUNT(DISTINCT slug) as "feeds", 95 | ARRAY_AGG(DISTINCT "lang") AS langs, 96 | ARRAY_AGG(DISTINCT "type") AS types 97 | FROM user_feeds 98 | JOIN (SELECT "url", "lang", "type" 99 | FROM feeds 100 | JOIN feed_types ON (feeds.url=feed_types.feed)) AS lang_type 101 | ON (user_feeds.feed=lang_type.url) 102 | WHERE user_feeds."public"=true 103 | GROUP BY "user" 104 | ORDER BY "user" ASC; 105 | 106 | CREATE OR REPLACE VIEW directory AS 107 | SELECT users.name AS "user", 108 | COALESCE(users.title, users.name) As title, 109 | users.image, 110 | user_feeds.slug, 111 | COALESCE(user_feeds.title, feeds.title) AS feed_title, 112 | feeds.lang, 113 | feed_types.types 114 | FROM users 115 | JOIN user_feeds ON (users.name=user_feeds."user") 116 | JOIN feeds ON (user_feeds.feed=feeds.url) 117 | JOIN (SELECT "feed", array_agg("type") AS types 118 | FROM feed_types 119 | GROUP BY "feed" 120 | ) AS feed_types ON (user_feeds.feed=feed_types.feed) 121 | WHERE user_feeds."public"=true 122 | ORDER BY users.name ASC, user_feeds.slug ASC; 123 | -------------------------------------------------------------------------------- /pg_migrate_enclosures_recheck.sql: -------------------------------------------------------------------------------- 1 | -- State fields 2 | ALTER TABLE enclosure_torrents ADD COLUMN "length" BIGINT; 3 | ALTER TABLE enclosure_torrents ADD COLUMN etag TEXT; 4 | ALTER TABLE enclosure_torrents ADD COLUMN last_modified TEXT; 5 | -- Prefill for migration 6 | UPDATE enclosure_torrents 7 | SET "length"=(SELECT "size" 8 | FROM torrents 9 | WHERE info_hash=enclosure_torrents.info_hash 10 | ); 11 | 12 | -- Scheduling 13 | ALTER TABLE enclosure_torrents ADD COLUMN next_recheck TIMESTAMP; 14 | CREATE INDEX enclosure_torrents_next_recheck ON enclosure_torrents (next_recheck ASC NULLS FIRST); 15 | CREATE OR REPLACE FUNCTION enclosure_to_recheck( 16 | OUT e_url TEXT, 17 | OUT e_length BIGINT, 18 | OUT e_etag TEXT, 19 | OUT e_last_modified TEXT 20 | ) RETURNS RECORD AS $$ 21 | DECLARE 22 | "next" RECORD; 23 | "age" INTERVAL; 24 | "next_interval" INTERVAL; 25 | BEGIN 26 | LOCK "enclosure_torrents" IN SHARE ROW EXCLUSIVE MODE; 27 | SELECT "url", enclosure_torrents."length", "etag", "last_modified" 28 | INTO "next" 29 | FROM enclosure_torrents 30 | WHERE next_recheck IS NULL 31 | OR next_recheck <= NOW() 32 | ORDER BY next_recheck ASC NULLS FIRST, last_update DESC 33 | LIMIT 1; 34 | 35 | IF FOUND THEN 36 | SELECT NOW() - COALESCE(published, '1970-01-01') INTO "age" 37 | FROM enclosures 38 | JOIN feed_items ON (enclosures.feed=feed_items.feed AND enclosures.item=feed_items.id) 39 | WHERE enclosures.url=next.url 40 | ORDER BY published DESC 41 | LIMIT 1; 42 | next_interval = CASE 43 | WHEN age < '6 hours' THEN '2 minutes' 44 | WHEN age < '24 hours' THEN '5 minutes' 45 | WHEN age < '7 days' THEN '15 minutes' 46 | ELSE '7 days' 47 | END; 48 | UPDATE enclosure_torrents 49 | SET next_recheck = NOW() + next_interval 50 | WHERE url=next.url; 51 | 52 | e_url = next.url; 53 | e_length = next.length; 54 | e_etag = next.etag; 55 | e_last_modified = next.last_modified; 56 | END IF; 57 | END; 58 | $$ LANGUAGE plpgsql; 59 | -------------------------------------------------------------------------------- /pg_search.sql: -------------------------------------------------------------------------------- 1 | -- TODO: search users 2 | 3 | -- search feeds 4 | ALTER TABLE feeds ADD COLUMN search tsvector; 5 | CREATE INDEX search_feeds ON feeds USING gin(search); 6 | 7 | CREATE OR REPLACE FUNCTION search_feeds_trigger() RETURNS trigger AS $$ 8 | DECLARE 9 | conf regconfig := 10 | CASE NEW.lang 11 | WHEN 'dk' THEN 'danish' 12 | WHEN 'nl' THEN 'dutch' 13 | WHEN 'fi' THEN 'finnish' 14 | WHEN 'fr' THEN 'french' 15 | WHEN 'de' THEN 'german' 16 | WHEN 'hu' THEN 'hungarian' 17 | WHEN 'it' THEN 'italian' 18 | WHEN 'no' THEN 'norwegian' 19 | WHEN 'pt' THEN 'portuguese' 20 | WHEN 'ro' THEN 'romanian' 21 | WHEN 'ru' THEN 'russian' 22 | WHEN 'es' THEN 'spanish' 23 | WHEN 'sw' THEN 'swedish' 24 | WHEN 'tr' THEN 'turkish' 25 | ELSE 'english' 26 | END; 27 | BEGIN 28 | NEW.search := 29 | setweight(to_tsvector(conf, coalesce(new.title,'')), 'A') || 30 | setweight(to_tsvector(conf, coalesce(new.summary,'')), 'B') || 31 | setweight(to_tsvector(conf, coalesce(new.url,'')), 'D') || 32 | setweight(to_tsvector(conf, coalesce(new.homepage,'')), 'D'); 33 | RETURN NEW; 34 | END 35 | $$ LANGUAGE plpgsql; 36 | 37 | CREATE TRIGGER search_feeds_trigger BEFORE INSERT OR UPDATE 38 | ON feeds FOR EACH ROW EXECUTE PROCEDURE search_feeds_trigger(); 39 | 40 | 41 | CREATE OR REPLACE FUNCTION search_feeds(needle TEXT) RETURNS SETOF feeds AS $$ 42 | DECLARE 43 | "query" TSQUERY := plainto_tsquery(needle); 44 | BEGIN 45 | RETURN QUERY 46 | SELECT * 47 | FROM feeds 48 | WHERE "search" @@ "query" 49 | ORDER BY ts_rank("search", "query") DESC; 50 | END; 51 | $$ LANGUAGE plpgsql; 52 | 53 | -- search items 54 | ALTER TABLE feed_items ADD COLUMN search tsvector; 55 | CREATE INDEX search_feed_items ON feed_items USING gist(search); 56 | 57 | CREATE OR REPLACE FUNCTION search_feed_items_trigger() RETURNS trigger AS $$ 58 | DECLARE 59 | conf regconfig := 60 | CASE NEW.lang 61 | WHEN 'dk' THEN 'danish' 62 | WHEN 'nl' THEN 'dutch' 63 | WHEN 'fi' THEN 'finnish' 64 | WHEN 'fr' THEN 'french' 65 | WHEN 'de' THEN 'german' 66 | WHEN 'hu' THEN 'hungarian' 67 | WHEN 'it' THEN 'italian' 68 | WHEN 'no' THEN 'norwegian' 69 | WHEN 'pt' THEN 'portuguese' 70 | WHEN 'ro' THEN 'romanian' 71 | WHEN 'ru' THEN 'russian' 72 | WHEN 'es' THEN 'spanish' 73 | WHEN 'sw' THEN 'swedish' 74 | WHEN 'tr' THEN 'turkish' 75 | ELSE 'english' 76 | END; 77 | BEGIN 78 | NEW.search := 79 | setweight(to_tsvector(conf, coalesce(new.title,'')), 'A') || 80 | setweight(to_tsvector(conf, coalesce(new.summary,'')), 'B') || 81 | setweight(to_tsvector(conf, coalesce(new.homepage,'')), 'D'); 82 | RETURN NEW; 83 | END 84 | $$ LANGUAGE plpgsql; 85 | 86 | CREATE TRIGGER search_feed_items_trigger BEFORE INSERT OR UPDATE 87 | ON feed_items FOR EACH ROW EXECUTE PROCEDURE search_feed_items_trigger(); 88 | 89 | 90 | CREATE OR REPLACE FUNCTION search_feed_items( 91 | "limit" INT, "offset" INT, needle TEXT 92 | ) RETURNS SETOF download AS $$ 93 | DECLARE 94 | "query" TSQUERY := plainto_tsquery(needle); 95 | BEGIN 96 | RETURN QUERY 97 | SELECT user_feeds."user", user_feeds."slug", user_feeds."feed", 98 | enclosures.item, enclosures.url AS enclosure, 99 | COALESCE(user_feeds.title, feeds.title) AS feed_title, user_feeds."public" AS feed_public, 100 | torrents.info_hash, torrents.name, torrents.size, enclosures.type, 101 | feed_items.title, feed_items.lang, feed_items.summary, feed_items.published, feed_items.homepage, feed_items.payment, feed_items.image, 102 | COALESCE(downloaded_stats.downloaded, 0) AS "downloaded" 103 | FROM (SELECT * FROM feed_items 104 | WHERE "search" @@ "query" 105 | ORDER BY ts_rank(feed_items."search", "query") DESC, "published" DESC 106 | LIMIT "limit" OFFSET "offset" 107 | ) AS feed_items 108 | JOIN feeds ON (feed_items.feed=feeds.url) 109 | JOIN user_feeds ON (feed_items.feed=user_feeds.feed) 110 | JOIN enclosures ON (enclosures.feed=feed_items.feed AND enclosures.item=feed_items.id) 111 | JOIN enclosure_torrents ON (enclosure_torrents.url=enclosures.url) 112 | JOIN torrents USING (info_hash) 113 | LEFT JOIN downloaded_stats ON (enclosure_torrents.info_hash=downloaded_stats.info_hash) 114 | WHERE user_feeds."public"; 115 | END; 116 | $$ LANGUAGE plpgsql; 117 | -------------------------------------------------------------------------------- /pg_stats.sql: -------------------------------------------------------------------------------- 1 | -- Gauges 2 | -- 3 | -- For values that change over time (seeders, leechers) 4 | CREATE TABLE gauges ( 5 | "kind" TEXT NOT NULL, 6 | "time" TIMESTAMP NOT NULL, 7 | "info_hash" BYTEA, 8 | "value" BIGINT DEFAULT 1 9 | ); 10 | CREATE INDEX gauges_kind_info_hash_time_value ON gauges ("kind","info_hash","time","value"); 11 | 12 | CREATE OR REPLACE FUNCTION set_gauge( 13 | "e_kind" TEXT, 14 | "e_info_hash" BYTEA, 15 | "e_value" BIGINT 16 | ) RETURNS void AS $$ 17 | DECLARE 18 | "prev_value" BIGINT; 19 | BEGIN 20 | -- Deduplication: 21 | SELECT "value" INTO "prev_value" 22 | FROM gauges 23 | WHERE "kind"="e_kind" 24 | AND "info_hash"="e_info_hash" 25 | ORDER BY "time" DESC 26 | LIMIT 1; 27 | IF prev_value IS NULL OR prev_value != e_value THEN 28 | -- Add new datum: 29 | INSERT INTO gauges 30 | ("kind", "time", "info_hash", "value") 31 | VALUES (e_kind, NOW(), e_info_hash, e_value); 32 | END IF; 33 | END; 34 | $$ LANGUAGE plpgsql; 35 | 36 | 37 | -- Counters 38 | -- 39 | -- For adding values (up, down, up_seeder) 40 | CREATE TABLE counters ( 41 | "kind" TEXT NOT NULL, 42 | "time" TIMESTAMP NOT NULL, 43 | "info_hash" BYTEA, 44 | "value" BIGINT NOT NULL 45 | ); 46 | 47 | CREATE INDEX counters_kind_info_hash_time_value ON counters ("kind","info_hash","time","value"); 48 | 49 | CREATE INDEX counters_get 50 | ON counters ("info_hash", "time") 51 | -- Must be mentioned explicitly to use this index 52 | WHERE info_hash LIKE 'GET /%.torrent'; 53 | 54 | CREATE OR REPLACE FUNCTION add_counter( 55 | "e_kind" TEXT, 56 | "e_info_hash" BYTEA, 57 | "e_value" BIGINT 58 | ) RETURNS void AS $$ 59 | DECLARE 60 | period_length BIGINT := 600; 61 | -- TODO: reuse align_timestamp() 62 | period TIMESTAMP := TO_TIMESTAMP( 63 | FLOOR( 64 | EXTRACT(EPOCH FROM NOW()) 65 | / period_length) 66 | * period_length); 67 | BEGIN 68 | IF e_value = 0 THEN 69 | -- Nothing to do 70 | RETURN; 71 | END IF; 72 | 73 | UPDATE counters SET "value"="value"+e_value 74 | WHERE "kind"="e_kind" 75 | AND "info_hash"="e_info_hash" 76 | AND "time"="period"; 77 | IF NOT FOUND THEN 78 | INSERT INTO counters 79 | ("kind", "time", "info_hash", "value") 80 | VALUES (e_kind, period, e_info_hash, e_value); 81 | END IF; 82 | END; 83 | $$ LANGUAGE plpgsql; 84 | 85 | 86 | CREATE OR REPLACE FUNCTION align_timestamp ( 87 | "ts" TIMESTAMP, 88 | "interval" INT 89 | ) RETURNS TIMESTAMP WITH TIME ZONE AS $$ 90 | SELECT TO_TIMESTAMP(FLOOR(EXTRACT(EPOCH FROM $1) / $2) * $2); 91 | $$ LANGUAGE SQL IMMUTABLE; 92 | 93 | 94 | CREATE TABLE downloaded_stats ( 95 | "info_hash" BYTEA PRIMARY KEY, 96 | downloaded BIGINT, 97 | downloaded30 BIGINT, 98 | downloaded7 BIGINT, 99 | downloaded1 BIGINT 100 | ); 101 | 102 | CREATE INDEX downloaded_stats_downloaded ON downloaded_stats (downloaded DESC); 103 | CREATE INDEX downloaded_stats_downloaded30 ON downloaded_stats (downloaded30 DESC); 104 | CREATE INDEX downloaded_stats_downloaded7 ON downloaded_stats (downloaded7 DESC); 105 | CREATE INDEX downloaded_stats_downloaded1 ON downloaded_stats (downloaded1 DESC); 106 | 107 | CREATE OR REPLACE FUNCTION update_downloaded_stats( 108 | "t_info_hash" BYTEA 109 | ) RETURNS void AS $$ 110 | DECLARE 111 | "t_downloaded" BIGINT; 112 | "t_downloaded30" BIGINT; 113 | "t_downloaded7" BIGINT; 114 | "t_downloaded1" BIGINT; 115 | BEGIN 116 | DELETE FROM downloaded_stats WHERE info_hash=t_info_hash; 117 | 118 | SELECT COALESCE(SUM("value"), 0) 119 | INTO "t_downloaded" 120 | FROM counters 121 | WHERE ("kind"='complete' OR "kind"='complete_w') 122 | AND "info_hash"=t_info_hash; 123 | SELECT COALESCE(SUM("value"), 0) 124 | INTO "t_downloaded30" 125 | FROM counters 126 | WHERE ("kind"='complete' OR "kind"='complete_w') 127 | AND "info_hash"=t_info_hash 128 | AND "time" > (NOW() - '30 days'::INTERVAL); 129 | SELECT COALESCE(SUM("value"), 0) 130 | INTO "t_downloaded7" 131 | FROM counters 132 | WHERE ("kind"='complete' OR "kind"='complete_w') 133 | AND "info_hash"=t_info_hash 134 | AND "time" > (NOW() - '7 days'::INTERVAL); 135 | SELECT COALESCE(SUM("value"), 0) 136 | INTO "t_downloaded1" 137 | FROM counters 138 | WHERE ("kind"='complete' OR "kind"='complete_w') 139 | AND "info_hash"=t_info_hash 140 | AND "time" > (NOW() - '1 day'::INTERVAL); 141 | 142 | INSERT INTO downloaded_stats (info_hash, downloaded, downloaded30, downloaded7, downloaded1) 143 | VALUES (t_info_hash, t_downloaded, t_downloaded30, t_downloaded7, t_downloaded1); 144 | END; 145 | $$ LANGUAGE plpgsql; 146 | 147 | 148 | -- Only for trigger when "kind"='complete' 149 | CREATE OR REPLACE FUNCTION counters_update_downloaded_stats() RETURNS trigger AS $$ 150 | BEGIN 151 | PERFORM update_downloaded_stats(NEW.info_hash); 152 | 153 | RETURN NEW; 154 | END; 155 | $$ LANGUAGE plpgsql; 156 | 157 | -- Push "downloaded" count to "scraped" cache 158 | CREATE TRIGGER counters_update_downloaded_stats AFTER INSERT OR UPDATE ON counters 159 | FOR EACH ROW 160 | WHEN (NEW.kind = 'complete') 161 | EXECUTE PROCEDURE counters_update_downloaded_stats(); 162 | 163 | 164 | 165 | -- Run periodically to update downloaded_stats.downloaded{30,7,1} 166 | CREATE OR REPLACE FUNCTION update_all_downloaded_stats() RETURNS void AS $$ 167 | DECLARE 168 | t_info_hash BYTEA; 169 | BEGIN 170 | FOR t_info_hash IN 171 | SELECT DISTINCT "info_hash" 172 | FROM counters 173 | WHERE (kind = 'complete' OR "kind"='complete_w') 174 | AND time >= (NOW() - '31 days'::INTERVAL) 175 | AND time <= (NOW() - '1 day'::INTERVAL) 176 | LOOP 177 | PERFORM update_downloaded_stats(t_info_hash); 178 | END LOOP; 179 | END; 180 | $$ LANGUAGE plpgsql; 181 | 182 | CREATE index counters_completes ON counters (kind, time, info_hash) WHERE kind = 'complete' OR kind = 'complete_w'; 183 | 184 | 185 | -- Compaction code 186 | CREATE OR REPLACE FUNCTION compact_counters( 187 | "p_start" TIMESTAMP, 188 | "p_end" TIMESTAMP 189 | ) RETURNS void AS $$ 190 | DECLARE 191 | "t_info_hash" BYTEA; 192 | "t_kind" TEXT; 193 | "t_total" BIGINT; 194 | "t_start" TIMESTAMP; 195 | BEGIN 196 | FOR t_kind, t_info_hash, t_total, t_start IN 197 | SELECT kind, info_hash, SUM("value"), MIN("time") 198 | FROM counters 199 | WHERE "time" >= p_start 200 | AND "time" < p_end 201 | GROUP BY kind, info_hash 202 | LOOP 203 | DELETE FROM counters 204 | WHERE "kind" = t_kind 205 | AND "info_hash" = t_info_hash 206 | AND "time" >= p_start 207 | AND "time" < p_end; 208 | INSERT INTO counters ("kind", "time", "info_hash", "value") 209 | VALUES (t_kind, t_start, t_info_hash, t_total); 210 | END LOOP; 211 | END; 212 | $$ LANGUAGE plpgsql; 213 | 214 | CREATE OR REPLACE FUNCTION compact_gauges( 215 | "p_start" TIMESTAMP, 216 | "p_end" TIMESTAMP 217 | ) RETURNS void AS $$ 218 | DECLARE 219 | "t_info_hash" BYTEA; 220 | "t_kind" TEXT; 221 | "t_total" BIGINT; 222 | "t_start" TIMESTAMP; 223 | BEGIN 224 | FOR t_kind, t_info_hash, t_total, t_start IN 225 | SELECT kind, info_hash, AVG("value"), MIN("time") 226 | FROM gauges 227 | WHERE "time" >= p_start 228 | AND "time" < p_end 229 | GROUP BY kind, info_hash 230 | LOOP 231 | DELETE FROM gauges 232 | WHERE "kind" = t_kind 233 | AND "info_hash" = t_info_hash 234 | AND "time" >= p_start 235 | AND "time" < p_end; 236 | INSERT INTO counters ("kind", "time", "info_hash", "value") 237 | VALUES (t_kind, t_start, t_info_hash, t_total); 238 | END LOOP; 239 | END; 240 | $$ LANGUAGE plpgsql; 241 | -------------------------------------------------------------------------------- /pg_var.sql: -------------------------------------------------------------------------------- 1 | 2 | CREATE TABLE feed_items ("feed" TEXT NOT NULL REFERENCES "feeds" ("url") ON DELETE CASCADE, 3 | "id" TEXT NOT NULL, 4 | "title" TEXT, 5 | "lang" TEXT, 6 | "summary" TEXT, 7 | "homepage" TEXT, 8 | "published" TIMESTAMP NOT NULL, 9 | "payment" TEXT, 10 | "image" TEXT, 11 | "updated" TIMESTAMP, 12 | PRIMARY KEY ("feed", "id")); 13 | 14 | CREATE INDEX feed_items_published ON feed_items ("published" DESC); 15 | 16 | -- Used for inserting items: 17 | CREATE OR REPLACE FUNCTION no_future( 18 | TIMESTAMP WITH TIME ZONE 19 | ) RETURNS TIMESTAMP WITH TIME ZONE 20 | AS $$ 21 | SELECT CASE 22 | WHEN $1 > NOW() THEN NOW() 23 | ELSE $1 24 | END; 25 | $$ LANGUAGE SQL; 26 | 27 | CREATE TABLE enclosures ("feed" TEXT NOT NULL, 28 | "item" TEXT NOT NULL, 29 | "url" TEXT NOT NULL, 30 | "type" TEXT, 31 | "title" TEXT, 32 | "guid" TEXT, 33 | PRIMARY KEY ("feed", "item", "url"), 34 | FOREIGN KEY ("feed", "item") 35 | REFERENCES "feed_items" ("feed", "id") 36 | ON DELETE CASCADE); 37 | CREATE INDEX enclosures_url ON enclosures ("url"); 38 | CREATE INDEX enclosures_guid ON enclosures ("guid"); 39 | 40 | -- Materialized distinct content types per feed 41 | CREATE TABLE feed_types ( 42 | "feed" TEXT NOT NULL, 43 | "type" TEXT, 44 | PRIMARY KEY ("feed", "type") 45 | ); 46 | CREATE OR REPLACE FUNCTION update_feed_type( 47 | "f_feed" TEXT, 48 | "f_type" TEXT 49 | ) RETURNS void AS $$ 50 | BEGIN 51 | PERFORM TRUE 52 | FROM enclosures 53 | WHERE "feed"=f_feed AND "type"=f_type; 54 | 55 | IF NOT FOUND THEN 56 | DELETE FROM enclosures 57 | WHERE "feed"=f_feed AND "type"=f_type; 58 | ELSE 59 | BEGIN 60 | INSERT INTO feed_types ("feed", "type") 61 | VALUES (f_feed, f_type); 62 | EXCEPTION 63 | WHEN integrity_constraint_violation 64 | THEN -- ignore 65 | END; 66 | END IF; 67 | END; 68 | $$ LANGUAGE plpgsql; 69 | 70 | CREATE OR REPLACE FUNCTION enclosures_update_feed_type( 71 | ) RETURNS trigger AS $$ 72 | BEGIN 73 | PERFORM update_feed_type(NEW.feed, NEW.type); 74 | RETURN NEW; 75 | END; 76 | $$ LANGUAGE plpgsql; 77 | CREATE TRIGGER enclosures_update_feed_type 78 | AFTER INSERT OR UPDATE ON enclosures 79 | FOR EACH ROW 80 | EXECUTE PROCEDURE enclosures_update_feed_type(); 81 | 82 | CREATE OR REPLACE FUNCTION enclosures_update_feed_type_delete( 83 | ) RETURNS trigger AS $$ 84 | BEGIN 85 | PERFORM update_feed_type(OLD.feed, OLD.type); 86 | RETURN OLD; 87 | END; 88 | $$ LANGUAGE plpgsql; 89 | CREATE TRIGGER enclosures_update_feed_type_delete 90 | AFTER INSERT OR UPDATE ON enclosures 91 | FOR EACH ROW 92 | EXECUTE PROCEDURE enclosures_update_feed_type_delete(); 93 | 94 | 95 | CREATE INDEX enclosure_torrents_info_hash 96 | ON enclosure_torrents (info_hash) 97 | WHERE LENGTH(info_hash) = 20; 98 | CREATE OR REPLACE VIEW enclosures_to_hash AS 99 | SELECT enclosures.url, 100 | enclosure_torrents.last_update AS last_update, 101 | enclosure_torrents.error AS error, 102 | enclosure_torrents.info_hash 103 | FROM enclosures 104 | LEFT JOIN enclosure_torrents 105 | ON (enclosures.url=enclosure_torrents.url) 106 | LEFT JOIN feeds 107 | ON (feeds.url=enclosures.feed) 108 | WHERE feeds.torrentify AND 109 | enclosures.url NOT LIKE '%.torrent' AND 110 | enclosures.type != 'application/x-bittorrent' AND 111 | (enclosure_torrents.info_hash IS NULL OR 112 | LENGTH(enclosure_torrents.info_hash)=0) 113 | ORDER BY last_update NULLS FIRST; 114 | 115 | CREATE INDEX enclosure_torrents_info_hash ON enclosure_torrents (info_hash); 116 | 117 | CREATE OR REPLACE FUNCTION enclosure_to_hash( 118 | min_inactivity INTERVAL DEFAULT '2 hours', 119 | OUT enclosure_url TEXT 120 | ) RETURNS TEXT AS $$ 121 | DECLARE 122 | next_url RECORD; 123 | BEGIN 124 | SELECT enclosures_to_hash.url, enclosures_to_hash.last_update 125 | INTO next_url 126 | FROM enclosures_to_hash 127 | LIMIT 1; 128 | IF next_url IS NULL OR next_url.url IS NULL THEN 129 | RETURN; 130 | END IF; 131 | 132 | IF next_url.last_update IS NULL THEN 133 | next_url.last_update = '1970-01-01 00:00:00'; 134 | END IF; 135 | IF next_url.last_update <= CURRENT_TIMESTAMP - min_inactivity THEN 136 | enclosure_url := next_url.url; 137 | IF EXISTS (SELECT "url" FROM enclosure_torrents WHERE "url"=enclosure_url) THEN 138 | UPDATE enclosure_torrents SET "last_update"=CURRENT_TIMESTAMP WHERE "url"=enclosure_url; 139 | ELSE 140 | INSERT INTO enclosure_torrents ("url", "last_update") VALUES (enclosure_url, CURRENT_TIMESTAMP); 141 | END IF; 142 | END IF; 143 | END; 144 | $$ LANGUAGE plpgsql; 145 | 146 | CREATE VIEW item_torrents AS 147 | SELECT enclosures.feed, enclosures.item, enclosures.url, 148 | enclosure_torrents.info_hash 149 | FROM enclosure_torrents LEFT JOIN enclosures ON (enclosures.url=enclosure_torrents.url) 150 | WHERE LENGTH(info_hash)=20; 151 | 152 | 153 | CREATE VIEW feed_errors AS 154 | SELECT feeds.url, 155 | MIN(COALESCE(feeds.error, enclosure.error)) AS error 156 | FROM feeds 157 | LEFT JOIN (SELECT feed, error 158 | FROM enclosures 159 | JOIN enclosure_torrents USING (url) 160 | WHERE error != '' 161 | ) AS enclosure ON (feeds.url = enclosure.feed) 162 | WHERE feeds.error IS NOT NULL 163 | OR enclosure.error != '' 164 | GROUP BY feeds.url; 165 | 166 | -- Login UI 167 | -- TODO: write reset functions 168 | CREATE TABLE user_tokens ( 169 | "kind" TEXT, 170 | "user" TEXT, 171 | "token" BYTEA PRIMARY KEY, 172 | "created" TIMESTAMP 173 | ); 174 | 175 | CREATE TABLE user_sessions ( 176 | "user" TEXT NOT NULL REFERENCES users ("name"), 177 | "sid" BYTEA PRIMARY KEY, 178 | "updated" TIMESTAMP 179 | ); 180 | -------------------------------------------------------------------------------- /rebar.config: -------------------------------------------------------------------------------- 1 | {sub_dirs, ["apps/shared", "apps/model", "apps/ui", "apps/feeds", "apps/hasher", "apps/seeder", 2 | "rel/ui", "rel/feeds", "rel/hasher", "rel/seeder"]}. 3 | 4 | {deps, [{cowboy, "0.*", 5 | {git, "git://github.com/extend/cowboy.git", {branch, "master"}}}, 6 | {epgsql, "VERSION", 7 | {git, "git://github.com/wg/epgsql.git", {tag, "1.4"}}}, 8 | {lhttpc, "1.3.*", 9 | {git, "git://github.com/astro/lhttpc.git", {branch, "master"}}}, 10 | {poolboy, "1.*", 11 | {git, "git://github.com/devinus/poolboy.git", {branch, "master"}}}, 12 | {exmpp, "git-head", 13 | {git, "git://github.com/astro/exmpp.git", {branch, "rebar-openssl-update"}}} 14 | ]}. 15 | 16 | {xref_warnings, true}. 17 | 18 | {relx, [ 19 | {release, {feeds, "1.0.0"}, 20 | [feeds]}, 21 | {release, {hasher, "1.0.0"}, 22 | [hasher]}, 23 | {release, {seeder, "1.0.0"}, 24 | [seeder]}, 25 | 26 | %% {dev_mode, true}, 27 | {include_erts, false}, 28 | 29 | {extended_start_script, false}, 30 | {vm_args, "config/vm.args"} 31 | ]}. 32 | -------------------------------------------------------------------------------- /tracker_load.erl: -------------------------------------------------------------------------------- 1 | -module(tracker_load). 2 | 3 | -export([run/2]). 4 | 5 | 6 | worker(Parent, Tracker, InfoHash) -> 7 | {MS, S, SS} = erlang:now(), 8 | random:seed(MS, S, SS), 9 | PeerId = <<"-<30000-", 10 | (<< << (random:uniform(255)):8 >> || _ <- lists:seq(1, 12) >>)/binary 11 | >>, 12 | worker(Parent, Tracker, InfoHash, PeerId, 0, 0). 13 | 14 | worker(Parent, Tracker, InfoHash, PeerId, Up, Down) -> 15 | URL = [Tracker, 16 | "?info_hash=", cowboy_http:urlencode(InfoHash), 17 | "&peer_id=", cowboy_http:urlencode(PeerId), 18 | "&port=6881&left=0&uploaded=", integer_to_list(Up), 19 | "&downloaded=", integer_to_list(Down) 20 | ], 21 | 22 | T1 = util:get_now_us(), 23 | case lhttpc:request(binary_to_list(list_to_binary(URL)), get, [], 1000) of 24 | {ok, {{200, _}, _Headers, _Body}} -> 25 | T2 = util:get_now_us(), 26 | Parent ! {req_done, T2 - T1}, 27 | ok 28 | end, 29 | worker(Parent, Tracker, InfoHash, PeerId, Up + 1, Down + 10). 30 | 31 | run(Tracker, InfoHash) -> 32 | run(Tracker, InfoHash, 1). 33 | 34 | run(Tracker, InfoHash, N) -> 35 | I = self(), 36 | spawn_link(fun() -> 37 | worker(I, Tracker, InfoHash) 38 | end), 39 | io:format("Running ~B clients~n", [N]), 40 | receive 41 | after 1000 -> 42 | {ok, Reqs, Time} = recv_all(), 43 | io:format("~B req/s, avg. ~B us~n", [Reqs, trunc(Time)]), 44 | run(Tracker, InfoHash, N + 4) 45 | end. 46 | 47 | 48 | recv_all() -> 49 | recv_all(0, 0). 50 | 51 | recv_all(Reqs, Time) -> 52 | receive 53 | {req_done, T} -> 54 | recv_all(Reqs + 1, Time + T) 55 | after 0 -> 56 | {ok, Reqs, Time / Reqs} 57 | end. 58 | --------------------------------------------------------------------------------