IsCheckedChanged { get; set; }
21 |
22 | [Parameter]
23 | public RenderFragment? ChildContent { get; set; }
24 |
25 | private bool InternalChecked
26 | {
27 | get => IsChecked;
28 | set
29 | {
30 | if (value == IsChecked) return;
31 | IsChecked = value;
32 | IsCheckedChanged.InvokeAsync(value);
33 | }
34 | }
35 | }
--------------------------------------------------------------------------------
/src/Zilean.ApiService/Features/Dashboard/Components/Pages/Dashboard/DashboardTorrentDetails.cs:
--------------------------------------------------------------------------------
1 | namespace Zilean.ApiService.Features.Dashboard.Components.Pages.Dashboard;
2 |
3 | public class DashboardTorrentDetails
4 | {
5 | [Required]
6 | [StringLength(40)]
7 | public string InfoHash { get; set; } = default!;
8 | public string Category { get; set; } = default!;
9 | [Required]
10 | public string? RawTitle { get; set; }
11 | public string? ParsedTitle { get; set; }
12 | public bool? Trash { get; set; } = false;
13 | [Range(0, 9999, ErrorMessage = "Please enter valid Year between 0 and 9999")]
14 | public string? Year { get; set; }
15 | [Required]
16 | [Range(0, long.MaxValue, ErrorMessage = "Please enter valid Filesize in Bytes")]
17 | public string? Size { get; set; }
18 | public string? ImdbId { get; set; }
19 | public bool IsAdult { get; set; }
20 | public bool ChangeCategory { get; set; }
21 | public bool ChangeTrash { get; set; }
22 | public bool ChangeYear { get; set; }
23 | public bool ChangeAdult { get; set; }
24 | public bool ChangeImdb { get; set; }
25 |
26 | public static TorrentInfo ToTorrentInfo(DashboardTorrentDetails dtd) => new()
27 | {
28 | InfoHash = dtd.InfoHash,
29 | RawTitle = dtd.RawTitle,
30 | ParsedTitle = dtd.ParsedTitle,
31 | Trash = dtd.Trash,
32 | Year = !dtd.Year.IsNullOrWhiteSpace() ? int.Parse(dtd.Year) : null,
33 | Category = dtd.Category,
34 | Size = dtd.Size,
35 | ImdbId = dtd.ImdbId,
36 | IsAdult = dtd.IsAdult
37 | };
38 |
39 | public static DashboardTorrentDetails FromTorrentInfo(TorrentInfo ti) => new()
40 | {
41 | InfoHash = ti.InfoHash,
42 | RawTitle = ti.RawTitle,
43 | ParsedTitle = ti.ParsedTitle,
44 | Trash = ti.Trash,
45 | Year = ti.Year.HasValue ? ti.Year.ToString() : null,
46 | Category = ti.Category,
47 | Size = ti.Size,
48 | ImdbId = ti.ImdbId,
49 | IsAdult = ti.IsAdult
50 | };
51 | }
52 |
--------------------------------------------------------------------------------
/src/Zilean.ApiService/Features/Dashboard/Components/Pages/Error.razor:
--------------------------------------------------------------------------------
1 | @page "/Error"
2 | @using System.Diagnostics
3 |
4 | Error
5 |
6 | Error.
7 | An error occurred while processing your request.
8 |
9 | @if (ShowRequestId)
10 | {
11 |
12 | Request ID: @RequestId
13 |
14 | }
15 |
16 | Development Mode
17 |
18 | Swapping to Development environment will display more detailed information about the error that occurred.
19 |
20 |
21 | The Development environment shouldn't be enabled for deployed applications.
22 | It can result in displaying sensitive information from exceptions to end users.
23 | For local debugging, enable the Development environment by setting the ASPNETCORE_ENVIRONMENT environment variable to Development
24 | and restarting the app.
25 |
26 |
27 | @code{
28 | [CascadingParameter]
29 | private HttpContext? HttpContext { get; set; }
30 |
31 | private string? RequestId { get; set; }
32 | private bool ShowRequestId => !string.IsNullOrEmpty(RequestId);
33 |
34 | protected override void OnInitialized() =>
35 | RequestId = Activity.Current?.Id ?? HttpContext?.TraceIdentifier;
36 | }
--------------------------------------------------------------------------------
/src/Zilean.ApiService/Features/Dashboard/Components/Routes.razor:
--------------------------------------------------------------------------------
1 |
2 |
3 |
6 |
--------------------------------------------------------------------------------
/src/Zilean.ApiService/Features/Dashboard/Components/ZileanWebApp.razor:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 | ();
28 |
29 | return group;
30 | }
31 |
32 | private static async Task> PerformSearch(HttpContext context, IImdbFileService imdbFileService, ZileanConfiguration configuration, ILogger logger, [AsParameters] ImdbFilteredRequest request)
33 | {
34 | try
35 | {
36 | if (string.IsNullOrEmpty(request.Query))
37 | {
38 | return TypedResults.Ok(Array.Empty());
39 | }
40 |
41 | logger.LogInformation("Performing imdb search for {@Request}", request);
42 |
43 | var results = await imdbFileService.SearchForImdbIdAsync(request.Query, request.Year, request.Category);
44 |
45 | logger.LogInformation("Filtered imdb search for {QueryText} returned {Count} results", request.Query, results.Length);
46 |
47 | return results.Length == 0
48 | ? TypedResults.Ok(Array.Empty())
49 | : TypedResults.Ok(results);
50 | }
51 | catch
52 | {
53 | return TypedResults.Ok(Array.Empty());
54 | }
55 | }
56 |
57 | private abstract class ImdbFilteredInstance;
58 | }
59 |
--------------------------------------------------------------------------------
/src/Zilean.ApiService/Features/Imdb/ImdbFilteredRequest.cs:
--------------------------------------------------------------------------------
1 | namespace Zilean.ApiService.Features.Imdb;
2 |
3 | public class ImdbFilteredRequest
4 | {
5 | public string? Query { get; init; }
6 | public int? Year { get; init; }
7 | public string? Category { get; init; }
8 | }
9 |
--------------------------------------------------------------------------------
/src/Zilean.ApiService/Features/Search/SearchFilteredRequest.cs:
--------------------------------------------------------------------------------
1 | namespace Zilean.ApiService.Features.Search;
2 |
3 | public class SearchFilteredRequest
4 | {
5 | public string? Query { get; init; }
6 | public int? Season { get; init; }
7 | public int? Episode { get; init; }
8 | public int? Year { get; init; }
9 | public string? Language { get; init; }
10 | public string? Resolution { get; init; }
11 | public string? ImdbId { get; init; }
12 | public string? Category { get; init; }
13 | }
14 |
--------------------------------------------------------------------------------
/src/Zilean.ApiService/Features/Sync/DmmSyncJob.cs:
--------------------------------------------------------------------------------
1 | namespace Zilean.ApiService.Features.Sync;
2 |
3 | public class DmmSyncJob(IShellExecutionService shellExecutionService, ILogger logger, ZileanDbContext dbContext) : IInvocable, ICancellableInvocable
4 | {
5 | public CancellationToken CancellationToken { get; set; }
6 | private const string DmmSyncArg = "dmm-sync";
7 |
8 | public async Task Invoke()
9 | {
10 | logger.LogInformation("Dmm SyncJob started");
11 |
12 | var argumentBuilder = ArgumentsBuilder.Create();
13 | argumentBuilder.AppendArgument(DmmSyncArg, string.Empty, false, false);
14 |
15 | await shellExecutionService.ExecuteCommand(new ShellCommandOptions
16 | {
17 | Command = Path.Combine(AppContext.BaseDirectory, "scraper"),
18 | ArgumentsBuilder = argumentBuilder,
19 | ShowOutput = true,
20 | CancellationToken = CancellationToken
21 | });
22 |
23 | logger.LogInformation("Dmm SyncJob completed");
24 | }
25 |
26 | // ReSharper disable once MethodSupportsCancellation
27 | public Task ShouldRunOnStartup() => dbContext.ParsedPages.AnyAsync();
28 | }
29 |
--------------------------------------------------------------------------------
/src/Zilean.ApiService/Features/Sync/GenericSyncJob.cs:
--------------------------------------------------------------------------------
1 | namespace Zilean.ApiService.Features.Sync;
2 |
3 | public class GenericSyncJob(IShellExecutionService shellExecutionService, ILogger logger, ZileanDbContext dbContext) : IInvocable, ICancellableInvocable
4 | {
5 | public CancellationToken CancellationToken { get; set; }
6 | private const string GenericSyncArg = "generic-sync";
7 |
8 | public async Task Invoke()
9 | {
10 | logger.LogInformation("Generic SyncJob started");
11 |
12 | var argumentBuilder = ArgumentsBuilder.Create();
13 | argumentBuilder.AppendArgument(GenericSyncArg, string.Empty, false, false);
14 |
15 | await shellExecutionService.ExecuteCommand(new ShellCommandOptions
16 | {
17 | Command = Path.Combine(AppContext.BaseDirectory, "scraper"),
18 | ArgumentsBuilder = argumentBuilder,
19 | ShowOutput = true,
20 | CancellationToken = CancellationToken
21 | });
22 |
23 | logger.LogInformation("Generic SyncJob completed");
24 | }
25 |
26 | // ReSharper disable once MethodSupportsCancellation
27 | public Task ShouldRunOnStartup() => dbContext.ParsedPages.AnyAsync();
28 | }
29 |
--------------------------------------------------------------------------------
/src/Zilean.ApiService/Features/Sync/SyncOnDemandState.cs:
--------------------------------------------------------------------------------
1 | namespace Zilean.ApiService.Features.Sync;
2 |
3 | public class SyncOnDemandState
4 | {
5 | public bool IsRunning { get; set; }
6 | }
7 |
--------------------------------------------------------------------------------
/src/Zilean.ApiService/Features/Torrents/CachedItem.cs:
--------------------------------------------------------------------------------
1 | namespace Zilean.ApiService.Features.Torrents;
2 |
3 | public class CachedItem
4 | {
5 | [JsonPropertyName("info_hash")]
6 | public string? InfoHash { get; set; }
7 | [JsonPropertyName("is_cached")]
8 | public bool? IsCached { get; set; }
9 | [JsonPropertyName("item")]
10 | public TorrentInfo? Item { get; set; }
11 | }
12 |
--------------------------------------------------------------------------------
/src/Zilean.ApiService/Features/Torrents/CheckCachedRequest.cs:
--------------------------------------------------------------------------------
1 | namespace Zilean.ApiService.Features.Torrents;
2 |
3 | public class CheckCachedRequest
4 | {
5 | public string? Hashes { get; set; }
6 | }
7 |
--------------------------------------------------------------------------------
/src/Zilean.ApiService/Features/Torrents/ErrorResponse.cs:
--------------------------------------------------------------------------------
1 | namespace Zilean.ApiService.Features.Torrents;
2 |
3 | public class ErrorResponse(string message)
4 | {
5 | [JsonPropertyName("message")]
6 | public string Message { get; } = message;
7 | }
8 |
--------------------------------------------------------------------------------
/src/Zilean.ApiService/Features/Torznab/TorznabRequest.cs:
--------------------------------------------------------------------------------
1 | // ReSharper disable InconsistentNaming
2 | namespace Zilean.ApiService.Features.Torznab;
3 |
4 | public class TorznabRequest
5 | {
6 | public string? q { get; set; }
7 | public string? imdbid { get; set; }
8 | public string? ep { get; set; }
9 | public string? t { get; set; }
10 | public string? extended { get; set; }
11 | public string? limit { get; set; }
12 | public string? offset { get; set; }
13 | public string? cat { get; set; }
14 | public string? season { get; set; }
15 | public string? year { get; set; }
16 | }
17 |
--------------------------------------------------------------------------------
/src/Zilean.ApiService/Features/Torznab/TorznabRequestExtensions.cs:
--------------------------------------------------------------------------------
1 | using Serilog;
2 | using ILogger = Serilog.ILogger;
3 |
4 | namespace Zilean.ApiService.Features.Torznab;
5 |
6 | public static class TorznabRequestExtensions
7 | {
8 | private static ILogger Logger => Log.ForContext(typeof(TorznabRequestExtensions));
9 |
10 | public static TorznabQuery? ToTorznabQuery(this TorznabRequest request)
11 | {
12 | try
13 | {
14 | var query = new TorznabQuery
15 | {
16 | QueryType = "search",
17 | SearchTerm = request.q,
18 | ImdbID = request.imdbid,
19 | };
20 | if (request.t != null)
21 | {
22 | query.QueryType = request.t;
23 | }
24 |
25 | if (!string.IsNullOrWhiteSpace(request.season))
26 | {
27 | query.Season = Parsing.CoerceInt(request.season);
28 | }
29 |
30 | if (!string.IsNullOrWhiteSpace(request.ep))
31 | {
32 | query.Episode = Parsing.CoerceInt(request.ep);
33 | }
34 |
35 | if (!string.IsNullOrWhiteSpace(request.extended))
36 | {
37 | query.Extended = Parsing.CoerceInt(request.extended);
38 | }
39 |
40 | if (!string.IsNullOrWhiteSpace(request.limit))
41 | {
42 | query.Limit = Parsing.CoerceInt(request.limit);
43 | }
44 |
45 | if (!string.IsNullOrWhiteSpace(request.offset))
46 | {
47 | query.Offset = Parsing.CoerceInt(request.offset);
48 | }
49 |
50 | query.Categories = request.cat != null
51 | ? request.cat.Split(',')
52 | .Where(s => !string.IsNullOrWhiteSpace(s))
53 | .Select(int.Parse)
54 | .ToArray()
55 | : query.QueryType switch
56 | {
57 | "movie" when !string.IsNullOrWhiteSpace(request.imdbid) => [TorznabCategoryTypes.Movies.Id],
58 | "tvSearch" when !string.IsNullOrWhiteSpace(request.imdbid) => [TorznabCategoryTypes.TV.Id],
59 | "xxx" when !string.IsNullOrWhiteSpace(request.imdbid) => [TorznabCategoryTypes.XXX.Id],
60 | _ => []
61 | };
62 |
63 |
64 | if (!string.IsNullOrWhiteSpace(request.season))
65 | {
66 | query.Season = int.Parse(request.season);
67 | }
68 |
69 | if (!string.IsNullOrWhiteSpace(request.year))
70 | {
71 | query.Year = int.Parse(request.year);
72 | }
73 |
74 | return query;
75 | }
76 | catch (Exception e)
77 | {
78 | Logger.Error(e, "Failed to convert TorznabRequest to TorznabQuery");
79 | return null;
80 | }
81 | }
82 | }
83 |
--------------------------------------------------------------------------------
/src/Zilean.ApiService/Features/Torznab/XmlResult.cs:
--------------------------------------------------------------------------------
1 | namespace Zilean.ApiService.Features.Torznab;
2 |
3 | public static class StreamManager
4 | {
5 | public static RecyclableMemoryStreamManager Instance { get; } = new();
6 | }
7 |
8 | public class XmlResult(T result, int statusCode) : IResult
9 | {
10 | private static readonly XmlSerializer _serializer = new(typeof(T));
11 |
12 | public async Task ExecuteAsync(HttpContext httpContext)
13 | {
14 | httpContext.Response.ContentType = "application/xml";
15 | httpContext.Response.StatusCode = statusCode;
16 |
17 | if (result is string xmlString)
18 | {
19 | // Handle string results directly to avoid wrapping in a tag
20 | await httpContext.Response.WriteAsync(xmlString);
21 | }
22 | else
23 | {
24 | // Serialize non-string objects to XML
25 | await using var ms = StreamManager.Instance.GetStream();
26 | _serializer.Serialize(ms, result);
27 |
28 | ms.Position = 0;
29 | await ms.CopyToAsync(httpContext.Response.Body);
30 | }
31 | }
32 | }
33 |
--------------------------------------------------------------------------------
/src/Zilean.ApiService/GlobalUsings.cs:
--------------------------------------------------------------------------------
1 | // Global using directives
2 |
3 | global using System.Collections;
4 | global using System.ComponentModel.DataAnnotations;
5 | global using System.Diagnostics;
6 | global using System.Diagnostics.CodeAnalysis;
7 | global using System.Globalization;
8 | global using System.Reflection;
9 | global using System.Security.Claims;
10 | global using System.Text.Encodings.Web;
11 | global using System.Text.Json;
12 | global using System.Text.Json.Serialization;
13 | global using System.Xml.Serialization;
14 | global using Coravel;
15 | global using Coravel.Invocable;
16 | global using Coravel.Scheduling.Schedule.Interfaces;
17 | global using Microsoft.AspNetCore.Authentication;
18 | global using Microsoft.AspNetCore.Authorization;
19 | global using Microsoft.AspNetCore.Http.HttpResults;
20 | global using Microsoft.AspNetCore.Mvc;
21 | global using Microsoft.AspNetCore.OpenApi;
22 | global using Microsoft.EntityFrameworkCore;
23 | global using Microsoft.Extensions.DependencyInjection;
24 | global using Microsoft.Extensions.Logging;
25 | global using Microsoft.Extensions.Options;
26 | global using Microsoft.IO;
27 | global using Microsoft.OpenApi.Models;
28 | global using Scalar.AspNetCore;
29 | global using SimCube.Aspire.Features.Otlp;
30 | global using Syncfusion.Blazor;
31 | global using Syncfusion.Blazor.Data;
32 | global using Zilean.ApiService.Features.Authentication;
33 | global using Zilean.ApiService.Features.Blacklist;
34 | global using Zilean.ApiService.Features.Bootstrapping;
35 | global using Zilean.ApiService.Features.Dashboard;
36 | global using Zilean.ApiService.Features.HealthChecks;
37 | global using Zilean.ApiService.Features.Imdb;
38 | global using Zilean.ApiService.Features.Search;
39 | global using Zilean.ApiService.Features.Sync;
40 | global using Zilean.ApiService.Features.Torrents;
41 | global using Zilean.ApiService.Features.Torznab;
42 | global using Zilean.Database;
43 | global using Zilean.Database.Bootstrapping;
44 | global using Zilean.Database.Services;
45 | global using Zilean.Shared.Extensions;
46 | global using Zilean.Shared.Features.Blacklist;
47 | global using Zilean.Shared.Features.Configuration;
48 | global using Zilean.Shared.Features.Dmm;
49 | global using Zilean.Shared.Features.Python;
50 | global using Zilean.Shared.Features.Scraping;
51 | global using Zilean.Shared.Features.Shell;
52 | global using Zilean.Shared.Features.Torznab;
53 | global using Zilean.Shared.Features.Torznab.Categories;
54 | global using Zilean.Shared.Features.Torznab.Info;
55 | global using Zilean.Shared.Features.Utilities;
56 |
--------------------------------------------------------------------------------
/src/Zilean.ApiService/Program.cs:
--------------------------------------------------------------------------------
1 | var builder = WebApplication.CreateBuilder(args);
2 |
3 | builder.Configuration.AddConfigurationFiles();
4 |
5 | var zileanConfiguration = builder.Configuration.GetZileanConfiguration();
6 |
7 | builder.AddOtlpServiceDefaults();
8 |
9 | builder.Services
10 | .AddConfiguration(zileanConfiguration)
11 | .AddSwaggerSupport()
12 | .AddSchedulingSupport()
13 | .AddShellExecutionService()
14 | .ConditionallyRegisterDmmJob(zileanConfiguration)
15 | .AddZileanDataServices(zileanConfiguration)
16 | .AddApiKeyAuthentication()
17 | .AddStartupHostedServices()
18 | .AddDashboardSupport(zileanConfiguration);
19 |
20 | var app = builder.Build();
21 |
22 | app.UseZileanRequired(zileanConfiguration);
23 | app.MapZileanEndpoints(zileanConfiguration);
24 | app.Services.SetupScheduling(zileanConfiguration);
25 |
26 | app.Run();
27 |
--------------------------------------------------------------------------------
/src/Zilean.ApiService/Properties/launchSettings.json:
--------------------------------------------------------------------------------
1 | {
2 | "$schema": "http://json.schemastore.org/launchsettings.json",
3 | "profiles": {
4 | "Zilean.ApiService": {
5 | "commandName": "Project",
6 | "launchBrowser": true,
7 | "launchUrl": "http://localhost:5000",
8 | "environmentVariables": {
9 | "ZILEAN_PYTHON_VENV": "C:\\Python311",
10 | "ASPNETCORE_ENVIRONMENT": "Development",
11 | "ASPNETCORE_URLS": "http://localhost:5000",
12 | "Zilean__Torrents__EnableEndpoint": "true",
13 | "Zilean__Ingestion__EnableScraping": "true",
14 | "Zilean__Dmm__EnableScraping": "true",
15 | "Zilean__EnableDashboard": "true",
16 | "ZILEAN__ApiKey": "test-123",
17 | "Zilean__Database__ConnectionString": "Host=localhost;Database=zilean-test;Username=postgres;Password=postgres;Include Error Detail=true;Timeout=30;CommandTimeout=3600;"
18 | }
19 | }
20 | }
21 | }
22 |
--------------------------------------------------------------------------------
/src/Zilean.ApiService/Zilean.ApiService.csproj:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 | Exe
5 | enable
6 | enable
7 | false
8 | zilean-api
9 |
10 |
11 |
12 |
24 |
25 |
26 |
29 |
30 |
31 |
32 | PreserveNewest
33 |
34 |
35 | PreserveNewest
36 |
37 |
38 | PreserveNewest
39 |
40 |
41 |
42 |
43 |
45 | Always
46 |
47 |
48 |
49 |
--------------------------------------------------------------------------------
/src/Zilean.ApiService/wwwroot/app.css:
--------------------------------------------------------------------------------
1 | html, body {
2 | font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif;
3 | background-color: #3a3a3a;
4 | }
5 |
6 | a, .btn-link {
7 | color: #006bb7;
8 | }
9 |
10 | .btn-primary {
11 | color: #fff;
12 | background-color: #1b6ec2;
13 | border-color: #1861ac;
14 | }
15 |
16 | .btn:focus, .btn:active:focus, .btn-link.nav-link:focus, .form-control:focus, .form-check-input:focus {
17 | box-shadow: 0 0 0 0.1rem white, 0 0 0 0.25rem #258cfb;
18 | }
19 |
20 | .content {
21 | padding-top: 1.1rem;
22 | }
23 |
24 | h1:focus {
25 | outline: none;
26 | }
27 |
28 | .valid.modified:not([type=checkbox]) {
29 | outline: 1px solid #26b050;
30 | }
31 |
32 | .invalid {
33 | outline: 1px solid #e50000;
34 | }
35 |
36 | .validation-message {
37 | color: #e50000;
38 | }
39 |
40 | .blazor-error-boundary {
41 | background: url(data:image/svg+xml;base64,PHN2ZyB3aWR0aD0iNTYiIGhlaWdodD0iNDkiIHhtbG5zPSJodHRwOi8vd3d3LnczLm9yZy8yMDAwL3N2ZyIgeG1sbnM6eGxpbms9Imh0dHA6Ly93d3cudzMub3JnLzE5OTkveGxpbmsiIG92ZXJmbG93PSJoaWRkZW4iPjxkZWZzPjxjbGlwUGF0aCBpZD0iY2xpcDAiPjxyZWN0IHg9IjIzNSIgeT0iNTEiIHdpZHRoPSI1NiIgaGVpZ2h0PSI0OSIvPjwvY2xpcFBhdGg+PC9kZWZzPjxnIGNsaXAtcGF0aD0idXJsKCNjbGlwMCkiIHRyYW5zZm9ybT0idHJhbnNsYXRlKC0yMzUgLTUxKSI+PHBhdGggZD0iTTI2My41MDYgNTFDMjY0LjcxNyA1MSAyNjUuODEzIDUxLjQ4MzcgMjY2LjYwNiA1Mi4yNjU4TDI2Ny4wNTIgNTIuNzk4NyAyNjcuNTM5IDUzLjYyODMgMjkwLjE4NSA5Mi4xODMxIDI5MC41NDUgOTIuNzk1IDI5MC42NTYgOTIuOTk2QzI5MC44NzcgOTMuNTEzIDI5MSA5NC4wODE1IDI5MSA5NC42NzgyIDI5MSA5Ny4wNjUxIDI4OS4wMzggOTkgMjg2LjYxNyA5OUwyNDAuMzgzIDk5QzIzNy45NjMgOTkgMjM2IDk3LjA2NTEgMjM2IDk0LjY3ODIgMjM2IDk0LjM3OTkgMjM2LjAzMSA5NC4wODg2IDIzNi4wODkgOTMuODA3MkwyMzYuMzM4IDkzLjAxNjIgMjM2Ljg1OCA5Mi4xMzE0IDI1OS40NzMgNTMuNjI5NCAyNTkuOTYxIDUyLjc5ODUgMjYwLjQwNyA1Mi4yNjU4QzI2MS4yIDUxLjQ4MzcgMjYyLjI5NiA1MSAyNjMuNTA2IDUxWk0yNjMuNTg2IDY2LjAxODNDMjYwLjczNyA2Ni4wMTgzIDI1OS4zMTMgNjcuMTI0NSAyNTkuMzEzIDY5LjMzNyAyNTkuMzEzIDY5LjYxMDIgMjU5LjMzMiA2OS44NjA4IDI1OS4zNzEgNzAuMDg4N0wyNjEuNzk1IDg0LjAxNjEgMjY1LjM4IDg0LjAxNjEgMjY3LjgyMSA2OS43NDc1QzI2Ny44NiA2OS43MzA5IDI2Ny44NzkgNjkuNTg3NyAyNjcuODc5IDY5LjMxNzkgMjY3Ljg3OSA2Ny4xMTgyIDI2Ni40NDggNjYuMDE4MyAyNjMuNTg2IDY2LjAxODNaTTI2My41NzYgODYuMDU0N0MyNjEuMDQ5IDg2LjA1NDcgMjU5Ljc4NiA4Ny4zMDA1IDI1OS43ODYgODkuNzkyMSAyNTkuNzg2IDkyLjI4MzcgMjYxLjA0OSA5My41Mjk1IDI2My41NzYgOTMuNTI5NSAyNjYuMTE2IDkzLjUyOTUgMjY3LjM4NyA5Mi4yODM3IDI2Ny4zODcgODkuNzkyMSAyNjcuMzg3IDg3LjMwMDUgMjY2LjExNiA4Ni4wNTQ3IDI2My41NzYgODYuMDU0N1oiIGZpbGw9IiNGRkU1MDAiIGZpbGwtcnVsZT0iZXZlbm9kZCIvPjwvZz48L3N2Zz4=) no-repeat 1rem/1.8rem, #b32121;
42 | padding: 1rem 1rem 1rem 3.7rem;
43 | color: white;
44 | }
45 |
46 | .blazor-error-boundary::after {
47 | content: "An error has occurred."
48 | }
49 |
50 | .darker-border-checkbox.form-check-input {
51 | border-color: #929292;
52 | }
53 |
--------------------------------------------------------------------------------
/src/Zilean.ApiService/wwwroot/images/zilean-logo.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/iPromKnight/zilean/8bfd20d49deda5e263a06476b47176d2a876a200/src/Zilean.ApiService/wwwroot/images/zilean-logo.png
--------------------------------------------------------------------------------
/src/Zilean.Benchmarks/Benchmarks/PythonParsing.cs:
--------------------------------------------------------------------------------
1 | using Zilean.Shared.Features.Python;
2 |
3 | namespace Zilean.Benchmarks.Benchmarks;
4 |
5 | public class PythonParsing
6 | {
7 | private ParseTorrentNameService _service = null!;
8 | private List? _oneK;
9 | private List? _fiveK;
10 | private List? _tenK;
11 | private List? _oneHundredK;
12 |
13 | [GlobalSetup]
14 | public void Setup()
15 | {
16 | Environment.SetEnvironmentVariable("PYTHONNET_PYDLL", "/opt/homebrew/opt/python@3.11/Frameworks/Python.framework/Versions/3.11/lib/libpython3.11.dylib");
17 | var logger = Substitute.For>();
18 | _service = new ParseTorrentNameService(logger);
19 | _oneK = GenerateTorrents(1000);
20 | _fiveK = GenerateTorrents(5000);
21 | _tenK = GenerateTorrents(10000);
22 | _oneHundredK = GenerateTorrents(100000);
23 | }
24 |
25 | [Benchmark]
26 | public async Task> ParseTorrent_1K_Success()
27 | {
28 | var results = await _service.ParseAndPopulateAsync(_oneK);
29 | return results;
30 | }
31 |
32 | [Benchmark]
33 | public async Task> ParseTorrent_5K_Success()
34 | {
35 | var results = await _service.ParseAndPopulateAsync(_fiveK);
36 | return results;
37 | }
38 |
39 | [Benchmark]
40 | public async Task> ParseTorrent_10k_Success()
41 | {
42 | var results = await _service.ParseAndPopulateAsync(_tenK);
43 | return results;
44 | }
45 |
46 | [Benchmark]
47 | public async Task> ParseTorrent_100k_Success()
48 | {
49 | var results = await _service.ParseAndPopulateAsync(_oneHundredK);
50 | return results;
51 | }
52 |
53 | private static List GenerateTorrents(int count)
54 | {
55 | var torrents = new List();
56 | var random = new Random();
57 | var titles = new[]
58 | {
59 | "Iron.Man.2008.INTERNAL.REMASTERED.2160p.UHD.BluRay.X265-IAMABLE",
60 | "Harry.Potter.and.the.Sorcerers.Stone.2001.2160p.UHD.BluRay.X265-IAMABLE",
61 | "The.Dark.Knight.2008.2160p.UHD.BluRay.X265-IAMABLE",
62 | "Inception.2010.2160p.UHD.BluRay.X265-IAMABLE",
63 | "The.Matrix.1999.2160p.UHD.BluRay.X265-IAMABLE"
64 | };
65 |
66 | for (int i = 0; i < count; i++)
67 | {
68 | var infoHash = $"1234562828797{i:D4}";
69 | var filename = titles[random.Next(titles.Length)];
70 | var filesize = (long)(random.NextDouble() * 100000000000);
71 |
72 | torrents.Add(new ExtractedDmmEntry(infoHash, filename, filesize, null));
73 | }
74 |
75 | return torrents;
76 | }
77 | }
78 |
--------------------------------------------------------------------------------
/src/Zilean.Benchmarks/GlobalUsings.cs:
--------------------------------------------------------------------------------
1 | global using BenchmarkDotNet.Attributes;
2 | global using BenchmarkDotNet.Running;
3 | global using Microsoft.Extensions.Logging;
4 | global using NSubstitute;
5 | global using Zilean.Shared.Features.Dmm;
6 |
--------------------------------------------------------------------------------
/src/Zilean.Benchmarks/Program.cs:
--------------------------------------------------------------------------------
1 | BenchmarkSwitcher.FromAssembly(typeof(Program).Assembly).Run(args);
2 |
--------------------------------------------------------------------------------
/src/Zilean.Benchmarks/Zilean.Benchmarks.csproj:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 | Exe
5 | enable
6 | enable
7 | true
8 |
9 |
10 |
11 |
14 |
15 |
16 |
18 |
19 |
20 |
22 | Always
23 |
24 |
25 |
26 |
27 |
--------------------------------------------------------------------------------
/src/Zilean.Database/Bootstrapping/ServiceCollectionExtensions.cs:
--------------------------------------------------------------------------------
1 | namespace Zilean.Database.Bootstrapping;
2 |
3 | public static class ServiceCollectionExtensions
4 | {
5 | public static IServiceCollection AddZileanDataServices(this IServiceCollection services, ZileanConfiguration configuration)
6 | {
7 | services.AddDbContext(options => options.UseNpgsql(configuration.Database.ConnectionString));
8 | services.AddTransient();
9 | services.AddTransient();
10 | services.RegisterImdbMatchingService(configuration);
11 |
12 | return services;
13 | }
14 |
15 | private static void RegisterImdbMatchingService(this IServiceCollection services, ZileanConfiguration configuration)
16 | {
17 | if (configuration.Imdb.UseLucene)
18 | {
19 | services.AddTransient();
20 | return;
21 | }
22 |
23 | services.AddTransient();
24 | }
25 | }
26 |
--------------------------------------------------------------------------------
/src/Zilean.Database/Dtos/ImdbSearchResult.cs:
--------------------------------------------------------------------------------
1 | namespace Zilean.Database.Dtos;
2 |
3 | public class ImdbSearchResult
4 | {
5 | public string? Title { get; set; }
6 | public string? ImdbId { get; set; }
7 | public int Year { get; set; }
8 | public double Score { get; set; }
9 | public string? Category { get; set; }
10 | }
11 |
--------------------------------------------------------------------------------
/src/Zilean.Database/Dtos/LuceneIndexEntry.cs:
--------------------------------------------------------------------------------
1 | namespace Zilean.Database.Dtos;
2 |
3 | public static class LuceneIndexEntry
4 | {
5 | public const string ImdbId = "imdbId";
6 | public const string Title = "title";
7 | public const string Year = "year";
8 | public const string Category = "category";
9 | }
10 |
--------------------------------------------------------------------------------
/src/Zilean.Database/Dtos/LuceneSession.cs:
--------------------------------------------------------------------------------
1 | using J2N.IO;
2 | using Lucene.Net.Analysis.Standard;
3 | using Lucene.Net.Index;
4 | using Lucene.Net.Util;
5 |
6 | namespace Zilean.Database.Dtos;
7 |
8 | public sealed class LuceneSession : IDisposable
9 | {
10 | public RAMDirectory? Directory { get; } = new();
11 | public StandardAnalyzer? Analyzer { get; } = new(LuceneVersion.LUCENE_48);
12 | public IndexWriterConfig? Config { get; private set; }
13 | public IndexWriter? Writer { get; private set; }
14 |
15 | public static LuceneSession NewInstance()
16 | {
17 | var instance = new LuceneSession();
18 |
19 | instance.Config = new(LuceneVersion.LUCENE_48, instance.Analyzer);
20 | instance.Writer = new(instance.Directory, instance.Config);
21 |
22 | return instance;
23 | }
24 |
25 | public void Dispose()
26 | {
27 | Directory?.Dispose();
28 | Analyzer?.Dispose();
29 | Writer?.Dispose();
30 | }
31 | }
32 |
--------------------------------------------------------------------------------
/src/Zilean.Database/Functions/SearchImdbProcedure.cs:
--------------------------------------------------------------------------------
1 | namespace Zilean.Database.Functions;
2 |
3 | public static class SearchImdbProcedure
4 | {
5 | internal const string CreateImdbProcedure =
6 | """
7 | CREATE OR REPLACE FUNCTION search_imdb_meta(search_term TEXT, category_param TEXT DEFAULT NULL, year_param INT DEFAULT NULL, limit_param INT DEFAULT 10, similarity_threshold REAL DEFAULT 0.85)
8 | RETURNS TABLE(imdb_id text, title text, category text, year INT, score REAL) AS $$
9 | BEGIN
10 | EXECUTE format('SET pg_trgm.similarity_threshold = %L', similarity_threshold);
11 | RETURN QUERY
12 | SELECT "ImdbId", "Title", "Category", "Year", similarity("Title", search_term) as score
13 | FROM public."ImdbFiles"
14 | WHERE ("Title" % search_term)
15 | AND ("Adult" = FALSE)
16 | AND (category_param IS NULL OR "Category" = category_param)
17 | AND (year_param IS NULL OR "Year" BETWEEN year_param - 1 AND year_param + 1)
18 | ORDER BY score DESC
19 | LIMIT limit_param;
20 | END; $$
21 | LANGUAGE plpgsql;
22 | """;
23 |
24 | internal const string RemoveImdbProcedure = "DROP FUNCTION IF EXISTS search_imdb_meta(TEXT, TEXT, INT, INT);";
25 | }
26 |
--------------------------------------------------------------------------------
/src/Zilean.Database/Functions/SearchImdbProcedureV2.cs:
--------------------------------------------------------------------------------
1 | namespace Zilean.Database.Functions;
2 |
3 | public static class SearchImdbProcedureV2
4 | {
5 | internal const string CreateImdbProcedure =
6 | """
7 | CREATE OR REPLACE FUNCTION search_imdb_meta(search_term TEXT, category_param TEXT DEFAULT NULL, year_param INT DEFAULT NULL, limit_param INT DEFAULT 10, similarity_threshold REAL DEFAULT 0.85)
8 | RETURNS TABLE(imdb_id text, title text, category text, year INT, score REAL) AS $$
9 | BEGIN
10 | EXECUTE format('SET pg_trgm.similarity_threshold = %L', similarity_threshold);
11 | RETURN QUERY
12 | SELECT "ImdbId", "Title", "Category", "Year", similarity("Title", search_term) as score
13 | FROM public."ImdbFiles"
14 | WHERE ("Title" % search_term)
15 | AND ("Adult" = FALSE)
16 | AND (
17 | category_param IS NULL
18 | OR (
19 | category_param = 'movie' AND "Category" IN ('movies', 'tvMovies')
20 | )
21 | OR (
22 | category_param = 'tvSeries' AND "Category" IN ('tvSeries', 'tvShort', 'tvMiniSeries', 'tvSpecial')
23 | )
24 | OR (
25 | category_param NOT IN ('movie', 'tvSeries') AND "Category" = category_param
26 | )
27 | )
28 | AND (year_param IS NULL OR "Year" BETWEEN year_param - 1 AND year_param + 1)
29 | ORDER BY score DESC
30 | LIMIT limit_param;
31 | END; $$
32 | LANGUAGE plpgsql;
33 | """;
34 |
35 | internal const string RemoveImdbProcedure = "DROP FUNCTION IF EXISTS search_imdb_meta(TEXT, TEXT, INT, INT);";
36 | }
37 |
--------------------------------------------------------------------------------
/src/Zilean.Database/Functions/SearchImdbProcedureV3.cs:
--------------------------------------------------------------------------------
1 | namespace Zilean.Database.Functions;
2 |
3 | public static class SearchImdbProcedureV3
4 | {
5 | internal const string CreateImdbProcedure =
6 | """
7 | CREATE OR REPLACE FUNCTION search_imdb_meta(search_term TEXT, category_param TEXT DEFAULT NULL, year_param INT DEFAULT NULL, limit_param INT DEFAULT 10, similarity_threshold REAL DEFAULT 0.85)
8 | RETURNS TABLE(imdb_id text, title text, category text, year INT, score REAL) AS $$
9 | BEGIN
10 | EXECUTE format('SET pg_trgm.similarity_threshold = %L', similarity_threshold);
11 | RETURN QUERY
12 | SELECT "ImdbId", "Title", "Category", "Year", similarity("Title", search_term) as score
13 | FROM public."ImdbFiles"
14 | WHERE ("Title" % search_term)
15 | AND ("Adult" = FALSE)
16 | AND (
17 | category_param IS NULL
18 | OR (
19 | category_param = 'movie' AND "Category" IN ('movie', 'tvMovie')
20 | )
21 | OR (
22 | category_param = 'tvSeries' AND "Category" IN ('tvSeries', 'tvShort', 'tvMiniSeries', 'tvSpecial')
23 | )
24 | OR (
25 | category_param NOT IN ('movie', 'tvSeries') AND "Category" = category_param
26 | )
27 | )
28 | AND (year_param IS NULL OR "Year" BETWEEN year_param - 1 AND year_param + 1)
29 | ORDER BY score DESC
30 | LIMIT limit_param;
31 | END; $$
32 | LANGUAGE plpgsql;
33 | """;
34 |
35 | internal const string RemoveImdbProcedure = "DROP FUNCTION IF EXISTS search_imdb_meta(TEXT, TEXT, INT, INT);";
36 | }
37 |
--------------------------------------------------------------------------------
/src/Zilean.Database/GlobalUsings.cs:
--------------------------------------------------------------------------------
1 | global using System.Collections.Concurrent;
2 | global using System.Data;
3 | global using System.Diagnostics;
4 | global using System.Text.Json;
5 | global using Dapper;
6 | global using EFCore.BulkExtensions;
7 | global using Lucene.Net.Store;
8 | global using Microsoft.EntityFrameworkCore;
9 | global using Microsoft.EntityFrameworkCore.Metadata.Builders;
10 | global using Microsoft.Extensions.DependencyInjection;
11 | global using Microsoft.Extensions.Logging;
12 | global using Npgsql;
13 | global using NpgsqlTypes;
14 | global using Spectre.Console;
15 | global using Zilean.Database.Dtos;
16 | global using Zilean.Database.ModelConfiguration;
17 | global using Zilean.Database.Services;
18 | global using Zilean.Database.Services.FuzzyString;
19 | global using Zilean.Database.Services.Lucene;
20 | global using Zilean.Shared.Features.Blacklist;
21 | global using Zilean.Shared.Features.Configuration;
22 | global using Zilean.Shared.Features.Dmm;
23 | global using Zilean.Shared.Features.Imdb;
24 | global using Zilean.Shared.Features.Statistics;
25 | global using Zilean.Shared.Features.Utilities;
26 |
--------------------------------------------------------------------------------
/src/Zilean.Database/Indexes/ImdbFilesIndexes.cs:
--------------------------------------------------------------------------------
1 | namespace Zilean.Database.Indexes;
2 |
3 | public static class ImdbFilesIndexes
4 | {
5 | internal const string CreateIndexes =
6 | """
7 | CREATE INDEX idx_imdb_metadata_adult ON public."ImdbFiles"("Adult");
8 | CREATE INDEX idx_imdb_metadata_category ON public."ImdbFiles"("Category");
9 | CREATE INDEX idx_imdb_metadata_year ON public."ImdbFiles"("Year");
10 | CREATE INDEX title_gin ON public."ImdbFiles" USING gin("Title" gin_trgm_ops);
11 | CREATE INDEX torrents_title_gin ON public."Torrents" USING gin("ParsedTitle" gin_trgm_ops);
12 | CREATE INDEX idx_torrents_infohash ON public."Torrents"("InfoHash");
13 | """;
14 |
15 | internal const string RemoveIndexes =
16 | """
17 | DROP INDEX IF EXISTS idx_imdb_metadata_adult;
18 | DROP INDEX IF EXISTS idx_imdb_metadata_category;
19 | DROP INDEX IF EXISTS idx_imdb_metadata_year;
20 | DROP INDEX IF EXISTS title_gin;
21 | DROP INDEX IF EXISTS torrents_title_gin;
22 | DROP INDEX IF EXISTS idx_torrents_infohash;
23 | """;
24 |
25 | }
26 |
--------------------------------------------------------------------------------
/src/Zilean.Database/Migrations/20240910121802_FunctionsAndIndexes.cs:
--------------------------------------------------------------------------------
1 | using Microsoft.EntityFrameworkCore.Migrations;
2 | using Zilean.Database.Functions;
3 | using Zilean.Database.Indexes;
4 |
5 | #nullable disable
6 |
7 | namespace Zilean.Database.Migrations;
8 |
9 | /// (
14 | name: "IngestedAt",
15 | table: "Torrents",
16 | type: "timestamp with time zone",
17 | nullable: false,
18 | defaultValueSql: "now() at time zone 'utc'");
19 |
20 | /// (type: "text", nullable: false),
19 | Reason = table.Column(type: "text", nullable: false),
20 | BlacklistedAt = table.Column(type: "timestamp with time zone", nullable: false, defaultValueSql: "now() at time zone 'utc'")
21 | },
22 | constraints: table =>
23 | {
24 | table.PrimaryKey("PK_BlacklistedItems", x => x.InfoHash);
25 | });
26 |
27 | migrationBuilder.CreateIndex(
28 | name: "IX_BlacklistedItems_InfoHash",
29 | table: "BlacklistedItems",
30 | column: "InfoHash",
31 | unique: true);
32 | }
33 |
34 | /// (
32 | name: "CleanedParsedTitle",
33 | table: "Torrents",
34 | type: "text",
35 | nullable: false,
36 | defaultValue: "");
37 |
38 | migrationBuilder.Sql(SearchTorrentsMetaV3.Remove);
39 | migrationBuilder.Sql(SearchTorrentsMetaV4.Create);
40 | migrationBuilder.Sql(UpdateTorrentsCleanedParsedTitle);
41 | }
42 |
43 | /// (
14 | name: "IsAdult",
15 | table: "Torrents",
16 | type: "boolean",
17 | nullable: false,
18 | defaultValue: false);
19 |
20 | migrationBuilder.CreateIndex(
21 | name: "idx_torrents_isadult",
22 | table: "Torrents",
23 | column: "IsAdult");
24 | }
25 |
26 | ///
4 | {
5 | public void Configure(EntityTypeBuilder builder)
6 | {
7 | builder.ToTable("BlacklistedItems");
8 |
9 | builder.HasKey(i => i.InfoHash);
10 |
11 | builder.Property(i => i.InfoHash)
12 | .HasColumnType("text")
13 | .HasAnnotation("Relational:JsonPropertyName", "info_hash");
14 |
15 | builder.Property(i => i.Reason)
16 | .IsRequired()
17 | .HasColumnType("text")
18 | .HasAnnotation("Relational:JsonPropertyName", "reason");
19 |
20 | builder.Property(t => t.BlacklistedAt)
21 | .IsRequired()
22 | .HasColumnType("timestamp with time zone")
23 | .HasDefaultValueSql("now() at time zone 'utc'")
24 | .HasAnnotation("Relational:JsonPropertyName", "blacklisted_at");
25 |
26 | builder.HasIndex(i => i.InfoHash)
27 | .IsUnique();
28 | }
29 | }
30 |
--------------------------------------------------------------------------------
/src/Zilean.Database/ModelConfiguration/ImdbFileConfiguration.cs:
--------------------------------------------------------------------------------
1 | namespace Zilean.Database.ModelConfiguration;
2 |
3 | public class ImdbFileConfiguration: IEntityTypeConfiguration
4 | {
5 | public void Configure(EntityTypeBuilder builder)
6 | {
7 | builder.ToTable("ImdbFiles");
8 |
9 | builder.HasKey(i => i.ImdbId);
10 |
11 | builder.Property(i => i.ImdbId)
12 | .HasColumnType("text");
13 |
14 | builder.Property(i => i.Category)
15 | .HasColumnType("text");
16 |
17 | builder.Property(i => i.Title)
18 | .HasColumnType("text");
19 |
20 | builder.Property(i => i.Adult)
21 | .HasColumnType("boolean");
22 |
23 | builder.Property(i => i.Year)
24 | .HasColumnType("integer");
25 |
26 | builder.HasIndex(i => i.ImdbId)
27 | .IsUnique();
28 | }
29 | }
30 |
--------------------------------------------------------------------------------
/src/Zilean.Database/ModelConfiguration/ImportMetadataConfiguration.cs:
--------------------------------------------------------------------------------
1 | namespace Zilean.Database.ModelConfiguration;
2 |
3 | public class ImportMetadataConfiguration : IEntityTypeConfiguration
4 | {
5 | public void Configure(EntityTypeBuilder builder)
6 | {
7 | builder.ToTable("ImportMetadata");
8 |
9 | builder.HasKey(x => x.Key);
10 |
11 | builder.Property(e => e.Value)
12 | .IsRequired()
13 | .HasColumnType("jsonb");
14 | }
15 | }
16 |
--------------------------------------------------------------------------------
/src/Zilean.Database/ModelConfiguration/ParsedPagesConfiguration.cs:
--------------------------------------------------------------------------------
1 | namespace Zilean.Database.ModelConfiguration;
2 |
3 | public class ParsedPagesConfiguration : IEntityTypeConfiguration
4 | {
5 | public void Configure(EntityTypeBuilder builder)
6 | {
7 | builder.ToTable("ParsedPages");
8 |
9 | builder.HasKey(x => x.Page);
10 | builder.Property(x => x.Page).IsRequired();
11 | builder.Property(x => x.EntryCount).IsRequired();
12 | }
13 | }
14 |
--------------------------------------------------------------------------------
/src/Zilean.Database/Services/BaseDapperService.cs:
--------------------------------------------------------------------------------
1 | namespace Zilean.Database.Services;
2 |
3 | public abstract class BaseDapperService(ILogger logger, ZileanConfiguration configuration)
4 | {
5 | protected ZileanConfiguration Configuration { get; } = configuration;
6 |
7 | protected async Task ExecuteCommandAsync(Func operation, string taskMessage, CancellationToken cancellationToken = default)
8 | {
9 | try
10 | {
11 | logger.LogInformation(taskMessage);
12 | await using var connection = new NpgsqlConnection(Configuration.Database.ConnectionString);
13 | await connection.OpenAsync(cancellationToken);
14 | await operation(connection);
15 | }
16 | catch (Exception ex)
17 | {
18 | logger.LogError(ex, "An error occurred while executing a command.");
19 | Process.GetCurrentProcess().Kill();
20 | }
21 | }
22 |
23 | protected async Task ExecuteCommandAsync(Func> operation,
24 | string errorMessage, CancellationToken cancellationToken = default)
25 | {
26 | try
27 | {
28 | await using var connection = new NpgsqlConnection(Configuration.Database.ConnectionString);
29 | await connection.OpenAsync(cancellationToken);
30 |
31 | var result = await operation(connection);
32 | return result;
33 | }
34 | catch (Exception e)
35 | {
36 | logger.LogError(e, errorMessage);
37 | throw;
38 | }
39 | }
40 | }
41 |
--------------------------------------------------------------------------------
/src/Zilean.Database/Services/DapperResult.cs:
--------------------------------------------------------------------------------
1 | namespace Zilean.Database.Services;
2 |
3 | public class DapperResult
4 | {
5 | public TSuccess Success { get; }
6 | public TFailure Failure { get; }
7 | public bool IsSuccess { get; }
8 |
9 | private DapperResult(TSuccess success, TFailure failure, bool isSuccess)
10 | {
11 | Success = success;
12 | Failure = failure;
13 | IsSuccess = isSuccess;
14 | }
15 |
16 | public static DapperResult Ok(TSuccess success) =>
17 | new(success, default, true);
18 |
19 | public static DapperResult Fail(TFailure failure) =>
20 | new(default, failure, false);
21 | }
22 |
--------------------------------------------------------------------------------
/src/Zilean.Database/Services/DmmService.cs:
--------------------------------------------------------------------------------
1 | namespace Zilean.Database.Services;
2 |
3 | public class DmmService(ILogger logger, ZileanConfiguration configuration, IServiceProvider serviceProvider) : BaseDapperService(logger, configuration)
4 | {
5 | public async Task GetDmmLastImportAsync(CancellationToken cancellationToken)
6 | {
7 | await using var serviceScope = serviceProvider.CreateAsyncScope();
8 | await using var dbContext = serviceScope.ServiceProvider.GetRequiredService();
9 |
10 | var dmmLastImport = await dbContext.ImportMetadata.AsNoTracking().FirstOrDefaultAsync(x => x.Key == MetadataKeys.DmmLastImport, cancellationToken: cancellationToken);
11 |
12 | return dmmLastImport?.Value.Deserialize();
13 | }
14 |
15 | public async Task SetDmmImportAsync(DmmLastImport dmmLastImport)
16 | {
17 | await using var serviceScope = serviceProvider.CreateAsyncScope();
18 | await using var dbContext = serviceScope.ServiceProvider.GetRequiredService();
19 |
20 | var metadata = await dbContext.ImportMetadata.FirstOrDefaultAsync(x => x.Key == MetadataKeys.DmmLastImport);
21 |
22 | if (metadata is null)
23 | {
24 | metadata = new ImportMetadata
25 | {
26 | Key = MetadataKeys.DmmLastImport,
27 | Value = JsonSerializer.SerializeToDocument(dmmLastImport),
28 | };
29 | await dbContext.ImportMetadata.AddAsync(metadata);
30 | await dbContext.SaveChangesAsync();
31 | return;
32 | }
33 |
34 | metadata.Value = JsonSerializer.SerializeToDocument(dmmLastImport);
35 | await dbContext.SaveChangesAsync();
36 | }
37 |
38 | public async Task AddPagesToIngestedAsync(IEnumerable pageNames, CancellationToken cancellationToken)
39 | {
40 | await using var serviceScope = serviceProvider.CreateAsyncScope();
41 | await using var dbContext = serviceScope.ServiceProvider.GetRequiredService();
42 | await dbContext.ParsedPages.AddRangeAsync(pageNames, cancellationToken);
43 | await dbContext.SaveChangesAsync(cancellationToken);
44 | }
45 |
46 | public async Task AddPageToIngestedAsync(ParsedPages pageNames, CancellationToken cancellationToken)
47 | {
48 | await using var serviceScope = serviceProvider.CreateAsyncScope();
49 | await using var dbContext = serviceScope.ServiceProvider.GetRequiredService();
50 | await dbContext.ParsedPages.AddAsync(pageNames, cancellationToken);
51 | await dbContext.SaveChangesAsync(cancellationToken);
52 | }
53 |
54 | public async Task> GetIngestedPagesAsync(CancellationToken cancellationToken)
55 | {
56 | await using var serviceScope = serviceProvider.CreateAsyncScope();
57 | await using var dbContext = serviceScope.ServiceProvider.GetRequiredService();
58 |
59 | return await dbContext.ParsedPages.AsNoTracking().ToListAsync(cancellationToken: cancellationToken);
60 | }
61 | }
62 |
--------------------------------------------------------------------------------
/src/Zilean.Database/Services/FuzzyString/ImdbFuzzyStringMatchingServiceLogger.cs:
--------------------------------------------------------------------------------
1 | namespace Zilean.Database.Services.FuzzyString;
2 |
3 | public static partial class ImdbFuzzyStringMatchingServiceLogger
4 | {
5 | [LoggerMessage(
6 | EventId = 1,
7 | Level = LogLevel.Warning,
8 | Message = "No suitable match found for Torrent '{Title}', Category: {Category}")]
9 | public static partial void NoSuitableMatchFound(this ILogger logger, string title, string category);
10 |
11 | [LoggerMessage(
12 | EventId = 2,
13 | Level = LogLevel.Information,
14 | Message = "Torrent '{Title}' updated from IMDb ID '{OldImdbId}' to '{NewImdbId}' with a score of {Score}, Category: {Category}, Imdb Title: {ImdbTitle}, Imdb Year: {ImdbYear}")]
15 | public static partial void TorrentUpdated(
16 | this ILogger logger,
17 | string title,
18 | string oldImdbId,
19 | string newImdbId,
20 | double score,
21 | string category,
22 | string imdbTitle,
23 | int imdbYear);
24 |
25 | [LoggerMessage(
26 | EventId = 3,
27 | Level = LogLevel.Information,
28 | Message = "Torrent '{Title}' retained its existing IMDb ID '{ImdbId}' with a best match score of {Score}, Category: {Category}, Imdb Title: {ImdbTitle}, Imdb Year: {ImdbYear}")]
29 | public static partial void TorrentRetained(
30 | this ILogger logger,
31 | string title,
32 | string imdbId,
33 | double score,
34 | string category,
35 | string imdbTitle,
36 | int imdbYear);
37 | }
38 |
--------------------------------------------------------------------------------
/src/Zilean.Database/Services/IDmmService.cs:
--------------------------------------------------------------------------------
1 | namespace Zilean.Database.Services;
2 |
3 | public interface IDmmService
4 | {
5 | Task GetDmmLastImportAsync(CancellationToken cancellationToken);
6 | Task SetDmmImportAsync(DmmLastImport dmmLastImport);
7 | Task AddPagesToIngestedAsync(IEnumerable pageNames);
8 | Task> GetIngestedPagesAsync();
9 | }
10 |
--------------------------------------------------------------------------------
/src/Zilean.Database/Services/IImdbFileService.cs:
--------------------------------------------------------------------------------
1 | namespace Zilean.Database.Services;
2 |
3 | public interface IImdbFileService
4 | {
5 | void AddImdbFile(ImdbFile imdbFile);
6 | Task StoreImdbFiles();
7 |
8 | Task SearchForImdbIdAsync(string query, int? year = null, string? category = null);
9 | Task SetImdbLastImportAsync(ImdbLastImport imdbLastImport);
10 | Task GetImdbLastImportAsync(CancellationToken cancellationToken);
11 | int ImdbFileCount { get; }
12 | Task VaccumImdbFilesIndexes(CancellationToken cancellationToken);
13 | }
14 |
--------------------------------------------------------------------------------
/src/Zilean.Database/Services/IImdbMatchingService.cs:
--------------------------------------------------------------------------------
1 | namespace Zilean.Database.Services;
2 |
3 | public interface IImdbMatchingService
4 | {
5 | Task> MatchImdbIdsForBatchAsync(IEnumerable batch);
6 | Task PopulateImdbData();
7 | void DisposeImdbData();
8 | }
9 |
--------------------------------------------------------------------------------
/src/Zilean.Database/Services/ITorrentInfoService.cs:
--------------------------------------------------------------------------------
1 | namespace Zilean.Database.Services;
2 |
3 | public interface ITorrentInfoService
4 | {
5 | Task StoreTorrentInfo(List torrents, int batchSize = 10000);
6 | Task SearchForTorrentInfoByOnlyTitle(string query);
7 | Task SearchForTorrentInfoFiltered(TorrentInfoFilter filter, int? limit = null);
8 | Task> GetExistingInfoHashesAsync(List infoHashes);
9 | Task> GetBlacklistedItems();
10 | Task VaccumTorrentsIndexes(CancellationToken cancellationToken);
11 | }
12 |
--------------------------------------------------------------------------------
/src/Zilean.Database/Services/Lucene/ImdbLuceneMatchingServiceLogger.cs:
--------------------------------------------------------------------------------
1 | namespace Zilean.Database.Services.Lucene;
2 |
3 | public static partial class ImdbLuceneMatchingServiceLogger
4 | {
5 | [LoggerMessage(
6 | EventId = 1,
7 | Level = LogLevel.Warning,
8 | Message = "No suitable match found for Torrent '{Title}', Category: {Category}")]
9 | public static partial void NoSuitableMatchFound(this ILogger logger, string title, string category);
10 |
11 | [LoggerMessage(
12 | EventId = 2,
13 | Level = LogLevel.Information,
14 | Message = "Torrent '{Title}' updated from IMDb ID '{OldImdbId}' to '{NewImdbId}' with a score of {Score}, Category: {Category}, Imdb Title: {ImdbTitle}, Imdb Year: {ImdbYear}")]
15 | public static partial void TorrentUpdated(
16 | this ILogger logger,
17 | string title,
18 | string oldImdbId,
19 | string newImdbId,
20 | double score,
21 | string category,
22 | string imdbTitle,
23 | int imdbYear);
24 |
25 | [LoggerMessage(
26 | EventId = 3,
27 | Level = LogLevel.Information,
28 | Message = "Torrent '{Title}' retained its existing IMDb ID '{ImdbId}' with a best match score of {Score}, Category: {Category}, Imdb Title: {ImdbTitle}, Imdb Year: {ImdbYear}")]
29 | public static partial void TorrentRetained(
30 | this ILogger logger,
31 | string title,
32 | string imdbId,
33 | double score,
34 | string category,
35 | string imdbTitle,
36 | int imdbYear);
37 | }
38 |
--------------------------------------------------------------------------------
/src/Zilean.Database/Services/TorrentInfoFilter.cs:
--------------------------------------------------------------------------------
1 | namespace Zilean.Database.Services;
2 |
3 | public class TorrentInfoFilter
4 | {
5 | public string? Query { get; init; }
6 | public int? Season { get; init; }
7 | public int? Episode { get; init; }
8 | public int? Year { get; init; }
9 | public string? Language { get; init; }
10 | public string? Resolution { get; init; }
11 | public string? ImdbId { get; init; }
12 | public string? Category { get; set; }
13 | }
14 |
--------------------------------------------------------------------------------
/src/Zilean.Database/Services/TorrentInfoQueryResult.cs:
--------------------------------------------------------------------------------
1 | namespace Zilean.Database.Services;
2 |
3 | public class TorrentInfoResult : TorrentInfo
4 | {
5 | // Aliased columns
6 | public string? ImdbCategory { get; set; } // Matches the alias in SQL
7 | public string? ImdbTitle { get; set; } // Matches the alias in SQL
8 | public int? ImdbYear { get; set; } // Matches the alias in SQL
9 | public bool ImdbAdult { get; set; } // Matches the alias in SQL
10 | }
11 |
--------------------------------------------------------------------------------
/src/Zilean.Database/Zilean.Database.csproj:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 | Library
5 | enable
6 | enable
7 | Zilean.Database
8 |
9 |
10 |
11 |
13 |
14 |
15 |
22 | all
23 | runtime; build; native; contentfiles; analyzers; buildtransitive
24 |
25 |
30 |
31 |
32 |
34 |
35 |
36 |
38 |
39 |
--------------------------------------------------------------------------------
/src/Zilean.Database/ZileanDbContext.cs:
--------------------------------------------------------------------------------
1 | namespace Zilean.Database;
2 |
3 | public class ZileanDbContext : DbContext
4 | {
5 | public ZileanDbContext()
6 | {
7 | }
8 |
9 | public ZileanDbContext(DbContextOptions options): base(options)
10 | {
11 | }
12 |
13 | protected override void OnConfiguring(DbContextOptionsBuilder optionsBuilder)
14 | {
15 | if (!optionsBuilder.IsConfigured)
16 | {
17 | optionsBuilder.UseNpgsql("Host=localhost;Port=5432;Database=zilean;Username=postgres;Password=postgres;CommandTimeout=0;Include Error Detail=true;");
18 | }
19 | base.OnConfiguring(optionsBuilder);
20 | }
21 |
22 | protected override void OnModelCreating(ModelBuilder modelBuilder)
23 | {
24 | base.OnModelCreating(modelBuilder);
25 |
26 | modelBuilder.ApplyConfiguration(new TorrentInfoConfiguration());
27 | modelBuilder.ApplyConfiguration(new ImdbFileConfiguration());
28 | modelBuilder.ApplyConfiguration(new ParsedPagesConfiguration());
29 | modelBuilder.ApplyConfiguration(new ImportMetadataConfiguration());
30 | modelBuilder.ApplyConfiguration(new BlacklistedItemConfiguration());
31 | }
32 |
33 | public DbSet Torrents => Set();
34 | public DbSet ImdbFiles => Set();
35 | public DbSet ParsedPages => Set();
36 | public DbSet ImportMetadata => Set();
37 | public DbSet BlacklistedItems => Set();
38 | }
39 |
--------------------------------------------------------------------------------
/src/Zilean.Scraper/Features/Bootstrapping/EnsureMigrated.cs:
--------------------------------------------------------------------------------
1 | namespace Zilean.Scraper.Features.Bootstrapping;
2 |
3 | public class EnsureMigrated(ImdbMetadataLoader metadataLoader, ILogger logger, ZileanDbContext dbContext, ZileanConfiguration configuration) : IHostedService
4 | {
5 | public async Task StartAsync(CancellationToken cancellationToken)
6 | {
7 | logger.LogInformation("Applying Migrations...");
8 | await dbContext.Database.MigrateAsync(cancellationToken: cancellationToken);
9 | logger.LogInformation("Migrations Applied.");
10 |
11 | if (configuration.Imdb.EnableImportMatching)
12 | {
13 | var imdbLoadedResult = await metadataLoader.Execute(cancellationToken);
14 |
15 | if (imdbLoadedResult == 1)
16 | {
17 | Environment.ExitCode = 1;
18 | Process.GetCurrentProcess().Kill();
19 | }
20 | }
21 | }
22 |
23 | public Task StopAsync(CancellationToken cancellationToken) => Task.CompletedTask;
24 | }
25 |
--------------------------------------------------------------------------------
/src/Zilean.Scraper/Features/Bootstrapping/HostingExtensions.cs:
--------------------------------------------------------------------------------
1 | namespace Zilean.Scraper.Features.Bootstrapping;
2 |
3 | public static class HostingExtensions
4 | {
5 | public static IServiceCollection AddCommandLine(
6 | this IServiceCollection services,
7 | Action configurator)
8 | {
9 | var app = new CommandApp(new TypeRegistrar(services));
10 | app.Configure(configurator);
11 | services.AddSingleton(app);
12 |
13 | return services;
14 | }
15 |
16 | public static IServiceCollection AddCommandLine(
17 | this IServiceCollection services,
18 | Action configurator)
19 | where TDefaultCommand : class, ICommand
20 | {
21 | var app = new CommandApp(new TypeRegistrar(services));
22 | app.Configure(configurator);
23 | services.AddSingleton(app);
24 |
25 | return services;
26 | }
27 |
28 | public static async Task RunAsync(this IHost host, string[] args)
29 | {
30 | ArgumentNullException.ThrowIfNull(host);
31 |
32 | await host.StartAsync();
33 |
34 | try
35 | {
36 | var app = host.Services.GetService() ??
37 | throw new InvalidOperationException("Command application has not been configured.");
38 |
39 | return await app.RunAsync(args);
40 | }
41 | finally
42 | {
43 | await host.StopAsync();
44 | await ((IAsyncDisposable)host).DisposeAsync();
45 | }
46 | }
47 | }
48 |
--------------------------------------------------------------------------------
/src/Zilean.Scraper/Features/Bootstrapping/ServiceCollectionExtensions.cs:
--------------------------------------------------------------------------------
1 | using Zilean.Scraper.Features.Ingestion.Dmm;
2 |
3 | namespace Zilean.Scraper.Features.Bootstrapping;
4 |
5 | public static class ServiceCollectionExtensions
6 | {
7 | public static void AddScrapers(this IServiceCollection services, IConfiguration configuration)
8 | {
9 | var zileanConfiguration = configuration.GetZileanConfiguration();
10 |
11 | services.AddHttpClient();
12 | services.AddSingleton(zileanConfiguration);
13 | services.AddImdbServices();
14 | services.AddDmmServices();
15 | services.AddGenericServices();
16 | services.AddZileanDataServices(zileanConfiguration);
17 | services.AddSingleton();
18 | services.AddHostedService();
19 | }
20 |
21 | private static void AddDmmServices(this IServiceCollection services)
22 | {
23 | services.AddSingleton();
24 | services.AddSingleton();
25 | services.AddTransient();
26 | }
27 |
28 | private static void AddGenericServices(this IServiceCollection services)
29 | {
30 | services.AddSingleton();
31 | services.AddSingleton();
32 | }
33 |
34 | private static void AddImdbServices(this IServiceCollection services)
35 | {
36 | services.AddSingleton();
37 | services.AddSingleton();
38 | services.AddSingleton();
39 | services.AddSingleton();
40 | services.AddSingleton();
41 | }
42 | }
43 |
--------------------------------------------------------------------------------
/src/Zilean.Scraper/Features/Bootstrapping/TypeRegistrar.cs:
--------------------------------------------------------------------------------
1 | namespace Zilean.Scraper.Features.Bootstrapping;
2 |
3 | internal sealed class TypeRegistrar(IServiceCollection provider) : ITypeRegistrar
4 | {
5 | public ITypeResolver Build() => new TypeResolver(provider.BuildServiceProvider());
6 |
7 | public void Register(Type service, Type implementation) => provider.AddSingleton(service, implementation);
8 |
9 | public void RegisterInstance(Type service, object implementation) => provider.AddSingleton(service, implementation);
10 |
11 | public void RegisterLazy(Type service, Func factory) => provider.AddSingleton(service, _ => factory());
12 | }
13 |
--------------------------------------------------------------------------------
/src/Zilean.Scraper/Features/Bootstrapping/TypeResolver.cs:
--------------------------------------------------------------------------------
1 | namespace Zilean.Scraper.Features.Bootstrapping;
2 |
3 | internal sealed class TypeResolver(IServiceProvider provider) : ITypeResolver
4 | {
5 | public object? Resolve(Type? type) =>
6 | type == null ? null : provider.GetService(type);
7 | }
8 |
--------------------------------------------------------------------------------
/src/Zilean.Scraper/Features/Commands/DefaultCommand.cs:
--------------------------------------------------------------------------------
1 | namespace Zilean.Scraper.Features.Commands;
2 |
3 | public sealed class DefaultCommand(ILogger logger) : Command
4 | {
5 | public sealed class Settings : CommandSettings
6 | {
7 | }
8 |
9 | public override int Execute(CommandContext context, Settings settings)
10 | {
11 | logger.LogInformation("Zilean Scraper: Execution Completed");
12 | return 0;
13 | }
14 | }
15 |
--------------------------------------------------------------------------------
/src/Zilean.Scraper/Features/Commands/DmmSyncCommand.cs:
--------------------------------------------------------------------------------
1 | using Zilean.Scraper.Features.Ingestion.Dmm;
2 |
3 | namespace Zilean.Scraper.Features.Commands;
4 |
5 | public class DmmSyncCommand(DmmScraping dmmScraping) : AsyncCommand
6 | {
7 | public override Task ExecuteAsync(CommandContext context) =>
8 | dmmScraping.Execute(CancellationToken.None);
9 | }
10 |
--------------------------------------------------------------------------------
/src/Zilean.Scraper/Features/Commands/GenericSyncCommand.cs:
--------------------------------------------------------------------------------
1 | namespace Zilean.Scraper.Features.Commands;
2 |
3 | public class GenericSyncCommand(GenericIngestionScraping genericIngestion) : AsyncCommand
4 | {
5 | public override Task ExecuteAsync(CommandContext context) =>
6 | genericIngestion.Execute(CancellationToken.None);
7 | }
8 |
--------------------------------------------------------------------------------
/src/Zilean.Scraper/Features/Imdb/ImdbFileDownloader.cs:
--------------------------------------------------------------------------------
1 | namespace Zilean.Scraper.Features.Imdb;
2 |
3 | public class ImdbFileDownloader(ILogger logger)
4 | {
5 | private static readonly string _dataFilePath = Path.Combine(AppContext.BaseDirectory, "data", TitleBasicsFileName);
6 | private const string TitleBasicsFileName = "title.basics.tsv";
7 | private const string ImdbDataBaseAddress = "https://datasets.imdbws.com/";
8 |
9 | public async Task DownloadMetadataFile(CancellationToken cancellationToken) =>
10 | await DownloadFileToTempPath(TitleBasicsFileName, cancellationToken);
11 |
12 | private async Task DownloadFileToTempPath(string fileName, CancellationToken cancellationToken)
13 | {
14 | if (File.Exists(_dataFilePath))
15 | {
16 | var fileInfo = new FileInfo(_dataFilePath);
17 | if (fileInfo.CreationTimeUtc <= DateTime.UtcNow.AddDays(30))
18 | {
19 | logger.LogInformation("IMDB data '{Filename}' already exists at {TempFile}. Will use records in that for import.", fileName, _dataFilePath);
20 | return _dataFilePath;
21 | }
22 |
23 | logger.LogInformation("IMDB data '{Filename}' is older than 30 days, deleting", fileName);
24 | File.Delete(_dataFilePath);
25 | }
26 |
27 | logger.LogInformation("Downloading IMDB data '{Filename}'", fileName);
28 |
29 | var client = CreateHttpClient();
30 | var response = await client.GetAsync($"{fileName}.gz", cancellationToken);
31 | response.EnsureSuccessStatusCode();
32 |
33 | await using var stream = await response.Content.ReadAsStreamAsync(cancellationToken);
34 | await using var gzipStream = new GZipStream(stream, CompressionMode.Decompress);
35 | await using var fileStream = File.Create(_dataFilePath);
36 |
37 | await gzipStream.CopyToAsync(fileStream, cancellationToken);
38 |
39 | logger.LogInformation("Downloaded IMDB data '{Filename}' to {TempFile}", fileName, _dataFilePath);
40 |
41 | fileStream.Close();
42 | return _dataFilePath;
43 | }
44 |
45 | private static HttpClient CreateHttpClient()
46 | {
47 | var httpClient = new HttpClient
48 | {
49 | BaseAddress = new Uri(ImdbDataBaseAddress),
50 | Timeout = TimeSpan.FromMinutes(30),
51 | };
52 |
53 | httpClient.DefaultRequestHeaders.UserAgent.ParseAdd("curl/7.54");
54 | return httpClient;
55 | }
56 | }
57 |
--------------------------------------------------------------------------------
/src/Zilean.Scraper/Features/Imdb/ImdbFileExtensions.cs:
--------------------------------------------------------------------------------
1 | namespace Zilean.Scraper.Features.Imdb;
2 |
3 | public static class ImdbFileExtensions
4 | {
5 | public static List GetCandidatesForYearRange(this ConcurrentDictionary> imdbFiles, int year)
6 | {
7 | var candidates = new List();
8 |
9 | for (int y = year - 1; y <= year + 1; y++)
10 | {
11 | if (imdbFiles.TryGetValue(y, out var files))
12 | {
13 | candidates.AddRange(files);
14 | }
15 | }
16 |
17 | return candidates;
18 | }
19 | }
20 |
--------------------------------------------------------------------------------
/src/Zilean.Scraper/Features/Imdb/ImdbFileProcessor.cs:
--------------------------------------------------------------------------------
1 | namespace Zilean.Scraper.Features.Imdb;
2 |
3 | public class ImdbFileProcessor(ILogger logger, IImdbFileService imdbFileService)
4 | {
5 | private static readonly List _requiredCategories = [
6 | "movie",
7 | "tvMovie",
8 | "tvSeries",
9 | "tvShort",
10 | "tvMiniSeries",
11 | "tvSpecial",
12 | ];
13 |
14 | public async Task Import(string fileName, CancellationToken cancellationToken)
15 | {
16 | logger.LogInformation("Importing Downloaded IMDB Basics data from {FilePath}", fileName);
17 |
18 | var csvConfig = new CsvConfiguration(CultureInfo.InvariantCulture)
19 | {
20 | Delimiter = "\t",
21 | BadDataFound = null,
22 | MissingFieldFound = null,
23 | HasHeaderRecord = true,
24 | ShouldSkipRecord = record => !_requiredCategories.Contains(record.Row.GetField(1))
25 | };
26 |
27 | using var reader = new StreamReader(fileName);
28 | using var csv = new CsvReader(reader, csvConfig);
29 |
30 | // skip header...
31 | await csv.ReadAsync();
32 |
33 | await ReadBasicEntries(csv, imdbFileService, cancellationToken);
34 |
35 | await imdbFileService.StoreImdbFiles();
36 |
37 | await imdbFileService.VaccumImdbFilesIndexes(cancellationToken);
38 | }
39 |
40 | private static async Task ReadBasicEntries(CsvReader csv, IImdbFileService imdbFileService, CancellationToken cancellationToken)
41 | {
42 | while (await csv.ReadAsync())
43 | {
44 | var isAdultSet = int.TryParse(csv.GetField(4), out var adult);
45 | var yearField = csv.GetField(5);
46 | var isYearValid = int.TryParse(yearField == @"\N" ? "0" : yearField, out var year);
47 |
48 | var movieData = new ImdbFile
49 | {
50 | ImdbId = csv.GetField(0),
51 | Category = csv.GetField(1),
52 | Title = csv.GetField(2),
53 | Adult = isAdultSet && adult == 1,
54 | Year = isYearValid ? year : 0
55 | };
56 |
57 | if (cancellationToken.IsCancellationRequested)
58 | {
59 | return;
60 | }
61 |
62 | imdbFileService.AddImdbFile(movieData);
63 | }
64 | }
65 | }
66 |
--------------------------------------------------------------------------------
/src/Zilean.Scraper/Features/Imdb/ImdbMetadataLoader.cs:
--------------------------------------------------------------------------------
1 | namespace Zilean.Scraper.Features.Imdb;
2 |
3 | public class ImdbMetadataLoader(ImdbFileDownloader downloader, ImdbFileProcessor processor, ILogger logger, ImdbFileService imdbFileService)
4 | {
5 | public async Task Execute(CancellationToken cancellationToken, bool skipLastImport = false)
6 | {
7 | try
8 | {
9 | if (!skipLastImport)
10 | {
11 | var imdbLastImport = await imdbFileService.GetImdbLastImportAsync(cancellationToken);
12 |
13 | if (imdbLastImport is not null)
14 | {
15 | logger.LogInformation("Last import date: {LastImportDate}", imdbLastImport.OccuredAt);
16 | if (DateTime.UtcNow - imdbLastImport.OccuredAt < TimeSpan.FromDays(14))
17 | {
18 | logger.LogInformation("Imdb Records import is not required as last import was less than 14 days ago");
19 | return 0;
20 | }
21 | }
22 | }
23 |
24 | var dataFile = await downloader.DownloadMetadataFile(cancellationToken);
25 |
26 | await processor.Import(dataFile, cancellationToken);
27 |
28 | logger.LogInformation("All IMDB records processed");
29 |
30 | logger.LogInformation("ImdbMetadataLoader Tasks Completed");
31 |
32 | return 0;
33 | }
34 | catch (TaskCanceledException)
35 | {
36 | logger.LogInformation("ImdbMetadataLoader Task Cancelled");
37 | return 1;
38 | }
39 | catch (OperationCanceledException)
40 | {
41 | logger.LogInformation("ImdbMetadataLoader Task Cancelled");
42 | return 1;
43 | }
44 | catch (Exception ex)
45 | {
46 | logger.LogError(ex, "Error occurred during ImdbMetadataLoader Task");
47 | return 1;
48 | }
49 | }
50 | }
51 |
--------------------------------------------------------------------------------
/src/Zilean.Scraper/Features/Ingestion/Dmm/DmmScraping.cs:
--------------------------------------------------------------------------------
1 | namespace Zilean.Scraper.Features.Ingestion.Dmm;
2 |
3 | public class DmmScraping(
4 | DmmFileDownloader downloader,
5 | ParseTorrentNameService parseTorrentNameService,
6 | ITorrentInfoService torrentInfoService,
7 | ZileanConfiguration configuration,
8 | ILogger logger,
9 | ILoggerFactory loggerFactory,
10 | DmmService dmmService)
11 | {
12 | public async Task Execute(CancellationToken cancellationToken)
13 | {
14 | try
15 | {
16 | var processor = new DmmFileEntryProcessor(dmmService, torrentInfoService, parseTorrentNameService, loggerFactory, configuration);
17 |
18 | var (dmmLastImport, created) = await RetrieveAndInitializeDmmLastImport(cancellationToken);
19 |
20 | var tempDirectory = await DownloadDmmFileToTempPath(dmmLastImport, created, cancellationToken);
21 |
22 | await UpdateDmmLastImportStatus(dmmLastImport, ImportStatus.InProgress, 0, 0);
23 |
24 | await processor.LoadParsedPages(cancellationToken);
25 |
26 | var files = Directory.GetFiles(tempDirectory, "*.html", SearchOption.AllDirectories)
27 | .Where(f => !processor.ExistingPages.ContainsKey(Path.GetFileName(f)))
28 | .ToList();
29 |
30 | logger.LogInformation("Found {Count} files to parse", files.Count);
31 |
32 | if (files.Count == 0)
33 | {
34 | logger.LogInformation("No files to parse, exiting");
35 | return 0;
36 | }
37 |
38 | await processor.ProcessFilesAsync(files, cancellationToken);
39 |
40 | logger.LogInformation("All files processed");
41 |
42 | await UpdateDmmLastImportStatus(dmmLastImport, ImportStatus.Complete, processor.NewPages.Count, processor.NewPages.Sum(x => x.Value));
43 |
44 | await torrentInfoService.VaccumTorrentsIndexes(cancellationToken);
45 |
46 | logger.LogInformation("DMM Internal Tasks Completed");
47 |
48 | return 0;
49 | }
50 | catch (TaskCanceledException)
51 | {
52 | return 0;
53 | }
54 | catch (OperationCanceledException)
55 | {
56 | return 0;
57 | }
58 | catch (Exception ex)
59 | {
60 | logger.LogError(ex, "Error occurred during DMM Scraper Task");
61 | return 1;
62 | }
63 | }
64 |
65 | private async Task DownloadDmmFileToTempPath(DmmLastImport dmmLastImport, bool created, CancellationToken cancellationToken) =>
66 | created
67 | ? await downloader.DownloadFileToTempPath(null, cancellationToken)
68 | : await downloader.DownloadFileToTempPath(dmmLastImport, cancellationToken);
69 |
70 | private async Task<(DmmLastImport DmmLastImport, bool Created)> RetrieveAndInitializeDmmLastImport(CancellationToken cancellationToken)
71 | {
72 | var dmmLastImport = await dmmService.GetDmmLastImportAsync(cancellationToken);
73 |
74 | if (dmmLastImport is not null)
75 | {
76 | return (dmmLastImport, false);
77 | }
78 |
79 | dmmLastImport = new DmmLastImport();
80 | return (dmmLastImport, true);
81 | }
82 |
83 | private async Task UpdateDmmLastImportStatus(DmmLastImport? dmmLastImport, ImportStatus status, int pageCount, int entryCount)
84 | {
85 | dmmLastImport.OccuredAt = DateTime.UtcNow;
86 | dmmLastImport.PageCount = pageCount;
87 | dmmLastImport.EntryCount = entryCount;
88 | dmmLastImport.Status = status;
89 |
90 | await dmmService.SetDmmImportAsync(dmmLastImport);
91 | }
92 | }
93 |
--------------------------------------------------------------------------------
/src/Zilean.Scraper/Features/Ingestion/Endpoints/KubernetesServiceDiscovery.cs:
--------------------------------------------------------------------------------
1 | namespace Zilean.Scraper.Features.Ingestion.Endpoints;
2 |
3 | public class KubernetesServiceDiscovery(
4 | ILogger logger,
5 | ZileanConfiguration configuration)
6 | {
7 | private record DiscoveredService(V1Service Service, KubernetesSelector Selector);
8 |
9 | public async Task> DiscoverUrlsAsync(CancellationToken cancellationToken = default)
10 | {
11 | var urls = new List();
12 |
13 | try
14 | {
15 | var clientConfig = configuration.Ingestion.Kubernetes.AuthenticationType switch
16 | {
17 | KubernetesAuthenticationType.ConfigFile => KubernetesClientConfiguration.BuildConfigFromConfigFile(configuration
18 | .Ingestion.Kubernetes.KubeConfigFile),
19 | KubernetesAuthenticationType.RoleBased => KubernetesClientConfiguration.InClusterConfig(),
20 | _ => throw new InvalidOperationException("Unknown authentication type")
21 | };
22 |
23 | var kubernetesClient = new Kubernetes(clientConfig);
24 |
25 | List discoveredServices = [];
26 |
27 | foreach (var selector in configuration.Ingestion.Kubernetes.KubernetesSelectors)
28 | {
29 | var services = await kubernetesClient.CoreV1.ListServiceForAllNamespacesAsync(
30 | labelSelector: selector.LabelSelector,
31 | cancellationToken: cancellationToken);
32 |
33 | discoveredServices.AddRange(services.Items.Select(service => new DiscoveredService(service, selector)));
34 | }
35 |
36 | foreach (var service in discoveredServices)
37 | {
38 | try
39 | {
40 | var url = BuildUrlFromService(service);
41 | if (!string.IsNullOrEmpty(url))
42 | {
43 | urls.Add(new GenericEndpoint
44 | {
45 | EndpointType = service.Selector.EndpointType,
46 | Url = url,
47 | });
48 | logger.LogInformation("Discovered service URL: {Url}", url);
49 | }
50 | }
51 | catch (Exception ex)
52 | {
53 | logger.LogError(ex, "Failed to build URL for service {ServiceName} in namespace {Namespace}",
54 | service.Service.Metadata.Name, service.Service.Metadata.NamespaceProperty);
55 | }
56 | }
57 | }
58 | catch (Exception ex)
59 | {
60 | logger.LogError(ex, "Failed to list services with label selectors {@LabelSelector}", configuration.Ingestion.Kubernetes.KubernetesSelectors);
61 | }
62 |
63 | return urls;
64 | }
65 |
66 | private string BuildUrlFromService(DiscoveredService service)
67 | {
68 | if (service.Service.Metadata?.NamespaceProperty == null)
69 | {
70 | throw new InvalidOperationException("Service metadata or namespace is missing.");
71 | }
72 |
73 | var namespaceName = service.Service.Metadata.NamespaceProperty;
74 | return string.Format(service.Selector.UrlTemplate, namespaceName);
75 | }
76 | }
77 |
--------------------------------------------------------------------------------
/src/Zilean.Scraper/Features/Ingestion/Processing/ProcessedCounts.cs:
--------------------------------------------------------------------------------
1 | namespace Zilean.Scraper.Features.Ingestion.Processing;
2 |
3 | public sealed class ProcessedCounts
4 | {
5 | private int _totalProcessed;
6 | private int _adultRemoved;
7 | private int _trashRemoved;
8 | private int _blacklistedRemoved;
9 |
10 | public void Reset()
11 | {
12 | Interlocked.Exchange(ref _totalProcessed, 0);
13 | Interlocked.Exchange(ref _adultRemoved, 0);
14 | Interlocked.Exchange(ref _trashRemoved, 0);
15 | Interlocked.Exchange(ref _blacklistedRemoved, 0);
16 | }
17 |
18 | public void AddProcessed(int count) => Interlocked.Add(ref _totalProcessed, count);
19 | public void AddAdultRemoved(int count) => Interlocked.Add(ref _adultRemoved, count);
20 | public void AddTrashRemoved(int count) => Interlocked.Add(ref _trashRemoved, count);
21 | public void AddBlacklistedRemoved(int count) => Interlocked.Add(ref _blacklistedRemoved, count);
22 |
23 | public void WriteOutput(ZileanConfiguration configuration, Stopwatch stopwatch, ConcurrentDictionary? newPages = null, GenericEndpoint? endpoint = null)
24 | {
25 | var table = new Table();
26 |
27 | table.AddColumn("Description");
28 | table.AddColumn("Count");
29 | table.AddColumn("Additional Info");
30 |
31 | if (newPages is not null)
32 | {
33 | table.AddRow("Processed new DMM pages", newPages.Count.ToString(), $"{newPages.Sum(x => x.Value)} entries");
34 | }
35 |
36 | if (endpoint is not null)
37 | {
38 | table.AddRow("Processed URL", endpoint.Url, $"Type: {endpoint.EndpointType}");
39 | }
40 |
41 | table.AddRow("Processed torrents", _totalProcessed.ToString(), $"Time Taken: {stopwatch.Elapsed.TotalSeconds:F2}s");
42 |
43 | if (_blacklistedRemoved > 0)
44 | {
45 | table.AddRow("Removed Blacklisted Content", _blacklistedRemoved.ToString(), "Due to identification by infohash");
46 | }
47 |
48 | AnsiConsole.Write(table);
49 | }
50 | }
51 |
--------------------------------------------------------------------------------
/src/Zilean.Scraper/Features/Ingestion/Processing/StreamedEntryProcessor.cs:
--------------------------------------------------------------------------------
1 | namespace Zilean.Scraper.Features.Ingestion.Processing;
2 |
3 | public class StreamedEntryProcessor(
4 | ITorrentInfoService torrentInfoService,
5 | ParseTorrentNameService parseTorrentNameService,
6 | ILoggerFactory loggerFactory,
7 | IHttpClientFactory clientFactory,
8 | ZileanConfiguration configuration) : GenericProcessor(loggerFactory, torrentInfoService, parseTorrentNameService, configuration)
9 | {
10 | private GenericEndpoint? _currentEndpoint;
11 |
12 | protected override ExtractedDmmEntry TransformToTorrent(StreamedEntry input) =>
13 | ExtractedDmmEntry.FromStreamedEntry(input);
14 |
15 | public async Task ProcessEndpointAsync(GenericEndpoint endpoint, CancellationToken cancellationToken)
16 | {
17 | var sw = Stopwatch.StartNew();
18 | _logger.LogInformation("Processing URL: {@Url}", endpoint);
19 | _processedCounts.Reset();
20 | _currentEndpoint = endpoint;
21 | await ProcessAsync(ProduceEntriesAsync, cancellationToken);
22 | _processedCounts.WriteOutput(_configuration, sw);
23 | sw.Stop();
24 | }
25 |
26 | private async Task ProduceEntriesAsync(ChannelWriter> writer, CancellationToken cancellationToken)
27 | {
28 | try
29 | {
30 | if (_currentEndpoint is null)
31 | {
32 | _logger.LogError("Endpoint not set before calling ProduceEntriesAsync.");
33 | throw new InvalidOperationException("Endpoint not set");
34 | }
35 |
36 | var httpClient = clientFactory.CreateClient();
37 | httpClient.Timeout = TimeSpan.FromSeconds(_configuration.Ingestion.RequestTimeout);
38 |
39 | var fullUrl = _currentEndpoint.EndpointType switch
40 | {
41 | GenericEndpointType.Zurg => $"{_currentEndpoint.Url}/debug/torrents",
42 | GenericEndpointType.Zilean => $"{_currentEndpoint.Url}/torrents/all",
43 | GenericEndpointType.Generic => $"{_currentEndpoint.Url}{_currentEndpoint.EndpointSuffix}",
44 | _ => throw new InvalidOperationException($"Unknown endpoint type: {_currentEndpoint.EndpointType}")
45 | };
46 |
47 | if (_currentEndpoint.EndpointType == GenericEndpointType.Zilean)
48 | {
49 | httpClient.DefaultRequestHeaders.Add("X-Api-Key", _currentEndpoint.ApiKey);
50 | }
51 | if (_currentEndpoint.EndpointType == GenericEndpointType.Generic)
52 | {
53 | if (!string.IsNullOrEmpty(_currentEndpoint.Authorization))
54 | {
55 | httpClient.DefaultRequestHeaders.Add("Authorization", _currentEndpoint.Authorization);
56 | }
57 | }
58 |
59 | var response = await httpClient.GetAsync(fullUrl, HttpCompletionOption.ResponseHeadersRead, cancellationToken);
60 | response.EnsureSuccessStatusCode();
61 |
62 | await using var stream = await response.Content.ReadAsStreamAsync(cancellationToken);
63 | var options = new JsonSerializerOptions { PropertyNameCaseInsensitive = true };
64 |
65 | await foreach (var item in JsonSerializer.DeserializeAsyncEnumerable(stream, options, cancellationToken))
66 | {
67 | if (item is not null)
68 | {
69 | await writer.WriteAsync(Task.FromResult(item), cancellationToken);
70 | }
71 | }
72 | }
73 | catch (Exception ex)
74 | {
75 | _logger.LogError(ex, "Error while fetching and producing entries for URL: {Url}", _currentEndpoint.Url);
76 | }
77 | finally
78 | {
79 | writer.Complete();
80 | }
81 | }
82 | }
83 |
--------------------------------------------------------------------------------
/src/Zilean.Scraper/Features/Ingestion/Processing/TorrentInfoExtensions.cs:
--------------------------------------------------------------------------------
1 | using ILogger = Microsoft.Extensions.Logging.ILogger;
2 |
3 | namespace Zilean.Scraper.Features.Ingestion.Processing;
4 |
5 | public static class TorrentInfoExtensions
6 | {
7 | public static bool IsBlacklisted(this TorrentInfo torrent, HashSet blacklistedItems) =>
8 | blacklistedItems.Any(x => x.Equals(torrent.InfoHash, StringComparison.OrdinalIgnoreCase));
9 |
10 | public static IEnumerable FilterBlacklistedTorrents(this IEnumerable finalizedTorrentsEnumerable,
11 | List parsedTorrents, HashSet blacklistedHashes, ZileanConfiguration configuration, ILogger logger,
12 | ProcessedCounts processedCount)
13 | {
14 | if (blacklistedHashes.Count <= 0)
15 | {
16 | return finalizedTorrentsEnumerable;
17 | }
18 |
19 | finalizedTorrentsEnumerable = finalizedTorrentsEnumerable.Where(t => !blacklistedHashes.Contains(t.InfoHash));
20 | var blacklistedCount = parsedTorrents.Count(x => blacklistedHashes.Contains(x.InfoHash));
21 | logger.LogInformation("Filtered out {Count} blacklisted torrents", blacklistedCount);
22 | processedCount.AddBlacklistedRemoved(blacklistedCount);
23 |
24 | return finalizedTorrentsEnumerable;
25 | }
26 | }
27 |
--------------------------------------------------------------------------------
/src/Zilean.Scraper/Features/LzString/StringBuilderCache.cs:
--------------------------------------------------------------------------------
1 | namespace Zilean.Scraper.Features.LzString;
2 |
3 | public static class StringBuilderCache
4 | {
5 | [ThreadStatic]
6 | private static StringBuilder? _cachedInstance;
7 |
8 | public static StringBuilder Acquire(int capacity = 16)
9 | {
10 | if (capacity > 360)
11 | {
12 | return new StringBuilder(capacity);
13 | }
14 |
15 | var sb = _cachedInstance;
16 |
17 | if (sb == null || capacity > sb.Capacity)
18 | {
19 | return new StringBuilder(capacity);
20 | }
21 |
22 | _cachedInstance = null;
23 | sb.Clear();
24 |
25 | return sb;
26 | }
27 |
28 | public static string GetStringAndRelease(StringBuilder sb)
29 | {
30 | string result = sb.ToString();
31 | Release(sb);
32 | return result;
33 | }
34 |
35 | private static void Release(StringBuilder sb)
36 | {
37 | if (sb.Capacity <= 360)
38 | {
39 | _cachedInstance = sb;
40 | }
41 | }
42 | }
43 |
--------------------------------------------------------------------------------
/src/Zilean.Scraper/GlobalUsings.cs:
--------------------------------------------------------------------------------
1 | global using System.Buffers;
2 | global using System.Collections.Concurrent;
3 | global using System.ComponentModel;
4 | global using System.Diagnostics;
5 | global using System.Diagnostics.CodeAnalysis;
6 | global using System.Globalization;
7 | global using System.IO.Compression;
8 | global using System.Runtime.CompilerServices;
9 | global using System.Runtime.InteropServices;
10 | global using System.Text;
11 | global using System.Text.Json;
12 | global using System.Text.RegularExpressions;
13 | global using System.Threading.Channels;
14 | global using CsvHelper;
15 | global using CsvHelper.Configuration;
16 | global using k8s;
17 | global using k8s.Models;
18 | global using Microsoft.EntityFrameworkCore;
19 | global using Microsoft.Extensions.Configuration;
20 | global using Microsoft.Extensions.DependencyInjection;
21 | global using Microsoft.Extensions.Hosting;
22 | global using Microsoft.Extensions.Logging;
23 | global using Microsoft.Extensions.ObjectPool;
24 | global using Npgsql;
25 | global using Python.Runtime;
26 | global using Serilog;
27 | global using Serilog.Sinks.Spectre;
28 | global using SimCube.Aspire.Features.Otlp;
29 | global using Spectre.Console;
30 | global using Spectre.Console.Cli;
31 | global using Zilean.Database;
32 | global using Zilean.Database.Bootstrapping;
33 | global using Zilean.Database.Services;
34 | global using Zilean.Scraper.Features.Bootstrapping;
35 | global using Zilean.Scraper.Features.Commands;
36 | global using Zilean.Scraper.Features.Ingestion;
37 | global using Zilean.Scraper.Features.Imdb;
38 | global using Zilean.Scraper.Features.Ingestion.Endpoints;
39 | global using Zilean.Scraper.Features.Ingestion.Processing;
40 | global using Zilean.Scraper.Features.LzString;
41 | global using Zilean.Shared.Extensions;
42 | global using Zilean.Shared.Features.Configuration;
43 | global using Zilean.Shared.Features.Dmm;
44 | global using Zilean.Shared.Features.Imdb;
45 | global using Zilean.Shared.Features.Python;
46 | global using Zilean.Shared.Features.Scraping;
47 | global using Zilean.Shared.Features.Statistics;
48 |
--------------------------------------------------------------------------------
/src/Zilean.Scraper/Program.cs:
--------------------------------------------------------------------------------
1 | var builder = Host.CreateDefaultBuilder();
2 |
3 | builder.ConfigureAppConfiguration(configuration =>
4 | {
5 | configuration.AddConfigurationFiles();
6 | });
7 |
8 | builder.ConfigureLogging((context, logging) =>
9 | {
10 | logging.ClearProviders();
11 | var loggingConfiguration = context.Configuration.GetLoggerConfiguration();
12 | Log.Logger = loggingConfiguration.CreateLogger();
13 | logging.AddSerilog();
14 | });
15 |
16 | builder.ConfigureServices((context, services) =>
17 | {
18 | services.AddScrapers(context.Configuration);
19 | services.AddCommandLine(config =>
20 | {
21 | config.SetApplicationName("zilean-scraper");
22 |
23 | config.AddCommand("dmm-sync")
24 | .WithDescription("Sync DMM Hashlists from Github.");
25 |
26 | config.AddCommand("generic-sync")
27 | .WithDescription("Sync data from Zurg and Zilean instances.");
28 |
29 | config.AddCommand("resync-imdb")
30 | .WithDescription("Force resync imdb data.");
31 | });
32 | });
33 |
34 | var host = builder.Build();
35 | return await host.RunAsync(args);
36 |
--------------------------------------------------------------------------------
/src/Zilean.Scraper/Properties/launchSettings.json:
--------------------------------------------------------------------------------
1 | {
2 | "$schema": "http://json.schemastore.org/launchsettings.json",
3 | "profiles": {
4 | "Zilean.Scraper: DMM Sync": {
5 | "commandName": "Project",
6 | "environmentVariables": {
7 | "ZILEAN_PYTHON_VENV": "C:\\Python311",
8 | "Zilean__Database__ConnectionString": "Host=localhost;Database=zilean;Username=postgres;Password=postgres;Include Error Detail=true;Timeout=30;CommandTimeout=3600;",
9 | "Zilean__Parsing__IncludeTrash": "false",
10 | "Zilean__Parsing__IncludeAdult": "false",
11 | "Zilean__Parsing__BatchSize": "5000"
12 | },
13 | "commandLineArgs": "dmm-sync"
14 | },
15 | "Zilean.Scraper: Generic Sync": {
16 | "commandName": "Project",
17 | "environmentVariables": {
18 | "ZILEAN_PYTHON_VENV": "C:\\Python311",
19 | "Zilean__Database__ConnectionString": "Host=localhost;Database=zilean;Username=postgres;Password=postgres;Include Error Detail=true;Timeout=300;CommandTimeout=3600;"
20 | },
21 | "commandLineArgs": "generic-sync"
22 | },
23 | "Zilean.Scraper: Resync Imdb Files": {
24 | "commandName": "Project",
25 | "environmentVariables": {
26 | "ZILEAN_PYTHON_VENV": "C:\\Python311",
27 | "Zilean__Database__ConnectionString": "Host=media-host;port=17000;Database=zilean;Username=postgres;Password=V8fVsqcFjiknykULWCA7egxMZAznkE;Include Error Detail=true;Timeout=30;CommandTimeout=36000;",
28 | "Zilean__Imdb__UseAllCores": "true",
29 | "Zilean__Imdb__MinimumScoreMatch": "0.85",
30 | "Zilean__Imdb__UseLucene": "true"
31 | },
32 | "commandLineArgs": "resync-imdb -t"
33 | }
34 | }
35 | }
36 |
--------------------------------------------------------------------------------
/src/Zilean.Scraper/Zilean.Scraper.csproj:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 | Exe
5 | enable
6 | enable
7 | scraper
8 | false
9 | true
10 |
11 |
12 |
13 |
15 |
16 |
17 |
20 |
21 |
22 |
26 |
27 |
28 |
30 | Always
31 |
32 |
33 |
34 |
--------------------------------------------------------------------------------
/src/Zilean.Shared/Extensions/CollectionExtensions.cs:
--------------------------------------------------------------------------------
1 | namespace Zilean.Shared.Extensions;
2 |
3 | public static class CollectionExtensions
4 | {
5 | public static IEnumerable> ToChunks(this List collection, int batchSize)
6 | {
7 | for (int i = 0; i < collection.Count; i += batchSize)
8 | {
9 | yield return collection.GetRange(i, Math.Min(batchSize, collection.Count - i));
10 | }
11 | }
12 |
13 | public static async IAsyncEnumerable> ToChunksAsync(this IAsyncEnumerable source, int size,
14 | [EnumeratorCancellation] CancellationToken cancellationToken = default)
15 | {
16 | if (size <= 0)
17 | {
18 | throw new ArgumentException("Chunk size must be greater than zero.", nameof(size));
19 | }
20 |
21 | var batch = new List(size);
22 |
23 | await foreach (var item in source.WithCancellation(cancellationToken))
24 | {
25 | batch.Add(item);
26 | if (batch.Count != size)
27 | {
28 | continue;
29 | }
30 |
31 | yield return batch;
32 | batch = new List(size);
33 | }
34 |
35 | if (batch.Count > 0)
36 | {
37 | yield return batch;
38 | }
39 | }
40 | }
41 |
--------------------------------------------------------------------------------
/src/Zilean.Shared/Extensions/DictionaryExtensions.cs:
--------------------------------------------------------------------------------
1 | namespace Zilean.Shared.Extensions;
2 |
3 | public static class DictionaryExtensions
4 | {
5 | public static ConcurrentDictionary ToConcurrentDictionary(
6 | this IEnumerable source,
7 | Func keySelector,
8 | Func valueSelector) where TKey : notnull
9 | {
10 | var concurrentDictionary = new ConcurrentDictionary();
11 |
12 | foreach (var element in source)
13 | {
14 | concurrentDictionary.TryAdd(keySelector(element), valueSelector(element));
15 | }
16 |
17 | return concurrentDictionary;
18 | }
19 | }
20 |
--------------------------------------------------------------------------------
/src/Zilean.Shared/Extensions/JsonExtensions.cs:
--------------------------------------------------------------------------------
1 | namespace Zilean.Shared.Extensions;
2 |
3 | public static class JsonExtensions
4 | {
5 | private static readonly JsonSerializerOptions _jsonSerializerOptions = new()
6 | {
7 | PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
8 | WriteIndented = false,
9 | ReferenceHandler = ReferenceHandler.IgnoreCycles,
10 | NumberHandling = JsonNumberHandling.Strict,
11 | };
12 |
13 | public static string AsJson(this T obj) => JsonSerializer.Serialize(obj, _jsonSerializerOptions);
14 | }
15 |
--------------------------------------------------------------------------------
/src/Zilean.Shared/Extensions/StringExtensions.cs:
--------------------------------------------------------------------------------
1 | namespace Zilean.Shared.Extensions;
2 |
3 | public static class StringExtensions
4 | {
5 | public static bool ContainsIgnoreCase(this string? source, string toCheck) =>
6 | source.Contains(toCheck, StringComparison.OrdinalIgnoreCase);
7 |
8 | public static bool ContainsIgnoreCase(this IEnumerable? source, string toCheck) =>
9 | source.Any(s => s.Contains(toCheck, StringComparison.OrdinalIgnoreCase));
10 |
11 | public static bool IsNullOrWhiteSpace(this string? source) =>
12 | string.IsNullOrWhiteSpace(source);
13 | }
14 |
--------------------------------------------------------------------------------
/src/Zilean.Shared/Features/Blacklist/BlacklistedItem.cs:
--------------------------------------------------------------------------------
1 | namespace Zilean.Shared.Features.Blacklist;
2 |
3 | public class BlacklistedItem
4 | {
5 | [JsonPropertyName("info_hash")]
6 | public string? InfoHash { get; set; }
7 |
8 | [JsonPropertyName("reason")]
9 | public string? Reason { get; set; }
10 |
11 | [JsonPropertyName("blacklisted_at")]
12 | public DateTime? BlacklistedAt { get; set; }
13 | }
14 |
--------------------------------------------------------------------------------
/src/Zilean.Shared/Features/Configuration/ConfigurationExtensions.cs:
--------------------------------------------------------------------------------
1 | using Microsoft.Extensions.Configuration;
2 |
3 | namespace Zilean.Shared.Features.Configuration;
4 |
5 | public static class ConfigurationExtensions
6 | {
7 | public static IConfigurationBuilder AddConfigurationFiles(this IConfigurationBuilder configuration)
8 | {
9 | var configurationFolderPath = Path.Combine(AppContext.BaseDirectory, ConfigurationLiterals.ConfigurationFolder);
10 |
11 | EnsureConfigurationDirectoryExists(configurationFolderPath);
12 |
13 | ZileanConfiguration.EnsureExists();
14 |
15 | configuration.SetBasePath(configurationFolderPath);
16 | configuration.AddLoggingConfiguration(configurationFolderPath);
17 | configuration.AddJsonFile(ConfigurationLiterals.SettingsConfigFilename, false, false);
18 | configuration.AddEnvironmentVariables();
19 |
20 | return configuration;
21 | }
22 |
23 | public static ZileanConfiguration GetZileanConfiguration(this IConfiguration configuration) =>
24 | configuration.GetSection(ConfigurationLiterals.MainSettingsSectionName).Get();
25 |
26 | private static void EnsureConfigurationDirectoryExists(string configurationFolderPath)
27 | {
28 | if (!Directory.Exists(configurationFolderPath))
29 | {
30 | Directory.CreateDirectory(configurationFolderPath);
31 | }
32 | }
33 | }
34 |
--------------------------------------------------------------------------------
/src/Zilean.Shared/Features/Configuration/DatabaseConfiguration.cs:
--------------------------------------------------------------------------------
1 | namespace Zilean.Shared.Features.Configuration;
2 |
3 | public class DatabaseConfiguration
4 | {
5 | public string ConnectionString { get; set; }
6 |
7 | public DatabaseConfiguration()
8 | {
9 | var password = Environment.GetEnvironmentVariable("POSTGRES_PASSWORD");
10 | if (string.IsNullOrWhiteSpace(password))
11 | {
12 | throw new InvalidOperationException("Environment variable POSTGRES_PASSWORD is not set.");
13 | }
14 |
15 | ConnectionString = $"Host=postgres;Database=zilean;Username=postgres;Password={password};Include Error Detail=true;Timeout=30;CommandTimeout=3600;";
16 | }
17 | }
18 |
--------------------------------------------------------------------------------
/src/Zilean.Shared/Features/Configuration/DmmConfiguration.cs:
--------------------------------------------------------------------------------
1 | namespace Zilean.Shared.Features.Configuration;
2 |
3 | public class DmmConfiguration
4 | {
5 | public bool EnableScraping { get; set; } = true;
6 | public bool EnableEndpoint { get; set; } = true;
7 | public string ScrapeSchedule { get; set; } = "0 * * * *";
8 | public int MinimumReDownloadIntervalMinutes { get; set; } = 30;
9 | public int MaxFilteredResults { get; set; } = 200;
10 | public double MinimumScoreMatch { get; set; } = 0.85;
11 | }
12 |
--------------------------------------------------------------------------------
/src/Zilean.Shared/Features/Configuration/ImdbConfiguration.cs:
--------------------------------------------------------------------------------
1 | namespace Zilean.Shared.Features.Configuration;
2 |
3 | public class ImdbConfiguration
4 | {
5 | public bool EnableImportMatching { get; set; } = true;
6 | public bool EnableEndpoint { get; set; } = true;
7 | public double MinimumScoreMatch { get; set; } = 0.85;
8 |
9 | public bool UseAllCores { get; set; } = false;
10 |
11 | public int NumberOfCores { get; set; } = 2;
12 |
13 | public bool UseLucene { get; set; } = false;
14 | }
15 |
--------------------------------------------------------------------------------
/src/Zilean.Shared/Features/Configuration/IngestionConfiguration.cs:
--------------------------------------------------------------------------------
1 | namespace Zilean.Shared.Features.Configuration;
2 |
3 | public class IngestionConfiguration
4 | {
5 | public List ZurgInstances { get; set; } = [];
6 | public List ZileanInstances { get; set; } = [];
7 | public List GenericInstances { get; set; } = [];
8 | public bool EnableScraping { get; set; } = false;
9 | public KubernetesConfiguration Kubernetes { get; set; } = new();
10 | public string ScrapeSchedule { get; set; } = "0 * * * *";
11 | public string ZurgEndpointSuffix { get; set; } = "/debug/torrents";
12 | public string ZileanEndpointSuffix { get; set; } = "/torrents/all";
13 | public int RequestTimeout { get; set; } = 10000;
14 | }
15 |
--------------------------------------------------------------------------------
/src/Zilean.Shared/Features/Configuration/KubernetesAuthenticationType.cs:
--------------------------------------------------------------------------------
1 | namespace Zilean.Shared.Features.Configuration;
2 |
3 | public enum KubernetesAuthenticationType
4 | {
5 | ConfigFile = 0,
6 | RoleBased = 1
7 | }
8 |
--------------------------------------------------------------------------------
/src/Zilean.Shared/Features/Configuration/KubernetesConfiguration.cs:
--------------------------------------------------------------------------------
1 | namespace Zilean.Shared.Features.Configuration;
2 |
3 | public class KubernetesConfiguration
4 | {
5 | public bool EnableServiceDiscovery { get; set; } = false;
6 | public List KubernetesSelectors { get; set; } = [];
7 | public string KubeConfigFile { get; set; } = "/$HOME/.kube/config";
8 |
9 | public KubernetesAuthenticationType AuthenticationType { get; set; } = KubernetesAuthenticationType.ConfigFile;
10 | }
11 |
--------------------------------------------------------------------------------
/src/Zilean.Shared/Features/Configuration/KubernetesSelector.cs:
--------------------------------------------------------------------------------
1 | namespace Zilean.Shared.Features.Configuration;
2 |
3 | public class KubernetesSelector
4 | {
5 | public string UrlTemplate { get; set; } = "http://zurg.{0}:9999/debug/torrents";
6 | public string LabelSelector { get; set; } = "app.elfhosted.com/name=zurg";
7 | public GenericEndpointType EndpointType { get; set; } = GenericEndpointType.Zurg;
8 | }
9 |
--------------------------------------------------------------------------------
/src/Zilean.Shared/Features/Configuration/Literals.cs:
--------------------------------------------------------------------------------
1 | namespace Zilean.Shared.Features.Configuration;
2 |
3 | public static class ConfigurationLiterals
4 | {
5 | public const string ConfigurationFolder = "data";
6 | public const string SettingsConfigFilename = "settings.json";
7 | public const string LoggingConfigFilename = "logging.json";
8 | public const string MainSettingsSectionName = "Zilean";
9 | }
10 |
--------------------------------------------------------------------------------
/src/Zilean.Shared/Features/Configuration/LoggingConfiguration.cs:
--------------------------------------------------------------------------------
1 | using Microsoft.Extensions.Configuration;
2 |
3 | namespace Zilean.Shared.Features.Configuration;
4 |
5 | public static class LoggingConfiguration
6 | {
7 | private const string DefaultLoggingContents =
8 | """
9 | {
10 | "Serilog": {
11 | "MinimumLevel": {
12 | "Default": "Information",
13 | "Override": {
14 | "Microsoft": "Warning",
15 | "System": "Warning",
16 | "System.Net.Http.HttpClient.Scraper.LogicalHandler": "Warning",
17 | "System.Net.Http.HttpClient.Scraper.ClientHandler": "Warning",
18 | "Microsoft.AspNetCore.Hosting.Diagnostics": "Error",
19 | "Microsoft.AspNetCore.DataProtection": "Error",
20 | }
21 | }
22 | }
23 | }
24 | """;
25 |
26 | public static IConfigurationBuilder AddLoggingConfiguration(this IConfigurationBuilder configuration, string configurationFolderPath)
27 | {
28 | EnsureExists(configurationFolderPath);
29 |
30 | configuration.AddJsonFile(ConfigurationLiterals.LoggingConfigFilename, false, false);
31 |
32 | return configuration;
33 | }
34 |
35 | private static void EnsureExists(string configurationFolderPath)
36 | {
37 | var loggingPath = Path.Combine(configurationFolderPath, ConfigurationLiterals.LoggingConfigFilename);
38 | File.WriteAllText(loggingPath, DefaultLoggingContents);
39 | }
40 | }
41 |
--------------------------------------------------------------------------------
/src/Zilean.Shared/Features/Configuration/ParsingConfiguration.cs:
--------------------------------------------------------------------------------
1 | namespace Zilean.Shared.Features.Configuration;
2 |
3 | public class ParsingConfiguration
4 | {
5 | public int BatchSize { get; set; } = 5000;
6 | }
7 |
--------------------------------------------------------------------------------
/src/Zilean.Shared/Features/Configuration/ServiceCollectionExtensions.cs:
--------------------------------------------------------------------------------
1 | namespace Zilean.Shared.Features.Configuration;
2 |
3 | public static class ServiceCollectionExtensions
4 | {
5 | public static IServiceCollection AddConfiguration(this IServiceCollection services, ZileanConfiguration configuration)
6 | {
7 | services.AddSingleton(configuration);
8 |
9 | return services;
10 | }
11 | }
12 |
--------------------------------------------------------------------------------
/src/Zilean.Shared/Features/Configuration/TorrentsConfiguration.cs:
--------------------------------------------------------------------------------
1 | namespace Zilean.Shared.Features.Configuration;
2 |
3 | public class TorrentsConfiguration
4 | {
5 | public bool EnableEndpoint { get; set; } = false;
6 | public int MaxHashesToCheck { get; set; } = 100;
7 | public bool EnableScrapeEndpoint { get; set; } = false;
8 | public bool EnableCacheCheckEndpoint { get; set; } = false;
9 | }
10 |
--------------------------------------------------------------------------------
/src/Zilean.Shared/Features/Configuration/TorznabConfiguration.cs:
--------------------------------------------------------------------------------
1 | namespace Zilean.Shared.Features.Configuration;
2 |
3 | public class TorznabConfiguration
4 | {
5 | public bool EnableEndpoint { get; set; } = true;
6 | }
7 |
--------------------------------------------------------------------------------
/src/Zilean.Shared/Features/Configuration/ZileanConfiguration.cs:
--------------------------------------------------------------------------------
1 | namespace Zilean.Shared.Features.Configuration;
2 |
3 | public class ZileanConfiguration
4 | {
5 | private static readonly JsonSerializerOptions? _jsonSerializerOptions = new()
6 | {
7 | WriteIndented = true,
8 | PropertyNamingPolicy = null,
9 | };
10 |
11 | public string? ApiKey { get; set; } = Utilities.ApiKey.Generate();
12 | public bool FirstRun { get; set; } = true;
13 | public bool EnableDashboard { get; set; } = false;
14 | public DmmConfiguration Dmm { get; set; } = new();
15 | public TorznabConfiguration Torznab { get; set; } = new();
16 | public DatabaseConfiguration Database { get; set; } = new();
17 | public TorrentsConfiguration Torrents { get; set; } = new();
18 | public ImdbConfiguration Imdb { get; set; } = new();
19 | public IngestionConfiguration Ingestion { get; set; } = new();
20 | public ParsingConfiguration Parsing { get; set; } = new();
21 |
22 | public static void EnsureExists()
23 | {
24 | var settingsFilePath = Path.Combine(AppContext.BaseDirectory, ConfigurationLiterals.ConfigurationFolder, ConfigurationLiterals.SettingsConfigFilename);
25 | if (!File.Exists(settingsFilePath))
26 | {
27 | File.WriteAllText(settingsFilePath, DefaultConfigurationContents());
28 | }
29 | }
30 |
31 | private static string DefaultConfigurationContents()
32 | {
33 | var mainSettings = new Dictionary
34 | {
35 | [ConfigurationLiterals.MainSettingsSectionName] = new ZileanConfiguration(),
36 | };
37 |
38 | return JsonSerializer.Serialize(mainSettings, _jsonSerializerOptions);
39 | }
40 | }
41 |
--------------------------------------------------------------------------------
/src/Zilean.Shared/Features/Dmm/DmmRecords.cs:
--------------------------------------------------------------------------------
1 | namespace Zilean.Shared.Features.Dmm;
2 |
3 | public class ExtractedDmmEntry(string? infoHash, string? filename, long filesize, TorrentInfo? parseResponse)
4 | {
5 | public string? Filename { get; set; } = filename;
6 | public string? InfoHash { get; set; } = infoHash;
7 | public long Filesize { get; set; } = filesize;
8 | public TorrentInfo? ParseResponse { get; set; } = parseResponse;
9 |
10 | public static ExtractedDmmEntry FromStreamedEntry(StreamedEntry streamedEntry) =>
11 | new(streamedEntry.InfoHash, streamedEntry.Name, streamedEntry.Size, null);
12 | }
13 |
14 | public class ExtractedDmmEntryResponse(TorrentInfo torrentInfo)
15 | {
16 | public string? Filename { get; set; } = torrentInfo.RawTitle;
17 | public string? InfoHash { get; set; } = torrentInfo.InfoHash;
18 | public string Filesize { get; set; } = torrentInfo.Size;
19 | public TorrentInfo ParseResponse { get; set; } = torrentInfo;
20 | }
21 |
22 | public record DmmQueryRequest(string QueryText);
23 |
--------------------------------------------------------------------------------
/src/Zilean.Shared/Features/Dmm/ParsedPages.cs:
--------------------------------------------------------------------------------
1 | namespace Zilean.Shared.Features.Dmm;
2 |
3 | public class ParsedPages
4 | {
5 | [Key]
6 | public string Page { get; set; } = default!;
7 | public int EntryCount { get; set; }
8 | }
9 |
--------------------------------------------------------------------------------
/src/Zilean.Shared/Features/Dmm/TorrentInfoExtensions.cs:
--------------------------------------------------------------------------------
1 | namespace Zilean.Shared.Features.Dmm;
2 |
3 | public static class TorrentInfoExtensions
4 | {
5 | public static string CacheKey(this TorrentInfo torrentInfo) =>
6 | $"{torrentInfo.ParsedTitle}-{torrentInfo.Category}-{torrentInfo.Year}";
7 | }
8 |
--------------------------------------------------------------------------------
/src/Zilean.Shared/Features/Expressions/ExpressionStringBuilder.cs:
--------------------------------------------------------------------------------
1 | namespace Zilean.Shared.Features.Expressions;
2 |
3 | public class ExpressionStringBuilder : ExpressionVisitor
4 | {
5 | private readonly StringBuilder _sb = new();
6 |
7 | public override Expression Visit(Expression? node)
8 | {
9 | if (node == null)
10 | {
11 | return null;
12 | }
13 |
14 | switch (node.NodeType)
15 | {
16 | case ExpressionType.Lambda:
17 | var lambda = (LambdaExpression)node;
18 | Visit(lambda.Body);
19 | break;
20 | case ExpressionType.MemberAccess:
21 | var member = (MemberExpression)node;
22 | Visit(member.Expression);
23 | _sb.Append($".{member.Member.Name}");
24 | break;
25 | case ExpressionType.Constant:
26 | var constant = (ConstantExpression)node;
27 | _sb.Append(constant.Value);
28 | break;
29 | case ExpressionType.Equal:
30 | var binaryEqual = (BinaryExpression)node;
31 | Visit(binaryEqual.Left);
32 | _sb.Append(" == ");
33 | Visit(binaryEqual.Right);
34 | break;
35 | case ExpressionType.NotEqual:
36 | var binaryNotEqual = (BinaryExpression)node;
37 | Visit(binaryNotEqual.Left);
38 | _sb.Append(" != ");
39 | Visit(binaryNotEqual.Right);
40 | break;
41 | case ExpressionType.AndAlso:
42 | var binaryAnd = (BinaryExpression)node;
43 | Visit(binaryAnd.Left);
44 | _sb.Append(" && ");
45 | Visit(binaryAnd.Right);
46 | break;
47 | case ExpressionType.OrElse:
48 | var binaryOr = (BinaryExpression)node;
49 | Visit(binaryOr.Left);
50 | _sb.Append(" || ");
51 | Visit(binaryOr.Right);
52 | break;
53 | case ExpressionType.Call:
54 | var methodCall = (MethodCallExpression)node;
55 | Visit(methodCall.Object);
56 | _sb.Append($".{methodCall.Method.Name}(");
57 | for (int i = 0; i < methodCall.Arguments.Count; i++)
58 | {
59 | if (i > 0)
60 | {
61 | _sb.Append(", ");
62 | }
63 |
64 | Visit(methodCall.Arguments[i]);
65 | }
66 |
67 | _sb.Append(")");
68 | break;
69 | default:
70 | _sb.Append(node);
71 | break;
72 | }
73 |
74 | return node;
75 | }
76 |
77 | public override string ToString() => _sb.ToString();
78 | }
79 |
--------------------------------------------------------------------------------
/src/Zilean.Shared/Features/Imdb/ImdbFile.cs:
--------------------------------------------------------------------------------
1 | namespace Zilean.Shared.Features.Imdb;
2 |
3 | public class ImdbFile
4 | {
5 | [Key]
6 | public string ImdbId { get; set; } = default!;
7 | public string? Category { get; set; }
8 | public string? Title { get; set; }
9 | public bool Adult { get; set; }
10 | public int Year { get; set; }
11 | }
12 |
--------------------------------------------------------------------------------
/src/Zilean.Shared/Features/Python/ParseTorrentTitleResponse.cs:
--------------------------------------------------------------------------------
1 | namespace Zilean.Shared.Features.Python;
2 |
3 | public record ParseTorrentTitleResponse(bool Success, TorrentInfo? Response);
4 |
--------------------------------------------------------------------------------
/src/Zilean.Shared/Features/Python/PyObjectExtensions.cs:
--------------------------------------------------------------------------------
1 | namespace Zilean.Shared.Features.Python;
2 |
3 | public static class PyObjectExtensions
4 | {
5 | public static bool HasKey(this PyObject dict, string key) =>
6 | dict.InvokeMethod("__contains__", new PyString(key)).As();
7 | }
8 |
--------------------------------------------------------------------------------
/src/Zilean.Shared/Features/Scraping/GenericEndpoint.cs:
--------------------------------------------------------------------------------
1 | namespace Zilean.Shared.Features.Scraping;
2 |
3 | public class GenericEndpoint
4 | {
5 | public string? Url { get; set; }
6 | public GenericEndpointType? EndpointType { get; set; }
7 | public string? ApiKey { get; set; }
8 | public string? Authorization { get; set; }
9 | public string? EndpointSuffix { get; set; }
10 | }
11 |
--------------------------------------------------------------------------------
/src/Zilean.Shared/Features/Scraping/GenericEndpointType.cs:
--------------------------------------------------------------------------------
1 | namespace Zilean.Shared.Features.Scraping;
2 |
3 | public enum GenericEndpointType
4 | {
5 | Zilean = 0,
6 | Zurg = 1,
7 | Generic = 2
8 | }
9 |
--------------------------------------------------------------------------------
/src/Zilean.Shared/Features/Scraping/StreamedEntry.cs:
--------------------------------------------------------------------------------
1 | namespace Zilean.Shared.Features.Scraping;
2 |
3 | public class StreamedEntry
4 | {
5 | [JsonPropertyName("name")]
6 | public required string Name { get; set; }
7 |
8 | [JsonPropertyName("size")]
9 | public required long Size { get; set; }
10 |
11 | [JsonPropertyName("hash")]
12 | public required string InfoHash { get; set; }
13 |
14 | public TorrentInfo? ParseResponse { get; set; }
15 | }
16 |
--------------------------------------------------------------------------------
/src/Zilean.Shared/Features/Shell/ArgumentsBuilder.cs:
--------------------------------------------------------------------------------
1 | namespace Zilean.Shared.Features.Shell;
2 |
3 | public class ArgumentsBuilder
4 | {
5 | private readonly Dictionary> _arguments = [];
6 |
7 | public static ArgumentsBuilder Create() => new();
8 |
9 | public ArgumentsBuilder Clear()
10 | {
11 | _arguments.Clear();
12 |
13 | return this;
14 | }
15 |
16 |
17 | public ArgumentsBuilder AppendArgument(string argument, string newValue, bool allowDuplicates = false, bool quoteValue = true)
18 | {
19 | if (!_arguments.TryGetValue(argument, out var value))
20 | {
21 | value = quoteValue ? [$"\"{newValue}\""] : [newValue];
22 | _arguments[argument] = value;
23 |
24 | return this;
25 | }
26 |
27 | if (allowDuplicates)
28 | {
29 | value.Add(quoteValue ? $"\"{newValue}\"" : newValue);
30 | }
31 |
32 | return this;
33 | }
34 |
35 | public string RenderArguments(char propertyKeySeparator = ' ')
36 | {
37 | var renderedArguments = new List();
38 |
39 | foreach (var arg in _arguments)
40 | {
41 | foreach (var value in arg.Value)
42 | {
43 | if (value == string.Empty)
44 | {
45 | renderedArguments.Add(arg.Key);
46 | continue;
47 | }
48 |
49 | if (arg.Key.StartsWith("-p"))
50 | {
51 | renderedArguments.Add($"{arg.Key}={value}");
52 | continue;
53 | }
54 |
55 | renderedArguments.Add($"{arg.Key}{propertyKeySeparator}{value}");
56 | }
57 | }
58 |
59 | return string.Join(" ", renderedArguments);
60 | }
61 |
62 | }
63 |
--------------------------------------------------------------------------------
/src/Zilean.Shared/Features/Shell/ServiceCollectionExtensions.cs:
--------------------------------------------------------------------------------
1 | namespace Zilean.Shared.Features.Shell;
2 |
3 | public static class ServiceCollectionExtensions
4 | {
5 | public static IServiceCollection AddShellExecutionService(this IServiceCollection services)
6 | {
7 | services.AddSingleton();
8 | return services;
9 | }
10 | }
11 |
--------------------------------------------------------------------------------
/src/Zilean.Shared/Features/Shell/ShellCommandOptions.cs:
--------------------------------------------------------------------------------
1 | namespace Zilean.Shared.Features.Shell;
2 |
3 | [ExcludeFromCodeCoverage]
4 | public sealed class ShellCommandOptions
5 | {
6 | public string? Command { get; set; }
7 | public ArgumentsBuilder? ArgumentsBuilder { get; set; } = new();
8 | public bool NonInteractive { get; set; }
9 | public bool ShowOutput { get; set; } = false;
10 | public string? WorkingDirectory { get; set; }
11 | public char PropertyKeySeparator { get; set; } = ' ';
12 | public string? PreCommandMessage { get; set; }
13 | public string? SuccessCommandMessage { get; set; }
14 | public string? FailureCommandMessage { get; set; }
15 | public Dictionary EnvironmentVariables { get; set; } = [];
16 | public CancellationToken CancellationToken { get; set; }
17 | }
18 |
--------------------------------------------------------------------------------
/src/Zilean.Shared/Features/Shell/ShellExecutionService.cs:
--------------------------------------------------------------------------------
1 | namespace Zilean.Shared.Features.Shell;
2 |
3 | public interface IShellExecutionService
4 | {
5 | Task ExecuteCommand(ShellCommandOptions options);
6 | }
7 |
8 | public class ShellExecutionService(ILogger logger) : IShellExecutionService
9 | {
10 | public async Task ExecuteCommand(ShellCommandOptions options)
11 | {
12 | try
13 | {
14 | var arguments = options.ArgumentsBuilder.RenderArguments(propertyKeySeparator: options.PropertyKeySeparator);
15 |
16 | if (options.ShowOutput)
17 | {
18 | logger.LogInformation(string.IsNullOrEmpty(options.PreCommandMessage)
19 | ? $"Executing: {options.Command} {arguments}"
20 | : options.PreCommandMessage);
21 | }
22 |
23 | var executionDirectory = string.IsNullOrEmpty(options.WorkingDirectory)
24 | ? Directory.GetCurrentDirectory()
25 | : options.WorkingDirectory;
26 |
27 | await using var stdOut = Console.OpenStandardOutput();
28 | await using var stdErr = Console.OpenStandardError();
29 |
30 | await Cli.Wrap(options.Command)
31 | .WithWorkingDirectory(executionDirectory)
32 | .WithArguments(arguments)
33 | .WithEnvironmentVariables(options.EnvironmentVariables)
34 | .WithValidation(CommandResultValidation.None)
35 | .WithStandardOutputPipe(PipeTarget.ToStream(stdOut))
36 | .WithStandardErrorPipe(PipeTarget.ToStream(stdErr))
37 | .ExecuteAsync(options.CancellationToken);
38 | }
39 | catch (TaskCanceledException)
40 | {
41 | logger.LogInformation("Command execution was cancelled");
42 | }
43 | catch (OperationCanceledException)
44 | {
45 | logger.LogInformation("Command execution was cancelled");
46 | }
47 | catch (Exception ex)
48 | {
49 | logger.LogError(ex, "Error executing command");
50 | }
51 | }
52 | }
53 |
--------------------------------------------------------------------------------
/src/Zilean.Shared/Features/Statistics/BaseLastImport.cs:
--------------------------------------------------------------------------------
1 | namespace Zilean.Shared.Features.Statistics;
2 |
3 | public abstract class BaseLastImport
4 | {
5 | public DateTime OccuredAt { get; set; } = DateTime.UtcNow;
6 | public ImportStatus Status { get; set; } = ImportStatus.InProgress;
7 | }
8 |
--------------------------------------------------------------------------------
/src/Zilean.Shared/Features/Statistics/DmmLastImport.cs:
--------------------------------------------------------------------------------
1 | namespace Zilean.Shared.Features.Statistics;
2 |
3 | public class DmmLastImport : BaseLastImport
4 | {
5 | public long PageCount { get; set; }
6 | public long EntryCount { get; set; }
7 | }
8 |
--------------------------------------------------------------------------------
/src/Zilean.Shared/Features/Statistics/ImdbLastImport.cs:
--------------------------------------------------------------------------------
1 | namespace Zilean.Shared.Features.Statistics;
2 |
3 | public class ImdbLastImport : BaseLastImport
4 | {
5 | public long EntryCount { get; set; }
6 | }
7 |
--------------------------------------------------------------------------------
/src/Zilean.Shared/Features/Statistics/ImportMetadata.cs:
--------------------------------------------------------------------------------
1 | namespace Zilean.Shared.Features.Statistics;
2 |
3 | public class ImportMetadata
4 | {
5 | [Key]
6 | public string Key { get; set; } = default!;
7 |
8 | public JsonDocument Value { get; set; } = JsonDocument.Parse("{}");
9 | }
10 |
--------------------------------------------------------------------------------
/src/Zilean.Shared/Features/Statistics/ImportStatus.cs:
--------------------------------------------------------------------------------
1 | namespace Zilean.Shared.Features.Statistics;
2 |
3 | public enum ImportStatus
4 | {
5 | InProgress,
6 | Complete,
7 | Failed
8 | }
9 |
--------------------------------------------------------------------------------
/src/Zilean.Shared/Features/Statistics/MetadataKeys.cs:
--------------------------------------------------------------------------------
1 | namespace Zilean.Shared.Features.Statistics;
2 |
3 | public static class MetadataKeys
4 | {
5 | public const string ImdbLastImport = "ImdbLastImport";
6 | public const string DmmLastImport = "DmmLastImport";
7 | }
8 |
--------------------------------------------------------------------------------
/src/Zilean.Shared/Features/Torznab/Categories/TorznabCategory.cs:
--------------------------------------------------------------------------------
1 | namespace Zilean.Shared.Features.Torznab.Categories;
2 |
3 | public class TorznabCategory(int id, string name)
4 | {
5 | public int Id { get; } = id;
6 | public string Name { get; set; } = name;
7 | public List SubCategories { get; private set; } = [];
8 |
9 | public bool Contains(TorznabCategory cat) =>
10 | Equals(this, cat) || SubCategories.Contains(cat);
11 |
12 | public JsonObject ToJson() =>
13 | new()
14 | {
15 | ["ID"] = Id,
16 | ["Name"] = Name
17 | };
18 |
19 | public override bool Equals(object? obj) => (obj as TorznabCategory)?.Id == Id;
20 |
21 | public override int GetHashCode() => Id;
22 | public TorznabCategory CopyWithoutSubCategories() => new(Id, Name);
23 | }
24 |
--------------------------------------------------------------------------------
/src/Zilean.Shared/Features/Torznab/Categories/TorznabCategoryExtensions.cs:
--------------------------------------------------------------------------------
1 | namespace Zilean.Shared.Features.Torznab.Categories;
2 |
3 | public static class TorznabCategoryExtensions
4 | {
5 | public static List GetTorznabCategoryTree(this List categories)
6 | {
7 | var sortedTree = categories
8 | .Select(c =>
9 | {
10 | var sortedSubCats = c.SubCategories.OrderBy(x => x.Id);
11 | var newCat = new TorznabCategory(c.Id, c.Name);
12 | newCat.SubCategories.AddRange(sortedSubCats);
13 | return newCat;
14 | }).OrderBy(x => x.Id >= 100000 ? "zzz" + x.Name : x.Id.ToString()).ToList();
15 |
16 | return sortedTree;
17 | }
18 | }
19 |
--------------------------------------------------------------------------------
/src/Zilean.Shared/Features/Torznab/Info/ChannelInfo.cs:
--------------------------------------------------------------------------------
1 | namespace Zilean.Shared.Features.Torznab.Info;
2 |
3 | public class ChannelInfo
4 | {
5 | public const string GitHubRepo = "https://github.com/iPromKnight/zilean";
6 | public const string Title = "Zilean Indexer";
7 | public const string Description = "DMM Cached RD Indexer";
8 | public const string Language = "en-US";
9 | public const string Category = "search";
10 | public static Uri Link => new(GitHubRepo);
11 | public static ChannelInfo ZileanIndexer => new();
12 | }
13 |
--------------------------------------------------------------------------------
/src/Zilean.Shared/Features/Torznab/Info/ReleaseInfo.cs:
--------------------------------------------------------------------------------
1 | namespace Zilean.Shared.Features.Torznab.Info;
2 |
3 | public class ReleaseInfo() : ICloneable
4 | {
5 | public const long Seeders = 999;
6 | public const long Peers = 999;
7 | public const string Origin = "Zilean";
8 | public string? Title { get; set; }
9 | public Guid? Guid { get; set; }
10 | public Uri? Magnet { get; set; }
11 | public Uri? Details { get; set; }
12 | public DateTime PublishDate { get; set; }
13 | public ICollection Category { get; set; } = [];
14 | public long? Size { get; set; }
15 | public string? Description { get; set; }
16 | public long? Imdb { get; set; }
17 | public ICollection Languages { get; set; } = [];
18 | public long? Year { get; set; }
19 | public string? InfoHash { get; set; }
20 | public static double? GigabytesFromBytes(double? size) => size / 1024.0 / 1024.0 / 1024.0;
21 |
22 | private ReleaseInfo(ReleaseInfo copyFrom) : this()
23 | {
24 | Title = copyFrom.Title;
25 | Guid = copyFrom.Guid;
26 | Magnet = copyFrom.Magnet;
27 | Details = copyFrom.Details;
28 | PublishDate = copyFrom.PublishDate;
29 | Category = copyFrom.Category;
30 | Size = copyFrom.Size;
31 | Description = copyFrom.Description;
32 | Imdb = copyFrom.Imdb;
33 | Languages = copyFrom.Languages;
34 | Year = copyFrom.Year;
35 | InfoHash = copyFrom.InfoHash;
36 | }
37 |
38 | public virtual object Clone() => new ReleaseInfo(this);
39 |
40 | public override string ToString() =>
41 | $"[ReleaseInfo: Title={Title}, Guid={Guid}, Link={Magnet}, Details={Details}, PublishDate={PublishDate}, Category={Category}, Size={Size}, Description={Description}, Imdb={Imdb}, Seeders={Seeders}, Peers={Peers}, InfoHash={InfoHash}]";
42 | }
43 |
--------------------------------------------------------------------------------
/src/Zilean.Shared/Features/Torznab/Parameters/MovieSearch.cs:
--------------------------------------------------------------------------------
1 | namespace Zilean.Shared.Features.Torznab.Parameters;
2 |
3 | public enum MovieSearch
4 | {
5 | Q,
6 | ImdbId,
7 | Year,
8 | }
9 |
--------------------------------------------------------------------------------
/src/Zilean.Shared/Features/Torznab/Parameters/TvSearch.cs:
--------------------------------------------------------------------------------
1 | namespace Zilean.Shared.Features.Torznab.Parameters;
2 |
3 | public enum TvSearch
4 | {
5 | Q,
6 | Season,
7 | Ep,
8 | ImdbId,
9 | Year,
10 | }
11 |
--------------------------------------------------------------------------------
/src/Zilean.Shared/Features/Torznab/Parameters/XxxSearch.cs:
--------------------------------------------------------------------------------
1 | namespace Zilean.Shared.Features.Torznab.Parameters;
2 |
3 | public enum XxxSearch
4 | {
5 | Q,
6 | ImdbId,
7 | Year,
8 | }
9 |
--------------------------------------------------------------------------------
/src/Zilean.Shared/Features/Torznab/TorznabErrorResponse.cs:
--------------------------------------------------------------------------------
1 | namespace Zilean.Shared.Features.Torznab;
2 |
3 | public static class TorznabErrorResponse
4 | {
5 | public static string Create(int code, string description)
6 | {
7 | var xdoc = new XDocument(
8 | new XDeclaration("1.0", "UTF-8", null),
9 | new XElement("error",
10 | new XAttribute("code", code.ToString()),
11 | new XAttribute("description", description)
12 | )
13 | );
14 |
15 | return xdoc.Declaration + Environment.NewLine + xdoc;
16 | }
17 | }
18 |
--------------------------------------------------------------------------------
/src/Zilean.Shared/Features/Utilities/ApiKey.cs:
--------------------------------------------------------------------------------
1 | namespace Zilean.Shared.Features.Utilities;
2 |
3 | public static class ApiKey
4 | {
5 | public static string Generate() => $"{Guid.NewGuid():N}{Guid.NewGuid():N}";
6 | }
7 |
--------------------------------------------------------------------------------
/src/Zilean.Shared/GlobalUsings.cs:
--------------------------------------------------------------------------------
1 | global using System.Collections.Concurrent;
2 | global using System.ComponentModel.DataAnnotations;
3 | global using System.Diagnostics.CodeAnalysis;
4 | global using System.Globalization;
5 | global using System.Linq.Expressions;
6 | global using System.Runtime.CompilerServices;
7 | global using System.Runtime.InteropServices;
8 | global using System.Text;
9 | global using System.Text.Json;
10 | global using System.Text.Json.Nodes;
11 | global using System.Text.Json.Serialization;
12 | global using System.Text.RegularExpressions;
13 | global using System.Xml.Linq;
14 | global using CliWrap;
15 | global using Microsoft.AspNetCore.WebUtilities;
16 | global using Microsoft.Extensions.DependencyInjection;
17 | global using Microsoft.Extensions.Logging;
18 | global using Python.Runtime;
19 | global using Zilean.Shared.Extensions;
20 | global using Zilean.Shared.Features.Configuration;
21 | global using Zilean.Shared.Features.Dmm;
22 | global using Zilean.Shared.Features.Imdb;
23 | global using Zilean.Shared.Features.Scraping;
24 | global using Zilean.Shared.Features.Torznab.Categories;
25 | global using Zilean.Shared.Features.Torznab.Info;
26 | global using Zilean.Shared.Features.Torznab.Parameters;
27 | global using Zilean.Shared.Features.Utilities;
28 |
--------------------------------------------------------------------------------
/src/Zilean.Shared/Zilean.Shared.csproj:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 | Library
5 | enable
6 | enable
7 |
8 |
9 |
10 |
12 |
13 |
14 |
21 |
22 |
--------------------------------------------------------------------------------
/tests/Zilean.Tests/Collections/ElasticTestCollectionDefinition.cs:
--------------------------------------------------------------------------------
1 | namespace Zilean.Tests.Collections;
2 |
3 | [CollectionDefinition(nameof(ElasticTestCollectionDefinition))]
4 | public class ElasticTestCollectionDefinition : ICollectionFixture;
5 |
--------------------------------------------------------------------------------
/tests/Zilean.Tests/Fixtures/PostgresLifecycleFixture.cs:
--------------------------------------------------------------------------------
1 | namespace Zilean.Tests.Fixtures;
2 |
3 | public class PostgresLifecycleFixture : IAsyncLifetime
4 | {
5 | private PostgreSqlContainer PostgresContainer { get; } = new PostgreSqlBuilder()
6 | .WithImage("postgres:16.3-alpine3.20")
7 | .WithPortBinding(5432, 5432)
8 | .WithEnvironment("POSTGRES_USER", "postgres")
9 | .WithEnvironment("POSTGRES_PASSWORD", "postgres")
10 | .WithEnvironment("POSTGRES_DB", "zilean")
11 | .Build();
12 |
13 | public ZileanConfiguration ZileanConfiguration { get; } = new();
14 |
15 | public PostgresLifecycleFixture() =>
16 | DerivePathInfo(
17 | (_, projectDirectory, type, method) => new(
18 | directory: Path.Combine(projectDirectory, "Verification"),
19 | typeName: type.Name,
20 | methodName: method.Name));
21 |
22 | public async Task InitializeAsync()
23 | {
24 | await PostgresContainer.StartAsync();
25 | ZileanConfiguration.Database.ConnectionString = PostgresContainer.GetConnectionString();
26 | }
27 |
28 | public Task DisposeAsync() => PostgresContainer.DisposeAsync().AsTask();
29 | }
30 |
--------------------------------------------------------------------------------
/tests/Zilean.Tests/GlobalUsings.cs:
--------------------------------------------------------------------------------
1 | global using System.Diagnostics;
2 | global using FluentAssertions;
3 | global using Microsoft.Extensions.Configuration;
4 | global using Microsoft.Extensions.Logging;
5 | global using NSubstitute;
6 | global using Testcontainers.PostgreSql;
7 | global using Xunit.Abstractions;
8 | global using Zilean.Scraper.Features.Imdb;
9 | global using Zilean.Shared.Features.Configuration;
10 | global using Zilean.Shared.Features.Dmm;
11 | global using Zilean.Shared.Features.Imdb;
12 | global using Zilean.Shared.Features.Scraping;
13 | global using Zilean.Tests.Fixtures;
14 |
--------------------------------------------------------------------------------
/tests/Zilean.Tests/Zilean.Tests.csproj:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 | enable
5 | enable
6 | false
7 | true
8 | true
9 | Zilean.Tests
10 |
11 |
12 |
13 |
23 |
24 |
25 |
27 |
28 |
29 |
31 |
32 |
33 |
--------------------------------------------------------------------------------
4 |
5 | Zilean is a service that allows you to search for [DebridMediaManager](https://github.com/debridmediamanager/debrid-media-manager) sourced content shared by users.
6 | This can then be configured as a Torznab indexer in your favorite content application.
7 | Newly added is the ability for Zilean to scrape from your running Zurg instance, and from other running Zilean instances.
8 |
9 | Documentation for zilean can be viewed at [https://ipromknight.github.io/zilean/](https://ipromknight.github.io/zilean/)
10 |
11 | ---
12 |
13 |
14 | 
4 |
5 | ## What is %Product%?
6 |
7 | %Product% is a service that allows you to search for [DebridMediaManager](https://github.com/debridmediamanager/debrid-media-manager) sourced content shared by users.
8 | The DMM import reruns on missing pages in the configured time interval see [](Configuration.md).
9 |
10 | This can then be configured as a Torznab indexer in your favorite content application.
11 |
12 | Newly added is the ability for %Product% to scrape from your running Zurg instance, and from other running %Product% instances.
13 |
14 | ## Installation
15 |
16 | The easiest way to get up and running with %Product% is to use the provided docker-compose file.
17 |
18 | Ensure you have the following installed:
19 | - Docker, Docker Desktop or Podman
20 |
21 | First, generate your postgres password with
22 |
23 | ```bash
24 | echo "POSTGRES_PASSWORD=$(openssl rand -base64 42 | tr -dc A-Za-z0-9 | cut -c -32 | tr -d '\n')" > .env
25 | ```
26 |
27 | The example compose file below can be copied, and used to get the system running locally.
28 |
29 | ```yaml
30 | ```
31 | { src="compose-file.yaml" }
32 |
33 | This compose file will start the following services:
34 | - %Product%
35 | - Postgres (version 17)
36 |
37 | The configuration and persistent storage of both services will be stored in docker volumes, but i recommend changing this if you are not on windows to the `./data` directory, next to where the compose file resides.
38 |
39 | ## Pulling Latest Image
40 |
41 | If you would like to pull the latest image from the docker registry, you can use the following command:
42 |
43 | ```bash
44 | docker compose pull %product%
45 | ```
46 |
47 | > Please Note - Always make sure you check the [github release notes](https://github.com/iPromKnight/zilean/releases) for the latest release, to ensure there are no breaking changes.
48 | > The changelog can also be viewed [here](https://github.com/iPromKnight/zilean/blob/main/CHANGELOG.md).
49 | { style="note" }
50 |
--------------------------------------------------------------------------------
/docs/Writerside/topics/Scraper.md:
--------------------------------------------------------------------------------
1 | # Scraper
2 |
3 | The zilean scraper is the service responsible for ingestion of all content.
4 | This also includes metadata from imdb.
5 |
6 | While the scraper does run as a scheduled task, it can also be run on demand, and manually from within the container.
7 | The scraper exists at `/app/scraper`
8 |
9 | ## Running the Scraper
10 |
11 | > Note: When doing this - ensure you have stopped Your api instance, as the scraper and the api scheduler scraping
12 | > tasks will cause data corruption.
13 | {style="warning"}
14 |
15 | To run the scraper manually, you can use the following command:
16 |
17 | ```bash
18 | docker compose stop zilean
19 |
20 | docker run -it --rm \
21 | -v ./settings.json:/app/data/settings.json \
22 | --name zileanscraper \
23 | ipromknight/zilean:latest \
24 | /app/scraper 
8 |