├── .dockerignore ├── .gitignore ├── Dockerfile.service ├── Dockerfile.web ├── LICENSE ├── README.md ├── docker-compose.yml ├── images └── screenshot.png ├── schema.sql └── src ├── .editorconfig ├── BeatSaverMatcher.Api.Spotify ├── BeatSaverMatcher.Api.Spotify.csproj ├── SpotifyConfiguration.cs └── SpotifyRepository.cs ├── BeatSaverMatcher.Api.Tidal ├── BeatSaverMatcher.Api.Tidal.csproj ├── TidalClient.cs ├── TidalConfiguration.cs └── TidalModels.cs ├── BeatSaverMatcher.Api ├── APIException.cs ├── BeatSaverMatcher.Api.csproj ├── IMusicServiceApi.cs ├── Playlist.cs └── PlaylistSong.cs ├── BeatSaverMatcher.Common ├── BeatSaver │ ├── BeatSaverDeletedSong.cs │ ├── BeatSaverRepository.cs │ ├── BeatSaverSong.cs │ ├── BeatSaverSongDeletedResponse.cs │ ├── BeatSaverSongSearchResponse.cs │ └── BeatSaverUtils.cs ├── BeatSaverMatcher.Common.csproj ├── CacheKeys.cs ├── Db │ ├── BeatSaberSong.cs │ ├── BeatSaberSongRatings.cs │ ├── BeatSaberSongRepository.cs │ ├── BeatSaberSongWithRatings.cs │ ├── DbConfiguration.cs │ ├── IBeatSaberSongRepository.cs │ ├── SongDifficulties.cs │ └── SqlServerRepository.cs └── MetricsServer.cs ├── BeatSaverMatcher.Crawler ├── BeatSaverMatcher.Crawler.csproj ├── CrawlerHost.cs ├── DeleteCrawlerHost.cs ├── MetricsScrapeHost.cs ├── Program.cs ├── RatingsCrawlerHost.cs └── config │ └── logging.json ├── BeatSaverMatcher.Web ├── BeatSaverMatcher.Web.csproj ├── Controllers │ ├── MatchesController.cs │ └── ModSaberPlaylistController.cs ├── MatchCleanupWorker.cs ├── MatchingService.cs ├── Models │ ├── BeatSaberSongViewModel.cs │ ├── ModSaberPlaylist.cs │ ├── ModSaberSong.cs │ ├── SongMatch.cs │ └── SongMatchResult.cs ├── Program.cs ├── Properties │ └── launchSettings.json ├── Result │ ├── SongMatchState.cs │ ├── WorkItemStore.cs │ └── WorkResultItem.cs ├── SongMatchWorker.cs ├── Startup.cs ├── config │ ├── appSettings.example.json │ └── logging.json ├── package-lock.json ├── package.json ├── src │ ├── icon.png │ ├── icon_32.png │ ├── icon_48.png │ ├── index.html │ ├── index.ts │ └── style.css ├── tsconfig.json └── webpack.config.js └── BeatSaverMatcher.sln /.dockerignore: -------------------------------------------------------------------------------- 1 | **/.vs/ 2 | **/bin/ 3 | **/obj/ 4 | **/appSettings.json 5 | **/wwwroot/ 6 | **/node_modules/ -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .vs/ 2 | *.user 3 | bin/ 4 | obj/ 5 | appSettings.json 6 | wwwroot 7 | node_modules/ -------------------------------------------------------------------------------- /Dockerfile.service: -------------------------------------------------------------------------------- 1 | FROM mcr.microsoft.com/dotnet/sdk:8.0 AS build 2 | WORKDIR /app/src 3 | COPY src/ . 4 | RUN dotnet restore BeatSaverMatcher.Crawler/BeatSaverMatcher.Crawler.csproj 5 | RUN dotnet publish -c Release BeatSaverMatcher.Crawler/BeatSaverMatcher.Crawler.csproj -o /app/build 6 | 7 | FROM mcr.microsoft.com/dotnet/runtime:8.0 8 | WORKDIR /app 9 | COPY --from=build /app/build/ ./ 10 | ENTRYPOINT ["dotnet", "BeatSaverMatcher.Crawler.dll"] -------------------------------------------------------------------------------- /Dockerfile.web: -------------------------------------------------------------------------------- 1 | FROM mcr.microsoft.com/dotnet/sdk:8.0 AS build 2 | RUN curl -sL https://deb.nodesource.com/setup_16.x | bash - && apt install -y nodejs && rm -rf /var/lib/apt/lists/* 3 | WORKDIR /app/src 4 | COPY src/ . 5 | RUN dotnet restore BeatSaverMatcher.Web/BeatSaverMatcher.Web.csproj 6 | RUN dotnet publish -c Release BeatSaverMatcher.Web/BeatSaverMatcher.Web.csproj -o /app/build 7 | 8 | FROM mcr.microsoft.com/dotnet/aspnet:8.0 9 | WORKDIR /app 10 | COPY --from=build /app/build/ ./ 11 | ENTRYPOINT ["dotnet", "BeatSaverMatcher.Web.dll"] -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2020 patagona 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # BeatSaverMatcher 2 | Web app to match Beat Saber beatmaps to Spotify/Tidal playlists. 3 | 4 | Currently hosted at [beat-saver-matcher.uwu.industries](https://beat-saver-matcher.uwu.industries/) 5 | 6 | ## Features 7 | - find all Beatsaver beatmaps for each song in a Spotify/Tidal playlist 8 | - progress indicator 9 | - manual selection of beatmaps 10 | - direct link to BeatSaver to preview beatmap 11 | - show the difficulties and ratings for each beatmap 12 | - download multiple beatmaps as a `bplist` playlist 13 | 14 | ## Screenshot 15 | 16 | ![app screenhot](images/screenshot.png) 17 | 18 | ## Installation 19 | 20 | This isn't really supposed to be hosted by anyone themselves, but if you need to do that anyways, here's a rough sketch of what you'll have to do: 21 | 22 | - create a Spotify App under https://developer.spotify.com/dashboard and copy the Client ID and Client Secret to the compose file 23 | - create a Tidal App under https://developer.tidal.com/dashboard/ and copy the Client ID and Client Secret to the compose file 24 | - start the app, redis and db using docker-compose (`docker-compose up -d`) 25 | - connect to the database (for example, with Microsoft SQL Server Management Studio), create the database (called `BeatSaverMatcher`) and run the `schema.sql` script to create the required table(s) 26 | - `docker-compose restart` 27 | - check logs of the service if beatsaver.com scraping works correctly 28 | - check if web app is running and working correctly (port 8080 by default) -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: "3.7" 2 | services: 3 | web: 4 | image: matcher-web 5 | build: 6 | context: . 7 | dockerfile: Dockerfile.web 8 | ports: 9 | - "8080:80" 10 | environment: 11 | - "ASPNETCORE_HTTP_PORTS=80" 12 | - "Spotify__ClientId=" 13 | - "Spotify__ClientSecret=" 14 | - "Tidal__ClientId=" 15 | - "Tidal__ClientSecret=" 16 | - "ConnectionString=Server=db,1433;Database=BeatSaverMatcher;User Id=sa;Password=!1Start1!;" 17 | - "RedisConnection=redis" 18 | restart: unless-stopped 19 | 20 | service: 21 | image: matcher-service 22 | build: 23 | context: . 24 | dockerfile: Dockerfile.service 25 | environment: 26 | - "ConnectionString=Server=db,1433;Database=BeatSaverMatcher;User Id=sa;Password=!1Start1!;" 27 | restart: unless-stopped 28 | 29 | db: 30 | image: benjaminabt/mssql-fts:2019-ubuntu-1804 31 | ports: 32 | - "127.0.0.1:1433:1433" 33 | environment: 34 | - "ACCEPT_EULA=Y" 35 | - "SA_PASSWORD=!1Start1!" 36 | volumes: 37 | - "db:/var/opt/mssql" 38 | restart: unless-stopped 39 | 40 | redis: 41 | image: redis:6.0.5 42 | restart: unless-stopped 43 | 44 | volumes: 45 | db: 46 | -------------------------------------------------------------------------------- /images/screenshot.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/patagonaa/BeatSaverMatcher/ef3e094db43f9c080c18b2f9b50c3147730db796/images/screenshot.png -------------------------------------------------------------------------------- /schema.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE [dbo].[BeatSaberSong]( 2 | [BeatSaverKey] [int] NOT NULL, 3 | [Hash] [binary](20) NOT NULL, 4 | [Uploader] [nvarchar](4000) NOT NULL, 5 | [Uploaded] [datetime2] NOT NULL, 6 | [Difficulties] [int] NOT NULL, 7 | [Bpm] [float] NOT NULL, 8 | [LevelAuthorName] [nvarchar](4000) NOT NULL, 9 | [SongAuthorName] [nvarchar](4000) NOT NULL, 10 | [SongName] [nvarchar](4000) NOT NULL, 11 | [SongSubName] [nvarchar](4000) NOT NULL, 12 | [Name] [nvarchar](4000) NOT NULL, 13 | [AutoMapper] [nvarchar](255) NULL, 14 | CONSTRAINT [PK_BeatSaberSong] PRIMARY KEY CLUSTERED 15 | ( 16 | [BeatSaverKey] ASC 17 | ) 18 | ) ON [PRIMARY] 19 | GO 20 | 21 | CREATE FULLTEXT CATALOG [BeatSaverCatalog] WITH ACCENT_SENSITIVITY = ON 22 | AS DEFAULT 23 | 24 | CREATE FULLTEXT INDEX ON BeatSaberSong(LevelAuthorName, SongAuthorName, SongName, SongSubName, Name) KEY INDEX PK_BeatSaberSong 25 | 26 | BEGIN TRANSACTION 27 | GO 28 | ALTER TABLE dbo.BeatSaberSong ADD 29 | CreatedAt datetime2(7) NULL, 30 | UpdatedAt datetime2(7) NULL, 31 | LastPublishedAt datetime2(7) NULL 32 | GO 33 | ALTER TABLE dbo.BeatSaberSong ADD 34 | DeletedAt datetime2(7) NULL 35 | GO 36 | COMMIT 37 | 38 | BEGIN TRANSACTION 39 | GO 40 | CREATE TABLE dbo.BeatSaberSongRatings 41 | ( 42 | BeatSaverKey int NOT NULL, 43 | Upvotes int NOT NULL, 44 | Downvotes int NOT NULL, 45 | Score float(53) NOT NULL, 46 | UpdatedAt datetime2(7) NOT NULL 47 | ) ON [PRIMARY] 48 | GO 49 | ALTER TABLE dbo.BeatSaberSongRatings ADD CONSTRAINT 50 | PK_BeatSaberSongRatings PRIMARY KEY CLUSTERED 51 | ( 52 | BeatSaverKey 53 | ) WITH( STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON) ON [PRIMARY] 54 | 55 | GO 56 | COMMIT 57 | 58 | 59 | CREATE NONCLUSTERED INDEX [IDX_BeatSaberSong_DeletedAt] ON [dbo].[BeatSaberSong] 60 | ( 61 | [DeletedAt] ASC 62 | )WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, SORT_IN_TEMPDB = OFF, DROP_EXISTING = OFF, ONLINE = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) 63 | 64 | GO 65 | 66 | 67 | -------------------------------------------------------------------------------- /src/.editorconfig: -------------------------------------------------------------------------------- 1 | [*.{cs,vb}] 2 | dotnet_naming_rule.private_members_with_underscore.symbols = private_fields 3 | dotnet_naming_rule.private_members_with_underscore.style = prefix_underscore 4 | dotnet_naming_rule.private_members_with_underscore.severity = suggestion 5 | 6 | dotnet_naming_symbols.private_fields.applicable_kinds = field 7 | dotnet_naming_symbols.private_fields.applicable_accessibilities = private 8 | 9 | dotnet_naming_style.prefix_underscore.capitalization = camel_case 10 | dotnet_naming_style.prefix_underscore.required_prefix = _ -------------------------------------------------------------------------------- /src/BeatSaverMatcher.Api.Spotify/BeatSaverMatcher.Api.Spotify.csproj: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | net8.0 14 | enable 15 | 16 | 17 | 18 | -------------------------------------------------------------------------------- /src/BeatSaverMatcher.Api.Spotify/SpotifyConfiguration.cs: -------------------------------------------------------------------------------- 1 | namespace BeatSaverMatcher.Api.Spotify 2 | { 3 | public class SpotifyConfiguration 4 | { 5 | public string ClientId { get; set; } 6 | public string ClientSecret { get; set; } 7 | } 8 | } 9 | -------------------------------------------------------------------------------- /src/BeatSaverMatcher.Api.Spotify/SpotifyRepository.cs: -------------------------------------------------------------------------------- 1 | using Microsoft.Extensions.Options; 2 | using SpotifyAPI.Web; 3 | using System; 4 | using System.Collections.Generic; 5 | using System.Linq; 6 | using System.Threading; 7 | using System.Threading.Tasks; 8 | 9 | namespace BeatSaverMatcher.Api.Spotify 10 | { 11 | public class SpotifyRepository : IMusicServiceApi 12 | { 13 | private readonly SpotifyClient _spotifyClient; 14 | 15 | public SpotifyRepository(IOptions options) 16 | { 17 | var spotifyConfig = options.Value; 18 | 19 | var config = SpotifyClientConfig 20 | .CreateDefault() 21 | .WithAuthenticator(new ClientCredentialsAuthenticator(spotifyConfig.ClientId, spotifyConfig.ClientSecret)); 22 | 23 | _spotifyClient = new SpotifyClient(config); 24 | } 25 | 26 | public async Task GetPlaylist(string playlistId, CancellationToken cancellationToken) 27 | { 28 | var spotifyPlaylist = await _spotifyClient.Playlists.Get(playlistId, cancellationToken); 29 | return new Playlist( 30 | spotifyPlaylist.Id ?? throw new Exception("Missing ID of playlist reponse"), 31 | spotifyPlaylist.Name, 32 | spotifyPlaylist.Owner?.DisplayName, 33 | spotifyPlaylist.Images?.LastOrDefault(x => Math.Max(x.Width, x.Height) >= 256)?.Url); 34 | } 35 | 36 | public async Task> GetTracksForPlaylist(string playlistId, Action progress, CancellationToken cancellationToken) 37 | { 38 | var toReturn = new List(); 39 | 40 | var request = new PlaylistGetItemsRequest(PlaylistGetItemsRequest.AdditionalTypes.Track); 41 | 42 | try 43 | { 44 | var currentPage = await _spotifyClient.Playlists.GetItems(playlistId, request, cancellationToken); 45 | while (currentPage?.Items != null) 46 | { 47 | if (currentPage.Offset.HasValue && currentPage.Total.HasValue) 48 | { 49 | progress(currentPage.Offset.Value + currentPage.Items.Count, currentPage.Total.Value); 50 | } 51 | cancellationToken.ThrowIfCancellationRequested(); 52 | toReturn.AddRange(currentPage.Items.Select(x => x.Track).OfType().Select(MapSong)); // OfType instead of Cast because there may be Episodes, etc. 53 | currentPage = currentPage.Next != null ? await _spotifyClient.NextPage(currentPage) : null; 54 | } 55 | } 56 | catch (SpotifyAPI.Web.APIException aex) 57 | { 58 | throw new APIException(aex.Response?.StatusCode, aex.Message, aex); 59 | } 60 | 61 | return toReturn; 62 | } 63 | 64 | private PlaylistSong MapSong(FullTrack track) 65 | { 66 | return new PlaylistSong(track.Name, track.Artists.Select(x => x.Name).ToList()); 67 | } 68 | } 69 | } 70 | -------------------------------------------------------------------------------- /src/BeatSaverMatcher.Api.Tidal/BeatSaverMatcher.Api.Tidal.csproj: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | net8.0 15 | enable 16 | 17 | 18 | 19 | -------------------------------------------------------------------------------- /src/BeatSaverMatcher.Api.Tidal/TidalClient.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Net; 5 | using System.Net.Http; 6 | using System.Net.Http.Headers; 7 | using System.Net.Http.Json; 8 | using System.Text; 9 | using System.Text.Json.Nodes; 10 | using System.Threading; 11 | using System.Threading.Tasks; 12 | using Microsoft.Extensions.Logging; 13 | using Microsoft.Extensions.Options; 14 | 15 | namespace BeatSaverMatcher.Api.Tidal; 16 | 17 | public class TidalClient : IMusicServiceApi, IDisposable 18 | { 19 | private readonly TidalRateLimitHandler _rateLimitHandler; 20 | private readonly ILogger _logger; 21 | private readonly string _clientId; 22 | private readonly string _clientSecret; 23 | private readonly HttpClient _httpClient; 24 | 25 | private DateTime _tokenExpiry = DateTime.MinValue; 26 | 27 | public TidalClient(TidalRateLimitHandler rateLimitHandler, ILogger logger, IOptions options) 28 | { 29 | _rateLimitHandler = rateLimitHandler; 30 | _logger = logger; 31 | var config = options.Value; 32 | _clientId = config.ClientId; 33 | _clientSecret = config.ClientSecret; 34 | 35 | _httpClient = new HttpClient() 36 | { 37 | BaseAddress = new Uri("https://openapi.tidal.com/v2/") 38 | }; 39 | } 40 | 41 | private async Task Authenticate() 42 | { 43 | if (_tokenExpiry >= DateTime.UtcNow) 44 | { 45 | return; 46 | } 47 | 48 | using var httpClient = new HttpClient(); 49 | using var request = new HttpRequestMessage(HttpMethod.Post, "https://auth.tidal.com/v1/oauth2/token") 50 | { 51 | Content = new FormUrlEncodedContent(new Dictionary { { "grant_type", "client_credentials" } }) 52 | }; 53 | request.Headers.Authorization = new AuthenticationHeaderValue("Basic", Convert.ToBase64String(Encoding.UTF8.GetBytes($"{_clientId}:{_clientSecret}"))); 54 | 55 | var response = await httpClient.SendAsync(request); 56 | response.EnsureSuccessStatusCode(); 57 | var responseJson = await response.Content.ReadFromJsonAsync(); 58 | 59 | if (responseJson == null) 60 | throw new Exception("Empty auth response"); 61 | 62 | var tokenType = responseJson["token_type"]?.GetValue(); 63 | var expiresIn = responseJson["expires_in"]?.GetValue(); 64 | var token = responseJson["access_token"]?.GetValue(); 65 | 66 | if (tokenType != "Bearer" || expiresIn == null || token == null) 67 | throw new Exception("Invalid auth response"); 68 | 69 | _httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", token); 70 | _tokenExpiry = DateTime.UtcNow.AddSeconds(expiresIn.Value); 71 | } 72 | 73 | public async Task GetPlaylist(string playlistId, CancellationToken cancellationToken) 74 | { 75 | if (!Guid.TryParse(playlistId, out _)) 76 | throw new APIException(message: "Invalid playlist ID!"); 77 | 78 | await Authenticate(); 79 | 80 | var elem = await GetWithRateLimit>(_httpClient, $"playlists/{playlistId}?countryCode=US&include=coverArt", cancellationToken); 81 | 82 | if (elem?.Data?.Attributes == null) 83 | throw new Exception("missing data from response!"); 84 | 85 | var attributes = elem.Data.Attributes; 86 | var artworks = elem.GetIncluded("artworks"); 87 | 88 | var coverArtRelationship = elem.Data.Relationships?.CoverArt?.Data; 89 | string? coverUrl = null; 90 | if (coverArtRelationship?.Id != null) 91 | { 92 | var includedArt = artworks.GetValueOrDefault(coverArtRelationship.Id)?.Attributes; 93 | coverUrl = includedArt?.Files?.LastOrDefault(x => Math.Max(x.Meta?.Width ?? 0, x.Meta?.Height ?? 0) >= 256)?.Href; 94 | } 95 | 96 | return new Playlist(elem.Data.Id ?? throw new Exception("missing ID"), attributes.Name, "???", coverUrl); 97 | } 98 | 99 | public async Task> GetTracksForPlaylist(string playlistId, Action progress, CancellationToken cancellationToken) 100 | { 101 | if (!Guid.TryParse(playlistId, out _)) 102 | throw new APIException(message: "Invalid playlist ID!"); 103 | 104 | await Authenticate(); 105 | 106 | var toReturn = new List(); 107 | 108 | var nextUri = $"playlists/{playlistId}/relationships/items?countryCode=US&include=items.artists"; 109 | 110 | while (nextUri != null) 111 | { 112 | // including items here is useless because those don't include the artist relationship data which we need. 113 | var itemsResponse = await GetWithRateLimit>>(_httpClient, nextUri, cancellationToken); 114 | 115 | if (itemsResponse?.Data == null) 116 | throw new Exception("missing data from response!"); 117 | 118 | nextUri = itemsResponse.Links?.GetValueOrDefault("next")?.TrimStart('/'); 119 | 120 | // Fix for API bug (next-page URI only has top level include) 121 | nextUri = nextUri?.Replace("?include=items&", "?include=items.artists&"); 122 | 123 | var tracks = itemsResponse.GetIncluded("tracks"); 124 | var artists = itemsResponse.GetIncluded("artists"); 125 | 126 | foreach (var item in itemsResponse.Data) 127 | { 128 | var track = tracks.GetValueOrDefault(item.Id ?? throw new Exception("missing ID")); 129 | if (track == null) 130 | continue; // missing / deleted track 131 | if (track?.Attributes?.Title == null || track?.Relationships?.Artists?.Data == null) 132 | throw new Exception("missing data from track!"); 133 | var trackArtists = new List(); 134 | foreach (var artistRel in track.Relationships.Artists.Data) 135 | { 136 | var artist = artists?.GetValueOrDefault(artistRel.Id ?? throw new Exception("missing ID")); 137 | if (artist == null) 138 | continue; // missing artist 139 | if (artist.Attributes?.Name == null) 140 | throw new Exception("missing data from artist!"); 141 | trackArtists.Add(artist.Attributes.Name); 142 | } 143 | toReturn.Add(new PlaylistSong(track.Attributes.Title, trackArtists)); 144 | } 145 | progress(toReturn.Count, null); 146 | } 147 | 148 | return toReturn; 149 | } 150 | 151 | private async Task GetWithRateLimit(HttpClient httpClient, string uri, CancellationToken token) 152 | { 153 | int tries = 0; 154 | int? requiredTokens = null; 155 | while (true) 156 | { 157 | tries++; 158 | if (tries > 10) 159 | throw new Exception("Too Many Retries"); 160 | 161 | await _rateLimitHandler.WaitForTokens(requiredTokens, token); 162 | 163 | using var request = new HttpRequestMessage(HttpMethod.Get, uri); 164 | foreach (var header in httpClient.DefaultRequestHeaders) 165 | { 166 | request.Headers.Add(header.Key, header.Value); 167 | } 168 | var response = await httpClient.SendAsync(request, token); 169 | 170 | if ( 171 | response.Headers.TryGetValues("X-RateLimit-Remaining", out var limitRemaining) && 172 | response.Headers.TryGetValues("X-RateLimit-Burst-Capacity", out var burstCapacity) && 173 | response.Headers.TryGetValues("X-RateLimit-Replenish-Rate", out var replenishRate) && 174 | response.Headers.TryGetValues("X-RateLimit-Requested-Tokens", out var requestedTokens) 175 | ) 176 | { 177 | requiredTokens = int.Parse(requestedTokens.First()); 178 | _rateLimitHandler.Update(int.Parse(limitRemaining.First()), int.Parse(burstCapacity.First()), int.Parse(replenishRate.First()), requiredTokens.Value); 179 | } 180 | 181 | if (response.StatusCode >= HttpStatusCode.OK && response.StatusCode < HttpStatusCode.MultipleChoices) 182 | { 183 | return await response.Content.ReadFromJsonAsync(token)!; 184 | } 185 | else if (response.StatusCode == HttpStatusCode.TooManyRequests) 186 | { 187 | _logger.LogInformation("Hit Tidal rate limit"); 188 | continue; 189 | } 190 | else if (response.StatusCode >= HttpStatusCode.InternalServerError) 191 | { 192 | _logger.LogWarning("Status 500 while fetching from Tidal"); 193 | await Task.Delay(10000, token); 194 | } 195 | else if (response.StatusCode >= HttpStatusCode.BadRequest) 196 | { 197 | string? message = null; 198 | Exception? exception = null; 199 | try 200 | { 201 | var error = await response.Content.ReadFromJsonAsync(token); 202 | if (error?.Errors?.Any(x => x.Detail != null) ?? false) 203 | { 204 | message = string.Join(", ", error.Errors.Select(x => x.Detail)); 205 | } 206 | } 207 | catch (Exception ex) 208 | { 209 | exception = ex; 210 | } 211 | throw new APIException(response.StatusCode, message, exception); 212 | } 213 | else 214 | { 215 | response.EnsureSuccessStatusCode(); // should throw 216 | throw new Exception($"Unknown Status {response.StatusCode}"); 217 | } 218 | } 219 | } 220 | 221 | public void Dispose() 222 | { 223 | _httpClient.Dispose(); 224 | } 225 | } 226 | 227 | public class TidalRateLimitHandler 228 | { 229 | private readonly SemaphoreSlim _semaphore = new(1, 1); 230 | private readonly ILogger _logger; 231 | private (DateTime LastUpdate, int LastTokenCount, int MaxTokenCount, int ReplenishRate)? _state = null; 232 | private int? _minimumRequestTokens; 233 | 234 | public TidalRateLimitHandler(ILogger logger) 235 | { 236 | _logger = logger; 237 | } 238 | 239 | public void Update(int remainingTokens, int burstCapacity, int replenishRate, int requestedTokens) 240 | { 241 | if (replenishRate <= 0) 242 | throw new ArgumentException("ReplenishRate must be greater than 0", nameof(replenishRate)); 243 | _semaphore.Wait(); 244 | try 245 | { 246 | _state = (DateTime.UtcNow, remainingTokens, burstCapacity, replenishRate); 247 | if (_minimumRequestTokens == null || requestedTokens < _minimumRequestTokens) 248 | { 249 | _minimumRequestTokens = requestedTokens; 250 | } 251 | } 252 | finally 253 | { 254 | _semaphore.Release(); 255 | } 256 | } 257 | 258 | public async Task WaitForTokens(int? requestedTokensHint, CancellationToken token) 259 | { 260 | var requestedTokens = requestedTokensHint ?? _minimumRequestTokens ?? 0; 261 | 262 | if (_state.HasValue && requestedTokens > _state.Value.MaxTokenCount) 263 | throw new Exception("may not request more tokens than maximum"); 264 | while (true) 265 | { 266 | TimeSpan waitTime; 267 | await _semaphore.WaitAsync(token); 268 | try 269 | { 270 | if (_state == null) 271 | return; 272 | 273 | var state = _state.Value; 274 | var now = DateTime.UtcNow; 275 | 276 | var elapsedSeconds = (int)(now - state.LastUpdate).TotalSeconds; 277 | var currentTokens = Math.Min(state.LastTokenCount + elapsedSeconds * state.ReplenishRate, state.MaxTokenCount); 278 | 279 | if (requestedTokens <= currentTokens) 280 | { 281 | _state = (now, currentTokens - requestedTokens, state.MaxTokenCount, state.ReplenishRate); 282 | return; 283 | } 284 | else 285 | { 286 | waitTime = TimeSpan.FromSeconds(Math.Ceiling((requestedTokens - currentTokens) / (double)state.ReplenishRate)); 287 | _logger.LogInformation("Waiting for Tidal rate limit, waiting {WaitTime}s to replenish ({CurrentTokens} / {RequestedTokens} tokens)", (int)waitTime.TotalSeconds, currentTokens, requestedTokensHint?.ToString() ?? "?"); 288 | } 289 | } 290 | finally 291 | { 292 | _semaphore.Release(); 293 | } 294 | await Task.Delay(waitTime, token); 295 | } 296 | 297 | } 298 | } 299 | -------------------------------------------------------------------------------- /src/BeatSaverMatcher.Api.Tidal/TidalConfiguration.cs: -------------------------------------------------------------------------------- 1 | namespace BeatSaverMatcher.Api.Tidal 2 | { 3 | public class TidalConfiguration 4 | { 5 | public string ClientId { get; set; } 6 | public string ClientSecret { get; set; } 7 | } 8 | } 9 | -------------------------------------------------------------------------------- /src/BeatSaverMatcher.Api.Tidal/TidalModels.cs: -------------------------------------------------------------------------------- 1 | using System.Collections.Generic; 2 | using System.Linq; 3 | using System.Text.Json; 4 | using System.Text.Json.Nodes; 5 | 6 | namespace BeatSaverMatcher.Api.Tidal; 7 | 8 | internal class TidalResponse 9 | { 10 | public TData? Data { get; set; } 11 | public Dictionary? Links { get; set; } 12 | public IList? Included { get; set; } 13 | public Dictionary GetIncluded(string type) 14 | where T : class 15 | { 16 | return Included? 17 | .Where(x => x["type"]?.GetValue() == type && x["id"] != null) 18 | .ToDictionary( 19 | x => x["id"]!.GetValue(), 20 | x => x.Deserialize(new JsonSerializerOptions { PropertyNameCaseInsensitive = true })!) ?? []; 21 | } 22 | } 23 | 24 | internal class TidalErrorResponse 25 | { 26 | public IList Errors { get; set; } 27 | } 28 | 29 | internal class TidalError 30 | { 31 | public string Id { get; set; } 32 | public string Status { get; set; } 33 | public string Code { get; set; } 34 | public string Detail { get; set; } 35 | //... 36 | } 37 | 38 | internal class TidalData 39 | { 40 | public string? Id { get; set; } 41 | public string? Type { get; set; } 42 | } 43 | 44 | internal class TidalPlaylistData : TidalData 45 | { 46 | public TidalPlaylistAttributes? Attributes { get; set; } 47 | public TidalPlaylistRelationships? Relationships { get; set; } 48 | } 49 | internal class TidalArtworkData : TidalData 50 | { 51 | public TidalArtworkAttributes? Attributes { get; set; } 52 | } 53 | 54 | internal class TidalTrackData : TidalData 55 | { 56 | public TidalTrackAttributes? Attributes { get; set; } 57 | public TidalTrackRelationships? Relationships { get; set; } 58 | } 59 | 60 | internal class TidalTrackRelationships 61 | { 62 | public TidalRelationship>? Artists { get; set; } 63 | } 64 | 65 | internal class TidalTrackArtistRelationshipData : TidalData 66 | { 67 | } 68 | 69 | internal class TidalTrackAttributes 70 | { 71 | public string? Title { get; set; } 72 | } 73 | 74 | internal class TidalArtistData : TidalData 75 | { 76 | public TidalArtistAttributes? Attributes { get; set; } 77 | } 78 | 79 | public class TidalArtistAttributes 80 | { 81 | public string? Name { get; set; } 82 | } 83 | 84 | internal class TidalPlaylistRelationships 85 | { 86 | public TidalRelationship? CoverArt { get; set; } 87 | } 88 | 89 | internal class TidalRelationship 90 | { 91 | public Dictionary? Links { get; set; } 92 | public TData? Data { get; set; } 93 | } 94 | internal class TidalPlaylistCoverArtRelationshipData : TidalData 95 | { 96 | } 97 | 98 | internal class TidalPlaylistItemRelationshipData : TidalData 99 | { 100 | // Meta 101 | } 102 | 103 | internal class TidalPlaylistAttributes 104 | { 105 | public string? Name { get; set; } 106 | } 107 | 108 | internal class TidalArtworkAttributes 109 | { 110 | public string? MediaType { get; set; } 111 | public IList? Files { get; set; } 112 | } 113 | internal class TidalArtworkAttributesFile 114 | { 115 | public string? Href { get; set; } 116 | public TidalArtworkAttributesFileMeta? Meta { get; set; } 117 | } 118 | internal class TidalArtworkAttributesFileMeta 119 | { 120 | public int? Width { get; set; } 121 | public int? Height { get; set; } 122 | } -------------------------------------------------------------------------------- /src/BeatSaverMatcher.Api/APIException.cs: -------------------------------------------------------------------------------- 1 | 2 | using System; 3 | using System.Net; 4 | 5 | public class APIException : Exception 6 | { 7 | public APIException(HttpStatusCode? statusCode = null, string? message = null, Exception? inner = null) 8 | : base(message, inner) 9 | { 10 | StatusCode = statusCode; 11 | } 12 | 13 | public HttpStatusCode? StatusCode { get; } 14 | } -------------------------------------------------------------------------------- /src/BeatSaverMatcher.Api/BeatSaverMatcher.Api.csproj: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | net8.0 5 | enable 6 | 7 | 8 | 9 | -------------------------------------------------------------------------------- /src/BeatSaverMatcher.Api/IMusicServiceApi.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Threading; 4 | using System.Threading.Tasks; 5 | 6 | namespace BeatSaverMatcher.Api; 7 | 8 | public interface IMusicServiceApi 9 | { 10 | Task GetPlaylist(string playlistId, CancellationToken cancellationToken); 11 | Task> GetTracksForPlaylist(string playlistId, Action progress, CancellationToken cancellationToken); 12 | } 13 | -------------------------------------------------------------------------------- /src/BeatSaverMatcher.Api/Playlist.cs: -------------------------------------------------------------------------------- 1 | namespace BeatSaverMatcher.Api; 2 | 3 | public record Playlist(string Id, string? Name, string? OwnerName, string? ImageUrl); 4 | -------------------------------------------------------------------------------- /src/BeatSaverMatcher.Api/PlaylistSong.cs: -------------------------------------------------------------------------------- 1 | using System.Collections.Generic; 2 | 3 | namespace BeatSaverMatcher.Api; 4 | public record PlaylistSong(string Name, IList Artists); 5 | -------------------------------------------------------------------------------- /src/BeatSaverMatcher.Common/BeatSaver/BeatSaverDeletedSong.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | 3 | namespace BeatSaverMatcher.Common.BeatSaver 4 | { 5 | public class BeatSaverDeletedSong 6 | { 7 | public string Id { get; set; } 8 | public DateTime DeletedAt { get; set; } 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /src/BeatSaverMatcher.Common/BeatSaver/BeatSaverRepository.cs: -------------------------------------------------------------------------------- 1 | using Microsoft.Extensions.Logging; 2 | using System; 3 | using System.Collections.Generic; 4 | using System.IO; 5 | using System.Linq; 6 | using System.Net; 7 | using System.Text.Json; 8 | using System.Text.Json.Serialization; 9 | using System.Threading; 10 | using System.Threading.Tasks; 11 | using System.Web; 12 | 13 | namespace BeatSaverMatcher.Common.BeatSaver 14 | { 15 | public class BeatSaverRepository 16 | { 17 | private readonly ILogger _logger; 18 | private static readonly JsonSerializerOptions _beatSaverSerializerOptions = new JsonSerializerOptions 19 | { 20 | PropertyNamingPolicy = JsonNamingPolicy.CamelCase, 21 | Converters = 22 | { 23 | new JsonStringEnumConverter() 24 | } 25 | }; 26 | 27 | public BeatSaverRepository(ILogger logger) 28 | { 29 | _logger = logger; 30 | } 31 | 32 | public async Task GetSong(int key, CancellationToken token) 33 | { 34 | return await DoWithRetries(async () => 35 | { 36 | try 37 | { 38 | var request = WebRequest.CreateHttp($"https://beatsaver.com/api/maps/id/{key:x}"); 39 | request.Headers.Add("user-agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.85 Safari/537.36"); 40 | request.Headers.Add("sec-fetch-mode", "navigate"); 41 | request.Headers.Add("sec-fetch-user", "?1"); 42 | request.Headers.Add("accept-language", "de-DE,de;q=0.9,en-US;q=0.8,en;q=0.7"); 43 | 44 | BeatSaverSong song; 45 | var response = (HttpWebResponse)await request.GetResponseAsync(); 46 | using (var sr = new StreamReader(response.GetResponseStream())) 47 | { 48 | song = JsonSerializer.Deserialize(sr.ReadToEnd(), _beatSaverSerializerOptions); 49 | } 50 | return song; 51 | } 52 | catch (WebException wex) 53 | { 54 | if (!(wex.Response is HttpWebResponse response)) 55 | throw; 56 | 57 | if (response.StatusCode == HttpStatusCode.NotFound) 58 | { 59 | return null; 60 | } 61 | throw; 62 | } 63 | }, token); 64 | } 65 | 66 | public async Task> GetSongsUpdatedAfter(DateTime lastUpdatedAt, CancellationToken token) 67 | { 68 | return await DoWithRetries(async () => 69 | { 70 | var url = $"https://beatsaver.com/api/maps/latest?sort=UPDATED&automapper=true&pageSize=100&after={HttpUtility.UrlEncode(DateTime.SpecifyKind(lastUpdatedAt, DateTimeKind.Utc).ToString("o"))}"; 71 | var request = WebRequest.CreateHttp(url); 72 | request.Headers.Add("user-agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.85 Safari/537.36"); 73 | request.Headers.Add("sec-fetch-mode", "navigate"); 74 | request.Headers.Add("sec-fetch-user", "?1"); 75 | request.Headers.Add("accept-language", "de-DE,de;q=0.9,en-US;q=0.8,en;q=0.7"); 76 | 77 | BeatSaverSongSearchResponse page; 78 | var response = (HttpWebResponse)await request.GetResponseAsync(); 79 | using (var sr = new StreamReader(response.GetResponseStream())) 80 | { 81 | page = JsonSerializer.Deserialize(sr.ReadToEnd(), _beatSaverSerializerOptions); 82 | } 83 | 84 | return page.Docs; 85 | }, token); 86 | } 87 | 88 | public async Task> GetSongsDeletedAfter(DateTime lastUpdatedAt, CancellationToken token) 89 | { 90 | return await DoWithRetries(async () => 91 | { 92 | var url = $"https://beatsaver.com/api/maps/deleted?pageSize=100&after={HttpUtility.UrlEncode(DateTime.SpecifyKind(lastUpdatedAt, DateTimeKind.Utc).ToString("o"))}"; 93 | var request = WebRequest.CreateHttp(url); 94 | request.Headers.Add("user-agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.85 Safari/537.36"); 95 | request.Headers.Add("sec-fetch-mode", "navigate"); 96 | request.Headers.Add("sec-fetch-user", "?1"); 97 | request.Headers.Add("accept-language", "de-DE,de;q=0.9,en-US;q=0.8,en;q=0.7"); 98 | 99 | BeatSaverSongDeletedResponse page; 100 | var response = (HttpWebResponse)await request.GetResponseAsync(); 101 | using (var sr = new StreamReader(response.GetResponseStream())) 102 | { 103 | page = JsonSerializer.Deserialize(sr.ReadToEnd(), _beatSaverSerializerOptions); 104 | } 105 | 106 | return page.Docs; 107 | }, token); 108 | } 109 | 110 | public async Task> GetSongs(IList keys, CancellationToken token) 111 | { 112 | return await DoWithRetries(async () => 113 | { 114 | var url = $"https://beatsaver.com/api/maps/ids/{string.Join(',', keys.Select(x => x.ToString("x")))}"; 115 | var request = WebRequest.CreateHttp(url); 116 | request.Headers.Add("user-agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.85 Safari/537.36"); 117 | request.Headers.Add("sec-fetch-mode", "navigate"); 118 | request.Headers.Add("sec-fetch-user", "?1"); 119 | request.Headers.Add("accept-language", "de-DE,de;q=0.9,en-US;q=0.8,en;q=0.7"); 120 | 121 | IDictionary result; 122 | var response = (HttpWebResponse)await request.GetResponseAsync(); 123 | using (var sr = new StreamReader(response.GetResponseStream())) 124 | { 125 | result = JsonSerializer.Deserialize>(sr.ReadToEnd(), _beatSaverSerializerOptions); 126 | } 127 | 128 | return result; 129 | }, token); 130 | } 131 | 132 | public async Task> GetScoresAfter(DateTime lastUpdatedAt, CancellationToken token) 133 | { 134 | return await DoWithRetries(async () => 135 | { 136 | var url = $"https://beatsaver.com/api/vote?since={HttpUtility.UrlEncode(DateTime.SpecifyKind(lastUpdatedAt, DateTimeKind.Utc).ToString("o"))}"; 137 | var request = WebRequest.CreateHttp(url); 138 | request.Headers.Add("user-agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.85 Safari/537.36"); 139 | request.Headers.Add("sec-fetch-mode", "navigate"); 140 | request.Headers.Add("sec-fetch-user", "?1"); 141 | request.Headers.Add("accept-language", "de-DE,de;q=0.9,en-US;q=0.8,en;q=0.7"); 142 | 143 | IList result; 144 | var response = (HttpWebResponse)await request.GetResponseAsync(); 145 | using (var sr = new StreamReader(response.GetResponseStream())) 146 | { 147 | result = JsonSerializer.Deserialize>(sr.ReadToEnd(), _beatSaverSerializerOptions); 148 | } 149 | 150 | return result; 151 | }, token); 152 | } 153 | 154 | private async Task DoWithRetries(Func> action, CancellationToken token) 155 | { 156 | int tries = 0; 157 | while (true) 158 | { 159 | try 160 | { 161 | return await action(); 162 | } 163 | catch (WebException wex) 164 | { 165 | var response = wex.Response as HttpWebResponse; 166 | if (response == null) 167 | throw; 168 | 169 | tries++; 170 | 171 | if ((int)response.StatusCode == 429) // Too Many Requests 172 | { 173 | if (_logger.IsEnabled(LogLevel.Debug)) 174 | { 175 | var allHeaders = Environment.NewLine + string.Join(Environment.NewLine, response.Headers.AllKeys.Select(headerKey => $"{headerKey}: {response.Headers[headerKey]}")); 176 | _logger.LogDebug("Error 429 Too Many Requests. Headers: {Headers}", allHeaders); 177 | } 178 | 179 | var timeout = TimeSpan.Zero; 180 | if (response.Headers.AllKeys.Contains("Rate-Limit-Reset") && int.TryParse(response.Headers["Rate-Limit-Reset"], out int epochReset)) 181 | { 182 | var resetTime = new DateTime(1970, 01, 01, 0, 0, 0, DateTimeKind.Utc).AddSeconds(epochReset); 183 | var delayTime = resetTime - DateTime.UtcNow; 184 | if (delayTime > timeout) 185 | { 186 | timeout = delayTime; 187 | } 188 | } 189 | else 190 | { 191 | timeout = TimeSpan.FromSeconds(9); 192 | } 193 | 194 | timeout += TimeSpan.FromSeconds(1); // always wait a minimum of one second to account for time tolerances 195 | _logger.LogInformation("Rate Limit Reached. Waiting {Milliseconds} ms", timeout.TotalMilliseconds); 196 | await Task.Delay(timeout, token); 197 | } 198 | else 199 | { 200 | _logger.LogError(wex, "Error while fetching."); 201 | throw; 202 | } 203 | 204 | if (tries > 10) 205 | throw; 206 | } 207 | } 208 | } 209 | } 210 | } 211 | -------------------------------------------------------------------------------- /src/BeatSaverMatcher.Common/BeatSaver/BeatSaverSong.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | 4 | namespace BeatSaverMatcher.Common.BeatSaver 5 | { 6 | #nullable enable 7 | public class BeatSaverSong 8 | { 9 | public string Id { get; set; } 10 | public string Name { get; set; } 11 | public string? Description { get; set; } 12 | public BeatSaverUploader Uploader { get; set; } 13 | public BeatSaverMetadata Metadata { get; set; } 14 | public BeatSaverStats Stats { get; set; } 15 | public DateTime Uploaded { get; set; } 16 | public DateTime CreatedAt { get; set; } 17 | public DateTime UpdatedAt { get; set; } 18 | public DateTime LastPublishedAt { get; set; } 19 | public bool Automapper { get; set; } 20 | public bool Ranked { get; set; } 21 | public IList Versions { get; set; } 22 | //... 23 | } 24 | 25 | public class BeatSaverUploader 26 | { 27 | public string Name { get; set; } 28 | //... 29 | } 30 | 31 | public class BeatSaverStats 32 | { 33 | public int Downloads { get; set; } 34 | public int Plays { get; set; } 35 | public int Downvotes { get; set; } 36 | public int Upvotes { get; set; } 37 | public double Score { get; set; } 38 | } 39 | 40 | public class BeatSaverMetadata 41 | { 42 | public double Duration { get; set; } 43 | public string LevelAuthorName { get; set; } 44 | public string SongAuthorName { get; set; } 45 | public string SongName { get; set; } 46 | public string SongSubName { get; set; } 47 | public double Bpm { get; set; } 48 | } 49 | 50 | public class BeatSaverVersion 51 | { 52 | public string Hash { get; set; } 53 | public string State { get; set; } 54 | public DateTime CreatedAt { get; set; } 55 | public IList Diffs { get; set; } 56 | } 57 | 58 | public class BeatSaverScore 59 | { 60 | //Hash 61 | public int MapId { get; set; } 62 | // Key64 63 | public int Upvotes { get; set; } 64 | public int Downvotes { get; set; } 65 | public double Score { get; set; } 66 | } 67 | 68 | public class BeatSaverDifficulty 69 | { 70 | public BeatSaverDifficultyType Difficulty { get; set; } 71 | public string Characteristic { get; set; } 72 | } 73 | 74 | public enum BeatSaverDifficultyType 75 | { 76 | Easy, 77 | Normal, 78 | Hard, 79 | Expert, 80 | ExpertPlus 81 | } 82 | #nullable restore 83 | } 84 | -------------------------------------------------------------------------------- /src/BeatSaverMatcher.Common/BeatSaver/BeatSaverSongDeletedResponse.cs: -------------------------------------------------------------------------------- 1 | using System.Collections.Generic; 2 | 3 | namespace BeatSaverMatcher.Common.BeatSaver 4 | { 5 | internal class BeatSaverSongDeletedResponse 6 | { 7 | public IList Docs { get; set; } 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /src/BeatSaverMatcher.Common/BeatSaver/BeatSaverSongSearchResponse.cs: -------------------------------------------------------------------------------- 1 | using System.Collections.Generic; 2 | 3 | namespace BeatSaverMatcher.Common.BeatSaver 4 | { 5 | internal class BeatSaverSongSearchResponse 6 | { 7 | public IList Docs { get; set; } 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /src/BeatSaverMatcher.Common/BeatSaver/BeatSaverUtils.cs: -------------------------------------------------------------------------------- 1 | using BeatSaverMatcher.Common.Db; 2 | using System; 3 | using System.Collections.Generic; 4 | using System.Linq; 5 | 6 | namespace BeatSaverMatcher.Common.BeatSaver 7 | { 8 | public static class BeatSaverUtils 9 | { 10 | public static byte[] MapHash(string hash) 11 | { 12 | byte[] toReturn = new byte[hash.Length / 2]; 13 | for (int i = 0; i < hash.Length; i += 2) 14 | toReturn[i / 2] = Convert.ToByte(hash.Substring(i, 2), 16); 15 | return toReturn; 16 | } 17 | 18 | public static SongDifficulties MapDifficulties(IList difficulties) 19 | { 20 | SongDifficulties toReturn = 0; 21 | if (difficulties.Any(x => x.Difficulty == BeatSaverDifficultyType.Easy)) 22 | toReturn |= SongDifficulties.Easy; 23 | if (difficulties.Any(x => x.Difficulty == BeatSaverDifficultyType.Normal)) 24 | toReturn |= SongDifficulties.Normal; 25 | if (difficulties.Any(x => x.Difficulty == BeatSaverDifficultyType.Hard)) 26 | toReturn |= SongDifficulties.Hard; 27 | if (difficulties.Any(x => x.Difficulty == BeatSaverDifficultyType.Expert)) 28 | toReturn |= SongDifficulties.Expert; 29 | if (difficulties.Any(x => x.Difficulty == BeatSaverDifficultyType.ExpertPlus)) 30 | toReturn |= SongDifficulties.ExpertPlus; 31 | return toReturn; 32 | } 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /src/BeatSaverMatcher.Common/BeatSaverMatcher.Common.csproj: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | net8.0 5 | 8.0 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | -------------------------------------------------------------------------------- /src/BeatSaverMatcher.Common/CacheKeys.cs: -------------------------------------------------------------------------------- 1 | namespace BeatSaverMatcher.Common 2 | { 3 | public class CacheKeys 4 | { 5 | public static string GetForBeatmap(int key) 6 | { 7 | return $"Beatmap_{key}"; 8 | } 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /src/BeatSaverMatcher.Common/Db/BeatSaberSong.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | 3 | namespace BeatSaverMatcher.Common.Db 4 | { 5 | public class BeatSaberSong 6 | { 7 | public string LevelAuthorName { get; set; } 8 | public string SongAuthorName { get; set; } 9 | public string SongName { get; set; } 10 | public string SongSubName { get; set; } 11 | public double Bpm { get; set; } 12 | 13 | public string Name { get; set; } 14 | 15 | public SongDifficulties Difficulties { get; set; } 16 | public string Uploader { get; set; } 17 | public DateTime Uploaded { get; set; } 18 | public byte[] Hash { get; set; } 19 | public int BeatSaverKey { get; set; } 20 | public string AutoMapper { get; set; } 21 | public DateTime? CreatedAt { get; set; } 22 | public DateTime? UpdatedAt { get; set; } 23 | public DateTime? LastPublishedAt { get; set; } 24 | public DateTime? DeletedAt { get; set; } 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /src/BeatSaverMatcher.Common/Db/BeatSaberSongRatings.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | 3 | namespace BeatSaverMatcher.Common.Db 4 | { 5 | public class BeatSaberSongRatings 6 | { 7 | public int BeatSaverKey { get; set; } 8 | public int Downvotes { get; set; } 9 | public int Upvotes { get; set; } 10 | public double Score { get; set; } 11 | public DateTime UpdatedAt { get; set; } 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /src/BeatSaverMatcher.Common/Db/BeatSaberSongRepository.cs: -------------------------------------------------------------------------------- 1 | using Dapper; 2 | using Microsoft.Data.SqlClient; 3 | using Microsoft.Extensions.Options; 4 | using System; 5 | using System.Collections.Generic; 6 | using System.Data; 7 | using System.Linq; 8 | using System.Threading; 9 | using System.Threading.Tasks; 10 | 11 | namespace BeatSaverMatcher.Common.Db 12 | { 13 | public class BeatSaberSongRepository : SqlServerRepository, IBeatSaberSongRepository 14 | { 15 | public BeatSaberSongRepository(IOptions options) 16 | : base(options) 17 | { 18 | SqlMapper.AddTypeMap(typeof(DateTime), DbType.DateTime2); // use datetime2 so datetimes are saved with max. precision 19 | } 20 | 21 | public async Task> GetMatches(string artistName, string trackName) 22 | { 23 | artistName = "\"" + new string(artistName.Where(x => x != '"' && x != '*').ToArray()) + "\""; 24 | trackName = "\"" + new string(trackName.Where(x => x != '"' && x != '*').ToArray()) + "\""; 25 | using (var connection = GetConnection()) 26 | { 27 | var query = 28 | @" 29 | SELECT 30 | song.*, 31 | rating.Upvotes, 32 | rating.Downvotes, 33 | rating.Score 34 | FROM [BeatSaberSong] song 35 | LEFT JOIN [BeatSaberSongRatings] rating ON song.[BeatSaverKey] = rating.[BeatSaverKey] 36 | WHERE [DeletedAt] IS NULL AND 37 | CONTAINS(song.*, @ArtistName) AND 38 | CONTAINS(song.*, @TrackName) AND 39 | song.AutoMapper IS NULL"; 40 | 41 | var results = await connection.QueryAsync(query, new { ArtistName = artistName, TrackName = trackName }); 42 | return results.ToList(); 43 | } 44 | } 45 | 46 | public async Task GetLatestBeatSaverKey() 47 | { 48 | using (var connection = GetConnection()) 49 | { 50 | var sqlStr = @"SELECT TOP 1 BeatSaverKey FROM [dbo].[BeatSaberSong] ORDER BY BeatSaverKey DESC"; 51 | 52 | using (var command = new SqlCommand(sqlStr, connection)) 53 | { 54 | using (var reader = await command.ExecuteReaderAsync()) 55 | { 56 | if (!await reader.ReadAsync()) 57 | { 58 | return null; 59 | } 60 | return reader.GetInt32(0); 61 | } 62 | } 63 | } 64 | } 65 | 66 | public async Task InsertOrUpdateSong(BeatSaberSong song) 67 | { 68 | using (var connection = GetConnection()) 69 | { 70 | var sqlUpdate = @" 71 | UPDATE [dbo].[BeatSaberSong] 72 | SET 73 | [Hash] = @Hash, 74 | [Uploader] = @Uploader, 75 | [Uploaded] = @Uploaded, 76 | [Difficulties] = @Difficulties, 77 | [Bpm] = @Bpm, 78 | [LevelAuthorName] = @LevelAuthorName, 79 | [SongAuthorName] = @SongAuthorName, 80 | [SongName] = @SongName, 81 | [SongSubName] = @SongSubName, 82 | [Name] = @Name, 83 | [AutoMapper] = @AutoMapper, 84 | [CreatedAt] = @CreatedAt, 85 | [UpdatedAt] = @UpdatedAt, 86 | [LastPublishedAt] = @LastPublishedAt, 87 | [DeletedAt] = NULL 88 | WHERE [BeatSaverKey] = @BeatSaverKey 89 | "; 90 | 91 | var sqlInsert = @" 92 | INSERT INTO [dbo].[BeatSaberSong] 93 | ([BeatSaverKey],[Hash],[Uploader],[Uploaded],[Difficulties],[Bpm],[LevelAuthorName],[SongAuthorName],[SongName],[SongSubName],[Name],[AutoMapper],[CreatedAt],[UpdatedAt],[LastPublishedAt]) 94 | VALUES (@BeatSaverKey, @Hash, @Uploader, @Uploaded, @Difficulties, @Bpm, @LevelAuthorName, @SongAuthorName, @SongName, @SongSubName, @Name, @AutoMapper, @CreatedAt, @UpdatedAt, @LastPublishedAt) 95 | "; 96 | 97 | bool inserted = false; 98 | using (var transaction = connection.BeginTransaction(IsolationLevel.Serializable)) 99 | { 100 | if (await connection.ExecuteAsync(sqlUpdate, song, transaction) == 0) 101 | { 102 | await connection.ExecuteAsync(sqlInsert, song, transaction); 103 | inserted = true; 104 | } 105 | transaction.Commit(); 106 | } 107 | return inserted; 108 | } 109 | } 110 | 111 | public async Task InsertOrUpdateSongRatings(BeatSaberSongRatings ratings) 112 | { 113 | using (var connection = GetConnection()) 114 | { 115 | var sqlUpdate = @" 116 | UPDATE [dbo].[BeatSaberSongRatings] 117 | SET 118 | [Upvotes] = @Upvotes, 119 | [Downvotes] = @Downvotes, 120 | [Score] = @Score, 121 | [UpdatedAt] = @UpdatedAt 122 | WHERE [BeatSaverKey] = @BeatSaverKey 123 | "; 124 | 125 | var sqlInsert = @" 126 | INSERT INTO [dbo].[BeatSaberSongRatings] 127 | ([BeatSaverKey],[Upvotes],[Downvotes],[Score],[UpdatedAt]) 128 | VALUES (@BeatSaverKey,@Upvotes,@Downvotes,@Score,@UpdatedAt) 129 | "; 130 | 131 | bool inserted = false; 132 | using (var transaction = connection.BeginTransaction(IsolationLevel.Serializable)) 133 | { 134 | if (await connection.ExecuteAsync(sqlUpdate, ratings, transaction) == 0) 135 | { 136 | await connection.ExecuteAsync(sqlInsert, ratings, transaction); 137 | inserted = true; 138 | } 139 | transaction.Commit(); 140 | } 141 | return inserted; 142 | } 143 | } 144 | 145 | public async Task MarkDeleted(int key, DateTime deletedAt) 146 | { 147 | using var connection = GetConnection(); 148 | var query = "UPDATE [dbo].[BeatSaberSong] SET DeletedAt = @DeletedAt WHERE BeatSaverKey = @BeatSaverKey"; 149 | 150 | await connection.ExecuteAsync(query, new { DeletedAt = deletedAt, BeatSaverKey = key}); 151 | } 152 | 153 | public async Task> GetSongCount() 154 | { 155 | using var connection = GetConnection(); 156 | var query = @" 157 | SELECT 158 | CAST(IIF(AutoMapper IS NOT NULL, '1', '0') AS bit) AS AutoMapper, 159 | Difficulties, 160 | COUNT(*) AS Count 161 | FROM [dbo].[BeatSaberSong] 162 | WHERE [DeletedAt] IS NULL 163 | GROUP BY AutoMapper, Difficulties"; 164 | 165 | var results = await connection.QueryAsync<(bool AutoMapper, SongDifficulties Difficulties, int Count)>(query); 166 | return results.ToList(); 167 | } 168 | 169 | public async Task GetLatestUpdatedAt(CancellationToken token) 170 | { 171 | using (var connection = GetConnection()) 172 | { 173 | var sqlStr = @"SELECT TOP 1 [UpdatedAt] FROM [dbo].[BeatSaberSong] ORDER BY [UpdatedAt] DESC"; 174 | 175 | return await connection.QueryFirstOrDefaultAsync(sqlStr); 176 | } 177 | } 178 | 179 | public async Task GetLatestDeletedAt(CancellationToken token) 180 | { 181 | using (var connection = GetConnection()) 182 | { 183 | var sqlStr = @"SELECT TOP 1 [DeletedAt] FROM [dbo].[BeatSaberSong] ORDER BY [DeletedAt] DESC"; 184 | 185 | return await connection.QueryFirstOrDefaultAsync(sqlStr); 186 | } 187 | } 188 | 189 | public async Task GetLatestScoreUpdatedAt(CancellationToken token) 190 | { 191 | using (var connection = GetConnection()) 192 | { 193 | var sqlStr = @"SELECT TOP 1 [UpdatedAt] FROM [dbo].[BeatSaberSongRatings] ORDER BY [UpdatedAt] DESC"; 194 | 195 | return await connection.QueryFirstOrDefaultAsync(sqlStr); 196 | } 197 | } 198 | } 199 | } 200 | -------------------------------------------------------------------------------- /src/BeatSaverMatcher.Common/Db/BeatSaberSongWithRatings.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | 3 | namespace BeatSaverMatcher.Common.Db 4 | { 5 | public class BeatSaberSongWithRatings 6 | { 7 | public string LevelAuthorName { get; set; } 8 | public string SongAuthorName { get; set; } 9 | public string SongName { get; set; } 10 | public string SongSubName { get; set; } 11 | public double Bpm { get; set; } 12 | 13 | public string Name { get; set; } 14 | 15 | public SongDifficulties Difficulties { get; set; } 16 | public string Uploader { get; set; } 17 | public DateTime Uploaded { get; set; } 18 | public byte[] Hash { get; set; } 19 | public int BeatSaverKey { get; set; } 20 | public string AutoMapper { get; set; } 21 | public DateTime? CreatedAt { get; set; } 22 | public DateTime? UpdatedAt { get; set; } 23 | public DateTime? LastPublishedAt { get; set; } 24 | public DateTime? DeletedAt { get; set; } 25 | public int? Upvotes { get; set; } 26 | public int? Downvotes { get; set; } 27 | public double? Score { get; set; } 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /src/BeatSaverMatcher.Common/Db/DbConfiguration.cs: -------------------------------------------------------------------------------- 1 | namespace BeatSaverMatcher.Common.Db 2 | { 3 | public class DbConfiguration 4 | { 5 | public string ConnectionString { get; set; } 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /src/BeatSaverMatcher.Common/Db/IBeatSaberSongRepository.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Threading; 4 | using System.Threading.Tasks; 5 | 6 | namespace BeatSaverMatcher.Common.Db 7 | { 8 | public interface IBeatSaberSongRepository 9 | { 10 | Task> GetMatches(string artistName, string trackName); 11 | Task GetLatestBeatSaverKey(); 12 | Task> GetSongCount(); 13 | Task GetLatestUpdatedAt(CancellationToken token); 14 | Task GetLatestScoreUpdatedAt(CancellationToken token); 15 | Task GetLatestDeletedAt(CancellationToken token); 16 | Task InsertOrUpdateSong(BeatSaberSong song); 17 | Task InsertOrUpdateSongRatings(BeatSaberSongRatings ratings); 18 | Task MarkDeleted(int key, DateTime deletedAt); 19 | } 20 | } -------------------------------------------------------------------------------- /src/BeatSaverMatcher.Common/Db/SongDifficulties.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | 3 | namespace BeatSaverMatcher.Common.Db 4 | { 5 | [Flags] 6 | public enum SongDifficulties 7 | { 8 | Easy = 1 << 0, 9 | Normal = 1 << 1, 10 | Hard = 1 << 2, 11 | Expert = 1 << 3, 12 | ExpertPlus = 1 << 4 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /src/BeatSaverMatcher.Common/Db/SqlServerRepository.cs: -------------------------------------------------------------------------------- 1 | using Microsoft.Data.SqlClient; 2 | using Microsoft.Extensions.Options; 3 | 4 | namespace BeatSaverMatcher.Common.Db 5 | { 6 | public abstract class SqlServerRepository 7 | { 8 | private readonly DbConfiguration _config; 9 | 10 | protected SqlServerRepository(IOptions options) 11 | { 12 | _config = options.Value; 13 | } 14 | 15 | protected SqlConnection GetConnection() 16 | { 17 | SqlConnection sqlConnection = new SqlConnection(_config.ConnectionString); 18 | sqlConnection.Open(); 19 | return sqlConnection; 20 | } 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /src/BeatSaverMatcher.Common/MetricsServer.cs: -------------------------------------------------------------------------------- 1 | using Microsoft.Extensions.Hosting; 2 | using Microsoft.Extensions.Logging; 3 | using Prometheus; 4 | using System.Net; 5 | using System.Threading; 6 | using System.Threading.Tasks; 7 | 8 | namespace BeatSaverMatcher.Common 9 | { 10 | public class MetricsServer : IHostedService 11 | { 12 | private readonly MetricServer _server; 13 | private readonly ILogger _logger; 14 | 15 | public MetricsServer(ILogger logger) 16 | { 17 | _server = new MetricServer(8080); 18 | _logger = logger; 19 | } 20 | 21 | public Task StartAsync(CancellationToken cancellationToken) 22 | { 23 | new Thread(Main).Start(); 24 | return Task.CompletedTask; 25 | } 26 | 27 | private void Main() 28 | { 29 | try 30 | { 31 | _server.Start(); 32 | } 33 | catch (HttpListenerException ex) 34 | { 35 | _logger.LogWarning(ex, "Metrics server could not be started"); 36 | } 37 | } 38 | 39 | public async Task StopAsync(CancellationToken cancellationToken) 40 | { 41 | await _server.StopAsync(); 42 | } 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /src/BeatSaverMatcher.Crawler/BeatSaverMatcher.Crawler.csproj: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | Exe 5 | net8.0 6 | enable 7 | 8 | 9 | 10 | 11 | 12 | PreserveNewest 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | PreserveNewest 36 | 37 | 38 | 39 | 40 | -------------------------------------------------------------------------------- /src/BeatSaverMatcher.Crawler/CrawlerHost.cs: -------------------------------------------------------------------------------- 1 | using BeatSaverMatcher.Common.BeatSaver; 2 | using BeatSaverMatcher.Common.Db; 3 | using Microsoft.Extensions.Hosting; 4 | using Microsoft.Extensions.Logging; 5 | using System; 6 | using System.Collections.Generic; 7 | using System.Globalization; 8 | using System.Linq; 9 | using System.Net; 10 | using System.Threading; 11 | using System.Threading.Tasks; 12 | 13 | namespace BeatSaverMatcher.Crawler 14 | { 15 | class CrawlerHost : BackgroundService 16 | { 17 | private readonly ILogger _logger; 18 | private readonly IBeatSaberSongRepository _songRepository; 19 | private readonly BeatSaverRepository _beatSaverRepository; 20 | 21 | public CrawlerHost(ILogger logger, 22 | IBeatSaberSongRepository songRepository, 23 | BeatSaverRepository beatSaverRepository) 24 | { 25 | _logger = logger; 26 | _songRepository = songRepository; 27 | _beatSaverRepository = beatSaverRepository; 28 | } 29 | 30 | protected override async Task ExecuteAsync(CancellationToken stoppingToken) 31 | { 32 | while (true) 33 | { 34 | stoppingToken.ThrowIfCancellationRequested(); 35 | await Scrape(stoppingToken); 36 | await Task.Delay(TimeSpan.FromMinutes(15), stoppingToken); 37 | } 38 | } 39 | 40 | private async Task Scrape(CancellationToken token) 41 | { 42 | DateTime lastUpdatedAt = await _songRepository.GetLatestUpdatedAt(token) ?? DateTime.UnixEpoch; 43 | 44 | _logger.LogInformation("Starting update crawl at {Date}", lastUpdatedAt); 45 | 46 | while (true) 47 | { 48 | token.ThrowIfCancellationRequested(); 49 | 50 | IList songs; 51 | try 52 | { 53 | songs = await _beatSaverRepository.GetSongsUpdatedAfter(lastUpdatedAt, token); 54 | _logger.LogInformation("Got {SongCount} songs from BeatSaver", songs.Count); 55 | } 56 | catch (WebException wex) 57 | { 58 | if (wex.Response is HttpWebResponse response) 59 | { 60 | if ((int)response.StatusCode < 200 || (int)response.StatusCode >= 300) 61 | { 62 | _logger.LogWarning("Error {StatusCode} - {StatusDescription} while scraping", response.StatusCode, response.StatusDescription); 63 | break; 64 | } 65 | 66 | _logger.LogWarning(wex, "Unknown Exception while fetching"); 67 | break; 68 | } 69 | else 70 | { 71 | _logger.LogWarning(wex, "Unknown WebException"); 72 | break; 73 | } 74 | } 75 | catch (Exception ex) 76 | { 77 | _logger.LogWarning(ex, "Unknown Exception while fetching"); 78 | break; 79 | } 80 | 81 | if(songs.Count > 0 && songs.Last().UpdatedAt <= lastUpdatedAt) 82 | { 83 | _logger.LogError("Beatsaver returned map before or at last update date, this would cause an endless loop!"); 84 | break; 85 | } 86 | 87 | try 88 | { 89 | foreach (var song in songs.Reverse()) 90 | { 91 | token.ThrowIfCancellationRequested(); 92 | await InsertOrUpdate(song, token); 93 | 94 | if (song.UpdatedAt < lastUpdatedAt) 95 | { 96 | throw new Exception("Song list must be sorted ascending by UpdatedAt"); 97 | } 98 | else 99 | { 100 | lastUpdatedAt = song.UpdatedAt; 101 | } 102 | } 103 | } 104 | catch (Exception ex) 105 | { 106 | _logger.LogWarning(ex, "Unknown Exception"); 107 | break; 108 | } 109 | 110 | if (songs.Count == 0) 111 | { 112 | break; 113 | } 114 | } 115 | 116 | _logger.LogInformation("Scrape done."); 117 | } 118 | 119 | private async Task InsertOrUpdate(BeatSaverSong song, CancellationToken cancellationToken) 120 | { 121 | var mappedSong = MapSong(song); 122 | if (mappedSong != null) 123 | { 124 | var inserted = await _songRepository.InsertOrUpdateSong(mappedSong); 125 | 126 | _logger.LogInformation("{Action} song {Key}: {SongName}", inserted ? "Inserted" : "Updated", mappedSong.BeatSaverKey.ToString("x"), mappedSong.Name); 127 | } 128 | } 129 | 130 | private BeatSaberSong MapSong(BeatSaverSong song) 131 | { 132 | var currentVersion = song.Versions 133 | .Where(x => x.State == "Published") 134 | .OrderByDescending(x => x.CreatedAt) 135 | .FirstOrDefault(); 136 | 137 | if (currentVersion == null) 138 | return null; 139 | 140 | return new BeatSaberSong 141 | { 142 | LevelAuthorName = song.Metadata.LevelAuthorName, 143 | SongAuthorName = song.Metadata.SongAuthorName, 144 | SongName = song.Metadata.SongName, 145 | SongSubName = song.Metadata.SongSubName, 146 | Bpm = song.Metadata.Bpm, 147 | Name = song.Name, 148 | AutoMapper = song.Automapper ? song.Metadata.LevelAuthorName : null, 149 | Difficulties = BeatSaverUtils.MapDifficulties(currentVersion.Diffs), 150 | Uploader = song.Uploader.Name, 151 | Uploaded = song.Uploaded, 152 | CreatedAt = song.CreatedAt, 153 | UpdatedAt = song.UpdatedAt, 154 | LastPublishedAt = song.LastPublishedAt, 155 | Hash = BeatSaverUtils.MapHash(currentVersion.Hash), 156 | BeatSaverKey = int.Parse(song.Id, NumberStyles.HexNumber) 157 | }; 158 | } 159 | } 160 | } 161 | -------------------------------------------------------------------------------- /src/BeatSaverMatcher.Crawler/DeleteCrawlerHost.cs: -------------------------------------------------------------------------------- 1 | using BeatSaverMatcher.Common.BeatSaver; 2 | using BeatSaverMatcher.Common.Db; 3 | using Microsoft.Extensions.Hosting; 4 | using Microsoft.Extensions.Logging; 5 | using System; 6 | using System.Collections.Generic; 7 | using System.Globalization; 8 | using System.Linq; 9 | using System.Net; 10 | using System.Threading; 11 | using System.Threading.Tasks; 12 | 13 | namespace BeatSaverMatcher.Crawler 14 | { 15 | class DeleteCrawlerHost : BackgroundService 16 | { 17 | private readonly ILogger _logger; 18 | private readonly IBeatSaberSongRepository _songRepository; 19 | private readonly BeatSaverRepository _beatSaverRepository; 20 | 21 | public DeleteCrawlerHost(ILogger logger, 22 | IBeatSaberSongRepository songRepository, 23 | BeatSaverRepository beatSaverRepository) 24 | { 25 | _logger = logger; 26 | _songRepository = songRepository; 27 | _beatSaverRepository = beatSaverRepository; 28 | } 29 | 30 | protected override async Task ExecuteAsync(CancellationToken stoppingToken) 31 | { 32 | while (true) 33 | { 34 | stoppingToken.ThrowIfCancellationRequested(); 35 | await Scrape(stoppingToken); 36 | await Task.Delay(TimeSpan.FromMinutes(15), stoppingToken); 37 | } 38 | } 39 | 40 | private async Task Scrape(CancellationToken token) 41 | { 42 | var latestDeleted = await _songRepository.GetLatestDeletedAt(token) ?? DateTime.UnixEpoch; 43 | _logger.LogInformation("Starting delete crawl at {Date}", latestDeleted); 44 | 45 | while (true) 46 | { 47 | token.ThrowIfCancellationRequested(); 48 | 49 | IList songs; 50 | try 51 | { 52 | songs = await _beatSaverRepository.GetSongsDeletedAfter(latestDeleted, token); 53 | _logger.LogInformation("Got {SongCount} deleted songs from BeatSaver", songs.Count); 54 | } 55 | catch (WebException wex) 56 | { 57 | if (wex.Response is HttpWebResponse response) 58 | { 59 | if ((int)response.StatusCode < 200 || (int)response.StatusCode >= 300) 60 | { 61 | _logger.LogWarning("Error {StatusCode} - {StatusDescription} while scraping", response.StatusCode, response.StatusDescription); 62 | break; 63 | } 64 | 65 | _logger.LogWarning(wex, "Unknown Exception while fetching"); 66 | break; 67 | } 68 | else 69 | { 70 | _logger.LogWarning(wex, "Unknown WebException"); 71 | break; 72 | } 73 | } 74 | catch (Exception ex) 75 | { 76 | _logger.LogWarning(ex, "Unknown Exception while fetching"); 77 | break; 78 | } 79 | 80 | if (songs.Count > 0 && songs.Last().DeletedAt <= latestDeleted) 81 | { 82 | _logger.LogError("Beatsaver returned map before or at last update date, this would cause an endless loop!"); 83 | break; 84 | } 85 | 86 | try 87 | { 88 | foreach (var song in songs.Reverse()) 89 | { 90 | token.ThrowIfCancellationRequested(); 91 | await _songRepository.MarkDeleted(int.Parse(song.Id, NumberStyles.HexNumber), song.DeletedAt); 92 | 93 | if (song.DeletedAt < latestDeleted) 94 | { 95 | throw new Exception("Song list must be sorted ascending by DeletedAt"); 96 | } 97 | else 98 | { 99 | latestDeleted = song.DeletedAt; 100 | } 101 | } 102 | } 103 | catch (Exception ex) 104 | { 105 | _logger.LogWarning(ex, "Unknown Exception"); 106 | break; 107 | } 108 | 109 | if (songs.Count == 0) 110 | { 111 | break; 112 | } 113 | } 114 | _logger.LogInformation("Delete scrape done."); 115 | } 116 | } 117 | } 118 | -------------------------------------------------------------------------------- /src/BeatSaverMatcher.Crawler/MetricsScrapeHost.cs: -------------------------------------------------------------------------------- 1 | using BeatSaverMatcher.Common.Db; 2 | using Microsoft.Extensions.Hosting; 3 | using Prometheus; 4 | using System; 5 | using System.Linq; 6 | using System.Threading; 7 | using System.Threading.Tasks; 8 | 9 | namespace BeatSaverMatcher.Crawler 10 | { 11 | internal class MetricsScrapeHost : IHostedService 12 | { 13 | private readonly Gauge _totalBeatmaps = Metrics.CreateGauge("beatsaver_beatmaps_total", "Total number of beatmaps", new GaugeConfiguration { SuppressInitialValue = true, LabelNames = new[] { "automapper" } }); 14 | private readonly Gauge _totalBeatmapsPerDifficulty = Metrics.CreateGauge("beatsaver_beatmaps_per_difficulty_total", "Total number of beatmaps by difficulty", new GaugeConfiguration { SuppressInitialValue = true, LabelNames = new[] { "automapper", "difficulty" } }); 15 | private readonly Counter _currentSongId = Metrics.CreateCounter("beatsaver_latest_song_id", "ID of the song that was most recently scraped", new CounterConfiguration { SuppressInitialValue = true }); 16 | 17 | private readonly IBeatSaberSongRepository _songRepository; 18 | 19 | public MetricsScrapeHost(IBeatSaberSongRepository songRepository) 20 | { 21 | _songRepository = songRepository; 22 | } 23 | 24 | public Task StartAsync(CancellationToken cancellationToken) 25 | { 26 | Metrics.DefaultRegistry.AddBeforeCollectCallback(GetMetrics); 27 | return Task.CompletedTask; 28 | } 29 | 30 | private async Task GetMetrics(CancellationToken cancellationToken) 31 | { 32 | var lastScraped = await _songRepository.GetLatestBeatSaverKey() ?? 0; 33 | _currentSongId.IncTo(lastScraped); 34 | 35 | var songGroups = await _songRepository.GetSongCount(); 36 | 37 | _totalBeatmaps.WithLabels("true").Set(songGroups.Where(x => x.AutoMapper).Sum(x => x.Count)); 38 | _totalBeatmaps.WithLabels("false").Set(songGroups.Where(x => !x.AutoMapper).Sum(x => x.Count)); 39 | 40 | var difficulties = Enum.GetValues(); 41 | foreach (var difficulty in difficulties) 42 | { 43 | var difficultyBeatmaps = songGroups.Where(x => x.Difficulties.HasFlag(difficulty)).ToList(); 44 | _totalBeatmapsPerDifficulty.WithLabels("true", difficulty.ToString()).Set(difficultyBeatmaps.Where(x => x.AutoMapper).Sum(x => x.Count)); 45 | _totalBeatmapsPerDifficulty.WithLabels("false", difficulty.ToString()).Set(difficultyBeatmaps.Where(x => !x.AutoMapper).Sum(x => x.Count)); 46 | } 47 | } 48 | 49 | public Task StopAsync(CancellationToken cancellationToken) 50 | { 51 | return Task.CompletedTask; 52 | } 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /src/BeatSaverMatcher.Crawler/Program.cs: -------------------------------------------------------------------------------- 1 | using BeatSaverMatcher.Common; 2 | using BeatSaverMatcher.Common.BeatSaver; 3 | using BeatSaverMatcher.Common.Db; 4 | using Microsoft.Extensions.Configuration; 5 | using Microsoft.Extensions.DependencyInjection; 6 | using Microsoft.Extensions.Hosting; 7 | using Microsoft.Extensions.Logging; 8 | using Serilog; 9 | using System.Threading.Tasks; 10 | 11 | namespace BeatSaverMatcher.Crawler 12 | { 13 | class Program 14 | { 15 | static async Task Main(string[] args) 16 | { 17 | await Host.CreateDefaultBuilder(args) 18 | .ConfigureAppConfiguration(ConfigureAppConfiguration) 19 | .ConfigureServices(ConfigureServices) 20 | .ConfigureLogging(ConfigureLogging) 21 | .RunConsoleAsync(); 22 | } 23 | 24 | private static void ConfigureLogging(HostBuilderContext hostContext, ILoggingBuilder loggingBuilder) 25 | { 26 | loggingBuilder.ClearProviders(); 27 | 28 | Log.Logger = new LoggerConfiguration() 29 | .ReadFrom.Configuration(hostContext.Configuration) 30 | .CreateLogger(); 31 | loggingBuilder.AddSerilog(Log.Logger); 32 | } 33 | 34 | private static void ConfigureAppConfiguration(HostBuilderContext ctx, IConfigurationBuilder configBuilder) 35 | { 36 | configBuilder 37 | .AddJsonFile("./config/appSettings.json", optional: true) 38 | .AddJsonFile("./config/logging.json", optional: true) 39 | .AddEnvironmentVariables() 40 | .Build(); 41 | } 42 | 43 | private static void ConfigureServices(HostBuilderContext ctx, IServiceCollection services) 44 | { 45 | services.Configure(ctx.Configuration); 46 | services.AddTransient(); 47 | services.AddTransient(); 48 | services.AddHostedService(); 49 | services.AddHostedService(); 50 | services.AddHostedService(); 51 | services.AddHostedService(); 52 | services.AddHostedService(); 53 | 54 | 55 | var hasRedisConnection = !string.IsNullOrEmpty(ctx.Configuration["RedisConnection"]); 56 | 57 | if (hasRedisConnection) 58 | { 59 | services.AddStackExchangeRedisCache(options => 60 | { 61 | options.Configuration = ctx.Configuration["RedisConnection"]; 62 | options.InstanceName = "BeatSaverMatcher"; 63 | }); 64 | } 65 | else 66 | { 67 | services.AddDistributedMemoryCache(); 68 | } 69 | } 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /src/BeatSaverMatcher.Crawler/RatingsCrawlerHost.cs: -------------------------------------------------------------------------------- 1 | using BeatSaverMatcher.Common.BeatSaver; 2 | using BeatSaverMatcher.Common.Db; 3 | using Microsoft.Extensions.Hosting; 4 | using Microsoft.Extensions.Logging; 5 | using System; 6 | using System.Threading; 7 | using System.Threading.Tasks; 8 | 9 | namespace BeatSaverMatcher.Crawler 10 | { 11 | class RatingsCrawlerHost : BackgroundService 12 | { 13 | private readonly ILogger _logger; 14 | private readonly IBeatSaberSongRepository _songRepository; 15 | private readonly BeatSaverRepository _beatSaverRepository; 16 | 17 | public RatingsCrawlerHost(ILogger logger, 18 | IBeatSaberSongRepository songRepository, 19 | BeatSaverRepository beatSaverRepository) 20 | { 21 | _logger = logger; 22 | _songRepository = songRepository; 23 | _beatSaverRepository = beatSaverRepository; 24 | } 25 | 26 | protected override async Task ExecuteAsync(CancellationToken stoppingToken) 27 | { 28 | while (true) 29 | { 30 | stoppingToken.ThrowIfCancellationRequested(); 31 | await Scrape(stoppingToken); 32 | await Task.Delay(TimeSpan.FromMinutes(15), stoppingToken); 33 | } 34 | } 35 | 36 | private async Task Scrape(CancellationToken token) 37 | { 38 | var lastUpdatedAt = await _songRepository.GetLatestScoreUpdatedAt(token) ?? DateTime.UnixEpoch; 39 | _logger.LogInformation("Starting ratings crawl at {Key}", lastUpdatedAt); 40 | 41 | var now = DateTime.UtcNow; 42 | var scores = await _beatSaverRepository.GetScoresAfter(lastUpdatedAt, token); 43 | 44 | foreach (var score in scores) 45 | { 46 | token.ThrowIfCancellationRequested(); 47 | var rating = new BeatSaberSongRatings 48 | { 49 | BeatSaverKey = score.MapId, 50 | Score = score.Score, 51 | Downvotes = score.Downvotes, 52 | Upvotes = score.Upvotes, 53 | UpdatedAt = now // technically this isn't 100% safe. if this crashes, we miss some votes but whatever 54 | }; 55 | 56 | var inserted = await _songRepository.InsertOrUpdateSongRatings(rating); 57 | _logger.LogInformation("{Action} score for 0x{Key}", inserted ? "Inserted" : "Updated", score.MapId.ToString("x")); 58 | } 59 | _logger.LogInformation("Ratings scrape done."); 60 | } 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /src/BeatSaverMatcher.Crawler/config/logging.json: -------------------------------------------------------------------------------- 1 | { 2 | "Serilog": { 3 | "Using": [ "Serilog.Sinks.Console" ], 4 | "MinimumLevel": { 5 | "Default": "Information" 6 | }, 7 | "WriteTo": [ 8 | { "Name": "Console" } 9 | ], 10 | "Enrich": [ "FromLogContext" ] 11 | } 12 | } -------------------------------------------------------------------------------- /src/BeatSaverMatcher.Web/BeatSaverMatcher.Web.csproj: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | net8.0 5 | true 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | PreserveNewest 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | %(DistFiles.Identity) 47 | PreserveNewest 48 | 49 | 50 | 51 | 52 | 53 | -------------------------------------------------------------------------------- /src/BeatSaverMatcher.Web/Controllers/MatchesController.cs: -------------------------------------------------------------------------------- 1 | using BeatSaverMatcher.Web.Result; 2 | using Microsoft.AspNetCore.Mvc; 3 | using Prometheus; 4 | 5 | namespace BeatSaverMatcher.Web.Controllers 6 | { 7 | [Route("api/[controller]")] 8 | [ApiController] 9 | public class MatchesController : ControllerBase 10 | { 11 | private readonly WorkItemStore _itemStore; 12 | private readonly Counter _startMatchCounter; 13 | 14 | public MatchesController(WorkItemStore itemStore) 15 | { 16 | _itemStore = itemStore; 17 | _startMatchCounter = Metrics.CreateCounter("beatsaver_start_match_count", "number of times match was started"); 18 | } 19 | 20 | [HttpPost("{playlistId}")] 21 | public ActionResult StartMatch([FromRoute] string playlistId) 22 | { 23 | if (!_itemStore.Enqueue(playlistId)) 24 | return Conflict(); 25 | 26 | _startMatchCounter.Inc(); 27 | return Ok(); 28 | } 29 | 30 | [HttpGet("{playlistId}")] 31 | public WorkResultItem GetMatchState([FromRoute] string playlistId) 32 | { 33 | return _itemStore.Get(playlistId); 34 | } 35 | } 36 | } -------------------------------------------------------------------------------- /src/BeatSaverMatcher.Web/Controllers/ModSaberPlaylistController.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Linq; 3 | using System.Net.Http; 4 | using System.Net.Http.Headers; 5 | using System.Threading; 6 | using System.Threading.Tasks; 7 | using BeatSaverMatcher.Api; 8 | using BeatSaverMatcher.Api.Spotify; 9 | using BeatSaverMatcher.Api.Tidal; 10 | using BeatSaverMatcher.Web.Models; 11 | using BeatSaverMatcher.Web.Result; 12 | using Microsoft.AspNetCore.Mvc; 13 | using Microsoft.Extensions.Logging; 14 | 15 | namespace BeatSaverMatcher.Web.Controllers 16 | { 17 | [Route("api/[controller]")] 18 | [ApiController] 19 | public class ModSaberPlaylistController : ControllerBase 20 | { 21 | private readonly WorkItemStore _itemStore; 22 | private readonly SpotifyRepository _spotifyClient; 23 | private readonly TidalClient _tidalClient; 24 | private readonly ILogger _logger; 25 | private readonly IHttpClientFactory _httpClientFactory; 26 | 27 | public ModSaberPlaylistController(WorkItemStore itemStore, SpotifyRepository spotifyClient, TidalClient tidalClient, ILogger logger, IHttpClientFactory httpClientFactory) 28 | { 29 | _itemStore = itemStore; 30 | _spotifyClient = spotifyClient; 31 | _tidalClient = tidalClient; 32 | _logger = logger; 33 | _httpClientFactory = httpClientFactory; 34 | } 35 | 36 | [HttpGet("{playlistId}.bplist")] 37 | [HttpGet("{keys}/{playlistId}.bplist")] 38 | public async Task> GetMatchesAsPlaylistDownload([FromRoute] string playlistId, [FromRoute] string keys, CancellationToken cancellationToken) 39 | { 40 | var workItem = _itemStore.Get(playlistId); 41 | if (workItem == null) 42 | return NotFound(); 43 | 44 | if (workItem.State != SongMatchState.Finished) 45 | return BadRequest(); 46 | 47 | IMusicServiceApi client = Guid.TryParse(playlistId, out _) ? _tidalClient : _spotifyClient; 48 | 49 | var playlist = await client.GetPlaylist(playlistId, cancellationToken); 50 | 51 | var beatmaps = workItem.Result.Matches.SelectMany(x => x.BeatMaps).ToList(); 52 | if (keys != null) 53 | { 54 | var keysList = keys.Split(','); 55 | beatmaps = beatmaps.Where(x => keysList.Contains(x.BeatSaverKey.ToString("x"))).ToList(); 56 | } 57 | var header = new ContentDispositionHeaderValue("attachment") { FileName = playlist.Id + ".bplist", FileNameStar = playlist.Name + ".bplist" }; 58 | Response.Headers.ContentDisposition = header.ToString(); 59 | 60 | return new ModSaberPlaylist 61 | { 62 | PlaylistTitle = playlist.Name, 63 | PlaylistAuthor = (playlist.OwnerName ?? "") + " using https://github.com/patagonaa/BeatSaverMatcher", 64 | Image = await GetImage(playlist, cancellationToken), 65 | Songs = beatmaps.Select(x => new ModSaberSong 66 | { 67 | Hash = string.Join("", x.Hash.Select(x => x.ToString("x2"))), 68 | Key = x.BeatSaverKey.ToString("x"), 69 | SongName = x.Name, 70 | Uploader = x.Uploader 71 | }).ToList() 72 | }; 73 | } 74 | 75 | private async Task GetImage(Playlist playlist, CancellationToken cancellationToken) 76 | { 77 | try 78 | { 79 | var imageUrl = playlist.ImageUrl; 80 | if (imageUrl == null) 81 | { 82 | return null; 83 | } 84 | 85 | var imageBytes = await _httpClientFactory.CreateClient().GetByteArrayAsync(imageUrl, cancellationToken); 86 | 87 | return Convert.ToBase64String(imageBytes); 88 | } 89 | catch (Exception ex) 90 | { 91 | _logger.LogWarning(ex, "Couldn't get image for playlist"); 92 | return null; 93 | } 94 | } 95 | } 96 | } 97 | -------------------------------------------------------------------------------- /src/BeatSaverMatcher.Web/MatchCleanupWorker.cs: -------------------------------------------------------------------------------- 1 | using BeatSaverMatcher.Web.Result; 2 | using Microsoft.Extensions.Hosting; 3 | using Microsoft.Extensions.Logging; 4 | using System; 5 | using System.Threading; 6 | using System.Threading.Tasks; 7 | 8 | namespace BeatSaverMatcher.Web 9 | { 10 | public class MatchCleanupWorker : BackgroundService 11 | { 12 | private readonly WorkItemStore _itemStore; 13 | private readonly ILogger _logger; 14 | 15 | public MatchCleanupWorker(WorkItemStore itemStore, ILogger logger) 16 | { 17 | _itemStore = itemStore; 18 | _logger = logger; 19 | } 20 | 21 | protected override async Task ExecuteAsync(CancellationToken stoppingToken) 22 | { 23 | while (true) 24 | { 25 | stoppingToken.ThrowIfCancellationRequested(); 26 | try 27 | { 28 | _itemStore.DoCleanup(); 29 | _logger.LogDebug("Cleanup successful!"); 30 | } 31 | catch (Exception ex) 32 | { 33 | _logger.LogError(ex, "Error while doing cleanup!"); 34 | } 35 | 36 | await Task.Delay(TimeSpan.FromMinutes(10), stoppingToken); 37 | } 38 | } 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /src/BeatSaverMatcher.Web/MatchingService.cs: -------------------------------------------------------------------------------- 1 | using BeatSaverMatcher.Api; 2 | using BeatSaverMatcher.Api.Spotify; 3 | using BeatSaverMatcher.Api.Tidal; 4 | using BeatSaverMatcher.Common.Db; 5 | using BeatSaverMatcher.Web.Models; 6 | using BeatSaverMatcher.Web.Result; 7 | using Microsoft.Extensions.Logging; 8 | using System; 9 | using System.Collections.Concurrent; 10 | using System.Collections.Generic; 11 | using System.Linq; 12 | using System.Net; 13 | using System.Text; 14 | using System.Threading; 15 | using System.Threading.Tasks; 16 | 17 | namespace BeatSaverMatcher.Web 18 | { 19 | public class MatchingService 20 | { 21 | private readonly SpotifyRepository _spotifyClient; 22 | private readonly TidalClient _tidalClient; 23 | private readonly IBeatSaberSongRepository _songRepository; 24 | private readonly ILogger _logger; 25 | 26 | public MatchingService(SpotifyRepository spotifyClient, TidalClient tidalClient, IBeatSaberSongRepository songRepository, ILogger logger, ILogger logger1) 27 | { 28 | _spotifyClient = spotifyClient; 29 | _tidalClient = tidalClient; 30 | _songRepository = songRepository; 31 | _logger = logger; 32 | } 33 | 34 | public async Task GetMatches(WorkResultItem item, CancellationToken cancellationToken) 35 | { 36 | try 37 | { 38 | _logger.LogInformation("Loading songs for playlist {PlaylistId}", item.PlaylistId); 39 | item.State = SongMatchState.LoadingPlaylistSongs; 40 | item.ItemsTotal = 1; 41 | item.ItemsProcessed = 0; 42 | 43 | IList tracks; 44 | 45 | IMusicServiceApi client = Guid.TryParse(item.PlaylistId, out _) ? _tidalClient : _spotifyClient; 46 | 47 | try 48 | { 49 | var progressCallback = (int current, int? total) => { item.ItemsProcessed = current; item.ItemsTotal = total; }; 50 | tracks = (await client.GetTracksForPlaylist(item.PlaylistId, progressCallback, cancellationToken)) 51 | .Where(x => x != null) 52 | .ToList(); 53 | } 54 | catch (APIException aex) 55 | { 56 | if (aex.StatusCode == HttpStatusCode.NotFound) 57 | { 58 | throw new MatchingException("Error 404 while loading playlist: Not Found (is it public?)", aex); 59 | } 60 | else 61 | { 62 | var sb = new StringBuilder(); 63 | sb.Append("Error "); 64 | if (aex.StatusCode != null) 65 | sb.Append($"{aex.StatusCode.Value} "); 66 | sb.Append("while loading playlist"); 67 | if (aex.Message != null) 68 | sb.Append($": {aex.Message}"); 69 | throw new MatchingException(sb.ToString(), aex); 70 | } 71 | } 72 | 73 | _logger.LogInformation("Finding beatmaps"); 74 | item.State = SongMatchState.SearchingBeatMaps; 75 | 76 | var matches = new ConcurrentBag(); 77 | 78 | item.ItemsTotal = tracks.Count; 79 | item.ItemsProcessed = 0; 80 | 81 | var processed = 0; 82 | await Parallel.ForEachAsync( 83 | tracks, 84 | new ParallelOptions { CancellationToken = cancellationToken, MaxDegreeOfParallelism = 8 }, 85 | async (track, _) => 86 | { 87 | var match = new SongMatch 88 | { 89 | PlaylistArtist = string.Join(", ", track.Artists), 90 | PlaylistTitle = track.Name 91 | }; 92 | 93 | var beatmaps = new List(); 94 | 95 | foreach (var artist in track.Artists) 96 | { 97 | try 98 | { 99 | var directMatches = await _songRepository.GetMatches(artist, track.Name); 100 | foreach (var beatmap in directMatches) 101 | { 102 | beatmaps.Add(beatmap); 103 | } 104 | } 105 | catch (Exception ex) 106 | { 107 | _logger.LogWarning(ex, "Error while searching song in DB: {ArtistName} - {SongName}", artist, track.Name); 108 | continue; 109 | } 110 | } 111 | 112 | if (beatmaps.Any()) 113 | { 114 | match.DbBeatMaps = beatmaps.GroupBy(x => x.BeatSaverKey).Select(x => x.First()).ToList(); 115 | matches.Add(match); 116 | } 117 | Interlocked.Increment(ref processed); 118 | 119 | item.ItemsProcessed = processed; 120 | }); 121 | item.ItemsProcessed = processed; 122 | 123 | _logger.LogInformation("Found {MatchCount} / {TrackCount} songs!", matches.Count, tracks.Count); 124 | 125 | foreach (var match in matches) 126 | { 127 | cancellationToken.ThrowIfCancellationRequested(); 128 | var foundBeatMaps = new List(); 129 | foreach (var dbBeatmap in match.DbBeatMaps) 130 | { 131 | try 132 | { 133 | foundBeatMaps.Add(new BeatSaberSongViewModel 134 | { 135 | BeatSaverKey = dbBeatmap.BeatSaverKey, 136 | Hash = dbBeatmap.Hash, 137 | Uploader = dbBeatmap.Uploader, 138 | Uploaded = dbBeatmap.Uploaded, 139 | Difficulties = dbBeatmap.Difficulties, 140 | Bpm = dbBeatmap.Bpm, 141 | LevelAuthorName = dbBeatmap.LevelAuthorName, 142 | SongAuthorName = dbBeatmap.SongAuthorName, 143 | SongName = dbBeatmap.SongName, 144 | SongSubName = dbBeatmap.SongSubName, 145 | Name = dbBeatmap.Name, 146 | Rating = dbBeatmap.Score, 147 | UpVotes = dbBeatmap.Upvotes, 148 | DownVotes = dbBeatmap.Downvotes 149 | }); 150 | item.ItemsProcessed++; 151 | } 152 | catch (Exception ex) 153 | { 154 | _logger.LogError(ex, "Error while mapping beatmap 0x{BeatMapKey}", dbBeatmap.BeatSaverKey.ToString("x")); 155 | } 156 | } 157 | match.BeatMaps = foundBeatMaps.OrderByDescending(x => x.Rating ?? 0).ToList(); 158 | } 159 | 160 | item.Result = new SongMatchResult 161 | { 162 | TotalPlaylistSongs = tracks.Count, 163 | MatchedPlaylistSongs = matches.Count, 164 | Matches = matches.ToList() 165 | }; 166 | _logger.LogInformation("Done."); 167 | item.State = SongMatchState.Finished; 168 | } 169 | catch (MatchingException ex) 170 | { 171 | item.State = SongMatchState.Error; 172 | item.ErrorMessage = ex.Message; 173 | _logger.LogError(ex, "Error while matching!"); 174 | } 175 | catch (Exception ex) 176 | { 177 | item.State = SongMatchState.Error; 178 | _logger.LogError(ex, "Error while matching!"); 179 | } 180 | } 181 | 182 | private class MatchingException : Exception 183 | { 184 | public MatchingException(string message, Exception inner = null) 185 | : base(message, inner) 186 | { 187 | } 188 | } 189 | } 190 | } 191 | -------------------------------------------------------------------------------- /src/BeatSaverMatcher.Web/Models/BeatSaberSongViewModel.cs: -------------------------------------------------------------------------------- 1 | using BeatSaverMatcher.Common.Db; 2 | using System; 3 | 4 | namespace BeatSaverMatcher.Web.Models 5 | { 6 | public class BeatSaberSongViewModel 7 | { 8 | public string LevelAuthorName { get; set; } 9 | public string SongAuthorName { get; set; } 10 | public string SongName { get; set; } 11 | public string SongSubName { get; set; } 12 | public double Bpm { get; set; } 13 | 14 | public string Name { get; set; } 15 | 16 | public SongDifficulties Difficulties { get; set; } 17 | public string Uploader { get; set; } 18 | public DateTime Uploaded { get; set; } 19 | public byte[] Hash { get; set; } 20 | public int BeatSaverKey { get; set; } 21 | public double? Rating { get; set; } 22 | public int? UpVotes { get; set; } 23 | public int? DownVotes { get; set; } 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /src/BeatSaverMatcher.Web/Models/ModSaberPlaylist.cs: -------------------------------------------------------------------------------- 1 | using System.Collections.Generic; 2 | 3 | namespace BeatSaverMatcher.Web.Models 4 | { 5 | public class ModSaberPlaylist 6 | { 7 | public string PlaylistTitle { get; set; } 8 | public string PlaylistAuthor { get; set; } 9 | public string Image { get; set; } 10 | public IList Songs { get; set; } 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /src/BeatSaverMatcher.Web/Models/ModSaberSong.cs: -------------------------------------------------------------------------------- 1 | namespace BeatSaverMatcher.Web.Models 2 | { 3 | public class ModSaberSong 4 | { 5 | public string Key { get; set; } 6 | public string Hash { get; set; } 7 | public string SongName { get; set; } 8 | public string Uploader { get; set; } 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /src/BeatSaverMatcher.Web/Models/SongMatch.cs: -------------------------------------------------------------------------------- 1 | using BeatSaverMatcher.Common.Db; 2 | using System.Collections.Generic; 3 | using System.Text.Json.Serialization; 4 | 5 | namespace BeatSaverMatcher.Web.Models 6 | { 7 | public class SongMatch 8 | { 9 | public string PlaylistArtist { get; set; } 10 | public string PlaylistTitle { get; set; } 11 | [JsonIgnore] 12 | public IList DbBeatMaps { get; set; } 13 | public IList BeatMaps { get; set; } 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /src/BeatSaverMatcher.Web/Models/SongMatchResult.cs: -------------------------------------------------------------------------------- 1 | using System.Collections.Generic; 2 | 3 | namespace BeatSaverMatcher.Web.Models 4 | { 5 | public class SongMatchResult 6 | { 7 | public int MatchedPlaylistSongs { get; set; } 8 | public int TotalPlaylistSongs { get; set; } 9 | public IList Matches { get; set; } 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /src/BeatSaverMatcher.Web/Program.cs: -------------------------------------------------------------------------------- 1 | using Microsoft.AspNetCore.Hosting; 2 | using Microsoft.Extensions.Configuration; 3 | using Microsoft.Extensions.Hosting; 4 | using Microsoft.Extensions.Logging; 5 | using Serilog; 6 | 7 | namespace BeatSaverMatcher.Web 8 | { 9 | public class Program 10 | { 11 | public static void Main(string[] args) 12 | { 13 | CreateHostBuilder(args).Build().Run(); 14 | } 15 | 16 | public static IHostBuilder CreateHostBuilder(string[] args) 17 | { 18 | return Host.CreateDefaultBuilder(args) 19 | .ConfigureAppConfiguration(config => 20 | config 21 | .AddJsonFile("./config/appSettings.json", optional: true) 22 | .AddJsonFile("./config/logging.json", optional: true) 23 | .AddEnvironmentVariables()) 24 | .ConfigureLogging(ConfigureLogging) 25 | .ConfigureWebHostDefaults(webBuilder => webBuilder.UseStartup()); 26 | } 27 | 28 | private static void ConfigureLogging(HostBuilderContext hostContext, ILoggingBuilder loggingBuilder) 29 | { 30 | loggingBuilder.ClearProviders(); 31 | 32 | Log.Logger = new LoggerConfiguration() 33 | .ReadFrom.Configuration(hostContext.Configuration) 34 | .CreateLogger(); 35 | loggingBuilder.AddSerilog(Log.Logger); 36 | } 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /src/BeatSaverMatcher.Web/Properties/launchSettings.json: -------------------------------------------------------------------------------- 1 | { 2 | "iisSettings": { 3 | "windowsAuthentication": false, 4 | "anonymousAuthentication": true, 5 | "iisExpress": { 6 | "applicationUrl": "http://localhost:23355", 7 | "sslPort": 0 8 | } 9 | }, 10 | "profiles": { 11 | "IIS Express": { 12 | "commandName": "IISExpress", 13 | "launchBrowser": true, 14 | "environmentVariables": { 15 | "ASPNETCORE_ENVIRONMENT": "Development" 16 | } 17 | }, 18 | "BeatSaverMatcher.Web": { 19 | "commandName": "Project", 20 | "launchBrowser": true, 21 | "applicationUrl": "http://localhost:5000", 22 | "environmentVariables": { 23 | "ASPNETCORE_ENVIRONMENT": "Development" 24 | } 25 | } 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /src/BeatSaverMatcher.Web/Result/SongMatchState.cs: -------------------------------------------------------------------------------- 1 | namespace BeatSaverMatcher.Web.Result 2 | { 3 | public enum SongMatchState 4 | { 5 | None, 6 | Waiting, 7 | LoadingPlaylistSongs, 8 | SearchingBeatMaps, 9 | Finished = 5, 10 | Error 11 | } 12 | } -------------------------------------------------------------------------------- /src/BeatSaverMatcher.Web/Result/WorkItemStore.cs: -------------------------------------------------------------------------------- 1 | using Prometheus; 2 | using System; 3 | using System.Collections.Concurrent; 4 | 5 | namespace BeatSaverMatcher.Web.Result 6 | { 7 | public class WorkItemStore 8 | { 9 | private static readonly TimeSpan _resultKeepTime = TimeSpan.FromHours(1); 10 | 11 | private readonly ConcurrentQueue _pendingItems; 12 | private readonly ConcurrentDictionary _items; 13 | private readonly Gauge _pendingRequestsGauge; 14 | 15 | public WorkItemStore() 16 | { 17 | _pendingItems = new ConcurrentQueue(); 18 | _items = new ConcurrentDictionary(); 19 | 20 | _pendingRequestsGauge = Metrics.CreateGauge("beatsaver_waiting_requests", "Requests waiting to be processed"); 21 | } 22 | 23 | public bool TryDequeue(out WorkResultItem item) 24 | { 25 | bool dequeued = _pendingItems.TryDequeue(out item); 26 | if (dequeued) 27 | _pendingRequestsGauge.Dec(); 28 | return dequeued; 29 | } 30 | 31 | public bool Enqueue(string playlistId) 32 | { 33 | if (_items.TryGetValue(playlistId, out WorkResultItem existingItem) && !existingItem.IsFinished) 34 | { 35 | return false; 36 | } 37 | var newWorkItem = _items.AddOrUpdate(playlistId, key => new WorkResultItem(playlistId), (key, oldItem) => new WorkResultItem(playlistId)); 38 | _pendingItems.Enqueue(newWorkItem); 39 | _pendingRequestsGauge.Inc(); 40 | return true; 41 | } 42 | 43 | public WorkResultItem Get(string playlistId) 44 | { 45 | if (_items.TryGetValue(playlistId, out var item)) 46 | return item; 47 | return null; 48 | } 49 | 50 | public void DoCleanup() 51 | { 52 | var items = _items.Values; 53 | 54 | foreach (var item in items) 55 | { 56 | if (item.CreatedAt < (DateTime.UtcNow - _resultKeepTime)) 57 | { 58 | _items.TryRemove(item.PlaylistId, out _); 59 | } 60 | } 61 | } 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /src/BeatSaverMatcher.Web/Result/WorkResultItem.cs: -------------------------------------------------------------------------------- 1 | using BeatSaverMatcher.Web.Models; 2 | using System; 3 | 4 | namespace BeatSaverMatcher.Web.Result 5 | { 6 | public class WorkResultItem 7 | { 8 | public WorkResultItem(string playlistId) 9 | { 10 | PlaylistId = playlistId; 11 | State = SongMatchState.Waiting; 12 | CreatedAt = DateTime.UtcNow; 13 | } 14 | 15 | public string PlaylistId { get; } 16 | public SongMatchState State { get; set; } 17 | public string ErrorMessage { get; set; } 18 | public int ItemsProcessed { get; set; } 19 | public int? ItemsTotal { get; set; } 20 | public SongMatchResult Result { get; set; } 21 | public DateTime CreatedAt { get; } 22 | public bool IsFinished => State == SongMatchState.Finished || State == SongMatchState.Error; 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /src/BeatSaverMatcher.Web/SongMatchWorker.cs: -------------------------------------------------------------------------------- 1 | using BeatSaverMatcher.Web.Result; 2 | using Microsoft.Extensions.Hosting; 3 | using Microsoft.Extensions.Logging; 4 | using Prometheus; 5 | using System; 6 | using System.Collections.Generic; 7 | using System.Linq; 8 | using System.Threading; 9 | using System.Threading.Tasks; 10 | 11 | namespace BeatSaverMatcher.Web 12 | { 13 | public sealed class SongMatchWorker : BackgroundService, IDisposable 14 | { 15 | private const int _maxRunningTasks = 8; 16 | private readonly ILogger _logger; 17 | private readonly MatchingService _matchingService; 18 | private readonly WorkItemStore _itemStore; 19 | private readonly Gauge _runningMatchesGauge; 20 | private readonly List _runningTasks = new List(); 21 | 22 | public SongMatchWorker(ILogger logger, MatchingService matchingService, WorkItemStore itemStore) 23 | { 24 | _logger = logger; 25 | _matchingService = matchingService; 26 | _itemStore = itemStore; 27 | _runningMatchesGauge = Metrics.CreateGauge("beatsaver_running_requests", "Requests currently running"); 28 | } 29 | 30 | protected override async Task ExecuteAsync(CancellationToken cancelToken) 31 | { 32 | while (true) 33 | { 34 | try 35 | { 36 | cancelToken.ThrowIfCancellationRequested(); 37 | _runningMatchesGauge.Set(_runningTasks.Count(x => x.Status == TaskStatus.Running || x.Status == TaskStatus.WaitingForActivation || x.Status == TaskStatus.WaitingForChildrenToComplete)); 38 | if (_runningTasks.Count > _maxRunningTasks) 39 | { 40 | var task = await Task.WhenAny(_runningTasks); 41 | _runningTasks.Remove(task); 42 | } 43 | else if (_itemStore.TryDequeue(out var item)) 44 | { 45 | var task = Task.Run(() => _matchingService.GetMatches(item, cancelToken), cancelToken); 46 | _runningTasks.Add(task); 47 | } 48 | else 49 | { 50 | await Task.Delay(1000, cancelToken); 51 | } 52 | } 53 | catch (OperationCanceledException) when (cancelToken.IsCancellationRequested) 54 | { 55 | throw; 56 | } 57 | catch (Exception ex) 58 | { 59 | _logger.LogError(ex, "Error while managing matching"); 60 | } 61 | } 62 | } 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /src/BeatSaverMatcher.Web/Startup.cs: -------------------------------------------------------------------------------- 1 | using BeatSaverMatcher.Api.Spotify; 2 | using BeatSaverMatcher.Api.Tidal; 3 | using BeatSaverMatcher.Common; 4 | using BeatSaverMatcher.Common.BeatSaver; 5 | using BeatSaverMatcher.Common.Db; 6 | using BeatSaverMatcher.Web.Result; 7 | using Microsoft.AspNetCore.Builder; 8 | using Microsoft.AspNetCore.Hosting; 9 | using Microsoft.Extensions.Configuration; 10 | using Microsoft.Extensions.DependencyInjection; 11 | using Microsoft.Extensions.Hosting; 12 | using Prometheus; 13 | 14 | namespace BeatSaverMatcher.Web 15 | { 16 | public class Startup 17 | { 18 | public Startup(IConfiguration configuration) 19 | { 20 | Configuration = configuration; 21 | } 22 | 23 | public IConfiguration Configuration { get; } 24 | 25 | // This method gets called by the runtime. Use this method to add services to the container. 26 | public void ConfigureServices(IServiceCollection services) 27 | { 28 | services.Configure(Configuration.GetSection("Spotify")); 29 | services.AddTransient(); 30 | 31 | services.Configure(Configuration.GetSection("Tidal")); 32 | services.AddTransient(); 33 | services.AddSingleton(); 34 | 35 | services.AddTransient(); 36 | services.AddTransient(); 37 | services.AddSingleton(); 38 | services.AddHostedService(); 39 | services.AddHostedService(); 40 | services.AddHostedService(); 41 | services.AddHttpClient(); 42 | 43 | var hasRedisConnection = !string.IsNullOrEmpty(Configuration["RedisConnection"]); 44 | 45 | if (hasRedisConnection) 46 | { 47 | services.AddStackExchangeRedisCache(options => 48 | { 49 | options.Configuration = Configuration["RedisConnection"]; 50 | options.InstanceName = "BeatSaverMatcher"; 51 | }); 52 | } 53 | else 54 | { 55 | services.AddDistributedMemoryCache(); 56 | } 57 | 58 | services.Configure(Configuration); 59 | services.AddTransient(); 60 | 61 | services.AddResponseCompression(); 62 | services.AddControllers(); 63 | } 64 | 65 | // This method gets called by the runtime. Use this method to configure the HTTP request pipeline. 66 | public void Configure(IApplicationBuilder app, IWebHostEnvironment env) 67 | { 68 | app.UseResponseCompression(); 69 | 70 | if (env.IsDevelopment()) 71 | { 72 | app.UseDeveloperExceptionPage(); 73 | } 74 | 75 | app.UseRouting(); 76 | app.UseHttpMetrics(); 77 | 78 | app.UseDefaultFiles(); 79 | 80 | app.UseStaticFiles(); 81 | 82 | app.UseAuthorization(); 83 | 84 | app.UseEndpoints(endpoints => 85 | { 86 | endpoints.MapControllers(); 87 | }); 88 | } 89 | } 90 | } 91 | -------------------------------------------------------------------------------- /src/BeatSaverMatcher.Web/config/appSettings.example.json: -------------------------------------------------------------------------------- 1 | { 2 | "Spotify": { 3 | "ClientId": "", 4 | "ClientSecret": "" 5 | }, 6 | "Tidal": { 7 | "ClientId": "", 8 | "ClientSecret": "" 9 | }, 10 | "RedisConnection": "", 11 | "ConnectionString": "Server=.;Database=BeatSaverMatcher;User Id=xxx;Password=yyy;Encrypt=False;" 12 | } -------------------------------------------------------------------------------- /src/BeatSaverMatcher.Web/config/logging.json: -------------------------------------------------------------------------------- 1 | { 2 | "Serilog": { 3 | "Using": [ "Serilog.Sinks.Console" ], 4 | "MinimumLevel": { 5 | "Default": "Information", 6 | "Override": { 7 | "Microsoft": "Warning" 8 | } 9 | }, 10 | "WriteTo": [ 11 | { "Name": "Console" } 12 | ], 13 | "Enrich": [ "FromLogContext" ] 14 | } 15 | } -------------------------------------------------------------------------------- /src/BeatSaverMatcher.Web/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": "1.0.0", 3 | "name": "asp.net", 4 | "private": true, 5 | "scripts": { 6 | "watch:dev": "webpack --mode development --watch", 7 | "build:dev": "webpack --mode development", 8 | "build:prod": "webpack --mode production" 9 | }, 10 | "devDependencies": { 11 | "@types/knockout": "^3.4.77", 12 | "clean-webpack-plugin": "^4.0.0", 13 | "copy-webpack-plugin": "^12.0.1", 14 | "css-loader": "^6.9.0", 15 | "html-webpack-plugin": "^5.6.0", 16 | "minimist": "^1.2.8", 17 | "source-map-loader": "^5.0.0", 18 | "style-loader": "^3.3.4", 19 | "ts-loader": "^9.5.1", 20 | "typescript": "^5.3.3", 21 | "webpack": "^5.89.0", 22 | "webpack-cli": "^5.1.4" 23 | }, 24 | "dependencies": { 25 | "bootstrap": "^4.6.2", 26 | "jquery": "^3.7.1", 27 | "knockout": "^3.5.1", 28 | "popper.js": "^1.16.1" 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /src/BeatSaverMatcher.Web/src/icon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/patagonaa/BeatSaverMatcher/ef3e094db43f9c080c18b2f9b50c3147730db796/src/BeatSaverMatcher.Web/src/icon.png -------------------------------------------------------------------------------- /src/BeatSaverMatcher.Web/src/icon_32.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/patagonaa/BeatSaverMatcher/ef3e094db43f9c080c18b2f9b50c3147730db796/src/BeatSaverMatcher.Web/src/icon_32.png -------------------------------------------------------------------------------- /src/BeatSaverMatcher.Web/src/icon_48.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/patagonaa/BeatSaverMatcher/ef3e094db43f9c080c18b2f9b50c3147730db796/src/BeatSaverMatcher.Web/src/icon_48.png -------------------------------------------------------------------------------- /src/BeatSaverMatcher.Web/src/index.html: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | 5 | BeatSaverMatcher 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 |
14 |
15 |

BeatSaverMatcher

16 |
Match Spotify / Tidal playlists to songs on beatsaver.com
17 |
18 |
19 | 20 | 21 | Note: Automapped Beatmaps are ignored 22 |
23 |
24 | 25 |
26 | 27 | 28 |
29 |
30 | 31 | 32 |
33 |
34 | 35 | 36 |
37 |
38 | 39 | 40 |
41 |
42 | 43 | 44 |
45 | Beatmaps with other difficulties will still be shown, but not preselected 46 |
47 | 48 |
49 |
50 |
51 | Status:
52 |  ( / ) 53 |
54 |
55 |
56 | 57 |
58 |
59 |
60 |
61 | Open selected as playlist
62 | Download selected as playlist
63 | / matched!
64 | 65 | Columns: 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | 75 | 76 | 77 | 78 | 79 | 80 | 81 | 82 | 83 | 84 | 85 | 86 | 87 | 88 | 89 | 90 | 91 | 92 | 93 | 94 | 95 | 96 | 97 | 98 | 99 | 100 | 101 | 102 | 103 | 104 | 105 | 106 | 107 | 108 | 109 | 110 | 111 | 112 | 113 | 114 | 115 | 116 | 117 | 118 | 119 | 120 | 121 | 122 | 123 | 124 | 125 | 126 | 127 | 128 | 129 | 130 | 131 | 132 | 133 | 134 | 137 | 140 | 144 | 149 | 150 | 151 | 152 | 153 | 154 |
PlaylistBeatSaber
Artist(s)TitleMap nameSong authorSong nameSong sub-nameMap authorUpload dateDifficultyRating👍/👎KeySelected
135 |
136 | 		138 | 139 | 141 |
142 | 143 | 		145 | | 146 | Download | 147 | View 148 |
155 |
156 |
157 |
158 | Website 159 | GitHub 160 |
161 |
162 | 163 | Privacy policy (DE)
164 | TLDR: everything your browser transmits (your IP, your browser, form data, etc.) is transferred encrypted and may be logged, for example to fix bugs in this software. 165 | 166 |
167 |
168 |
169 | 170 | -------------------------------------------------------------------------------- /src/BeatSaverMatcher.Web/src/index.ts: -------------------------------------------------------------------------------- 1 | import * as ko from "knockout"; 2 | import 'bootstrap/dist/css/bootstrap.min.css'; 3 | import './style.css'; 4 | 5 | class AppViewModel { 6 | public playlistId = ko.observable(); 7 | public playlistUri = ko.observable(''); 8 | public preferredDifficulties = ko.observableArray(['1', '2', '4', '8', '16']); 9 | public preferredDifficultiesFlags = ko.computed(() => this.preferredDifficulties().map(x => +x).reduce((a, b) => a | b, 0)); 10 | public stateName = ko.observable('None'); 11 | public workItem = ko.observable(); 12 | public result = ko.observable(); 13 | public visibleColumns = ko.observableArray(['uploaded', 'levelAuthorName']); 14 | 15 | 16 | public async run() { 17 | let matches = this.playlistId().match(/(?:playlist[\/:])?([\w-]+)(?:\?.+)?$/); 18 | 19 | if (matches) { 20 | this.playlistId(matches[1]); 21 | } 22 | 23 | await fetch(`/api/Matches/${this.playlistId()}`, { 24 | method: 'POST' 25 | }); 26 | 27 | var result: SongMatchResult; 28 | var item: WorkResultItem = null; 29 | while (result == null) { 30 | try { 31 | var response = await fetch(`/api/Matches/${this.playlistId()}`); 32 | item = await response.json(); 33 | } catch (e) { 34 | this.stateName(SongMatchState[SongMatchState.Error]); 35 | throw 'Something went wrong!'; 36 | } 37 | 38 | this.workItem(item); 39 | if (item.state == SongMatchState.Error) { 40 | this.stateName(SongMatchState[item.state]); 41 | throw 'Something went wrong!'; 42 | } 43 | 44 | if (item.state == SongMatchState.Finished) { 45 | result = item.result; 46 | break; 47 | } 48 | this.stateName(SongMatchState[item.state]); 49 | await new Promise(r => setTimeout(r, 1000)); 50 | } 51 | 52 | for (let match of result.matches) { 53 | match.beatMaps.forEach(x => x.selected = ko.observable(false)); 54 | 55 | let firstPreferredDifficulty = match.beatMaps.find(x => (x.difficulties & this.preferredDifficultiesFlags()) > 0); 56 | if (firstPreferredDifficulty != null) 57 | firstPreferredDifficulty.selected(true); 58 | 59 | match.beatMaps.forEach(x => x.selected.subscribe(() => this.updatePlaylistUri())); 60 | } 61 | this.result(result); 62 | this.updatePlaylistUri(); 63 | this.stateName(SongMatchState[item.state]); 64 | } 65 | 66 | public getDifficulties(difficulties: number) { 67 | var toReturn = []; 68 | if (difficulties & 1) 69 | toReturn.push('Easy'); 70 | if (difficulties & 2) 71 | toReturn.push('Normal'); 72 | if (difficulties & 4) 73 | toReturn.push('Hard'); 74 | if (difficulties & 8) 75 | toReturn.push('Expert'); 76 | if (difficulties & 16) 77 | toReturn.push('ExpertPlus'); 78 | return toReturn; 79 | } 80 | 81 | private updatePlaylistUri() { 82 | let keys: string[] = []; 83 | 84 | for (let match of this.result().matches) { 85 | for (let beatSaberMatch of match.beatMaps) { 86 | if (beatSaberMatch.selected()) 87 | keys.push(beatSaberMatch.beatSaverKey.toString(16)); 88 | } 89 | } 90 | var uri = `${window.location.protocol}//${window.location.host}/api/ModSaberPlaylist/${keys.join(',')}/${this.playlistId()}.bplist`; 91 | 92 | this.playlistUri(uri); 93 | } 94 | } 95 | 96 | interface WorkResultItem { 97 | playlistId: string; 98 | state: SongMatchState; 99 | errorMessage: string; 100 | result: SongMatchResult; 101 | itemsProcessed: number; 102 | itemsTotal: number; 103 | } 104 | 105 | enum SongMatchState { 106 | None, 107 | Waiting, 108 | LoadingPlaylistSongs, 109 | SearchingBeatMaps, 110 | Finished = 5, 111 | Error 112 | } 113 | 114 | interface SongMatchResult { 115 | matchedPlaylistSongs: number; 116 | totalPlaylistSongs: number; 117 | matches: SongMatch[]; 118 | } 119 | 120 | interface SongMatch { 121 | playlistArtist: string; 122 | playlistTitle: string; 123 | beatMaps: BeatSaberSong[]; 124 | } 125 | 126 | interface BeatSaberSong { 127 | levelAuthorName: string; 128 | songAuthorName: string; 129 | songName: string; 130 | songSubName: string; 131 | bpm: number; 132 | name: string; 133 | difficulties: number; 134 | uploader: string; 135 | uploaded: string; 136 | hash: string; 137 | beatSaverKey: number; 138 | selected: KnockoutObservable; 139 | rating: number; 140 | upVotes: number; 141 | downVotes: number; 142 | } 143 | 144 | function init() { 145 | ko.applyBindings(new AppViewModel(), document.getElementById("main")); 146 | } 147 | 148 | window.onload = init; -------------------------------------------------------------------------------- /src/BeatSaverMatcher.Web/src/style.css: -------------------------------------------------------------------------------- 1 | body { 2 | } 3 | 4 | form { 5 | margin-bottom: 5px; 6 | } 7 | 8 | form, .status { 9 | max-width: 600px; 10 | } 11 | 12 | .status > div { 13 | padding: 5px 0; 14 | } 15 | 16 | progress { 17 | vertical-align: middle; 18 | } 19 | 20 | .links a { 21 | display: inline-block; 22 | margin: 0 5px 0 0; 23 | } 24 | 25 | .subtitle { 26 | margin-top: -0.375rem; 27 | } 28 | 29 | span.badge.is-easy { 30 | color: #fff; 31 | background-color: #3cb371; 32 | } 33 | 34 | span.badge.is-normal { 35 | color: #fff; 36 | background-color: #59b0f4; 37 | } 38 | 39 | span.badge.is-hard { 40 | color: #fff; 41 | background-color: tomato; 42 | } 43 | 44 | span.badge.is-expert { 45 | color: #fff; 46 | background-color: #bf2a42; 47 | } 48 | 49 | span.badge.is-expertplus { 50 | color: #fff; 51 | background-color: #8f48db; 52 | } 53 | -------------------------------------------------------------------------------- /src/BeatSaverMatcher.Web/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "noImplicitAny": false, 4 | "noEmitOnError": true, 5 | "removeComments": false, 6 | "sourceMap": true, 7 | "target": "ES2019" 8 | }, 9 | "exclude": [ 10 | "node_modules", 11 | "wwwroot" 12 | ] 13 | } 14 | -------------------------------------------------------------------------------- /src/BeatSaverMatcher.Web/webpack.config.js: -------------------------------------------------------------------------------- 1 | const path = require("path"); 2 | const HtmlWebpackPlugin = require("html-webpack-plugin"); 3 | const { CleanWebpackPlugin } = require('clean-webpack-plugin'); 4 | const CopyWebpackPlugin = require('copy-webpack-plugin'); 5 | const package = require('./package.json'); 6 | 7 | module.exports = { 8 | entry: { 9 | app: "./src/index.ts", 10 | vendor: Object.keys(package.dependencies) 11 | }, 12 | output: { 13 | path: path.resolve(__dirname, "wwwroot"), 14 | filename: "[name].js", 15 | publicPath: "/" 16 | }, 17 | resolve: { 18 | extensions: [".js", ".ts"] 19 | }, 20 | module: { 21 | rules: [ 22 | { 23 | test: /\.ts$/, 24 | use: "ts-loader" 25 | }, 26 | { 27 | test: /\.css$/, 28 | use: ['style-loader', 'css-loader'] 29 | } 30 | ] 31 | }, 32 | plugins: [ 33 | new CleanWebpackPlugin(), 34 | new HtmlWebpackPlugin({ 35 | template: "./src/index.html", 36 | filename: "index.html", 37 | chunks: ['vendor', 'app'], 38 | minify: { 39 | collapseWhitespace: true, 40 | keepClosingSlash: true, 41 | removeComments: false, 42 | removeRedundantAttributes: true, 43 | removeScriptTypeAttributes: true, 44 | removeStyleLinkTypeAttributes: true, 45 | useShortDoctype: true 46 | } 47 | }), 48 | new CopyWebpackPlugin({ 49 | patterns: [ 50 | { context: './src', from: 'icon*.png' } 51 | ] 52 | }) 53 | ] 54 | }; -------------------------------------------------------------------------------- /src/BeatSaverMatcher.sln: -------------------------------------------------------------------------------- 1 |  2 | Microsoft Visual Studio Solution File, Format Version 12.00 3 | # Visual Studio Version 16 4 | VisualStudioVersion = 16.0.30204.135 5 | MinimumVisualStudioVersion = 10.0.40219.1 6 | Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BeatSaverMatcher.Web", "BeatSaverMatcher.Web\BeatSaverMatcher.Web.csproj", "{BCEC6EBE-31EF-4B8C-A6BE-10B4FB4820C2}" 7 | EndProject 8 | Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BeatSaverMatcher.Crawler", "BeatSaverMatcher.Crawler\BeatSaverMatcher.Crawler.csproj", "{7F8CDC06-0AE7-4B3D-B999-7A89226B94A9}" 9 | EndProject 10 | Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BeatSaverMatcher.Common", "BeatSaverMatcher.Common\BeatSaverMatcher.Common.csproj", "{004D261B-DCA3-438A-8D57-8F9029F40E21}" 11 | EndProject 12 | Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BeatSaverMatcher.Api.Tidal", "BeatSaverMatcher.Api.Tidal\BeatSaverMatcher.Api.Tidal.csproj", "{74136C7D-1FA0-43F1-9652-DDD832B86675}" 13 | EndProject 14 | Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BeatSaverMatcher.Api", "BeatSaverMatcher.Api\BeatSaverMatcher.Api.csproj", "{B5AD31CD-1050-42DA-973A-5CC70DE92C6F}" 15 | EndProject 16 | Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BeatSaverMatcher.Api.Spotify", "BeatSaverMatcher.Api.Spotify\BeatSaverMatcher.Api.Spotify.csproj", "{B9CAA35C-F438-4F97-87AA-C0E6CE67C3F8}" 17 | EndProject 18 | Global 19 | GlobalSection(SolutionConfigurationPlatforms) = preSolution 20 | Debug|Any CPU = Debug|Any CPU 21 | Release|Any CPU = Release|Any CPU 22 | EndGlobalSection 23 | GlobalSection(ProjectConfigurationPlatforms) = postSolution 24 | {BCEC6EBE-31EF-4B8C-A6BE-10B4FB4820C2}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 25 | {BCEC6EBE-31EF-4B8C-A6BE-10B4FB4820C2}.Debug|Any CPU.Build.0 = Debug|Any CPU 26 | {BCEC6EBE-31EF-4B8C-A6BE-10B4FB4820C2}.Release|Any CPU.ActiveCfg = Release|Any CPU 27 | {BCEC6EBE-31EF-4B8C-A6BE-10B4FB4820C2}.Release|Any CPU.Build.0 = Release|Any CPU 28 | {7F8CDC06-0AE7-4B3D-B999-7A89226B94A9}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 29 | {7F8CDC06-0AE7-4B3D-B999-7A89226B94A9}.Debug|Any CPU.Build.0 = Debug|Any CPU 30 | {7F8CDC06-0AE7-4B3D-B999-7A89226B94A9}.Release|Any CPU.ActiveCfg = Release|Any CPU 31 | {7F8CDC06-0AE7-4B3D-B999-7A89226B94A9}.Release|Any CPU.Build.0 = Release|Any CPU 32 | {004D261B-DCA3-438A-8D57-8F9029F40E21}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 33 | {004D261B-DCA3-438A-8D57-8F9029F40E21}.Debug|Any CPU.Build.0 = Debug|Any CPU 34 | {004D261B-DCA3-438A-8D57-8F9029F40E21}.Release|Any CPU.ActiveCfg = Release|Any CPU 35 | {004D261B-DCA3-438A-8D57-8F9029F40E21}.Release|Any CPU.Build.0 = Release|Any CPU 36 | {74136C7D-1FA0-43F1-9652-DDD832B86675}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 37 | {74136C7D-1FA0-43F1-9652-DDD832B86675}.Debug|Any CPU.Build.0 = Debug|Any CPU 38 | {74136C7D-1FA0-43F1-9652-DDD832B86675}.Release|Any CPU.ActiveCfg = Release|Any CPU 39 | {74136C7D-1FA0-43F1-9652-DDD832B86675}.Release|Any CPU.Build.0 = Release|Any CPU 40 | {B5AD31CD-1050-42DA-973A-5CC70DE92C6F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 41 | {B5AD31CD-1050-42DA-973A-5CC70DE92C6F}.Debug|Any CPU.Build.0 = Debug|Any CPU 42 | {B5AD31CD-1050-42DA-973A-5CC70DE92C6F}.Release|Any CPU.ActiveCfg = Release|Any CPU 43 | {B5AD31CD-1050-42DA-973A-5CC70DE92C6F}.Release|Any CPU.Build.0 = Release|Any CPU 44 | {B9CAA35C-F438-4F97-87AA-C0E6CE67C3F8}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 45 | {B9CAA35C-F438-4F97-87AA-C0E6CE67C3F8}.Debug|Any CPU.Build.0 = Debug|Any CPU 46 | {B9CAA35C-F438-4F97-87AA-C0E6CE67C3F8}.Release|Any CPU.ActiveCfg = Release|Any CPU 47 | {B9CAA35C-F438-4F97-87AA-C0E6CE67C3F8}.Release|Any CPU.Build.0 = Release|Any CPU 48 | EndGlobalSection 49 | GlobalSection(SolutionProperties) = preSolution 50 | HideSolutionNode = FALSE 51 | EndGlobalSection 52 | GlobalSection(ExtensibilityGlobals) = postSolution 53 | SolutionGuid = {81DF8EA8-1D14-468C-8967-A8C66B864CC6} 54 | EndGlobalSection 55 | EndGlobal 56 | --------------------------------------------------------------------------------