├── .gitattributes ├── .gitignore ├── LICENSE ├── README.md └── SongDataCore ├── .editorconfig ├── BeatStar ├── BeatStarDataFile.cs ├── BeatStarDatabase.cs ├── BeatStarSong.cs └── CacheableBeatStarDownloadHandler.cs ├── Directory.Build.props ├── Directory.Build.targets ├── Downloader ├── CacheableDownloaderHandler.cs ├── DatabaseDownloader.cs └── IDatabaseDownloadHandler.cs ├── Internal └── JsonExtensions.cs ├── Plugin.cs ├── SongDataCore.csproj ├── SongDataCore.sln ├── Template.txt └── manifest.json /.gitattributes: -------------------------------------------------------------------------------- 1 | * text=auto eol=lf 2 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | ## Ignore Visual Studio temporary files, build results, and 2 | ## files generated by popular Visual Studio add-ons. 3 | ## 4 | ## Get latest from https://github.com/github/gitignore/blob/master/VisualStudio.gitignore 5 | 6 | # User-specific files 7 | *.suo 8 | *.user 9 | *.userosscache 10 | *.sln.docstates 11 | 12 | # User-specific files (MonoDevelop/Xamarin Studio) 13 | *.userprefs 14 | 15 | # Build results 16 | [Dd]ebug/ 17 | [Dd]ebugPublic/ 18 | [Rr]elease/ 19 | [Rr]eleases/ 20 | x64/ 21 | x86/ 22 | bld/ 23 | [Bb]in/ 24 | [Oo]bj/ 25 | [Ll]og/ 26 | 27 | # Visual Studio 2015 cache/options directory 28 | .vs/ 29 | # Uncomment if you have tasks that create the project's static files in wwwroot 30 | #wwwroot/ 31 | 32 | # MSTest test Results 33 | [Tt]est[Rr]esult*/ 34 | [Bb]uild[Ll]og.* 35 | 36 | # NUNIT 37 | *.VisualState.xml 38 | TestResult.xml 39 | 40 | # Build Results of an ATL Project 41 | [Dd]ebugPS/ 42 | [Rr]eleasePS/ 43 | dlldata.c 44 | 45 | # .NET Core 46 | project.lock.json 47 | project.fragment.lock.json 48 | artifacts/ 49 | **/Properties/launchSettings.json 50 | 51 | *_i.c 52 | *_p.c 53 | *_i.h 54 | *.ilk 55 | *.meta 56 | *.obj 57 | *.pch 58 | *.pdb 59 | *.pgc 60 | *.pgd 61 | *.rsp 62 | *.sbr 63 | *.tlb 64 | *.tli 65 | *.tlh 66 | *.tmp 67 | *.tmp_proj 68 | *.log 69 | *.vspscc 70 | *.vssscc 71 | .builds 72 | *.pidb 73 | *.svclog 74 | *.scc 75 | 76 | # Chutzpah Test files 77 | _Chutzpah* 78 | 79 | # Visual C++ cache files 80 | ipch/ 81 | *.aps 82 | *.ncb 83 | *.opendb 84 | *.opensdf 85 | *.sdf 86 | *.cachefile 87 | *.VC.db 88 | *.VC.VC.opendb 89 | 90 | # Visual Studio profiler 91 | *.psess 92 | *.vsp 93 | *.vspx 94 | *.sap 95 | 96 | # TFS 2012 Local Workspace 97 | $tf/ 98 | 99 | # Guidance Automation Toolkit 100 | *.gpState 101 | 102 | # ReSharper is a .NET coding add-in 103 | _ReSharper*/ 104 | *.[Rr]e[Ss]harper 105 | *.DotSettings.user 106 | 107 | # JustCode is a .NET coding add-in 108 | .JustCode 109 | 110 | # TeamCity is a build add-in 111 | _TeamCity* 112 | 113 | # DotCover is a Code Coverage Tool 114 | *.dotCover 115 | 116 | # Visual Studio code coverage results 117 | *.coverage 118 | *.coveragexml 119 | 120 | # NCrunch 121 | _NCrunch_* 122 | .*crunch*.local.xml 123 | nCrunchTemp_* 124 | 125 | # MightyMoose 126 | *.mm.* 127 | AutoTest.Net/ 128 | 129 | # Web workbench (sass) 130 | .sass-cache/ 131 | 132 | # Installshield output folder 133 | [Ee]xpress/ 134 | 135 | # DocProject is a documentation generator add-in 136 | DocProject/buildhelp/ 137 | DocProject/Help/*.HxT 138 | DocProject/Help/*.HxC 139 | DocProject/Help/*.hhc 140 | DocProject/Help/*.hhk 141 | DocProject/Help/*.hhp 142 | DocProject/Help/Html2 143 | DocProject/Help/html 144 | 145 | # Click-Once directory 146 | publish/ 147 | 148 | # Publish Web Output 149 | *.[Pp]ublish.xml 150 | *.azurePubxml 151 | # TODO: Comment the next line if you want to checkin your web deploy settings 152 | # but database connection strings (with potential passwords) will be unencrypted 153 | *.pubxml 154 | *.publishproj 155 | 156 | # Microsoft Azure Web App publish settings. Comment the next line if you want to 157 | # checkin your Azure Web App publish settings, but sensitive information contained 158 | # in these scripts will be unencrypted 159 | PublishScripts/ 160 | 161 | # NuGet Packages 162 | *.nupkg 163 | # The packages folder can be ignored because of Package Restore 164 | **/packages/* 165 | # except build/, which is used as an MSBuild target. 166 | !**/packages/build/ 167 | # Uncomment if necessary however generally it will be regenerated when needed 168 | #!**/packages/repositories.config 169 | # NuGet v3's project.json files produces more ignorable files 170 | *.nuget.props 171 | *.nuget.targets 172 | 173 | # Microsoft Azure Build Output 174 | csx/ 175 | *.build.csdef 176 | 177 | # Microsoft Azure Emulator 178 | ecf/ 179 | rcf/ 180 | 181 | # Windows Store app package directories and files 182 | AppPackages/ 183 | BundleArtifacts/ 184 | Package.StoreAssociation.xml 185 | _pkginfo.txt 186 | 187 | # Visual Studio cache files 188 | # files ending in .cache can be ignored 189 | *.[Cc]ache 190 | # but keep track of directories ending in .cache 191 | !*.[Cc]ache/ 192 | 193 | # Others 194 | ClientBin/ 195 | ~$* 196 | *~ 197 | *.dbmdl 198 | *.dbproj.schemaview 199 | *.jfm 200 | *.pfx 201 | *.publishsettings 202 | orleans.codegen.cs 203 | 204 | # Since there are multiple workflows, uncomment next line to ignore bower_components 205 | # (https://github.com/github/gitignore/pull/1529#issuecomment-104372622) 206 | #bower_components/ 207 | 208 | # RIA/Silverlight projects 209 | Generated_Code/ 210 | 211 | # Backup & report files from converting an old project file 212 | # to a newer Visual Studio version. Backup files are not needed, 213 | # because we have git ;-) 214 | _UpgradeReport_Files/ 215 | Backup*/ 216 | UpgradeLog*.XML 217 | UpgradeLog*.htm 218 | 219 | # SQL Server files 220 | *.mdf 221 | *.ldf 222 | *.ndf 223 | 224 | # Business Intelligence projects 225 | *.rdl.data 226 | *.bim.layout 227 | *.bim_*.settings 228 | 229 | # Microsoft Fakes 230 | FakesAssemblies/ 231 | 232 | # GhostDoc plugin setting file 233 | *.GhostDoc.xml 234 | 235 | # Node.js Tools for Visual Studio 236 | .ntvs_analysis.dat 237 | node_modules/ 238 | 239 | # Typescript v1 declaration files 240 | typings/ 241 | 242 | # Visual Studio 6 build log 243 | *.plg 244 | 245 | # Visual Studio 6 workspace options file 246 | *.opt 247 | 248 | # Visual Studio 6 auto-generated workspace file (contains which files were open etc.) 249 | *.vbw 250 | 251 | # Visual Studio LightSwitch build output 252 | **/*.HTMLClient/GeneratedArtifacts 253 | **/*.DesktopClient/GeneratedArtifacts 254 | **/*.DesktopClient/ModelManifest.xml 255 | **/*.Server/GeneratedArtifacts 256 | **/*.Server/ModelManifest.xml 257 | _Pvt_Extensions 258 | 259 | # Paket dependency manager 260 | .paket/paket.exe 261 | paket-files/ 262 | 263 | # FAKE - F# Make 264 | .fake/ 265 | 266 | # JetBrains Rider 267 | .idea/ 268 | *.sln.iml 269 | 270 | # CodeRush 271 | .cr/ 272 | 273 | # Python Tools for Visual Studio (PTVS) 274 | __pycache__/ 275 | *.pyc 276 | 277 | # Cake - Uncomment if you are using it 278 | # tools/** 279 | # !tools/packages.config 280 | 281 | # Telerik's JustMock configuration file 282 | *.jmconfig 283 | 284 | # BizTalk build output 285 | *.btp.cs 286 | *.btm.cs 287 | *.odx.cs 288 | *.xsd.cs 289 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2019 Halsafar 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # BeatSaberSongDataCore 2 | 3 | A Beat Saber Plugin that manages scraped data from various sources. 4 | 5 | ## Status 6 | - Working with BeatSaber 1.23.0 7 | 8 | ## Supports 9 | - Cached Downloads (ETAG of resource) 10 | - BeatSaver data dump. 11 | - ScoreSaber data dump. 12 | - Songs from each source are mapped by song hash. 13 | 14 | ## Example Usage: 15 | Data Available: 16 | ```c# 17 | SongDataCore.Plugin.BeatSaver.IsDataAvailable() 18 | SongDataCore.Plugin.ScoreSaber.IsDataAvailable() 19 | ``` 20 | 21 | Get song: 22 | ```c# 23 | BeatSaverSong beatSaverSong = SongDataCore.Plugin.BeatSaver.Data.Songs[hash]; 24 | ScoreSaberSong scoreSaberSong = SongDataCore.Plugin.ScoreSaber.Data.Songs[hash]; 25 | ``` 26 | 27 | ## Building on Windows 28 | To compile SongDataCore from source: 29 | 30 | 1. Install Beat Saber and Microsoft Visual Studio. 31 | 2. Download and extract the BeatSaberSongDataCore source code. 32 | 3. Create a new file `/SongDataCore/SongDataCore.csproj.user` with the following. (Make sure to replace BeatSaberDir with your real Beat Saber installation folder) 33 | ``` 34 | 35 | 36 | 37 | ProjectFiles 38 | C:\Program Files (x86)\Steam\steamapps\common\Beat Saber 39 | 40 | 41 | ``` 42 | 4. Open `/BeatSaberSongDataCore/SongDataCore.sln` in Microsoft Visual Studio. 43 | 5. Build the project with *Build -> Build Solution*. 44 | -------------------------------------------------------------------------------- /SongDataCore/.editorconfig: -------------------------------------------------------------------------------- 1 | # To learn more about .editorconfig see https://aka.ms/editorconfigdocs 2 | root = true 3 | 4 | # All files 5 | [*] 6 | indent_style = space 7 | end_of_line = lf 8 | 9 | # Xml files 10 | [*.xml] 11 | indent_size = 2 12 | -------------------------------------------------------------------------------- /SongDataCore/BeatStar/BeatStarDataFile.cs: -------------------------------------------------------------------------------- 1 | using Newtonsoft.Json; 2 | using System; 3 | using System.Collections.Generic; 4 | using System.Diagnostics; 5 | using System.IO; 6 | using System.IO.Compression; 7 | using System.Linq; 8 | 9 | namespace SongDataCore.BeatStar 10 | { 11 | public class BeatStarDataFile 12 | { 13 | public Dictionary Songs = null; 14 | 15 | public BeatStarDataFile(byte[] data) 16 | { 17 | Plugin.Log.Info("Constructing BeatStarDataFile"); 18 | 19 | //System.Threading.Thread.Sleep(2000); 20 | //Plugin.Log.Debug($"BeatSaber Total Memory - Before BeatStar Load: {GC.GetTotalMemory(true)}"); 21 | 22 | Stopwatch timer = new Stopwatch(); 23 | timer.Start(); 24 | 25 | try 26 | { 27 | string result; 28 | using (var compressedStream = new MemoryStream(data)) 29 | using (var zipStream = new GZipStream(compressedStream, CompressionMode.Decompress)) 30 | using (var resultStream = new MemoryStream()) 31 | { 32 | zipStream.CopyTo(resultStream); 33 | result = System.Text.Encoding.UTF8.GetString(resultStream.ToArray()); 34 | } 35 | 36 | var tmpSongs = JsonConvert.DeserializeObject>(result); 37 | Songs = new Dictionary(tmpSongs, StringComparer.OrdinalIgnoreCase); 38 | tmpSongs = null; 39 | 40 | HashSet userSongKeys = SongCore.Loader.CustomLevels.Select(x => x.Value.levelID.Remove(0, 13)).ToHashSet(); 41 | 42 | List removeSongs = new List(); 43 | foreach (var pair in Songs) 44 | { 45 | bool userHasSong = userSongKeys.Contains(pair.Key); 46 | if (!userHasSong) 47 | { 48 | removeSongs.Add(pair.Key); 49 | continue; 50 | } 51 | 52 | pair.Value.characteristics = new Dictionary>(); 53 | foreach (var diff in pair.Value.diffs) 54 | { 55 | var characteristic = (BeatStarCharacteristics)diff.type; 56 | if (!pair.Value.characteristics.ContainsKey(characteristic)) 57 | { 58 | pair.Value.characteristics.Add(characteristic, new Dictionary()); 59 | } 60 | 61 | // TODO - REMOVE when the scrape isnt duplicating diffs. 62 | if (pair.Value.characteristics[characteristic].ContainsKey(diff.diff)) 63 | { 64 | continue; 65 | } 66 | 67 | //Plugin.Log.Info($"Adding {characteristic}, {diff.diff}, {diff}"); 68 | pair.Value.characteristics[characteristic].Add(diff.diff, diff); 69 | } 70 | } 71 | 72 | Plugin.Log.Debug($"Removing {removeSongs.Count} songs from BeatStar database."); 73 | foreach (var key in removeSongs) 74 | { 75 | Songs.Remove(key); 76 | } 77 | 78 | timer.Stop(); 79 | Plugin.Log.Debug($"Processing BeatStar data took {timer.ElapsedMilliseconds}ms"); 80 | 81 | //System.GC.Collect(); 82 | //Plugin.Log.Debug($"BeatSaber Total Memory - After BeatStar Load: {GC.GetTotalMemory(true)}"); 83 | } 84 | catch (Exception e) 85 | { 86 | Plugin.Log.Error($"BeatStar data corrupted, sometimes JSON dump returns from BeatSaver corrupted: {e.Message}"); 87 | return; 88 | } 89 | } 90 | } 91 | } 92 | -------------------------------------------------------------------------------- /SongDataCore/BeatStar/BeatStarDatabase.cs: -------------------------------------------------------------------------------- 1 | using SongDataCore.Downloader; 2 | using System; 3 | using System.Collections; 4 | using UnityEngine.Networking; 5 | 6 | namespace SongDataCore.BeatStar 7 | { 8 | public class BeatStarDatabase : DatabaseDownloader, IDatabaseDownloadHandler 9 | { 10 | public const String SCRAPED_SCORE_SABER_ALL_JSON_URL = "https://cdn.wes.cloud/beatstar/bssb/v2-all.json.gz"; 11 | 12 | public BeatStarDataFile Data = null; 13 | 14 | protected byte[] Buffer = new byte[4096]; 15 | 16 | CacheableBeatStarDownloaderHandler _lastCacheHandler; 17 | 18 | /// 19 | /// Start downloading the BeatSaver database. 20 | /// 21 | public override void Load() 22 | { 23 | StartCoroutine(DownloadBeatStarDatabases()); 24 | } 25 | 26 | /// 27 | /// Attempt to reduce memory usage. 28 | /// 29 | public override void Unload() 30 | { 31 | StartCoroutine(WaitAndUnload()); 32 | } 33 | 34 | /// 35 | /// Cancel and Wait for any existing operations to complete then clean up. 36 | /// 37 | /// 38 | private IEnumerator WaitAndUnload() 39 | { 40 | yield return StartCoroutine(CancelDownload()); 41 | 42 | UnloadNow(); 43 | } 44 | 45 | public void UnloadNow() 46 | { 47 | Plugin.Log.Debug("Unload SongDataCore data."); 48 | 49 | if (Data != null) 50 | { 51 | //Plugin.Log.Debug($"BeatSaber Total Memory - Before BeatStar Unload: {GC.GetTotalMemory(false)}"); 52 | Data = null; 53 | System.GC.Collect(); 54 | //Plugin.Log.Debug($"BeatSaber Total Memory - After BeatStar Unload: {GC.GetTotalMemory(false)}"); 55 | } 56 | else 57 | { 58 | Plugin.Log.Debug("BeatStar Database not loaded..."); 59 | } 60 | } 61 | 62 | /// 63 | /// Helper to download both databases. They will be stacked together, ranked is more updated. 64 | /// 65 | /// 66 | private IEnumerator DownloadBeatStarDatabases() 67 | { 68 | Data = null; 69 | _isDownloading = true; 70 | _cancelRequested = false; 71 | 72 | yield return DownloadDatabase(SCRAPED_SCORE_SABER_ALL_JSON_URL, this); 73 | 74 | _isDownloading = false; 75 | } 76 | 77 | /// 78 | /// Return the appropriate handler for this database. 79 | /// 80 | /// 81 | /// 82 | public CacheableDownloadHandler GetDownloadHandler(UnityWebRequest www) 83 | { 84 | if (!this._firstSuccess || _lastCacheHandler == null) 85 | { 86 | _lastCacheHandler = new CacheableBeatStarDownloaderHandler(www, Buffer); 87 | } 88 | 89 | www.SetCacheable(_lastCacheHandler); 90 | return _lastCacheHandler; 91 | } 92 | 93 | /// 94 | /// Acquire the results from the download handler. 95 | /// 96 | /// 97 | public void HandleDownloadResults(DownloadHandler handler) 98 | { 99 | if (Data == null) 100 | { 101 | Data = (handler as CacheableBeatStarDownloaderHandler).DataFile; 102 | } 103 | 104 | _firstSuccess = true; 105 | } 106 | 107 | /// 108 | /// Do we have data. 109 | /// 110 | /// 111 | public bool IsDataAvailable() 112 | { 113 | return Data != null && Data.Songs != null; 114 | } 115 | 116 | /// 117 | /// No need to interrupt BeatStar yet, it parses often faster than we can even interrupt it. 118 | /// 119 | public void CancelHandler(DownloadHandler handler) 120 | { 121 | return; 122 | } 123 | } 124 | } 125 | -------------------------------------------------------------------------------- /SongDataCore/BeatStar/BeatStarSong.cs: -------------------------------------------------------------------------------- 1 | using Newtonsoft.Json; 2 | using System.Collections.Generic; 3 | 4 | namespace SongDataCore.BeatStar 5 | { 6 | public class BeatStarSong 7 | { 8 | public string key { get; set; } 9 | //public string song { get; set; } 10 | //public string mapper { get; set; } 11 | public List diffs { get; set; } 12 | 13 | public float bpm { get; set; } 14 | public int downloadCount { get; set; } 15 | public int upVotes { get; set; } 16 | public int downVotes { get; set; } 17 | public float heat { get; set; } 18 | public float rating { get; set; } 19 | 20 | [JsonIgnore] 21 | public Dictionary> characteristics { get; set; } 22 | } 23 | 24 | public class BeatStarSongDifficultyStats 25 | { 26 | public string diff { get; set; } 27 | public long scores { get; set; } 28 | public double star { get; set; } 29 | public double pp { get; set; } 30 | 31 | public int type { get; set; } 32 | public int len { get; set; } 33 | public int njs { get; set; } 34 | public int bmb { get; set; } 35 | public int nts { get; set; } 36 | public int obs{ get; set; } 37 | } 38 | 39 | public enum BeatStarCharacteristics 40 | { 41 | Unkown, 42 | Standard, 43 | OneSaber, 44 | NoArrows, 45 | Lightshow, 46 | Degree90, 47 | Degree360, 48 | Lawless, 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /SongDataCore/BeatStar/CacheableBeatStarDownloadHandler.cs: -------------------------------------------------------------------------------- 1 | using SongDataCore.Downloader; 2 | using UnityEngine.Networking; 3 | 4 | namespace SongDataCore.BeatStar 5 | { 6 | /// 7 | /// Cacheable download handler for score saber v2 json. 8 | /// 9 | public class CacheableBeatStarDownloaderHandler : CacheableDownloadHandler 10 | { 11 | BeatStarDataFile _dataFile; 12 | 13 | public CacheableBeatStarDownloaderHandler(UnityWebRequest www, byte[] preallocateBuffer) 14 | : base(www, preallocateBuffer) 15 | { 16 | } 17 | 18 | /// 19 | /// Returns the downloaded score saber data file or null. 20 | /// 21 | public BeatStarDataFile DataFile 22 | { 23 | get 24 | { 25 | if (_dataFile == null) 26 | { 27 | _dataFile = new BeatStarDataFile(GetData()); 28 | } 29 | return _dataFile; 30 | } 31 | } 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /SongDataCore/Directory.Build.props: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | true 9 | true 10 | true 11 | 12 | 13 | false 14 | true 15 | true 16 | 17 | -------------------------------------------------------------------------------- /SongDataCore/Directory.Build.targets: -------------------------------------------------------------------------------- 1 |  2 | 4 | 5 | 6 | 2.0 7 | 8 | false 9 | 10 | $(OutputPath)$(AssemblyName) 11 | 12 | $(OutputPath)Final 13 | True 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | $(BasePluginVersion) 30 | $(BasePluginVersion) 31 | $(BasePluginVersion) 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | $(AssemblyName) 41 | $(ArtifactName)-$(PluginVersion) 42 | $(ArtifactName)-bs$(GameVersion) 43 | $(ArtifactName)-$(CommitHash) 44 | 45 | 46 | 47 | 48 | 49 | 50 | $(AssemblyName) 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | $(AssemblyName) 65 | $(OutDir)zip\ 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | 75 | 76 | 77 | 78 | $(BeatSaberDir)\Plugins 79 | True 80 | Unable to copy assembly to game folder, did you set 'BeatSaberDir' correctly in your 'csproj.user' file? Plugins folder doesn't exist: '$(PluginDir)'. 81 | 82 | Unable to copy to Plugins folder, '$(BeatSaberDir)' does not appear to be a Beat Saber game install. 83 | 84 | Unable to copy to Plugins folder, 'BeatSaberDir' has not been set in your 'csproj.user' file. 85 | False 86 | 87 | 88 | 89 | 90 | 91 | 92 | 93 | $(BeatSaberDir)\IPA\Pending\Plugins 94 | 95 | 96 | 97 | 98 | 99 | 100 | 101 | -------------------------------------------------------------------------------- /SongDataCore/Downloader/CacheableDownloaderHandler.cs: -------------------------------------------------------------------------------- 1 | using UnityEngine.Networking; 2 | using System.IO; 3 | using System.Security.Cryptography; 4 | using System.Text; 5 | using System; 6 | using UnityEngine; 7 | 8 | namespace SongDataCore.Downloader 9 | { 10 | // Modified Version of: 11 | // https://github.com/mob-sakai/AssetSystem/blob/master/Assets/Mobcast/Coffee/AssetSystem/CacheableDownloadHandler.cs 12 | // MIT-LICENSE - https://github.com/mob-sakai/AssetSystem/blob/master/LICENSE 13 | // Modified to use custom logging only. 14 | public static class UnityWebRequestCachingExtensions 15 | { 16 | /// 17 | /// Set UnityWebRequest to be cacheable(Etag). 18 | /// 19 | public static void SetCacheable(this UnityWebRequest www, CacheableDownloadHandler handler) 20 | { 21 | var etag = handler.GetCacheEtag(handler.OriginalUrl); 22 | if (etag != null) 23 | www.SetRequestHeader("If-None-Match", etag); 24 | www.downloadHandler = handler; 25 | } 26 | } 27 | 28 | /// 29 | /// Cacheable download handler. 30 | /// 31 | public abstract class CacheableDownloadHandler : DownloadHandlerScript 32 | { 33 | const string kLog = "[WebRequestCaching] "; 34 | const string kDataSufix = "_d"; 35 | const string kEtagSufix = "_e"; 36 | 37 | private string m_WebCachePath; 38 | private SHA1CryptoServiceProvider m_SHA1 = new SHA1CryptoServiceProvider(); 39 | 40 | /// 41 | /// Is the download already finished? 42 | /// 43 | public new bool isDone { get; private set; } 44 | 45 | private UnityWebRequest m_WebRequest; 46 | private MemoryStream m_Stream; 47 | 48 | protected byte[] m_Buffer; 49 | 50 | private String m_originalUrl; 51 | public String OriginalUrl { get { return m_originalUrl; } } 52 | 53 | internal CacheableDownloadHandler(UnityWebRequest www, byte[] preallocateBuffer) 54 | : base(preallocateBuffer) 55 | { 56 | this.m_WebRequest = www; 57 | m_originalUrl = www.url; 58 | m_Stream = new MemoryStream(preallocateBuffer.Length); 59 | } 60 | 61 | /// 62 | /// Get path for web-caching. 63 | /// 64 | public string GetCachePath(string url) 65 | { 66 | if (m_WebCachePath == null) 67 | { 68 | m_WebCachePath = Application.temporaryCachePath + "/WebCache/"; 69 | Plugin.Log.Debug($"{kLog}WebCachePath : {m_WebCachePath}"); 70 | } 71 | 72 | if (!Directory.Exists(m_WebCachePath)) 73 | Directory.CreateDirectory(m_WebCachePath); 74 | 75 | return m_WebCachePath + Convert.ToBase64String(m_SHA1.ComputeHash(UTF8Encoding.Default.GetBytes(url))).Replace('/', '_'); 76 | } 77 | 78 | /// 79 | /// Get cached Etag for url. 80 | /// 81 | public string GetCacheEtag(string url) 82 | { 83 | var path = GetCachePath(url); 84 | var infoPath = path + kEtagSufix; 85 | var dataPath = path + kDataSufix; 86 | return File.Exists(infoPath) && File.Exists(dataPath) 87 | ? File.ReadAllText(infoPath) 88 | : null; 89 | } 90 | 91 | /// 92 | /// Load cached data for url. 93 | /// 94 | public byte[] LoadCache(string url) 95 | { 96 | return File.ReadAllBytes(GetCachePath(url) + kDataSufix); 97 | } 98 | 99 | /// 100 | /// Save cache data for url. 101 | /// 102 | public void SaveCache(string url, string etag, byte[] datas) 103 | { 104 | var path = GetCachePath(url); 105 | 106 | File.WriteAllText(path + kEtagSufix, etag); 107 | File.WriteAllBytes(path + kDataSufix, datas); 108 | } 109 | 110 | /// 111 | /// Callback, invoked when the data property is accessed. 112 | /// 113 | protected override byte[] GetData() 114 | { 115 | try 116 | { 117 | if (!isDone) 118 | { 119 | Plugin.Log.Error($"{kLog}Downloading is not completed : {m_WebRequest.url}"); 120 | throw new InvalidOperationException("Downloading is not completed. " + m_WebRequest.url); 121 | } 122 | else if (m_Buffer == null) 123 | { 124 | // Etag cache hit! 125 | if (m_WebRequest.responseCode == 304) 126 | { 127 | Plugin.Log.Debug($"{kLog}Etag cache hit : {m_WebRequest.url}"); 128 | m_Buffer = LoadCache(m_originalUrl); 129 | } 130 | // Download is completed successfully. 131 | else if (m_WebRequest.responseCode == 200) 132 | { 133 | Plugin.Log.Debug($"{kLog}Download is completed successfully : {m_WebRequest.url}"); 134 | m_Buffer = m_Stream.GetBuffer(); 135 | SaveCache(m_originalUrl, m_WebRequest.GetResponseHeader("Etag"), m_Buffer); 136 | } 137 | } 138 | 139 | if (m_Stream != null) 140 | { 141 | m_Stream.Dispose(); 142 | m_Stream = null; 143 | } 144 | return m_Buffer; 145 | } 146 | catch (Exception e) 147 | { 148 | Plugin.Log.Critical(e); 149 | return null; 150 | } 151 | } 152 | 153 | /// 154 | /// Callback, invoked as data is received from the remote server. 155 | /// 156 | protected override bool ReceiveData(byte[] data, int dataLength) 157 | { 158 | m_Stream.Write(data, 0, dataLength); 159 | return true; 160 | } 161 | 162 | /// 163 | /// Callback, invoked when all data has been received from the remote server. 164 | /// 165 | protected override void CompleteContent() 166 | { 167 | base.CompleteContent(); 168 | isDone = true; 169 | } 170 | 171 | /// 172 | /// Signals that this [DownloadHandler] is no longer being used, and should clean up any resources it is using. 173 | /// 174 | public new void Dispose() 175 | { 176 | base.Dispose(); 177 | if (m_Stream != null) 178 | { 179 | m_Stream.Dispose(); 180 | m_Stream = null; 181 | } 182 | } 183 | } 184 | } 185 | -------------------------------------------------------------------------------- /SongDataCore/Downloader/DatabaseDownloader.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections; 3 | using UnityEngine; 4 | using UnityEngine.Networking; 5 | using System.Threading.Tasks; 6 | using UnityEngine.SceneManagement; 7 | using System.Threading; 8 | using System.IO; 9 | 10 | namespace SongDataCore.Downloader 11 | { 12 | public abstract class DatabaseDownloader : MonoBehaviour 13 | { 14 | public Action OnStartDownloading; 15 | public Action OnFinishDownloading; 16 | public Action OnDataFinishedProcessing; 17 | public Action OnFailedDownload; 18 | 19 | private bool _backgroundHandlerRunning; 20 | 21 | protected bool _cancelRequested = false; 22 | public bool CancelRequested { get => _cancelRequested; set => _cancelRequested = value; } 23 | 24 | protected bool _isDownloading = false; 25 | public bool IsDownloading { get => _isDownloading; } 26 | 27 | protected bool _firstSuccess = false; 28 | 29 | public abstract void Load(); 30 | public abstract void Unload(); 31 | 32 | /// 33 | /// Block and wait for download operation to cancel. 34 | /// 35 | /// 36 | protected IEnumerator CancelDownload() 37 | { 38 | Plugin.Log.Debug("Attempting to cancel download..."); 39 | 40 | _cancelRequested = true; 41 | 42 | while (_isDownloading) 43 | { 44 | yield return null; 45 | } 46 | } 47 | 48 | /// 49 | /// Download a database, invoke the appropriate actions to inform listerns. 50 | /// 51 | /// 52 | protected IEnumerator DownloadDatabase(String url, IDatabaseDownloadHandler handler) 53 | { 54 | OnStartDownloading?.Invoke(); 55 | 56 | Plugin.Log.Info($"Preparing to download: {url}"); 57 | using (UnityWebRequest www = UnityWebRequest.Get(url)) 58 | { 59 | var cacheHandler = handler.GetDownloadHandler(www); 60 | if (cacheHandler == null) 61 | { 62 | OnFailedDownload?.Invoke(); 63 | Plugin.Log.Error($"Could not acquire a download handler for URL {url}"); 64 | yield break; 65 | } 66 | 67 | // && File.Exists(cacheHandler.GetCachePath(cacheHandler.OriginalUrl)) 68 | if (!_firstSuccess) 69 | { 70 | Plugin.Log.Debug($"Sending Web Request: {url}"); 71 | 72 | yield return www.SendWebRequest(); 73 | 74 | if (www.isNetworkError) 75 | { 76 | Plugin.Log.Error($"Network error downloading: {url}"); 77 | yield break; 78 | } 79 | 80 | Plugin.Log.Debug($"Success downloading data!"); 81 | 82 | if (_cancelRequested) 83 | { 84 | Plugin.Log.Debug("Cancel requested after download."); 85 | yield break; 86 | } 87 | } 88 | else 89 | { 90 | Plugin.Log.Debug($"Force the use of the cached data without causing a server request..."); 91 | } 92 | 93 | OnFinishDownloading?.Invoke(); 94 | 95 | Task myTask = null; 96 | CancellationTokenSource tokenSource = new CancellationTokenSource(); 97 | try 98 | { 99 | Plugin.Log.Debug($"Started data processing thread!"); 100 | _backgroundHandlerRunning = true; 101 | myTask = Task.Run(() => BackgroundDownloadHandler(handler, www)); 102 | } 103 | catch (System.InvalidOperationException) 104 | { 105 | Plugin.Log.Error($"Failed to download data file..."); 106 | OnFailedDownload?.Invoke(); 107 | } 108 | catch (Exception e) 109 | { 110 | Plugin.Log.Critical($"Exception trying to download data file... {e}"); 111 | OnFailedDownload?.Invoke(); 112 | } 113 | 114 | // Wait for the thread to finish/die 115 | bool cancelled = false; 116 | while (_backgroundHandlerRunning) 117 | { 118 | if (!cancelled && _cancelRequested) 119 | { 120 | Plugin.Log.Debug("Cancel requested during parsing"); 121 | cancelled = true; 122 | handler.CancelHandler(cacheHandler); 123 | } 124 | 125 | yield return null; 126 | } 127 | 128 | yield return new WaitForEndOfFrame(); 129 | 130 | if (_cancelRequested) 131 | { 132 | Plugin.Log.Debug("Cancel requested after parsing, aborting."); 133 | yield break; 134 | } 135 | 136 | Scene scene = SceneManager.GetActiveScene(); 137 | if (scene.name == "GameCore") 138 | { 139 | Plugin.Log.Debug("SongDataCore cannot fire events outside of menu scene."); 140 | yield break; 141 | } 142 | 143 | OnDataFinishedProcessing?.Invoke(); 144 | } 145 | } 146 | 147 | private void BackgroundDownloadHandler(IDatabaseDownloadHandler handler, UnityWebRequest www) 148 | { 149 | try 150 | { 151 | if (_cancelRequested) 152 | { 153 | Plugin.Log.Debug("Cancel requested mid-download, aborting."); 154 | return; 155 | } 156 | 157 | handler.HandleDownloadResults(www.downloadHandler); 158 | Plugin.Log.Info($"Success processing data: {www.url}"); 159 | } 160 | finally 161 | { 162 | _backgroundHandlerRunning = false; 163 | } 164 | } 165 | } 166 | } 167 | -------------------------------------------------------------------------------- /SongDataCore/Downloader/IDatabaseDownloadHandler.cs: -------------------------------------------------------------------------------- 1 | using UnityEngine.Networking; 2 | 3 | namespace SongDataCore.Downloader 4 | { 5 | public interface IDatabaseDownloadHandler 6 | { 7 | CacheableDownloadHandler GetDownloadHandler(UnityWebRequest www); 8 | 9 | void HandleDownloadResults(DownloadHandler handler); 10 | 11 | void CancelHandler(DownloadHandler handler); 12 | 13 | bool IsDataAvailable(); 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /SongDataCore/Internal/JsonExtensions.cs: -------------------------------------------------------------------------------- 1 | using Newtonsoft.Json; 2 | using System.Collections.Generic; 3 | using System.IO; 4 | 5 | namespace SongDataCore.Internal 6 | { 7 | /// 8 | /// IEnumerable way to Deserialize a JsonStream. 9 | /// Taken from: https://stackoverflow.com/a/37819584/543700 10 | /// 11 | public static partial class JsonExtensions 12 | { 13 | public static IEnumerable DeserializeValues(Stream stream) 14 | { 15 | return DeserializeValues(new StreamReader(stream)); 16 | } 17 | 18 | public static IEnumerable DeserializeValues(TextReader textReader) 19 | { 20 | var serializer = JsonSerializer.CreateDefault(); 21 | var reader = new JsonTextReader(textReader); 22 | reader.SupportMultipleContent = true; 23 | while (reader.Read()) 24 | { 25 | if (reader.TokenType == JsonToken.StartArray) 26 | { 27 | while (reader.Read()) 28 | { 29 | if (reader.TokenType == JsonToken.Comment) 30 | continue; 31 | else if (reader.TokenType == JsonToken.EndArray) 32 | break; 33 | else 34 | yield return serializer.Deserialize(reader); 35 | } 36 | } 37 | else if (reader.TokenType == JsonToken.StartObject) 38 | { 39 | while (reader.Read()) 40 | { 41 | if (reader.TokenType == JsonToken.Comment) 42 | continue; 43 | else if (reader.TokenType == JsonToken.PropertyName) 44 | continue; 45 | else if (reader.TokenType == JsonToken.EndObject) 46 | break; 47 | else 48 | yield return serializer.Deserialize(reader); 49 | } 50 | } 51 | } 52 | } 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /SongDataCore/Plugin.cs: -------------------------------------------------------------------------------- 1 | using UnityEngine.SceneManagement; 2 | using UnityEngine; 3 | using SongDataCore.BeatStar; 4 | using IPA; 5 | using IPA.Logging; 6 | using BS_Utils.Utilities; 7 | using IPA.Loader; 8 | using System.Reflection; 9 | using System.Collections.Generic; 10 | using System.Linq; 11 | 12 | namespace SongDataCore 13 | { 14 | [Plugin(RuntimeOptions.SingleStartInit)] 15 | public class Plugin 16 | { 17 | public static string VersionNumber { get; private set; } 18 | 19 | public static Plugin Instance; 20 | public static IPA.Logging.Logger Log; 21 | 22 | public static BeatStarDatabase Songs; 23 | 24 | public bool DatabasesLoaded; 25 | 26 | bool hasDependent = false; 27 | 28 | [Init] 29 | public void Init(IPA.Logging.Logger logger, PluginMetadata metadata) 30 | { 31 | Log = logger; 32 | VersionNumber = metadata.Version?.ToString() ?? Assembly.GetExecutingAssembly().GetName().Version.ToString(3); 33 | } 34 | 35 | [OnStart] 36 | public void OnStart() 37 | { 38 | Instance = this; 39 | DatabasesLoaded = false; 40 | 41 | BSEvents.OnLoad(); 42 | 43 | BSEvents.lateMenuSceneLoadedFresh += OnMenuSceneLoadedFresh; 44 | BSEvents.menuSceneLoaded += OnMenuSceneLoaded; 45 | BSEvents.gameSceneLoaded += OnGameSceneLoaded; 46 | } 47 | 48 | [OnExit] 49 | public void OnExit() 50 | { 51 | 52 | } 53 | 54 | private void OnMenuSceneLoadedFresh(ScenesTransitionSetupDataSO data) 55 | { 56 | Log.Info("OnMenuSceneLoadedFresh()"); 57 | 58 | Songs = new GameObject("SongDataCore_BeatStar").AddComponent(); 59 | 60 | // Force false, always load the database here. 61 | DatabasesLoaded = false; 62 | 63 | hasDependent = false; 64 | foreach(var x in PluginManager.EnabledPlugins) { 65 | hasDependent = IPA.Utilities.ReflectionUtil.GetProperty, PluginMetadata>(x, "Dependencies").Any(x => x.Id == "SongDataCore"); 66 | 67 | if(hasDependent) 68 | break; 69 | } 70 | LoadDatabases(); 71 | } 72 | 73 | private void OnMenuSceneLoaded() 74 | { 75 | Log.Info("OnMenuSceneLoaded()"); 76 | 77 | LoadDatabases(); 78 | } 79 | 80 | private void OnGameSceneLoaded() 81 | { 82 | Log.Info("OnGameSceneLoaded()"); 83 | 84 | UnloadDatabases(); 85 | } 86 | 87 | private void LoadDatabases() 88 | { 89 | if (DatabasesLoaded || !hasDependent) return; 90 | 91 | Songs.Load(); 92 | 93 | DatabasesLoaded = true; 94 | } 95 | 96 | private void UnloadDatabases() 97 | { 98 | if (!DatabasesLoaded) return; 99 | 100 | if (Songs.isActiveAndEnabled) 101 | { 102 | Songs.Unload(); 103 | } else 104 | { 105 | Songs.UnloadNow(); 106 | } 107 | 108 | DatabasesLoaded = false; 109 | } 110 | } 111 | } 112 | -------------------------------------------------------------------------------- /SongDataCore/SongDataCore.csproj: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | Library 5 | Properties 6 | SongDataCore 7 | ..\Refs 8 | $(LocalRefsDir) 9 | $(MSBuildProjectDirectory)\ 10 | net472 11 | 8 12 | 13 | 14 | 15 | full 16 | 17 | 18 | 19 | pdbonly 20 | 21 | 22 | 23 | True 24 | 25 | 26 | 27 | True 28 | True 29 | 30 | 31 | 32 | 33 | $(BeatSaberDir)\Plugins\BS_Utils.dll 34 | 35 | 36 | $(BeatSaberDir)\Beat Saber_Data\Managed\HMLib.dll 37 | 38 | 39 | False 40 | $(BeatSaberDir)\Beat Saber_Data\Managed\IPA.Loader.dll 41 | 42 | 43 | $(BeatSaberDir)\Beat Saber_Data\Managed\Main.dll 44 | 45 | 46 | False 47 | $(BeatSaberDir)\Libs\Newtonsoft.Json.dll 48 | 49 | 50 | $(BeatSaberDir)\Libs\SemVer.dll 51 | 52 | 53 | $(BeatSaberDir)\Plugins\SongCore.dll 54 | 55 | 56 | $(BeatSaberDir)\Beat Saber_Data\Managed\UnityEngine.dll 57 | 58 | 59 | $(BeatSaberDir)\Beat Saber_Data\Managed\UnityEngine.CoreModule.dll 60 | 61 | 62 | $(BeatSaberDir)\Beat Saber_Data\Managed\UnityEngine.UI.dll 63 | 64 | 65 | False 66 | $(BeatSaberDir)\Beat Saber_Data\Managed\UnityEngine.UnityWebRequestModule.dll 67 | 68 | 69 | False 70 | $(BeatSaberDir)\Beat Saber_Data\Managed\UnityEngine.UnityWebRequestWWWModule.dll 71 | 72 | 73 | 74 | 75 | 76 | 77 | 78 | 79 | 80 | 81 | 82 | 83 | all 84 | runtime; build; native; contentfiles; analyzers; buildtransitive 85 | 86 | 87 | -------------------------------------------------------------------------------- /SongDataCore/SongDataCore.sln: -------------------------------------------------------------------------------- 1 |  2 | Microsoft Visual Studio Solution File, Format Version 12.00 3 | # Visual Studio 15 4 | VisualStudioVersion = 15.0.28307.572 5 | MinimumVisualStudioVersion = 10.0.40219.1 6 | Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SongDataCore", "SongDataCore.csproj", "{1135EA6F-EBC2-4C90-B3FA-7690665B295E}" 7 | EndProject 8 | Global 9 | GlobalSection(SolutionConfigurationPlatforms) = preSolution 10 | Debug|Any CPU = Debug|Any CPU 11 | Release|Any CPU = Release|Any CPU 12 | EndGlobalSection 13 | GlobalSection(ProjectConfigurationPlatforms) = postSolution 14 | {1135EA6F-EBC2-4C90-B3FA-7690665B295E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 15 | {1135EA6F-EBC2-4C90-B3FA-7690665B295E}.Debug|Any CPU.Build.0 = Debug|Any CPU 16 | {1135EA6F-EBC2-4C90-B3FA-7690665B295E}.Release|Any CPU.ActiveCfg = Release|Any CPU 17 | {1135EA6F-EBC2-4C90-B3FA-7690665B295E}.Release|Any CPU.Build.0 = Release|Any CPU 18 | EndGlobalSection 19 | GlobalSection(SolutionProperties) = preSolution 20 | HideSolutionNode = FALSE 21 | EndGlobalSection 22 | GlobalSection(ExtensibilityGlobals) = postSolution 23 | SolutionGuid = {F0655D52-0B13-4DFE-9D45-BD421BBBF901} 24 | EndGlobalSection 25 | EndGlobal 26 | -------------------------------------------------------------------------------- /SongDataCore/Template.txt: -------------------------------------------------------------------------------- 1 | SongDataCore 2 | 3 | 1.4.8 4 | 5 | BSIPA@4.2.2,SongCore@3.9.6,BS Utils@1.12.0 6 | 7 | Download and cache database dumps of ScoreSaber and BeatSaver. Map the information by song hash and make available to other plugins. 8 | 9 | https://github.com/halsafar/BeatSaberSongDataCore/ 10 | -------------------------------------------------------------------------------- /SongDataCore/manifest.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://raw.githubusercontent.com/bsmg/BSIPA-MetadataFileSchema/master/Schema.json", 3 | "author": "Halsafar", 4 | "description": "Manage various databases of Beat Saber song data.", 5 | "gameVersion": "1.21.0", 6 | "id": "SongDataCore", 7 | "name": "SongDataCore", 8 | "version": "1.4.8", 9 | "dependsOn": { 10 | "BSIPA": "^4.2.2", 11 | "BS Utils": "^1.12.0", 12 | "SongCore": "^3.9.6" 13 | }, 14 | "misc": { 15 | "plugin-hint": "SongDataCore.Plugin" 16 | } 17 | } 18 | --------------------------------------------------------------------------------