├── .github └── workflows │ ├── build.yml │ └── comments.yml ├── .gitignore ├── README.md ├── ct.bat ├── hsync.sln ├── hsync ├── AppProvider.cs ├── CL │ ├── CommandLine.cs │ └── IConsole.cs ├── CharacterTest.cs ├── Command.cs ├── Component │ ├── EH.cs │ ├── Hitomi.cs │ └── HitomiIndex.cs ├── Crypto │ ├── Hash.cs │ └── Random.cs ├── DataBaseCreator.cs ├── DataBaseCreatorLowPerf.cs ├── Internals.cs ├── Log │ └── Logs.cs ├── Network │ ├── NetField.cs │ ├── NetQueue.cs │ ├── NetTask.cs │ ├── NetTaskPass.cs │ └── NetTools.cs ├── Program.cs ├── Progress.cs ├── RelatedTagTest.cs ├── SeriesTest.cs ├── Setting │ └── Settings.cs ├── Syncronizer.cs ├── SyncronizerLowPerf.cs ├── Utils │ ├── Compress.cs │ ├── Extends.cs │ ├── Heap.cs │ ├── ILazy.cs │ └── Strings.cs ├── Version.cs ├── hsync-src.7z ├── hsync.csproj ├── publish-linux-x64-aot.bat ├── publish-linux-x64.bat ├── publish-osx.10.15-x64-aot.bat ├── publish-osx.10.15-x64.bat ├── publish-win10-arm-aot.bat ├── publish-win10-arm.bat ├── publish-win10-x64-aot.bat ├── publish-win10-x64.bat ├── publish-win10-x86-aot.bat └── publish-win10-x86.bat ├── rtt.bat └── sync.py /.github/workflows/build.yml: -------------------------------------------------------------------------------- 1 | name: build 2 | 3 | on: 4 | workflow_dispatch: 5 | 6 | jobs: 7 | build: 8 | runs-on: ubuntu-latest 9 | steps: 10 | - name: Checkout 11 | uses: actions/checkout@v2 12 | with: 13 | fetch-depth: 0 14 | - name: Install .NET Core 15 | uses: actions/setup-dotnet@v1 16 | with: 17 | dotnet-version: 5.0.x 18 | - name: Build 19 | run: | 20 | cd hsync 21 | dotnet publish -r linux-x64 -c Release /p:PublishSingleFile=true /p:PublishTrimmed=true /p:PublishReadyToRun=true 22 | cd bin/Release 23 | ls -al 24 | - name: Upload build artifacts 25 | uses: actions/upload-artifact@v2 26 | with: 27 | name: MSIX Package 28 | path: ./hsync/bin/Release/ 29 | -------------------------------------------------------------------------------- /.github/workflows/comments.yml: -------------------------------------------------------------------------------- 1 | name: comments 2 | 3 | on: 4 | workflow_dispatch: 5 | 6 | jobs: 7 | build: 8 | runs-on: ubuntu-latest 9 | steps: 10 | - name: Build 11 | run: | 12 | wget https://github.com/project-violet/hsync/releases/download/2021.08.17.1/hsync 13 | chmod 777 hsync 14 | wget https://github.com/violet-dev/sync-data/releases/download/db_1633821555/data.db 15 | mkdir ex 16 | ./hsync --save-exhentai-page 0 17 | zip -r a.zip ex 18 | - name: Upload build artifacts 19 | uses: actions/upload-artifact@v2 20 | with: 21 | name: Raw HTMLS 22 | path: ./a.zip 23 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | ## Ignore Visual Studio temporary files, build results, and 2 | ## files generated by popular Visual Studio add-ons. 3 | ## 4 | ## Get latest from https://github.com/github/gitignore/blob/master/VisualStudio.gitignore 5 | 6 | # User-specific files 7 | *.rsuser 8 | *.suo 9 | *.user 10 | *.userosscache 11 | *.sln.docstates 12 | 13 | # User-specific files (MonoDevelop/Xamarin Studio) 14 | *.userprefs 15 | 16 | # Mono auto generated files 17 | mono_crash.* 18 | 19 | # Build results 20 | [Dd]ebug/ 21 | [Dd]ebugPublic/ 22 | [Rr]elease/ 23 | [Rr]eleases/ 24 | x64/ 25 | x86/ 26 | [Aa][Rr][Mm]/ 27 | [Aa][Rr][Mm]64/ 28 | bld/ 29 | [Bb]in/ 30 | [Oo]bj/ 31 | [Ll]og/ 32 | [Ll]ogs/ 33 | 34 | # Visual Studio 2015/2017 cache/options directory 35 | .vs/ 36 | # Uncomment if you have tasks that create the project's static files in wwwroot 37 | #wwwroot/ 38 | 39 | # Visual Studio 2017 auto generated files 40 | Generated\ Files/ 41 | 42 | # MSTest test Results 43 | [Tt]est[Rr]esult*/ 44 | [Bb]uild[Ll]og.* 45 | 46 | # NUnit 47 | *.VisualState.xml 48 | TestResult.xml 49 | nunit-*.xml 50 | 51 | # Build Results of an ATL Project 52 | [Dd]ebugPS/ 53 | [Rr]eleasePS/ 54 | dlldata.c 55 | 56 | # Benchmark Results 57 | BenchmarkDotNet.Artifacts/ 58 | 59 | # .NET Core 60 | project.lock.json 61 | project.fragment.lock.json 62 | artifacts/ 63 | 64 | # StyleCop 65 | StyleCopReport.xml 66 | 67 | # Files built by Visual Studio 68 | *_i.c 69 | *_p.c 70 | *_h.h 71 | *.ilk 72 | *.meta 73 | *.obj 74 | *.iobj 75 | *.pch 76 | *.pdb 77 | *.ipdb 78 | *.pgc 79 | *.pgd 80 | *.rsp 81 | *.sbr 82 | *.tlb 83 | *.tli 84 | *.tlh 85 | *.tmp 86 | *.tmp_proj 87 | *_wpftmp.csproj 88 | *.log 89 | *.vspscc 90 | *.vssscc 91 | .builds 92 | *.pidb 93 | *.svclog 94 | *.scc 95 | 96 | # Chutzpah Test files 97 | _Chutzpah* 98 | 99 | # Visual C++ cache files 100 | ipch/ 101 | *.aps 102 | *.ncb 103 | *.opendb 104 | *.opensdf 105 | *.sdf 106 | *.cachefile 107 | *.VC.db 108 | *.VC.VC.opendb 109 | 110 | # Visual Studio profiler 111 | *.psess 112 | *.vsp 113 | *.vspx 114 | *.sap 115 | 116 | # Visual Studio Trace Files 117 | *.e2e 118 | 119 | # TFS 2012 Local Workspace 120 | $tf/ 121 | 122 | # Guidance Automation Toolkit 123 | *.gpState 124 | 125 | # ReSharper is a .NET coding add-in 126 | _ReSharper*/ 127 | *.[Rr]e[Ss]harper 128 | *.DotSettings.user 129 | 130 | # JustCode is a .NET coding add-in 131 | .JustCode 132 | 133 | # TeamCity is a build add-in 134 | _TeamCity* 135 | 136 | # DotCover is a Code Coverage Tool 137 | *.dotCover 138 | 139 | # AxoCover is a Code Coverage Tool 140 | .axoCover/* 141 | !.axoCover/settings.json 142 | 143 | # Visual Studio code coverage results 144 | *.coverage 145 | *.coveragexml 146 | 147 | # NCrunch 148 | _NCrunch_* 149 | .*crunch*.local.xml 150 | nCrunchTemp_* 151 | 152 | # MightyMoose 153 | *.mm.* 154 | AutoTest.Net/ 155 | 156 | # Web workbench (sass) 157 | .sass-cache/ 158 | 159 | # Installshield output folder 160 | [Ee]xpress/ 161 | 162 | # DocProject is a documentation generator add-in 163 | DocProject/buildhelp/ 164 | DocProject/Help/*.HxT 165 | DocProject/Help/*.HxC 166 | DocProject/Help/*.hhc 167 | DocProject/Help/*.hhk 168 | DocProject/Help/*.hhp 169 | DocProject/Help/Html2 170 | DocProject/Help/html 171 | 172 | # Click-Once directory 173 | publish/ 174 | 175 | # Publish Web Output 176 | *.[Pp]ublish.xml 177 | *.azurePubxml 178 | # Note: Comment the next line if you want to checkin your web deploy settings, 179 | # but database connection strings (with potential passwords) will be unencrypted 180 | *.pubxml 181 | *.publishproj 182 | 183 | # Microsoft Azure Web App publish settings. Comment the next line if you want to 184 | # checkin your Azure Web App publish settings, but sensitive information contained 185 | # in these scripts will be unencrypted 186 | PublishScripts/ 187 | 188 | # NuGet Packages 189 | *.nupkg 190 | # NuGet Symbol Packages 191 | *.snupkg 192 | # The packages folder can be ignored because of Package Restore 193 | **/[Pp]ackages/* 194 | # except build/, which is used as an MSBuild target. 195 | !**/[Pp]ackages/build/ 196 | # Uncomment if necessary however generally it will be regenerated when needed 197 | #!**/[Pp]ackages/repositories.config 198 | # NuGet v3's project.json files produces more ignorable files 199 | *.nuget.props 200 | *.nuget.targets 201 | 202 | # Microsoft Azure Build Output 203 | csx/ 204 | *.build.csdef 205 | 206 | # Microsoft Azure Emulator 207 | ecf/ 208 | rcf/ 209 | 210 | # Windows Store app package directories and files 211 | AppPackages/ 212 | BundleArtifacts/ 213 | Package.StoreAssociation.xml 214 | _pkginfo.txt 215 | *.appx 216 | *.appxbundle 217 | *.appxupload 218 | 219 | # Visual Studio cache files 220 | # files ending in .cache can be ignored 221 | *.[Cc]ache 222 | # but keep track of directories ending in .cache 223 | !?*.[Cc]ache/ 224 | 225 | # Others 226 | ClientBin/ 227 | ~$* 228 | *~ 229 | *.dbmdl 230 | *.dbproj.schemaview 231 | *.jfm 232 | *.pfx 233 | *.publishsettings 234 | orleans.codegen.cs 235 | 236 | # Including strong name files can present a security risk 237 | # (https://github.com/github/gitignore/pull/2483#issue-259490424) 238 | #*.snk 239 | 240 | # Since there are multiple workflows, uncomment next line to ignore bower_components 241 | # (https://github.com/github/gitignore/pull/1529#issuecomment-104372622) 242 | #bower_components/ 243 | 244 | # RIA/Silverlight projects 245 | Generated_Code/ 246 | 247 | # Backup & report files from converting an old project file 248 | # to a newer Visual Studio version. Backup files are not needed, 249 | # because we have git ;-) 250 | _UpgradeReport_Files/ 251 | Backup*/ 252 | UpgradeLog*.XML 253 | UpgradeLog*.htm 254 | ServiceFabricBackup/ 255 | *.rptproj.bak 256 | 257 | # SQL Server files 258 | *.mdf 259 | *.ldf 260 | *.ndf 261 | 262 | # Business Intelligence projects 263 | *.rdl.data 264 | *.bim.layout 265 | *.bim_*.settings 266 | *.rptproj.rsuser 267 | *- [Bb]ackup.rdl 268 | *- [Bb]ackup ([0-9]).rdl 269 | *- [Bb]ackup ([0-9][0-9]).rdl 270 | 271 | # Microsoft Fakes 272 | FakesAssemblies/ 273 | 274 | # GhostDoc plugin setting file 275 | *.GhostDoc.xml 276 | 277 | # Node.js Tools for Visual Studio 278 | .ntvs_analysis.dat 279 | node_modules/ 280 | 281 | # Visual Studio 6 build log 282 | *.plg 283 | 284 | # Visual Studio 6 workspace options file 285 | *.opt 286 | 287 | # Visual Studio 6 auto-generated workspace file (contains which files were open etc.) 288 | *.vbw 289 | 290 | # Visual Studio LightSwitch build output 291 | **/*.HTMLClient/GeneratedArtifacts 292 | **/*.DesktopClient/GeneratedArtifacts 293 | **/*.DesktopClient/ModelManifest.xml 294 | **/*.Server/GeneratedArtifacts 295 | **/*.Server/ModelManifest.xml 296 | _Pvt_Extensions 297 | 298 | # Paket dependency manager 299 | .paket/paket.exe 300 | paket-files/ 301 | 302 | # FAKE - F# Make 303 | .fake/ 304 | 305 | # CodeRush personal settings 306 | .cr/personal 307 | 308 | # Python Tools for Visual Studio (PTVS) 309 | __pycache__/ 310 | *.pyc 311 | 312 | # Cake - Uncomment if you are using it 313 | # tools/** 314 | # !tools/packages.config 315 | 316 | # Tabs Studio 317 | *.tss 318 | 319 | # Telerik's JustMock configuration file 320 | *.jmconfig 321 | 322 | # BizTalk build output 323 | *.btp.cs 324 | *.btm.cs 325 | *.odx.cs 326 | *.xsd.cs 327 | 328 | # OpenCover UI analysis results 329 | OpenCover/ 330 | 331 | # Azure Stream Analytics local run output 332 | ASALocalRun/ 333 | 334 | # MSBuild Binary and Structured Log 335 | *.binlog 336 | 337 | # NVidia Nsight GPU debugger configuration file 338 | *.nvuser 339 | 340 | # MFractors (Xamarin productivity tool) working folder 341 | .mfractor/ 342 | 343 | # Local History for Visual Studio 344 | .localhistory/ 345 | 346 | # BeatPulse healthcheck temp database 347 | healthchecksdb 348 | 349 | # Backup folder for Package Reference Convert tool in Visual Studio 2017 350 | MigrationBackup/ 351 | 352 | # Ionide (cross platform F# VS Code tools) working folder 353 | .ionide/ 354 | 355 | ################################################################# 356 | ################################################################# 357 | ################################################################# 358 | ################################################################# 359 | ################################################################# 360 | ################################################################# 361 | 362 | # Logs 363 | logs 364 | *.log 365 | npm-debug.log* 366 | yarn-debug.log* 367 | yarn-error.log* 368 | lerna-debug.log* 369 | 370 | # Diagnostic reports (https://nodejs.org/api/report.html) 371 | report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json 372 | 373 | # Runtime data 374 | pids 375 | *.pid 376 | *.seed 377 | *.pid.lock 378 | 379 | # Directory for instrumented libs generated by jscoverage/JSCover 380 | lib-cov 381 | 382 | # Coverage directory used by tools like istanbul 383 | coverage 384 | *.lcov 385 | 386 | # nyc test coverage 387 | .nyc_output 388 | 389 | # Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files) 390 | .grunt 391 | 392 | # Bower dependency directory (https://bower.io/) 393 | bower_components 394 | 395 | # node-waf configuration 396 | .lock-wscript 397 | 398 | # Compiled binary addons (https://nodejs.org/api/addons.html) 399 | build/Release 400 | 401 | # Dependency directories 402 | node_modules/ 403 | jspm_packages/ 404 | 405 | # Snowpack dependency directory (https://snowpack.dev/) 406 | web_modules/ 407 | 408 | # TypeScript cache 409 | *.tsbuildinfo 410 | 411 | # Optional npm cache directory 412 | .npm 413 | 414 | # Optional eslint cache 415 | .eslintcache 416 | 417 | # Microbundle cache 418 | .rpt2_cache/ 419 | .rts2_cache_cjs/ 420 | .rts2_cache_es/ 421 | .rts2_cache_umd/ 422 | 423 | # Optional REPL history 424 | .node_repl_history 425 | 426 | # Output of 'npm pack' 427 | *.tgz 428 | 429 | # Yarn Integrity file 430 | .yarn-integrity 431 | 432 | # dotenv environment variables file 433 | .env 434 | .env.test 435 | 436 | # parcel-bundler cache (https://parceljs.org/) 437 | .cache 438 | .parcel-cache 439 | 440 | # Next.js build output 441 | .next 442 | out 443 | 444 | # Nuxt.js build / generate output 445 | .nuxt 446 | dist 447 | 448 | # Gatsby files 449 | .cache/ 450 | # Comment in the public line in if your project uses Gatsby and not Next.js 451 | # https://nextjs.org/blog/next-9-1#public-directory-support 452 | # public 453 | 454 | # vuepress build output 455 | .vuepress/dist 456 | 457 | # Serverless directories 458 | .serverless/ 459 | 460 | # FuseBox cache 461 | .fusebox/ 462 | 463 | # DynamoDB Local files 464 | .dynamodb/ 465 | 466 | # TernJS port file 467 | .tern-port 468 | 469 | # Stores VSCode versions used for testing VSCode extensions 470 | .vscode-test 471 | 472 | # yarn v2 473 | .yarn/cache 474 | .yarn/unplugged 475 | .yarn/build-state.yml 476 | .yarn/install-state.gz 477 | .pnp.* 478 | 479 | blacklist.txt 480 | 481 | log/* 482 | config/production.json 483 | export.cmd -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # hsync 2 | 3 | ``` 4 | hsync 2021.4.16 5 | Build Date: Saturday, 17 April 2021 6 | /T /I 7 | / |/ | .-~/ 8 | T\ Y I |/ / _ 9 | /T | \I | I Y.-~/ 10 | I l /I T\ | | l | T / 11 | T\ | \ Y l /T | \I l \ ` l Y 12 | __ | \l \l \I l __l l \ ` _. | 13 | \ ~-l `\ `\ \ \\ ~\ \ `. .-~ | 14 | \ ~-. "-. ` \ ^._ ^. "-. / \ | 15 | .--~-._ ~- ` _ ~-_.-"-." ._ /._ ." ./ 16 | >--. ~-. ._ ~>-" "\\ 7 7 ] 17 | ^.___~"--._ ~-{ .-~ . `\ Y . / | 18 | <__ ~"-. ~ /_/ \ \I Y : | 19 | ^-.__ ~(_/ \ >._: | l______ 20 | ^--.,___.-~" /_/ ! `-.~"--l_ / ~"-. 21 | (_/ . ~( /' "~"--,Y -=b-. _) 22 | (_/ . \ : / l c"~o \ 23 | \ / `. . .^ \_.-~"~--. ) 24 | (_/ . ` / / ! )/ 25 | / / _. '. .': / ' 26 | ~(_/ . / _ ` .-<_ 27 | /_/ . ' .-~" `. / \ \ ,z=. 28 | ~( / ' : | K "-.~-.______// 29 | "-,. l I/ \_ __{--->._(==. 30 | //( \ < ~"~" // 31 | /' /\ \ \ ,v=. (( 32 | .^. / /\ " }__ //===- ` 33 | / / ' ' "-.,__ {---(==- 34 | .^ ' : T ~" ll 35 | / . . . : | :! \\ 36 | (_/ / | | j-" ~^ 37 | ~-<_(_.^-~" 38 | 39 | Copyright (C) 2020-2021. project violet-server. 40 | Usage: ./hsync [OPTIONS...] 41 | --help 42 | -v, --version Show version information. 43 | --recover-settings Recover settings.json 44 | -r, --related-tag-test Related Tag Test [use --related-tag-test ] 45 | -h, --character-test Character Test [use --character-tag-test ] 46 | -p, --series-test Series Test [use --series-tag-test ] 47 | --create-ehentai-inv-table create e/exhentai hash inverse table [use --create-ehentai-inv-table] 48 | --create-datetime-estimator create datetime estimator [use --create-datetime-estimator] 49 | --init-server Upload all data to server database [use --init-server] 50 | --init-server-pages Upload all data to server article pages [use --init-server-pages] 51 | --export-for-es Export database bulk datas for elastic-search to json [use --export-for-es] 52 | --export-for-es-range Export database bulk datas for elastic-search to json using id range [--export-for-es-range] 53 | --export-for-db-range Upload data to server database by user range [--export-for-db-range] 54 | -s, --start Starts hsync [use --start] 55 | -c, --compress Compress exists data [use --compress] 56 | -x, --include-exhentai Include ExHentai Database [use --include-exhentai] 57 | -l, --low-perf hsync run on low performance system [use --low-perf] 58 | -n, --sync-only Sync only when start [use --sync-only] 59 | --hitomi-sync-range Set lookup id range manually [use --hitomi-sync-range ] 60 | --hitomi-sync-lookup-range Set hitomi id lookup range. (default: 4,000 [-4,000 ~ 4,000]) [use --hitomi-sync-lookup-range ] 61 | --exhentai-lookup-page Set exhentai lookup page. (default: 200) [use --exhentai-lookup-page ] 62 | -e, --use-server Upload sync data to server database [use --use-server] 63 | -a, --use-elastic-search Upload sync data to elastic-search server [use --use-elastic-search] 64 | --sync-only-hitomi Sync only hitomi [use --sync-only-hitomi] 65 | -t, --test hysnc test option [use --test ] 66 | ``` 67 | 68 | High-Performance E-Hentai/EX-Hentai/Hitomi Works Data Synchronizer 69 | 70 | ## Public Sync Data 71 | 72 | You can see real-time synchronization information from https://koromo.xyz/version.txt 73 | See sync.py for more information. 74 | 75 | ## How to use? 76 | 77 | At least 8 GB of memory is required. 78 | 79 | ``` 80 | 1. Set your own hitomi.la crawling range 81 | https://github.com/project-violet/violet-server/blob/master/tools/hsync/hsync/Syncronizer.cs#L32 82 | This use exhentai-based ID. 83 | The default of 6,000 means to crawl 157,000 to 163,000 based on 160,000. 84 | 85 | 2. Set exhentai.org search range 86 | https://github.com/project-violet/violet-server/blob/master/tools/hsync/hsync/Syncronizer.cs#L175 87 | This use exhentai search result page number. 88 | You need to set up how many pages you want to browse. The default is 150 pages. 89 | 90 | 3. Build 91 | I recommend the debug build, but if you want to release, 92 | check the batch files in https://github.com/project-violet/violet-server/tree/master/tools/hsync/hsyncc folder. 93 | 94 | 4. Download base database 95 | Download base database from https://github.com/project-violet/violet-server/releases 96 | 97 | 5. Run ./hsync.exe 98 | 99 | 6. Wait for complete 100 | ``` 101 | 102 | ### (Option) For Server or Embedded System 103 | 104 | This option is implemented to run on low performance system. 105 | 106 | #### Linux x64 107 | 108 | ``` 109 | 1. Install 110 | mkdir sync 111 | cd sync 112 | wget https://github.com/project-violet/hsync/releases/download/2020.10.11/hsync 113 | chmod 777 hsync 114 | wget https://github.com/project-violet/hsync/releases/download/2020.10.11/data.db 115 | mkdir runtimes 116 | cd runtimes 117 | wget https://github.com/project-violet/hsync/releases/download/2020.10.11/runtimes.zip 118 | unzip runtimes.zip 119 | cd .. 120 | 121 | 2. Syncronize 122 | ./hsync --start --low-perf 123 | ``` 124 | 125 | #### Customize Performance 126 | 127 | ``` 128 | 1. Adjust NetQueue thread pool min threads count 129 | https://github.com/project-violet/hsync/blob/master/hsync/Network/NetQueue.cs#L30 130 | 131 | 2. Adjust database query buffer size 132 | https://github.com/project-violet/hsync/blob/master/hsync/DataBaseCreatorLowPerf.cs#L62 133 | ``` 134 | -------------------------------------------------------------------------------- /ct.bat: -------------------------------------------------------------------------------- 1 | hsync.exe -h rawdata/data.db 0.0000001 2 | hsync.exe -h rawdata-chinese/data.db 0.0000001 3 | hsync.exe -h rawdata-english/data.db 0.0000001 4 | hsync.exe -h rawdata-japanese/data.db 0.0000001 5 | hsync.exe -h rawdata-korean/data.db 0.0000001 -------------------------------------------------------------------------------- /hsync.sln: -------------------------------------------------------------------------------- 1 |  2 | Microsoft Visual Studio Solution File, Format Version 12.00 3 | # Visual Studio Version 16 4 | VisualStudioVersion = 16.0.29709.97 5 | MinimumVisualStudioVersion = 10.0.40219.1 6 | Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "hsync", "hsync\hsync.csproj", "{9E14C781-D828-4E0A-8735-A867754F139E}" 7 | EndProject 8 | Global 9 | GlobalSection(SolutionConfigurationPlatforms) = preSolution 10 | Debug|Any CPU = Debug|Any CPU 11 | Debug|x64 = Debug|x64 12 | Release|Any CPU = Release|Any CPU 13 | Release|x64 = Release|x64 14 | EndGlobalSection 15 | GlobalSection(ProjectConfigurationPlatforms) = postSolution 16 | {9E14C781-D828-4E0A-8735-A867754F139E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 17 | {9E14C781-D828-4E0A-8735-A867754F139E}.Debug|Any CPU.Build.0 = Debug|Any CPU 18 | {9E14C781-D828-4E0A-8735-A867754F139E}.Debug|x64.ActiveCfg = Debug|x64 19 | {9E14C781-D828-4E0A-8735-A867754F139E}.Debug|x64.Build.0 = Debug|x64 20 | {9E14C781-D828-4E0A-8735-A867754F139E}.Release|Any CPU.ActiveCfg = Release|Any CPU 21 | {9E14C781-D828-4E0A-8735-A867754F139E}.Release|Any CPU.Build.0 = Release|Any CPU 22 | {9E14C781-D828-4E0A-8735-A867754F139E}.Release|x64.ActiveCfg = Release|x64 23 | {9E14C781-D828-4E0A-8735-A867754F139E}.Release|x64.Build.0 = Release|x64 24 | EndGlobalSection 25 | GlobalSection(SolutionProperties) = preSolution 26 | HideSolutionNode = FALSE 27 | EndGlobalSection 28 | GlobalSection(ExtensibilityGlobals) = postSolution 29 | SolutionGuid = {22A5A121-015F-4D9D-BDEF-EF24D7E8A108} 30 | EndGlobalSection 31 | EndGlobal 32 | -------------------------------------------------------------------------------- /hsync/AppProvider.cs: -------------------------------------------------------------------------------- 1 | // This source code is a part of project violet-server. 2 | // Copyright (C)2020-2021. violet-team. Licensed under the MIT Licence. 3 | 4 | using hsync.Log; 5 | using hsync.Network; 6 | using hsync.Setting; 7 | using hsync.Utils; 8 | using System; 9 | using System.Collections.Generic; 10 | using System.Globalization; 11 | using System.IO; 12 | using System.Linq; 13 | using System.Net; 14 | using System.Reflection; 15 | using System.Runtime; 16 | using System.Runtime.CompilerServices; 17 | using System.Text; 18 | using System.Threading; 19 | 20 | namespace hsync 21 | { 22 | public class AppProvider 23 | { 24 | public static string ApplicationPath = Directory.GetCurrentDirectory(); 25 | public static string DefaultSuperPath = Directory.GetCurrentDirectory(); 26 | 27 | public static Dictionary Instance => 28 | InstanceMonitor.Instances; 29 | 30 | public static NetQueue DownloadQueue { get; set; } 31 | 32 | public static DateTime StartTime = DateTime.Now; 33 | 34 | public static void Initialize() 35 | { 36 | // Initialize logs instance 37 | GCLatencyMode oldMode = GCSettings.LatencyMode; 38 | RuntimeHelpers.PrepareConstrainedRegions(); 39 | GCSettings.LatencyMode = GCLatencyMode.Batch; 40 | 41 | ServicePointManager.DefaultConnectionLimit = int.MaxValue; 42 | 43 | DownloadQueue = new NetQueue(Settings.Instance.Model.ThreadCount); 44 | 45 | GC.Collect(GC.MaxGeneration, GCCollectionMode.Forced); 46 | } 47 | 48 | public static void Deinitialize() 49 | { 50 | } 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /hsync/CL/CommandLine.cs: -------------------------------------------------------------------------------- 1 | // This source code is a part of project violet-server. 2 | // Copyright (C)2020-2021. violet-team. Licensed under the MIT Licence. 3 | 4 | using System; 5 | using System.Collections.Generic; 6 | using System.Linq; 7 | using System.Reflection; 8 | using System.Text; 9 | 10 | namespace hsync.CL 11 | { 12 | public enum CommandType 13 | { 14 | OPTION, 15 | ARGUMENTS, 16 | EQUAL, 17 | } 18 | 19 | /// 20 | /// Attribute model for the command line parser. 21 | /// 22 | [AttributeUsage(AttributeTargets.Field)] 23 | public class CommandLine : Attribute 24 | { 25 | public CommandType CType { get; private set; } 26 | public string Option { get; private set; } 27 | 28 | /// 29 | /// Usage information of the command. 30 | /// 31 | public string Info { get; set; } 32 | 33 | /// 34 | /// Default argument to insert if the argument does not match. 35 | /// 36 | public bool DefaultArgument { get; set; } 37 | 38 | /// 39 | /// Indicates that the variable will use a pipe. 40 | /// 41 | public bool Pipe { get; set; } 42 | 43 | /// 44 | /// This message is displayed when the command syntax is incorrect. 45 | /// 46 | public string Help { get; set; } 47 | 48 | /// 49 | /// This value is set to true when nothing is entered in the pipe. 50 | /// 51 | public bool PipeDefault { get; set; } = false; 52 | 53 | /// 54 | /// This value is set to true when nothing is entered. 55 | /// 56 | public bool Default { get; set; } = false; 57 | 58 | /// 59 | /// For ARGUMENTS type, specifies the number of arguments. 60 | /// 61 | public int ArgumentsCount { get; set; } = 1; 62 | 63 | /// 64 | /// One character option 65 | /// 66 | public string ShortOption { get; set; } 67 | 68 | /// 69 | /// 70 | /// 71 | /// Option token. 72 | /// 73 | public CommandLine(string option, CommandType type) 74 | { 75 | Option = option; 76 | CType = type; 77 | } 78 | } 79 | 80 | /// 81 | /// Tools for organizing the command line. 82 | /// 83 | public class CommandLineUtil 84 | { 85 | /// 86 | /// Checks whether there is an option in the argument array. 87 | /// 88 | /// 89 | /// 90 | public static bool AnyOption(string[] args) 91 | { 92 | return args.ToList().Any(x => x[0] == '-'); 93 | } 94 | 95 | /// 96 | /// Checks whether the argument array contains a string. 97 | /// 98 | /// 99 | /// 100 | public static bool AnyStrings(string[] args) 101 | { 102 | return args.ToList().Any(x => x[0] != '-'); 103 | } 104 | 105 | /// 106 | /// Gets whether a specific argument is included. 107 | /// 108 | /// 109 | /// 110 | /// 111 | public static bool AnyArgument(string[] args, string arg) 112 | { 113 | return args.ToList().Any(x => x == arg); 114 | } 115 | 116 | /// 117 | /// Delete a specific argument. 118 | /// 119 | /// 120 | /// 121 | public static string[] DeleteArgument(string[] args, string arg) 122 | { 123 | var list = args.ToList(); 124 | list.Remove(arg); 125 | return list.ToArray(); 126 | } 127 | 128 | /// 129 | /// Put specific options at the beginning. 130 | /// 131 | /// 132 | /// 133 | /// 134 | public static string[] PushFront(string[] args, string option) 135 | { 136 | var list = args.ToList(); 137 | list.Insert(0, option); 138 | return list.ToArray(); 139 | } 140 | 141 | /// 142 | /// Put specific options in specific locations. 143 | /// 144 | /// 145 | /// 146 | /// 147 | public static string[] Insert(string[] args, string option, int index) 148 | { 149 | var list = args.ToList(); 150 | list.Insert(index, option); 151 | return list.ToArray(); 152 | } 153 | 154 | /// 155 | /// If there is a mismatched argument, get it. 156 | /// 157 | /// 158 | /// 159 | /// 160 | public static List GetWeirdArguments(string[] argv) 161 | where T : IConsoleOption, new() 162 | { 163 | var field = CommandLineParser.GetFields(typeof(T)); 164 | List result = new List(); 165 | 166 | for (int i = 0; i < argv.Length; i++) 167 | { 168 | string token = argv[i].Split('=')[0]; 169 | if (field.ContainsKey(token)) 170 | { 171 | var cl = field[token]; 172 | if (cl.Item2.CType == CommandType.ARGUMENTS) 173 | i += cl.Item2.ArgumentsCount; 174 | } 175 | else 176 | { 177 | result.Add(i); 178 | } 179 | } 180 | 181 | return result; 182 | } 183 | 184 | /// 185 | /// Insert default arguments. 186 | /// 187 | /// 188 | /// 189 | /// 190 | /// 191 | public static string[] InsertWeirdArguments(string[] args, bool pipe, string option) 192 | where T : IConsoleOption, new() 193 | { 194 | var weird = GetWeirdArguments(args); 195 | 196 | if (weird.Count > 0 && pipe) 197 | args = Insert(args, option, weird[0]); 198 | 199 | return args; 200 | } 201 | 202 | /// 203 | /// Separate the combined factors. 204 | /// 205 | /// 206 | /// 207 | public static string[] SplitCombinedOptions(string[] args) 208 | { 209 | List result = new List(); 210 | foreach (var arg in args) 211 | { 212 | if (arg.Length > 1 && arg.StartsWith("-") && !arg.StartsWith("--") && !arg.Contains("=")) 213 | { 214 | for (int i = 1; i < arg.Length; i++) 215 | result.Add($"-{arg[i]}"); 216 | } 217 | else 218 | { 219 | result.Add(arg); 220 | } 221 | } 222 | return result.ToArray(); 223 | } 224 | } 225 | 226 | /// 227 | /// The command line parser. 228 | /// 229 | /// 230 | public class CommandLineParser 231 | { 232 | /// 233 | /// Get field information that contains Attribute information. 234 | /// 235 | /// 236 | public static Dictionary> GetFields(Type type) 237 | { 238 | FieldInfo[] fields = type.GetFields(); 239 | var field = new Dictionary>(); 240 | 241 | foreach (FieldInfo m in fields) 242 | { 243 | object[] attrs = m.GetCustomAttributes(false); 244 | 245 | foreach (var cl in attrs) 246 | { 247 | if (cl is CommandLine clcast) 248 | { 249 | field.Add(clcast.Option, Tuple.Create(m.Name, clcast)); 250 | if (!string.IsNullOrEmpty(clcast.ShortOption)) 251 | field.Add(clcast.ShortOption, Tuple.Create(m.Name, clcast)); 252 | } 253 | } 254 | } 255 | 256 | return field; 257 | } 258 | 259 | /// 260 | /// Parse command lines based on attributes. 261 | /// 262 | /// 263 | /// 264 | /// 265 | public static T Parse(T model, string[] argv, bool pipe = false, string contents = "") where T : IConsoleOption, new() 266 | { 267 | var field = GetFields(typeof(T)); 268 | 269 | // 270 | // This flag is enabled if there is no option 271 | // 272 | bool any_option = true; 273 | 274 | for (int i = 0; i < argv.Length; i++) 275 | { 276 | string token = argv[i].Split('=')[0]; 277 | if (field.ContainsKey(token)) 278 | { 279 | var cl = field[token]; 280 | if (cl.Item2.CType == CommandType.OPTION) 281 | { 282 | // 283 | // In the case of the OPTION type, the variable must be set to true. 284 | // 285 | typeof(T).GetField(cl.Item1).SetValue(model, true); 286 | } 287 | else if (cl.Item2.CType == CommandType.ARGUMENTS) 288 | { 289 | List sub_args = new List(); 290 | 291 | int arguments_count = cl.Item2.ArgumentsCount; 292 | 293 | if (cl.Item2.Pipe == true && pipe == true) 294 | { 295 | arguments_count--; 296 | sub_args.Add(contents); 297 | } 298 | 299 | for (int j = 1; j <= arguments_count; j++) 300 | { 301 | if (i + j == argv.Length) 302 | { 303 | typeof(T).GetField("Error").SetValue(model, true); 304 | typeof(T).GetField("ErrorMessage").SetValue(model, $"'{argv[i]}' require {arguments_count - j + 1} more sub arguments."); 305 | typeof(T).GetField("HelpMessage").SetValue(model, cl.Item2.Help); 306 | return model; 307 | } 308 | 309 | sub_args.Add(argv[i + j]); 310 | } 311 | 312 | i += cl.Item2.ArgumentsCount; 313 | 314 | typeof(T).GetField(cl.Item1).SetValue(model, sub_args.ToArray()); 315 | } 316 | else if (cl.Item2.CType == CommandType.EQUAL) 317 | { 318 | string[] split = argv[i].Split('='); 319 | 320 | if (split.Length == 1) 321 | { 322 | typeof(T).GetField("Error").SetValue(model, true); 323 | typeof(T).GetField("ErrorMessage").SetValue(model, $"'{split[0]}' must have equal delimiter."); 324 | typeof(T).GetField("HelpMessage").SetValue(model, cl.Item2.Help); 325 | return model; 326 | } 327 | 328 | typeof(T).GetField(cl.Item1).SetValue(model, split[1]); 329 | } 330 | any_option = false; 331 | } 332 | else 333 | { 334 | typeof(T).GetField("Error").SetValue(model, true); 335 | typeof(T).GetField("ErrorMessage").SetValue(model, $"'{argv[i]}' is not correct arguments."); 336 | return model; 337 | } 338 | } 339 | 340 | if (any_option) 341 | { 342 | // 343 | // Find and activate the first Default 344 | // 345 | foreach (var kv in field) 346 | { 347 | if (!pipe && kv.Value.Item2.Default) 348 | { 349 | typeof(T).GetField(kv.Value.Item1).SetValue(model, true); 350 | break; 351 | } 352 | else if (pipe && kv.Value.Item2.PipeDefault) 353 | { 354 | typeof(T).GetField(kv.Value.Item1).SetValue(model, new[] { contents }); 355 | break; 356 | } 357 | } 358 | } 359 | 360 | return model; 361 | } 362 | 363 | /// 364 | /// Parse command lines based on attributes. 365 | /// 366 | /// 367 | /// 368 | /// 369 | public static T Parse(string[] argv, bool pipe = false, string contents = "") where T : IConsoleOption, new() 370 | { 371 | return Parse(new T(), argv, pipe, contents); 372 | } 373 | } 374 | } 375 | -------------------------------------------------------------------------------- /hsync/CL/IConsole.cs: -------------------------------------------------------------------------------- 1 | // This source code is a part of project violet-server. 2 | // Copyright (C)2020-2021. violet-team. Licensed under the MIT Licence. 3 | 4 | using System; 5 | using System.Collections.Generic; 6 | using System.Text; 7 | 8 | namespace hsync.CL 9 | { 10 | /// 11 | /// This is a console option that must be included. 12 | /// 13 | public class IConsoleOption 14 | { 15 | public bool Error; 16 | public string ErrorMessage; 17 | public string HelpMessage; 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /hsync/CharacterTest.cs: -------------------------------------------------------------------------------- 1 | // This source code is a part of project violet-server. 2 | // Copyright (C)2020-2021. violet-team. Licensed under the MIT Licence. 3 | 4 | using hsync.Component; 5 | using Newtonsoft.Json; 6 | using Newtonsoft.Json.Linq; 7 | using SQLite; 8 | using System; 9 | using System.Collections.Generic; 10 | using System.IO; 11 | using System.Linq; 12 | using System.Text; 13 | using System.Threading; 14 | using System.Threading.Tasks; 15 | 16 | namespace hsync 17 | { 18 | public class CharacterTest 19 | { 20 | public string dbdir; 21 | public List target; 22 | public double threshold; 23 | 24 | public CharacterTest(string dbpath, double threshold) 25 | { 26 | var db = new SQLiteConnection(dbpath); 27 | var info = db.GetTableInfo(typeof(HitomiColumnModel).Name); 28 | 29 | if (!info.Any()) 30 | { 31 | Console.WriteLine($"{typeof(HitomiColumnModel).Name} table is not found."); 32 | return; 33 | } 34 | 35 | Console.Write("Load database... "); 36 | target = db.Query("SELECT * FROM HitomiColumnModel"); 37 | Console.WriteLine("Ready"); 38 | 39 | this.threshold = threshold; 40 | this.dbdir = Path.GetDirectoryName(dbpath); 41 | } 42 | 43 | public Dictionary>> result = new Dictionary>>(); 44 | 45 | public Dictionary> tags_dic = new Dictionary>(); 46 | public List>> tags_list; 47 | public List> results = new List>(); 48 | 49 | private void Initialize() 50 | { 51 | result.Clear(); 52 | results.Clear(); 53 | tags_dic.Clear(); 54 | if (tags_list != null) tags_list.Clear(); 55 | 56 | foreach (var data in target) 57 | { 58 | if (data.Characters != null) 59 | { 60 | foreach (var tag in data.Characters.Split('|')) 61 | { 62 | if (tag == "") continue; 63 | if (tags_dic.ContainsKey(tag)) 64 | tags_dic[tag].Add(data.Id); 65 | else 66 | tags_dic.Add(tag, new List { data.Id }); 67 | } 68 | } 69 | } 70 | 71 | tags_list = tags_dic.ToList(); 72 | 73 | tags_list.ForEach(x => x.Value.Sort()); 74 | tags_list.Sort((a, b) => a.Value.Count.CompareTo(b.Value.Count)); 75 | } 76 | 77 | private static int manually_intersect(List a, List b) 78 | { 79 | int intersect = 0; 80 | int i = 0, j = 0; 81 | for (; i < a.Count && j < b.Count;) 82 | { 83 | if (a[i] == b[j]) 84 | { 85 | intersect++; 86 | i++; 87 | j++; 88 | } 89 | else if (a[i] < b[j]) 90 | { 91 | i++; 92 | } 93 | else 94 | { 95 | j++; 96 | } 97 | } 98 | return intersect; 99 | } 100 | 101 | private List> Intersect(int i) 102 | { 103 | List> result = new List>(); 104 | 105 | for (int j = i + 1; j < tags_list.Count; j++) 106 | { 107 | int intersect = manually_intersect(tags_list[i].Value, tags_list[j].Value); 108 | int i_size = tags_list[i].Value.Count; 109 | int j_size = tags_list[j].Value.Count; 110 | double rate = (double)(intersect) / (i_size + j_size - intersect); 111 | if (rate >= threshold) 112 | result.Add(new Tuple(tags_list[i].Key, tags_list[j].Key, 113 | rate)); 114 | } 115 | 116 | return result; 117 | } 118 | 119 | private void Merge() 120 | { 121 | foreach (var tuple in results) 122 | { 123 | if (result.ContainsKey(tuple.Item1)) 124 | result[tuple.Item1].Add(new Tuple(tuple.Item2, tuple.Item3)); 125 | else 126 | result.Add(tuple.Item1, new List> { new Tuple(tuple.Item2, tuple.Item3) }); 127 | if (result.ContainsKey(tuple.Item2)) 128 | result[tuple.Item2].Add(new Tuple(tuple.Item1, tuple.Item3)); 129 | else 130 | result.Add(tuple.Item2, new List> { new Tuple(tuple.Item1, tuple.Item3) }); 131 | } 132 | result.ToList().ForEach(x => x.Value.Sort((a, b) => b.Item2.CompareTo(a.Item2))); 133 | results.Clear(); 134 | } 135 | 136 | int max; 137 | int progress; 138 | int mtl; 139 | public void Start() 140 | { 141 | Initialize(); 142 | 143 | max = tags_list.Count; 144 | progress = 0; 145 | mtl = Environment.ProcessorCount; 146 | 147 | Console.Write("Process... "); 148 | using (var pb = new ExtractingProgressBar()) 149 | { 150 | Task.WhenAll(Enumerable.Range(0, mtl).Select(no => Task.Run(() => process(no, pb)))).Wait(); 151 | } 152 | Console.WriteLine("Complete"); 153 | 154 | Console.Write("Merge... "); 155 | Merge(); 156 | Console.WriteLine("Complete"); 157 | 158 | Console.Write("Save... "); 159 | var rr = result.ToList(); 160 | rr.Sort((x, y) => y.Value.Count.CompareTo(x.Value.Count)); 161 | JArray arr = new JArray(); 162 | rr.ForEach(x => 163 | { 164 | JArray tags = new JArray(); 165 | x.Value.ForEach(y => 166 | { 167 | tags.Add(new JObject { { y.Item1, y.Item2 } }); 168 | }); 169 | arr.Add(new JObject { { x.Key, tags } }); 170 | }); 171 | File.WriteAllText(Path.Combine(dbdir, "ct-result.json"), JsonConvert.SerializeObject(arr, Formatting.Indented)); 172 | Console.WriteLine("Complete"); 173 | } 174 | 175 | private void process(int i, ExtractingProgressBar pb) 176 | { 177 | int min = this.max / mtl * i; 178 | int max = this.max / mtl * (i + 1); 179 | if (max > this.max) 180 | max = this.max; 181 | 182 | List> result = new List>(); 183 | 184 | for (int j = max - 1; j >= min; j--) 185 | { 186 | result.AddRange(Intersect(j)); 187 | 188 | pb.Report(this.max, Interlocked.Increment(ref progress)); 189 | } 190 | 191 | lock (results) 192 | results.AddRange(result); 193 | } 194 | } 195 | } 196 | -------------------------------------------------------------------------------- /hsync/Component/EH.cs: -------------------------------------------------------------------------------- 1 | // This source code is a part of project violet-server. 2 | // Copyright (C)2020-2021. violet-team. Licensed under the MIT Licence. 3 | 4 | using HtmlAgilityPack; 5 | using System; 6 | using System.Collections.Generic; 7 | using System.Linq; 8 | using System.Net; 9 | using System.Text; 10 | using hsync.Log; 11 | using System.Text.RegularExpressions; 12 | 13 | namespace hsync.Component 14 | { 15 | public class EHentaiArticle 16 | { 17 | public string Thumbnail { get; set; } 18 | 19 | public string Title { get; set; } 20 | public string SubTitle { get; set; } 21 | 22 | public string Type { get; set; } 23 | public string Uploader { get; set; } 24 | 25 | public string Posted; 26 | public string Parent; 27 | public string Visible; 28 | public string Language; 29 | public string FileSize; 30 | public int Length; 31 | public int Favorited; 32 | 33 | public string reclass; 34 | public string[] language; 35 | public string[] group; 36 | public string[] parody; 37 | public string[] character; 38 | public string[] artist; 39 | public string[] male; 40 | public string[] female; 41 | public string[] misc; 42 | 43 | public Tuple[] comment; 44 | public List ImagesLink { get; set; } 45 | public string Archive { get; set; } 46 | } 47 | 48 | public class EHentaiResultArticle 49 | { 50 | public string URL; 51 | 52 | public string Thumbnail; 53 | public string Title; 54 | 55 | public string Uploader; 56 | public string Published; 57 | public string Files; 58 | public string Type; 59 | 60 | public Dictionary> Descripts; 61 | } 62 | 63 | public class EHentaiParser 64 | { 65 | public static EHentaiArticle ParseArticleData(string source) 66 | { 67 | EHentaiArticle article = new EHentaiArticle(); 68 | 69 | HtmlDocument document = new HtmlDocument(); 70 | document.LoadHtml(source); 71 | HtmlNode nodes = document.DocumentNode.SelectNodes("//div[@class='gm']")[0]; 72 | 73 | article.Thumbnail = Regex.Match(nodes.SelectSingleNode(".//div[@id='gleft']//div//div").GetAttributeValue("style", ""), @"https://ehgt.org/.*?(?=\))").Groups[0].Value; 74 | 75 | article.Title = nodes.SelectSingleNode(".//div[@id='gd2']//h1[@id='gn']").InnerText; 76 | article.SubTitle = nodes.SelectSingleNode(".//div[@id='gd2']//h1[@id='gj']").InnerText; 77 | 78 | // article.Type = nodes.SelectSingleNode(".//div[@id='gmid']//div//div[@id='gdc']//a//img").GetAttributeValue("alt", ""); 79 | article.Uploader = nodes.SelectSingleNode(".//div[@id='gmid']//div//div[@id='gdn']//a").InnerText; 80 | 81 | HtmlNodeCollection nodes_static = nodes.SelectNodes(".//div[@id='gmid']//div//div[@id='gdd']//table//tr"); 82 | 83 | article.Posted = nodes_static[0].SelectSingleNode(".//td[@class='gdt2']").InnerText; 84 | article.Parent = nodes_static[1].SelectSingleNode(".//td[@class='gdt2']").InnerText; 85 | article.Visible = nodes_static[2].SelectSingleNode(".//td[@class='gdt2']").InnerText; 86 | article.Language = nodes_static[3].SelectSingleNode(".//td[@class='gdt2']").InnerText.Split(' ')[0].ToLower(); 87 | article.FileSize = nodes_static[4].SelectSingleNode(".//td[@class='gdt2']").InnerText; 88 | int.TryParse(nodes_static[5].SelectSingleNode(".//td[@class='gdt2']").InnerText.Split(' ')[0], out article.Length); 89 | int.TryParse(nodes_static[6].SelectSingleNode(".//td[@class='gdt2']").InnerText.Split(' ')[0], out article.Favorited); 90 | 91 | HtmlNodeCollection nodes_data = nodes.SelectNodes(".//div[@id='gmid']//div[@id='gd4']//table//tr"); 92 | 93 | Dictionary information = new Dictionary(); 94 | 95 | foreach (var i in nodes_data) 96 | { 97 | try 98 | { 99 | information.Add(i.SelectNodes(".//td")[0].InnerText.Trim(), 100 | i.SelectNodes(".//td")[1].SelectNodes(".//div").Select(e => e.SelectSingleNode(".//a").InnerText).ToArray()); 101 | } 102 | catch { } 103 | } 104 | 105 | if (information.ContainsKey("language:")) article.language = information["language:"]; 106 | if (information.ContainsKey("group:")) article.group = information["group:"]; 107 | if (information.ContainsKey("parody:")) article.parody = information["parody:"]; 108 | if (information.ContainsKey("character:")) article.character = information["character:"]; 109 | if (information.ContainsKey("artist:")) article.artist = information["artist:"]; 110 | if (information.ContainsKey("male:")) article.male = information["male:"]; 111 | if (information.ContainsKey("female:")) article.female = information["female:"]; 112 | if (information.ContainsKey("misc:")) article.misc = information["misc:"]; 113 | 114 | HtmlNode nodesc = document.DocumentNode.SelectNodes("//div[@id='cdiv']")[0]; 115 | HtmlNodeCollection nodes_datac = nodesc.SelectNodes(".//div[@class='c1']"); 116 | List> comments = new List>(); 117 | 118 | foreach (var i in nodes_datac ?? Enumerable.Empty()) 119 | { 120 | try 121 | { 122 | string date = WebUtility.HtmlDecode(i.SelectNodes(".//div[@class='c2']//div[@class='c3']")[0].InnerText.Trim()); 123 | string author = WebUtility.HtmlDecode(i.SelectNodes(".//div[@class='c2']//div[@class='c3']//a")[0].InnerText.Trim()); 124 | string contents = Regex.Replace(WebUtility.HtmlDecode(i.SelectNodes(".//div[@class='c6']")[0].InnerHtml.Trim()), @"
", "\r\n"); 125 | comments.Add(new Tuple( 126 | DateTime.Parse(date.Remove(date.IndexOf(" UTC")).Substring("Posted on ".Length) + "Z"), 127 | author, 128 | contents)); 129 | } 130 | catch { } 131 | } 132 | 133 | comments.Sort((a, b) => a.Item1.CompareTo(b.Item1)); 134 | article.comment = comments.ToArray(); 135 | 136 | return article; 137 | } 138 | } 139 | 140 | public class ExHentaiParser 141 | { 142 | public static EHentaiArticle ParseArticleData(string source) 143 | { 144 | EHentaiArticle article = new EHentaiArticle(); 145 | 146 | HtmlDocument document = new HtmlDocument(); 147 | document.LoadHtml(source); 148 | HtmlNode nodes = document.DocumentNode.SelectNodes("//div[@class='gm']")[0]; 149 | 150 | article.Thumbnail = Regex.Match(nodes.SelectSingleNode(".//div[@id='gleft']//div//div").GetAttributeValue("style", ""), @"https://exhentai.org/.*?(?=\))").Groups[0].Value; 151 | 152 | article.Title = nodes.SelectSingleNode(".//div[@id='gd2']//h1[@id='gn']").InnerText; 153 | article.SubTitle = nodes.SelectSingleNode(".//div[@id='gd2']//h1[@id='gj']").InnerText; 154 | 155 | //article.Type = nodes.SelectSingleNode(".//div[@id='gmid']//div//div[@id='gdc']//a//img").GetAttributeValue("alt", ""); 156 | article.Uploader = nodes.SelectSingleNode(".//div[@id='gmid']//div//div[@id='gdn']//a").InnerText; 157 | 158 | HtmlNodeCollection nodes_static = nodes.SelectNodes(".//div[@id='gmid']//div//div[@id='gdd']//table//tr"); 159 | 160 | article.Posted = nodes_static[0].SelectSingleNode(".//td[@class='gdt2']").InnerText; 161 | article.Parent = nodes_static[1].SelectSingleNode(".//td[@class='gdt2']").InnerText; 162 | article.Visible = nodes_static[2].SelectSingleNode(".//td[@class='gdt2']").InnerText; 163 | article.Language = nodes_static[3].SelectSingleNode(".//td[@class='gdt2']").InnerText.Split(' ')[0].ToLower(); 164 | article.FileSize = nodes_static[4].SelectSingleNode(".//td[@class='gdt2']").InnerText; 165 | int.TryParse(nodes_static[5].SelectSingleNode(".//td[@class='gdt2']").InnerText.Split(' ')[0], out article.Length); 166 | int.TryParse(nodes_static[6].SelectSingleNode(".//td[@class='gdt2']").InnerText.Split(' ')[0], out article.Favorited); 167 | 168 | HtmlNodeCollection nodes_data = nodes.SelectNodes(".//div[@id='gmid']//div[@id='gd4']//table//tr"); 169 | 170 | Dictionary information = new Dictionary(); 171 | 172 | foreach (var i in nodes_data) 173 | { 174 | try 175 | { 176 | information.Add(i.SelectNodes(".//td")[0].InnerText.Trim(), 177 | i.SelectNodes(".//td")[1].SelectNodes(".//div").Select(e => e.SelectSingleNode(".//a").InnerText).ToArray()); 178 | } 179 | catch { } 180 | } 181 | 182 | if (information.ContainsKey("language:")) article.language = information["language:"]; 183 | if (information.ContainsKey("group:")) article.group = information["group:"]; 184 | if (information.ContainsKey("parody:")) article.parody = information["parody:"]; 185 | if (information.ContainsKey("character:")) article.character = information["character:"]; 186 | if (information.ContainsKey("artist:")) article.artist = information["artist:"]; 187 | if (information.ContainsKey("male:")) article.male = information["male:"]; 188 | if (information.ContainsKey("female:")) article.female = information["female:"]; 189 | if (information.ContainsKey("misc:")) article.misc = information["misc:"]; 190 | 191 | HtmlNode nodesc = document.DocumentNode.SelectNodes("//div[@id='cdiv']")[0]; 192 | HtmlNodeCollection nodes_datac = nodesc.SelectNodes(".//div[@class='c1']"); 193 | List> comments = new List>(); 194 | 195 | foreach (var i in nodes_datac ?? Enumerable.Empty()) 196 | { 197 | try 198 | { 199 | string date = WebUtility.HtmlDecode(i.SelectNodes(".//div[@class='c2']//div[@class='c3']")[0].InnerText.Trim()); 200 | string author = WebUtility.HtmlDecode(i.SelectNodes(".//div[@class='c2']//div[@class='c3']//a")[0].InnerText.Trim()); 201 | string contents = Regex.Replace(WebUtility.HtmlDecode(i.SelectNodes(".//div[@class='c6']")[0].InnerHtml.Trim()), @"
", "\r\n"); 202 | comments.Add(new Tuple( 203 | DateTime.Parse(date.Remove(date.IndexOf(" by")).Substring("Posted on ".Length) + "Z"), 204 | author, 205 | contents)); 206 | } 207 | catch (Exception e) 208 | { 209 | Logs.Instance.PushError("[Fail] \r\n" + Logs.SerializeObject(e)); 210 | } 211 | } 212 | 213 | comments.Sort((a, b) => a.Item1.CompareTo(b.Item1)); 214 | article.comment = comments.ToArray(); 215 | 216 | return article; 217 | } 218 | 219 | /// 220 | /// 결과 페이지를 분석합니다. 221 | /// ex: https://exhentai.org/ 222 | /// ex: https://exhentai.org/?inline_set=dm_t 223 | /// ex: https://exhentai.org/?page=1 224 | /// 225 | /// 226 | /// 227 | public static List ParseResultPageThumbnailView(string html) 228 | { 229 | var result = new List(); 230 | 231 | var document = new HtmlDocument(); 232 | document.LoadHtml(html); 233 | var nodes = document.DocumentNode.SelectNodes("//div[@class='itg']/div[@class='id1']"); 234 | 235 | foreach (var node in nodes) 236 | { 237 | try 238 | { 239 | var article = new EHentaiResultArticle(); 240 | 241 | article.URL = node.SelectSingleNode("./div[@class='id2']/a").GetAttributeValue("href", ""); 242 | 243 | try { article.Thumbnail = node.SelectSingleNode("./div[@class='id3']/a/img").GetAttributeValue("src", ""); } catch { } 244 | article.Title = node.SelectSingleNode("./div[@class='id2']/a").InnerText; 245 | 246 | article.Files = node.SelectSingleNode(".//div[@class='id42']").InnerText; 247 | article.Type = node.SelectSingleNode(".//div[@class='id41']").GetAttributeValue("title", ""); 248 | 249 | result.Add(article); 250 | } 251 | catch { } 252 | } 253 | 254 | return result; 255 | } 256 | 257 | /// 258 | /// 결과 페이지를 분석합니다. 259 | /// ex: https://exhentai.org/ 260 | /// ex: https://exhentai.org/?inline_set=dm_l 261 | /// ex: https://exhentai.org/?page=1 262 | /// 263 | /// 264 | /// 265 | public static List ParseResultPageListView(string html) 266 | { 267 | var result = new List(); 268 | 269 | var document = new HtmlDocument(); 270 | document.LoadHtml(html); 271 | var nodes = document.DocumentNode.SelectNodes("//table[@class='itg']/tr"); 272 | 273 | if (nodes.Count > 1) nodes.RemoveAt(0); 274 | 275 | foreach (var node in nodes) 276 | { 277 | try 278 | { 279 | var article = new EHentaiResultArticle(); 280 | 281 | article.URL = node.SelectSingleNode("./td[3]/div/div[@class='it5']/a").GetAttributeValue("href", ""); 282 | 283 | try { article.Thumbnail = node.SelectSingleNode("./td[3]/div/div[@class='it2']/img").GetAttributeValue("src", ""); } catch { } 284 | article.Title = node.SelectSingleNode("./td[3]/div/div[@class='it5']/a").InnerText; 285 | 286 | article.Uploader = node.SelectSingleNode("./td[4]/div/a").GetAttributeValue("href", ""); 287 | article.Published = node.SelectSingleNode("./td[2]").InnerText; 288 | article.Type = node.SelectSingleNode("./td/a/img").GetAttributeValue("alt", ""); 289 | 290 | result.Add(article); 291 | } 292 | catch { } 293 | } 294 | 295 | return result; 296 | } 297 | 298 | /// 299 | /// 결과 페이지를 분석합니다. 300 | /// ex: https://exhentai.org/?inline_set=dm_e 301 | /// 302 | /// 303 | /// 304 | public static List ParseResultPageExtendedListView(string html) 305 | { 306 | var result = new List(); 307 | 308 | var document = new HtmlDocument(); 309 | document.LoadHtml(html); 310 | 311 | Queue nodes = new Queue(); 312 | var fn = document.DocumentNode.SelectNodes("//table[@class='itg glte']/tr"); 313 | fn.ToList().ForEach(x => nodes.Enqueue(x)); 314 | 315 | while (nodes.Count != 0) 316 | { 317 | var node = nodes.Dequeue(); 318 | try 319 | { 320 | var article = new EHentaiResultArticle(); 321 | 322 | article.URL = node.SelectSingleNode(".//a").GetAttributeValue("href", ""); 323 | try { article.Thumbnail = node.SelectSingleNode(".//img").GetAttributeValue("src", ""); } catch { } 324 | 325 | var g13 = node.SelectSingleNode("./td[2]/div/div"); 326 | 327 | article.Type = g13.SelectSingleNode("./div").InnerText.ToLower(); 328 | article.Published = g13.SelectSingleNode("./div[2]").InnerText; 329 | article.Uploader = g13.SelectSingleNode("./div[4]").InnerText; 330 | article.Files = g13.SelectSingleNode("./div[5]").InnerText; 331 | 332 | var gref = node.SelectSingleNode("./td[2]/div/a/div"); 333 | 334 | article.Title = gref.SelectSingleNode("./div").InnerText; 335 | 336 | if (article.Title.Contains("느와카나")) 337 | ; 338 | 339 | try 340 | { 341 | var desc = gref.SelectNodes("./div//tr"); 342 | var desc_dic = new Dictionary>(); 343 | 344 | foreach (var nn in desc) 345 | { 346 | var cont = nn.SelectSingleNode("./td").InnerText.Trim(); 347 | cont = cont.Remove(cont.Length - 1); 348 | 349 | var cc = new List(); 350 | 351 | foreach (var ccc in nn.SelectNodes("./td[2]//div")) 352 | { 353 | cc.Add(ccc.InnerText); 354 | } 355 | 356 | desc_dic.Add(cont, cc); 357 | } 358 | 359 | article.Descripts = desc_dic; 360 | } 361 | catch { } 362 | result.Add(article); 363 | 364 | var next = node.SelectNodes("./tr"); 365 | if (next != null) 366 | next.ToList().ForEach(x => nodes.Enqueue(x)); 367 | } 368 | catch { } 369 | } 370 | 371 | return result; 372 | } 373 | 374 | /// 375 | /// 결과 페이지를 분석합니다. 376 | /// ex: https://exhentai.org/?inline_set=dm_m 377 | /// 378 | /// 379 | /// 380 | public static List ParseResultPageMinimalListView(string html) 381 | { 382 | var result = new List(); 383 | 384 | var document = new HtmlDocument(); 385 | document.LoadHtml(html); 386 | 387 | HtmlNodeCollection nodes = document.DocumentNode.SelectNodes("//table[@class='itg gltm']/tr"); 388 | 389 | for (int i = 1; i < nodes.Count; i++) 390 | { 391 | var node = nodes[i]; 392 | 393 | var article = new EHentaiResultArticle(); 394 | 395 | article.Type = node.SelectSingleNode("./td/div").InnerText.Trim().ToLower(); 396 | article.Thumbnail = node.SelectSingleNode(".//img").GetAttributeValue("src", ""); 397 | if (article.Thumbnail.StartsWith("data")) 398 | article.Thumbnail = node.SelectSingleNode(".//img").GetAttributeValue("data-src", ""); 399 | article.Published = node.SelectSingleNode("./td[2]/div[2]/div[2]/div[1]/div[2]").InnerText.Trim(); 400 | article.Files = node.SelectSingleNode("./td[2]/div[2]/div[2]/div[2]/div[2]").InnerText.Trim(); 401 | 402 | article.URL = node.SelectSingleNode("./td[4]/a").GetAttributeValue("href", ""); 403 | article.Title = node.SelectSingleNode("./td[4]/a/div").InnerText.Trim(); 404 | article.Uploader = node.SelectSingleNode("./td[6]/div/a").InnerText.Trim(); 405 | 406 | result.Add(article); 407 | } 408 | 409 | return result; 410 | } 411 | } 412 | } 413 | -------------------------------------------------------------------------------- /hsync/Component/Hitomi.cs: -------------------------------------------------------------------------------- 1 | // This source code is a part of project violet-server. 2 | // Copyright (C)2020-2021. violet-team. Licensed under the MIT Licence. 3 | 4 | using hsync.Utils; 5 | using HtmlAgilityPack; 6 | using Newtonsoft.Json; 7 | using Newtonsoft.Json.Converters; 8 | using SQLite; 9 | using System; 10 | using System.Collections.Generic; 11 | using System.IO; 12 | using System.Linq; 13 | 14 | namespace hsync.Component 15 | { 16 | public struct HitomiMetadata 17 | { 18 | [JsonProperty(PropertyName = "a")] 19 | public string[] Artists { get; set; } 20 | [JsonProperty(PropertyName = "g")] 21 | public string[] Groups { get; set; } 22 | [JsonProperty(PropertyName = "p")] 23 | public string[] Parodies { get; set; } 24 | [JsonProperty(PropertyName = "t")] 25 | public string[] Tags { get; set; } 26 | [JsonProperty(PropertyName = "c")] 27 | public string[] Characters { get; set; } 28 | [JsonProperty(PropertyName = "l")] 29 | public string Language { get; set; } 30 | [JsonProperty(PropertyName = "n")] 31 | public string Name { get; set; } 32 | [JsonProperty(PropertyName = "type")] 33 | public string Type { get; set; } 34 | [JsonProperty(PropertyName = "id")] 35 | public int ID { get; set; } 36 | 37 | [JsonIgnore] 38 | public int Files { get; set; } 39 | 40 | [JsonIgnore] 41 | public DateTime? DateTime; 42 | } 43 | 44 | public class HitomiArticle 45 | { 46 | public string[] Artists { get; set; } 47 | public string[] Characters { get; set; } 48 | public string[] Groups { get; set; } 49 | public string Language { get; set; } 50 | public string[] Series { get; set; } 51 | public string[] Tags { get; set; } 52 | public string Type { get; set; } 53 | public bool ManualPathOrdering { get; set; } 54 | public string ManualAdditionalPath { get; set; } 55 | public string DateTime { get; set; } 56 | 57 | public string Thumbnail { get; set; } 58 | public string Magic { get; set; } 59 | public string Title { get; set; } 60 | public string Files { get; set; } 61 | } 62 | 63 | public abstract class SQLiteColumnModel 64 | { 65 | [PrimaryKey] 66 | public int Id { get; set; } 67 | } 68 | 69 | public class HitomiColumnModel : SQLiteColumnModel 70 | { 71 | public string Title { get; set; } 72 | public string EHash { get; set; } 73 | public string Type { get; set; } 74 | public string Artists { get; set; } 75 | public string Characters { get; set; } 76 | public string Groups { get; set; } 77 | public string Language { get; set; } 78 | public string Series { get; set; } 79 | public string Tags { get; set; } 80 | public string Uploader { get; set; } 81 | public DateTime? Published { get; set; } 82 | public int Files { get; set; } 83 | public string Class { get; set; } 84 | public int ExistOnHitomi { get; set; } 85 | public string Thumbnail { get; set; } 86 | } 87 | 88 | public class HitomiIndexArtistsColumnModel : SQLiteColumnModel { public string Tag { get; set; } } 89 | public class HitomiIndexGroupsColumnModel : SQLiteColumnModel { public string Tag { get; set; } } 90 | public class HitomiIndexSeriesColumnModel : SQLiteColumnModel { public string Tag { get; set; } } 91 | public class HitomiIndexCharactersColumnModel : SQLiteColumnModel { public string Tag { get; set; } } 92 | public class HitomiIndexLanguagesColumnModel : SQLiteColumnModel { public string Tag { get; set; } } 93 | public class HitomiIndexTypesColumnModel : SQLiteColumnModel { public string Tag { get; set; } } 94 | public class HitomiIndexTagsColumnModel : SQLiteColumnModel { public string Tag { get; set; } } 95 | 96 | //public class HitomiIndexModel : SQLiteColumnModel 97 | //{ 98 | // public string Name { get; set; } 99 | // public int Index { get; set; } 100 | //} 101 | 102 | public class HitomiLegalize 103 | { 104 | public static HitomiMetadata ArticleToMetadata(HitomiArticle article) 105 | { 106 | HitomiMetadata metadata = new HitomiMetadata(); 107 | if (article.Artists != null) metadata.Artists = article.Artists; 108 | if (article.Characters != null) metadata.Characters = article.Characters; 109 | if (article.Groups != null) metadata.Groups = article.Groups; 110 | try 111 | { 112 | if (article.Magic.Contains("-")) 113 | metadata.ID = Convert.ToInt32(article.Magic.Split('-').Last().Split('.')[0]); 114 | else if (article.Magic.Contains("galleries")) 115 | metadata.ID = Convert.ToInt32(article.Magic.Split('/').Last().Split('.')[0]); 116 | else 117 | metadata.ID = Convert.ToInt32(article.Magic); 118 | } 119 | catch 120 | { 121 | ; 122 | } 123 | metadata.Language = LegalizeLanguage(article.Language); 124 | metadata.Name = article.Title; 125 | if (article.Series != null) metadata.Parodies = article.Series; 126 | if (article.Tags != null) metadata.Tags = article.Tags.Select(x => LegalizeTag(x)).ToArray(); 127 | metadata.Type = article.Type; 128 | if (article.Files != null) metadata.Files = Convert.ToInt32(article.Files); 129 | if (article.DateTime != null) 130 | metadata.DateTime = DateTime.Parse(article.DateTime); 131 | return metadata; 132 | } 133 | 134 | public static string LegalizeTag(string tag) 135 | { 136 | if (tag.Trim().EndsWith("♀")) return "female:" + tag.Trim('♀').Trim(); 137 | if (tag.Trim().EndsWith("♂")) return "male:" + tag.Trim('♂').Trim(); 138 | return tag.Trim(); 139 | } 140 | 141 | public static string LegalizeLanguage(string lang) 142 | { 143 | switch (lang) 144 | { 145 | case "모든 언어": return "all"; 146 | case "한국어": return "korean"; 147 | case "N/A": return "n/a"; 148 | case "日本語": return "japanese"; 149 | case "English": return "english"; 150 | case "Español": return "spanish"; 151 | case "ไทย": return "thai"; 152 | case "Deutsch": return "german"; 153 | case "中文": return "chinese"; 154 | case "Português": return "portuguese"; 155 | case "Français": return "french"; 156 | case "Tagalog": return "tagalog"; 157 | case "Русский": return "russian"; 158 | case "Italiano": return "italian"; 159 | case "polski": return "polish"; 160 | case "tiếng việt": return "vietnamese"; 161 | case "magyar": return "hungarian"; 162 | case "Čeština": return "czech"; 163 | case "Bahasa Indonesia": return "indonesian"; 164 | case "العربية": return "arabic"; 165 | } 166 | 167 | return lang; 168 | } 169 | 170 | public static string DeLegalizeLanguage(string lang) 171 | { 172 | switch (lang) 173 | { 174 | case "all": return "모든 언어"; 175 | case "korean": return "한국어"; 176 | case "n/a": return "N/A"; 177 | case "japanese": return "日本語"; 178 | case "english": return "English"; 179 | case "spanish": return "Español"; 180 | case "thai": return "ไทย"; 181 | case "german": return "Deutsch"; 182 | case "chinese": return "中文"; 183 | case "portuguese": return "Português"; 184 | case "french": return "Français"; 185 | case "tagalog": return "Tagalog"; 186 | case "russian": return "Русский"; 187 | case "italian": return "Italiano"; 188 | case "polish": return "polski"; 189 | case "vietnamese": return "tiếng việt"; 190 | case "hungarian": return "magyar"; 191 | case "czech": return "Čeština"; 192 | case "indonesian": return "Bahasa Indonesia"; 193 | case "arabic": return "العربية"; 194 | } 195 | 196 | return lang; 197 | } 198 | } 199 | 200 | public class HitomiParser 201 | { 202 | static public HitomiArticle ParseGalleryBlock(string source) 203 | { 204 | HitomiArticle article = new HitomiArticle(); 205 | 206 | HtmlDocument document = new HtmlDocument(); 207 | document.LoadHtml(source); 208 | HtmlNode nodes = document.DocumentNode.SelectNodes("/div")[0]; 209 | 210 | article.Magic = nodes.SelectSingleNode("./a").GetAttributeValue("href", ""); 211 | try { article.Thumbnail = nodes.SelectSingleNode("./a//img").GetAttributeValue("data-src", "").Substring("//tn.hitomi.la/".Length).Replace("smallbig", "big"); } 212 | catch 213 | { article.Thumbnail = nodes.SelectSingleNode("./a//img").GetAttributeValue("src", "").Substring("//tn.hitomi.la/".Length); } 214 | article.Title = nodes.SelectSingleNode("./h1").InnerText; 215 | 216 | try { article.Artists = nodes.SelectNodes(".//div[@class='artist-list']//li").Select(node => node.SelectSingleNode("./a").InnerText).ToArray(); } 217 | catch { article.Artists = new[] { "N/A" }; } 218 | 219 | var contents = nodes.SelectSingleNode("./div[2]/table"); 220 | try { article.Series = contents.SelectNodes("./tr[1]/td[2]/ul/li").Select(node => node.SelectSingleNode(".//a").InnerText).ToArray(); } catch { } 221 | article.Type = contents.SelectSingleNode("./tr[2]/td[2]/a").InnerText; 222 | try { article.Language = HitomiLegalize.LegalizeLanguage(contents.SelectSingleNode("./tr[3]/td[2]/a").InnerText); } catch { } 223 | try { article.Tags = contents.SelectNodes("./tr[4]/td[2]/ul/li").Select(node => HitomiLegalize.LegalizeTag(node.SelectSingleNode(".//a").InnerText)).ToArray(); } catch { } 224 | 225 | article.DateTime = nodes.SelectSingleNode("./div[2]/p").InnerText; 226 | 227 | return article; 228 | } 229 | 230 | static public HitomiArticle ParseGallery(string source) 231 | { 232 | HitomiArticle article = new HitomiArticle(); 233 | 234 | HtmlDocument document = new HtmlDocument(); 235 | document.LoadHtml(source); 236 | HtmlNode nodes = document.DocumentNode.SelectSingleNode("//div[@class='content']"); 237 | 238 | try 239 | { 240 | article.Magic = nodes.SelectSingleNode("./div[3]/h1/a").GetAttributeValue("href", "").Split('/')[2].Split('.')[0]; 241 | } 242 | catch 243 | { 244 | ; 245 | } 246 | //article.Title = nodes.SelectSingleNode("./div[3]/h1").InnerText.Trim(); 247 | //article.Thumbnail = nodes.SelectSingleNode("./div[2]/div/a/img").GetAttributeValue("src", ""); 248 | //article.Artists = nodes.SelectSingleNode(".") 249 | 250 | foreach (var tr in document.DocumentNode.SelectNodes("//div[@class='gallery-info']/table/tr").ToList()) 251 | { 252 | var tt = tr.SelectSingleNode("./td").InnerText.ToLower().Trim(); 253 | if (tt == "group") 254 | { 255 | article.Groups = tr.SelectNodes(".//a")?.Select(x => x.InnerText.Trim()).ToArray(); 256 | } 257 | else if (tt == "characters") 258 | { 259 | article.Characters = tr.SelectNodes(".//a")?.Select(x => x.InnerText.Trim()).ToArray(); 260 | } 261 | } 262 | 263 | return article; 264 | } 265 | 266 | static public void FillGallery(string source, HitomiArticle article) 267 | { 268 | HtmlDocument document = new HtmlDocument(); 269 | document.LoadHtml(source); 270 | HtmlNode nodes = document.DocumentNode.SelectSingleNode("/div[@class='gallery-info']/table/tbody"); 271 | 272 | foreach (var tr in nodes.SelectNodes("./tr").ToList()) 273 | { 274 | var tt = tr.SelectSingleNode("./td").InnerText.ToLower().Trim(); 275 | if (tt == "group") 276 | { 277 | article.Groups = tr.SelectNodes(".//a").Select(x => x.InnerText.Trim()).ToArray(); 278 | } 279 | else if (tt == "characters") 280 | { 281 | article.Characters = tr.SelectNodes(".//a").Select(x => x.InnerText.Trim()).ToArray(); 282 | } 283 | } 284 | } 285 | } 286 | 287 | public class HitomiData : ILazy 288 | { 289 | public List metadata_collection; 290 | 291 | public void Load() 292 | { 293 | Log.Logs.Instance.Push("Load metadata files..."); 294 | 295 | try 296 | { 297 | metadata_collection = JsonConvert.DeserializeObject>(File.ReadAllText("metadata.json")); 298 | 299 | var articles = JsonConvert.DeserializeObject>(File.ReadAllText("hiddendata.json")); 300 | var overlap = new HashSet(); 301 | if (metadata_collection == null) 302 | metadata_collection = new List(); 303 | metadata_collection.ForEach(x => overlap.Add(x.ID.ToString())); 304 | foreach (var article in articles) 305 | { 306 | if (overlap.Contains(article.Magic)) continue; 307 | metadata_collection.Add(HitomiLegalize.ArticleToMetadata(article)); 308 | } 309 | metadata_collection.Sort((a, b) => b.ID.CompareTo(a.ID)); 310 | } 311 | catch (Exception e) 312 | { 313 | Log.Logs.Instance.PushError("Metadata loading error!"); 314 | Log.Logs.Instance.PushException(e); 315 | Log.Logs.Instance.Panic(); 316 | } 317 | } 318 | 319 | public void SaveWithNewData(List data) 320 | { 321 | var articles = JsonConvert.DeserializeObject>(File.ReadAllText("hiddendata.json")); 322 | File.Move("hiddendata.json", $"hiddendata-{DateTime.Now.Ticks}.json"); 323 | 324 | articles.AddRange(data); 325 | var overlap = new HashSet(); 326 | var pure = new List(); 327 | foreach (var article in articles) 328 | { 329 | if (!overlap.Contains(article.Magic)) 330 | { 331 | pure.Add(article); 332 | overlap.Add(article.Magic); 333 | } 334 | } 335 | 336 | JsonSerializer serializer = new JsonSerializer(); 337 | serializer.Converters.Add(new JavaScriptDateTimeConverter()); 338 | serializer.NullValueHandling = NullValueHandling.Ignore; 339 | 340 | using (StreamWriter sw = new StreamWriter("hiddendata.json")) 341 | using (JsonWriter writer = new JsonTextWriter(sw)) 342 | { 343 | serializer.Serialize(writer, pure); 344 | } 345 | } 346 | } 347 | } 348 | -------------------------------------------------------------------------------- /hsync/Component/HitomiIndex.cs: -------------------------------------------------------------------------------- 1 | using hsync.Utils; 2 | using MessagePack; 3 | using Newtonsoft.Json; 4 | using System; 5 | using System.Collections.Generic; 6 | using System.IO; 7 | using System.Linq; 8 | using System.Text; 9 | 10 | namespace hsync.Component 11 | { 12 | [MessagePackObject] 13 | public class HitomiIndexModel 14 | { 15 | [Key(0)] 16 | public string[] Artists; 17 | [Key(1)] 18 | public string[] Groups; 19 | [Key(2)] 20 | public string[] Series; 21 | [Key(3)] 22 | public string[] Characters; 23 | [Key(4)] 24 | public string[] Languages; 25 | [Key(5)] 26 | public string[] Types; 27 | [Key(6)] 28 | public string[] Tags; 29 | } 30 | 31 | [MessagePackObject] 32 | public struct HitomiIndexMetadata 33 | { 34 | [Key(0)] 35 | public int[] Artists { get; set; } 36 | [Key(1)] 37 | public int[] Groups { get; set; } 38 | [Key(2)] 39 | public int[] Parodies { get; set; } 40 | [Key(3)] 41 | public int[] Tags { get; set; } 42 | [Key(4)] 43 | public int[] Characters { get; set; } 44 | [Key(5)] 45 | public int Language { get; set; } 46 | [Key(6)] 47 | public string Name { get; set; } 48 | [Key(7)] 49 | public int Type { get; set; } 50 | [Key(8)] 51 | public int ID { get; set; } 52 | 53 | [JsonIgnore] 54 | public DateTime? DateTime; 55 | } 56 | 57 | [MessagePackObject] 58 | public class HitomiIndexDataModel 59 | { 60 | [Key(0)] 61 | public HitomiIndexModel index; 62 | [Key(1)] 63 | public List metadata; 64 | } 65 | 66 | public class HitomiIndex : ILazy 67 | { 68 | private static void add(Dictionary dic, string arr) 69 | { 70 | if (arr == null) return; 71 | if (!dic.ContainsKey(arr)) 72 | dic.Add(arr, dic.Count); 73 | } 74 | 75 | private static void add(Dictionary dic, string[] arr) 76 | { 77 | if (arr == null) return; 78 | foreach (var item in arr) 79 | if (!dic.ContainsKey(item)) 80 | dic.Add(item, dic.Count); 81 | } 82 | 83 | private static string[] pp(Dictionary dic) 84 | { 85 | var list = dic.ToList(); 86 | list.Sort((x, y) => x.Value.CompareTo(y.Value)); 87 | return list.Select(x => x.Key).ToArray(); 88 | } 89 | 90 | public static (HitomiIndexModel, List) MakeIndexF() 91 | { 92 | var artists = new Dictionary(); 93 | var groups = new Dictionary(); 94 | var series = new Dictionary(); 95 | var characters = new Dictionary(); 96 | var languages = new Dictionary(); 97 | var types = new Dictionary(); 98 | var tags = new Dictionary(); 99 | 100 | foreach (var md in HitomiData.Instance.metadata_collection) 101 | { 102 | add(artists, md.Artists); 103 | add(groups, md.Groups); 104 | add(series, md.Parodies); 105 | add(characters, md.Characters); 106 | if (md.Language != null) 107 | add(languages, md.Language.ToLower()); 108 | else 109 | add(languages, md.Language); 110 | if (md.Type != null) 111 | add(types, md.Type.ToLower()); 112 | else 113 | add(types, md.Type); 114 | add(tags, md.Tags); 115 | } 116 | 117 | var index = new HitomiIndexModel(); 118 | 119 | index.Artists = pp(artists); 120 | index.Groups = pp(groups); 121 | index.Series = pp(series); 122 | index.Characters = pp(characters); 123 | index.Languages = pp(languages); 124 | index.Types = pp(types); 125 | index.Tags = pp(tags); 126 | 127 | var mdl = new List(); 128 | 129 | foreach (var md in HitomiData.Instance.metadata_collection) 130 | { 131 | var him = new HitomiIndexMetadata(); 132 | him.ID = md.ID; 133 | him.Name = md.Name; 134 | if (md.Artists != null) him.Artists = md.Artists.Select(x => artists[x]).ToArray(); 135 | if (md.Groups != null) him.Groups = md.Groups.Select(x => groups[x]).ToArray(); 136 | if (md.Parodies != null) him.Parodies = md.Parodies.Select(x => series[x]).ToArray(); 137 | if (md.Characters != null) him.Characters = md.Characters.Select(x => characters[x]).ToArray(); 138 | if (md.Language != null) him.Language = languages[md.Language.ToLower()]; else him.Language = -1; 139 | if (md.Type != null) him.Type = types[md.Type.ToLower()]; else him.Type = -1; 140 | if (md.Tags != null) him.Tags = md.Tags.Select(x => tags[x]).ToArray(); 141 | him.DateTime = md.DateTime; 142 | mdl.Add(him); 143 | } 144 | 145 | return (index, mdl); 146 | } 147 | 148 | public static void MakeIndex() 149 | { 150 | var artists = new Dictionary(); 151 | var groups = new Dictionary(); 152 | var series = new Dictionary(); 153 | var characters = new Dictionary(); 154 | var languages = new Dictionary(); 155 | var types = new Dictionary(); 156 | var tags = new Dictionary(); 157 | 158 | foreach (var md in HitomiData.Instance.metadata_collection) 159 | { 160 | add(artists, md.Artists); 161 | add(groups, md.Groups); 162 | add(series, md.Parodies); 163 | add(characters, md.Characters); 164 | if (md.Language != null) 165 | add(languages, md.Language.ToLower()); 166 | else 167 | add(languages, md.Language); 168 | if (md.Type != null) 169 | add(types, md.Type.ToLower()); 170 | else 171 | add(types, md.Type); 172 | add(tags, md.Tags); 173 | } 174 | 175 | var index = new HitomiIndexModel(); 176 | 177 | index.Artists = pp(artists); 178 | index.Groups = pp(groups); 179 | index.Series = pp(series); 180 | index.Characters = pp(characters); 181 | index.Languages = pp(languages); 182 | index.Types = pp(types); 183 | index.Tags = pp(tags); 184 | 185 | var mdl = new List(); 186 | 187 | foreach (var md in HitomiData.Instance.metadata_collection) 188 | { 189 | var him = new HitomiIndexMetadata(); 190 | him.ID = md.ID; 191 | him.Name = md.Name; 192 | if (md.Artists != null) him.Artists = md.Artists.Select(x => artists[x]).ToArray(); 193 | if (md.Groups != null) him.Groups = md.Groups.Select(x => groups[x]).ToArray(); 194 | if (md.Parodies != null) him.Parodies = md.Parodies.Select(x => series[x]).ToArray(); 195 | if (md.Characters != null) him.Characters = md.Characters.Select(x => characters[x]).ToArray(); 196 | if (md.Language != null) him.Language = languages[md.Language.ToLower()]; else him.Language = -1; 197 | if (md.Type != null) him.Type = types[md.Type.ToLower()]; else him.Type = -1; 198 | if (md.Tags != null) him.Tags = md.Tags.Select(x => tags[x]).ToArray(); 199 | mdl.Add(him); 200 | } 201 | 202 | var result = new HitomiIndexDataModel(); 203 | result.index = index; 204 | result.metadata = mdl; 205 | 206 | var bbb = MessagePackSerializer.Serialize(result); 207 | using (FileStream fsStream = new FileStream("index-metadata.json", FileMode.Create)) 208 | using (BinaryWriter sw = new BinaryWriter(fsStream)) 209 | { 210 | sw.Write(bbb); 211 | } 212 | } 213 | 214 | } 215 | } 216 | -------------------------------------------------------------------------------- /hsync/Crypto/Hash.cs: -------------------------------------------------------------------------------- 1 | // This source code is a part of project violet-server. 2 | // Copyright (C)2020-2021. violet-team. Licensed under the MIT Licence. 3 | 4 | using System; 5 | using System.Collections.Generic; 6 | using System.IO; 7 | using System.Security.Cryptography; 8 | using System.Text; 9 | 10 | namespace hsync.Crypto 11 | { 12 | public static class Hash 13 | { 14 | public static string GetFileHash(this string file) 15 | { 16 | using (FileStream stream = File.OpenRead(file)) 17 | { 18 | SHA512Managed sha = new SHA512Managed(); 19 | byte[] hash = sha.ComputeHash(stream); 20 | return BitConverter.ToString(hash).Replace("-", String.Empty); 21 | } 22 | } 23 | 24 | public static string GetHashSHA1(this string str) 25 | { 26 | SHA1Managed sha = new SHA1Managed(); 27 | byte[] hash = sha.ComputeHash(Encoding.UTF8.GetBytes(str)); 28 | return BitConverter.ToString(hash).Replace("-", String.Empty); 29 | } 30 | 31 | public static string GetHashSHA256(this string str) 32 | { 33 | SHA256Managed sha = new SHA256Managed(); 34 | byte[] hash = sha.ComputeHash(Encoding.UTF8.GetBytes(str)); 35 | return BitConverter.ToString(hash).Replace("-", String.Empty); 36 | } 37 | 38 | public static string GetHashSHA512(this string str) 39 | { 40 | SHA512Managed sha = new SHA512Managed(); 41 | byte[] hash = sha.ComputeHash(Encoding.UTF8.GetBytes(str)); 42 | return BitConverter.ToString(hash).Replace("-", String.Empty); 43 | } 44 | 45 | public static string GetHashMD5(this string str) 46 | { 47 | var md5 = MD5.Create(); 48 | byte[] hash = md5.ComputeHash(Encoding.UTF8.GetBytes(str)); 49 | return BitConverter.ToString(hash).Replace("-", String.Empty); 50 | } 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /hsync/Crypto/Random.cs: -------------------------------------------------------------------------------- 1 | // This source code is a part of project violet-server. 2 | // Copyright (C)2020-2021. violet-team. Licensed under the MIT Licence. 3 | 4 | using System; 5 | using System.Collections.Generic; 6 | using System.Linq; 7 | using System.Text; 8 | 9 | namespace hsync.Crypto 10 | { 11 | public class Random 12 | { 13 | private static System.Random random = new System.Random(); 14 | public static string RandomString(int length) 15 | { 16 | const string chars = "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"; 17 | return new string(Enumerable.Repeat(chars, length) 18 | .Select(s => s[random.Next(s.Length)]).ToArray()); 19 | } 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /hsync/DataBaseCreatorLowPerf.cs: -------------------------------------------------------------------------------- 1 | // This source code is a part of project violet-server. 2 | // Copyright (C)2020-2021. violet-team. Licensed under the MIT Licence. 3 | 4 | using hsync.Component; 5 | using Newtonsoft.Json; 6 | using SQLite; 7 | using System; 8 | using System.Collections.Generic; 9 | using System.IO; 10 | using System.Linq; 11 | using System.Text; 12 | 13 | namespace hsync 14 | { 15 | public class DataBaseCreatorLowPerf 16 | { 17 | public void ExtractRawDatabase(string filename = "rawdata", string language = null, bool include_exhentai = false) 18 | { 19 | var db = new SQLiteConnection("data.db"); 20 | var count = db.ExecuteScalar("SELECT COUNT(*) FROM HitomiColumnModel"); 21 | 22 | Action, string> insert = (map, qr) => 23 | { 24 | if (qr == null) return; 25 | foreach (var tag in qr.Split('|')) 26 | { 27 | if (tag != "") 28 | { 29 | if (!map.ContainsKey(tag)) 30 | map.Add(tag, 0); 31 | map[tag] += 1; 32 | } 33 | } 34 | }; 35 | 36 | Action, string> insertSingle = (map, qr) => 37 | { 38 | if (qr == null || qr == "") return; 39 | if (!map.ContainsKey(qr)) 40 | map.Add(qr, 0); 41 | map[qr] += 1; 42 | }; 43 | 44 | if (Directory.Exists(filename)) 45 | Directory.Delete(filename, true); 46 | Directory.CreateDirectory(filename); 47 | 48 | var index = new IndexData(); 49 | var result_artist = new Dictionary>(); 50 | var result_group = new Dictionary>(); 51 | var result_uploader = new Dictionary>(); 52 | var result_series = new Dictionary>(); 53 | var result_character = new Dictionary>(); 54 | var result_characterseries = new Dictionary>(); 55 | var result_seriescharacter = new Dictionary>(); 56 | var result_charactercharacter = new Dictionary>(); 57 | var result_seriesseries = new Dictionary>(); 58 | var ff = new Dictionary(); 59 | 60 | var rdb = new SQLiteConnection(filename + "/data.db"); 61 | var info = rdb.GetTableInfo(typeof(HitomiColumnModel).Name); 62 | rdb.CreateTable(); 63 | 64 | const int perLoop = 50000; 65 | 66 | for (int i = 0; i < count; i += perLoop) 67 | { 68 | var query = db.Query($"SELECT * FROM HitomiColumnModel ORDER BY Id LIMIT {perLoop} OFFSET {i}"); 69 | 70 | Console.WriteLine($"{i}/{count}"); 71 | 72 | var fquery = query.Where(article => 73 | { 74 | if (article == null) return false; 75 | 76 | if (language != null && article.Language != language && article.Language != "n/a") 77 | return false; 78 | 79 | if (include_exhentai && article.ExistOnHitomi == 0) 80 | return false; 81 | 82 | return true; 83 | }); 84 | 85 | rdb.InsertAll(fquery); 86 | 87 | foreach (var article in fquery) 88 | { 89 | if (article == null) continue; 90 | 91 | if (language != null && article.Language != language && article.Language != "n/a") 92 | continue; 93 | 94 | if (include_exhentai && article.ExistOnHitomi == 0) 95 | continue; 96 | 97 | insert(index.tags, article.Tags); 98 | insert(index.artists, article.Artists); 99 | insert(index.groups, article.Groups); 100 | insert(index.series, article.Series); 101 | insert(index.characters, article.Characters); 102 | insertSingle(index.languages, article.Language); 103 | insertSingle(index.types, article.Type); 104 | insertSingle(index.uploaders, article.Uploader); 105 | insertSingle(index.classes, article.Class); 106 | 107 | if (article.Tags == null || article.Tags.Length == 0) 108 | continue; 109 | if (article.Artists != null) 110 | { 111 | foreach (var artist in article.Artists.Split('|')) 112 | { 113 | if (artist == "") 114 | continue; 115 | if (!result_artist.ContainsKey(artist)) 116 | result_artist.Add(artist, new Dictionary()); 117 | foreach (var tag in article.Tags.Split('|')) 118 | { 119 | if (tag == "") 120 | continue; 121 | if (!ff.ContainsKey(tag)) 122 | ff.Add(tag, ff.Count); 123 | if (!result_artist[artist].ContainsKey(ff[tag])) 124 | result_artist[artist].Add(ff[tag], 0); 125 | result_artist[artist][ff[tag]] += 1; 126 | } 127 | } 128 | } 129 | if (article.Groups != null) 130 | { 131 | foreach (var artist in article.Groups.Split('|')) 132 | { 133 | if (artist == "") 134 | continue; 135 | if (!result_group.ContainsKey(artist)) 136 | result_group.Add(artist, new Dictionary()); 137 | foreach (var tag in article.Tags.Split('|')) 138 | { 139 | if (tag == "") 140 | continue; 141 | if (!ff.ContainsKey(tag)) 142 | ff.Add(tag, ff.Count); 143 | if (!result_group[artist].ContainsKey(ff[tag])) 144 | result_group[artist].Add(ff[tag], 0); 145 | result_group[artist][ff[tag]] += 1; 146 | } 147 | } 148 | } 149 | if (article.Uploader != null) 150 | { 151 | foreach (var artist in article.Uploader.Split('|')) 152 | { 153 | if (artist == "") 154 | continue; 155 | if (!result_uploader.ContainsKey(artist)) 156 | result_uploader.Add(artist, new Dictionary()); 157 | foreach (var tag in article.Tags.Split('|')) 158 | { 159 | if (tag == "") 160 | continue; 161 | if (!ff.ContainsKey(tag)) 162 | ff.Add(tag, ff.Count); 163 | if (!result_uploader[artist].ContainsKey(ff[tag])) 164 | result_uploader[artist].Add(ff[tag], 0); 165 | result_uploader[artist][ff[tag]] += 1; 166 | } 167 | } 168 | } 169 | if (article.Series != null) 170 | { 171 | foreach (var artist in article.Series.Split('|')) 172 | { 173 | if (artist == "") 174 | continue; 175 | if (!result_series.ContainsKey(artist)) 176 | result_series.Add(artist, new Dictionary()); 177 | foreach (var tag in article.Tags.Split('|')) 178 | { 179 | if (tag == "") 180 | continue; 181 | if (!ff.ContainsKey(tag)) 182 | ff.Add(tag, ff.Count); 183 | if (!result_series[artist].ContainsKey(ff[tag])) 184 | result_series[artist].Add(ff[tag], 0); 185 | result_series[artist][ff[tag]] += 1; 186 | } 187 | } 188 | } 189 | if (article.Characters != null) 190 | { 191 | foreach (var artist in article.Characters.Split('|')) 192 | { 193 | if (artist == "") 194 | continue; 195 | if (!result_character.ContainsKey(artist)) 196 | result_character.Add(artist, new Dictionary()); 197 | foreach (var tag in article.Tags.Split('|')) 198 | { 199 | if (tag == "") 200 | continue; 201 | if (!ff.ContainsKey(tag)) 202 | ff.Add(tag, ff.Count); 203 | if (!result_character[artist].ContainsKey(ff[tag])) 204 | result_character[artist].Add(ff[tag], 0); 205 | result_character[artist][ff[tag]] += 1; 206 | } 207 | } 208 | } 209 | if (article.Series != null && article.Characters != null) 210 | { 211 | foreach (var series in article.Series.Split('|')) 212 | { 213 | if (series == "") 214 | continue; 215 | if (!result_characterseries.ContainsKey(series)) 216 | result_characterseries.Add(series, new Dictionary()); 217 | foreach (var character in article.Characters.Split('|')) 218 | { 219 | if (character == "") 220 | continue; 221 | if (!result_characterseries[series].ContainsKey(character)) 222 | result_characterseries[series].Add(character, 0); 223 | result_characterseries[series][character] += 1; 224 | } 225 | } 226 | foreach (var character in article.Characters.Split('|')) 227 | { 228 | if (character == "") 229 | continue; 230 | if (!result_seriescharacter.ContainsKey(character)) 231 | result_seriescharacter.Add(character, new Dictionary()); 232 | foreach (var series in article.Series.Split('|')) 233 | { 234 | if (series == "") 235 | continue; 236 | if (!result_seriescharacter[character].ContainsKey(series)) 237 | result_seriescharacter[character].Add(series, 0); 238 | result_seriescharacter[character][series] += 1; 239 | } 240 | } 241 | } 242 | if (article.Series != null) 243 | { 244 | foreach (var series in article.Series.Split('|')) 245 | { 246 | if (series == "") 247 | continue; 248 | if (!result_seriesseries.ContainsKey(series)) 249 | result_seriesseries.Add(series, new Dictionary()); 250 | foreach (var series2 in article.Series.Split('|')) 251 | { 252 | if (series2 == "" || series == series2) 253 | continue; 254 | if (!result_seriesseries[series].ContainsKey(series2)) 255 | result_seriesseries[series].Add(series2, 0); 256 | result_seriesseries[series][series2] += 1; 257 | } 258 | } 259 | } 260 | if (article.Characters != null) 261 | { 262 | foreach (var character in article.Characters.Split('|')) 263 | { 264 | if (character == "") 265 | continue; 266 | if (!result_charactercharacter.ContainsKey(character)) 267 | result_charactercharacter.Add(character, new Dictionary()); 268 | foreach (var character2 in article.Characters.Split('|')) 269 | { 270 | if (character2 == "" || character == character2) 271 | continue; 272 | if (!result_charactercharacter[character].ContainsKey(character2)) 273 | result_charactercharacter[character].Add(character2, 0); 274 | result_charactercharacter[character][character2] += 1; 275 | } 276 | } 277 | } 278 | } 279 | } 280 | rdb.Close(); 281 | 282 | File.WriteAllText(filename + "/index.json", JsonConvert.SerializeObject(index)); 283 | File.WriteAllText(filename + "/tag-index.json", JsonConvert.SerializeObject(ff)); 284 | File.WriteAllText(filename + "/tag-artist.json", JsonConvert.SerializeObject(result_artist)); 285 | File.WriteAllText(filename + "/tag-group.json", JsonConvert.SerializeObject(result_group)); 286 | File.WriteAllText(filename + "/tag-uploader.json", JsonConvert.SerializeObject(result_uploader)); 287 | File.WriteAllText(filename + "/tag-series.json", JsonConvert.SerializeObject(result_series)); 288 | File.WriteAllText(filename + "/tag-character.json", JsonConvert.SerializeObject(result_character)); 289 | File.WriteAllText(filename + "/character-series.json", JsonConvert.SerializeObject(result_characterseries)); 290 | File.WriteAllText(filename + "/series-character.json", JsonConvert.SerializeObject(result_seriescharacter)); 291 | File.WriteAllText(filename + "/character-character.json", JsonConvert.SerializeObject(result_charactercharacter)); 292 | File.WriteAllText(filename + "/series-series.json", JsonConvert.SerializeObject(result_seriesseries)); 293 | } 294 | 295 | class IndexData 296 | { 297 | [JsonProperty(PropertyName = "tag")] 298 | public Dictionary tags = new Dictionary(); 299 | [JsonProperty(PropertyName = "lang")] 300 | public Dictionary languages = new Dictionary(); 301 | [JsonProperty(PropertyName = "artist")] 302 | public Dictionary artists = new Dictionary(); 303 | [JsonProperty(PropertyName = "group")] 304 | public Dictionary groups = new Dictionary(); 305 | [JsonProperty(PropertyName = "type")] 306 | public Dictionary types = new Dictionary(); 307 | [JsonProperty(PropertyName = "uploader")] 308 | public Dictionary uploaders = new Dictionary(); 309 | [JsonProperty(PropertyName = "series")] 310 | public Dictionary series = new Dictionary(); 311 | [JsonProperty(PropertyName = "character")] 312 | public Dictionary characters = new Dictionary(); 313 | [JsonProperty(PropertyName = "class")] 314 | public Dictionary classes = new Dictionary(); 315 | } 316 | } 317 | } 318 | -------------------------------------------------------------------------------- /hsync/Internals.cs: -------------------------------------------------------------------------------- 1 | // This source code is a part of project violet-server. 2 | // Copyright (C)2020-2021. violet-team. Licensed under the MIT Licence. 3 | 4 | using System; 5 | using System.Collections.Generic; 6 | using System.Globalization; 7 | using System.IO; 8 | using System.Linq; 9 | using System.Reflection; 10 | using System.Text; 11 | 12 | namespace hsync 13 | { 14 | public class Internals 15 | { 16 | public static DateTime GetBuildDate() 17 | { 18 | const string BuildVersionMetadataPrefix = "+build"; 19 | 20 | var attribute = Assembly.GetExecutingAssembly().GetCustomAttribute(); 21 | if (attribute?.InformationalVersion != null) 22 | { 23 | var value = attribute.InformationalVersion; 24 | var index = value.IndexOf(BuildVersionMetadataPrefix); 25 | if (index > 0) 26 | { 27 | value = value.Substring(index + BuildVersionMetadataPrefix.Length); 28 | if (DateTime.TryParseExact(value, "yyyyMMddHHmmss", CultureInfo.InvariantCulture, DateTimeStyles.None, out var result)) 29 | { 30 | return result; 31 | } 32 | } 33 | } 34 | 35 | return default; 36 | } 37 | 38 | #region Low Level 39 | 40 | public const BindingFlags DefaultBinding = BindingFlags.NonPublic | 41 | BindingFlags.Instance | BindingFlags.IgnoreCase | BindingFlags.Public | BindingFlags.FlattenHierarchy; 42 | 43 | public const BindingFlags CommonBinding = BindingFlags.Instance | BindingFlags.Public; 44 | 45 | public static List get_all_fields(Type t, BindingFlags flags) 46 | { 47 | if (t == null) 48 | return new List(); 49 | 50 | var list = t.GetFields(flags).ToList(); 51 | list.AddRange(get_all_fields(t.BaseType, flags)); 52 | return list; 53 | } 54 | 55 | public static List enum_recursion(object obj, string[] bb, int ptr) 56 | { 57 | if (bb.Length == ptr) 58 | { 59 | return get_all_fields(obj.GetType(), DefaultBinding); 60 | } 61 | return enum_recursion(obj.GetType().GetField(bb[ptr], DefaultBinding).GetValue(obj), bb, ptr + 1); 62 | } 63 | 64 | public static List enum_recursion(object obj, string[] bb, int ptr, BindingFlags option) 65 | { 66 | if (bb.Length == ptr) 67 | { 68 | return obj.GetType().GetFields(option).ToList(); 69 | } 70 | var x = obj.GetType().GetField(bb[ptr], DefaultBinding); 71 | return enum_recursion(obj.GetType().GetField(bb[ptr], DefaultBinding).GetValue(obj), bb, ptr + 1, option); 72 | } 73 | 74 | public static object get_recursion(object obj, string[] bb, int ptr) 75 | { 76 | if (bb.Length == ptr) 77 | { 78 | return obj; 79 | } 80 | return get_recursion(obj.GetType().GetField(bb[ptr], DefaultBinding).GetValue(obj), bb, ptr + 1); 81 | } 82 | 83 | public static void set_recursion(object obj, string[] bb, int ptr, object val) 84 | { 85 | if (bb.Length - 1 == ptr) 86 | { 87 | obj.GetType().GetField(bb[ptr]).SetValue(obj, 88 | Convert.ChangeType(val, obj.GetType().GetField(bb[ptr], DefaultBinding).GetValue(obj).GetType())); 89 | return; 90 | } 91 | set_recursion(obj.GetType().GetField(bb[ptr]).GetValue(obj), bb, ptr + 1, val); 92 | } 93 | 94 | public static List enum_methods(object obj, string[] bb, int ptr, BindingFlags option) 95 | { 96 | if (bb.Length == ptr) 97 | { 98 | return obj.GetType().GetMethods(option).ToList(); 99 | } 100 | var x = obj.GetType().GetField(bb[ptr], DefaultBinding); 101 | return enum_methods(obj.GetType().GetField(bb[ptr], DefaultBinding).GetValue(obj), bb, ptr + 1, option); 102 | } 103 | 104 | public static object call_method(object obj, string[] bb, int ptr, BindingFlags option, object[] param) 105 | { 106 | if (bb.Length - 1 == ptr) 107 | { 108 | return obj.GetType().GetMethods(option | BindingFlags.Static).Where(y => y.Name == bb[ptr]).ToList()[0].Invoke(obj, param); 109 | } 110 | var x = obj.GetType().GetField(bb[ptr], DefaultBinding | BindingFlags.Static); 111 | return call_method(obj.GetType().GetField(bb[ptr], DefaultBinding | BindingFlags.Static).GetValue(obj), bb, ptr + 1, option, param); 112 | } 113 | 114 | public static ParameterInfo[] get_method_paraminfo(object obj, string[] bb, int ptr, BindingFlags option) 115 | { 116 | if (bb.Length - 1 == ptr) 117 | { 118 | return obj.GetType().GetMethods(option).Where(y => y.Name == bb[ptr]).ToList()[0].GetParameters(); 119 | } 120 | var x = obj.GetType().GetField(bb[ptr], DefaultBinding); 121 | return get_method_paraminfo(obj.GetType().GetField(bb[ptr], DefaultBinding).GetValue(obj), bb, ptr + 1, option); 122 | } 123 | 124 | #endregion 125 | } 126 | } 127 | -------------------------------------------------------------------------------- /hsync/Log/Logs.cs: -------------------------------------------------------------------------------- 1 | // This source code is a part of project violet-server. 2 | // Copyright (C) 2020. violet-team. Licensed under the MIT Licence. 3 | 4 | using hsync.Utils; 5 | using Newtonsoft.Json; 6 | using System; 7 | using System.Collections.Generic; 8 | using System.Collections.ObjectModel; 9 | using System.Collections.Specialized; 10 | using System.ComponentModel; 11 | using System.Globalization; 12 | using System.IO; 13 | using System.Linq; 14 | using System.Reflection; 15 | using System.Text; 16 | using System.Xml.Serialization; 17 | 18 | namespace hsync.Log 19 | { 20 | public class Logs : ILazy 21 | { 22 | /// 23 | /// Serialize an object. 24 | /// 25 | /// 26 | /// 27 | public static string SerializeObject(object toSerialize) 28 | { 29 | try 30 | { 31 | return JsonConvert.SerializeObject(toSerialize, Formatting.Indented, new JsonSerializerSettings 32 | { 33 | ReferenceLoopHandling = ReferenceLoopHandling.Ignore 34 | }); 35 | } 36 | catch 37 | { 38 | try 39 | { 40 | XmlSerializer xmlSerializer = new XmlSerializer(toSerialize.GetType()); 41 | 42 | using (StringWriter textWriter = new StringWriter()) 43 | { 44 | xmlSerializer.Serialize(textWriter, toSerialize); 45 | return textWriter.ToString(); 46 | } 47 | } 48 | catch 49 | { 50 | return toSerialize.ToString(); 51 | } 52 | } 53 | } 54 | 55 | public delegate void NotifyEvent(object sender, EventArgs e); 56 | event EventHandler LogCollectionChange; 57 | event EventHandler LogErrorCollectionChange; 58 | event EventHandler LogWarningCollectionChange; 59 | object event_lock = new object(); 60 | 61 | /// 62 | /// Attach your own notify event. 63 | /// 64 | /// 65 | public void AddLogNotify(NotifyEvent notify_event) 66 | { 67 | LogCollectionChange += new EventHandler(notify_event); 68 | } 69 | 70 | public void ClearLogNotify() 71 | { 72 | LogCollectionChange = null; 73 | } 74 | 75 | /// 76 | /// Attach your own notify event. 77 | /// 78 | /// 79 | public void AddLogErrorNotify(NotifyEvent notify_event) 80 | { 81 | LogErrorCollectionChange += new EventHandler(notify_event); 82 | } 83 | 84 | /// 85 | /// Attach your own notify event. 86 | /// 87 | /// 88 | public void AddLogWarningNotify(NotifyEvent notify_event) 89 | { 90 | LogWarningCollectionChange += new EventHandler(notify_event); 91 | } 92 | 93 | /// 94 | /// Push some message to log. 95 | /// 96 | /// 97 | public void Push(string str) 98 | { 99 | write_log(DateTime.Now, str); 100 | lock (event_lock) LogCollectionChange?.Invoke(Tuple.Create(DateTime.Now, str, false), null); 101 | } 102 | 103 | /// 104 | /// Push some object to log. 105 | /// 106 | /// 107 | public void Push(object obj) 108 | { 109 | write_log(DateTime.Now, obj.ToString()); 110 | write_log(DateTime.Now, SerializeObject(obj)); 111 | lock (event_lock) 112 | { 113 | LogCollectionChange?.Invoke(Tuple.Create(DateTime.Now, obj.ToString(), false), null); 114 | LogCollectionChange?.Invoke(Tuple.Create(DateTime.Now, SerializeObject(obj), true), null); 115 | } 116 | } 117 | 118 | /// 119 | /// Push some message to log. 120 | /// 121 | /// 122 | public void PushError(string str) 123 | { 124 | write_error_log(DateTime.Now, str); 125 | lock (event_lock) LogErrorCollectionChange?.Invoke(Tuple.Create(DateTime.Now, str, false), null); 126 | } 127 | 128 | /// 129 | /// Push some object to log. 130 | /// 131 | /// 132 | public void PushError(object obj) 133 | { 134 | write_error_log(DateTime.Now, obj.ToString()); 135 | write_error_log(DateTime.Now, SerializeObject(obj)); 136 | lock (event_lock) 137 | { 138 | LogErrorCollectionChange?.Invoke(Tuple.Create(DateTime.Now, obj.ToString(), false), null); 139 | LogErrorCollectionChange?.Invoke(Tuple.Create(DateTime.Now, SerializeObject(obj), true), null); 140 | } 141 | } 142 | 143 | /// 144 | /// Push some message to log. 145 | /// 146 | /// 147 | public void PushWarning(string str) 148 | { 149 | write_warning_log(DateTime.Now, str); 150 | lock (event_lock) LogWarningCollectionChange?.Invoke(Tuple.Create(DateTime.Now, str, false), null); 151 | } 152 | 153 | /// 154 | /// Push some object to log. 155 | /// 156 | /// 157 | public void PushWarning(object obj) 158 | { 159 | write_warning_log(DateTime.Now, obj.ToString()); 160 | write_warning_log(DateTime.Now, SerializeObject(obj)); 161 | lock (event_lock) 162 | { 163 | LogWarningCollectionChange?.Invoke(Tuple.Create(DateTime.Now, obj.ToString(), false), null); 164 | LogWarningCollectionChange?.Invoke(Tuple.Create(DateTime.Now, SerializeObject(obj), true), null); 165 | } 166 | } 167 | 168 | public void PushException(Exception e) 169 | { 170 | PushError($"Message: {e.Message}\n" + e.StackTrace); 171 | if (e.InnerException != null) 172 | PushException(e.InnerException); 173 | } 174 | 175 | public void Panic() 176 | { 177 | Environment.Exit(1); 178 | } 179 | 180 | object log_lock = new object(); 181 | 182 | private void write_log(DateTime dt, string message) 183 | { 184 | CultureInfo en = new CultureInfo("en-US"); 185 | lock (log_lock) 186 | { 187 | File.AppendAllText(Path.Combine(AppProvider.ApplicationPath, "log.txt"), $"[{dt.ToString(en)}] {message}\r\n"); 188 | } 189 | } 190 | 191 | private void write_error_log(DateTime dt, string message) 192 | { 193 | CultureInfo en = new CultureInfo("en-US"); 194 | lock (log_lock) 195 | { 196 | File.AppendAllText(Path.Combine(AppProvider.ApplicationPath, "log.txt"), $"[{dt.ToString(en)}] [Error] {message}\r\n"); 197 | } 198 | } 199 | 200 | private void write_warning_log(DateTime dt, string message) 201 | { 202 | CultureInfo en = new CultureInfo("en-US"); 203 | lock (log_lock) 204 | { 205 | File.AppendAllText(Path.Combine(AppProvider.ApplicationPath, "log.txt"), $"[{dt.ToString(en)}] [Warning] {message}\r\n"); 206 | } 207 | } 208 | } 209 | } 210 | -------------------------------------------------------------------------------- /hsync/Network/NetField.cs: -------------------------------------------------------------------------------- 1 | // This source code is a part of project violet-server. 2 | // Copyright (C)2020-2021. violet-team. Licensed under the MIT Licence. 3 | 4 | using System; 5 | using System.Collections.Generic; 6 | using System.IO; 7 | using System.Linq; 8 | using System.Net; 9 | using System.Text; 10 | using System.Threading; 11 | 12 | namespace hsync.Network 13 | { 14 | /// 15 | /// Implementaion of real download procedure 16 | /// 17 | public class NetField 18 | { 19 | public static void Do(NetTask content) 20 | { 21 | var retry_count = 0; 22 | 23 | RETRY_PROCEDURE: 24 | 25 | if (content.Cancel != null && content.Cancel.IsCancellationRequested) 26 | { 27 | content.CancleCallback(); 28 | return; 29 | } 30 | 31 | NetTaskPass.RunOnField(ref content); 32 | 33 | //if (content.DownloadString) 34 | // Log.Logs.Instance.Push("[NetField] Start download string... " + content.Url); 35 | //else if (content.MemoryCache) 36 | // Log.Logs.Instance.Push("[NetField] Start download to memory... " + content.Url); 37 | //else if (content.SaveFile) 38 | // Log.Logs.Instance.Push("[NetField] Start download file... " + content.Url + " to " + content.Filename); 39 | 40 | REDIRECTION: 41 | 42 | if (content.Cancel != null && content.Cancel.IsCancellationRequested) 43 | { 44 | content.CancleCallback(); 45 | return; 46 | } 47 | 48 | content.StartCallback?.Invoke(); 49 | 50 | try 51 | { 52 | // 53 | // Initialize http-web-request 54 | // 55 | 56 | var request = (HttpWebRequest)WebRequest.Create(content.Url); 57 | content.Request = request; 58 | 59 | request.Accept = content.Accept; 60 | request.UserAgent = content.UserAgent; 61 | 62 | if (content.Referer != null) 63 | request.Referer = content.Referer; 64 | else 65 | request.Referer = (content.Url.StartsWith("https://") ? "https://" : (content.Url.Split(':')[0] + "//")) + request.RequestUri.Host; 66 | 67 | if (content.Cookie != null) 68 | request.Headers.Add(HttpRequestHeader.Cookie, content.Cookie); 69 | 70 | if (content.Headers != null) 71 | content.Headers.ToList().ForEach(p => request.Headers.Add(p.Key, p.Value)); 72 | 73 | if (content.Proxy != null) 74 | request.Proxy = content.Proxy; 75 | 76 | if (content.TimeoutInfinite) 77 | request.Timeout = Timeout.Infinite; 78 | else 79 | request.Timeout = content.TimeoutMillisecond; 80 | 81 | request.AllowAutoRedirect = content.AutoRedirection; 82 | 83 | // 84 | // POST Data 85 | // 86 | 87 | if (content.Query != null) 88 | { 89 | request.Method = "POST"; 90 | request.ContentType = "application/x-www-form-urlencoded"; 91 | 92 | var request_stream = new StreamWriter(request.GetRequestStream()); 93 | var query = string.Join("&", content.Query.ToList().Select(x => $"{x.Key}={x.Value}")); 94 | request_stream.Write(query); 95 | request_stream.Close(); 96 | 97 | if (content.Cancel != null && content.Cancel.IsCancellationRequested) 98 | { 99 | content.CancleCallback(); 100 | return; 101 | } 102 | } 103 | 104 | // 105 | // Wait request 106 | // 107 | 108 | using (HttpWebResponse response = (HttpWebResponse)request.GetResponse()) 109 | { 110 | if (response.StatusCode == HttpStatusCode.NotFound || 111 | response.StatusCode == HttpStatusCode.Forbidden || 112 | response.StatusCode == HttpStatusCode.Unauthorized || 113 | response.StatusCode == HttpStatusCode.BadRequest || 114 | response.StatusCode == HttpStatusCode.InternalServerError) 115 | { 116 | // 117 | // Cannot continue 118 | // 119 | 120 | content.ErrorCallback?.Invoke(NetTask.NetError.CannotContinueByCriticalError); 121 | return; 122 | } 123 | else if (response.StatusCode == HttpStatusCode.Moved || 124 | response.StatusCode == HttpStatusCode.Redirect) 125 | { 126 | if (content.AutoRedirection) 127 | { 128 | var old = content.Url; 129 | content.Url = response.Headers.Get("Location"); 130 | Log.Logs.Instance.Push("[NetField] Redirection " + old + " to " + content.Url); 131 | goto REDIRECTION; 132 | } 133 | } 134 | else if (response.StatusCode == HttpStatusCode.OK) 135 | { 136 | if (content.Cancel != null && content.Cancel.IsCancellationRequested) 137 | { 138 | content.CancleCallback(); 139 | return; 140 | } 141 | 142 | content.HeaderReceive?.Invoke(response.Headers.ToString()); 143 | content.CookieReceive?.Invoke(response.Cookies); 144 | 145 | Stream istream = response.GetResponseStream(); 146 | Stream ostream = null; 147 | 148 | if (content.DownloadString || content.MemoryCache) 149 | { 150 | ostream = new MemoryStream(); 151 | } 152 | else if (content.DriveCache) 153 | { 154 | // TODO: 155 | } 156 | else 157 | { 158 | ostream = File.OpenWrite(content.Filename); 159 | } 160 | 161 | content.SizeCallback?.Invoke(response.ContentLength); 162 | 163 | if (content.NotifyOnlySize) 164 | { 165 | ostream.Close(); 166 | istream.Close(); 167 | return; 168 | } 169 | 170 | if (content.Cancel != null && content.Cancel.IsCancellationRequested) 171 | { 172 | content.CancleCallback(); 173 | return; 174 | } 175 | 176 | byte[] buffer = new byte[content.DownloadBufferSize]; 177 | long byte_read = 0; 178 | 179 | // 180 | // Download loop 181 | // 182 | 183 | do 184 | { 185 | if (content.Cancel != null && content.Cancel.IsCancellationRequested) 186 | { 187 | content.CancleCallback(); 188 | return; 189 | } 190 | 191 | byte_read = istream.Read(buffer, 0, buffer.Length); 192 | ostream.Write(buffer, 0, (int)byte_read); 193 | 194 | if (content.Cancel != null && content.Cancel.IsCancellationRequested) 195 | { 196 | content.CancleCallback(); 197 | return; 198 | } 199 | 200 | content.DownloadCallback?.Invoke(byte_read); 201 | 202 | } while (byte_read != 0); 203 | 204 | // 205 | // Notify Complete 206 | // 207 | 208 | if (content.DownloadString) 209 | { 210 | if (content.Encoding == null) 211 | content.CompleteCallbackString(Encoding.UTF8.GetString(((MemoryStream)ostream).ToArray())); 212 | else 213 | content.CompleteCallbackString(content.Encoding.GetString(((MemoryStream)ostream).ToArray())); 214 | } 215 | else if (content.MemoryCache) 216 | { 217 | content.CompleteCallbackBytes(((MemoryStream)ostream).ToArray()); 218 | } 219 | else 220 | { 221 | content.CompleteCallback?.Invoke(); 222 | } 223 | 224 | ostream.Close(); 225 | istream.Close(); 226 | 227 | return; 228 | } 229 | } 230 | } 231 | catch (WebException e) 232 | { 233 | var response = (HttpWebResponse)e.Response; 234 | 235 | if (response != null && response.StatusCode == HttpStatusCode.Moved) 236 | { 237 | if (content.AutoRedirection) 238 | { 239 | var old = content.Url; 240 | content.Url = response.Headers.Get("Location"); 241 | Log.Logs.Instance.Push("[NetField] Redirection " + old + " to " + content.Url); 242 | goto REDIRECTION; 243 | } 244 | } 245 | 246 | //lock (Log.Logs.Instance) 247 | //{ 248 | // Log.Logs.Instance.PushError("[NetField] Web Excpetion - " + e.Message + "\r\n" + e.StackTrace); 249 | // Log.Logs.Instance.PushError(content); 250 | //} 251 | 252 | if (content.FailUrls != null && retry_count < content.FailUrls.Count) 253 | { 254 | content.Url = content.FailUrls[retry_count++]; 255 | content.RetryCallback?.Invoke(retry_count); 256 | 257 | lock (Log.Logs.Instance) 258 | { 259 | Log.Logs.Instance.Push($"[NetField] Retry [{retry_count}/{content.RetryCount}]"); 260 | Log.Logs.Instance.Push(content); 261 | } 262 | goto RETRY_PROCEDURE; 263 | } 264 | 265 | if ((response != null && ( 266 | response.StatusCode == HttpStatusCode.NotFound || 267 | response.StatusCode == HttpStatusCode.Forbidden || 268 | response.StatusCode == HttpStatusCode.Unauthorized || 269 | response.StatusCode == HttpStatusCode.BadRequest || 270 | response.StatusCode == HttpStatusCode.InternalServerError)) || 271 | e.Status == WebExceptionStatus.NameResolutionFailure || 272 | e.Status == WebExceptionStatus.UnknownError) 273 | { 274 | if (response != null && response.StatusCode == HttpStatusCode.Forbidden && response.Cookies != null) 275 | { 276 | content.CookieReceive?.Invoke(response.Cookies); 277 | return; 278 | } 279 | 280 | // 281 | // Cannot continue 282 | // 283 | 284 | if (e.Status == WebExceptionStatus.UnknownError) 285 | { 286 | lock (Log.Logs.Instance) 287 | { 288 | Log.Logs.Instance.PushError("[NetField] Check your Firewall, Router or DPI settings."); 289 | Log.Logs.Instance.PushError("[NetField] If you continue to receive this error, please contact developer."); 290 | } 291 | 292 | content.ErrorCallback?.Invoke(NetTask.NetError.UnknowError); 293 | } 294 | else 295 | { 296 | content.ErrorCallback?.Invoke(NetTask.NetError.CannotContinueByCriticalError); 297 | } 298 | 299 | return; 300 | } 301 | } 302 | catch (UriFormatException e) 303 | { 304 | lock (Log.Logs.Instance) 305 | { 306 | Log.Logs.Instance.PushError("[NetField] URI Exception - " + e.Message + "\r\n" + e.StackTrace); 307 | Log.Logs.Instance.PushError(content); 308 | } 309 | 310 | // 311 | // Cannot continue 312 | // 313 | 314 | content.ErrorCallback?.Invoke(NetTask.NetError.UriFormatError); 315 | return; 316 | } 317 | catch (Exception e) 318 | { 319 | lock (Log.Logs.Instance) 320 | { 321 | Log.Logs.Instance.PushError("[NetField] Unhandled Excpetion - " + e.Message + "\r\n" + e.StackTrace); 322 | Log.Logs.Instance.PushError(content); 323 | } 324 | } 325 | 326 | // 327 | // Request Aborted 328 | // 329 | 330 | if (content.Aborted) 331 | { 332 | content.ErrorCallback?.Invoke(NetTask.NetError.Aborted); 333 | return; 334 | } 335 | 336 | // 337 | // Retry 338 | // 339 | 340 | //if (content.FailUrls != null && retry_count < content.FailUrls.Count) 341 | //{ 342 | // content.Url = content.FailUrls[retry_count++]; 343 | // content.RetryCallback?.Invoke(retry_count); 344 | 345 | // lock (Log.Logs.Instance) 346 | // { 347 | // Log.Logs.Instance.Push($"[NetField] Retry [{retry_count}/{content.RetryCount}]"); 348 | // Log.Logs.Instance.Push(content); 349 | // } 350 | // goto RETRY_PROCEDURE; 351 | //} 352 | 353 | //if (content.RetryWhenFail) 354 | //{ 355 | // if (content.RetryCount > retry_count) 356 | // { 357 | // retry_count += 1; 358 | 359 | // content.RetryCallback?.Invoke(retry_count); 360 | 361 | // lock (Log.Logs.Instance) 362 | // { 363 | // Log.Logs.Instance.Push($"[NetField] Retry [{retry_count}/{content.RetryCount}]"); 364 | // Log.Logs.Instance.Push(content); 365 | // } 366 | // goto RETRY_PROCEDURE; 367 | // } 368 | 369 | // // 370 | // // Many retry 371 | // // 372 | 373 | // lock (Log.Logs.Instance) 374 | // { 375 | // Log.Logs.Instance.Push($"[NetField] Many Retry"); 376 | // Log.Logs.Instance.Push(content); 377 | // } 378 | // content.ErrorCallback?.Invoke(NetTask.NetError.ManyRetry); 379 | //} 380 | 381 | content.ErrorCallback?.Invoke(NetTask.NetError.Unhandled); 382 | } 383 | } 384 | } 385 | -------------------------------------------------------------------------------- /hsync/Network/NetQueue.cs: -------------------------------------------------------------------------------- 1 | // This source code is a part of project violet-server. 2 | // Copyright (C)2020-2021. violet-team. Licensed under the MIT Licence. 3 | 4 | using hsync.Utils; 5 | using System; 6 | using System.Collections.Generic; 7 | using System.Text; 8 | using System.Threading; 9 | using System.Threading.Tasks; 10 | 11 | namespace hsync.Network 12 | { 13 | /// 14 | /// Download Queue Implementation 15 | /// 16 | public class NetQueue 17 | { 18 | public Queue queue = new Queue(); 19 | 20 | SemaphoreSlim semaphore; 21 | int capacity = 0; 22 | 23 | public NetQueue(int capacity = 0) 24 | { 25 | this.capacity = capacity; 26 | 27 | if (this.capacity == 0) 28 | this.capacity = Environment.ProcessorCount; 29 | 30 | int count = 50; //816; 31 | ThreadPool.SetMinThreads(count, count); 32 | semaphore = new SemaphoreSlim(count, count); 33 | } 34 | 35 | public Task Add(NetTask task) 36 | { 37 | return Task.Run(async () => 38 | { 39 | await semaphore.WaitAsync().ConfigureAwait(false); 40 | _ = Task.Run(() => 41 | { 42 | NetField.Do(task); 43 | semaphore.Release(); 44 | }).ConfigureAwait(false); 45 | }); 46 | } 47 | 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /hsync/Network/NetTask.cs: -------------------------------------------------------------------------------- 1 | // This source code is a part of project violet-server. 2 | // Copyright (C)2020-2021. violet-team. Licensed under the MIT Licence. 3 | 4 | using hsync.Setting; 5 | using Newtonsoft.Json; 6 | using System; 7 | using System.Collections.Generic; 8 | using System.Net; 9 | using System.Text; 10 | using System.Threading; 11 | 12 | namespace hsync.Network 13 | { 14 | /// 15 | /// Information of what download for 16 | /// 17 | [JsonObject(MemberSerialization.OptIn)] 18 | public class NetTask 19 | { 20 | public static NetTask MakeDefault(string url, string cookie = "") 21 | => new NetTask 22 | { 23 | Accept = "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8", 24 | UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.110 Safari/537.3", 25 | TimeoutInfinite = Settings.Instance.Network.TimeoutInfinite, 26 | TimeoutMillisecond = Settings.Instance.Network.TimeoutMillisecond, 27 | AutoRedirection = true, 28 | RetryWhenFail = true, 29 | RetryCount = Settings.Instance.Network.RetryCount, 30 | DownloadBufferSize = Settings.Instance.Network.DownloadBufferSize, 31 | Proxy = !string.IsNullOrEmpty(Settings.Instance.Network.Proxy) ? new WebProxy(Settings.Instance.Network.Proxy) : null, 32 | Cookie = cookie, 33 | Url = url 34 | }; 35 | 36 | public static NetTask MakeDefaultMobile(string url, string cookie = "") 37 | => new NetTask 38 | { 39 | Accept = "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8", 40 | UserAgent = "Mozilla/5.0 (Android 7.0; Mobile; rv:54.0) Gecko/54.0 Firefox/54.0 AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.125 Mobile Safari/603.2.4", 41 | TimeoutInfinite = Settings.Instance.Network.TimeoutInfinite, 42 | TimeoutMillisecond = Settings.Instance.Network.TimeoutMillisecond, 43 | AutoRedirection = true, 44 | RetryWhenFail = true, 45 | RetryCount = Settings.Instance.Network.RetryCount, 46 | DownloadBufferSize = Settings.Instance.Network.DownloadBufferSize, 47 | Proxy = !string.IsNullOrEmpty(Settings.Instance.Network.Proxy) ? new WebProxy(Settings.Instance.Network.Proxy) : null, 48 | Cookie = cookie, 49 | Url = url 50 | }; 51 | 52 | public enum NetError 53 | { 54 | Unhandled = 0, 55 | CannotContinueByCriticalError, 56 | UnknowError, // Check DPI Blocker 57 | UriFormatError, 58 | Aborted, 59 | ManyRetry, 60 | } 61 | 62 | /* Task Information */ 63 | 64 | [JsonProperty] 65 | public int Index { get; set; } 66 | 67 | /* Http Information */ 68 | 69 | [JsonProperty] 70 | public string Url { get; set; } 71 | [JsonProperty] 72 | public List FailUrls { get; set; } 73 | [JsonProperty] 74 | public string Accept { get; set; } 75 | [JsonProperty] 76 | public string Referer { get; set; } 77 | [JsonProperty] 78 | public string UserAgent { get; set; } 79 | [JsonProperty] 80 | public string Cookie { get; set; } 81 | [JsonProperty] 82 | public Dictionary Headers { get; set; } 83 | [JsonProperty] 84 | public Dictionary Query { get; set; } 85 | [JsonProperty] 86 | public IWebProxy Proxy { get; set; } 87 | 88 | /* Detail Information */ 89 | 90 | /// 91 | /// Text Encoding Information 92 | /// 93 | public Encoding Encoding { get; set; } 94 | 95 | /// 96 | /// Set if you want to download and save file to your own device. 97 | /// 98 | [JsonProperty] 99 | public bool SaveFile { get; set; } 100 | [JsonProperty] 101 | public string Filename { get; set; } 102 | 103 | /// 104 | /// Set if needing only string datas. 105 | /// 106 | [JsonProperty] 107 | public bool DownloadString { get; set; } 108 | 109 | /// 110 | /// Download data to temporary directory on your device. 111 | /// 112 | [JsonProperty] 113 | public bool DriveCache { get; set; } 114 | 115 | /// 116 | /// Download data to memory. 117 | /// 118 | [JsonProperty] 119 | public bool MemoryCache { get; set; } 120 | 121 | /// 122 | /// Retry download when fail to download. 123 | /// 124 | [JsonProperty] 125 | public bool RetryWhenFail { get; set; } 126 | [JsonProperty] 127 | public int RetryCount { get; set; } 128 | 129 | /// 130 | /// Timeout settings 131 | /// 132 | [JsonProperty] 133 | public bool TimeoutInfinite { get; set; } 134 | [JsonProperty] 135 | public int TimeoutMillisecond { get; set; } 136 | 137 | [JsonProperty] 138 | public int DownloadBufferSize { get; set; } 139 | 140 | [JsonProperty] 141 | public bool AutoRedirection { get; set; } 142 | 143 | [JsonProperty] 144 | public bool NotifyOnlySize { get; set; } 145 | 146 | /* Callback Functions */ 147 | 148 | public Action SizeCallback; 149 | public Action DownloadCallback; 150 | public Action StartCallback; 151 | public Action CompleteCallback; 152 | public Action CompleteCallbackString; 153 | public Action CompleteCallbackBytes; 154 | public Action CookieReceive; 155 | public Action HeaderReceive; 156 | public Action CancleCallback; 157 | 158 | /// 159 | /// Return total downloaded size 160 | /// 161 | public Action RetryCallback; 162 | public Action ErrorCallback; 163 | 164 | /* For NetField */ 165 | 166 | public bool Aborted; 167 | public HttpWebRequest Request; 168 | public CancellationToken Cancel; 169 | } 170 | } 171 | -------------------------------------------------------------------------------- /hsync/Network/NetTaskPass.cs: -------------------------------------------------------------------------------- 1 | // This source code is a part of project violet-server. 2 | // Copyright (C)2020-2021. violet-team. Licensed under the MIT Licence. 3 | 4 | using System; 5 | using System.Collections.Generic; 6 | using System.Text; 7 | 8 | namespace hsync.Network 9 | { 10 | public abstract class NetTaskPass 11 | { 12 | public static List Passes = new List(); 13 | 14 | public static void RunOnField(ref NetTask content) 15 | { 16 | foreach (var pass in Passes) 17 | pass.RunOnPass(ref content); 18 | } 19 | 20 | public static void RemoveFromPasses() where T : NetTaskPass, new() 21 | { 22 | lock (Passes) 23 | { 24 | var class_name = (new T()).GetType().Name; 25 | for (int i = 0; i < Passes.Count; i++) 26 | { 27 | if (Passes[i].GetType().Name == class_name) 28 | { 29 | Passes.RemoveAt(i); 30 | break; 31 | } 32 | } 33 | } 34 | } 35 | 36 | public abstract void RunOnPass(ref NetTask content); 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /hsync/Network/NetTools.cs: -------------------------------------------------------------------------------- 1 | // This source code is a part of project violet-server. 2 | // Copyright (C)2020-2021. violet-team. Licensed under the MIT Licence. 3 | 4 | using System; 5 | using System.Collections.Generic; 6 | using System.Linq; 7 | using System.Text; 8 | using System.Threading; 9 | using System.Threading.Tasks; 10 | 11 | namespace hsync.Network 12 | { 13 | public class NetTools 14 | { 15 | public static async Task> DownloadStrings(List urls, string cookie = "", Action complete = null, Action error = null) 16 | { 17 | var interrupt = new ManualResetEvent(false); 18 | var result = new string[urls.Count]; 19 | var count = urls.Count; 20 | int iter = 0; 21 | 22 | foreach (var url in urls) 23 | { 24 | var itertmp = iter; 25 | var task = NetTask.MakeDefault(url); 26 | task.DownloadString = true; 27 | var headers = new Dictionary(); 28 | // headers["accept"] = "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8"; 29 | // headers["user-agent"] = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.110 Safari/537.3"; 30 | // headers["referer"] = "https://hitomi.la/reader/1234.html"; 31 | task.Headers = headers; 32 | task.CompleteCallbackString = (str) => 33 | { 34 | result[itertmp] = str; 35 | if (Interlocked.Decrement(ref count) == 0) 36 | interrupt.Set(); 37 | complete?.Invoke(); 38 | }; 39 | task.ErrorCallback = (code) => 40 | { 41 | if (Interlocked.Decrement(ref count) == 0) 42 | interrupt.Set(); 43 | error?.Invoke(); 44 | }; 45 | task.Cookie = cookie; 46 | await AppProvider.DownloadQueue.Add(task).ConfigureAwait(false); 47 | iter++; 48 | } 49 | 50 | interrupt.WaitOne(); 51 | 52 | return result.ToList(); 53 | } 54 | 55 | public static async Task> DownloadStrings(List tasks, string cookie = "", Action complete = null) 56 | { 57 | var interrupt = new ManualResetEvent(false); 58 | var result = new string[tasks.Count]; 59 | var count = tasks.Count; 60 | int iter = 0; 61 | 62 | foreach (var task in tasks) 63 | { 64 | var itertmp = iter; 65 | task.DownloadString = true; 66 | task.CompleteCallbackString = (str) => 67 | { 68 | result[itertmp] = str; 69 | if (Interlocked.Decrement(ref count) == 0) 70 | interrupt.Set(); 71 | complete?.Invoke(); 72 | }; 73 | task.ErrorCallback = (code) => 74 | { 75 | if (Interlocked.Decrement(ref count) == 0) 76 | interrupt.Set(); 77 | }; 78 | task.Cookie = cookie; 79 | await AppProvider.DownloadQueue.Add(task).ConfigureAwait(false); 80 | iter++; 81 | } 82 | 83 | interrupt.WaitOne(); 84 | 85 | return result.ToList(); 86 | } 87 | 88 | public static string DownloadString(string url) 89 | { 90 | return DownloadStringAsync(NetTask.MakeDefault(url)).Result; 91 | } 92 | 93 | public static string DownloadString(NetTask task) 94 | { 95 | return DownloadStringAsync(task).Result; 96 | } 97 | 98 | public static async Task DownloadStringAsync(NetTask task) 99 | { 100 | return await Task.Run(async () => 101 | { 102 | var interrupt = new ManualResetEvent(false); 103 | string result = null; 104 | 105 | task.DownloadString = true; 106 | task.CompleteCallbackString = (string str) => 107 | { 108 | result = str; 109 | interrupt.Set(); 110 | }; 111 | 112 | task.ErrorCallback = (code) => 113 | { 114 | task.ErrorCallback = null; 115 | interrupt.Set(); 116 | }; 117 | 118 | await AppProvider.DownloadQueue.Add(task).ConfigureAwait(false); 119 | 120 | interrupt.WaitOne(); 121 | 122 | return result; 123 | }).ConfigureAwait(false); 124 | } 125 | 126 | public static async Task> DownloadFiles(List<(string, string)> url_path, string cookie = "", Action download = null, Action complete = null) 127 | { 128 | var interrupt = new ManualResetEvent(false); 129 | var result = new string[url_path.Count]; 130 | var count = url_path.Count; 131 | int iter = 0; 132 | 133 | foreach (var up in url_path) 134 | { 135 | var itertmp = iter; 136 | var task = NetTask.MakeDefault(up.Item1); 137 | task.SaveFile = true; 138 | task.Filename = up.Item2; 139 | task.DownloadCallback = (sz) => 140 | { 141 | download?.Invoke(sz); 142 | }; 143 | task.CompleteCallback = () => 144 | { 145 | if (Interlocked.Decrement(ref count) == 0) 146 | interrupt.Set(); 147 | complete?.Invoke(); 148 | }; 149 | task.ErrorCallback = (code) => 150 | { 151 | if (Interlocked.Decrement(ref count) == 0) 152 | interrupt.Set(); 153 | }; 154 | task.Cookie = cookie; 155 | await AppProvider.DownloadQueue.Add(task).ConfigureAwait(false); 156 | iter++; 157 | } 158 | 159 | interrupt.WaitOne(); 160 | 161 | return result.ToList(); 162 | } 163 | 164 | public static void DownloadFile(string url, string filename) 165 | { 166 | var task = NetTask.MakeDefault(url); 167 | task.SaveFile = true; 168 | task.Filename = filename; 169 | DownloadFileAsync(task).Wait(); 170 | } 171 | 172 | public static void DownloadFile(NetTask task) 173 | { 174 | DownloadFileAsync(task).Wait(); 175 | } 176 | 177 | public static async Task DownloadFileAsync(NetTask task) 178 | { 179 | await Task.Run(async () => 180 | { 181 | var interrupt = new ManualResetEvent(false); 182 | 183 | task.SaveFile = true; 184 | task.CompleteCallback = () => 185 | { 186 | interrupt.Set(); 187 | }; 188 | 189 | task.ErrorCallback = (code) => 190 | { 191 | task.ErrorCallback = null; 192 | interrupt.Set(); 193 | }; 194 | 195 | await AppProvider.DownloadQueue.Add(task).ConfigureAwait(false); 196 | 197 | interrupt.WaitOne(); 198 | }).ConfigureAwait(false); 199 | } 200 | 201 | public static byte[] DownloadData(string url) 202 | { 203 | return DownloadDataAsync(NetTask.MakeDefault(url)).Result; 204 | } 205 | 206 | public static byte[] DownloadData(NetTask task) 207 | { 208 | return DownloadDataAsync(task).Result; 209 | } 210 | 211 | public static async Task DownloadDataAsync(NetTask task) 212 | { 213 | return await Task.Run(async () => 214 | { 215 | var interrupt = new ManualResetEvent(false); 216 | byte[] result = null; 217 | 218 | task.MemoryCache = true; 219 | task.CompleteCallbackBytes = (byte[] bytes) => 220 | { 221 | result = bytes; 222 | interrupt.Set(); 223 | }; 224 | 225 | task.ErrorCallback = (code) => 226 | { 227 | task.ErrorCallback = null; 228 | interrupt.Set(); 229 | }; 230 | 231 | await AppProvider.DownloadQueue.Add(task).ConfigureAwait(false); 232 | 233 | interrupt.WaitOne(); 234 | 235 | return result; 236 | }).ConfigureAwait(false); 237 | } 238 | } 239 | } 240 | -------------------------------------------------------------------------------- /hsync/Program.cs: -------------------------------------------------------------------------------- 1 | // This source code is a part of project violet-server. 2 | // Copyright (C)2020-2021. violet-team. Licensed under the MIT Licence. 3 | 4 | using hsync.Log; 5 | using System; 6 | using System.Text; 7 | using System.Globalization; 8 | 9 | namespace hsync 10 | { 11 | class Program 12 | { 13 | static void Main(string[] args) 14 | { 15 | Encoding.RegisterProvider(CodePagesEncodingProvider.Instance); 16 | 17 | AppProvider.Initialize(); 18 | 19 | Logs.Instance.AddLogNotify((s, e) => 20 | { 21 | var tuple = s as Tuple; 22 | CultureInfo en = new CultureInfo("en-US"); 23 | Console.ForegroundColor = ConsoleColor.Green; 24 | Console.Write("info: "); 25 | Console.ResetColor(); 26 | Console.WriteLine($"[{tuple.Item1.ToString(en)}] {tuple.Item2}"); 27 | }); 28 | 29 | Logs.Instance.AddLogErrorNotify((s, e) => { 30 | var tuple = s as Tuple; 31 | CultureInfo en = new CultureInfo("en-US"); 32 | Console.ForegroundColor = ConsoleColor.Red; 33 | Console.Error.Write("error: "); 34 | Console.ResetColor(); 35 | Console.Error.WriteLine($"[{tuple.Item1.ToString(en)}] {tuple.Item2}"); 36 | }); 37 | 38 | Logs.Instance.AddLogWarningNotify((s, e) => { 39 | var tuple = s as Tuple; 40 | CultureInfo en = new CultureInfo("en-US"); 41 | Console.ForegroundColor = ConsoleColor.Yellow; 42 | Console.Error.Write("warning: "); 43 | Console.ResetColor(); 44 | Console.Error.WriteLine($"[{tuple.Item1.ToString(en)}] {tuple.Item2}"); 45 | }); 46 | 47 | AppDomain.CurrentDomain.UnhandledException += (s, e) => 48 | { 49 | Logs.Instance.PushError("unhandled: " + (e.ExceptionObject as Exception).ToString()); 50 | }; 51 | 52 | try 53 | { 54 | Command.Start(args); 55 | //Command.Start(new string[] {"-ls", "--sync-only"}); 56 | } 57 | catch (Exception e) 58 | { 59 | Console.WriteLine("An error occured! " + e.Message); 60 | Console.WriteLine(e.StackTrace); 61 | Console.WriteLine("Please, check log.txt file."); 62 | } 63 | 64 | AppProvider.Deinitialize(); 65 | 66 | Environment.Exit(0); 67 | } 68 | } 69 | } 70 | -------------------------------------------------------------------------------- /hsync/Progress.cs: -------------------------------------------------------------------------------- 1 | // This source code is a part of project violet-server. 2 | // Copyright (C)2020-2021. violet-team. Licensed under the MIT Licence. 3 | 4 | using System; 5 | using System.Collections.Generic; 6 | using System.Text; 7 | using System.Threading; 8 | 9 | namespace hsync 10 | { 11 | public abstract class ProgressBase : IDisposable 12 | { 13 | protected readonly TimeSpan animationInterval = TimeSpan.FromSeconds(1.0 / 8); 14 | protected readonly Timer timer; 15 | 16 | protected string currentText = string.Empty; 17 | protected bool disposed = false; 18 | 19 | public ProgressBase() 20 | { 21 | timer = new Timer(TimerHandler); 22 | 23 | if (!System.Console.IsOutputRedirected) 24 | { 25 | ResetTimer(); 26 | } 27 | } 28 | 29 | protected abstract void TimerHandler(object state); 30 | 31 | protected void UpdateText(string text) 32 | { 33 | int commonPrefixLength = 0; 34 | int commonLength = Math.Min(currentText.Length, text.Length); 35 | while (commonPrefixLength < commonLength && text[commonPrefixLength] == currentText[commonPrefixLength]) 36 | { 37 | commonPrefixLength++; 38 | } 39 | 40 | StringBuilder outputBuilder = new StringBuilder(); 41 | outputBuilder.Append('\b', currentText.Length - commonPrefixLength); 42 | 43 | outputBuilder.Append(text.Substring(commonPrefixLength)); 44 | 45 | int overlapCount = currentText.Length - text.Length; 46 | if (overlapCount > 0) 47 | { 48 | outputBuilder.Append(' ', overlapCount); 49 | outputBuilder.Append('\b', overlapCount); 50 | } 51 | 52 | System.Console.Write(outputBuilder); 53 | currentText = text; 54 | } 55 | 56 | protected void ResetTimer() 57 | { 58 | timer.Change(animationInterval, TimeSpan.FromMilliseconds(-1)); 59 | } 60 | 61 | public void Dispose() 62 | { 63 | lock (timer) 64 | { 65 | disposed = true; 66 | UpdateText(string.Empty); 67 | } 68 | } 69 | } 70 | 71 | 72 | /// 73 | /// An ASCII progress bar 74 | /// 75 | /// Reference[MIT]: DanielSWolf - https://gist.github.com/DanielSWolf/0ab6a96899cc5377bf54 76 | /// 77 | 78 | public class ExtractingProgressBar : ProgressBase, IDisposable 79 | { 80 | private const int blockCount = 20; 81 | private double currentProgress = 0; 82 | private long total = 0; 83 | private long complete = 0; 84 | 85 | public ExtractingProgressBar() 86 | : base() 87 | { 88 | } 89 | 90 | public void Report(long total, long complete) 91 | { 92 | var value = Math.Max(0, Math.Min(1, complete / (double)total)); 93 | Interlocked.Exchange(ref currentProgress, value); 94 | this.total = total; 95 | this.complete = complete; 96 | } 97 | 98 | protected override void TimerHandler(object state) 99 | { 100 | lock (timer) 101 | { 102 | if (disposed) return; 103 | 104 | int progressBlockCount = (int)(currentProgress * blockCount); 105 | int percent = (int)(currentProgress * 100); 106 | 107 | string text = string.Format("[{0}{1}] {2,3}% [{3}/{4}]", 108 | new string('#', progressBlockCount), new string('-', blockCount - progressBlockCount), 109 | percent, complete, total); 110 | UpdateText(text); 111 | 112 | ResetTimer(); 113 | } 114 | } 115 | } 116 | 117 | public class WaitProgress : ProgressBase, IDisposable 118 | { 119 | private const string animation = @"|/-\"; 120 | private int animationIndex = 0; 121 | 122 | public WaitProgress() 123 | : base() 124 | { 125 | } 126 | 127 | protected override void TimerHandler(object state) 128 | { 129 | lock (timer) 130 | { 131 | if (disposed) return; 132 | 133 | UpdateText(animation[animationIndex++ % animation.Length].ToString()); 134 | 135 | ResetTimer(); 136 | } 137 | } 138 | } 139 | 140 | public class ProgressBar : ProgressBase 141 | { 142 | private const int blockCount = 20; 143 | private double currentProgress = 0; 144 | private long total = 0; 145 | private long complete = 0; 146 | private long error = 0; 147 | 148 | public ProgressBar() 149 | : base() 150 | { 151 | } 152 | 153 | public void Report(long total, long complete, long error) 154 | { 155 | var value = Math.Max(0, Math.Min(1, (complete + error) / (double)total)); 156 | this.total = total; 157 | this.complete = complete; 158 | this.error = error; 159 | Interlocked.Exchange(ref currentProgress, value); 160 | } 161 | 162 | protected override void TimerHandler(object state) 163 | { 164 | lock (timer) 165 | { 166 | if (disposed) return; 167 | 168 | int progressBlockCount = (int)(currentProgress * blockCount); 169 | int percent = (int)(currentProgress * 100); 170 | 171 | string text = string.Format("[{0}{1}] {2,3}% [{3}/{4}] (Find: {5}, Error: {6})", 172 | new string('#', progressBlockCount), new string('-', blockCount - progressBlockCount), 173 | percent, 174 | complete + error, total, complete, error); 175 | UpdateText(text); 176 | 177 | ResetTimer(); 178 | } 179 | } 180 | } 181 | 182 | public class DownloadProgressBar : ProgressBase, IDisposable 183 | { 184 | private const int blockCount = 20; 185 | private double currentProgress = 0; 186 | private long total_read_bytes = 0; 187 | private long current_speed = 0; 188 | private long tick_speed = 0; 189 | private object report_lock = new object(); 190 | private long total = 0; 191 | private long complete = 0; 192 | private Queue speed_save = new Queue(); 193 | 194 | public DownloadProgressBar() 195 | : base() 196 | { 197 | } 198 | 199 | public void Report(long total, long complete, long read_bytes) 200 | { 201 | var value = Math.Max(0, Math.Min(1, complete / (double)total)); 202 | this.total = total; 203 | this.complete = complete; 204 | Interlocked.Exchange(ref currentProgress, value); 205 | lock (report_lock) 206 | { 207 | total_read_bytes += read_bytes; 208 | current_speed += read_bytes; 209 | tick_speed += read_bytes; 210 | } 211 | } 212 | 213 | protected override void TimerHandler(object state) 214 | { 215 | lock (timer) 216 | { 217 | if (disposed) return; 218 | double cs = 0; 219 | lock (report_lock) 220 | { 221 | speed_save.Enqueue(tick_speed); 222 | tick_speed = 0; 223 | cs = current_speed * (8 / (double)speed_save.Count); 224 | if (speed_save.Count >= 8) 225 | { 226 | current_speed -= speed_save.Peek(); 227 | speed_save.Dequeue(); 228 | } 229 | } 230 | 231 | int progressBlockCount = (int)(currentProgress * blockCount); 232 | int percent = (int)(currentProgress * 100); 233 | 234 | string speed; 235 | if (cs > 1024 * 1024) 236 | speed = (cs / (1024 * 1024)).ToString("#,0.0") + " MB/S"; 237 | else if (cs > 1024) 238 | speed = (cs / 1024).ToString("#,0.0") + " KB/S"; 239 | else 240 | speed = cs.ToString("#,0") + " Byte/S"; 241 | 242 | string downloads; 243 | if (total_read_bytes > 1024 * 1024 * 1024) 244 | downloads = (total_read_bytes / (double)(1024 * 1024 * 1024)).ToString("#,0.0") + " GB"; 245 | else if (total_read_bytes > 1024 * 1024) 246 | downloads = (total_read_bytes / (double)(1024 * 1024)).ToString("#,0.0") + " MB"; 247 | else if (total_read_bytes > 1024) 248 | downloads = (total_read_bytes / (double)(1024)).ToString("#,0.0") + " KB"; 249 | else 250 | downloads = (total_read_bytes).ToString("#,0") + " Byte"; 251 | 252 | string text = string.Format("[{0}{1}] {2,3}% [{5}/{6}] ({3} {4})", 253 | new string('#', progressBlockCount), new string('-', blockCount - progressBlockCount), 254 | percent, 255 | speed, downloads, complete, total); 256 | UpdateText(text); 257 | 258 | ResetTimer(); 259 | } 260 | } 261 | } 262 | 263 | public class SingleFileProgressBar : ProgressBase, IDisposable 264 | { 265 | private const int blockCount = 20; 266 | private double currentProgress = 0; 267 | private long total_read_bytes = 0; 268 | private long current_speed = 0; 269 | private long tick_speed = 0; 270 | private object report_lock = new object(); 271 | private Queue speed_save = new Queue(); 272 | 273 | public SingleFileProgressBar() 274 | : base() 275 | { 276 | } 277 | 278 | public void Report(long size, long read_bytes) 279 | { 280 | var value = Math.Max(0, Math.Min(1, total_read_bytes / (double)size)); 281 | Interlocked.Exchange(ref currentProgress, value); 282 | lock (report_lock) 283 | { 284 | total_read_bytes += read_bytes; 285 | current_speed += read_bytes; 286 | tick_speed += read_bytes; 287 | } 288 | } 289 | 290 | protected override void TimerHandler(object state) 291 | { 292 | lock (timer) 293 | { 294 | if (disposed) return; 295 | double cs = 0; 296 | lock (report_lock) 297 | { 298 | speed_save.Enqueue(tick_speed); 299 | tick_speed = 0; 300 | cs = current_speed * (8 / (double)speed_save.Count); 301 | if (speed_save.Count >= 8) 302 | { 303 | current_speed -= speed_save.Peek(); 304 | speed_save.Dequeue(); 305 | } 306 | } 307 | 308 | int progressBlockCount = (int)(currentProgress * blockCount); 309 | int percent = (int)(currentProgress * 100); 310 | 311 | string speed; 312 | if (cs > 1024 * 1024) 313 | speed = (cs / (1024 * 1024)).ToString("#,0.0") + " MB/S"; 314 | else if (cs > 1024) 315 | speed = (cs / 1024).ToString("#,0.0") + " KB/S"; 316 | else 317 | speed = cs.ToString("#,0") + " Byte/S"; 318 | 319 | string downloads; 320 | if (total_read_bytes > 1024 * 1024 * 1024) 321 | downloads = (total_read_bytes / (double)(1024 * 1024 * 1024)).ToString("#,0.0") + " GB"; 322 | else if (total_read_bytes > 1024 * 1024) 323 | downloads = (total_read_bytes / (double)(1024 * 1024)).ToString("#,0.0") + " MB"; 324 | else if (total_read_bytes > 1024) 325 | downloads = (total_read_bytes / (double)(1024)).ToString("#,0.0") + " KB"; 326 | else 327 | downloads = (total_read_bytes).ToString("#,0") + " Byte"; 328 | 329 | string text = string.Format("[{0}{1}] {2,3}% ({3} {4})", 330 | new string('#', progressBlockCount), new string('-', blockCount - progressBlockCount), 331 | percent, 332 | speed, downloads); 333 | UpdateText(text); 334 | 335 | ResetTimer(); 336 | } 337 | } 338 | } 339 | 340 | public class WaitPostprocessor : ProgressBase, IDisposable 341 | { 342 | private long wait; 343 | private const string animation = @"|/-\"; 344 | private int animationIndex = 0; 345 | private object report_lock = new object(); 346 | 347 | public WaitPostprocessor() 348 | : base() 349 | { 350 | } 351 | 352 | public void Report(long wait) 353 | { 354 | lock (report_lock) 355 | { 356 | this.wait = wait; 357 | } 358 | } 359 | 360 | protected override void TimerHandler(object state) 361 | { 362 | lock (timer) 363 | { 364 | if (disposed) return; 365 | 366 | UpdateText(animation[animationIndex++ % animation.Length].ToString() + $" [{wait} jobs remained]"); 367 | 368 | ResetTimer(); 369 | } 370 | } 371 | } 372 | 373 | } 374 | -------------------------------------------------------------------------------- /hsync/RelatedTagTest.cs: -------------------------------------------------------------------------------- 1 | // This source code is a part of project violet-server. 2 | // Copyright (C)2020-2021. violet-team. Licensed under the MIT Licence. 3 | 4 | using hsync.Component; 5 | using Newtonsoft.Json; 6 | using Newtonsoft.Json.Linq; 7 | using SQLite; 8 | using System; 9 | using System.Collections.Generic; 10 | using System.IO; 11 | using System.Linq; 12 | using System.Text; 13 | using System.Threading; 14 | using System.Threading.Tasks; 15 | 16 | namespace hsync 17 | { 18 | public class RelatedTagTest 19 | { 20 | public string dbdir; 21 | public List target; 22 | public double threshold; 23 | 24 | public RelatedTagTest(string dbpath, double threshold) 25 | { 26 | var db = new SQLiteConnection(dbpath); 27 | var info = db.GetTableInfo(typeof(HitomiColumnModel).Name); 28 | 29 | if (!info.Any()) 30 | { 31 | Console.WriteLine($"{typeof(HitomiColumnModel).Name} table is not found."); 32 | return; 33 | } 34 | 35 | Console.Write("Load database... "); 36 | target = db.Query("SELECT * FROM HitomiColumnModel"); 37 | Console.WriteLine("Ready"); 38 | 39 | this.threshold = threshold; 40 | this.dbdir = Path.GetDirectoryName(dbpath); 41 | } 42 | 43 | public Dictionary>> result = new Dictionary>>(); 44 | 45 | public Dictionary> tags_dic = new Dictionary>(); 46 | public List>> tags_list; 47 | public List> results = new List>(); 48 | 49 | private void Initialize() 50 | { 51 | result.Clear(); 52 | results.Clear(); 53 | tags_dic.Clear(); 54 | if (tags_list != null) tags_list.Clear(); 55 | 56 | foreach (var data in target) 57 | { 58 | if (data.Tags != null) 59 | { 60 | foreach (var tag in data.Tags.Split('|')) 61 | { 62 | if (tag == "") continue; 63 | if (tags_dic.ContainsKey(tag)) 64 | tags_dic[tag].Add(data.Id); 65 | else 66 | tags_dic.Add(tag, new List { data.Id }); 67 | } 68 | } 69 | } 70 | 71 | tags_list = tags_dic.ToList(); 72 | 73 | tags_list.ForEach(x => x.Value.Sort()); 74 | tags_list.Sort((a, b) => a.Value.Count.CompareTo(b.Value.Count)); 75 | } 76 | 77 | private static int manually_intersect(List a, List b) 78 | { 79 | int intersect = 0; 80 | int i = 0, j = 0; 81 | for (; i < a.Count && j < b.Count;) 82 | { 83 | if (a[i] == b[j]) 84 | { 85 | intersect++; 86 | i++; 87 | j++; 88 | } 89 | else if (a[i] < b[j]) 90 | { 91 | i++; 92 | } 93 | else 94 | { 95 | j++; 96 | } 97 | } 98 | return intersect; 99 | } 100 | 101 | private List> Intersect(int i) 102 | { 103 | List> result = new List>(); 104 | 105 | for (int j = i + 1; j < tags_list.Count; j++) 106 | { 107 | int intersect = manually_intersect(tags_list[i].Value, tags_list[j].Value); 108 | int i_size = tags_list[i].Value.Count; 109 | int j_size = tags_list[j].Value.Count; 110 | double rate = (double)(intersect) / (i_size + j_size - intersect); 111 | if (rate >= threshold) 112 | result.Add(new Tuple(tags_list[i].Key, tags_list[j].Key, 113 | rate)); 114 | } 115 | 116 | return result; 117 | } 118 | 119 | private void Merge() 120 | { 121 | foreach (var tuple in results) 122 | { 123 | if (result.ContainsKey(tuple.Item1)) 124 | result[tuple.Item1].Add(new Tuple(tuple.Item2, tuple.Item3)); 125 | else 126 | result.Add(tuple.Item1, new List> { new Tuple(tuple.Item2, tuple.Item3) }); 127 | if (result.ContainsKey(tuple.Item2)) 128 | result[tuple.Item2].Add(new Tuple(tuple.Item1, tuple.Item3)); 129 | else 130 | result.Add(tuple.Item2, new List> { new Tuple(tuple.Item1, tuple.Item3) }); 131 | } 132 | result.ToList().ForEach(x => x.Value.Sort((a, b) => b.Item2.CompareTo(a.Item2))); 133 | results.Clear(); 134 | } 135 | 136 | int max; 137 | int progress; 138 | int mtl; 139 | public void Start() 140 | { 141 | Initialize(); 142 | 143 | max = tags_list.Count; 144 | progress = 0; 145 | mtl = Environment.ProcessorCount; 146 | 147 | Console.Write("Process... "); 148 | using (var pb = new ExtractingProgressBar()) 149 | { 150 | Task.WhenAll(Enumerable.Range(0, mtl).Select(no => Task.Run(() => process(no, pb)))).Wait(); 151 | } 152 | Console.WriteLine("Complete"); 153 | 154 | Console.Write("Merge... "); 155 | Merge(); 156 | Console.WriteLine("Complete"); 157 | 158 | Console.Write("Save... "); 159 | var rr = result.ToList(); 160 | rr.Sort((x, y) => y.Value.Count.CompareTo(x.Value.Count)); 161 | JArray arr = new JArray(); 162 | rr.ForEach(x => 163 | { 164 | JArray tags = new JArray(); 165 | x.Value.ForEach(y => 166 | { 167 | tags.Add(new JObject{ { y.Item1, y.Item2 } }); 168 | }); 169 | arr.Add(new JObject { { x.Key, tags } }); 170 | }); 171 | File.WriteAllText(Path.Combine(dbdir, "rtt-result.json"), JsonConvert.SerializeObject(arr)); 172 | Console.WriteLine("Complete"); 173 | } 174 | 175 | private void process(int i, ExtractingProgressBar pb) 176 | { 177 | int min = this.max / mtl * i; 178 | int max = this.max / mtl * (i + 1); 179 | if (max > this.max) 180 | max = this.max; 181 | 182 | List> result = new List>(); 183 | 184 | for (int j = max - 1; j >= min; j--) 185 | { 186 | result.AddRange(Intersect(j)); 187 | 188 | pb.Report(this.max, Interlocked.Increment(ref progress)); 189 | } 190 | 191 | lock(results) 192 | results.AddRange(result); 193 | } 194 | } 195 | } 196 | -------------------------------------------------------------------------------- /hsync/SeriesTest.cs: -------------------------------------------------------------------------------- 1 | // This source code is a part of project violet-server. 2 | // Copyright (C)2020-2021. violet-team. Licensed under the MIT Licence. 3 | 4 | using hsync.Component; 5 | using Newtonsoft.Json; 6 | using Newtonsoft.Json.Linq; 7 | using SQLite; 8 | using System; 9 | using System.Collections.Generic; 10 | using System.IO; 11 | using System.Linq; 12 | using System.Text; 13 | using System.Threading; 14 | using System.Threading.Tasks; 15 | 16 | namespace hsync 17 | { 18 | public class SeriesTest 19 | { 20 | public string dbdir; 21 | public List target; 22 | public double threshold; 23 | 24 | public SeriesTest(string dbpath) 25 | { 26 | var db = new SQLiteConnection(dbpath); 27 | var info = db.GetTableInfo(typeof(HitomiColumnModel).Name); 28 | 29 | if (!info.Any()) 30 | { 31 | Console.WriteLine($"{typeof(HitomiColumnModel).Name} table is not found."); 32 | return; 33 | } 34 | 35 | Console.Write("Load database... "); 36 | target = db.Query("SELECT * FROM HitomiColumnModel"); 37 | Console.WriteLine("Ready"); 38 | 39 | this.dbdir = Path.GetDirectoryName(dbpath); 40 | } 41 | 42 | public void Start() 43 | { 44 | var tags_dic = new Dictionary>(); 45 | 46 | foreach (var data in target) 47 | { 48 | if (data.Series != null && data.Characters != null) 49 | { 50 | foreach (var series in data.Series.Split('|')) 51 | { 52 | if (series == "") continue; 53 | foreach (var tag in data.Characters.Split('|')) 54 | { 55 | if (tag == "") continue; 56 | if (tags_dic.ContainsKey(series)) 57 | { 58 | if (tags_dic[series].ContainsKey(tag)) 59 | tags_dic[series][tag] += 1; 60 | else 61 | tags_dic[series].Add(tag, 1); 62 | } 63 | else 64 | tags_dic.Add(series, new Dictionary { { tag, 1 } }); 65 | } 66 | } 67 | } 68 | } 69 | 70 | Console.Write("Save... "); 71 | var rr = tags_dic.ToList(); 72 | rr.Sort((x, y) => y.Value.Count.CompareTo(x.Value.Count)); 73 | JArray arr = new JArray(); 74 | rr.ForEach(x => 75 | { 76 | JArray tags = new JArray(); 77 | var rx = x.Value.ToList(); 78 | rx.Sort((x, y) => y.Value.CompareTo(x.Value)); 79 | rx.ForEach(y => 80 | { 81 | tags.Add(new JObject { { y.Key, y.Value } }); 82 | }); 83 | arr.Add(new JObject { { x.Key, tags } }); 84 | }); 85 | File.WriteAllText(Path.Combine(dbdir, "st-result.json"), JsonConvert.SerializeObject(arr, Formatting.Indented)); 86 | Console.WriteLine("Complete"); 87 | } 88 | } 89 | } 90 | -------------------------------------------------------------------------------- /hsync/Setting/Settings.cs: -------------------------------------------------------------------------------- 1 | // This source code is a part of project violet-server. 2 | // Copyright (C)2020-2021. violet-team. Licensed under the MIT Licence. 3 | 4 | using hsync.Log; 5 | using hsync.Utils; 6 | using Newtonsoft.Json; 7 | using System; 8 | using System.IO; 9 | using System.Threading; 10 | 11 | namespace hsync.Setting 12 | { 13 | public class SettingModel 14 | { 15 | public class NetworkSetting 16 | { 17 | public bool TimeoutInfinite; 18 | public int TimeoutMillisecond; 19 | public int DownloadBufferSize; 20 | public int RetryCount; 21 | public string Proxy; 22 | } 23 | 24 | public class HitomiSetting 25 | { 26 | public string SaveDirectoryFormat; 27 | public string RemainInfoFile; 28 | } 29 | 30 | public HitomiSetting HitomiSettings; 31 | 32 | public NetworkSetting NetworkSettings; 33 | 34 | /// 35 | /// Scheduler Thread Count 36 | /// 37 | public int ThreadCount; 38 | 39 | /// 40 | /// Postprocessor Scheduler Thread Count 41 | /// 42 | public int PostprocessorThreadCount; 43 | 44 | /// 45 | /// Provider Language 46 | /// 47 | public string Language; 48 | 49 | /// 50 | /// Parent Path for Downloading 51 | /// 52 | public string SuperPath; 53 | 54 | /// 55 | /// Server Connection String 56 | /// 57 | public string ServerConnection; 58 | 59 | /// 60 | /// Elastic Search Server HostName 61 | /// 62 | public string ElasticSearchHost; 63 | } 64 | 65 | public class Settings : ILazy 66 | { 67 | public const string Name = "settings.json"; 68 | 69 | public SettingModel Model { get; set; } 70 | public SettingModel.NetworkSetting Network { get { return Model.NetworkSettings; } } 71 | 72 | public Settings() 73 | { 74 | var full_path = Path.Combine(AppProvider.ApplicationPath, Name); 75 | if (File.Exists(full_path)) 76 | Model = JsonConvert.DeserializeObject(File.ReadAllText(full_path)); 77 | 78 | if (Model == null) 79 | { 80 | Model = new SettingModel 81 | { 82 | Language = GetLanguageKey(), 83 | ThreadCount = Environment.ProcessorCount * 6, 84 | PostprocessorThreadCount = 3, 85 | SuperPath = AppProvider.DefaultSuperPath, 86 | ServerConnection = "Server=localhost;Database=test;Uid=root;Pwd=123;CharSet=utf8mb4", 87 | ElasticSearchHost = "", 88 | 89 | HitomiSettings = new SettingModel.HitomiSetting 90 | { 91 | SaveDirectoryFormat = "%(artist)/[%(id)] %(title)" 92 | }, 93 | 94 | NetworkSettings = new SettingModel.NetworkSetting 95 | { 96 | TimeoutInfinite = false, 97 | TimeoutMillisecond = 10000, 98 | DownloadBufferSize = 131072, 99 | RetryCount = 10, 100 | }, 101 | }; 102 | } 103 | Save(); 104 | } 105 | 106 | public static string GetLanguageKey() 107 | { 108 | var lang = Thread.CurrentThread.CurrentCulture.ToString(); 109 | var language = "all"; 110 | switch (lang) 111 | { 112 | case "ko-KR": 113 | language = "korean"; 114 | break; 115 | 116 | case "ja-JP": 117 | language = "japanese"; 118 | break; 119 | 120 | case "en-US": 121 | language = "english"; 122 | break; 123 | } 124 | return language; 125 | } 126 | 127 | /// 128 | /// Recover incorrect configuration. 129 | /// 130 | public void Recover() 131 | { 132 | if (string.IsNullOrWhiteSpace(Model.Language)) 133 | Model.Language = GetLanguageKey(); 134 | 135 | if (Model.ThreadCount <= 0 || Model.ThreadCount >= 128) 136 | Model.ThreadCount = Environment.ProcessorCount; 137 | if (Model.PostprocessorThreadCount <= 0 || Model.PostprocessorThreadCount >= 128) 138 | Model.ThreadCount = 3; 139 | if (string.IsNullOrWhiteSpace(Model.SuperPath)) 140 | Model.SuperPath = AppProvider.DefaultSuperPath; 141 | 142 | if (Model.NetworkSettings == null) 143 | { 144 | Model.NetworkSettings = new SettingModel.NetworkSetting 145 | { 146 | TimeoutInfinite = false, 147 | TimeoutMillisecond = 10000, 148 | DownloadBufferSize = 131072, 149 | RetryCount = 10, 150 | }; 151 | } 152 | 153 | if (Model.NetworkSettings.TimeoutMillisecond < 1000) 154 | Model.NetworkSettings.TimeoutMillisecond = 10000; 155 | if (Model.NetworkSettings.DownloadBufferSize < 100000) 156 | Model.NetworkSettings.DownloadBufferSize = 131072; 157 | if (Model.NetworkSettings.RetryCount < 0) 158 | Model.NetworkSettings.RetryCount = 10; 159 | } 160 | 161 | public void Save() 162 | { 163 | var full_path = Path.Combine(AppProvider.ApplicationPath, Name); 164 | var json = JsonConvert.SerializeObject(Model, Formatting.Indented); 165 | using (var fs = new StreamWriter(new FileStream(full_path, FileMode.Create, FileAccess.Write))) 166 | { 167 | fs.Write(json); 168 | } 169 | AppProvider.DefaultSuperPath = Model.SuperPath; 170 | } 171 | } 172 | } 173 | -------------------------------------------------------------------------------- /hsync/Syncronizer.cs: -------------------------------------------------------------------------------- 1 | // This source code is a part of project violet-server. 2 | // Copyright (C)2020-2021. violet-team. Licensed under the MIT Licence. 3 | 4 | using hsync.Component; 5 | using hsync.Log; 6 | using hsync.Network; 7 | using hsync.Utils; 8 | using Newtonsoft.Json; 9 | using Newtonsoft.Json.Converters; 10 | using System; 11 | using System.Collections.Generic; 12 | using System.IO; 13 | using System.Linq; 14 | using System.Net; 15 | using System.Text; 16 | using System.Threading; 17 | using System.Threading.Tasks; 18 | 19 | namespace hsync 20 | { 21 | public class Syncronizer 22 | { 23 | int latestId; 24 | int hitomiSyncRange; 25 | 26 | int starts=0, ends=0; 27 | bool useManualRange = false; 28 | 29 | int exhentaiLookupPage; 30 | 31 | public List newedDataHitomi; 32 | public List newedDataEH; 33 | 34 | bool hitomi_sync_ignore_exists = false; 35 | 36 | public Syncronizer(string[] hitomi_sync_range, string[] hitomi_sync_lookup_range, bool hitomi_sync_ignore_exists, string[] exhentai_lookup_page) 37 | { 38 | HitomiData.Instance.Load(); 39 | latestId = HitomiData.Instance.metadata_collection.First().ID; 40 | 41 | int lookup_range; 42 | if (hitomi_sync_lookup_range != null && int.TryParse(hitomi_sync_lookup_range[0], out lookup_range)) 43 | hitomiSyncRange = lookup_range; 44 | else 45 | hitomiSyncRange = 4000; 46 | 47 | if (hitomi_sync_range != null 48 | && int.TryParse(hitomi_sync_range[0], out starts) 49 | && int.TryParse(hitomi_sync_range[1], out ends)) 50 | useManualRange = true; 51 | 52 | if (exhentai_lookup_page != null && int.TryParse(exhentai_lookup_page[0], out exhentaiLookupPage)) 53 | ; 54 | else 55 | exhentaiLookupPage = 200; 56 | 57 | newedDataHitomi = new List(); 58 | newedDataEH = new List(); 59 | 60 | this.hitomi_sync_ignore_exists = hitomi_sync_ignore_exists; 61 | } 62 | 63 | public void SyncHitomi() 64 | { 65 | var exists = new HashSet(); 66 | foreach (var metadata in HitomiData.Instance.metadata_collection) 67 | exists.Add(metadata.ID); 68 | 69 | var gburls = Enumerable.Range(useManualRange ? starts : latestId - hitomiSyncRange, useManualRange ? ends - starts + 1 : hitomiSyncRange * 2) 70 | //var gburls = Enumerable.Range(1000, latestId + hitomiSyncRange / 2) 71 | .Where(x => !exists.Contains(x) || hitomi_sync_ignore_exists).Select(x => $"https://ltn.hitomi.la/galleryblock/{x}.html").ToList(); 72 | var dcnt = 0; 73 | var ecnt = 0; 74 | Console.Write("Running galleryblock tester... "); 75 | List htmls; 76 | using (var pb = new ProgressBar()) 77 | { 78 | htmls = NetTools.DownloadStrings(gburls, "", 79 | () => 80 | { 81 | pb.Report(gburls.Count, Interlocked.Increment(ref dcnt), ecnt); 82 | }, 83 | () => 84 | { 85 | pb.Report(gburls.Count, dcnt, Interlocked.Increment(ref ecnt)); 86 | }).Result; 87 | } 88 | Console.WriteLine("Complete"); 89 | 90 | var gurls = new List(gburls.Count); 91 | for (int i = 0; i < gburls.Count; i++) 92 | { 93 | if (htmls[i] == null) 94 | continue; 95 | var aa = HitomiParser.ParseGalleryBlock(htmls[i]); 96 | if (aa.Magic.Contains("-")) 97 | gurls.Add("https://hitomi.la/" + aa.Magic); 98 | else 99 | gurls.Add("https://hitomi.la/galleries/" + i + ".html"); 100 | } 101 | 102 | dcnt = 0; 103 | ecnt = 0; 104 | Console.Write("Running gallery tester... "); 105 | List htmls2 = null; 106 | if (gurls.Count != 0) 107 | using (var pb = new ProgressBar()) 108 | { 109 | htmls2 = NetTools.DownloadStrings(gurls, "", 110 | () => 111 | { 112 | pb.Report(gburls.Count, Interlocked.Increment(ref dcnt), ecnt); 113 | }, 114 | () => 115 | { 116 | pb.Report(gburls.Count, dcnt, Interlocked.Increment(ref ecnt)); 117 | }).Result; 118 | } 119 | Console.WriteLine("Complete"); 120 | 121 | //Console.Write("Check redirect gallery html... "); 122 | //for (int i = 0; i < htmls2.Count; i++) 123 | //{ 124 | // if (htmls2[i] == null) 125 | // continue; 126 | // var node = htmls2[i].ToHtmlNode(); 127 | // var title = node.SelectSingleNode("//title"); 128 | // if (title != null && title.InnerText == "Redirect") 129 | // { 130 | // htmls2.RemoveAt(i--); 131 | // } 132 | //} 133 | //Console.WriteLine("Complete"); 134 | 135 | var result = new List(); 136 | for (int i = 0, j = 0; i < gburls.Count; i++) 137 | { 138 | if (htmls[i] == null) 139 | continue; 140 | var aa = HitomiParser.ParseGalleryBlock(htmls[i]); 141 | if (htmls2[j] != null) 142 | { 143 | var node = htmls2[j].ToHtmlNode(); 144 | var title = node.SelectSingleNode("//title"); 145 | if (!(title != null && title.InnerText == "Redirect")) 146 | { 147 | var ab = HitomiParser.ParseGallery(htmls2[j]); 148 | aa.Groups = ab.Groups; 149 | aa.Characters = ab.Characters; 150 | } 151 | } 152 | try 153 | { 154 | if (aa.Magic.Contains("-")) 155 | newedDataHitomi.Add(Convert.ToInt32(aa.Magic.Split('-').Last().Split('.')[0])); 156 | else if (aa.Magic.Contains("galleries")) 157 | newedDataHitomi.Add(Convert.ToInt32(aa.Magic.Split('/').Last().Split('.')[0])); 158 | else 159 | newedDataHitomi.Add(Convert.ToInt32(aa.Magic)); 160 | } 161 | catch 162 | { 163 | ; 164 | } 165 | result.Add(aa); 166 | j++; 167 | } 168 | 169 | Console.Write("Save to hiddendata.json... "); 170 | HitomiData.Instance.SaveWithNewData(result); 171 | Console.WriteLine("Complete"); 172 | } 173 | 174 | public void SyncExHentai() 175 | { 176 | var result = new List(); 177 | 178 | for (int i = 0; i < 9999999; i++) 179 | { 180 | try 181 | { 182 | //var task = NetTask.MakeDefault($"https://exhentai.org/?page={i}&f_doujinshi=on&f_manga=on&f_artistcg=on&f_gamecg=on&&f_cats=0&f_sname=on&f_stags=on&f_sh=on&advsearch=1&f_srdd=2&f_sname=on&f_stags=on&f_sdesc=on&f_sh=on"); 183 | //task.Cookie = "igneous=30e0c0a66;ipb_member_id=2742770;ipb_pass_hash=6042be35e994fed920ee7dd11180b65f;sl=dm_2"; 184 | //var html = NetTools.DownloadString(task); 185 | var url = $"https://exhentai.org/?page={i}&f_doujinshi=on&f_manga=on&f_artistcg=on&f_gamecg=on&&f_cats=0&f_sname=on&f_stags=on&f_sh=on&advsearch=1&f_srdd=2&f_sname=on&f_stags=on&f_sdesc=on&f_sh=on"; 186 | var wc = new WebClient(); 187 | wc.Encoding = Encoding.UTF8; 188 | wc.Headers.Add(HttpRequestHeader.Accept, "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8"); 189 | wc.Headers.Add(HttpRequestHeader.UserAgent, "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.139 Safari/537.36"); 190 | wc.Headers.Add(HttpRequestHeader.Cookie, "igneous=30e0c0a66;ipb_member_id=2742770;ipb_pass_hash=6042be35e994fed920ee7dd11180b65f;sl=dm_2"); 191 | var html = wc.DownloadString(url); 192 | 193 | try 194 | { 195 | var exh = ExHentaiParser.ParseResultPageExtendedListView(html); 196 | result.AddRange(exh); 197 | if (exh.Count != 25) 198 | Logs.Instance.PushWarning("[Miss] " + url); 199 | if (i > exhentaiLookupPage && exh.Min(x => x.URL.Split('/')[4].ToInt()) < latestId) 200 | break; 201 | Logs.Instance.Push("Parse exh page - " + i); 202 | } 203 | catch (Exception e) 204 | { 205 | Logs.Instance.PushError("[Fail] " + url); 206 | } 207 | } 208 | catch (Exception e) 209 | { 210 | Logs.Instance.PushError($"{i} {e.Message}"); 211 | } 212 | Thread.Sleep(100); 213 | 214 | if (i % 1000 == 999) 215 | Thread.Sleep(60000); 216 | } 217 | 218 | var xxx = JsonConvert.DeserializeObject>(File.ReadAllText("ex-hentai-archive.json")); 219 | File.Move("ex-hentai-archive.json", $"ex-hentai-archive-{DateTime.Now.Ticks}.json"); 220 | 221 | var exists = new HashSet(); 222 | xxx.ForEach(x => exists.Add(x.URL.Split('/')[4].ToInt())); 223 | 224 | foreach (var z in result) 225 | { 226 | var nn = z.URL.Split('/')[4].ToInt(); 227 | 228 | if (!exists.Contains(nn)) 229 | { 230 | newedDataEH.Add(nn); 231 | xxx.Add(z); 232 | } 233 | } 234 | 235 | JsonSerializer serializer = new JsonSerializer(); 236 | serializer.Converters.Add(new JavaScriptDateTimeConverter()); 237 | serializer.NullValueHandling = NullValueHandling.Ignore; 238 | 239 | Logs.Instance.Push("Write file: ex-hentai-archive.json"); 240 | using (StreamWriter sw = new StreamWriter("ex-hentai-archive.json")) 241 | using (JsonWriter writer = new JsonTextWriter(sw)) 242 | { 243 | serializer.Serialize(writer, xxx); 244 | } 245 | } 246 | } 247 | } 248 | -------------------------------------------------------------------------------- /hsync/Utils/Compress.cs: -------------------------------------------------------------------------------- 1 | // This source code is a part of project violet-server. 2 | // Copyright (C)2020-2021. violet-team. Licensed under the MIT Licence. 3 | 4 | using System; 5 | using System.Collections.Generic; 6 | using System.IO; 7 | using System.IO.Compression; 8 | using System.Text; 9 | 10 | namespace hsync.Utils 11 | { 12 | public class CompressUtils 13 | { 14 | public static byte[] Compress(byte[] data) 15 | { 16 | MemoryStream output = new MemoryStream(); 17 | using (DeflateStream dstream = new DeflateStream(output, CompressionLevel.Optimal)) 18 | { 19 | dstream.Write(data, 0, data.Length); 20 | } 21 | return output.ToArray(); 22 | } 23 | 24 | public static byte[] Decompress(byte[] data) 25 | { 26 | MemoryStream input = new MemoryStream(data); 27 | MemoryStream output = new MemoryStream(); 28 | using (DeflateStream dstream = new DeflateStream(input, CompressionMode.Decompress)) 29 | { 30 | dstream.CopyTo(output); 31 | } 32 | return output.ToArray(); 33 | } 34 | 35 | public static byte[] Zip(byte[] data) 36 | { 37 | throw new NotImplementedException(); 38 | } 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /hsync/Utils/Extends.cs: -------------------------------------------------------------------------------- 1 | // This source code is a part of project violet-server. 2 | // Copyright (C)2020-2021. violet-team. Licensed under the MIT Licence. 3 | 4 | using HtmlAgilityPack; 5 | using Newtonsoft.Json; 6 | using System; 7 | using System.Collections.Concurrent; 8 | using System.Collections.Generic; 9 | using System.IO; 10 | using System.Linq; 11 | using System.Text; 12 | using System.Threading.Tasks; 13 | 14 | namespace hsync.Utils 15 | { 16 | public static class Extends 17 | { 18 | public static int ToInt(this string str) => Convert.ToInt32(str); 19 | 20 | public static string MyText(this HtmlNode node) => 21 | string.Join("", node.ChildNodes.Where(x => x.Name == "#text").Select(x => x.InnerText.Trim())); 22 | 23 | public static HtmlNode ToHtmlNode(this string html) 24 | { 25 | var document = new HtmlDocument(); 26 | document.LoadHtml(html); 27 | return document.DocumentNode; 28 | } 29 | 30 | public static Task ForEachAsync(this IEnumerable source, int countdvd, Func body) 31 | { 32 | return Task.WhenAll( 33 | from partition in Partitioner.Create(source).GetPartitions(countdvd) 34 | select Task.Run(async delegate 35 | { 36 | using (partition) 37 | while (partition.MoveNext()) 38 | await body(partition.Current); 39 | })); 40 | } 41 | 42 | public static async Task ReadJson(string path) 43 | { 44 | using (var fs = new StreamReader(new FileStream(path, FileMode.Open, FileAccess.Read))) 45 | { 46 | return JsonConvert.DeserializeObject(await fs.ReadToEndAsync()); 47 | } 48 | } 49 | 50 | public static async void WriteJson(string path, T value) 51 | { 52 | var json = JsonConvert.SerializeObject(value, Formatting.Indented); 53 | using (var fs = new StreamWriter(new FileStream(path, FileMode.Create, FileAccess.Write))) 54 | { 55 | await fs.WriteAsync(json); 56 | } 57 | } 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /hsync/Utils/Heap.cs: -------------------------------------------------------------------------------- 1 | // This source code is a part of project violet-server. 2 | // Copyright (C)2020-2021. violet-team. Licensed under the MIT Licence. 3 | 4 | using System; 5 | using System.Collections.Generic; 6 | using System.Text; 7 | 8 | namespace hsync.Utils 9 | { 10 | /// 11 | /// Priority queue data structure for C# 12 | /// 13 | /// Type of data 14 | /// Comparator of data 15 | public class Heap 16 | where T : IComparable 17 | where C : IComparer, new() 18 | { 19 | List heap; 20 | C comp; 21 | 22 | public Heap(int capacity = 256) 23 | { 24 | heap = new List(capacity); 25 | comp = new C(); 26 | } 27 | 28 | public void Push(T d) 29 | { 30 | heap.Add(d); 31 | leaf_to_root(); 32 | } 33 | 34 | public void Pop() 35 | { 36 | heap[0] = heap[heap.Count - 1]; 37 | heap.RemoveAt(heap.Count - 1); 38 | root_to_leaf(); 39 | } 40 | 41 | public T Front => heap[0]; 42 | 43 | private void root_to_leaf() 44 | { 45 | int x = 0; 46 | int l = heap.Count - 1; 47 | while (x < l) 48 | { 49 | int c1 = x * 2 + 1; 50 | int c2 = c1 + 1; 51 | 52 | // 53 | // x 54 | // / \ 55 | // / \ 56 | // c1 c2 57 | // 58 | 59 | int c = c1; 60 | if (c2 < l && comp.Compare(heap[c2], heap[c1]) > 0) 61 | c = c2; 62 | 63 | if (c < l && comp.Compare(heap[c], heap[x]) > 0) 64 | { 65 | swap(c, x); 66 | x = c; 67 | } 68 | else 69 | { 70 | break; 71 | } 72 | } 73 | } 74 | 75 | private void leaf_to_root() 76 | { 77 | int x = heap.Count - 1; 78 | while (x > 0) 79 | { 80 | int p = (x - 1) >> 1; 81 | if (comp.Compare(heap[x], heap[p]) > 0) 82 | { 83 | swap(p, x); 84 | x = p; 85 | } 86 | else 87 | break; 88 | } 89 | } 90 | 91 | private void swap(int i, int j) 92 | { 93 | T t = heap[i]; 94 | heap[i] = heap[j]; 95 | heap[j] = t; 96 | } 97 | } 98 | 99 | public class DefaultHeapComparer : Comparer where T : IComparable 100 | { 101 | public override int Compare(T x, T y) 102 | => x.CompareTo(y); 103 | } 104 | 105 | public class MinHeapComparer : Comparer where T : IComparable 106 | { 107 | public override int Compare(T x, T y) 108 | => y.CompareTo(x); 109 | } 110 | 111 | public class Heap : Heap> where T : IComparable { } 112 | public class MinHeap : Heap> where T : IComparable { } 113 | public class MaxHeap : Heap> where T : IComparable { } 114 | 115 | public class UpdatableHeapElements : IComparable 116 | where T : IComparable 117 | { 118 | public T data; 119 | public int index; 120 | public static UpdatableHeapElements Create(T data, int index) 121 | => new UpdatableHeapElements { data = data, index = index }; 122 | public int CompareTo(T obj) 123 | => data.CompareTo(obj); 124 | } 125 | 126 | public class UpdatableHeap 127 | where S : IComparable 128 | where T : UpdatableHeapElements, IComparable 129 | where C : IComparer, new() 130 | { 131 | List heap; 132 | C comp; 133 | 134 | public UpdatableHeap(int capacity = 256) 135 | { 136 | heap = new List(capacity); 137 | comp = new C(); 138 | } 139 | 140 | public T Push(S d) 141 | { 142 | var dd = (T)UpdatableHeapElements.Create(d, heap.Count - 1); 143 | heap.Add(dd); 144 | top_down(heap.Count - 1); 145 | return dd; 146 | } 147 | 148 | public void Pop() 149 | { 150 | heap[0] = heap[heap.Count - 1]; 151 | heap[0].index = 0; 152 | heap.RemoveAt(heap.Count - 1); 153 | bottom_up(); 154 | } 155 | 156 | public void Update(T d) 157 | { 158 | int p = (d.index - 1) >> 1; 159 | if (p == d.index) 160 | bottom_up(); 161 | else 162 | { 163 | if (comp.Compare(heap[p].data, heap[d.index].data) > 0) 164 | top_down(d.index); 165 | else 166 | bottom_up(d.index); 167 | } 168 | } 169 | 170 | public S Front => heap[0].data; 171 | 172 | public int Count { get { return heap.Count; } } 173 | 174 | private void bottom_up(int x = 0) 175 | { 176 | int l = heap.Count - 1; 177 | while (x < l) 178 | { 179 | int c1 = x * 2 + 1; 180 | int c2 = c1 + 1; 181 | 182 | // 183 | // x 184 | // / \ 185 | // / \ 186 | // c1 c2 187 | // 188 | 189 | int c = c1; 190 | if (c2 < l && comp.Compare(heap[c2].data, heap[c1].data) > 0) 191 | c = c2; 192 | 193 | if (c < l && comp.Compare(heap[c].data, heap[x].data) > 0) 194 | { 195 | swap(c, x); 196 | x = c; 197 | } 198 | else 199 | { 200 | break; 201 | } 202 | } 203 | } 204 | 205 | private void top_down(int x) 206 | { 207 | while (x > 0) 208 | { 209 | int p = (x - 1) >> 1; 210 | if (comp.Compare(heap[x].data, heap[p].data) > 0) 211 | { 212 | swap(p, x); 213 | x = p; 214 | } 215 | else 216 | break; 217 | } 218 | } 219 | 220 | private void swap(int i, int j) 221 | { 222 | T t = heap[i]; 223 | heap[i] = heap[j]; 224 | heap[j] = t; 225 | 226 | int tt = heap[i].index; 227 | heap[i].index = heap[j].index; 228 | heap[j].index = tt; 229 | } 230 | } 231 | 232 | public class UpdatableHeap : UpdatableHeap, DefaultHeapComparer> where T : IComparable { } 233 | public class UpdatableMinHeap : UpdatableHeap, MinHeapComparer> where T : IComparable { } 234 | public class UpdatableMaxHeap : UpdatableHeap, DefaultHeapComparer> where T : IComparable { } 235 | } 236 | -------------------------------------------------------------------------------- /hsync/Utils/ILazy.cs: -------------------------------------------------------------------------------- 1 | // This source code is a part of project violet-server. 2 | // Copyright (C)2020-2021. violet-team. Licensed under the MIT Licence. 3 | 4 | using System; 5 | using System.Collections.Generic; 6 | using System.Text; 7 | 8 | namespace hsync.Utils 9 | { 10 | /// 11 | /// Contains all instance information. 12 | /// 13 | public class InstanceMonitor 14 | { 15 | public static Dictionary Instances = new Dictionary(); 16 | } 17 | 18 | /// 19 | /// Class to make lazy instance easier to implement. 20 | /// 21 | /// 22 | public class ILazy 23 | where T : new() 24 | { 25 | private static readonly Lazy instance = new Lazy(() => 26 | { 27 | T instance = new T(); 28 | InstanceMonitor.Instances.Add(instance.GetType().Name.ToLower(), instance); 29 | return instance; 30 | }); 31 | public static T Instance => instance.Value; 32 | public static bool IsValueCreated => instance.IsValueCreated; 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /hsync/Utils/Strings.cs: -------------------------------------------------------------------------------- 1 | // This source code is a part of project violet-server. 2 | // Copyright (C)2020-2021. violet-team. Licensed under the MIT Licence. 3 | 4 | using System; 5 | using System.Collections.Generic; 6 | using System.IO; 7 | using System.IO.Compression; 8 | using System.Text; 9 | 10 | namespace hsync.Utils 11 | { 12 | public static class Strings 13 | { 14 | // https://stackoverflow.com/questions/11743160/how-do-i-encode-and-decode-a-base64-string 15 | public static string ToBase64(this string text) 16 | { 17 | return ToBase64(text, Encoding.UTF8); 18 | } 19 | 20 | public static string ToBase64(this string text, Encoding encoding) 21 | { 22 | if (string.IsNullOrEmpty(text)) 23 | { 24 | return text; 25 | } 26 | 27 | byte[] textAsBytes = encoding.GetBytes(text); 28 | return Convert.ToBase64String(textAsBytes); 29 | } 30 | 31 | public static bool TryParseBase64(this string text, out string decodedText) 32 | { 33 | return TryParseBase64(text, Encoding.UTF8, out decodedText); 34 | } 35 | 36 | public static bool TryParseBase64(this string text, Encoding encoding, out string decodedText) 37 | { 38 | if (string.IsNullOrEmpty(text)) 39 | { 40 | decodedText = text; 41 | return false; 42 | } 43 | 44 | try 45 | { 46 | byte[] textAsBytes = Convert.FromBase64String(text); 47 | decodedText = encoding.GetString(textAsBytes); 48 | return true; 49 | } 50 | catch (Exception) 51 | { 52 | decodedText = null; 53 | return false; 54 | } 55 | } 56 | 57 | #region Compression 58 | 59 | // https://stackoverflow.com/questions/7343465/compression-decompression-string-with-c-sharp 60 | private static void CopyTo(Stream src, Stream dest) 61 | { 62 | byte[] bytes = new byte[4096]; 63 | 64 | int cnt; 65 | 66 | while ((cnt = src.Read(bytes, 0, bytes.Length)) != 0) 67 | { 68 | dest.Write(bytes, 0, cnt); 69 | } 70 | } 71 | 72 | public static byte[] Zip(this string str) 73 | { 74 | var bytes = Encoding.UTF8.GetBytes(str); 75 | 76 | using (var msi = new MemoryStream(bytes)) 77 | using (var mso = new MemoryStream()) 78 | { 79 | using (var gs = new GZipStream(mso, CompressionMode.Compress)) 80 | { 81 | CopyTo(msi, gs); 82 | } 83 | 84 | return mso.ToArray(); 85 | } 86 | } 87 | 88 | public static byte[] ZipByte(this byte[] bytes) 89 | { 90 | using (var msi = new MemoryStream(bytes)) 91 | using (var mso = new MemoryStream()) 92 | { 93 | using (var gs = new GZipStream(mso, CompressionMode.Compress)) 94 | { 95 | CopyTo(msi, gs); 96 | } 97 | 98 | return mso.ToArray(); 99 | } 100 | } 101 | 102 | 103 | public static string Unzip(this byte[] bytes) 104 | { 105 | using (var msi = new MemoryStream(bytes)) 106 | using (var mso = new MemoryStream()) 107 | { 108 | using (var gs = new GZipStream(msi, CompressionMode.Decompress)) 109 | { 110 | CopyTo(gs, mso); 111 | } 112 | 113 | return Encoding.UTF8.GetString(mso.ToArray()); 114 | } 115 | } 116 | 117 | public static byte[] UnzipByte(this byte[] bytes) 118 | { 119 | using (var msi = new MemoryStream(bytes)) 120 | using (var mso = new MemoryStream()) 121 | { 122 | using (var gs = new GZipStream(msi, CompressionMode.Decompress)) 123 | { 124 | CopyTo(gs, mso); 125 | } 126 | 127 | return mso.ToArray(); 128 | } 129 | } 130 | 131 | #endregion 132 | 133 | public static int ComputeLevenshteinDistance(this string a, string b) 134 | { 135 | int x = a.Length; 136 | int y = b.Length; 137 | int i, j; 138 | 139 | if (x == 0) return x; 140 | if (y == 0) return y; 141 | int[] v0 = new int[(y + 1) << 1]; 142 | 143 | for (i = 0; i < y + 1; i++) v0[i] = i; 144 | for (i = 0; i < x; i++) 145 | { 146 | v0[y + 1] = i + 1; 147 | for (j = 0; j < y; j++) 148 | v0[y + j + 2] = Math.Min(Math.Min(v0[y + j + 1], v0[j + 1]) + 1, v0[j] + ((a[i] == b[j]) ? 0 : 1)); 149 | for (j = 0; j < y + 1; j++) v0[j] = v0[y + j + 1]; 150 | } 151 | return v0[y + y + 1]; 152 | } 153 | 154 | public static int[] GetLevenshteinDistance(this string src, string tar) 155 | { 156 | int[,] dist = new int[src.Length + 1, tar.Length + 1]; 157 | for (int i = 1; i <= src.Length; i++) dist[i, 0] = i; 158 | for (int j = 1; j <= tar.Length; j++) dist[0, j] = j; 159 | 160 | for (int j = 1; j <= tar.Length; j++) 161 | { 162 | for (int i = 1; i <= src.Length; i++) 163 | { 164 | if (src[i - 1] == tar[j - 1]) dist[i, j] = dist[i - 1, j - 1]; 165 | else dist[i, j] = Math.Min(dist[i - 1, j - 1] + 1, Math.Min(dist[i, j - 1] + 1, dist[i - 1, j] + 1)); 166 | } 167 | } 168 | 169 | // Get diff through backtracking. 170 | int[] route = new int[src.Length + 1]; 171 | int fz = dist[src.Length, tar.Length]; 172 | for (int i = src.Length, j = tar.Length; i >= 0 && j >= 0;) 173 | { 174 | int lu = int.MaxValue; 175 | int u = int.MaxValue; 176 | int l = int.MaxValue; 177 | if (i - 1 >= 0 && j - 1 >= 0) lu = dist[i - 1, j - 1]; 178 | if (i - 1 >= 0) u = dist[i - 1, j]; 179 | if (j - 1 >= 0) l = dist[i, j - 1]; 180 | int min = Math.Min(lu, Math.Min(l, u)); 181 | if (min == fz) route[i] = 1; 182 | if (min == lu) 183 | { 184 | i--; j--; 185 | } 186 | else if (min == u) i--; 187 | else j--; 188 | fz = min; 189 | } 190 | return route; 191 | } 192 | 193 | 194 | // https://stackoverflow.com/questions/1601834/c-implementation-of-or-alternative-to-strcmplogicalw-in-shlwapi-dll 195 | public class NaturalComparer : IComparer 196 | { 197 | public int Compare(string x, string y) 198 | { 199 | if (x == null && y == null) return 0; 200 | if (x == null) return -1; 201 | if (y == null) return 1; 202 | 203 | int lx = x.Length, ly = y.Length; 204 | 205 | for (int mx = 0, my = 0; mx < lx && my < ly; mx++, my++) 206 | { 207 | if (char.IsDigit(x[mx]) && char.IsDigit(y[my])) 208 | { 209 | long vx = 0, vy = 0; 210 | 211 | for (; mx < lx && char.IsDigit(x[mx]); mx++) 212 | vx = vx * 10 + x[mx] - '0'; 213 | 214 | for (; my < ly && char.IsDigit(y[my]); my++) 215 | vy = vy * 10 + y[my] - '0'; 216 | 217 | if (vx != vy) 218 | return vx > vy ? 1 : -1; 219 | } 220 | 221 | if (mx < lx && my < ly && x[mx] != y[my]) 222 | return x[mx] > y[my] ? 1 : -1; 223 | } 224 | 225 | return lx - ly; 226 | } 227 | } 228 | } 229 | } 230 | -------------------------------------------------------------------------------- /hsync/Version.cs: -------------------------------------------------------------------------------- 1 | // This source code is a part of project violet-server. 2 | // Copyright (C)2020-2021. violet-team. Licensed under the MIT Licence. 3 | 4 | using System; 5 | using System.Collections.Generic; 6 | using System.Text; 7 | 8 | namespace hsync 9 | { 10 | public class Version 11 | { 12 | public const int MajorVersion = 2021; 13 | public const int MinorVersion = 04; 14 | public const int BuildVersion = 16; 15 | 16 | public const string Name = "hsync"; 17 | public static string Text { get; } = $"{MajorVersion}.{MinorVersion}.{BuildVersion}"; 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /hsync/hsync-src.7z: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/project-violet/hsync/18c3cffed3b2fe2c477648dce59be4cc331d4984/hsync/hsync-src.7z -------------------------------------------------------------------------------- /hsync/hsync.csproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | Exe 5 | netcoreapp3.1 6 | AnyCPU;x64 7 | build$([System.DateTime]::UtcNow.ToString("yyyyMMddHHmmss")) 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | -------------------------------------------------------------------------------- /hsync/publish-linux-x64-aot.bat: -------------------------------------------------------------------------------- 1 | dotnet publish -r linux-x64 -c Release /p:PublishSingleFile=true /p:PublishTrimmed=true /p:PublishReadyToRun=true -------------------------------------------------------------------------------- /hsync/publish-linux-x64.bat: -------------------------------------------------------------------------------- 1 | dotnet publish -r linux-x64 -c Release /p:PublishSingleFile=true /p:PublishTrimmed=true -------------------------------------------------------------------------------- /hsync/publish-osx.10.15-x64-aot.bat: -------------------------------------------------------------------------------- 1 | dotnet publish -r osx.10.15-x64 -c Release /p:PublishSingleFile=true /p:PublishTrimmed=true /p:PublishReadyToRun=true -------------------------------------------------------------------------------- /hsync/publish-osx.10.15-x64.bat: -------------------------------------------------------------------------------- 1 | dotnet publish -r osx.10.15-x64 -c Release /p:PublishSingleFile=true /p:PublishTrimmed=true -------------------------------------------------------------------------------- /hsync/publish-win10-arm-aot.bat: -------------------------------------------------------------------------------- 1 | dotnet publish -r win10-arm -c Release /p:PublishSingleFile=true /p:PublishTrimmed=true /p:PublishReadyToRun=true -------------------------------------------------------------------------------- /hsync/publish-win10-arm.bat: -------------------------------------------------------------------------------- 1 | dotnet publish -r win10-arm -c Release /p:PublishSingleFile=true /p:PublishTrimmed=true -------------------------------------------------------------------------------- /hsync/publish-win10-x64-aot.bat: -------------------------------------------------------------------------------- 1 | dotnet publish -r win10-x64 -c Release /p:PublishSingleFile=true /p:PublishReadyToRun=true -------------------------------------------------------------------------------- /hsync/publish-win10-x64.bat: -------------------------------------------------------------------------------- 1 | dotnet publish -r win10-x64 -c Release /p:PublishSingleFile=true -------------------------------------------------------------------------------- /hsync/publish-win10-x86-aot.bat: -------------------------------------------------------------------------------- 1 | dotnet publish -r win10-x86 -c Release /p:PublishSingleFile=true /p:PublishReadyToRun=true -------------------------------------------------------------------------------- /hsync/publish-win10-x86.bat: -------------------------------------------------------------------------------- 1 | dotnet publish -r win10-x86 -c Release /p:PublishSingleFile=true -------------------------------------------------------------------------------- /rtt.bat: -------------------------------------------------------------------------------- 1 | hsync.exe -r rawdata/data.db 0.0000001 2 | hsync.exe -r rawdata-chinese/data.db 0.0000001 3 | hsync.exe -r rawdata-english/data.db 0.0000001 4 | hsync.exe -r rawdata-japanese/data.db 0.0000001 5 | hsync.exe -r rawdata-korean/data.db 0.0000001 -------------------------------------------------------------------------------- /sync.py: -------------------------------------------------------------------------------- 1 | # This source code is a part of project violet-server. 2 | # Copyright (C) 2020. violet-team. Licensed under the MIT Licence. 3 | 4 | import sys 5 | import os 6 | import os.path 7 | import time 8 | import shutil 9 | from subprocess import Popen, PIPE 10 | from datetime import datetime 11 | 12 | token = 'faketoken' 13 | dbmetapath = '/home/violet.dev.master/violet-server/frontend/build/version.txt' 14 | 15 | def sync(): 16 | # 17 | # Sync (Low Performance Starting) 18 | # 19 | shutil.rmtree('chunk', ignore_errors=True) 20 | process = Popen(['./hsync', '-ls', '--sync-only']) 21 | process.wait() 22 | 23 | def upload_chunk(): 24 | # donot use utcnow() 25 | timestamp = str(int(datetime.now().timestamp())) 26 | filename1 = os.listdir('chunk')[0] 27 | filename2 = os.listdir('chunk')[1] 28 | chunkfile1 = 'chunk/' + filename1 29 | chunkfile2 = 'chunk/' + filename2 30 | size1 = os.path.getsize(chunkfile1) 31 | size2 = os.path.getsize(chunkfile2) 32 | 33 | process = Popen(['github-release', 34 | 'upload', 35 | '--owner=violet-dev', 36 | '--repo=chunk', 37 | '--tag=' + timestamp, 38 | '--release-name=chunk ' + timestamp + '', 39 | '--body=""', 40 | '--prerelease=false', 41 | '--token=' + token, 42 | chunkfile1, 43 | chunkfile2, 44 | ]) 45 | process.wait() 46 | 47 | url = 'https://github.com/violet-dev/chunk/releases/download/'+timestamp+'/'+filename1 48 | with open(dbmetapath, "a") as myfile: 49 | myfile.write('chunk ' + timestamp + ' ' + url + ' ' + str(size1) + '\n') 50 | 51 | url = 'https://github.com/violet-dev/chunk/releases/download/'+timestamp+'/'+filename2 52 | with open(dbmetapath, "a") as myfile: 53 | myfile.write('chunkraw ' + timestamp + ' ' + url + ' ' + str(size2) + '\n') 54 | 55 | def release(): 56 | # 57 | # Create database 58 | # 59 | process = Popen(['./hsync', '-lc']) 60 | process.wait() 61 | 62 | # 63 | # Compress 64 | # 65 | process = Popen(['7z', 'a', 'rawdata.7z', 'rawdata/*'], stdout=open(os.devnull, 'wb')) 66 | process.wait() 67 | process = Popen(['7z', 'a', 'rawdata-chinese.7z', 'rawdata-chinese/*'], stdout=open(os.devnull, 'wb')) 68 | process.wait() 69 | process = Popen(['7z', 'a', 'rawdata-english.7z', 'rawdata-english/*'], stdout=open(os.devnull, 'wb')) 70 | process.wait() 71 | process = Popen(['7z', 'a', 'rawdata-japanese.7z', 'rawdata-japanese/*'], stdout=open(os.devnull, 'wb')) 72 | process.wait() 73 | process = Popen(['7z', 'a', 'rawdata-korean.7z', 'rawdata-korean/*'], stdout=open(os.devnull, 'wb')) 74 | process.wait() 75 | 76 | # 77 | # Rename rawdata/data.db 78 | # 79 | process = Popen(['mv', 'rawdata/data.db', 'rawdata/rawdata.db']) 80 | process.wait() 81 | process = Popen(['mv', 'rawdata-chinese/data.db', 'rawdata-chinese/rawdata-chinese.db']) 82 | process.wait() 83 | process = Popen(['mv', 'rawdata-english/data.db', 'rawdata-english/rawdata-english.db']) 84 | process.wait() 85 | process = Popen(['mv', 'rawdata-japanese/data.db', 'rawdata-japanese/rawdata-japanese.db']) 86 | process.wait() 87 | process = Popen(['mv', 'rawdata-korean/data.db', 'rawdata-korean/rawdata-korean.db']) 88 | process.wait() 89 | 90 | # 91 | # Upload 92 | # 93 | date = datetime.utcnow().strftime('%Y.%m.%d') 94 | process = Popen(['github-release', 95 | 'upload', 96 | '--owner=violet-dev', 97 | '--repo=db', 98 | '--tag=' + date, 99 | '--release-name=db ' + date + '', 100 | '--body=""', 101 | '--prerelease=false', 102 | '--token=' + token, 103 | 'rawdata.7z', 104 | 'rawdata-chinese.7z', 105 | 'rawdata-english.7z', 106 | 'rawdata-japanese.7z', 107 | 'rawdata-korean.7z', 108 | 'rawdata/rawdata.db', 109 | 'rawdata-chinese/rawdata-chinese.db', 110 | 'rawdata-english/rawdata-english.db', 111 | 'rawdata-japanese/rawdata-japanese.db', 112 | 'rawdata-korean/rawdata-korean.db', 113 | ]) 114 | process.wait() 115 | 116 | timestamp = str(int(datetime.now().timestamp())) 117 | url = 'https://github.com/violet-dev/db/releases/download/'+date+'/rawdata' 118 | with open(dbmetapath, "a") as myfile: 119 | myfile.write('db ' + timestamp + ' ' + url + '\n') 120 | 121 | def remove_exists(path): 122 | if os.path.exists(path): 123 | os.remove(path) 124 | 125 | def clean(): 126 | shutil.rmtree('rawdata', ignore_errors=True) 127 | shutil.rmtree('rawdata-chinese', ignore_errors=True) 128 | shutil.rmtree('rawdata-english', ignore_errors=True) 129 | shutil.rmtree('rawdata-japanese', ignore_errors=True) 130 | shutil.rmtree('rawdata-korean', ignore_errors=True) 131 | 132 | remove_exists('rawdata.7z') 133 | remove_exists('rawdata-chinese.7z') 134 | remove_exists('rawdata-english.7z') 135 | remove_exists('rawdata-japanese.7z') 136 | remove_exists('rawdata-korean.7z') 137 | 138 | latest_sync_date = '' 139 | 140 | while True: 141 | sync() 142 | upload_chunk() 143 | cur_date = datetime.utcnow().strftime('%Y.%m.%d') 144 | if latest_sync_date != cur_date: 145 | latest_sync_date = cur_date 146 | clean() 147 | release() 148 | # 1 hour 149 | time.sleep(60 * 60) --------------------------------------------------------------------------------