├── .gitattributes ├── .gitignore ├── .gitmodules ├── LICENSE.md ├── README.md ├── build.gradle.kts ├── buildSrc ├── build.gradle.kts └── src │ └── main │ └── kotlin │ └── de │ └── skyrising │ └── mc │ └── scanner │ └── gen │ └── random-ticks.kt ├── gradle.properties ├── gradle └── wrapper │ ├── gradle-wrapper.jar │ └── gradle-wrapper.properties ├── gradlew ├── gradlew.bat ├── settings.gradle.kts └── src ├── main ├── kotlin │ └── de │ │ └── skyrising │ │ └── mc │ │ └── scanner │ │ ├── constants.kt │ │ ├── inventories.kt │ │ ├── io.kt │ │ ├── nbt.kt │ │ ├── needles.kt │ │ ├── region.kt │ │ ├── region │ │ ├── io.kt │ │ ├── tree.kt │ │ └── visitors.kt │ │ ├── scanner.kt │ │ ├── script │ │ └── script.kt │ │ ├── utils.kt │ │ └── zlib │ │ ├── api.kt │ │ ├── bitstream.kt │ │ ├── constants.kt │ │ ├── deflate.kt │ │ └── huffman.kt └── resources │ ├── META-INF │ └── kotlin │ │ └── script │ │ └── templates │ │ └── de.skyrising.mc.scanner.script.ScannerScript.classname │ ├── flattening │ ├── block_states.json │ └── items.json │ └── scripts │ ├── geode.scan.kts │ ├── search.scan.kts │ └── stats.scan.kts └── test └── kotlin └── de └── skyrising └── mc └── scanner └── zlib └── test-deflate.kt /.gitattributes: -------------------------------------------------------------------------------- 1 | # 2 | # https://help.github.com/articles/dealing-with-line-endings/ 3 | # 4 | # These are explicitly windows files and should use crlf 5 | *.bat text eol=crlf 6 | 7 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Ignore Gradle project-specific cache directory 2 | .gradle 3 | 4 | # Ignore Gradle build output directory 5 | build/ 6 | **/build/ 7 | tmp/ 8 | out/ 9 | **/out/ 10 | 11 | *.iml 12 | .idea 13 | 14 | benchmark*.txt 15 | results.txt 16 | *.log 17 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SciCraft/mc-scanner/e29653496645c35df7f67849eeb89ba63b4dd453/.gitmodules -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | 2 | The MIT License (MIT) 3 | 4 | Copyright (c) 2021 5 | 6 | Permission is hereby granted, free of charge, to any person obtaining a copy 7 | of this software and associated documentation files (the "Software"), to deal 8 | in the Software without restriction, including without limitation the rights 9 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | copies of the Software, and to permit persons to whom the Software is 11 | furnished to do so, subject to the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be included in all 14 | copies or substantial portions of the Software. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | SOFTWARE. 23 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # mc-scanner 2 | 3 | Scan Minecraft worlds for certain blocks/items or 4 | collect statistics about inventory contents 5 | 6 | ## Compiling 7 | ```shell 8 | > ./gradlew build 9 | ``` 10 | 11 | ## Example Usage 12 | 13 | ### Searching for Sponges 14 | ```shell 15 | > java -jar mc-scanner-.jar -i sponge -i wet_sponge -b sponge -b wet_sponge sponges.zip 16 | # ... 20s later ... 17 | # 2671/2671 5.9GiB/5.9GiB 298.6MiB/s 30 results 18 | ``` 19 | 20 | ### Collecting Statistics 21 | Into a directory: 22 | ```shell 23 | > java -jar mc-scanner- --stats result/ 24 | ``` 25 | 26 | Into a zip file: 27 | ```shell 28 | > java -jar mc-scanner- --stats result.zip 29 | ``` 30 | 31 | Of only one region file: 32 | ```shell 33 | > java -jar mc-scanner-.jar --stats /region/r.92.-83.mca quarry-items.zip 34 | # 1/1 10.4MiB/10.4MiB 11.1MiB/s 51224 results 35 | ``` -------------------------------------------------------------------------------- /build.gradle.kts: -------------------------------------------------------------------------------- 1 | import org.jetbrains.kotlin.gradle.tasks.KotlinCompile 2 | import com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar 3 | import de.skyrising.mc.scanner.gen.generateRandomTicksKt 4 | 5 | plugins { 6 | kotlin("jvm") version "1.8.0" 7 | kotlin("plugin.serialization") version "1.8.0" 8 | id("com.github.johnrengelman.shadow") version "7.1.2" 9 | application 10 | } 11 | 12 | 13 | repositories { 14 | mavenCentral() 15 | } 16 | 17 | dependencies { 18 | implementation(kotlin("reflect")) 19 | implementation("it.unimi.dsi:fastutil:8.5.8") 20 | implementation("net.sf.jopt-simple:jopt-simple:6.0-alpha-3") 21 | implementation("org.jetbrains.kotlinx:kotlinx-serialization-json:1.4.1") 22 | 23 | implementation("org.jetbrains.kotlin:kotlin-scripting-common") 24 | implementation("org.jetbrains.kotlin:kotlin-scripting-jvm") 25 | //implementation("org.jetbrains.kotlin:kotlin-scripting-dependencies") 26 | //implementation("org.jetbrains.kotlin:kotlin-scripting-dependencies-maven") 27 | implementation("org.jetbrains.kotlin:kotlin-scripting-jvm-host") 28 | 29 | testImplementation(kotlin("test")) 30 | } 31 | 32 | application { 33 | mainClassName = "de.skyrising.mc.scanner.ScannerKt" 34 | } 35 | 36 | tasks { 37 | named("shadowJar") { 38 | classifier = "" 39 | mergeServiceFiles() 40 | minimize { 41 | exclude(dependency("org.jetbrains.kotlin:.*")) 42 | } 43 | dependsOn(distTar, distZip) 44 | } 45 | } 46 | 47 | tasks.test { 48 | useJUnitPlatform() 49 | } 50 | 51 | tasks.withType { 52 | kotlinOptions.jvmTarget = "1.8" 53 | } 54 | 55 | val generatedKotlinDir = project.buildDir.resolve("generated/kotlin") 56 | 57 | tasks.create("generateSources") { 58 | doFirst { 59 | generateRandomTicksKt().writeTo(generatedKotlinDir) 60 | } 61 | } 62 | 63 | tasks.compileKotlin { 64 | dependsOn("generateSources") 65 | } 66 | 67 | sourceSets { 68 | main { 69 | java { 70 | srcDir(generatedKotlinDir) 71 | } 72 | } 73 | } -------------------------------------------------------------------------------- /buildSrc/build.gradle.kts: -------------------------------------------------------------------------------- 1 | plugins { 2 | `kotlin-dsl` 3 | } 4 | 5 | repositories { 6 | mavenCentral() 7 | gradlePluginPortal() 8 | } 9 | 10 | dependencies { 11 | implementation("com.squareup:kotlinpoet:1.10.0") 12 | } -------------------------------------------------------------------------------- /buildSrc/src/main/kotlin/de/skyrising/mc/scanner/gen/random-ticks.kt: -------------------------------------------------------------------------------- 1 | package de.skyrising.mc.scanner.gen 2 | 3 | import com.squareup.kotlinpoet.* 4 | import com.squareup.kotlinpoet.ParameterizedTypeName.Companion.parameterizedBy 5 | import java.lang.Long.bitCount 6 | import java.lang.Long.toUnsignedString 7 | 8 | val STEP = 1 / 32.0 9 | 10 | data class ChunkOffset(val x: Int, val z: Int) 11 | data class Vec2d(val x: Double, val z: Double) 12 | 13 | fun isRandomTicked(player: Vec2d, chunk: ChunkOffset): Boolean { 14 | val cx = chunk.x * 16 + 8 15 | val cz = chunk.z * 16 + 8 16 | val dx = player.x - cx 17 | val dz = player.z - cz 18 | return dx * dx + dz * dz < 128 * 128 19 | } 20 | 21 | fun mantissaBits(d: Double) = bitCount(d.toRawBits() and ((1L shl 52) - 1)) 22 | 23 | fun generateRandomTicksKt(): FileSpec { 24 | val permaloaded = linkedSetOf() 25 | val semiloaded = linkedSetOf() 26 | val regions = mutableMapOf() 27 | val corners = arrayOf( 28 | Vec2d(0.0, 0.0), 29 | Vec2d(0.0, 16.0), 30 | Vec2d(16.0, 0.0), 31 | Vec2d(16.0, 16.0) 32 | ) 33 | for (z in -8..8) { 34 | for (x in -8..8) { 35 | val chunk = ChunkOffset(x, z) 36 | val ticked = corners.map { isRandomTicked(it, chunk) } 37 | if (ticked.all { it }) { 38 | permaloaded.add(chunk) 39 | } else if (ticked.any { it }) { 40 | semiloaded.add(chunk) 41 | } 42 | } 43 | } 44 | require(semiloaded.size <= 64) { "Semi-loaded bits can't fit into Long" } 45 | var z = 0.0 46 | while (z <= 16.0) { 47 | var x = 0.0 48 | while (x <= 16.0) { 49 | val pos = Vec2d(x, z) 50 | val chunks = semiloaded 51 | .mapIndexed() { i, chunk -> (if (isRandomTicked(pos, chunk)) 1L else 0L) shl i } 52 | .reduce(Long::or) 53 | regions.compute(chunks) { _, old -> 54 | if (old == null) { 55 | pos 56 | } else { 57 | val oldBits = Math.pow(mantissaBits(old.x).toDouble(), 2.0) + Math.pow(mantissaBits(old.z).toDouble(), 2.0) 58 | val newBits = Math.pow(mantissaBits(pos.x).toDouble(), 2.0) + Math.pow(mantissaBits(pos.z).toDouble(), 2.0) 59 | if (newBits < oldBits) { 60 | pos 61 | } else { 62 | old 63 | } 64 | } 65 | } 66 | x += STEP 67 | } 68 | z += STEP 69 | } 70 | 71 | println(regions.size) 72 | val typeVec2i = ClassName("de.skyrising.mc.scanner", "Vec2i") 73 | val typeVec2d = ClassName("de.skyrising.mc.scanner", "Vec2d") 74 | val typeRegion = ClassName("de.skyrising.mc.scanner", "RandomTickRegion") 75 | val typeSpecRegion = TypeSpec.classBuilder("RandomTickRegion").addModifiers(KModifier.DATA) 76 | .primaryConstructor(FunSpec.constructorBuilder() 77 | .addParameter("pos", typeVec2d) 78 | .addParameter("chunks", U_LONG).build()) 79 | .addProperty(PropertySpec.builder("pos", typeVec2d).initializer("pos").build()) 80 | .addProperty(PropertySpec.builder("chunks", U_LONG).initializer("chunks").build()) 81 | .build() 82 | 83 | val permaVal = PropertySpec.builder("ALWAYS_RANDOM_TICKED", ARRAY.parameterizedBy(typeVec2i)) 84 | .initializer("arrayOf(%L)", permaloaded.map { 85 | CodeBlock.of("%T(%L,·%L)", typeVec2i, it.x, it.z) 86 | }.joinToCode(", ")) 87 | .build() 88 | 89 | val semiVal = PropertySpec.builder("SOMETIMES_RANDOM_TICKED", ARRAY.parameterizedBy(typeVec2i)) 90 | .initializer("arrayOf(%L)", semiloaded.map { 91 | CodeBlock.of("%T(%L,·%L)", typeVec2i, it.x, it.z) 92 | }.joinToCode(", ")) 93 | .build() 94 | 95 | val regionsVal = PropertySpec.builder("RANDOM_TICK_REGIONS", ARRAY.parameterizedBy(typeRegion)) 96 | .initializer("arrayOf(\n%L\n)", regions.map { 97 | CodeBlock.of("%T(%T(%L, %L), %L)", typeRegion, typeVec2d, it.value.x, it.value.z, "0x" + toUnsignedString(it.key, 16) + "UL") 98 | }.joinToCode(",\n")) 99 | .build() 100 | 101 | return FileSpec.builder("de.skyrising.mc.scanner", "random-ticks") 102 | .addType(typeSpecRegion) 103 | .addProperty(permaVal) 104 | .addProperty(semiVal) 105 | .addProperty(regionsVal) 106 | .build() 107 | } 108 | -------------------------------------------------------------------------------- /gradle.properties: -------------------------------------------------------------------------------- 1 | version = 0.6.0 -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SciCraft/mc-scanner/e29653496645c35df7f67849eeb89ba63b4dd453/gradle/wrapper/gradle-wrapper.jar -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.properties: -------------------------------------------------------------------------------- 1 | distributionBase=GRADLE_USER_HOME 2 | distributionPath=wrapper/dists 3 | distributionUrl=https\://services.gradle.org/distributions/gradle-7.6-bin.zip 4 | zipStoreBase=GRADLE_USER_HOME 5 | zipStorePath=wrapper/dists 6 | -------------------------------------------------------------------------------- /gradlew: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | 3 | # 4 | # Copyright 2015 the original author or authors. 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # https://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | # 18 | 19 | ############################################################################## 20 | ## 21 | ## Gradle start up script for UN*X 22 | ## 23 | ############################################################################## 24 | 25 | # Attempt to set APP_HOME 26 | # Resolve links: $0 may be a link 27 | PRG="$0" 28 | # Need this for relative symlinks. 29 | while [ -h "$PRG" ] ; do 30 | ls=`ls -ld "$PRG"` 31 | link=`expr "$ls" : '.*-> \(.*\)$'` 32 | if expr "$link" : '/.*' > /dev/null; then 33 | PRG="$link" 34 | else 35 | PRG=`dirname "$PRG"`"/$link" 36 | fi 37 | done 38 | SAVED="`pwd`" 39 | cd "`dirname \"$PRG\"`/" >/dev/null 40 | APP_HOME="`pwd -P`" 41 | cd "$SAVED" >/dev/null 42 | 43 | APP_NAME="Gradle" 44 | APP_BASE_NAME=`basename "$0"` 45 | 46 | # Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. 47 | DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' 48 | 49 | # Use the maximum available, or set MAX_FD != -1 to use that value. 50 | MAX_FD="maximum" 51 | 52 | warn () { 53 | echo "$*" 54 | } 55 | 56 | die () { 57 | echo 58 | echo "$*" 59 | echo 60 | exit 1 61 | } 62 | 63 | # OS specific support (must be 'true' or 'false'). 64 | cygwin=false 65 | msys=false 66 | darwin=false 67 | nonstop=false 68 | case "`uname`" in 69 | CYGWIN* ) 70 | cygwin=true 71 | ;; 72 | Darwin* ) 73 | darwin=true 74 | ;; 75 | MINGW* ) 76 | msys=true 77 | ;; 78 | NONSTOP* ) 79 | nonstop=true 80 | ;; 81 | esac 82 | 83 | CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar 84 | 85 | 86 | # Determine the Java command to use to start the JVM. 87 | if [ -n "$JAVA_HOME" ] ; then 88 | if [ -x "$JAVA_HOME/jre/sh/java" ] ; then 89 | # IBM's JDK on AIX uses strange locations for the executables 90 | JAVACMD="$JAVA_HOME/jre/sh/java" 91 | else 92 | JAVACMD="$JAVA_HOME/bin/java" 93 | fi 94 | if [ ! -x "$JAVACMD" ] ; then 95 | die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME 96 | 97 | Please set the JAVA_HOME variable in your environment to match the 98 | location of your Java installation." 99 | fi 100 | else 101 | JAVACMD="java" 102 | which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 103 | 104 | Please set the JAVA_HOME variable in your environment to match the 105 | location of your Java installation." 106 | fi 107 | 108 | # Increase the maximum file descriptors if we can. 109 | if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then 110 | MAX_FD_LIMIT=`ulimit -H -n` 111 | if [ $? -eq 0 ] ; then 112 | if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then 113 | MAX_FD="$MAX_FD_LIMIT" 114 | fi 115 | ulimit -n $MAX_FD 116 | if [ $? -ne 0 ] ; then 117 | warn "Could not set maximum file descriptor limit: $MAX_FD" 118 | fi 119 | else 120 | warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT" 121 | fi 122 | fi 123 | 124 | # For Darwin, add options to specify how the application appears in the dock 125 | if $darwin; then 126 | GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\"" 127 | fi 128 | 129 | # For Cygwin or MSYS, switch paths to Windows format before running java 130 | if [ "$cygwin" = "true" -o "$msys" = "true" ] ; then 131 | APP_HOME=`cygpath --path --mixed "$APP_HOME"` 132 | CLASSPATH=`cygpath --path --mixed "$CLASSPATH"` 133 | 134 | JAVACMD=`cygpath --unix "$JAVACMD"` 135 | 136 | # We build the pattern for arguments to be converted via cygpath 137 | ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null` 138 | SEP="" 139 | for dir in $ROOTDIRSRAW ; do 140 | ROOTDIRS="$ROOTDIRS$SEP$dir" 141 | SEP="|" 142 | done 143 | OURCYGPATTERN="(^($ROOTDIRS))" 144 | # Add a user-defined pattern to the cygpath arguments 145 | if [ "$GRADLE_CYGPATTERN" != "" ] ; then 146 | OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)" 147 | fi 148 | # Now convert the arguments - kludge to limit ourselves to /bin/sh 149 | i=0 150 | for arg in "$@" ; do 151 | CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -` 152 | CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option 153 | 154 | if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition 155 | eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"` 156 | else 157 | eval `echo args$i`="\"$arg\"" 158 | fi 159 | i=`expr $i + 1` 160 | done 161 | case $i in 162 | 0) set -- ;; 163 | 1) set -- "$args0" ;; 164 | 2) set -- "$args0" "$args1" ;; 165 | 3) set -- "$args0" "$args1" "$args2" ;; 166 | 4) set -- "$args0" "$args1" "$args2" "$args3" ;; 167 | 5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;; 168 | 6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;; 169 | 7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;; 170 | 8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;; 171 | 9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;; 172 | esac 173 | fi 174 | 175 | # Escape application args 176 | save () { 177 | for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done 178 | echo " " 179 | } 180 | APP_ARGS=`save "$@"` 181 | 182 | # Collect all arguments for the java command, following the shell quoting and substitution rules 183 | eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS" 184 | 185 | exec "$JAVACMD" "$@" 186 | -------------------------------------------------------------------------------- /gradlew.bat: -------------------------------------------------------------------------------- 1 | @rem 2 | @rem Copyright 2015 the original author or authors. 3 | @rem 4 | @rem Licensed under the Apache License, Version 2.0 (the "License"); 5 | @rem you may not use this file except in compliance with the License. 6 | @rem You may obtain a copy of the License at 7 | @rem 8 | @rem https://www.apache.org/licenses/LICENSE-2.0 9 | @rem 10 | @rem Unless required by applicable law or agreed to in writing, software 11 | @rem distributed under the License is distributed on an "AS IS" BASIS, 12 | @rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | @rem See the License for the specific language governing permissions and 14 | @rem limitations under the License. 15 | @rem 16 | 17 | @if "%DEBUG%" == "" @echo off 18 | @rem ########################################################################## 19 | @rem 20 | @rem Gradle startup script for Windows 21 | @rem 22 | @rem ########################################################################## 23 | 24 | @rem Set local scope for the variables with windows NT shell 25 | if "%OS%"=="Windows_NT" setlocal 26 | 27 | set DIRNAME=%~dp0 28 | if "%DIRNAME%" == "" set DIRNAME=. 29 | set APP_BASE_NAME=%~n0 30 | set APP_HOME=%DIRNAME% 31 | 32 | @rem Resolve any "." and ".." in APP_HOME to make it shorter. 33 | for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi 34 | 35 | @rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. 36 | set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" 37 | 38 | @rem Find java.exe 39 | if defined JAVA_HOME goto findJavaFromJavaHome 40 | 41 | set JAVA_EXE=java.exe 42 | %JAVA_EXE% -version >NUL 2>&1 43 | if "%ERRORLEVEL%" == "0" goto execute 44 | 45 | echo. 46 | echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 47 | echo. 48 | echo Please set the JAVA_HOME variable in your environment to match the 49 | echo location of your Java installation. 50 | 51 | goto fail 52 | 53 | :findJavaFromJavaHome 54 | set JAVA_HOME=%JAVA_HOME:"=% 55 | set JAVA_EXE=%JAVA_HOME%/bin/java.exe 56 | 57 | if exist "%JAVA_EXE%" goto execute 58 | 59 | echo. 60 | echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 61 | echo. 62 | echo Please set the JAVA_HOME variable in your environment to match the 63 | echo location of your Java installation. 64 | 65 | goto fail 66 | 67 | :execute 68 | @rem Setup the command line 69 | 70 | set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar 71 | 72 | 73 | @rem Execute Gradle 74 | "%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %* 75 | 76 | :end 77 | @rem End local scope for the variables with windows NT shell 78 | if "%ERRORLEVEL%"=="0" goto mainEnd 79 | 80 | :fail 81 | rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of 82 | rem the _cmd.exe /c_ return code! 83 | if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 84 | exit /b 1 85 | 86 | :mainEnd 87 | if "%OS%"=="Windows_NT" endlocal 88 | 89 | :omega 90 | -------------------------------------------------------------------------------- /settings.gradle.kts: -------------------------------------------------------------------------------- 1 | rootProject.name = "mc-scanner" 2 | 3 | pluginManagement { 4 | repositories { 5 | gradlePluginPortal() 6 | mavenCentral() 7 | } 8 | } -------------------------------------------------------------------------------- /src/main/kotlin/de/skyrising/mc/scanner/constants.kt: -------------------------------------------------------------------------------- 1 | package de.skyrising.mc.scanner 2 | 3 | object DataVersion { 4 | const val FLATTENING = 1451 // 17w47a 5 | const val NON_PACKED_BLOCK_STATES = 2529 // 20w17a 6 | const val BLOCK_STATES_CONTAINERIZED = 2832 // 1.18_experimental-snapshot-7 + 1 7 | const val REMOVE_LEVEL_TAG = 2842 // 21w42a + 2 8 | } -------------------------------------------------------------------------------- /src/main/kotlin/de/skyrising/mc/scanner/inventories.kt: -------------------------------------------------------------------------------- 1 | package de.skyrising.mc.scanner 2 | 3 | import it.unimi.dsi.fastutil.objects.Object2LongMap 4 | import it.unimi.dsi.fastutil.objects.Object2LongOpenHashMap 5 | import java.nio.ByteBuffer 6 | import java.nio.file.Files 7 | import java.nio.file.Path 8 | import java.util.* 9 | import java.util.function.LongPredicate 10 | import kotlin.collections.LinkedHashMap 11 | import kotlin.collections.LinkedHashSet 12 | 13 | data class PlayerFile(private val path: Path) : Scannable { 14 | override val size: Long = Files.size(path) 15 | val uuid: UUID 16 | init { 17 | val nameParts = path.fileName.toString().split('.') 18 | if (nameParts.size != 2 || nameParts[1] != "dat") throw IllegalArgumentException("Invalid player file: ${path.fileName}") 19 | uuid = UUID.fromString(nameParts[0]) 20 | } 21 | 22 | override fun scan(needles: Collection, statsMode: Boolean): List { 23 | val itemNeedles = needles.filterIsInstance() 24 | if (itemNeedles.isEmpty()) return emptyList() 25 | val raw = Files.readAllBytes(path) 26 | val data = Tag.read(ByteBufferDataInput(DECOMPRESSOR.decodeGzip(ByteBuffer.wrap(raw)))) 27 | if (data !is CompoundTag) return emptyList() 28 | val results = mutableListOf() 29 | if (data.has("Inventory", Tag.LIST)) { 30 | val invScan = scanInventory(data.getList("Inventory"), itemNeedles, statsMode) 31 | addResults(results, PlayerInventory(uuid, false), invScan, statsMode) 32 | } 33 | if (data.has("EnderItems", Tag.LIST)) { 34 | val enderScan = scanInventory(data.getList("EnderItems"), itemNeedles, statsMode) 35 | addResults(results, PlayerInventory(uuid, true), enderScan, statsMode) 36 | } 37 | return results 38 | } 39 | } 40 | 41 | fun scanInventory(slots: ListTag, needles: Collection, statsMode: Boolean): List> { 42 | val byId = LinkedHashMap>() 43 | for (needle in needles) { 44 | byId.computeIfAbsent(needle.id) { LinkedHashSet() }.add(needle) 45 | } 46 | val result = Object2LongOpenHashMap() 47 | val inventories = mutableListOf>(result) 48 | for (slot in slots) { 49 | if (!slot.has("id", Tag.STRING)) continue 50 | val id = Identifier.of(slot.getString("id")) 51 | val contained = getSubResults(slot, needles, statsMode) 52 | val matchingTypes = byId[id] 53 | if (matchingTypes != null || (byId.isEmpty() && statsMode && contained.isEmpty())) { 54 | val dmg = if (slot.has("Damage", Tag.INTEGER)) slot.getInt("Damage") else null 55 | var bestMatch = if (statsMode) ItemType(id, dmg ?: -1) else null 56 | if (matchingTypes != null) { 57 | for (type in matchingTypes) { 58 | if (dmg == type.damage || (bestMatch == null && type.damage < 0)) { 59 | bestMatch = type 60 | } 61 | } 62 | } 63 | if (bestMatch != null) { 64 | result.addTo(bestMatch, slot.getInt("Count").toLong()) 65 | } 66 | } 67 | if (contained.isEmpty()) continue 68 | if (statsMode) { 69 | contained.forEach(::flatten) 70 | inventories.addAll(contained) 71 | } 72 | for (e in contained[0].object2LongEntrySet()) { 73 | result.addTo(e.key, e.longValue) 74 | } 75 | } 76 | flatten(result) 77 | return inventories 78 | } 79 | 80 | fun flatten(items: Object2LongMap) { 81 | val updates = Object2LongOpenHashMap() 82 | for (e in items.object2LongEntrySet()) { 83 | if (e.key.flattened) continue 84 | val flattened = e.key.flatten() 85 | if (flattened == e.key) continue 86 | updates[flattened] = if (flattened in updates) updates.getLong(flattened) else items.getLong(flattened) + e.longValue 87 | e.setValue(0) 88 | } 89 | items.putAll(updates) 90 | items.values.removeIf(LongPredicate{ it == 0L }) 91 | } 92 | 93 | fun getSubResults(slot: CompoundTag, needles: Collection, statsMode: Boolean): List> { 94 | if (!slot.has("tag", Tag.COMPOUND)) return emptyList() 95 | val tag = slot.getCompound("tag") 96 | if (!tag.has("BlockEntityTag", Tag.COMPOUND)) return emptyList() 97 | val blockEntityTag = tag.getCompound("BlockEntityTag") 98 | if (!blockEntityTag.has("Items", Tag.LIST)) return emptyList() 99 | return scanInventory(blockEntityTag.getList("Items"), needles, statsMode) 100 | } 101 | 102 | fun tallyStats(scan: Object2LongMap): StatsResults { 103 | val types = scan.keys.toTypedArray() 104 | val stride = types.size 105 | val results = DoubleArray(stride * stride) 106 | val counts = scan.values.toLongArray() 107 | var total = counts.sum() 108 | for (i in types.indices) { 109 | // val count = counts[i] 110 | for (j in types.indices) { 111 | val otherCount = counts[j] 112 | val avg = otherCount.toDouble() / total 113 | results[i * stride + j] = avg 114 | } 115 | } 116 | return StatsResults(types, results) 117 | } -------------------------------------------------------------------------------- /src/main/kotlin/de/skyrising/mc/scanner/io.kt: -------------------------------------------------------------------------------- 1 | package de.skyrising.mc.scanner 2 | 3 | import java.io.ByteArrayInputStream 4 | import java.io.DataInput 5 | import java.io.EOFException 6 | import java.io.UTFDataFormatException 7 | import java.nio.ByteBuffer 8 | import java.util.zip.GZIPInputStream 9 | import java.util.zip.Inflater 10 | import java.util.zip.ZipException 11 | 12 | class ByteBufferDataInput(private val buf: ByteBuffer) : DataInput { 13 | private var chars = CharArray(128) 14 | 15 | override fun readFully(b: ByteArray) { 16 | if (buf.remaining() < b.size) throw EOFException() 17 | buf.get(b) 18 | } 19 | 20 | override fun readFully(b: ByteArray, off: Int, len: Int) { 21 | if (buf.remaining() < len) throw EOFException() 22 | buf.get(b, off, len) 23 | } 24 | 25 | override fun skipBytes(n: Int): Int { 26 | buf.position(buf.position() + n) 27 | return n 28 | } 29 | 30 | override fun readBoolean() = buf.get() != 0.toByte() 31 | override fun readByte() = buf.get() 32 | override fun readUnsignedByte() = buf.get().toInt() and 0xff 33 | override fun readShort() = buf.short 34 | override fun readUnsignedShort() = buf.short.toInt() and 0xffff 35 | override fun readChar() = buf.char 36 | override fun readInt() = buf.int 37 | override fun readLong() = buf.long 38 | override fun readFloat() = buf.float 39 | override fun readDouble() = buf.double 40 | 41 | override fun readLine(): String { 42 | TODO("Not yet implemented") 43 | } 44 | 45 | override fun readUTF(): String { 46 | val utflen = readUnsignedShort() 47 | if (buf.remaining() < utflen) throw EOFException() 48 | var count = 0 49 | val pos = buf.position() 50 | val chars = if (chars.size >= utflen) chars else CharArray(utflen).also { chars = it } 51 | while (count < utflen) { 52 | val c = buf[pos + count].toInt() and 0xff 53 | chars[count] = c.toChar() 54 | if (c > 127) break 55 | count++ 56 | } 57 | if (count == utflen) { 58 | skipBytes(count) 59 | return String(chars, 0, count) 60 | } 61 | return readUTF(chars, count, pos, utflen) 62 | } 63 | 64 | private fun readUTF(chararr: CharArray, start: Int, pos: Int, len: Int): String { 65 | var count = start 66 | var char2: Int 67 | var char3: Int 68 | var chararrCount = 0 69 | while (count < len) { 70 | val c = buf[pos + count].toInt() and 0xff 71 | when (c shr 4) { 72 | 0, 1, 2, 3, 4, 5, 6, 7 -> { /* 0xxxxxxx*/ 73 | count++ 74 | chararr[chararrCount++] = c.toChar() 75 | } 76 | 12, 13 -> { /* 110x xxxx 10xx xxxx*/ 77 | count += 2 78 | if (count > chararr.size) throw UTFDataFormatException( 79 | "malformed input: partial character at end") 80 | char2 = buf[pos + count - 1].toInt() 81 | if (char2 and 0xC0 != 0x80) throw UTFDataFormatException( 82 | "malformed input around byte $count") 83 | chararr[chararrCount++] = (c and 0x1F shl 6 or 84 | (char2 and 0x3F)).toChar() 85 | } 86 | 14 -> { /* 1110 xxxx 10xx xxxx 10xx xxxx */ 87 | count += 3 88 | if (count > chararr.size) throw UTFDataFormatException( 89 | "malformed input: partial character at end") 90 | char2 = buf[pos + count - 2].toInt() 91 | char3 = buf[pos + count - 1].toInt() 92 | if (char2 and 0xC0 != 0x80 || char3 and 0xC0 != 0x80) throw UTFDataFormatException( 93 | "malformed input around byte " + (count - 1)) 94 | chararr[chararrCount++] = (c and 0x0F shl 12 or 95 | (char2 and 0x3F shl 6) or 96 | (char3 and 0x3F shl 0)).toChar() 97 | } 98 | else -> throw UTFDataFormatException( 99 | "malformed input around byte $count") 100 | } 101 | } 102 | skipBytes(len) 103 | return String(chararr, 0, chararrCount) 104 | } 105 | } 106 | 107 | enum class Decompressor { 108 | INTERNAL { 109 | override fun decodeZlib(buf: ByteBuffer, arr: ByteArray?) = de.skyrising.mc.scanner.zlib.decodeZlib(buf, arr) 110 | override fun decodeGzip(buf: ByteBuffer, arr: ByteArray?) = de.skyrising.mc.scanner.zlib.decodeGzip(buf, arr) 111 | }, 112 | JAVA { 113 | override fun decodeZlib(buf: ByteBuffer, arr: ByteArray?): ByteBuffer { 114 | val length = buf.remaining() 115 | var bytes = arr ?: ByteArray(length) 116 | if (bytes.size < length) bytes = ByteArray(length) 117 | buf.get(bytes, 0, length) 118 | val inflater = Inflater() 119 | inflater.setInput(bytes.copyOfRange(0, length)) 120 | var uncompressedBytes = inflater.inflate(bytes) 121 | while (!inflater.finished()) { 122 | if (inflater.needsInput() || inflater.needsDictionary()) throw ZipException() 123 | bytes = bytes.copyOf(bytes.size * 2) 124 | uncompressedBytes += inflater.inflate(bytes, uncompressedBytes, bytes.size - uncompressedBytes) 125 | } 126 | inflater.end() 127 | return ByteBuffer.wrap(bytes, 0, uncompressedBytes) 128 | } 129 | 130 | override fun decodeGzip(buf: ByteBuffer, arr: ByteArray?): ByteBuffer { 131 | val length = buf.remaining() 132 | var bytes = arr ?: ByteArray(length) 133 | if (bytes.size < length) bytes = ByteArray(length) 134 | buf.get(bytes, 0, length) 135 | val gzip = GZIPInputStream(ByteArrayInputStream(bytes, 0, length)) 136 | return ByteBuffer.wrap(gzip.readBytes()) 137 | } 138 | }; 139 | 140 | abstract fun decodeZlib(buf: ByteBuffer, arr: ByteArray? = null): ByteBuffer 141 | abstract fun decodeGzip(buf: ByteBuffer, arr: ByteArray? = null): ByteBuffer 142 | } -------------------------------------------------------------------------------- /src/main/kotlin/de/skyrising/mc/scanner/nbt.kt: -------------------------------------------------------------------------------- 1 | package de.skyrising.mc.scanner 2 | 3 | import it.unimi.dsi.fastutil.objects.Reference2IntOpenHashMap 4 | import java.io.DataInput 5 | import java.io.DataOutput 6 | import java.io.Writer 7 | import java.util.* 8 | import kotlin.reflect.KClass 9 | 10 | interface TagType { 11 | fun read(din: DataInput): T 12 | } 13 | 14 | sealed class Tag { 15 | abstract fun write(out: DataOutput) 16 | abstract fun toString(sb: StringBuilder, depth: Int, path: LinkedList, indentString: String) 17 | 18 | fun writeSnbt(writer: Writer) { 19 | val sb = StringBuilder() 20 | toString(sb, 0, LinkedList(), " ") 21 | writer.write(sb.toString()) 22 | } 23 | 24 | @Suppress("MemberVisibilityCanBePrivate") 25 | companion object { 26 | const val END = 0 27 | const val BYTE = 1 28 | const val SHORT = 2 29 | const val INT = 3 30 | const val LONG = 4 31 | const val FLOAT = 5 32 | const val DOUBLE = 6 33 | const val BYTE_ARRAY = 7 34 | const val STRING = 8 35 | const val LIST = 9 36 | const val COMPOUND = 10 37 | const val INT_ARRAY = 11 38 | const val LONG_ARRAY = 12 39 | const val INTEGER = 99 40 | 41 | val NO_INDENT = setOf( 42 | listOf("{}", "size", "[]"), 43 | listOf("{}", "data", "[]", "{}"), 44 | listOf("{}", "palette", "[]", "{}"), 45 | listOf("{}", "entities", "[]", "{}") 46 | ) 47 | 48 | private val idToTag = Array>(13) { EndTag::class.java } 49 | private val idToType = Array>(13) { EndTag } 50 | private val tagToId = Reference2IntOpenHashMap>(13) 51 | 52 | private fun register(cls: KClass, id: Int, type: TagType) { 53 | idToTag[id] = cls.java 54 | idToType[id] = type 55 | tagToId[cls.java] = id 56 | } 57 | 58 | fun getId(tag: Tag) = tagToId.getInt(tag::class.java) 59 | fun getId(type: KClass) = tagToId.getInt(type.java) 60 | 61 | fun getReader(id: Int) = try { 62 | idToType[id] 63 | } catch (_: ArrayIndexOutOfBoundsException) { 64 | throw IllegalArgumentException("Unknown tag type $id") 65 | } 66 | 67 | inline fun read(id: Int, din: DataInput) = getReader(id).read(din) 68 | 69 | fun read(input: DataInput): Tag { 70 | val id = input.readByte().toInt() 71 | if (id == END) return EndTag 72 | input.readUTF() 73 | return read(id, input) 74 | } 75 | 76 | init { 77 | register(EndTag::class, END, EndTag) 78 | register(ByteTag::class, BYTE, ByteTag.Companion) 79 | register(ShortTag::class, SHORT, ShortTag.Companion) 80 | register(IntTag::class, INT, IntTag.Companion) 81 | register(LongTag::class, LONG, LongTag.Companion) 82 | register(FloatTag::class, FLOAT, FloatTag.Companion) 83 | register(DoubleTag::class, DOUBLE, DoubleTag.Companion) 84 | register(ByteArrayTag::class, BYTE_ARRAY, ByteArrayTag.Companion) 85 | register(StringTag::class, STRING, StringTag.Companion) 86 | register(ListTag::class, LIST, ListTag.Companion) 87 | register(CompoundTag::class, COMPOUND, CompoundTag.Companion) 88 | register(IntArrayTag::class, INT_ARRAY, IntArrayTag.Companion) 89 | register(LongArrayTag::class, LONG_ARRAY, LongArrayTag.Companion) 90 | } 91 | } 92 | } 93 | 94 | object EndTag : Tag(), TagType { 95 | override fun read(din: DataInput) = EndTag 96 | override fun write(out: DataOutput) {} 97 | override fun toString(sb: StringBuilder, depth: Int, path: LinkedList, indentString: String) {} 98 | } 99 | 100 | data class ByteTag(val value: Byte) : Tag() { 101 | override fun write(out: DataOutput) = out.writeByte(value.toInt()) 102 | companion object : TagType { 103 | override fun read(din: DataInput) = ByteTag(din.readByte()) 104 | } 105 | 106 | override fun toString(sb: StringBuilder, depth: Int, path: LinkedList, indentString: String) { 107 | sb.append(value.toInt()).append('b') 108 | } 109 | } 110 | 111 | data class ShortTag(val value: Short) : Tag() { 112 | override fun write(out: DataOutput) = out.writeShort(value.toInt()) 113 | companion object : TagType { 114 | override fun read(din: DataInput) = ShortTag(din.readShort()) 115 | } 116 | 117 | override fun toString(sb: StringBuilder, depth: Int, path: LinkedList, indentString: String) { 118 | sb.append(value.toInt()).append('s') 119 | } 120 | } 121 | 122 | data class IntTag(val value: Int) : Tag() { 123 | override fun write(out: DataOutput) = out.writeInt(value) 124 | companion object : TagType { 125 | override fun read(din: DataInput) = IntTag(din.readInt()) 126 | } 127 | 128 | override fun toString(sb: StringBuilder, depth: Int, path: LinkedList, indentString: String) { 129 | sb.append(value) 130 | } 131 | } 132 | 133 | data class LongTag(val value: Long) : Tag() { 134 | override fun write(out: DataOutput) = out.writeLong(value) 135 | companion object : TagType { 136 | override fun read(din: DataInput) = LongTag(din.readLong()) 137 | } 138 | 139 | override fun toString(sb: StringBuilder, depth: Int, path: LinkedList, indentString: String) { 140 | sb.append(value).append('L') 141 | } 142 | } 143 | 144 | data class FloatTag(val value: Float) : Tag() { 145 | override fun write(out: DataOutput) = out.writeFloat(value) 146 | companion object : TagType { 147 | override fun read(din: DataInput) = FloatTag(din.readFloat()) 148 | } 149 | 150 | override fun toString(sb: StringBuilder, depth: Int, path: LinkedList, indentString: String) { 151 | sb.append(value).append('f') 152 | } 153 | } 154 | 155 | data class DoubleTag(val value: Double) : Tag() { 156 | override fun write(out: DataOutput) = out.writeDouble(value) 157 | companion object : TagType { 158 | override fun read(din: DataInput) = DoubleTag(din.readDouble()) 159 | } 160 | 161 | override fun toString(sb: StringBuilder, depth: Int, path: LinkedList, indentString: String) { 162 | sb.append(value).append('d') 163 | } 164 | } 165 | 166 | data class ByteArrayTag(val value: ByteArray) : Tag() { 167 | override fun write(out: DataOutput) { 168 | out.writeInt(value.size) 169 | out.write(value) 170 | } 171 | 172 | override fun equals(other: Any?): Boolean { 173 | if (this === other) return true 174 | if (javaClass != other?.javaClass) return false 175 | other as ByteArrayTag 176 | if (!value.contentEquals(other.value)) return false 177 | return true 178 | } 179 | 180 | override fun hashCode(): Int = value.contentHashCode() 181 | 182 | override fun toString(sb: StringBuilder, depth: Int, path: LinkedList, indentString: String) { 183 | sb.append("[B;") 184 | for (i in value.indices) { 185 | if (i > 0) sb.append(',') 186 | sb.append(' ').append(value[i]).append('B') 187 | } 188 | sb.append(']') 189 | } 190 | 191 | companion object : TagType { 192 | override fun read(din: DataInput): ByteArrayTag { 193 | val value = ByteArray(din.readInt()) 194 | din.readFully(value) 195 | return ByteArrayTag(value) 196 | } 197 | } 198 | } 199 | 200 | data class StringTag(val value: String) : Tag() { 201 | override fun write(out: DataOutput) = out.writeUTF(value) 202 | 203 | override fun toString(sb: StringBuilder, depth: Int, path: LinkedList, indentString: String) { 204 | escape(sb, value) 205 | } 206 | 207 | companion object : TagType { 208 | val SIMPLE = Regex("[A-Za-z0-9._+-]+") 209 | 210 | override fun read(din: DataInput) = StringTag(din.readUTF()) 211 | 212 | fun escape(sb: StringBuilder, s: String) { 213 | val start = sb.length 214 | sb.append(' ') 215 | var quoteChar = 0.toChar() 216 | for (c in s) { 217 | if (c == '\\') sb.append('\\') 218 | else if (c == '"' || c == '\'') { 219 | if (quoteChar == 0.toChar()) { 220 | quoteChar = if (c == '"') '\'' else '"' 221 | } 222 | if (quoteChar == c) sb.append('\\') 223 | } 224 | sb.append(c) 225 | } 226 | if (quoteChar == 0.toChar()) quoteChar = '"' 227 | sb[start] = quoteChar 228 | sb.append(quoteChar) 229 | } 230 | } 231 | } 232 | 233 | data class ListTag(val value: MutableList) : Tag(), MutableList by value { 234 | constructor(value: Collection) : this(value.toMutableList()) 235 | 236 | fun verify(): Int { 237 | var id = 0 238 | for (elem in value) { 239 | val elemId = getId(elem) 240 | if (id == 0) id = elemId 241 | else if (elemId != id) throw IllegalStateException("Conflicting types in ListTag") 242 | } 243 | return id 244 | } 245 | 246 | override fun write(out: DataOutput) { 247 | out.writeByte(verify()) 248 | out.writeInt(value.size) 249 | for (tag in value) tag.write(out) 250 | } 251 | 252 | override fun toString(sb: StringBuilder, depth: Int, path: LinkedList, indentString: String) { 253 | if (value.isEmpty()) { 254 | sb.append("[]") 255 | return 256 | } 257 | sb.append('[') 258 | path.addLast("[]") 259 | val indent = indentString.isNotEmpty() && !NO_INDENT.contains(path) 260 | var first = true 261 | for (e in value) { 262 | if (!first) sb.append(',') 263 | if (indent) { 264 | sb.append('\n') 265 | for (i in 0 .. depth) sb.append(indentString) 266 | } else if (!first) { 267 | sb.append(' ') 268 | } 269 | first = false 270 | e.toString(sb, depth + 1, path, if (indent) indentString else "") 271 | } 272 | path.removeLast() 273 | if (indent) { 274 | sb.append('\n') 275 | for (i in 0 until depth) sb.append(" ") 276 | } 277 | sb.append(']') 278 | } 279 | 280 | override fun toString() = "ListTag$value" 281 | 282 | companion object : TagType> { 283 | override fun read(din: DataInput): ListTag<*> { 284 | val id = din.readByte().toInt() 285 | val size = din.readInt() 286 | val value = ArrayList(size) 287 | val reader = getReader(id) 288 | for (i in 0 until size) { 289 | value.add(reader.read(din)) 290 | } 291 | return ListTag(value) 292 | } 293 | } 294 | } 295 | 296 | data class CompoundTag(val value: MutableMap) : Tag(), MutableMap by value { 297 | override fun write(out: DataOutput) { 298 | for ((k, v) in value) { 299 | if (v is EndTag) throw IllegalStateException("EndTag in CompoundTag") 300 | out.writeByte(getId(v)) 301 | out.writeUTF(k) 302 | v.write(out) 303 | } 304 | out.write(END) 305 | } 306 | 307 | fun has(key: String, type: Int) = get(key, type) != null 308 | 309 | private fun get(key: String, type: Int): Tag? { 310 | val value = this[key] ?: return null 311 | if (type == INTEGER) { 312 | if (value !is ByteTag && value !is ShortTag && value !is IntTag) return null 313 | } else if (getId(value) != type) { 314 | return null 315 | } 316 | return value 317 | } 318 | 319 | private inline fun getTyped(key: String, value: (T) -> U): U { 320 | val tag = this[key] 321 | if (tag !is T) throw IllegalArgumentException("No ${T::class.java.name} for $key") 322 | return value(tag) 323 | } 324 | 325 | fun getCompound(key: String) = getTyped(key) { it } 326 | fun getByteArray(key: String) = getTyped(key, ByteArrayTag::value) 327 | fun getString(key: String) = getTyped(key, StringTag::value) 328 | fun getLongArray(key: String) = getTyped(key, LongArrayTag::value) 329 | 330 | fun getInt(key: String): Int { 331 | val tag = get(key, INTEGER) ?: throw IllegalArgumentException("No int value for $key") 332 | return when (tag) { 333 | is IntTag -> tag.value 334 | is ShortTag -> tag.value.toInt() 335 | is ByteTag -> tag.value.toInt() 336 | else -> error("Unexpected type $tag") 337 | } 338 | } 339 | 340 | inline fun getList(key: String): ListTag { 341 | val tag = this[key] 342 | if (tag !is ListTag<*>) throw IllegalArgumentException("No list tag for $key") 343 | val listType = tag.verify() 344 | @Suppress("UNCHECKED_CAST") 345 | if (listType == END || getId(T::class) == listType) return tag as ListTag 346 | throw IllegalArgumentException("Invalid list tag for $key") 347 | } 348 | 349 | override fun toString(sb: StringBuilder, depth: Int, path: LinkedList, indentString: String) { 350 | if (value.isEmpty()) { 351 | sb.append("{}") 352 | return 353 | } 354 | sb.append('{') 355 | path.addLast("{}") 356 | val indent = indentString.isNotEmpty() && !NO_INDENT.contains(path) 357 | var first = true 358 | for (k in getOrderedKeys(path)) { 359 | val v = value[k]!! 360 | if (!first) sb.append(',') 361 | if (indent) { 362 | sb.append('\n') 363 | for (i in 0 .. depth) sb.append(" ") 364 | } else if (!first) { 365 | sb.append(' ') 366 | } 367 | first = false 368 | if (StringTag.SIMPLE.matches(k)) { 369 | sb.append(k) 370 | } else { 371 | StringTag.escape(sb, k) 372 | } 373 | sb.append(": ") 374 | path.addLast(k) 375 | v.toString(sb, depth + 1, path, if (indent) indentString else "") 376 | path.removeLast() 377 | } 378 | path.removeLast() 379 | if (indent) { 380 | sb.append('\n') 381 | for (i in 0 until depth) sb.append(" ") 382 | } 383 | sb.append('}') 384 | } 385 | 386 | override fun toString() = "CompoundTag$value" 387 | 388 | private fun getOrderedKeys(path: List): List { 389 | var set = keys 390 | val ordered = mutableListOf() 391 | KEY_ORDER[path]?.let { 392 | set = HashSet(set) 393 | for (key in it) { 394 | if (set.remove(key)) ordered.add(key) 395 | } 396 | } 397 | ordered.addAll(set.sorted()) 398 | return ordered 399 | } 400 | 401 | companion object : TagType { 402 | private val KEY_ORDER = mapOf( 403 | listOf("{}") to listOf("DataVersion", "author", "size", "data", "entities", "palette", "palettes"), 404 | listOf("{}", "data", "[]", "{}") to listOf("pos", "state", "nbt"), 405 | listOf("{}", "entities", "[]", "{}") to listOf("blockPos", "pos") 406 | ) 407 | 408 | override fun read(din: DataInput): CompoundTag { 409 | val map = LinkedHashMap(8) 410 | while (true) { 411 | val id = din.readByte().toInt() 412 | if (id == 0) break 413 | map[din.readUTF()] = read(id, din) 414 | } 415 | return CompoundTag(map) 416 | } 417 | } 418 | } 419 | 420 | data class IntArrayTag(val value: IntArray) : Tag() { 421 | override fun write(out: DataOutput) { 422 | out.writeInt(value.size) 423 | for (i in value) out.writeInt(i) 424 | } 425 | 426 | override fun equals(other: Any?): Boolean { 427 | if (this === other) return true 428 | if (javaClass != other?.javaClass) return false 429 | other as IntArrayTag 430 | if (!value.contentEquals(other.value)) return false 431 | return true 432 | } 433 | 434 | override fun hashCode() = value.contentHashCode() 435 | 436 | override fun toString(sb: StringBuilder, depth: Int, path: LinkedList, indentString: String) { 437 | sb.append("[I;") 438 | for (i in value.indices) { 439 | if (i > 0) sb.append(',') 440 | sb.append(' ').append(value[i]) 441 | } 442 | sb.append(']') 443 | } 444 | 445 | companion object : TagType { 446 | override fun read(din: DataInput): IntArrayTag { 447 | val value = IntArray(din.readInt()) 448 | for (i in value.indices) value[i] = din.readInt() 449 | return IntArrayTag(value) 450 | } 451 | } 452 | } 453 | 454 | data class LongArrayTag(val value: LongArray) : Tag() { 455 | override fun write(out: DataOutput) { 456 | out.writeInt(value.size) 457 | for (l in value) out.writeLong(l) 458 | } 459 | 460 | override fun equals(other: Any?): Boolean { 461 | if (this === other) return true 462 | if (javaClass != other?.javaClass) return false 463 | other as LongArrayTag 464 | if (!value.contentEquals(other.value)) return false 465 | return true 466 | } 467 | 468 | override fun hashCode() = value.contentHashCode() 469 | 470 | override fun toString(sb: StringBuilder, depth: Int, path: LinkedList, indentString: String) { 471 | sb.append("[L;") 472 | for (i in value.indices) { 473 | if (i > 0) sb.append(',') 474 | sb.append(' ').append(value[i]).append('L') 475 | } 476 | sb.append(']') 477 | } 478 | 479 | companion object : TagType { 480 | override fun read(din: DataInput): LongArrayTag { 481 | val value = LongArray(din.readInt()) 482 | for (i in value.indices) value[i] = din.readLong() 483 | return LongArrayTag(value) 484 | } 485 | } 486 | } -------------------------------------------------------------------------------- /src/main/kotlin/de/skyrising/mc/scanner/needles.kt: -------------------------------------------------------------------------------- 1 | package de.skyrising.mc.scanner 2 | 3 | import kotlinx.serialization.decodeFromString 4 | import kotlinx.serialization.json.Json 5 | import kotlinx.serialization.json.JsonObject 6 | import kotlinx.serialization.json.jsonPrimitive 7 | 8 | interface Needle 9 | 10 | private val BLOCK_STATE_MAP = readBlockStateMap() 11 | private val ITEM_MAP = readItemMap() 12 | 13 | data class Identifier(val namespace: String, val path: String) : Comparable { 14 | override fun compareTo(other: Identifier): Int { 15 | val namespaceCompare = namespace.compareTo(other.namespace) 16 | if (namespaceCompare != 0) return namespaceCompare 17 | return path.compareTo(other.path) 18 | } 19 | 20 | override fun toString() = "$namespace:$path" 21 | 22 | companion object { 23 | fun of(id: String): Identifier { 24 | val colon = id.indexOf(':') 25 | if (colon < 0) return ofMinecraft(id) 26 | val namespace = id.substring(0, colon) 27 | val path = id.substring(colon + 1) 28 | if (namespace == "minecraft") return ofMinecraft(path) 29 | return Identifier(namespace, path) 30 | } 31 | fun ofMinecraft(path: String) = Identifier("minecraft", path) 32 | } 33 | } 34 | 35 | data class BlockState(val id: Identifier, val properties: Map = emptyMap()) : Needle, Comparable { 36 | fun unflatten(): List { 37 | val list = mutableListOf() 38 | var id: Int? = null 39 | var mask = 0 40 | for (i in BLOCK_STATE_MAP.indices) { 41 | val mapped = BLOCK_STATE_MAP[i] ?: continue 42 | if (!mapped.matches(this)) continue 43 | val currentId = i shr 4 44 | if (id != null && currentId != id) { 45 | list.add(BlockIdMask(id, mask, this)) 46 | mask = 0 47 | } 48 | id = currentId 49 | mask = mask or (1 shl (i and 0xf)) 50 | } 51 | if (id != null) list.add(BlockIdMask(id, mask, this)) 52 | return list 53 | } 54 | 55 | fun matches(predicate: BlockState): Boolean { 56 | if (id != predicate.id) return false 57 | for (e in predicate.properties.entries) { 58 | if (!properties.containsKey(e.key) || properties[e.key] != e.value) return false 59 | } 60 | return true 61 | } 62 | 63 | override fun compareTo(other: BlockState): Int { 64 | val idComp = id.compareTo(other.id) 65 | if (idComp != 0) return idComp 66 | if (properties == other.properties) return 0 67 | return properties.hashCode().compareTo(other.properties.hashCode()) or 1 68 | } 69 | 70 | fun format(): String { 71 | if (properties.isEmpty()) return id.toString() 72 | val sb = StringBuilder(id.toString()).append('[') 73 | var first = true 74 | for (e in properties.entries) { 75 | if (!first) sb.append(',') 76 | first = false 77 | sb.append(e.key).append('=').append(e.value) 78 | } 79 | return sb.append(']').toString() 80 | } 81 | 82 | override fun toString() = "BlockState(${format()})" 83 | 84 | companion object { 85 | fun parse(desc: String): BlockState { 86 | if (!desc.contains('[')) return BlockState(Identifier.of(desc)) 87 | val bracketIndex = desc.indexOf('[') 88 | val closingBracketIndex = desc.indexOf(']', bracketIndex + 1) 89 | if (closingBracketIndex != desc.lastIndex) throw IllegalArgumentException("Expected closing ]") 90 | val id = Identifier.of(desc.substring(0, bracketIndex)) 91 | val properties = LinkedHashMap() 92 | for (kvPair in desc.substring(bracketIndex + 1, closingBracketIndex).split(',')) { 93 | val equalsIndex = kvPair.indexOf('=') 94 | if (equalsIndex < 0) throw IllegalArgumentException("Invalid key-value pair") 95 | properties[kvPair.substring(0, equalsIndex)] = kvPair.substring(equalsIndex + 1) 96 | } 97 | return BlockState(id, properties) 98 | } 99 | 100 | fun from(nbt: CompoundTag): BlockState { 101 | val id = Identifier.of(nbt.getString("Name")) 102 | if (!nbt.has("Properties", Tag.COMPOUND)) return BlockState(id) 103 | val properties = LinkedHashMap() 104 | for (e in nbt.getCompound("Properties").entries) { 105 | properties[e.key] = (e.value as StringTag).value 106 | } 107 | return BlockState(id, properties) 108 | } 109 | } 110 | } 111 | 112 | data class BlockIdMask(val id: Int, val metaMask: Int, val blockState: BlockState? = null) : Needle { 113 | fun matches(id: Int, meta: Int) = this.id == id && (1 shl meta) and metaMask != 0 114 | 115 | infix fun or(other: BlockIdMask): BlockIdMask { 116 | if (other.id != id) throw IllegalArgumentException("Cannot combine different ids") 117 | return BlockIdMask(id, metaMask or other.metaMask) 118 | } 119 | 120 | override fun toString(): String { 121 | if (blockState == null) return "BlockIdMask(%d:0x%04x)".format(id, metaMask) 122 | return "BlockIdMask(%d:0x%04x %s)".format(id, metaMask, blockState.format()) 123 | } 124 | } 125 | 126 | data class ItemType(val id: Identifier, val damage: Int = -1, val flattened: Boolean = damage < 0) : Needle, Comparable { 127 | fun flatten(): ItemType { 128 | if (this.flattened) return this 129 | var flattened = ITEM_MAP[this] 130 | if (flattened == null) flattened = ITEM_MAP[ItemType(id, 0)] 131 | if (flattened == null) return ItemType(id, damage, true) 132 | return ItemType(flattened, -1, true) 133 | } 134 | 135 | fun unflatten(): List { 136 | if (!flattened) return emptyList() 137 | val list = mutableListOf() 138 | for (e in ITEM_MAP.entries) { 139 | if (e.value == this.id && e.key.id != this.id) { 140 | list.add(e.key) 141 | } 142 | } 143 | return list 144 | } 145 | 146 | override fun toString(): String { 147 | return "ItemType(${format()})" 148 | } 149 | 150 | fun format() = if (damage < 0 || (flattened && damage == 0)) "$id" else "$id.$damage" 151 | 152 | override fun compareTo(other: ItemType): Int { 153 | val idComp = id.compareTo(other.id) 154 | if (idComp != 0) return idComp 155 | return damage.compareTo(other.damage) 156 | } 157 | 158 | companion object { 159 | fun parse(str: String): ItemType { 160 | if (!str.contains('.')) return ItemType(Identifier.of(str)) 161 | return ItemType(Identifier.of(str.substringBefore('.')), str.substringAfter('.').toInt()) 162 | } 163 | } 164 | } 165 | 166 | private fun getFlatteningMap(name: String): JsonObject = Json.decodeFromString(BlockIdMask::class.java.getResourceAsStream("/flattening/$name.json")!!.reader().readText()) 167 | 168 | private fun readBlockStateMap(): Array { 169 | val jsonMap = getFlatteningMap("block_states") 170 | val map = Array(256 * 16) { null } 171 | for (e in jsonMap.entries) { 172 | val id = if (e.key.contains(':')) { 173 | e.key.substringBefore(':').toInt() shl 4 or e.key.substringAfter(':').toInt() 174 | } else { 175 | e.key.toInt() shl 4 176 | } 177 | map[id] = BlockState.parse(e.value.jsonPrimitive.content) 178 | } 179 | for (i in map.indices) { 180 | if (map[i] != null) continue 181 | map[i] = map[i and 0xff0] 182 | } 183 | return map 184 | } 185 | 186 | private fun readItemMap(): Map { 187 | val jsonMap = getFlatteningMap("items") 188 | val map = LinkedHashMap() 189 | for (e in jsonMap.entries) { 190 | map[ItemType.parse(e.key)] = Identifier.of(e.value.jsonPrimitive.content) 191 | } 192 | return map 193 | } 194 | -------------------------------------------------------------------------------- /src/main/kotlin/de/skyrising/mc/scanner/region.kt: -------------------------------------------------------------------------------- 1 | package de.skyrising.mc.scanner 2 | 3 | import de.skyrising.mc.scanner.region.RegionReader 4 | import de.skyrising.mc.scanner.region.RegionVisitor 5 | import it.unimi.dsi.fastutil.ints.Int2ObjectMap 6 | import it.unimi.dsi.fastutil.ints.Int2ObjectOpenHashMap 7 | import it.unimi.dsi.fastutil.objects.Object2IntMap 8 | import it.unimi.dsi.fastutil.objects.Object2IntOpenHashMap 9 | import java.nio.file.Files 10 | import java.nio.file.Path 11 | import java.nio.file.StandardOpenOption 12 | import kotlin.math.ceil 13 | import kotlin.math.log2 14 | 15 | data class RegionFile(private val path: Path) : Scannable { 16 | val x: Int 17 | val z: Int 18 | val dimension = when (val dim = path.getName(path.nameCount - 3).toString()) { 19 | "." -> "overworld" 20 | "DIM-1" -> "the_nether" 21 | "DIM1" -> "the_end" 22 | else -> dim 23 | } 24 | override val size = Files.size(path) 25 | init { 26 | val fileName = path.fileName.toString() 27 | val parts = fileName.split('.') 28 | if (parts.size != 4 || parts[0] != "r" || parts[3] != "mca") { 29 | throw IllegalArgumentException("Not a valid region file name: $fileName") 30 | } 31 | x = parts[1].toInt() 32 | z = parts[2].toInt() 33 | } 34 | 35 | fun reader() = RegionReader(Files.newByteChannel(path, StandardOpenOption.READ)) 36 | 37 | fun forEachChunk(visitor: (x: Int, z: Int, version: Int, data: CompoundTag) -> Unit) { 38 | reader().use { 39 | it.accept(RegionVisitor.visitAllChunks(visitor)) 40 | } 41 | } 42 | 43 | fun scanChunks(needles: Collection, statsMode: Boolean): List { 44 | val results = mutableListOf() 45 | val blockIdNeedles: Set = needles.filterIsInstanceTo(mutableSetOf()) 46 | val blockStateNeedles: Set = needles.filterIsInstanceTo(mutableSetOf()) 47 | val itemNeedles: Set = needles.filterIsInstanceTo(mutableSetOf()) 48 | forEachChunk { x, z, version, data -> 49 | scanChunk(results, blockIdNeedles, blockStateNeedles, itemNeedles, statsMode, ChunkPos(dimension, (this.x shl 5) or x, (this.z shl 5) or z), version, data) 50 | } 51 | return results 52 | } 53 | 54 | override fun scan(needles: Collection, statsMode: Boolean): List { 55 | return scanChunks(needles, statsMode) 56 | } 57 | 58 | override fun toString(): String { 59 | return "RegionFile($dimension, x=$x, z=$z)" 60 | } 61 | } 62 | 63 | fun getPalette(section: CompoundTag, version: Int): List? { 64 | if (version <= DataVersion.BLOCK_STATES_CONTAINERIZED) { 65 | return if (section.has("Palette", Tag.LIST)) section.getList("Palette") else null 66 | } 67 | if (!section.has("block_states", Tag.COMPOUND)) return null 68 | val container = section.getCompound("block_states") 69 | return if (container.has("palette", Tag.LIST)) container.getList("palette") else null 70 | } 71 | 72 | fun getBlockStates(section: CompoundTag, version: Int): LongArray? { 73 | if (version <= DataVersion.BLOCK_STATES_CONTAINERIZED) { 74 | return if (section.has("BlockStates", Tag.LONG_ARRAY)) section.getLongArray("BlockStates") else null 75 | } 76 | if (!section.has("block_states", Tag.COMPOUND)) return null 77 | val container = section.getCompound("block_states") 78 | return if (container.has("data", Tag.LONG_ARRAY)) container.getLongArray("data") else null 79 | } 80 | 81 | fun scanChunk(results: MutableList, blockIdNeedles: Set, blockStateNeedles: Set, itemNeedles: Set, statsMode: Boolean, chunkPos: ChunkPos, version: Int, data: CompoundTag) { 82 | val dimension = chunkPos.dimension 83 | val flattened = version >= DataVersion.FLATTENING 84 | if (!flattened && blockIdNeedles.isNotEmpty()) { 85 | scanUnflattenedChunk(blockIdNeedles, data, results, chunkPos) 86 | } 87 | if (flattened && blockStateNeedles.isNotEmpty()) { 88 | scanFlattenedChunk(data, version, blockStateNeedles, results, chunkPos) 89 | } 90 | if (itemNeedles.isNotEmpty() || statsMode) { 91 | scanChunkItems(version, data, itemNeedles, statsMode, dimension, results) 92 | } 93 | } 94 | 95 | private fun scanUnflattenedChunk(blockIdNeedles: Set, data: CompoundTag, results: MutableList, chunkPos: ChunkPos) { 96 | val sections = data.getList("Sections") 97 | val matches = Object2IntOpenHashMap() 98 | for (section in sections) { 99 | val y = section.getInt("Y") 100 | val blocks = section.getByteArray("Blocks") 101 | //val add = if (section.has("Add", Tag.BYTE_ARRAY)) section.getByteArray("Add") else null 102 | for (blockNeedle in blockIdNeedles) { 103 | val count = blocks.count { it == blockNeedle.id.toByte() } 104 | if (count != 0) { 105 | matches[blockNeedle] = matches.getInt(blockNeedle) + count 106 | } 107 | } 108 | if (matches.size == blockIdNeedles.size) break 109 | } 110 | for (match in matches) { 111 | results.add(SearchResult(match.key.blockState ?: match.key, chunkPos, match.value.toLong())) 112 | } 113 | } 114 | 115 | private fun scanFlattenedChunk(data: CompoundTag, version: Int, blockStateNeedles: Set, results: MutableList, chunkPos: ChunkPos) { 116 | val sections = data.getList(if (version < DataVersion.REMOVE_LEVEL_TAG) "Sections" else "sections") 117 | val matches = Object2IntOpenHashMap() 118 | for (section in sections) { 119 | val palette = getPalette(section, version) ?: continue 120 | val blockStates = getBlockStates(section, version) ?: continue 121 | val matchingPaletteEntries = Int2ObjectOpenHashMap() 122 | palette.forEachIndexed { index, paletteEntry -> 123 | val state = BlockState.from(paletteEntry) 124 | for (blockNeedle in blockStateNeedles) { 125 | if (state.matches(blockNeedle)) matchingPaletteEntries[index] = blockNeedle 126 | } 127 | } 128 | if (matchingPaletteEntries.isEmpty()) continue 129 | val counts = scanBlockStates(matchingPaletteEntries, blockStates, palette.size, version < DataVersion.NON_PACKED_BLOCK_STATES) 130 | for (e in counts.object2IntEntrySet()) { 131 | matches[e.key] = matches.getInt(e.key) + e.intValue 132 | } 133 | } 134 | for (match in matches) { 135 | results.add(SearchResult(match.key, chunkPos, match.value.toLong())) 136 | } 137 | } 138 | 139 | private fun scanChunkItems(version: Int, data: CompoundTag, itemNeedles: Set, statsMode: Boolean, dimension: String, results: MutableList) { 140 | val blockEntitiesTag = if (version >= DataVersion.REMOVE_LEVEL_TAG) "block_entities" else "TileEntities" 141 | if (data.has(blockEntitiesTag, Tag.LIST)) { 142 | for (blockEntity in data.getList(blockEntitiesTag)) { 143 | if (!blockEntity.has("Items", Tag.LIST)) continue 144 | val contents = scanInventory(blockEntity.getList("Items"), itemNeedles, statsMode) 145 | val pos = BlockPos(dimension, blockEntity.getInt("x"), blockEntity.getInt("y"), blockEntity.getInt("z")) 146 | val container = Container(blockEntity.getString("id"), pos) 147 | addResults(results, container, contents, statsMode) 148 | } 149 | } 150 | val entitiesTag = if (version >= DataVersion.REMOVE_LEVEL_TAG) "entities" else "Entities" 151 | if (data.has(entitiesTag, Tag.LIST)) { 152 | for (entity in data.getList(entitiesTag)) { 153 | val id = entity.getString("id") 154 | val items = mutableListOf() 155 | if (entity.has("HandItems", Tag.LIST)) items.addAll(entity.getList("HandItems")) 156 | if (entity.has("ArmorItems", Tag.LIST)) items.addAll(entity.getList("ArmorItems")) 157 | if (entity.has("Inventory", Tag.LIST)) items.addAll(entity.getList("Inventory")) 158 | if (entity.has("Item", Tag.COMPOUND)) items.add(entity.getCompound("Item")) 159 | val listTag = ListTag(items.filter(CompoundTag::isNotEmpty)) 160 | if (listTag.isNotEmpty()) { 161 | val posTag = entity.getList("Pos") 162 | val pos = Vec3d(dimension, posTag[0].value, posTag[1].value, posTag[2].value) 163 | val entityLocation = Entity(id, pos) 164 | val contents = scanInventory(listTag, itemNeedles, statsMode) 165 | addResults(results, entityLocation, contents, statsMode) 166 | } 167 | } 168 | } 169 | } 170 | 171 | fun scanBlockStates(ids: Int2ObjectMap, blockStates: LongArray, paletteSize: Int, packed: Boolean): Object2IntMap { 172 | val counts = Object2IntOpenHashMap() 173 | val bits = maxOf(4, ceil(log2(paletteSize.toDouble())).toInt()) 174 | val mask = (1 shl bits) - 1 175 | var longIndex = 0 176 | var subIndex = 0 177 | repeat(16 * 16 * 16) { 178 | if (subIndex + bits > 64 && !packed) { 179 | longIndex++ 180 | subIndex = 0 181 | } 182 | val id = if (subIndex + bits > 64) { 183 | val loBitsCount = 64 - subIndex 184 | val loBits = (blockStates[longIndex] ushr subIndex).toInt() 185 | val hiBits = blockStates[longIndex + 1].toInt() 186 | longIndex++ 187 | (loBits or (hiBits shl loBitsCount)) and mask 188 | } else { 189 | (blockStates[longIndex] ushr subIndex).toInt() and mask 190 | } 191 | if (id in ids) { 192 | val state = ids[id] 193 | counts[state] = counts.getInt(state) + 1 194 | } 195 | if (subIndex + bits == 64) longIndex++ 196 | subIndex = (subIndex + bits) and 0x3f 197 | } 198 | return counts 199 | } -------------------------------------------------------------------------------- /src/main/kotlin/de/skyrising/mc/scanner/region/io.kt: -------------------------------------------------------------------------------- 1 | package de.skyrising.mc.scanner.region 2 | 3 | import de.skyrising.mc.scanner.* 4 | import java.io.IOException 5 | import java.nio.ByteBuffer 6 | import java.nio.channels.SeekableByteChannel 7 | import kotlin.concurrent.getOrSet 8 | 9 | private val decompressedBuf = ThreadLocal() 10 | 11 | class RegionReader(private val channel: SeekableByteChannel): AutoCloseable { 12 | private var fileSize: Long = 0 13 | private var chunkBuf: ByteBuffer? = null 14 | 15 | fun accept(visitor: RegionVisitor) { 16 | channel.position(0) 17 | fileSize = channel.size() 18 | if (fileSize == 0L) return 19 | if (fileSize < 1024 * 2 * 4) throw IOException("File is too small") 20 | val tableBuf = ByteBuffer.allocateDirect(1024 * 2 * 4) 21 | readFully(channel, tableBuf) 22 | tableBuf.flip() 23 | val tableInts = tableBuf.asIntBuffer() 24 | for (i in 0 until 1024) { 25 | val location = tableInts[i] 26 | if (location == 0) continue 27 | val chunkX = i and 0x1f 28 | val chunkZ = i shr 5 29 | val chunkVisitor = visitor.visitChunk(chunkX, chunkZ) ?: continue 30 | val offset = location shr 8 31 | val sectors = location and 0xff 32 | visitChunk(chunkVisitor, offset, sectors, chunkX, chunkZ) 33 | } 34 | } 35 | 36 | override fun close() { 37 | channel.close() 38 | } 39 | 40 | private fun readChunk(offset: Int, sectors: Int, chunkX: Int, chunkZ: Int): ByteBuffer { 41 | var chunkBuf = this.chunkBuf 42 | if (chunkBuf == null || chunkBuf.capacity() < sectors * 4096) { 43 | chunkBuf = ByteBuffer.allocateDirect(sectors * 4096)!! 44 | } 45 | chunkBuf.position(0) 46 | val byteOffset = offset * 4096L 47 | chunkBuf.limit(minOf(sectors * 4096, (fileSize - byteOffset).toInt())) 48 | channel.position(byteOffset) 49 | try { 50 | readFully(channel, chunkBuf) 51 | } catch (e: IOException) { 52 | throw IOException("Could not read chunk $chunkX, $chunkZ at offset ${offset * 4096L}, size ${sectors * 4096}, file size $fileSize", e) 53 | } 54 | chunkBuf.flip() 55 | return chunkBuf 56 | } 57 | 58 | private fun visitChunk(visitor: ChunkVisitor, offset: Int, sectors: Int, chunkX: Int, chunkZ: Int) { 59 | val chunkBuf = readChunk(offset, sectors, chunkX, chunkZ) 60 | val length = chunkBuf.int - 1 61 | chunkBuf.limit(length + 5) 62 | val compression = chunkBuf.get() 63 | //println("$chunkPos, offset=$offset, sectors=$sectors, length=$length, compression=$compression, buf=$chunkBuf") 64 | val dbuf = decompressedBuf.getOrSet { 65 | ByteArray(length) 66 | } 67 | 68 | val chunk = when (compression.toInt()) { 69 | 1 -> { 70 | try { 71 | val buf = DECOMPRESSOR.decodeGzip(chunkBuf, dbuf) 72 | decompressedBuf.set(buf.array()) 73 | Tag.read(ByteBufferDataInput(buf)) 74 | } catch (e: Exception) { 75 | visitor.onInvalidData(e) 76 | return 77 | } 78 | } 79 | 2 -> { 80 | try { 81 | val buf = DECOMPRESSOR.decodeZlib(chunkBuf, dbuf) 82 | val copy = buf.slice() 83 | decompressedBuf.set(buf.array()) 84 | try { 85 | Tag.read(ByteBufferDataInput(buf)) 86 | } catch (e: Exception) { 87 | println(hexdump(copy)) 88 | throw e 89 | } 90 | } catch (e: Exception) { 91 | visitor.onInvalidData(e) 92 | return 93 | } 94 | } 95 | else -> { 96 | visitor.onUnsupportedCompressionType(compression.toInt()) 97 | return 98 | } 99 | } 100 | if (chunk !is CompoundTag) { 101 | visitor.onInvalidData(IllegalArgumentException("Expected chunk to be a CompoundTag")) 102 | return 103 | } 104 | if (!chunk.has("DataVersion", Tag.INTEGER)) { 105 | visitor.onInvalidData(IllegalArgumentException("No data version")) 106 | return 107 | } 108 | val version = chunk.getInt("DataVersion"); 109 | if (!chunk.has("Level", Tag.COMPOUND) && version < DataVersion.REMOVE_LEVEL_TAG) { 110 | visitor.onInvalidData(IllegalArgumentException("No level tag")) 111 | return 112 | } 113 | visitor.visit(version, if (version >= DataVersion.REMOVE_LEVEL_TAG) chunk else chunk.getCompound("Level")) 114 | } 115 | } -------------------------------------------------------------------------------- /src/main/kotlin/de/skyrising/mc/scanner/region/tree.kt: -------------------------------------------------------------------------------- 1 | package de.skyrising.mc.scanner.region 2 | 3 | import de.skyrising.mc.scanner.CompoundTag 4 | 5 | class RegionNode : RegionVisitor() { 6 | private val chunks = Array(32 * 32) { null } 7 | 8 | fun accept(visitor: RegionVisitor) { 9 | for (i in chunks.indices) { 10 | val chunk = chunks[i] ?: continue 11 | visitor.visitChunk(i and 0x1f, i shr 5)?.let { chunk.accept(it) } 12 | } 13 | } 14 | 15 | override fun visitChunk(x: Int, z: Int): ChunkNode { 16 | val node = ChunkNode() 17 | chunks[(z shl 5) or x] = node 18 | return node 19 | } 20 | } 21 | 22 | class ChunkNode : ChunkVisitor() { 23 | private var version = 0 24 | private var data: CompoundTag? = null 25 | 26 | fun accept(visitor: ChunkVisitor) { 27 | val data = this.data 28 | if (data != null) visitor.visit(version, data) 29 | } 30 | } -------------------------------------------------------------------------------- /src/main/kotlin/de/skyrising/mc/scanner/region/visitors.kt: -------------------------------------------------------------------------------- 1 | package de.skyrising.mc.scanner.region 2 | 3 | import de.skyrising.mc.scanner.CompoundTag 4 | 5 | abstract class RegionVisitor() { 6 | protected var delegate: RegionVisitor? = null 7 | 8 | constructor(regionVisitor: RegionVisitor) : this() { 9 | this.delegate = regionVisitor 10 | } 11 | 12 | open fun visitChunk(x: Int, z: Int): ChunkVisitor? { 13 | return delegate?.visitChunk(x, z) 14 | } 15 | 16 | companion object { 17 | fun visitAllChunks(visit: (x: Int, z: Int, version: Int, data: CompoundTag) -> Unit) = object : RegionVisitor() { 18 | override fun visitChunk(x: Int, z: Int): ChunkVisitor { 19 | return object : ChunkVisitor() { 20 | override fun visit(version: Int, data: CompoundTag) { 21 | visit(x, z, version, data) 22 | } 23 | 24 | override fun onInvalidData(e: Exception) { 25 | e.printStackTrace() 26 | } 27 | } 28 | } 29 | } 30 | } 31 | } 32 | 33 | abstract class ChunkVisitor() { 34 | protected var delegate: ChunkVisitor? = null 35 | 36 | constructor(chunkVisitor: ChunkVisitor) : this() { 37 | this.delegate = chunkVisitor 38 | } 39 | 40 | open fun visit(version: Int, data: CompoundTag) { 41 | delegate?.visit(version, data) 42 | } 43 | 44 | open fun onUnsupportedCompressionType(type: Int) { 45 | delegate?.onUnsupportedCompressionType(type) 46 | } 47 | 48 | open fun onInvalidData(ex: Exception) { 49 | delegate?.onInvalidData(ex) 50 | } 51 | } -------------------------------------------------------------------------------- /src/main/kotlin/de/skyrising/mc/scanner/scanner.kt: -------------------------------------------------------------------------------- 1 | package de.skyrising.mc.scanner 2 | 3 | import de.skyrising.mc.scanner.script.Scan 4 | import de.skyrising.mc.scanner.script.ScannerScript 5 | import it.unimi.dsi.fastutil.objects.* 6 | import joptsimple.OptionException 7 | import joptsimple.OptionParser 8 | import joptsimple.ValueConverter 9 | import java.io.PrintStream 10 | import java.net.URI 11 | import java.nio.file.* 12 | import java.util.* 13 | import java.util.concurrent.CompletableFuture 14 | import java.util.concurrent.ExecutorService 15 | import java.util.concurrent.Executors 16 | import java.util.concurrent.atomic.AtomicInteger 17 | import java.util.concurrent.atomic.AtomicLong 18 | import java.util.function.ToIntFunction 19 | import kotlin.script.experimental.api.* 20 | import kotlin.script.experimental.host.toScriptSource 21 | import kotlin.script.experimental.jvmhost.BasicJvmScriptingHost 22 | 23 | var DECOMPRESSOR = Decompressor.INTERNAL 24 | 25 | fun main(args: Array) { 26 | val parser = OptionParser() 27 | val helpArg = parser.accepts("help").forHelp() 28 | val nonOptions = parser.nonOptions() 29 | val blockArg = parser.acceptsAll(listOf("b", "block"), "Add a block to search for").withRequiredArg() 30 | val itemArg = parser.acceptsAll(listOf("i", "item"), "Add an item to search for").withRequiredArg() 31 | val statsArg = parser.accepts("stats", "Calculate statistics for storage tech") 32 | val geode = parser.accepts("geode", "Calculate AFK spots for geodes") 33 | val threadsArg = parser.acceptsAll(listOf("t", "threads"), "Set the number of threads to use").withRequiredArg().ofType(Integer::class.java) 34 | val loopArg = parser.accepts("loop").withOptionalArg().ofType(Integer::class.java) 35 | val decompressorArg = parser.accepts("decompressor", "Decompressor to use").withOptionalArg().withValuesConvertedBy(object : ValueConverter { 36 | override fun convert(value: String) = Decompressor.valueOf(value.uppercase()) 37 | override fun valueType() = Decompressor::class.java 38 | override fun valuePattern() = "internal|java" 39 | }) 40 | val needles = mutableListOf() 41 | fun printUsage() { 42 | System.err.println("Usage: mc-scanner (-i | -b )* [options] [output]") 43 | parser.printHelpOn(System.err) 44 | } 45 | var threads = 0 46 | var loopCount = 0 47 | val path: Path 48 | val outPath: Path 49 | val zip: FileSystem? 50 | val script: Path 51 | try { 52 | val options = parser.parse(*args) 53 | if (options.has(helpArg)) { 54 | printUsage() 55 | return 56 | } 57 | for (block in options.valuesOf(blockArg)) { 58 | val state = BlockState.parse(block) 59 | needles.add(state) 60 | needles.addAll(state.unflatten()) 61 | } 62 | for (item in options.valuesOf(itemArg)) { 63 | val itemType = ItemType.parse(item) 64 | needles.add(itemType) 65 | needles.addAll(itemType.unflatten()) 66 | } 67 | if (options.has(threadsArg)) threads = options.valueOf(threadsArg).toInt() 68 | if (options.has(decompressorArg)) DECOMPRESSOR = options.valueOf(decompressorArg) 69 | if (options.has(loopArg)) { 70 | loopCount = if (options.hasArgument(loopArg)) { 71 | options.valueOf(loopArg).toInt() 72 | } else { 73 | -1 74 | } 75 | } 76 | val paths = options.valuesOf(nonOptions).toMutableList() 77 | script = when { 78 | options.has(statsArg) -> builtinScript("stats") 79 | options.has(geode) -> builtinScript("geode") 80 | paths.isNotEmpty() && paths[0].endsWith(".scan.kts") -> Paths.get(paths.removeAt(0)) 81 | else -> builtinScript("search") 82 | } 83 | if (paths.size > 2 || paths.isEmpty()) throw IllegalArgumentException("Expected 1 or 2 paths") 84 | path = Paths.get(paths[0]) 85 | if (paths.size == 1) { 86 | outPath = Paths.get("") 87 | zip = null 88 | } else { 89 | val out = Paths.get(paths[1]) 90 | if (Files.exists(out) && Files.isDirectory(out)) { 91 | outPath = out 92 | zip = null 93 | } else if (!out.fileName.toString().endsWith(".zip")) { 94 | Files.createDirectories(out) 95 | outPath = out 96 | zip = null 97 | } else { 98 | val uri = out.toUri() 99 | val fsUri = URI("jar:${uri.scheme}", uri.userInfo, uri.host, uri.port, uri.path, uri.query, uri.fragment) 100 | zip = FileSystems.newFileSystem(fsUri, mapOf("create" to "true")) 101 | outPath = zip.getPath("/") 102 | } 103 | } 104 | } catch (e: RuntimeException) { 105 | if (e is OptionException || e is IllegalArgumentException) { 106 | System.err.println(e.message) 107 | } else { 108 | e.printStackTrace() 109 | } 110 | println() 111 | printUsage() 112 | return 113 | } 114 | val executor = when { 115 | threads <= 0 -> Executors.newWorkStealingPool() 116 | threads == 1 -> Executors.newSingleThreadExecutor() 117 | else -> Executors.newWorkStealingPool(threads) 118 | } 119 | do { 120 | runScript(path, outPath, executor, needles, script) 121 | } while (loopCount == -1 || loopCount-- > 0) 122 | zip?.close() 123 | executor.shutdownNow() 124 | } 125 | 126 | fun getHaystack(path: Path): Set { 127 | val haystack = mutableSetOf() 128 | if (Files.isRegularFile(path) && path.fileName.toString().endsWith(".mca")) { 129 | return setOf(RegionFile(path)) 130 | } 131 | val playerDataPath = path.resolve("playerdata") 132 | if (Files.exists(playerDataPath)) { 133 | Files.list(playerDataPath).forEach { 134 | val fileName = it.fileName.toString() 135 | if (fileName.endsWith(".dat") && fileName.split('-').size == 5) { 136 | try { 137 | haystack.add(PlayerFile(it)) 138 | } catch (e: Exception) { 139 | e.printStackTrace() 140 | } 141 | } 142 | } 143 | } 144 | for (dim in listOf(".", "DIM-1", "DIM1")) { 145 | val dimPath = path.resolve(dim) 146 | if (!Files.exists(dimPath)) continue 147 | val dimRegionPath = dimPath.resolve("region") 148 | if (!Files.exists(dimRegionPath)) continue 149 | Files.list(dimRegionPath).forEach { 150 | if (it.fileName.toString().endsWith(".mca")) { 151 | try { 152 | haystack.add(RegionFile(it)) 153 | } catch (e: Exception) { 154 | e.printStackTrace() 155 | } 156 | } 157 | } 158 | } 159 | return haystack 160 | } 161 | 162 | fun builtinScript(name: String): Path { 163 | val url = ScannerScript::class.java.getResource("/scripts/$name.scan.kts") 164 | return Paths.get(url?.toURI() ?: throw IllegalArgumentException("Script not found: $name")) 165 | } 166 | 167 | fun evalScript(path: Path, scan: Scan): ResultWithDiagnostics { 168 | val source = Files.readAllBytes(path).toString(Charsets.UTF_8).toScriptSource(path.fileName.toString()) 169 | return BasicJvmScriptingHost().evalWithTemplate(source, evaluation = { 170 | constructorArgs(scan) 171 | }) 172 | } 173 | 174 | fun runScript(path: Path, outPath: Path, executor: ExecutorService, needles: List, script: Path) { 175 | val scan = Scan(outPath, needles) 176 | evalScript(script, scan).valueOrThrow() 177 | val haystack = getHaystack(path).filterTo(mutableSetOf(), scan.haystackPredicate) 178 | var totalSize = 0L 179 | for (s in haystack) totalSize += s.size 180 | val progressSize = AtomicLong() 181 | var speed = 0.0 182 | var resultCount = 0 183 | val lock = Object() 184 | fun printStatus(i: Int, current: Any? = null) { 185 | synchronized(lock) { 186 | print("\u001b[2K$i/${haystack.size} ") 187 | print("${formatSize(progressSize.toDouble())}/${formatSize(totalSize.toDouble())} ") 188 | print("${formatSize(speed)}/s $resultCount result${if (resultCount == 1) "" else "s"} ") 189 | if (current != null) print(current) 190 | print("\u001B[G") 191 | } 192 | } 193 | val index = AtomicInteger() 194 | printStatus(0) 195 | val before = System.nanoTime() 196 | val futures = haystack.map { CompletableFuture.runAsync({ 197 | val results = try { 198 | scan.scanner(it) 199 | } catch (e: Exception) { 200 | print("\u001b[2K") 201 | System.err.println("Error scanning $it") 202 | e.printStackTrace() 203 | println() 204 | return@runAsync 205 | } 206 | val time = System.nanoTime() - before 207 | val progressAfter = progressSize.addAndGet(it.size) 208 | speed = progressAfter * 1e9 / time 209 | synchronized(scan) { 210 | scan.onResults(results) 211 | resultCount += results.size 212 | } 213 | printStatus(index.incrementAndGet(), it) 214 | }, executor)} 215 | CompletableFuture.allOf(*futures.toTypedArray()).join() 216 | scan.postProcess() 217 | printStatus(haystack.size) 218 | println() 219 | 220 | } 221 | 222 | interface Scannable { 223 | val size: Long 224 | fun scan(needles: Collection, statsMode: Boolean): List 225 | } 226 | interface Location 227 | 228 | data class SubLocation(val parent: Location, val index: Int): Location 229 | data class ChunkPos(val dimension: String, val x: Int, val z: Int) : Location 230 | data class Container(val type: String, val location: Location) : Location 231 | data class Entity(val type: String, val location: Location) : Location 232 | data class BlockPos(val dimension: String, val x: Int, val y: Int, val z: Int) : Location 233 | data class Vec3d(val dimension: String, val x: Double, val y: Double, val z: Double) : Location 234 | data class PlayerInventory(val player: UUID, val enderChest: Boolean) : Location 235 | data class StatsResults(val types: Array, val matrix: DoubleArray): Needle { 236 | override fun equals(other: Any?): Boolean { 237 | if (this === other) return true 238 | if (javaClass != other?.javaClass) return false 239 | 240 | other as StatsResults 241 | 242 | if (!types.contentEquals(other.types)) return false 243 | if (!matrix.contentEquals(other.matrix)) return false 244 | 245 | return true 246 | } 247 | 248 | override fun hashCode(): Int { 249 | var result = types.contentHashCode() 250 | result = 31 * result + matrix.contentHashCode() 251 | return result 252 | } 253 | } 254 | 255 | data class SearchResult(val needle: Needle, val location: Location, val count: Long) 256 | 257 | fun addResults(results: MutableList, location: Location, contents: List>, statsMode: Boolean) { 258 | for (e in contents[0].object2LongEntrySet()) { 259 | results.add(SearchResult(e.key, location, e.longValue)) 260 | } 261 | if (statsMode) { 262 | results.add(SearchResult(tallyStats(contents[0]), location, 1)) 263 | for (i in 1 until contents.size) { 264 | val subLocation = SubLocation(location, i) 265 | for (e in contents[i].object2LongEntrySet()) { 266 | results.add(SearchResult(e.key, subLocation, e.longValue)) 267 | } 268 | } 269 | } 270 | } -------------------------------------------------------------------------------- /src/main/kotlin/de/skyrising/mc/scanner/script/script.kt: -------------------------------------------------------------------------------- 1 | package de.skyrising.mc.scanner.script 2 | 3 | import de.skyrising.mc.scanner.* 4 | import de.skyrising.mc.scanner.region.ChunkNode 5 | import java.nio.file.Path 6 | import kotlin.script.experimental.annotations.KotlinScript 7 | import kotlin.script.experimental.api.ScriptCompilationConfiguration 8 | import kotlin.script.experimental.api.defaultImports 9 | import kotlin.script.experimental.jvm.dependenciesFromCurrentContext 10 | import kotlin.script.experimental.jvm.jvm 11 | 12 | @KotlinScript( 13 | fileExtension = "scan.kts", 14 | compilationConfiguration = ScannerScriptConfiguration::class 15 | ) 16 | abstract class ScannerScript(val scan: Scan) : ScanBase by scan { 17 | fun collect(predicate: Scannable.() -> Boolean) { 18 | scan.haystackPredicate = predicate 19 | } 20 | 21 | fun scan(scanner: Scannable.() -> List) { 22 | scan.scanner = scanner 23 | } 24 | 25 | fun onResults(block: List.() -> Unit) { 26 | scan.onResults = block 27 | } 28 | 29 | fun after(block: () -> Unit) { 30 | scan.postProcess = block 31 | } 32 | } 33 | 34 | object ScannerScriptConfiguration : ScriptCompilationConfiguration({ 35 | defaultImports( 36 | Scannable::class, 37 | Location::class, 38 | SubLocation::class, 39 | ChunkPos::class, 40 | Container::class, 41 | Entity::class, 42 | BlockPos::class, 43 | Vec3d::class, 44 | PlayerInventory::class, 45 | StatsResults::class, 46 | SearchResult::class, 47 | Identifier::class, 48 | BlockState::class, 49 | BlockIdMask::class, 50 | ItemType::class, 51 | RegionFile::class, 52 | ChunkNode::class, 53 | ) 54 | jvm { 55 | dependenciesFromCurrentContext(wholeClasspath = true) 56 | } 57 | }) 58 | 59 | interface ScanBase { 60 | val outPath: Path 61 | var needles: List 62 | } 63 | 64 | class Scan(override val outPath: Path, override var needles: List) : ScanBase { 65 | var haystackPredicate: (Scannable) -> Boolean = { true } 66 | var scanner: (Scannable) -> List = { it.scan(needles, false) } 67 | var onResults: (List) -> Unit = {} 68 | var postProcess: () -> Unit = {} 69 | } -------------------------------------------------------------------------------- /src/main/kotlin/de/skyrising/mc/scanner/utils.kt: -------------------------------------------------------------------------------- 1 | package de.skyrising.mc.scanner 2 | 3 | import java.io.EOFException 4 | 5 | import java.nio.ByteBuffer 6 | 7 | import java.nio.channels.ReadableByteChannel 8 | import java.util.* 9 | 10 | data class Vec2i(val x: Int, val y: Int) 11 | data class Vec2d(val x: Double, val y: Double) 12 | 13 | fun readFully(channel: ReadableByteChannel, b: ByteBuffer) { 14 | val expectedLength: Int = b.remaining() 15 | var read = 0 16 | do { 17 | val readNow = channel.read(b) 18 | if (readNow <= 0) { 19 | break 20 | } 21 | read += readNow 22 | } while (read < expectedLength) 23 | if (read < expectedLength) { 24 | throw EOFException() 25 | } 26 | } 27 | 28 | fun formatSize(size: Double): String { 29 | val kib = size / 1024 30 | if (kib < 1024) return String.format(Locale.ROOT, "%.1fKiB", kib) 31 | val mib = kib / 1024 32 | if (mib < 1024) return String.format(Locale.ROOT, "%.1fMiB", mib) 33 | val gib = mib / 1024 34 | if (gib < 1024) return String.format(Locale.ROOT, "%.1fGiB", gib) 35 | return String.format(Locale.ROOT, "%.1fTiB", gib / 1024) 36 | } 37 | 38 | fun hexdump(buf: ByteBuffer, addr: Int = 0): String { 39 | val readerIndex = buf.position() 40 | val writerIndex = buf.limit() 41 | val line = ByteArray(16) 42 | val length = writerIndex - readerIndex 43 | if (length == 0) return "empty" 44 | var digits = 0 45 | for (i in 0..30) { 46 | if ((addr + length - 1) shr i == 0) { 47 | digits = i + 3 shr 2 48 | break 49 | } 50 | } 51 | var i = 0 52 | val sb = StringBuilder() 53 | sb.append(String.format("%0" + digits + "x: ", addr + i)) 54 | while (i < length) { 55 | val b = buf[readerIndex + i] 56 | if (i > 0 && i and 0xf == 0) { 57 | sb.append("| ") 58 | for (j in 0..15) { 59 | val c = line[j] 60 | sb.append(if (c < 0x20 || c >= 0x7f) '.' else c.toChar()) 61 | } 62 | sb.append('\n') 63 | sb.append(String.format("%0" + digits + "x: ", addr + i)) 64 | } 65 | sb.append(String.format("%02x ", b.toInt() and 0xff)) 66 | line[i and 0xf] = b 67 | i++ 68 | } 69 | if (i > 0) { 70 | i = (i - 1 and 0xf) + 1 71 | for (j in 16 downTo i + 1) sb.append(" ") 72 | sb.append("| ") 73 | for (j in 0 until i) { 74 | val c = line[j] 75 | sb.append(if (c < 0x20 || c >= 0x7f) '.' else c.toChar()) 76 | } 77 | sb.append('\n') 78 | } 79 | return sb.toString() 80 | } -------------------------------------------------------------------------------- /src/main/kotlin/de/skyrising/mc/scanner/zlib/api.kt: -------------------------------------------------------------------------------- 1 | package de.skyrising.mc.scanner.zlib 2 | 3 | import java.nio.ByteBuffer 4 | import java.nio.ByteOrder 5 | import java.util.zip.Adler32 6 | import java.util.zip.CRC32 7 | import java.util.zip.DataFormatException 8 | 9 | private const val VERIFY_CHECKSUM = false 10 | 11 | fun decodeZlib(buf: ByteBuffer, arr: ByteArray? = null): ByteBuffer { 12 | val cmf = buf.get().toUInt() and 0xffu 13 | val flg = buf.get().toUInt() and 0xffu 14 | if ((cmf * 256u + flg) % 31u != 0u) throw DataFormatException("Invalid ZLIB header checksum") 15 | val method = cmf and 0x0fu 16 | if (method != ZLIB_METHOD_DEFLATE) throw UnsupportedOperationException("ZLIB method $method not supported") 17 | val fdict = (flg and 0x20u != 0u) 18 | if (fdict) throw UnsupportedOperationException("ZLIB dictionary not supported") 19 | 20 | val result = inflate(buf, arr) 21 | 22 | buf.order(ByteOrder.BIG_ENDIAN) 23 | val checksum = buf.int 24 | if (VERIFY_CHECKSUM) { 25 | val resultChecksum = Adler32().apply { update(result.slice()) }.value.toInt() 26 | if (resultChecksum != checksum) throw DataFormatException("ZLIB checksum mismatch") 27 | } 28 | 29 | return result 30 | } 31 | 32 | fun decodeGzip(buf: ByteBuffer, arr: ByteArray? = null): ByteBuffer { 33 | buf.order(ByteOrder.LITTLE_ENDIAN) 34 | val magic = buf.short.toUInt() and 0xffffu 35 | if (magic != GZIP_MAGIC) throw DataFormatException("Invalid GZIP header") 36 | val method = buf.get().toUInt() and 0xffu 37 | if (method != GZIP_METHOD_DEFLATE) throw UnsupportedOperationException("GZIP method $method not supported") 38 | val flags = buf.get().toUInt() and 0xffu 39 | buf.position(buf.position() + 6) 40 | if ((flags and GZIP_FLAG_EXTRA) != 0u) { 41 | buf.position(buf.position() + buf.short.toInt() and 0xffff) 42 | } 43 | if ((flags and GZIP_FLAG_NAME) != 0u) { 44 | while (buf.get() != 0u.toByte()) {/**/} 45 | } 46 | if ((flags and GZIP_FLAG_COMMENT) != 0u) { 47 | while (buf.get() != 0u.toByte()) {/**/} 48 | } 49 | if ((flags and GZIP_FLAG_HCRC) != 0u) { 50 | buf.short 51 | } 52 | 53 | val result = inflate(buf, arr) 54 | 55 | buf.order(ByteOrder.LITTLE_ENDIAN) 56 | val checksum = buf.int 57 | if (VERIFY_CHECKSUM) { 58 | val resultChecksum = CRC32().apply { update(result.slice()) }.value.toInt() 59 | if (resultChecksum != checksum) throw DataFormatException("GZIP checksum mismatch") 60 | } 61 | val size = buf.int 62 | if (result.remaining() != size) throw DataFormatException("GZIP size mismatch") 63 | 64 | return result 65 | } 66 | 67 | fun inflate(buf: ByteBuffer, arr: ByteArray? = null) = inflate(BitStream(buf), arr ?: ByteArray(buf.remaining())) -------------------------------------------------------------------------------- /src/main/kotlin/de/skyrising/mc/scanner/zlib/bitstream.kt: -------------------------------------------------------------------------------- 1 | package de.skyrising.mc.scanner.zlib 2 | 3 | import java.io.EOFException 4 | import java.nio.ByteBuffer 5 | import java.nio.ByteOrder 6 | import java.util.zip.DataFormatException 7 | 8 | 9 | class BitStream(private val buf: ByteBuffer) { 10 | init { 11 | buf.order(ByteOrder.LITTLE_ENDIAN) 12 | } 13 | private var offset = buf.position() 14 | private var overrun = 0 15 | private var bitbuf: Long = 0 16 | private var bitsleft = 0 17 | 18 | private fun fillBitsSlow(bytes: Int) { 19 | for(i in 0 until bytes) { 20 | val off = offset++ 21 | val buf = this.buf 22 | val bl = bitsleft 23 | if (off < buf.limit()) { 24 | bitbuf = bitbuf or ((buf.get(off).toLong() and 0xffL) shl bl) 25 | } else { 26 | overrun++ 27 | offset-- 28 | } 29 | bitsleft = bl + 8 30 | } 31 | } 32 | 33 | private fun fillBitsFast(bytes: Int, off: Int, bl: Int) { 34 | bitbuf = bitbuf or (buf.getLong(off) shl bl) 35 | bitsleft = bl + (bytes shl 3) 36 | offset = off + bytes 37 | } 38 | 39 | private fun fillBits() { 40 | val bl = bitsleft 41 | val off = offset 42 | val desired = (64 - bl) shr 3 43 | if (buf.limit() - off >= 8) { 44 | fillBitsFast(desired, off, bl) 45 | } else { 46 | fillBitsSlow(desired) 47 | } 48 | } 49 | 50 | fun ensureBits(bits: Int) { 51 | if (bits > BITS_MAX) throw IllegalArgumentException() 52 | if (bitsleft < bits) fillBits() 53 | if (bitsleft < bits) throw EOFException() 54 | } 55 | 56 | fun align() { 57 | if (overrun > bitsleft / 8) throw DataFormatException() 58 | offset -= bitsleft / 8 - overrun 59 | overrun = 0 60 | bitbuf = 0 61 | bitsleft = 0 62 | } 63 | 64 | fun end() { 65 | align() 66 | buf.position(offset) 67 | } 68 | 69 | fun peekBits(n: Int): Int { 70 | ensureBits(n) 71 | return bitbuf.toInt() and ((1 shl n) - 1) 72 | } 73 | 74 | fun removeBits(n: Int) { 75 | bitbuf = bitbuf ushr n 76 | bitsleft -= n 77 | } 78 | 79 | fun popBits(n: Int): Int { 80 | val bits = peekBits(n) 81 | removeBits(n) 82 | return bits 83 | } 84 | 85 | fun readUnsignedShort(): Int { 86 | val value = buf.getShort(offset).toInt() and 0xffff 87 | offset += 2 88 | return value 89 | } 90 | 91 | fun copyBytes(b: ByteArray, off: Int, len: Int) { 92 | val p = buf.position() 93 | buf.position(offset) 94 | buf.get(b, off, len) 95 | buf.position(p) 96 | offset += len 97 | } 98 | } -------------------------------------------------------------------------------- /src/main/kotlin/de/skyrising/mc/scanner/zlib/constants.kt: -------------------------------------------------------------------------------- 1 | package de.skyrising.mc.scanner.zlib 2 | 3 | const val ZLIB_METHOD_DEFLATE = 8u 4 | const val GZIP_MAGIC = 0x8b1fu 5 | const val GZIP_FLAG_TEXT = 0x01u 6 | const val GZIP_FLAG_HCRC = 0x02u 7 | const val GZIP_FLAG_EXTRA = 0x04u 8 | const val GZIP_FLAG_NAME = 0x08u 9 | const val GZIP_FLAG_COMMENT = 0x10u 10 | const val GZIP_METHOD_DEFLATE = 8u 11 | 12 | const val DEFLATE_BLOCKTYPE_UNCOMPRESSED = 0 13 | const val DEFLATE_BLOCKTYPE_FIXED_HUFFMAN = 1 14 | const val DEFLATE_BLOCKTYPE_DYNAMIC_HUFFMAN = 2 15 | 16 | const val DEFLATE_CODE_LENGTHS_COPY: UShort = 16u 17 | const val DEFLATE_CODE_LENGTHS_REPEAT_3: UShort = 17u 18 | const val DEFLATE_CODE_LENGTHS_REPEAT_7: UShort = 18u 19 | 20 | const val BITS_MAX = 64 - 8 21 | 22 | 23 | // RFC 1951 - 3.2.5 24 | private data class PackedLengthSymbol(val symbol: UShort, val baseLength: UShort, val extraBits: UShort) 25 | private val PACKED_LENGTH_SYMBOLS = arrayOf( 26 | PackedLengthSymbol(257u, 3u, 0u), 27 | PackedLengthSymbol(258u, 4u, 0u), 28 | PackedLengthSymbol(259u, 5u, 0u), 29 | PackedLengthSymbol(260u, 6u, 0u), 30 | PackedLengthSymbol(261u, 7u, 0u), 31 | PackedLengthSymbol(262u, 8u, 0u), 32 | PackedLengthSymbol(263u, 9u, 0u), 33 | PackedLengthSymbol(264u, 10u, 0u), 34 | PackedLengthSymbol(265u, 11u, 1u), 35 | PackedLengthSymbol(266u, 13u, 1u), 36 | PackedLengthSymbol(267u, 15u, 1u), 37 | PackedLengthSymbol(268u, 17u, 1u), 38 | PackedLengthSymbol(269u, 19u, 1u), 39 | PackedLengthSymbol(270u, 23u, 2u), 40 | PackedLengthSymbol(271u, 27u, 2u), 41 | PackedLengthSymbol(272u, 31u, 2u), 42 | PackedLengthSymbol(273u, 35u, 3u), 43 | PackedLengthSymbol(274u, 43u, 3u), 44 | PackedLengthSymbol(275u, 51u, 3u), 45 | PackedLengthSymbol(276u, 59u, 4u), 46 | PackedLengthSymbol(277u, 67u, 4u), 47 | PackedLengthSymbol(278u, 83u, 4u), 48 | PackedLengthSymbol(279u, 99u, 5u), 49 | PackedLengthSymbol(280u, 115u, 5u), 50 | PackedLengthSymbol(281u, 131u, 5u), 51 | PackedLengthSymbol(282u, 163u, 5u), 52 | PackedLengthSymbol(283u, 195u, 5u), 53 | PackedLengthSymbol(284u, 227u, 5u), 54 | PackedLengthSymbol(285u, 258u, 0u) 55 | ) 56 | 57 | val LENGTH_TO_SYMBOL = generateLengthToSymbol() 58 | private fun generateLengthToSymbol(): UShortArray { 59 | val array = UShortArray(259) 60 | for (i in 0 until 3) array[i] = UShort.MAX_VALUE 61 | var baseLength = 0 62 | for (len in 3 until 259) { 63 | if (len == PACKED_LENGTH_SYMBOLS[baseLength + 1].baseLength.toInt()) { 64 | baseLength++ 65 | } 66 | array[len] = PACKED_LENGTH_SYMBOLS[baseLength].symbol 67 | } 68 | return array 69 | } 70 | 71 | // RFC 1951 - 3.2.5 72 | private data class PackedDistance(val symbol: UShort, val baseDistance: UShort, val extraBits: UShort) 73 | private val PACKED_DISTANCES = arrayOf( 74 | PackedDistance(0u, 1u, 0u), 75 | PackedDistance(1u, 2u, 0u), 76 | PackedDistance(2u, 3u, 0u), 77 | PackedDistance(3u, 4u, 0u), 78 | PackedDistance(4u, 5u, 1u), 79 | PackedDistance(5u, 7u, 1u), 80 | PackedDistance(6u, 9u, 2u), 81 | PackedDistance(7u, 13u, 2u), 82 | PackedDistance(8u, 17u, 3u), 83 | PackedDistance(9u, 25u, 3u), 84 | PackedDistance(10u, 33u, 4u), 85 | PackedDistance(11u, 49u, 4u), 86 | PackedDistance(12u, 65u, 5u), 87 | PackedDistance(13u, 97u, 5u), 88 | PackedDistance(14u, 129u, 6u), 89 | PackedDistance(15u, 193u, 6u), 90 | PackedDistance(16u, 257u, 7u), 91 | PackedDistance(17u, 385u, 7u), 92 | PackedDistance(18u, 513u, 8u), 93 | PackedDistance(19u, 769u, 8u), 94 | PackedDistance(20u, 1025u, 9u), 95 | PackedDistance(21u, 1537u, 9u), 96 | PackedDistance(22u, 2049u, 10u), 97 | PackedDistance(23u, 3073u, 10u), 98 | PackedDistance(24u, 4097u, 11u), 99 | PackedDistance(25u, 6145u, 11u), 100 | PackedDistance(26u, 8193u, 12u), 101 | PackedDistance(27u, 12289u, 12u), 102 | PackedDistance(28u, 16385u, 13u), 103 | PackedDistance(29u, 24577u, 13u), 104 | PackedDistance(30u, 32769u, 0u) 105 | ) 106 | 107 | // RFC 1951 - 3.2.6 108 | private data class FixedLiteralBits(val baseValue: UShort, val bits: UShort) 109 | private val FIXED_LITERAL_BITS_TABLE = arrayOf( 110 | FixedLiteralBits(0u, 8u), 111 | FixedLiteralBits(144u, 9u), 112 | FixedLiteralBits(256u, 7u), 113 | FixedLiteralBits(280u, 8u), 114 | FixedLiteralBits(288u, 0u) 115 | ) 116 | 117 | val FIXED_LITERAL_BIT_LENGTHS = generateFixedLiteralBitLengths() 118 | fun generateFixedLiteralBitLengths(): ByteArray { 119 | val lengths = ByteArray(288) 120 | for (i in 0 until 4) { 121 | val current = FIXED_LITERAL_BITS_TABLE[i] 122 | val next = FIXED_LITERAL_BITS_TABLE[i + 1] 123 | for (j in current.baseValue until next.baseValue) { 124 | lengths[j.toInt()] = current.bits.toByte() 125 | } 126 | } 127 | return lengths 128 | } 129 | 130 | val FIXED_DISTANCE_BIT_LENGTHS = ByteArray(32) { 5 } 131 | 132 | // RFC 1951 - 3.2.7 133 | val CODE_LENGTHS_CODE_LENGTHS_ORDER = byteArrayOf(16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15) -------------------------------------------------------------------------------- /src/main/kotlin/de/skyrising/mc/scanner/zlib/deflate.kt: -------------------------------------------------------------------------------- 1 | package de.skyrising.mc.scanner.zlib 2 | 3 | import java.nio.ByteBuffer 4 | import java.util.* 5 | import java.util.zip.DataFormatException 6 | 7 | private const val DEFLATE_DEBUG = false 8 | 9 | private fun readUncompressedBlockLength(stream: BitStream): Int { 10 | stream.align() 11 | val len = stream.readUnsignedShort() 12 | val nLen = stream.readUnsignedShort() 13 | if (len xor 0xffff != nLen) throw DataFormatException("invalid uncompressed block") 14 | return len 15 | } 16 | 17 | private fun decodeLength(symbol: Int, stream: BitStream) = when { 18 | symbol <= 264 -> symbol - 254 19 | symbol <= 284 -> { 20 | val extraBits = (symbol - 261) / 4 21 | (((symbol - 265) % 4 + 4) shl extraBits) + 3 + stream.popBits(extraBits) 22 | } 23 | symbol == 285 -> 258 24 | else -> throw DataFormatException("invalid length symbol") 25 | } 26 | 27 | fun decodeDistance(symbol: Int, stream: BitStream) = when { 28 | symbol <= 3 -> symbol + 1 29 | symbol <= 29 -> { 30 | val extraBits = symbol / 2 - 1 31 | ((symbol % 2 + 2) shl extraBits) + 1 + stream.popBits(extraBits) 32 | } 33 | else -> throw DataFormatException("invalid distance symbol $symbol") 34 | } 35 | 36 | private fun readCompressedBlock(stream: BitStream, literalCode: HuffmanDecoder, distanceCode: HuffmanDecoder?, out: OutputBuffer) { 37 | if (DEFLATE_DEBUG) println("compressed $literalCode $distanceCode") 38 | while (true) { 39 | val symbol = literalCode.decode(stream).toInt() 40 | when { 41 | symbol >= 286 -> throw DataFormatException("invalid symbol") 42 | symbol < 256 -> { 43 | if (DEFLATE_DEBUG) print("literal '") 44 | out.write(symbol.toUByte()) 45 | if (DEFLATE_DEBUG) println("'") 46 | continue 47 | } 48 | symbol == 256 -> return 49 | else -> { 50 | if (distanceCode == null) throw DataFormatException("invalid symbol") 51 | val length = decodeLength(symbol, stream) 52 | val distanceSymbol = distanceCode.decode(stream).toInt() 53 | if (distanceSymbol >= 30) throw DataFormatException("invalid distance symbol") 54 | val distance = decodeDistance(distanceSymbol, stream) 55 | if (DEFLATE_DEBUG) print("backref -$distance,$length '") 56 | out.outputBackref(distance, length) 57 | if (DEFLATE_DEBUG) println("'") 58 | } 59 | } 60 | } 61 | } 62 | 63 | private fun decodeCodes(state: InflatorState, stream: BitStream) { 64 | val literalCodeCount = stream.popBits(5) + 257 65 | val distanceCodeCount = stream.popBits(5) + 1 66 | val codeLengthCount = stream.popBits(4) + 4 67 | 68 | val codeLengthsCodeLengths = state.codeLengthsCodeLengths 69 | Arrays.fill(codeLengthsCodeLengths, 0) 70 | for (i in 0 until codeLengthCount) { 71 | codeLengthsCodeLengths[CODE_LENGTHS_CODE_LENGTHS_ORDER[i].toInt()] = stream.popBits(3).toByte() 72 | } 73 | 74 | val codeLengthCode = HuffmanDecoder.fromBytes(codeLengthsCodeLengths) ?: throw DataFormatException("invalid code lengths code") 75 | 76 | val codeLengths = ByteArray(literalCodeCount + distanceCodeCount) 77 | var i = 0 78 | while (i < codeLengths.size) { 79 | val symbol = codeLengthCode.decode(stream) 80 | if (symbol > UShort.MAX_VALUE) throw DataFormatException("invalid code length") 81 | when (symbol.toUShort()) { 82 | DEFLATE_CODE_LENGTHS_COPY -> { 83 | val repeat = stream.popBits(2) + 3 84 | val value = codeLengths[i - 1] 85 | for (j in 0 until repeat) { 86 | codeLengths[i++] = value 87 | } 88 | } 89 | DEFLATE_CODE_LENGTHS_REPEAT_3 -> { 90 | for (j in 0 until stream.popBits(3) + 3) codeLengths[i++] = 0 91 | } 92 | DEFLATE_CODE_LENGTHS_REPEAT_7 -> { 93 | for (j in 0 until stream.popBits(7) + 11) codeLengths[i++] = 0 94 | } 95 | else -> { 96 | codeLengths[i++] = symbol.toByte() 97 | } 98 | } 99 | } 100 | 101 | state.literalCode = HuffmanDecoder.fromBytes(codeLengths, 0, literalCodeCount) ?: throw DataFormatException("invalid literal code") 102 | 103 | if (distanceCodeCount == 1) { 104 | val length = codeLengths[literalCodeCount].toInt() 105 | if (length == 0) { 106 | state.distanceCode = null 107 | return 108 | } 109 | if (length != 1) throw DataFormatException("invalid distance code") 110 | } 111 | 112 | state.distanceCode = HuffmanDecoder.fromBytes(codeLengths, literalCodeCount, distanceCodeCount) ?: throw DataFormatException("invalid distance code") 113 | } 114 | 115 | private fun updateCodes(state: InflatorState, type: Int, stream: BitStream) = when (type) { 116 | DEFLATE_BLOCKTYPE_FIXED_HUFFMAN -> { 117 | state.literalCode = HuffmanDecoder.FIXED_LITERAL_CODES 118 | state.distanceCode = HuffmanDecoder.FIXED_DISTANCE_CODES 119 | } 120 | DEFLATE_BLOCKTYPE_DYNAMIC_HUFFMAN -> { 121 | decodeCodes(state, stream) 122 | } 123 | else -> throw DataFormatException("invalid block type") 124 | } 125 | 126 | private fun printChar(c: Char) { 127 | if (c in ' '..'~') { 128 | print(c) 129 | } else { 130 | print("\\x%02x".format(c.code and 0xff)) 131 | } 132 | } 133 | 134 | class OutputBuffer(internal var arr: ByteArray) { 135 | internal var pos = 0 136 | 137 | fun ensureSpace(space: Int) { 138 | if (pos + space > arr.size) { 139 | expand(space) 140 | } 141 | } 142 | 143 | private fun expand(space: Int) { 144 | var newSize = arr.size 145 | while (pos + space > newSize) newSize *= 2 146 | arr = arr.copyOf(newSize) 147 | } 148 | 149 | fun write(b: UByte) { 150 | ensureSpace(1) 151 | arr[pos++] = b.toByte() 152 | if (DEFLATE_DEBUG) { 153 | printChar(b.toInt().toChar()) 154 | } 155 | } 156 | 157 | fun outputBackref(distance: Int, length: Int) { 158 | ensureSpace(length) 159 | outputBackref(arr, pos, distance, length) 160 | if (DEFLATE_DEBUG) { 161 | for (i in 0 until length) { 162 | printChar(arr[pos + i].toInt().toChar()) 163 | } 164 | } 165 | pos += length 166 | } 167 | 168 | fun buffer(): ByteBuffer { 169 | return ByteBuffer.wrap(arr, 0, pos) 170 | } 171 | } 172 | 173 | private fun outputBackref(arr: ByteArray, pos: Int, distance: Int, length: Int) { 174 | if (distance == 1) { 175 | Arrays.fill(arr, pos, pos + length, arr[pos - 1]) 176 | } else if (length > distance) { 177 | outputBackrefBytewise(arr, pos, distance, length) 178 | } else { 179 | System.arraycopy(arr, pos - distance, arr, pos, length) 180 | } 181 | } 182 | 183 | private fun outputBackrefBytewise(arr: ByteArray, pos: Int, distance: Int, length: Int) { 184 | val srcPos = pos - distance 185 | for (i in 0 until length) { 186 | arr[pos + i] = arr[srcPos + i] 187 | } 188 | } 189 | 190 | private class InflatorState { 191 | var literalCode = HuffmanDecoder.FIXED_LITERAL_CODES 192 | var distanceCode: HuffmanDecoder? = HuffmanDecoder.FIXED_DISTANCE_CODES 193 | val codeLengthsCodeLengths = ByteArray(19) 194 | } 195 | 196 | fun inflate(stream: BitStream, arr: ByteArray): ByteBuffer { 197 | val state = InflatorState() 198 | val out = OutputBuffer(arr) 199 | var finalBlock = false 200 | while (!finalBlock) { 201 | stream.ensureBits(3) 202 | finalBlock = stream.popBits(1) != 0 203 | when (val type = stream.popBits(2)) { 204 | DEFLATE_BLOCKTYPE_UNCOMPRESSED -> { 205 | val len = readUncompressedBlockLength(stream) 206 | out.ensureSpace(len) 207 | stream.copyBytes(out.arr, out.pos, len) 208 | if (DEFLATE_DEBUG) { 209 | print("uncompressed $len '") 210 | for (i in out.pos until out.pos + len) { 211 | val c = out.arr[i].toInt().toChar() 212 | if (c in ' '..'~') { 213 | print(c) 214 | } else { 215 | print("\\u%04x".format(c.toInt())) 216 | } 217 | } 218 | println("'") 219 | } 220 | out.pos += len 221 | } 222 | DEFLATE_BLOCKTYPE_FIXED_HUFFMAN, DEFLATE_BLOCKTYPE_DYNAMIC_HUFFMAN -> { 223 | updateCodes(state, type, stream) 224 | readCompressedBlock(stream, state.literalCode, state.distanceCode, out) 225 | } 226 | else -> throw DataFormatException() 227 | } 228 | } 229 | stream.end() 230 | return out.buffer() 231 | } -------------------------------------------------------------------------------- /src/main/kotlin/de/skyrising/mc/scanner/zlib/huffman.kt: -------------------------------------------------------------------------------- 1 | @file:OptIn(ExperimentalUnsignedTypes::class) 2 | 3 | package de.skyrising.mc.scanner.zlib 4 | 5 | interface HuffmanDecoder { 6 | fun decode(stream: BitStream): UInt 7 | 8 | companion object { 9 | val FIXED_LITERAL_CODES = fromBytes(FIXED_LITERAL_BIT_LENGTHS)!! 10 | val FIXED_DISTANCE_CODES = fromBytes(FIXED_DISTANCE_BIT_LENGTHS)!! 11 | 12 | fun fromBytes(buf: ByteArray) = fromBytes(buf, 0, buf.size) 13 | fun fromBytes(buf: ByteArray, off: Int, len: Int): HuffmanDecoder? = SmartHuffmanDecoder.fromBytes(buf, off, len) 14 | } 15 | } 16 | 17 | const val HUFFMAN_LUT_BITS = 9 18 | const val HUFFMAN_LUT_MASK = (1 shl HUFFMAN_LUT_BITS) - 1 19 | const val HUFFMAN_LUT_SYM_BITS = 9 20 | const val HUFFMAN_LUT_SYM_SHIFT = 0 21 | const val HUFFMAN_LUT_SYM_MASK = ((1 shl HUFFMAN_LUT_SYM_BITS) - 1) shl HUFFMAN_LUT_SYM_SHIFT 22 | const val HUFFMAN_LUT_LEN_BITS = 7 23 | const val HUFFMAN_LUT_LEN_SHIFT = HUFFMAN_LUT_SYM_SHIFT + HUFFMAN_LUT_SYM_BITS 24 | const val HUFFMAN_LUT_LEN_MASK = ((1 shl HUFFMAN_LUT_LEN_BITS) - 1) shl HUFFMAN_LUT_LEN_SHIFT 25 | 26 | const val MAX_HUFFMAN_BITS = 16 27 | const val MAX_HUFFMAN_SYMBOLS = 288 28 | 29 | class SmartHuffmanDecoder( 30 | private val lookupTable: UShortArray, 31 | private val sentinelBits: UIntArray, 32 | private val offsetFirstSymbolIndex: UIntArray, 33 | private val symbols: UShortArray 34 | ) : HuffmanDecoder { 35 | constructor() : this( 36 | UShortArray(1 shl HUFFMAN_LUT_BITS), 37 | UIntArray(MAX_HUFFMAN_BITS + 1), 38 | UIntArray(MAX_HUFFMAN_BITS + 1), 39 | UShortArray(MAX_HUFFMAN_SYMBOLS) 40 | ) 41 | override fun decode(stream: BitStream): UInt { 42 | var bits = stream.peekBits(16) 43 | val lookupBits = bits and HUFFMAN_LUT_MASK 44 | val lutEntry = lookupTable[lookupBits].toInt() 45 | if (lutEntry and HUFFMAN_LUT_LEN_MASK != 0) { 46 | val len = (lutEntry and HUFFMAN_LUT_LEN_MASK) ushr HUFFMAN_LUT_LEN_SHIFT 47 | val sym = (lutEntry and HUFFMAN_LUT_SYM_MASK) ushr HUFFMAN_LUT_SYM_SHIFT 48 | stream.removeBits(len) 49 | return sym.toUInt() 50 | } 51 | bits = Integer.reverse(bits) ushr (32 - MAX_HUFFMAN_BITS) 52 | for (l in HUFFMAN_LUT_BITS + 1 .. MAX_HUFFMAN_BITS) { 53 | if (bits >= sentinelBits[l].toInt()) continue 54 | bits = bits ushr (MAX_HUFFMAN_BITS - l) 55 | val symIdx = (offsetFirstSymbolIndex[l].toInt() + bits) and 0xffff 56 | stream.removeBits(l) 57 | return symbols[symIdx].toUInt() 58 | } 59 | return UInt.MAX_VALUE 60 | } 61 | 62 | override fun toString(): String { 63 | val sb = StringBuilder("HuffmanDecoder([") 64 | for (i in lookupTable.indices) { 65 | if (i > 0) sb.append(", ") 66 | val lutEntry = lookupTable[i].toInt() 67 | val len = (lutEntry and HUFFMAN_LUT_LEN_MASK) ushr HUFFMAN_LUT_LEN_SHIFT 68 | val sym = (lutEntry and HUFFMAN_LUT_SYM_MASK) ushr HUFFMAN_LUT_SYM_SHIFT 69 | sb.append('(').append(len).append(',').append(sym).append(')') 70 | } 71 | sb.append("], ").append(sentinelBits.contentToString()) 72 | .append(", ").append(offsetFirstSymbolIndex.contentToString()) 73 | .append(", ").append(symbols.contentToString()) 74 | .append(')') 75 | return sb.toString() 76 | } 77 | 78 | companion object { 79 | private fun count(buf: ByteArray, off: Int, len: Int): UShortArray { 80 | val count = UShortArray(MAX_HUFFMAN_BITS + 1) 81 | for (i in 0 until len) { 82 | count[buf[off + i].toInt()]++ 83 | } 84 | count[0] = 0u 85 | return count 86 | } 87 | 88 | fun fromBytes(buf: ByteArray, off: Int, len: Int): SmartHuffmanDecoder? { 89 | val d = SmartHuffmanDecoder() 90 | 91 | val count = count(buf, off, len) 92 | val code = UShortArray(MAX_HUFFMAN_BITS + 1) 93 | val symbolIndex = UShortArray(MAX_HUFFMAN_BITS + 1) 94 | 95 | for (l in 1..MAX_HUFFMAN_BITS) { 96 | val codePlusCountPrev = code[l - 1] + count[l - 1] 97 | code[l] = (codePlusCountPrev shl 1).toUShort() 98 | val codePlusCount = code[l] + count[l] 99 | if (count[l] != 0u.toUShort() && codePlusCount > 1u shl l) return null 100 | d.sentinelBits[l] = codePlusCount shl (MAX_HUFFMAN_BITS - l) 101 | 102 | symbolIndex[l] = (symbolIndex[l - 1] + count[l - 1]).toUShort() 103 | d.offsetFirstSymbolIndex[l] = symbolIndex[l] - code[l] 104 | } 105 | 106 | for (i in 0 until len) { 107 | val l = buf[off + i].toInt() 108 | if (l == 0) continue 109 | d.symbols[symbolIndex[l].toInt()] = i.toUShort() 110 | symbolIndex[l]++ 111 | if (l <= HUFFMAN_LUT_BITS) { 112 | tableInsert(d.lookupTable, i, l, code[l]) 113 | code[l]++ 114 | } 115 | } 116 | return d 117 | } 118 | 119 | private fun tableInsert(lookupTable: UShortArray, symbol: Int, length: Int, code: UShort) { 120 | val reverseCode = Integer.reverse(code.toInt()) ushr (32 - length) 121 | val padLen = HUFFMAN_LUT_BITS - length 122 | for (padding in 0 until (1 shl padLen)) { 123 | val index = reverseCode or (padding shl length) 124 | lookupTable[index] = ((length shl HUFFMAN_LUT_LEN_SHIFT) or (symbol shl HUFFMAN_LUT_SYM_SHIFT)).toUShort() 125 | } 126 | } 127 | } 128 | } -------------------------------------------------------------------------------- /src/main/resources/META-INF/kotlin/script/templates/de.skyrising.mc.scanner.script.ScannerScript.classname: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SciCraft/mc-scanner/e29653496645c35df7f67849eeb89ba63b4dd453/src/main/resources/META-INF/kotlin/script/templates/de.skyrising.mc.scanner.script.ScannerScript.classname -------------------------------------------------------------------------------- /src/main/resources/flattening/items.json: -------------------------------------------------------------------------------- 1 | { 2 | "minecraft:stone.0": "minecraft:stone", 3 | "minecraft:stone.1": "minecraft:granite", 4 | "minecraft:stone.2": "minecraft:polished_granite", 5 | "minecraft:stone.3": "minecraft:diorite", 6 | "minecraft:stone.4": "minecraft:polished_diorite", 7 | "minecraft:stone.5": "minecraft:andesite", 8 | "minecraft:stone.6": "minecraft:polished_andesite", 9 | "minecraft:dirt.0": "minecraft:dirt", 10 | "minecraft:dirt.1": "minecraft:coarse_dirt", 11 | "minecraft:dirt.2": "minecraft:podzol", 12 | "minecraft:leaves.0": "minecraft:oak_leaves", 13 | "minecraft:leaves.1": "minecraft:spruce_leaves", 14 | "minecraft:leaves.2": "minecraft:birch_leaves", 15 | "minecraft:leaves.3": "minecraft:jungle_leaves", 16 | "minecraft:leaves2.0": "minecraft:acacia_leaves", 17 | "minecraft:leaves2.1": "minecraft:dark_oak_leaves", 18 | "minecraft:log.0": "minecraft:oak_log", 19 | "minecraft:log.1": "minecraft:spruce_log", 20 | "minecraft:log.2": "minecraft:birch_log", 21 | "minecraft:log.3": "minecraft:jungle_log", 22 | "minecraft:log2.0": "minecraft:acacia_log", 23 | "minecraft:log2.1": "minecraft:dark_oak_log", 24 | "minecraft:sapling.0": "minecraft:oak_sapling", 25 | "minecraft:sapling.1": "minecraft:spruce_sapling", 26 | "minecraft:sapling.2": "minecraft:birch_sapling", 27 | "minecraft:sapling.3": "minecraft:jungle_sapling", 28 | "minecraft:sapling.4": "minecraft:acacia_sapling", 29 | "minecraft:sapling.5": "minecraft:dark_oak_sapling", 30 | "minecraft:planks.0": "minecraft:oak_planks", 31 | "minecraft:planks.1": "minecraft:spruce_planks", 32 | "minecraft:planks.2": "minecraft:birch_planks", 33 | "minecraft:planks.3": "minecraft:jungle_planks", 34 | "minecraft:planks.4": "minecraft:acacia_planks", 35 | "minecraft:planks.5": "minecraft:dark_oak_planks", 36 | "minecraft:sand.0": "minecraft:sand", 37 | "minecraft:sand.1": "minecraft:red_sand", 38 | "minecraft:quartz_block.0": "minecraft:quartz_block", 39 | "minecraft:quartz_block.1": "minecraft:chiseled_quartz_block", 40 | "minecraft:quartz_block.2": "minecraft:quartz_pillar", 41 | "minecraft:anvil.0": "minecraft:anvil", 42 | "minecraft:anvil.1": "minecraft:chipped_anvil", 43 | "minecraft:anvil.2": "minecraft:damaged_anvil", 44 | "minecraft:wool.0": "minecraft:white_wool", 45 | "minecraft:wool.1": "minecraft:orange_wool", 46 | "minecraft:wool.2": "minecraft:magenta_wool", 47 | "minecraft:wool.3": "minecraft:light_blue_wool", 48 | "minecraft:wool.4": "minecraft:yellow_wool", 49 | "minecraft:wool.5": "minecraft:lime_wool", 50 | "minecraft:wool.6": "minecraft:pink_wool", 51 | "minecraft:wool.7": "minecraft:gray_wool", 52 | "minecraft:wool.8": "minecraft:light_gray_wool", 53 | "minecraft:wool.9": "minecraft:cyan_wool", 54 | "minecraft:wool.10": "minecraft:purple_wool", 55 | "minecraft:wool.11": "minecraft:blue_wool", 56 | "minecraft:wool.12": "minecraft:brown_wool", 57 | "minecraft:wool.13": "minecraft:green_wool", 58 | "minecraft:wool.14": "minecraft:red_wool", 59 | "minecraft:wool.15": "minecraft:black_wool", 60 | "minecraft:carpet.0": "minecraft:white_carpet", 61 | "minecraft:carpet.1": "minecraft:orange_carpet", 62 | "minecraft:carpet.2": "minecraft:magenta_carpet", 63 | "minecraft:carpet.3": "minecraft:light_blue_carpet", 64 | "minecraft:carpet.4": "minecraft:yellow_carpet", 65 | "minecraft:carpet.5": "minecraft:lime_carpet", 66 | "minecraft:carpet.6": "minecraft:pink_carpet", 67 | "minecraft:carpet.7": "minecraft:gray_carpet", 68 | "minecraft:carpet.8": "minecraft:light_gray_carpet", 69 | "minecraft:carpet.9": "minecraft:cyan_carpet", 70 | "minecraft:carpet.10": "minecraft:purple_carpet", 71 | "minecraft:carpet.11": "minecraft:blue_carpet", 72 | "minecraft:carpet.12": "minecraft:brown_carpet", 73 | "minecraft:carpet.13": "minecraft:green_carpet", 74 | "minecraft:carpet.14": "minecraft:red_carpet", 75 | "minecraft:carpet.15": "minecraft:black_carpet", 76 | "minecraft:hardened_clay.0": "minecraft:terracotta", 77 | "minecraft:stained_hardened_clay.0": "minecraft:white_terracotta", 78 | "minecraft:stained_hardened_clay.1": "minecraft:orange_terracotta", 79 | "minecraft:stained_hardened_clay.2": "minecraft:magenta_terracotta", 80 | "minecraft:stained_hardened_clay.3": "minecraft:light_blue_terracotta", 81 | "minecraft:stained_hardened_clay.4": "minecraft:yellow_terracotta", 82 | "minecraft:stained_hardened_clay.5": "minecraft:lime_terracotta", 83 | "minecraft:stained_hardened_clay.6": "minecraft:pink_terracotta", 84 | "minecraft:stained_hardened_clay.7": "minecraft:gray_terracotta", 85 | "minecraft:stained_hardened_clay.8": "minecraft:light_gray_terracotta", 86 | "minecraft:stained_hardened_clay.9": "minecraft:cyan_terracotta", 87 | "minecraft:stained_hardened_clay.10": "minecraft:purple_terracotta", 88 | "minecraft:stained_hardened_clay.11": "minecraft:blue_terracotta", 89 | "minecraft:stained_hardened_clay.12": "minecraft:brown_terracotta", 90 | "minecraft:stained_hardened_clay.13": "minecraft:green_terracotta", 91 | "minecraft:stained_hardened_clay.14": "minecraft:red_terracotta", 92 | "minecraft:stained_hardened_clay.15": "minecraft:black_terracotta", 93 | "minecraft:silver_glazed_terracotta.0": "minecraft:light_gray_glazed_terracotta", 94 | "minecraft:stained_glass.0": "minecraft:white_stained_glass", 95 | "minecraft:stained_glass.1": "minecraft:orange_stained_glass", 96 | "minecraft:stained_glass.2": "minecraft:magenta_stained_glass", 97 | "minecraft:stained_glass.3": "minecraft:light_blue_stained_glass", 98 | "minecraft:stained_glass.4": "minecraft:yellow_stained_glass", 99 | "minecraft:stained_glass.5": "minecraft:lime_stained_glass", 100 | "minecraft:stained_glass.6": "minecraft:pink_stained_glass", 101 | "minecraft:stained_glass.7": "minecraft:gray_stained_glass", 102 | "minecraft:stained_glass.8": "minecraft:light_gray_stained_glass", 103 | "minecraft:stained_glass.9": "minecraft:cyan_stained_glass", 104 | "minecraft:stained_glass.10": "minecraft:purple_stained_glass", 105 | "minecraft:stained_glass.11": "minecraft:blue_stained_glass", 106 | "minecraft:stained_glass.12": "minecraft:brown_stained_glass", 107 | "minecraft:stained_glass.13": "minecraft:green_stained_glass", 108 | "minecraft:stained_glass.14": "minecraft:red_stained_glass", 109 | "minecraft:stained_glass.15": "minecraft:black_stained_glass", 110 | "minecraft:stained_glass_pane.0": "minecraft:white_stained_glass_pane", 111 | "minecraft:stained_glass_pane.1": "minecraft:orange_stained_glass_pane", 112 | "minecraft:stained_glass_pane.2": "minecraft:magenta_stained_glass_pane", 113 | "minecraft:stained_glass_pane.3": "minecraft:light_blue_stained_glass_pane", 114 | "minecraft:stained_glass_pane.4": "minecraft:yellow_stained_glass_pane", 115 | "minecraft:stained_glass_pane.5": "minecraft:lime_stained_glass_pane", 116 | "minecraft:stained_glass_pane.6": "minecraft:pink_stained_glass_pane", 117 | "minecraft:stained_glass_pane.7": "minecraft:gray_stained_glass_pane", 118 | "minecraft:stained_glass_pane.8": "minecraft:light_gray_stained_glass_pane", 119 | "minecraft:stained_glass_pane.9": "minecraft:cyan_stained_glass_pane", 120 | "minecraft:stained_glass_pane.10": "minecraft:purple_stained_glass_pane", 121 | "minecraft:stained_glass_pane.11": "minecraft:blue_stained_glass_pane", 122 | "minecraft:stained_glass_pane.12": "minecraft:brown_stained_glass_pane", 123 | "minecraft:stained_glass_pane.13": "minecraft:green_stained_glass_pane", 124 | "minecraft:stained_glass_pane.14": "minecraft:red_stained_glass_pane", 125 | "minecraft:stained_glass_pane.15": "minecraft:black_stained_glass_pane", 126 | "minecraft:prismarine.0": "minecraft:prismarine", 127 | "minecraft:prismarine.1": "minecraft:prismarine_bricks", 128 | "minecraft:prismarine.2": "minecraft:dark_prismarine", 129 | "minecraft:concrete.0": "minecraft:white_concrete", 130 | "minecraft:concrete.1": "minecraft:orange_concrete", 131 | "minecraft:concrete.2": "minecraft:magenta_concrete", 132 | "minecraft:concrete.3": "minecraft:light_blue_concrete", 133 | "minecraft:concrete.4": "minecraft:yellow_concrete", 134 | "minecraft:concrete.5": "minecraft:lime_concrete", 135 | "minecraft:concrete.6": "minecraft:pink_concrete", 136 | "minecraft:concrete.7": "minecraft:gray_concrete", 137 | "minecraft:concrete.8": "minecraft:light_gray_concrete", 138 | "minecraft:concrete.9": "minecraft:cyan_concrete", 139 | "minecraft:concrete.10": "minecraft:purple_concrete", 140 | "minecraft:concrete.11": "minecraft:blue_concrete", 141 | "minecraft:concrete.12": "minecraft:brown_concrete", 142 | "minecraft:concrete.13": "minecraft:green_concrete", 143 | "minecraft:concrete.14": "minecraft:red_concrete", 144 | "minecraft:concrete.15": "minecraft:black_concrete", 145 | "minecraft:concrete_powder.0": "minecraft:white_concrete_powder", 146 | "minecraft:concrete_powder.1": "minecraft:orange_concrete_powder", 147 | "minecraft:concrete_powder.2": "minecraft:magenta_concrete_powder", 148 | "minecraft:concrete_powder.3": "minecraft:light_blue_concrete_powder", 149 | "minecraft:concrete_powder.4": "minecraft:yellow_concrete_powder", 150 | "minecraft:concrete_powder.5": "minecraft:lime_concrete_powder", 151 | "minecraft:concrete_powder.6": "minecraft:pink_concrete_powder", 152 | "minecraft:concrete_powder.7": "minecraft:gray_concrete_powder", 153 | "minecraft:concrete_powder.8": "minecraft:light_gray_concrete_powder", 154 | "minecraft:concrete_powder.9": "minecraft:cyan_concrete_powder", 155 | "minecraft:concrete_powder.10": "minecraft:purple_concrete_powder", 156 | "minecraft:concrete_powder.11": "minecraft:blue_concrete_powder", 157 | "minecraft:concrete_powder.12": "minecraft:brown_concrete_powder", 158 | "minecraft:concrete_powder.13": "minecraft:green_concrete_powder", 159 | "minecraft:concrete_powder.14": "minecraft:red_concrete_powder", 160 | "minecraft:concrete_powder.15": "minecraft:black_concrete_powder", 161 | "minecraft:cobblestone_wall.0": "minecraft:cobblestone_wall", 162 | "minecraft:cobblestone_wall.1": "minecraft:mossy_cobblestone_wall", 163 | "minecraft:sandstone.0": "minecraft:sandstone", 164 | "minecraft:sandstone.1": "minecraft:chiseled_sandstone", 165 | "minecraft:sandstone.2": "minecraft:cut_sandstone", 166 | "minecraft:red_sandstone.0": "minecraft:red_sandstone", 167 | "minecraft:red_sandstone.1": "minecraft:chiseled_red_sandstone", 168 | "minecraft:red_sandstone.2": "minecraft:cut_red_sandstone", 169 | "minecraft:stonebrick.0": "minecraft:stone_bricks", 170 | "minecraft:stonebrick.1": "minecraft:mossy_stone_bricks", 171 | "minecraft:stonebrick.2": "minecraft:cracked_stone_bricks", 172 | "minecraft:stonebrick.3": "minecraft:chiseled_stone_bricks", 173 | "minecraft:monster_egg.0": "minecraft:infested_stone", 174 | "minecraft:monster_egg.1": "minecraft:infested_cobblestone", 175 | "minecraft:monster_egg.2": "minecraft:infested_stone_bricks", 176 | "minecraft:monster_egg.3": "minecraft:infested_mossy_stone_bricks", 177 | "minecraft:monster_egg.4": "minecraft:infested_cracked_stone_bricks", 178 | "minecraft:monster_egg.5": "minecraft:infested_chiseled_stone_bricks", 179 | "minecraft:yellow_flower.0": "minecraft:dandelion", 180 | "minecraft:red_flower.0": "minecraft:poppy", 181 | "minecraft:red_flower.1": "minecraft:blue_orchid", 182 | "minecraft:red_flower.2": "minecraft:allium", 183 | "minecraft:red_flower.3": "minecraft:azure_bluet", 184 | "minecraft:red_flower.4": "minecraft:red_tulip", 185 | "minecraft:red_flower.5": "minecraft:orange_tulip", 186 | "minecraft:red_flower.6": "minecraft:white_tulip", 187 | "minecraft:red_flower.7": "minecraft:pink_tulip", 188 | "minecraft:red_flower.8": "minecraft:oxeye_daisy", 189 | "minecraft:double_plant.0": "minecraft:sunflower", 190 | "minecraft:double_plant.1": "minecraft:lilac", 191 | "minecraft:double_plant.2": "minecraft:tall_grass", 192 | "minecraft:double_plant.3": "minecraft:large_fern", 193 | "minecraft:double_plant.4": "minecraft:rose_bush", 194 | "minecraft:double_plant.5": "minecraft:peony", 195 | "minecraft:deadbush.0": "minecraft:dead_bush", 196 | "minecraft:tallgrass.0": "minecraft:dead_bush", 197 | "minecraft:tallgrass.1": "minecraft:grass", 198 | "minecraft:tallgrass.2": "minecraft:fern", 199 | "minecraft:sponge.0": "minecraft:sponge", 200 | "minecraft:sponge.1": "minecraft:wet_sponge", 201 | "minecraft:purpur_slab.0": "minecraft:purpur_slab", 202 | "minecraft:stone_slab.0": "minecraft:stone_slab", 203 | "minecraft:stone_slab.1": "minecraft:sandstone_slab", 204 | "minecraft:stone_slab.2": "minecraft:petrified_oak_slab", 205 | "minecraft:stone_slab.3": "minecraft:cobblestone_slab", 206 | "minecraft:stone_slab.4": "minecraft:brick_slab", 207 | "minecraft:stone_slab.5": "minecraft:stone_brick_slab", 208 | "minecraft:stone_slab.6": "minecraft:nether_brick_slab", 209 | "minecraft:stone_slab.7": "minecraft:quartz_slab", 210 | "minecraft:stone_slab2.0": "minecraft:red_sandstone_slab", 211 | "minecraft:wooden_slab.0": "minecraft:oak_slab", 212 | "minecraft:wooden_slab.1": "minecraft:spruce_slab", 213 | "minecraft:wooden_slab.2": "minecraft:birch_slab", 214 | "minecraft:wooden_slab.3": "minecraft:jungle_slab", 215 | "minecraft:wooden_slab.4": "minecraft:acacia_slab", 216 | "minecraft:wooden_slab.5": "minecraft:dark_oak_slab", 217 | "minecraft:coal.0": "minecraft:coal", 218 | "minecraft:coal.1": "minecraft:charcoal", 219 | "minecraft:fish.0": "minecraft:cod", 220 | "minecraft:fish.1": "minecraft:salmon", 221 | "minecraft:fish.2": "minecraft:clownfish", 222 | "minecraft:fish.3": "minecraft:pufferfish", 223 | "minecraft:cooked_fish.0": "minecraft:cooked_cod", 224 | "minecraft:cooked_fish.1": "minecraft:cooked_salmon", 225 | "minecraft:skull.0": "minecraft:skeleton_skull", 226 | "minecraft:skull.1": "minecraft:wither_skeleton_skull", 227 | "minecraft:skull.2": "minecraft:zombie_head", 228 | "minecraft:skull.3": "minecraft:player_head", 229 | "minecraft:skull.4": "minecraft:creeper_head", 230 | "minecraft:skull.5": "minecraft:dragon_head", 231 | "minecraft:golden_apple.0": "minecraft:golden_apple", 232 | "minecraft:golden_apple.1": "minecraft:enchanted_golden_apple", 233 | "minecraft:fireworks.0": "minecraft:firework_rocket", 234 | "minecraft:firework_charge.0": "minecraft:firework_star", 235 | "minecraft:dye.0": "minecraft:ink_sac", 236 | "minecraft:dye.1": "minecraft:rose_red", 237 | "minecraft:dye.2": "minecraft:cactus_green", 238 | "minecraft:dye.3": "minecraft:cocoa_beans", 239 | "minecraft:dye.4": "minecraft:lapis_lazuli", 240 | "minecraft:dye.5": "minecraft:purple_dye", 241 | "minecraft:dye.6": "minecraft:cyan_dye", 242 | "minecraft:dye.7": "minecraft:light_gray_dye", 243 | "minecraft:dye.8": "minecraft:gray_dye", 244 | "minecraft:dye.9": "minecraft:pink_dye", 245 | "minecraft:dye.10": "minecraft:lime_dye", 246 | "minecraft:dye.11": "minecraft:dandelion_yellow", 247 | "minecraft:dye.12": "minecraft:light_blue_dye", 248 | "minecraft:dye.13": "minecraft:magenta_dye", 249 | "minecraft:dye.14": "minecraft:orange_dye", 250 | "minecraft:dye.15": "minecraft:bone_meal", 251 | "minecraft:silver_shulker_box.0": "minecraft:light_gray_shulker_box", 252 | "minecraft:fence.0": "minecraft:oak_fence", 253 | "minecraft:fence_gate.0": "minecraft:oak_fence_gate", 254 | "minecraft:wooden_door.0": "minecraft:oak_door", 255 | "minecraft:boat.0": "minecraft:oak_boat", 256 | "minecraft:lit_pumpkin.0": "minecraft:jack_o_lantern", 257 | "minecraft:pumpkin.0": "minecraft:carved_pumpkin", 258 | "minecraft:trapdoor.0": "minecraft:oak_trapdoor", 259 | "minecraft:nether_brick.0": "minecraft:nether_bricks", 260 | "minecraft:red_nether_brick.0": "minecraft:red_nether_bricks", 261 | "minecraft:netherbrick.0": "minecraft:nether_brick", 262 | "minecraft:wooden_button.0": "minecraft:oak_button", 263 | "minecraft:wooden_pressure_plate.0": "minecraft:oak_pressure_plate", 264 | "minecraft:noteblock.0": "minecraft:note_block", 265 | "minecraft:bed.0": "minecraft:white_bed", 266 | "minecraft:bed.1": "minecraft:orange_bed", 267 | "minecraft:bed.2": "minecraft:magenta_bed", 268 | "minecraft:bed.3": "minecraft:light_blue_bed", 269 | "minecraft:bed.4": "minecraft:yellow_bed", 270 | "minecraft:bed.5": "minecraft:lime_bed", 271 | "minecraft:bed.6": "minecraft:pink_bed", 272 | "minecraft:bed.7": "minecraft:gray_bed", 273 | "minecraft:bed.8": "minecraft:light_gray_bed", 274 | "minecraft:bed.9": "minecraft:cyan_bed", 275 | "minecraft:bed.10": "minecraft:purple_bed", 276 | "minecraft:bed.11": "minecraft:blue_bed", 277 | "minecraft:bed.12": "minecraft:brown_bed", 278 | "minecraft:bed.13": "minecraft:green_bed", 279 | "minecraft:bed.14": "minecraft:red_bed", 280 | "minecraft:bed.15": "minecraft:black_bed", 281 | "minecraft:banner.15": "minecraft:white_banner", 282 | "minecraft:banner.14": "minecraft:orange_banner", 283 | "minecraft:banner.13": "minecraft:magenta_banner", 284 | "minecraft:banner.12": "minecraft:light_blue_banner", 285 | "minecraft:banner.11": "minecraft:yellow_banner", 286 | "minecraft:banner.10": "minecraft:lime_banner", 287 | "minecraft:banner.9": "minecraft:pink_banner", 288 | "minecraft:banner.8": "minecraft:gray_banner", 289 | "minecraft:banner.7": "minecraft:light_gray_banner", 290 | "minecraft:banner.6": "minecraft:cyan_banner", 291 | "minecraft:banner.5": "minecraft:purple_banner", 292 | "minecraft:banner.4": "minecraft:blue_banner", 293 | "minecraft:banner.3": "minecraft:brown_banner", 294 | "minecraft:banner.2": "minecraft:green_banner", 295 | "minecraft:banner.1": "minecraft:red_banner", 296 | "minecraft:banner.0": "minecraft:black_banner", 297 | "minecraft:grass.0": "minecraft:grass_block", 298 | "minecraft:brick_block.0": "minecraft:bricks", 299 | "minecraft:end_bricks.0": "minecraft:end_stone_bricks", 300 | "minecraft:golden_rail.0": "minecraft:powered_rail", 301 | "minecraft:magma.0": "minecraft:magma_block", 302 | "minecraft:quartz_ore.0": "minecraft:nether_quartz_ore", 303 | "minecraft:reeds.0": "minecraft:sugar_cane", 304 | "minecraft:slime.0": "minecraft:slime_block", 305 | "minecraft:stone_stairs.0": "minecraft:cobblestone_stairs", 306 | "minecraft:waterlily.0": "minecraft:lily_pad", 307 | "minecraft:web.0": "minecraft:cobweb", 308 | "minecraft:snow.0": "minecraft:snow_block", 309 | "minecraft:snow_layer.0": "minecraft:snow", 310 | "minecraft:record_11.0": "minecraft:music_disc_11", 311 | "minecraft:record_13.0": "minecraft:music_disc_13", 312 | "minecraft:record_blocks.0": "minecraft:music_disc_blocks", 313 | "minecraft:record_cat.0": "minecraft:music_disc_cat", 314 | "minecraft:record_chirp.0": "minecraft:music_disc_chirp", 315 | "minecraft:record_far.0": "minecraft:music_disc_far", 316 | "minecraft:record_mall.0": "minecraft:music_disc_mall", 317 | "minecraft:record_mellohi.0": "minecraft:music_disc_mellohi", 318 | "minecraft:record_stal.0": "minecraft:music_disc_stal", 319 | "minecraft:record_strad.0": "minecraft:music_disc_strad", 320 | "minecraft:record_wait.0": "minecraft:music_disc_wait", 321 | "minecraft:record_ward.0": "minecraft:music_disc_ward" 322 | } -------------------------------------------------------------------------------- /src/main/resources/scripts/geode.scan.kts: -------------------------------------------------------------------------------- 1 | package scripts 2 | 3 | import de.skyrising.mc.scanner.* 4 | import it.unimi.dsi.fastutil.objects.Object2IntMap 5 | import it.unimi.dsi.fastutil.objects.Object2IntOpenHashMap 6 | import java.io.PrintStream 7 | import java.nio.file.Files 8 | 9 | needles = listOf(BlockState(Identifier("minecraft", "budding_amethyst"), emptyMap())) 10 | 11 | collect { 12 | this is RegionFile && dimension == "overworld" 13 | } 14 | 15 | scan { 16 | if (this is RegionFile) { 17 | scanChunks(needles, false) 18 | } else { 19 | emptyList() 20 | } 21 | } 22 | 23 | val resultsFile = PrintStream(Files.newOutputStream(outPath.resolve("results.txt")), false, "UTF-8") 24 | val amethystCounts = Object2IntOpenHashMap() 25 | 26 | onResults { 27 | forEach { result -> 28 | if (result.needle is BlockState) { 29 | amethystCounts.addTo(result.location as ChunkPos, result.count.toInt()) 30 | } 31 | } 32 | } 33 | 34 | fun RandomTickRegion.sumChunks(mapFn: (Vec2i) -> Int): Int { 35 | var sum = 0 36 | for (i in 0 until 64) { 37 | if (chunks and (1UL shl i) == 0UL) continue 38 | sum += mapFn(SOMETIMES_RANDOM_TICKED[i]) 39 | } 40 | return sum 41 | } 42 | 43 | after { 44 | val afkChunks = mutableMapOf>() 45 | for (pos in amethystCounts.keys) { 46 | for (x in pos.x - 8 .. pos.x + 8) { 47 | for (z in pos.z - 8 .. pos.z + 8) { 48 | afkChunks.getOrPut(ChunkPos(pos.dimension, x, z)) { Object2IntOpenHashMap() }.put(pos, amethystCounts.getInt(pos)) 49 | } 50 | } 51 | } 52 | var bestPos: Vec3d 53 | var bestCount = 0 54 | for ((pos, neighbors) in afkChunks) { 55 | val maxPossible = neighbors.values.sum() 56 | if (maxPossible < bestCount) continue 57 | val base = ALWAYS_RANDOM_TICKED.sumOf { (x, z) -> neighbors.getInt(ChunkPos(pos.dimension, pos.x + x, pos.z + z)) } 58 | for (region in RANDOM_TICK_REGIONS) { 59 | val count = base + region.sumChunks { (x, z) -> neighbors.getInt(ChunkPos(pos.dimension, pos.x + x, pos.z + z)) } 60 | if (count > bestCount) { 61 | bestPos = Vec3d("overworld", pos.x * 16 + region.pos.x, 0.0, pos.z * 16 + region.pos.y) 62 | bestCount = count 63 | resultsFile.println("$bestPos,$bestCount") 64 | } 65 | } 66 | } 67 | resultsFile.close() 68 | } -------------------------------------------------------------------------------- /src/main/resources/scripts/search.scan.kts: -------------------------------------------------------------------------------- 1 | package scripts 2 | 3 | import de.skyrising.mc.scanner.Needle 4 | import it.unimi.dsi.fastutil.objects.Object2LongOpenHashMap 5 | import java.io.PrintStream 6 | import java.nio.file.Files 7 | import kotlin.system.exitProcess 8 | 9 | val resultsFile = PrintStream(Files.newOutputStream(outPath.resolve("results.txt")), false, "UTF-8") 10 | val total = Object2LongOpenHashMap() 11 | 12 | if (needles.isEmpty()) { 13 | println("Nothing to search for.") 14 | exitProcess(0) 15 | } 16 | 17 | println(needles) 18 | 19 | onResults { 20 | for (result in this) { 21 | resultsFile.println("${result.location}: ${result.needle} x ${result.count}") 22 | if (result.location is SubLocation) continue 23 | total.addTo(result.needle, result.count) 24 | } 25 | resultsFile.flush() 26 | } 27 | 28 | after { 29 | val totalTypes = total.keys.sortedWith { a, b -> 30 | if (a.javaClass != b.javaClass) return@sortedWith a.javaClass.hashCode() - b.javaClass.hashCode() 31 | if (a is ItemType && b is ItemType) return@sortedWith a.compareTo(b) 32 | if (a is BlockState && b is BlockState) return@sortedWith a.compareTo(b) 33 | 0 34 | } 35 | for (type in totalTypes) { 36 | resultsFile.println("Total $type: ${total.getLong(type)}") 37 | } 38 | } -------------------------------------------------------------------------------- /src/main/resources/scripts/stats.scan.kts: -------------------------------------------------------------------------------- 1 | package scripts 2 | 3 | import de.skyrising.mc.scanner.Needle 4 | import it.unimi.dsi.fastutil.objects.* 5 | import java.io.PrintStream 6 | import java.nio.file.Files 7 | import java.util.* 8 | import java.util.function.ToIntFunction 9 | 10 | val inventoriesFile = PrintStream(Files.newOutputStream(outPath.resolve("inventories.csv")), false, "UTF-8") 11 | val total = Object2LongOpenHashMap() 12 | val stats = Object2ObjectOpenHashMap>() 13 | val inventoryCounts = Object2IntOpenHashMap() 14 | val locationIds = Object2IntLinkedOpenHashMap() 15 | 16 | scan { 17 | scan(needles, true) 18 | } 19 | 20 | onResults { 21 | for (result in this) { 22 | val location = result.location 23 | val needle = result.needle 24 | when (needle) { 25 | is StatsResults -> { 26 | val types = needle.types 27 | val stride = types.size 28 | val matrix = needle.matrix 29 | for (i in 0 until stride) { 30 | inventoryCounts.addTo(types[i], 1) 31 | val map = stats.computeIfAbsent(types[i], Object2ObjectFunction { Object2DoubleOpenHashMap() }) 32 | for (j in 0 until stride) { 33 | val type = types[j] 34 | val value = matrix[i * stride + j] 35 | map[type] = map.getDouble(type) + value 36 | } 37 | } 38 | continue 39 | } 40 | 41 | is ItemType -> { 42 | val loc = if (location is SubLocation) location else SubLocation(location, 0) 43 | val locId = locationIds.computeIfAbsent(loc.parent, ToIntFunction { locationIds.size }) 44 | //val locStr = loc.parent.toString().replace("\"", "\\\"") 45 | //resultsFile.print('"') 46 | //resultsFile.print(locStr) 47 | //resultsFile.print('"') 48 | inventoriesFile.print(locId) 49 | inventoriesFile.print(',') 50 | inventoriesFile.print(loc.index) 51 | inventoriesFile.print(',') 52 | inventoriesFile.print(needle.id) 53 | inventoriesFile.print(',') 54 | inventoriesFile.print(result.count) 55 | inventoriesFile.println() 56 | } 57 | } 58 | if (location is SubLocation) continue 59 | total.addTo(needle, result.count) 60 | } 61 | inventoriesFile.flush() 62 | } 63 | 64 | after { 65 | PrintStream(Files.newOutputStream(outPath.resolve("locations.csv")), false, "UTF-8").use { locationsFile -> 66 | locationsFile.println("Id,Location") 67 | for ((location, id) in locationIds) { 68 | locationsFile.print(id) 69 | locationsFile.print(',') 70 | locationsFile.print('"') 71 | locationsFile.print(location.toString().replace("\"", "\\\"")) 72 | locationsFile.println('"') 73 | } 74 | } 75 | val types = stats.keys.sorted() 76 | PrintStream(Files.newOutputStream(outPath.resolve("counts.csv")), false, "UTF-8").use { countsFile -> 77 | countsFile.println("Type,Total,Number of Inventories") 78 | for (type in types) { 79 | countsFile.print(type.format()) 80 | countsFile.print(',') 81 | countsFile.print(total.getLong(type)) 82 | countsFile.print(',') 83 | countsFile.println(inventoryCounts.getInt(type)) 84 | } 85 | } 86 | PrintStream(Files.newOutputStream(outPath.resolve("stats.csv")), false, "UTF-8").use { statsFile -> 87 | for (type in types) { 88 | statsFile.print(',') 89 | statsFile.print(type.format()) 90 | } 91 | statsFile.println() 92 | for (a in types) { 93 | statsFile.print(a.id) 94 | val map = stats[a]!! 95 | val sum = map.values.sum() 96 | for (b in types) { 97 | statsFile.print(',') 98 | val value = map.getDouble(b) 99 | statsFile.printf(Locale.ROOT, "%1.5f", value / sum) 100 | } 101 | statsFile.println() 102 | } 103 | } 104 | } -------------------------------------------------------------------------------- /src/test/kotlin/de/skyrising/mc/scanner/zlib/test-deflate.kt: -------------------------------------------------------------------------------- 1 | @file:OptIn(ExperimentalUnsignedTypes::class) 2 | 3 | package de.skyrising.mc.scanner.zlib 4 | 5 | import org.junit.jupiter.api.Test 6 | import java.nio.ByteBuffer 7 | import kotlin.test.assertEquals 8 | import kotlin.test.assertNotNull 9 | 10 | class TestDeflate { 11 | @Test 12 | fun smartHuffmanSimple() { 13 | val code = byteArrayOf( 14 | 5, 5, 5, 5, 5, 5, 5, 5, 15 | 5, 5, 5, 5, 5, 5, 5, 5, 16 | 5, 5, 5, 5, 5, 5, 5, 5, 17 | 5, 5, 5, 5, 5, 5, 5, 5, 18 | ) 19 | val input = ubyteArrayOf(0b000_00000u, 0b0_10000_10u, 0b1000_0100u, 0b10_10100_1u, 0b10110_000u, 0b000_10101u) 20 | val output = uintArrayOf(0x00u, 0x01u, 0x01u, 0x02u, 0x03u, 0x05u, 0x08u, 0x0du, 0x15u) 21 | val huffman = SmartHuffmanDecoder.fromBytes(code, 0, code.size) 22 | assertNotNull(huffman) 23 | val stream = BitStream(ByteBuffer.wrap(input.toByteArray())) 24 | for (i in output) { 25 | assertEquals(i, huffman.decode(stream)) 26 | } 27 | } 28 | 29 | @Test 30 | fun smartHuffmanComplex() { 31 | val code = byteArrayOf(3, 2, 3, 3, 2, 3) 32 | val input = ubyteArrayOf(0b101_00_001u, 0b111_10_011u, 0b101_00_001u, 0b111_10_011u) 33 | val output = uintArrayOf(0u, 1u, 2u, 3u, 4u, 5u, 0u, 1u, 2u, 3u, 4u, 5u) 34 | val huffman = SmartHuffmanDecoder.fromBytes(code, 0, code.size) 35 | assertNotNull(huffman) 36 | val stream = BitStream(ByteBuffer.wrap(input.toByteArray())) 37 | for (i in output) { 38 | assertEquals(i, huffman.decode(stream)) 39 | } 40 | } 41 | 42 | @Test 43 | fun decompressCompressedBlock() { 44 | val compressed = ubyteArrayOf( 45 | 0x0Bu, 0xC9u, 0xC8u, 0x2Cu, 0x56u, 0x00u, 0xA2u, 0x44u, 46 | 0x85u, 0xE2u, 0xCCu, 0xDCu, 0x82u, 0x9Cu, 0x54u, 0x85u, 47 | 0x92u, 0xD4u, 0x8Au, 0x12u, 0x85u, 0xB4u, 0x4Cu, 0x20u, 48 | 0xCBu, 0x4Au, 0x13u, 0x00u 49 | ) 50 | val decompressed = inflate(ByteBuffer.wrap(compressed.toByteArray())) 51 | val decompressedArray = ByteArray(decompressed.remaining()) 52 | decompressed.get(decompressedArray) 53 | assertEquals("This is a simple text file :)", decompressedArray.toString(Charsets.UTF_8)) 54 | } 55 | } --------------------------------------------------------------------------------