├── .gitignore ├── README.md ├── pom.xml ├── proxy-smf-manifest.xml ├── smf.xml ├── tiny-maven-indexer ├── pom.xml └── src │ └── main │ └── java │ └── com │ └── mastfrog │ └── tiny │ └── maven │ └── indexer │ ├── IndexerImpl.java │ └── Main.java └── tiny-maven-proxy ├── pom.xml └── src ├── main ├── java │ └── com │ │ └── mastfrog │ │ └── tinymavenproxy │ │ ├── Browse.java │ │ ├── Config.java │ │ ├── DownloadReceiver.java │ │ ├── DownloadResult.java │ │ ├── DownloaderV2A.java │ │ ├── FileFinder.java │ │ ├── GetActeur.java │ │ ├── GetIndex.java │ │ ├── Noop.java │ │ ├── TempFiles.java │ │ ├── TinyMavenProxy.java │ │ └── VersionActeur.java └── resources │ └── com │ └── mastfrog │ └── tinymavenproxy │ └── index.html └── test ├── java └── com │ └── mastfrog │ └── tinymavenproxy │ ├── FakeMavenServer.java │ ├── FakeMavenServers.java │ ├── GeneralProxyingTest.java │ └── TestBug5.java └── resources └── com └── mastfrog └── tinymavenproxy ├── GeneralProxyingTest.properties ├── GetActeurTest.properties └── fakepom.txt /.gitignore: -------------------------------------------------------------------------------- 1 | *.class 2 | nbactions.xml 3 | nb-configuration.xml 4 | tiny-maven-proxy/.uids 5 | # Package Files # 6 | *.jar 7 | *.war 8 | *.ear 9 | *.orig 10 | *.iml 11 | .idea/* 12 | target/ 13 | 14 | /tiny-maven-proxy/nbproject/ -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Tiny Maven Proxy 2 | ================ 3 | 4 | Is exactly what it says it is - a tiny proxy server for Maven, which you can 5 | run on your local network. The *only* thing it does (at present) is proxy 6 | stuff Maven downloads and cache it. 7 | 8 | If you have a slow-ish internet connection, and you have multiple machines 9 | or a team that will all be building and downloading, this is the project for 10 | you. 11 | 12 | It is a tiny server you can run with `java -jar` somewhere on your network, 13 | and configure Maven to use. It is written with [acteur](http://timboudreau.com/blog/updatedActeur/read) 14 | and Netty, meaning that it is asynchronous, with a small memory footprint 15 | and low memory usage (microscopic if you're used to Java EE - `-Xmx16M` is 16 | reasonable). 17 | 18 | By default it runs on port 5956. It has a minimal web-ui. 19 | 20 | You give it a list of repository URLs to proxy, and a folder to cache files 21 | in, and run it. Then configure your `~/.m2/settings.xml` to use it. That's all. 22 | 23 | Download [a recent build here](https://timboudreau.com/builds/job/tiny-maven-proxy/lastSuccessfulBuild/artifact/tiny-maven-proxy/target/tiny-maven-proxy.jar) 24 | 25 | ![Screenshot](http://timboudreau.com/files/screen/d17d10e6-9e65-4314-9c68-cecb4a57023a.png) 26 | 27 | 28 | Configuration 29 | ------------- 30 | 31 | There are two properties you'll want to set. You can either set them from 32 | the command-line, or in a `tiny-maven-proxy.properties` file that can live in 33 | `/etc`, `/opt/local/etc`, `/~` or `./` (these override each other in that order). 34 | 35 | 36 | #### Example 37 | 38 | ``` 39 | java -jar tiny-maven-proxy.jar --maven.dir /var/lib/maven --mirror https://repo1.maven.org/maven2,http://bits.netbeans.org/maven2/ 40 | ``` 41 | 42 | or you could create `/etc/tiny-maven-proxy.properties` and put in it: 43 | 44 | ``` 45 | maven.dir=/var/lib/maven 46 | mirror=https://repo1.maven.org/maven2,http://bits.netbeans.org/maven2/ 47 | ``` 48 | 49 | Other properties that affect Acteur that may be useful: 50 | 51 | * `port` - the port to run on 52 | * `cors.enabled` - whether or not to answer CORS preflight requests affirmitively - on by default 53 | * `workerThreads` - the number of threads used to answer requests (one thread *can* work on multiple requests at a time with netty, so 4-8 is usually enough) 54 | * `log.file` - log to a file 55 | 56 | #### Defaults 57 | 58 | If `maven.dir` is not set, it will create a `/maven` directory in the system 59 | temporary dir (on most OSs this is wiped on reboot). 60 | 61 | The following is the list of Maven repositories it proxies by default, if you 62 | do not set the `mirror` setting: 63 | 64 | * https://repo1.maven.org/maven2 65 | * http://bits.netbeans.org/maven2/ 66 | * http://bits.netbeans.org/nexus/content/repositories/snapshots/ 67 | * https://timboudreau.com/builds/plugin/repository/everything/ 68 | * https://maven.java.net/content/groups/public/ 69 | * https://oss.sonatype.org/ 70 | 71 | 72 | What This Project Is Not 73 | ------------------------ 74 | 75 | It is not a full-featured Maven proxy, such as Nexus or Artifactory. Those 76 | are great if you need to manage complex mirroring setups, authentication, etc. 77 | 78 | It does no authentication, validation, checksum checking (but your Maven client 79 | will, so you'll get the same result as if you'd downloaded things directly). 80 | 81 | 82 | Indexing 83 | -------- 84 | 85 | IDEs and other tools will try to download a Nexus-style maven index. A companion 86 | project next to this makes it simple to automate index generation. It embeds the 87 | nexus-cli tool in a single fat-jar and runs it appropriately. 88 | 89 | Since plexus depends on a version of Guice from the dawn of time, we cannot embed 90 | the indexer directly in tiny-maven-proxy. However, it is trivial to set up as a 91 | cron job. Here is an example crontab: 92 | 93 | ``` 94 | @hourly /usr/bin/java -jar /opt/tiny-maven-proxy/tiny-maven-indexer.jar --repositoryId dr.timboudreau.org --repositoryDir /space/maven/repository 95 | ``` 96 | 97 | To build it, simply build the companion project with `mvn install`, copy `tiny-maven-indexer.jar` to 98 | wherever you need it, and set it up to run periodically _as the same user tiny-maven-proxy runs as_ on the 99 | target server. 100 | 101 | Logging 102 | ------- 103 | 104 | The project uses [bunyan-java](https://github.com/timboudreau/bunyan-java) for 105 | logging in JSON format - [more about bunyan-java here](http://timboudreau.com/blog/bunyan/read). 106 | That makes it easy to collect metrics and stats and process log files using the 107 | `bunyan` command-line utility (to get that, install [NodeJS](http://nodejs.org) 108 | and then run `nbm install -g bunyan` on the command-line). 109 | 110 | 111 | To-Dos 112 | ------ 113 | 114 | * Clean out `-SNAPSHOT` dependencies periodically 115 | 116 | 117 | Under The Hood 118 | -------------- 119 | 120 | Tiny Maven Proxy uses [netty-http-client](https://github.com/timboudreau/netty-http-client) 121 | for downloads, and [acteur](https://github.com/timboudreau/acteur) for the server piece. 122 | On a request for a non-cached file, it simultaneously attempts downloads from all the 123 | servers it knows about, and when one succeeds, cancels the others. 124 | 125 | Command-line and configuration file management is done [with giulius](https://github.com/timboudreau/giulius). 126 | 127 | 128 | Footprint 129 | --------- 130 | 131 | While the default Java 64Mb heap is preferred, especially if the server will be heavily used, just to prove 132 | you can run this with a minimal memory footprint, you *can* run it and use it with an 7Mb heap - the following 133 | command-line sets up a JDK 8 vm appropriately: 134 | 135 | ``` 136 | java -XX:-UseConcMarkSweepGC -Xmx7M -jar tiny-maven-proxy.jar --log.level=fatal 137 | --acteur.fork.join false --download.chunk.size 256 138 | ``` 139 | 140 | A bunch of care is taken to ensure as few memory copies as possible are performed, and that downloads are 141 | read and written chunk by chunk, so the whole file is never dragged into memory at once. 142 | 143 | Hopefully this demonstrates what you can do with non-blocking I/O and a bit of care :-) 144 | -------------------------------------------------------------------------------- /pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 6 | 4.0.0 7 | 1.12 8 | tiny-maven-proxy-parent 9 | 10 | com.mastfrog 11 | Tiny Maven Proxy (modules) 12 | pom 13 | 14 | tiny-maven-indexer 15 | tiny-maven-proxy 16 | 17 | 18 | 19 | 2.9.1 20 | 1.8 21 | 1.8 22 | UTF-8 23 | 24 | 25 | 26 | 27 | 28 | 29 | com.mastfrog 30 | maven-merge-configuration 31 | ${mastfrog.version} 32 | 33 | 34 | org.apache.maven.plugins 35 | maven-compiler-plugin 36 | 3.10.1 37 | 38 | 39 | org.apache.maven.plugins 40 | maven-surefire-plugin 41 | 3.0.0-M7 42 | 43 | 44 | com.mastfrog 45 | revision-info-plugin 46 | 0.21 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | ${project.groupId} 55 | mastfrog-parent 56 | ${mastfrog.version} 57 | pom 58 | import 59 | 60 | 61 | org.codehaus.plexus 62 | plexus-cli 63 | 1.6 64 | 65 | 66 | org.hamcrest 67 | hamcrest-core 68 | 1.3 69 | 70 | 71 | org.sonatype.nexus.plugins 72 | nexus-indexer-lucene-plugin 73 | 3.0.0-b2015020701 74 | 75 | 76 | org.sonatype.nexus 77 | nexus-indexer 78 | 3.1-M1 79 | 80 | 81 | 82 | 83 | 84 | git@github.com:timboudreau/tiny-maven-proxy.git 85 | scm:git:https://github.com:timboudreau/tiny-maven-proxy.git 86 | git@github.com:timboudreau/tiny-maven-proxy.git 87 | 88 | 89 | Github 90 | https://github.com/timboudreau/tiny-maven-proxy/issues 91 | 92 | 93 | 94 | Mastfrog Technologies 95 | https://mastfrog.com 96 | 97 | 98 | 99 | 100 | MIT 101 | https://opensource.org/licenses/MIT 102 | repo 103 | 104 | 105 | 106 | 107 | -------------------------------------------------------------------------------- /proxy-smf-manifest.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 37 | 38 | 39 | 40 | -------------------------------------------------------------------------------- /smf.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 36 | 37 | 38 | 39 | -------------------------------------------------------------------------------- /tiny-maven-indexer/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 6 | 4.0.0 7 | 8 | 9 | com.mastfrog 10 | tiny-maven-proxy-parent 11 | 1.12 12 | 13 | tiny-maven-indexer 14 | 15 | 16 | git@github.com:timboudreau/tiny-maven-proxy.git 17 | scm:git:https://github.com:timboudreau/tiny-maven-proxy.git 18 | git@github.com:timboudreau/tiny-maven-proxy.git 19 | 20 | 21 | Github 22 | https://github.com/timboudreau/tiny-maven-proxy/issues 23 | 24 | 25 | 26 | Mastfrog Technologies 27 | https://mastfrog.com 28 | 29 | 30 | 31 | 32 | MIT 33 | https://opensource.org/licenses/MIT 34 | repo 35 | 36 | 37 | 38 | 39 | ${project.groupId} 40 | giulius-settings 41 | 42 | 43 | ${project.groupId} 44 | util-strings 45 | 46 | 47 | ${project.groupId} 48 | util-streams 49 | 50 | 51 | org.sonatype.nexus 52 | nexus-indexer 53 | 54 | 55 | org.sonatype.nexus.plugins 56 | nexus-indexer-lucene-plugin 57 | provided 58 | 59 | 60 | org.codehaus.plexus 61 | plexus-cli 62 | 63 | 64 | 65 | 66 | 67 | 68 | com.mastfrog 69 | maven-merge-configuration 70 | 71 | 72 | package 73 | compile 74 | 75 | merge-configuration 76 | 77 | 78 | 79 | 80 | com.mastfrog.tiny.maven.indexer.Main 81 | tiny-maven-indexer 82 | true 83 | true 84 | true 85 | org.junit 86 | org.hamcrest 87 | com.mastfrog.tiny.http.server 88 | com.mastfrog.util.search 89 | com.google.common.eventbus 90 | com.google.common.graph 91 | com.google.common.annotations 92 | com.google.common.escape 93 | com.google.common.io 94 | com.mastfrog.giulius.annotations.processors 95 | com.mastfrog.parameters.processor 96 | com.mastfrog.util.perf 97 | com.mastfrog.util.search 98 | org.openide.xml 99 | META-INF.maven 100 | .netbeans_automatic_build 101 | 102 | 103 | 104 | 105 | 106 | -------------------------------------------------------------------------------- /tiny-maven-indexer/src/main/java/com/mastfrog/tiny/maven/indexer/IndexerImpl.java: -------------------------------------------------------------------------------- 1 | /* 2 | * To change this license header, choose License Headers in Project Properties. 3 | * To change this template file, choose Tools | Templates 4 | * and open the template in the editor. 5 | */ 6 | package com.mastfrog.tiny.maven.indexer; 7 | 8 | import com.google.inject.Inject; 9 | import com.mastfrog.settings.Settings; 10 | import static com.mastfrog.tiny.maven.indexer.Main.SETTINGS_KEY_INDEX_DIR; 11 | import static com.mastfrog.tiny.maven.indexer.Main.SETTINGS_KEY_REPOSITORY_BASE_DIR; 12 | import static com.mastfrog.tiny.maven.indexer.Main.SETTINGS_KEY_REPOSITORY_ID; 13 | import com.mastfrog.util.preconditions.ConfigurationError; 14 | import com.mastfrog.util.strings.Strings; 15 | import java.nio.file.Files; 16 | import java.nio.file.Path; 17 | import java.nio.file.Paths; 18 | import javax.inject.Named; 19 | 20 | /** 21 | * 22 | * @author Tim Boudreau 23 | */ 24 | public class IndexerImpl { 25 | 26 | private final Path indexPath; 27 | private final Path repoPath; 28 | private final String id; 29 | 30 | @Inject 31 | IndexerImpl(@Named(SETTINGS_KEY_REPOSITORY_BASE_DIR) String repoDir, @Named(SETTINGS_KEY_INDEX_DIR) String indexDir, 32 | @Named(SETTINGS_KEY_REPOSITORY_ID) String id) throws Exception { 33 | repoPath = Paths.get(repoDir); 34 | if (!Files.exists(repoPath)) { 35 | throw new ConfigurationError("Repo path " + repoDir + " does not exist"); 36 | } 37 | if ("_".equals(indexDir)) { 38 | indexDir = repoPath.resolve(Paths.get(".index")).toString(); 39 | } 40 | indexPath = Paths.get(indexDir); 41 | if (!Files.exists(indexPath.getParent())) { 42 | Files.createDirectories(indexPath.getParent()); 43 | } 44 | this.id = id; 45 | System.out.println("INDEX " + repoPath + " to index " + indexPath + " for repo " + id); 46 | } 47 | 48 | void start() throws Exception { 49 | // export REPODIR=/path/to/your/local/repo/ && 50 | // java org.sonatype.nexus.index.cli.NexusIndexerCli -r $REPODIR -i $REPODIR/.index -d $REPODIR/.index -n localrepo 51 | 52 | String[] args = new String[]{ 53 | "-r", repoPath.toString(), 54 | "-i", indexPath.toString(), 55 | "-d", indexPath.toString(), 56 | "-n", id, 57 | "--checksums", "sha1", "--type", "full" 58 | }; 59 | System.out.println("WILL RUN " + Strings.join(' ', args)); 60 | org.sonatype.nexus.index.cli.NexusIndexerCli.main(args); 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /tiny-maven-indexer/src/main/java/com/mastfrog/tiny/maven/indexer/Main.java: -------------------------------------------------------------------------------- 1 | package com.mastfrog.tiny.maven.indexer; 2 | 3 | import com.google.inject.AbstractModule; 4 | import com.google.inject.Guice; 5 | import com.google.inject.name.Names; 6 | import com.mastfrog.settings.Settings; 7 | import com.mastfrog.settings.SettingsBuilder; 8 | import java.io.File; 9 | import java.io.IOException; 10 | import org.sonatype.nexus.index.DefaultNexusIndexer; 11 | import org.sonatype.nexus.index.NexusIndexer; 12 | 13 | /** 14 | * 15 | * @author Tim Boudreau 16 | */ 17 | public class Main extends AbstractModule { 18 | 19 | public static final String SETTINGS_KEY_REPOSITORY_ID = "repositoryId"; 20 | public static final String SETTINGS_KEY_REPOSITORY_BASE_DIR = "repositoryDir"; 21 | public static final String SETTINGS_KEY_INDEX_DIR = "indexDir"; 22 | private final Settings settings; 23 | 24 | public static void main(String[] args) throws IOException, Exception { 25 | Settings settings = new SettingsBuilder("tiny-maven-proxy") 26 | .add("application.name", "tiny-maven-proxy") 27 | // The repository id 28 | .add(SETTINGS_KEY_REPOSITORY_ID, "tiny") 29 | // Default to system temp dir and ~/.m2/repository for testing - these should be 30 | // set in tiny-maven-proxy.properties in /etc or working dir, or passed on the 31 | // command-line 32 | .add(SETTINGS_KEY_REPOSITORY_BASE_DIR, System.getProperty("user.home") + File.separatorChar + ".m2" + File.separatorChar + "repository") 33 | // .index/nexus-maven-repository-index.properties 34 | 35 | .add(SETTINGS_KEY_INDEX_DIR, "_") // by default, use $REPO_DIR/.index 36 | .addFilesystemAndClasspathLocations() 37 | .parseCommandLineArguments(args) 38 | .build(); 39 | IndexerImpl impl = Guice.createInjector(new Main(settings)).getInstance(IndexerImpl.class); 40 | impl.start(); 41 | } 42 | 43 | Main(Settings settings) { 44 | this.settings = settings; 45 | } 46 | 47 | @Override 48 | protected void configure() { 49 | bind(NexusIndexer.class).to(DefaultNexusIndexer.class); 50 | bind(IndexerImpl.class).asEagerSingleton(); 51 | bind(Settings.class).toInstance(settings); 52 | for (String s : new String[]{SETTINGS_KEY_INDEX_DIR, SETTINGS_KEY_REPOSITORY_BASE_DIR, SETTINGS_KEY_REPOSITORY_ID}) { 53 | bind(String.class).annotatedWith(Names.named(s)).toInstance(settings.getString(s)); 54 | } 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /tiny-maven-proxy/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 6 | 4.0.0 7 | 8 | 9 | com.mastfrog 10 | 1.12 11 | tiny-maven-proxy-parent 12 | 13 | tiny-maven-proxy 14 | 15 | 16 | git@github.com:timboudreau/tiny-maven-proxy.git 17 | scm:git:https://github.com:timboudreau/tiny-maven-proxy.git 18 | git@github.com:timboudreau/tiny-maven-proxy.git 19 | 20 | 21 | Github 22 | https://github.com/timboudreau/tiny-maven-proxy/issues 23 | 24 | 25 | 26 | Mastfrog Technologies 27 | https://mastfrog.com 28 | 29 | 30 | 31 | 32 | MIT 33 | https://opensource.org/licenses/MIT 34 | repo 35 | 36 | 37 | 38 | 39 | ${project.groupId} 40 | injection-reflection-indexer 41 | compile 42 | 43 | 44 | ${project.groupId} 45 | acteur-headers 46 | 47 | 48 | ${project.groupId} 49 | util-misc 50 | 51 | 52 | ${project.groupId} 53 | util-function 54 | 55 | 56 | ${project.groupId} 57 | acteur-bunyan-v2 58 | 59 | 60 | com.google.inject 61 | guice 62 | 63 | 64 | ${project.groupId} 65 | giulius 66 | 67 | 68 | ${project.groupId} 69 | acteur 70 | 71 | 72 | ${project.groupId} 73 | simple-webserver 74 | ${mastfrog.version} 75 | test 76 | 77 | 78 | ${project.groupId} 79 | http-test-harness 80 | 0.9.8-dev 81 | test 82 | 83 | 84 | com.telenav.cactus 85 | wordy 86 | 1.5.24 87 | test 88 | 89 | 90 | junit 91 | junit 92 | test 93 | 94 | 95 | org.hamcrest 96 | hamcrest-core 97 | test 98 | 99 | 100 | ${project.groupId} 101 | giulius-tests 102 | test 103 | 104 | 105 | ${project.groupId} 106 | tiny-http-server 107 | test 108 | 109 | 110 | com.mastfrog 111 | util-net 112 | test 113 | 114 | 115 | 116 | 117 | 118 | timboudreau-plugins 119 | timboudreau.com plugins 120 | http://timboudreau.com/maven/ 121 | 122 | true 123 | never 124 | 125 | 126 | true 127 | never 128 | 129 | 130 | 131 | 132 | 133 | 134 | 135 | com.mastfrog 136 | maven-merge-configuration 137 | 138 | 139 | package 140 | compile 141 | 142 | merge-configuration 143 | 144 | 145 | 146 | 147 | com.mastfrog.tinymavenproxy.TinyMavenProxy 148 | tiny-maven-proxy 149 | true 150 | true 151 | true 152 | com.mastfrog.tiny.http.server 153 | com.mastfrog.util.search 154 | com.mastfrog.abstractions.list 155 | com.mastfrog.annotation.processor 156 | com.mastfrog.annotation.registries 157 | com.mastfrog.annotation.validation 158 | com.mastfrog.graal.injection.processor 159 | com.mastfrog.util.path 160 | com.mastfrog.java.vogon 161 | org.codehaus.mojo.animal_sniffer 162 | org.checkerframework 163 | com.google.common.eventbus 164 | com.google.common.graph 165 | com.google.common.annotations 166 | com.google.common.escape 167 | com.google.common.io 168 | com.mastfrog.giulius.annotation.processors 169 | com.mastfrog.parameters.processor 170 | com.mastfrog.util.perf 171 | org.openide.xml 172 | META-INF.maven 173 | .netbeans_automatic_build 174 | 175 | 176 | 177 | maven-compiler-plugin 178 | 179 | 180 | 11 181 | 11 182 | true 183 | ${project.build.sourceEncoding} 184 | ${maven.compiler.argument} 185 | -Xlint:unchecked 186 | -Xdoclint:none 187 | -verbose 188 | -XprintRounds 189 | -XprintProcessorInfo 190 | -Xdiags:verbose 191 | true 192 | true 193 | 194 | 195 | 196 | ${project.groupId} 197 | annotation-processors 198 | ${mastfrog.version} 199 | 200 | 201 | ${project.groupId} 202 | injection-reflection-indexer 203 | ${mastfrog.version} 204 | 205 | 206 | ${project.groupId} 207 | util-fileformat 208 | ${mastfrog.version} 209 | 210 | 211 | ${project.groupId} 212 | giulius-annotation-processors 213 | ${mastfrog.version} 214 | 215 | 216 | ${project.groupId} 217 | acteur-annotation-processors 218 | ${mastfrog.version} 219 | 220 | 221 | 222 | 223 | 224 | org.apache.maven.plugins 225 | maven-surefire-plugin 226 | 227 | none 228 | none 229 | false 230 | 0 231 | 237 | false 238 | 239 | /tmp 240 | 241 | ${surefire.forkNumber} 242 | true 243 | 244 | false 245 | 246 | 247 | 248 | 249 | com.mastfrog 250 | revision-info-plugin 251 | 252 | 253 | revision-info-plugin 254 | generate-sources 255 | 256 | revision-info 257 | 258 | 259 | 260 | 261 | true 262 | 263 | 264 | 265 | 266 | 267 | -------------------------------------------------------------------------------- /tiny-maven-proxy/src/main/java/com/mastfrog/tinymavenproxy/Browse.java: -------------------------------------------------------------------------------- 1 | /* 2 | * The MIT License 3 | * 4 | * Copyright 2015 Tim Boudreau. 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in 14 | * all copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 22 | * THE SOFTWARE. 23 | */ 24 | package com.mastfrog.tinymavenproxy; 25 | 26 | import com.fasterxml.jackson.core.JsonProcessingException; 27 | import com.fasterxml.jackson.databind.ObjectMapper; 28 | import com.google.inject.Inject; 29 | import com.google.inject.Provider; 30 | import com.google.inject.name.Named; 31 | import com.google.inject.util.Providers; 32 | import com.mastfrog.acteur.Acteur; 33 | import com.mastfrog.acteur.HttpEvent; 34 | import com.mastfrog.acteur.annotations.HttpCall; 35 | import com.mastfrog.acteur.header.entities.CacheControl; 36 | import com.mastfrog.acteur.headers.Headers; 37 | import static com.mastfrog.acteur.headers.Method.GET; 38 | import static com.mastfrog.acteur.headers.Method.HEAD; 39 | import com.mastfrog.acteur.preconditions.Description; 40 | import com.mastfrog.acteur.preconditions.Methods; 41 | import com.mastfrog.acteur.server.PathFactory; 42 | import com.mastfrog.mime.MimeType; 43 | import static com.mastfrog.mime.MimeType.JSON_UTF_8; 44 | import com.mastfrog.url.Path; 45 | import static com.mastfrog.util.collections.CollectionUtils.map; 46 | import com.mastfrog.util.strings.Strings; 47 | import com.mastfrog.util.time.TimeUtil; 48 | import static com.mastfrog.util.time.TimeUtil.GMT; 49 | import io.netty.buffer.ByteBuf; 50 | import io.netty.channel.ChannelFuture; 51 | import io.netty.channel.ChannelFutureListener; 52 | import io.netty.handler.codec.http.DefaultLastHttpContent; 53 | import static io.netty.handler.codec.http.HttpResponseStatus.NOT_MODIFIED; 54 | import java.io.File; 55 | import java.security.MessageDigest; 56 | import java.security.NoSuchAlgorithmException; 57 | import java.time.ZonedDateTime; 58 | import java.util.ArrayList; 59 | import java.util.Base64; 60 | import java.util.List; 61 | import java.util.Map; 62 | 63 | /** 64 | * 65 | * @author Tim Boudreau 66 | */ 67 | @HttpCall(order = Integer.MAX_VALUE) 68 | @Methods({GET, HEAD}) 69 | @Description(category = "Download", value = "Serves directory listings and the HTML index page") 70 | public class Browse extends Acteur { 71 | 72 | @Inject 73 | Browse(HttpEvent evt, FileFinder finder, ZonedDateTime startTime, PathFactory paths, 74 | @Named("index") Provider indexPage, 75 | @Named("indexHash") Provider indexPageHash, 76 | ObjectMapper mapper) throws NoSuchAlgorithmException, JsonProcessingException { 77 | Path path = evt.path(); 78 | add(Headers.CACHE_CONTROL, CacheControl.PUBLIC_MUST_REVALIDATE_MAX_AGE_1_DAY); 79 | if (path.size() == 0 && !"true".equals(evt.urlParameter("browse"))) { 80 | ZonedDateTime headerTime = evt.header(Headers.IF_MODIFIED_SINCE); 81 | // if (headerTime != null && (TimeUtil.equalsToSeconds(startTime, headerTime) || headerTime.isAfter(startTime))) { 82 | CharSequence ifNoneMatch = evt.header(Headers.IF_NONE_MATCH); 83 | if (ifNoneMatch != null && Strings.charSequencesEqual(ifNoneMatch, indexPageHash.get())) { 84 | reply(NOT_MODIFIED); 85 | return; 86 | } else if (TimeUtil.equalsToSecondsOrAfter(startTime, headerTime)) { 87 | reply(NOT_MODIFIED); 88 | return; 89 | } 90 | add(Headers.ETAG, indexPageHash.get()); 91 | add(Headers.CONTENT_TYPE, MimeType.HTML_UTF_8); 92 | if (HEAD.is(evt.method())) { 93 | ok(); 94 | } else { 95 | ByteBuf buf = indexPage.get(); 96 | add(Headers.CONTENT_LENGTH, buf.readableBytes()); 97 | ok(indexPage.get()); 98 | } 99 | return; 100 | } 101 | File f = finder.folder(path); 102 | if (f == null) { 103 | notFound(); 104 | return; 105 | } 106 | MessageDigest digest = MessageDigest.getInstance("MD5"); 107 | long newest = 0; 108 | File[] kids = f.listFiles(); 109 | List> result = new ArrayList(kids.length); 110 | for (File file : f.listFiles()) { 111 | String name = file.getName(); 112 | if ("index.html".equals(name) || ".index".equals(name)) { 113 | continue; 114 | } else if (name.charAt(0) == '_') { 115 | // Gzipped cache files 116 | continue; 117 | } else if ("maven-metadata-local.xml".equals(name)) { 118 | continue; 119 | } 120 | 121 | long lastModified = file.lastModified(); 122 | newest = Math.max(lastModified, newest); 123 | result.add(map("name").to(file.getName()).map("file").to(file.isFile()) 124 | .maybeMap(file::isFile, mb -> { 125 | mb.map("length").to(file.length()); 126 | }).map("lastModified").finallyTo(lastModified)); 127 | } 128 | add(Headers.LAST_MODIFIED, TimeUtil.fromUnixTimestamp(newest, GMT)); 129 | String etag = Base64.getEncoder().encodeToString(digest.digest()); 130 | add(Headers.ETAG, etag); 131 | add(Headers.CONTENT_TYPE, JSON_UTF_8); 132 | CharSequence inm = evt.header(Headers.IF_NONE_MATCH); 133 | if (inm != null && Strings.charSequencesEqual(etag, inm)) { 134 | reply(NOT_MODIFIED); 135 | return; 136 | } else if (TimeUtil.equalsToSecondsOrAfter(TimeUtil.fromUnixTimestamp(newest, GMT), evt.header(Headers.IF_MODIFIED_SINCE))) { 137 | reply(NOT_MODIFIED); 138 | return; 139 | } 140 | if (evt.method().is(HEAD)) { 141 | ok(); 142 | return; 143 | } 144 | setChunked(true); 145 | ok(); 146 | byte[] bytes = mapper.writeValueAsBytes(result); 147 | ByteBuf buf = evt.channel().alloc().ioBuffer(bytes.length, bytes.length); 148 | buf.writeBytes(bytes); 149 | setResponseBodyWriter(new IndexPageWriter(Providers.of(buf))); 150 | } 151 | 152 | static class IndexPageWriter implements ChannelFutureListener { 153 | 154 | private final Provider indexBytes; 155 | 156 | @Inject 157 | IndexPageWriter(@Named("index") Provider buf) { 158 | indexBytes = buf; 159 | } 160 | 161 | @Override 162 | public void operationComplete(ChannelFuture f) throws Exception { 163 | if (!f.isDone() || f.isSuccess()) { 164 | f.channel().writeAndFlush(new DefaultLastHttpContent(indexBytes.get())); 165 | } 166 | } 167 | } 168 | } 169 | -------------------------------------------------------------------------------- /tiny-maven-proxy/src/main/java/com/mastfrog/tinymavenproxy/Config.java: -------------------------------------------------------------------------------- 1 | /* 2 | * The MIT License 3 | * 4 | * Copyright 2015 Tim Boudreau. 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in 14 | * all copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 22 | * THE SOFTWARE. 23 | */ 24 | package com.mastfrog.tinymavenproxy; 25 | 26 | import com.fasterxml.jackson.annotation.JsonProperty; 27 | import com.google.inject.Inject; 28 | import com.google.inject.Singleton; 29 | import com.mastfrog.settings.Settings; 30 | import static com.mastfrog.tinymavenproxy.TinyMavenProxy.SETTINGS_KEY_DOWNLOAD_CHUNK_SIZE; 31 | import com.mastfrog.url.Path; 32 | import com.mastfrog.url.PathElement; 33 | import com.mastfrog.url.URL; 34 | import com.mastfrog.url.URLBuilder; 35 | import com.mastfrog.util.preconditions.ConfigurationError; 36 | import com.mastfrog.util.strings.Strings; 37 | import java.io.File; 38 | import java.io.IOException; 39 | import java.util.ArrayList; 40 | import java.util.Arrays; 41 | import java.util.Collection; 42 | import java.util.HashSet; 43 | import java.util.Iterator; 44 | import java.util.List; 45 | import java.util.Set; 46 | import java.util.function.Supplier; 47 | import org.netbeans.validation.api.Problems; 48 | 49 | /** 50 | * 51 | * @author Tim Boudreau 52 | */ 53 | @Singleton 54 | public class Config implements Iterable { 55 | 56 | public static final String SETTINGS_KEY_MIRROR_URLS = "mirror"; 57 | public static final String MAVEN_CACHE_DIR = "maven.dir"; 58 | public static final String SETTINGS_KEY_CACHE_FAILED_PATHS_MINUTES = "failed.path.cache.minutes"; 59 | public static final String SETTINGS_KEY_INDEX_DIR = "index.dir"; 60 | private static final String DEFAULT_URLS 61 | = "https://repo.maven.apache.org/maven2/," 62 | + "https://repo1.maven.org/maven2/," 63 | + ",https://oss.sonatype.org/content/repositories/snapshots/," 64 | + ",https://oss.sonatype.org/content/repositories/releases/," 65 | + ",https://maven.atlassian.com/3rdparty/"; 66 | 67 | private final URL[] urls; 68 | public final File dir; 69 | final File indexDir; 70 | final boolean debugLog; 71 | final int bufferSize; 72 | final int failedPathCacheMinutes; 73 | 74 | @Inject 75 | Config(Settings s) throws IOException { 76 | failedPathCacheMinutes = s.getInt(SETTINGS_KEY_CACHE_FAILED_PATHS_MINUTES, 90); 77 | bufferSize = s.getInt(SETTINGS_KEY_DOWNLOAD_CHUNK_SIZE, 1480); 78 | debugLog = s.getBoolean("maven.proxy.debug", false); 79 | String[] u = s.getString(SETTINGS_KEY_MIRROR_URLS, DEFAULT_URLS).split(","); 80 | List urls = new ArrayList<>(); 81 | for (int i = 0; i < u.length; i++) { 82 | String mirror = u[i].trim(); 83 | if (mirror.isEmpty()) { 84 | continue; 85 | } 86 | URL uu = URL.parse(mirror); 87 | if (!uu.isValid()) { 88 | Problems p = uu.getProblems(); 89 | if (p.hasFatal()) { 90 | throw new ConfigurationError("Fatal problem with " + uu 91 | + " for " + mirror + ": " + p); 92 | } 93 | } 94 | urls.add(uu); 95 | } 96 | if (urls.isEmpty()) { 97 | throw new ConfigurationError("No urls to proxy"); 98 | } 99 | this.urls = urls.toArray(URL[]::new); 100 | debugLog("START WITH URLS ", () -> new Object[]{Strings.commas(this.urls)}); 101 | String dirname = s.getString(MAVEN_CACHE_DIR); 102 | if (dirname == null) { 103 | File tmp = new File(System.getProperty("java.io.tmpdir")); 104 | dir = new File(tmp, "maven"); 105 | if (dir.exists() && !dir.isDirectory()) { 106 | throw new IOException("Not a folder: " + dir); 107 | } 108 | if (!dir.exists()) { 109 | if (!dir.mkdirs()) { 110 | throw new IOException("Could not create " + dir); 111 | } 112 | } 113 | } else { 114 | dir = new File(dirname); 115 | if (dir.exists() && !dir.isDirectory()) { 116 | throw new IOException("Not a folder: " + dir); 117 | } 118 | if (!dir.exists()) { 119 | if (!dir.mkdirs()) { 120 | throw new IOException("Could not create " + dir); 121 | } 122 | } 123 | } 124 | String indexDir = s.getString(SETTINGS_KEY_INDEX_DIR, "_"); 125 | if ("_".equals(indexDir)) { 126 | indexDir = new File(dir, ".index").getAbsolutePath(); 127 | } 128 | this.indexDir = new File(indexDir); 129 | if (!this.indexDir.exists()) { 130 | if (!this.indexDir.mkdirs()) { 131 | throw new ConfigurationError("Could not create index dirs " + this.indexDir); 132 | } 133 | } 134 | } 135 | 136 | @JsonProperty("mirroring") 137 | Set urlStrings() { 138 | Set result = new HashSet<>(); 139 | for (URL url : urls) { 140 | result.add(url.toString()); 141 | } 142 | return result; 143 | } 144 | 145 | @JsonProperty("dir") 146 | String path() { 147 | return dir.getAbsolutePath(); 148 | } 149 | 150 | File indexDir() { 151 | return indexDir; 152 | } 153 | 154 | public Collection withPath(Path path) { 155 | List result = new ArrayList(urls.length); 156 | for (URL u : this) { 157 | URLBuilder b = URL.builder(u); 158 | for (PathElement p : path) { 159 | b.add(p); 160 | } 161 | result.add(b.create()); 162 | } 163 | return result; 164 | } 165 | 166 | @Override 167 | public Iterator iterator() { 168 | return Arrays.asList(urls).iterator(); 169 | } 170 | 171 | public boolean isDebug() { 172 | return debugLog; 173 | } 174 | 175 | final void debugLog(String msg, Supplier lazy) { 176 | if (debugLog) { 177 | debugLog(msg, lazy.get()); 178 | } 179 | } 180 | 181 | final void debugLog(String msg, Object... objs) { 182 | if (debugLog) { 183 | StringBuilder sb = new StringBuilder(msg); 184 | for (int i = 0; i < objs.length; i++) { 185 | sb.append(' ').append(objs[i]); 186 | if (i != objs.length - 1) { 187 | sb.append(','); 188 | } 189 | } 190 | System.out.println(sb); 191 | } 192 | } 193 | } 194 | -------------------------------------------------------------------------------- /tiny-maven-proxy/src/main/java/com/mastfrog/tinymavenproxy/DownloadReceiver.java: -------------------------------------------------------------------------------- 1 | /* 2 | * The MIT License 3 | * 4 | * Copyright 2022 Mastfrog Technologies. 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in 14 | * all copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 22 | * THE SOFTWARE. 23 | */ 24 | package com.mastfrog.tinymavenproxy; 25 | 26 | import io.netty.buffer.ByteBuf; 27 | import io.netty.handler.codec.http.HttpHeaders; 28 | import io.netty.handler.codec.http.HttpResponseStatus; 29 | import java.io.File; 30 | 31 | /** 32 | * 33 | * @author Tim Boudreau 34 | */ 35 | interface DownloadReceiver { 36 | 37 | void receive(HttpResponseStatus status, ByteBuf buf, HttpHeaders headers); 38 | 39 | void receive(HttpResponseStatus status, File file, HttpHeaders headers); 40 | 41 | void failed(HttpResponseStatus status); 42 | 43 | void failed(HttpResponseStatus status, String msg); 44 | 45 | } 46 | -------------------------------------------------------------------------------- /tiny-maven-proxy/src/main/java/com/mastfrog/tinymavenproxy/DownloadResult.java: -------------------------------------------------------------------------------- 1 | /* 2 | * The MIT License 3 | * 4 | * Copyright 2015 Tim Boudreau. 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in 14 | * all copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 22 | * THE SOFTWARE. 23 | */ 24 | package com.mastfrog.tinymavenproxy; 25 | 26 | import io.netty.buffer.ByteBuf; 27 | import io.netty.handler.codec.http.HttpHeaders; 28 | import io.netty.handler.codec.http.HttpResponseStatus; 29 | import java.io.File; 30 | 31 | /** 32 | * 33 | * @author Tim Boudreau 34 | */ 35 | class DownloadResult { 36 | 37 | final HttpResponseStatus status; 38 | ByteBuf buf; 39 | HttpHeaders headers; 40 | File file; 41 | 42 | DownloadResult(HttpResponseStatus status, File file, HttpHeaders headers) { 43 | this.file = file; 44 | this.status = status; 45 | this.headers = headers; 46 | } 47 | 48 | DownloadResult(HttpResponseStatus status, ByteBuf message) { 49 | this(status, message, null); 50 | } 51 | 52 | DownloadResult(HttpResponseStatus status, ByteBuf buf, HttpHeaders headers) { 53 | // This method is currently unused, but if we enhance the server to accept 54 | // uploads, we will likely need code a lot like this 55 | this.status = status; 56 | this.buf = buf; 57 | this.headers = headers; 58 | } 59 | 60 | DownloadResult(HttpResponseStatus status) { 61 | this.status = status; 62 | } 63 | 64 | public String toString() { 65 | StringBuilder sb = new StringBuilder(status.toString()); 66 | if (file != null) { 67 | sb.append(" ").append(file.getPath()); 68 | } 69 | if (headers != null) { 70 | sb.append(" ").append(headers); 71 | } 72 | if (buf != null) { 73 | sb.append(" bytes=").append(buf.readableBytes()); 74 | } 75 | return sb.toString(); 76 | } 77 | 78 | boolean isFile() { 79 | return this.file != null; 80 | } 81 | 82 | boolean isFail() { 83 | return status.code() > 399 || (buf == null && file == null); 84 | } 85 | 86 | } 87 | -------------------------------------------------------------------------------- /tiny-maven-proxy/src/main/java/com/mastfrog/tinymavenproxy/DownloaderV2A.java: -------------------------------------------------------------------------------- 1 | /* 2 | * The MIT License 3 | * 4 | * Copyright 2022 Mastfrog Technologies. 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in 14 | * all copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 22 | * THE SOFTWARE. 23 | */ 24 | package com.mastfrog.tinymavenproxy; 25 | 26 | import com.google.common.cache.Cache; 27 | import com.google.common.cache.CacheBuilder; 28 | import com.google.inject.Inject; 29 | import com.google.inject.name.Named; 30 | import com.mastfrog.acteur.errors.ResponseException; 31 | import com.mastfrog.acteur.headers.Headers; 32 | import com.mastfrog.acteur.server.ServerModule; 33 | import com.mastfrog.acteur.spi.ApplicationControl; 34 | import com.mastfrog.acteur.util.RequestID; 35 | import com.mastfrog.bunyan.java.v2.Log; 36 | import com.mastfrog.bunyan.java.v2.Logs; 37 | import com.mastfrog.function.state.Int; 38 | import com.mastfrog.tinymavenproxy.TempFiles.TempFile; 39 | import static com.mastfrog.tinymavenproxy.TinyMavenProxy.DOWNLOAD_LOGGER; 40 | import com.mastfrog.url.Path; 41 | import com.mastfrog.url.URL; 42 | import com.mastfrog.util.libversion.VersionInfo; 43 | import io.netty.handler.codec.http.DefaultHttpHeaders; 44 | import io.netty.handler.codec.http.HttpHeaders; 45 | import io.netty.handler.codec.http.HttpResponseStatus; 46 | import static io.netty.handler.codec.http.HttpResponseStatus.GONE; 47 | import static io.netty.handler.codec.http.HttpResponseStatus.OK; 48 | import java.io.File; 49 | import java.io.IOException; 50 | import java.net.URISyntaxException; 51 | import java.net.http.HttpClient; 52 | import java.net.http.HttpRequest; 53 | import java.net.http.HttpResponse; 54 | import java.net.http.HttpResponse.BodyHandler; 55 | import java.net.http.HttpResponse.BodySubscriber; 56 | import java.nio.ByteBuffer; 57 | import java.time.Duration; 58 | import java.time.ZoneId; 59 | import java.time.ZonedDateTime; 60 | import java.util.ArrayList; 61 | import java.util.Collection; 62 | import java.util.List; 63 | import java.util.concurrent.CancellationException; 64 | import java.util.concurrent.CompletableFuture; 65 | import java.util.concurrent.CompletionStage; 66 | import java.util.concurrent.ExecutorService; 67 | import java.util.concurrent.Flow; 68 | import java.util.concurrent.TimeUnit; 69 | import java.util.concurrent.atomic.AtomicLong; 70 | import java.util.function.Consumer; 71 | import java.util.logging.Level; 72 | import java.util.logging.Logger; 73 | 74 | /** 75 | * 76 | * @author Tim Boudreau 77 | */ 78 | public class DownloaderV2A { 79 | 80 | private final HttpClient client; 81 | private final Config config; 82 | private final FileFinder finder; 83 | private final Cache failedURLs; 84 | private final Logs logger; 85 | private final ApplicationControl control; 86 | 87 | static final AtomicLong counter = new AtomicLong(); 88 | private final TempFiles tempFiles; 89 | private final String runId; 90 | private final String userAgent; 91 | private final ExecutorService pool; 92 | 93 | @Inject 94 | public DownloaderV2A(HttpClient client, Config config, FileFinder finder, 95 | @Named(DOWNLOAD_LOGGER) Logs logger, ApplicationControl control, 96 | @Named("runId") String runId, TempFiles tempFiles, VersionInfo ver, 97 | @Named(ServerModule.BACKGROUND_THREAD_POOL_NAME) ExecutorService pool) { 98 | this.pool = pool; 99 | failedURLs = CacheBuilder.newBuilder().expireAfterWrite(config.failedPathCacheMinutes, TimeUnit.MINUTES).build(); 100 | this.client = client; 101 | this.config = config; 102 | this.finder = finder; 103 | this.logger = logger.child("rid", runId); 104 | this.control = control; 105 | this.runId = runId; 106 | this.tempFiles = tempFiles; 107 | userAgent = "tmpx-" + ver.version; 108 | } 109 | 110 | String nextDownloadId() { 111 | return runId + Long.toString(counter.getAndIncrement(), 36); 112 | } 113 | 114 | public boolean isFailedPath(Path path) { 115 | return failedURLs.getIfPresent(path) != null; 116 | } 117 | 118 | CompletableFuture download(Path path, RequestID rid, DownloadReceiver recv) throws URISyntaxException { 119 | CompletableFuture tf = download(path, rid); 120 | Logs requestLog = logger.child("download", rid); 121 | tf.whenComplete((file, thrown) -> { 122 | if (thrown != null) { 123 | failedURLs.put(path, path); 124 | // System.out.println("DO FAIL FOR " + thrown + " " + path); 125 | recv.failed(GONE, thrown.getMessage()); 126 | } else if (file != null) { 127 | HttpResponseStatus status = file.info().map(info -> { 128 | return HttpResponseStatus.valueOf(info.statusCode()); 129 | }).orElse(OK); 130 | HttpHeaders nettyHeaders = new DefaultHttpHeaders(false); 131 | file.info().ifPresent(info -> { 132 | info.headers().map().forEach(nettyHeaders::add); 133 | }); 134 | recv.receive(status, file.path().toFile(), nettyHeaders); 135 | } 136 | }); 137 | return tf; 138 | } 139 | 140 | public CompletableFuture download(Path path, RequestID rid) throws URISyntaxException { 141 | Collection urls = config.withPath(path); 142 | List> futures = new ArrayList<>(urls.size()); 143 | Int remainder = Int.createAtomic(); 144 | remainder.set(urls.size()); 145 | CompletableFuture result = new CompletableFuture<>(); 146 | Logs requestLog = logger.child("req", rid); 147 | 148 | final Object lock = new Object(); 149 | 150 | Consumer> cancelOthers = fut -> { 151 | synchronized (lock) { 152 | List> copy = new ArrayList<>(futures); 153 | futures.clear(); 154 | for (CompletableFuture f : copy) { 155 | if (f != fut) { 156 | f.cancel(false); 157 | } 158 | } 159 | } 160 | }; 161 | 162 | result.whenCompleteAsync((file, thrown) -> { 163 | if (thrown instanceof CancellationException) { 164 | requestLog.info("request-cancelled-killing-downloads") 165 | .add("tasks", futures.size()) 166 | .add("alive", remainder.get()).close(); 167 | cancelOthers.accept(null); 168 | } else if (thrown != null) { 169 | Log l = requestLog.warn("all-failed"); 170 | if (!(thrown instanceof ResponseException)) { 171 | l.add(thrown).close(); 172 | } 173 | } 174 | }); 175 | 176 | for (URL u : urls) { 177 | String dlId = nextDownloadId(); 178 | Logs perUrl = requestLog.child("dl", dlId) 179 | .child("url", u.toString()); 180 | HttpRequest req 181 | = HttpRequest.newBuilder(u.toURI()) 182 | .header("User-Agent", userAgent) 183 | .timeout(Duration.ofMinutes(2)) 184 | .GET() 185 | .build(); 186 | CompletableFuture fut = new CompletableFuture<>(); 187 | fut.whenComplete((file, thrown) -> { 188 | // synchronized (lock) { 189 | remainder.decrement(); 190 | int remaining = remainder.getAsInt(); 191 | if (remaining < 0) { 192 | throw new IllegalStateException("" + remaining); 193 | } 194 | try (Log lr = perUrl.debug("completed")) { 195 | if (result.isDone()) { 196 | if (file != null) { 197 | file.close(); 198 | } 199 | return; 200 | } 201 | if (file != null) { 202 | cancelOthers.accept(fut); 203 | if (fut.isCancelled()) { 204 | return; 205 | } 206 | lr.add("file", file.path().toString()); 207 | file.info().ifPresent(info -> { 208 | lr.add("status", info.statusCode()); 209 | }); 210 | file.lastModified().ifPresent(lm -> lr.add("lastModified", lm)); 211 | ZonedDateTime zdt = file.lastModified().map(ins -> ZonedDateTime.ofInstant(ins, ZoneId.of("Z"))).orElse(ZonedDateTime.now()); 212 | File dest = finder.put(path, file); 213 | lr.add("saved", dest.toString()); 214 | result.complete(file); 215 | } else if (thrown instanceof CancellationException) { 216 | lr.add("cancelled"); 217 | if (remaining == 0) { 218 | // System.out.println("REMAINING 0 COMPLETE 1 " + path); 219 | result.completeExceptionally(new ResponseException(GONE, "No result " + path)); 220 | } 221 | } else if (thrown != null) { 222 | if (remaining == 0) { 223 | // System.out.println("REMAINING 0 COMPLETE 2 " + path); 224 | result.completeExceptionally(thrown); 225 | } 226 | lr.add(thrown); 227 | } else { 228 | lr.add("state", "No info."); 229 | if (remaining == 0) { 230 | // System.out.println("REMAINING 0 COMPLETE 3 " + path); 231 | result.completeExceptionally(new ResponseException(GONE, "No result " + path)); 232 | } 233 | } 234 | } catch (IOException ex) { 235 | result.completeExceptionally(ex); 236 | } 237 | // } 238 | }); 239 | // fut.whenComplete(onComplete); 240 | BH bh = new BH(dlId, u, fut, perUrl); 241 | futures.add(fut); 242 | client.sendAsync(req, bh); 243 | } 244 | return result; 245 | } 246 | 247 | static void sleep() { 248 | try { 249 | Thread.sleep(50); 250 | } catch (InterruptedException ex) { 251 | ex.printStackTrace(); 252 | } 253 | } 254 | 255 | private final static Noop NO_OP = new Noop<>(); 256 | 257 | private static final String LAST_MODIFIED = Headers.LAST_MODIFIED.name().toString(); 258 | 259 | class BH implements BodyHandler { 260 | 261 | private final CompletableFuture result; 262 | private final String downloadId; 263 | private final URL url; 264 | private final Logs logs; 265 | 266 | public BH(String downloadId, URL url, CompletableFuture result, Logs logs) { 267 | this.downloadId = downloadId; 268 | this.url = url; 269 | this.result = result; 270 | this.logs = logs; 271 | } 272 | 273 | @Override 274 | public BodySubscriber apply(HttpResponse.ResponseInfo info) { 275 | if (info.statusCode() > 399) { 276 | logs.warn("request-failed") 277 | .add("status", info.statusCode()).close(); 278 | // .add("headers", info.headers().map()).close(); 279 | result.cancel(true); 280 | return NO_OP; 281 | } else { 282 | logs.info("potential-success") 283 | .add("status", info.statusCode()).close(); 284 | // .add("headers", info.headers().map()).close(); 285 | TempFile tempFile = tempFiles.tempFile(downloadId); 286 | info.headers().firstValue(LAST_MODIFIED) 287 | .map(Headers.LAST_MODIFIED) 288 | .ifPresent(tempFile::setLastModified); 289 | return new BS(tempFile.withResponseInfo(info), result, logs); 290 | } 291 | } 292 | } 293 | 294 | static class BS implements BodySubscriber { 295 | 296 | private final TempFile file; 297 | private Flow.Subscription subscription; 298 | private final CompletableFuture result; 299 | private final Logs logs; 300 | 301 | BS(TempFile file, CompletableFuture result, Logs logs) { 302 | this.file = file; 303 | this.result = result; 304 | this.logs = logs; 305 | } 306 | 307 | synchronized void cancel() { 308 | logs.info("cancelled").close(); 309 | try { 310 | file.close(); 311 | result.cancel(true); 312 | } catch (Exception ex) { 313 | result.completeExceptionally(ex); 314 | } 315 | } 316 | 317 | @Override 318 | public CompletionStage getBody() { 319 | return result; 320 | } 321 | 322 | @Override 323 | public void onSubscribe(Flow.Subscription subscription) { 324 | if (result.isDone()) { 325 | return; 326 | } 327 | if (subscription == null) { 328 | logs.warn("null subscription").close(); 329 | return; 330 | } 331 | synchronized (this) { 332 | this.subscription = subscription; 333 | } 334 | subscription.request(Long.MAX_VALUE); 335 | } 336 | 337 | @Override 338 | public void onNext(List item) { 339 | if (result.isDone()) { 340 | subscription.cancel(); 341 | return; 342 | } 343 | try (Log log = logs.info("onNext")) { 344 | log.add("buffers", item.size()); 345 | for (int i = 0; i < item.size(); i++) { 346 | ByteBuffer b = item.get(i); 347 | log.add("buf_" + i, b.remaining()); 348 | } 349 | for (ByteBuffer b : item) { 350 | try { 351 | file.append(b); 352 | } catch (IOException ex) { 353 | log.add(ex); 354 | onError(ex); 355 | return; 356 | } 357 | } 358 | } 359 | } 360 | 361 | @Override 362 | public void onError(Throwable throwable) { 363 | try { 364 | file.close(); 365 | } catch (Exception ex) { 366 | throwable.addSuppressed(ex); 367 | } 368 | logs.error("onError").add(throwable).close(); 369 | result.completeExceptionally(throwable); 370 | } 371 | 372 | @Override 373 | public void onComplete() { 374 | result.complete(file); 375 | if (result.isDone()) { 376 | logs.warn("done-but-already-complete").close(); 377 | } 378 | } 379 | } 380 | } 381 | -------------------------------------------------------------------------------- /tiny-maven-proxy/src/main/java/com/mastfrog/tinymavenproxy/FileFinder.java: -------------------------------------------------------------------------------- 1 | /* 2 | * The MIT License 3 | * 4 | * Copyright 2015 Tim Boudreau. 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in 14 | * all copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 22 | * THE SOFTWARE. 23 | */ 24 | package com.mastfrog.tinymavenproxy; 25 | 26 | import com.google.inject.Inject; 27 | import com.google.inject.name.Named; 28 | import com.mastfrog.acteur.server.ServerModule; 29 | import static com.mastfrog.tinymavenproxy.GetActeur.isGzipCacheFile; 30 | import com.mastfrog.tinymavenproxy.TempFiles.TempFile; 31 | import com.mastfrog.url.Path; 32 | import com.mastfrog.util.streams.Streams; 33 | import com.mastfrog.util.time.TimeUtil; 34 | import io.netty.buffer.ByteBuf; 35 | import io.netty.buffer.ByteBufInputStream; 36 | import java.io.BufferedOutputStream; 37 | import java.io.File; 38 | import java.io.FileOutputStream; 39 | import java.io.IOException; 40 | import java.io.OutputStream; 41 | import java.nio.file.FileAlreadyExistsException; 42 | import java.nio.file.Files; 43 | import java.nio.file.Paths; 44 | import java.time.ZonedDateTime; 45 | import java.util.concurrent.ExecutorService; 46 | 47 | /** 48 | * 49 | * @author Tim Boudreau 50 | */ 51 | public class FileFinder { 52 | 53 | private final Config config; 54 | private final ExecutorService threadPool; 55 | 56 | @Inject 57 | FileFinder(Config config, @Named(ServerModule.BACKGROUND_THREAD_POOL_NAME) ExecutorService threadPool) { 58 | this.config = config; 59 | this.threadPool = threadPool; 60 | } 61 | 62 | public File find(Path path) { 63 | File f = new File(config.dir, path.toString()); 64 | if (f.exists() && f.isFile()) { 65 | return f; 66 | } 67 | return null; 68 | } 69 | 70 | public File folder(Path path) { 71 | if (path.size() == 0) { 72 | return config.dir; 73 | } 74 | File f = new File(config.dir, path.toString()); 75 | if (f.exists() && f.isDirectory()) { 76 | return f; 77 | } 78 | return null; 79 | } 80 | 81 | public synchronized File put(final Path path, final TempFile file) throws IOException { 82 | java.nio.file.Path target = config.dir.toPath() 83 | .resolve(path.toString()); 84 | file.close(target); 85 | return target.toFile(); 86 | } 87 | 88 | public synchronized void put(final Path path, final ByteBuf content, final ZonedDateTime lastModified) { 89 | // This method is currently unused, but if we enhance the server to accept 90 | // uploads, we will likely need code a lot like this 91 | if (content.readableBytes() == 0) { 92 | return; 93 | } 94 | final ByteBuf buf = content.duplicate(); 95 | threadPool.submit(() -> { 96 | final File target = new File(config.dir, path.toString().replace('/', File.separatorChar)); 97 | buf.retain(); 98 | if (!target.exists()) { 99 | if (!target.getParentFile().exists()) { 100 | if (!target.getParentFile().mkdirs()) { 101 | throw new IOException("Could not create " + target.getParentFile()); 102 | } 103 | } 104 | if (!target.createNewFile()) { 105 | throw new IOException("Could not create " + target); 106 | } 107 | } 108 | try (ByteBufInputStream in = new ByteBufInputStream(buf)) { 109 | try (OutputStream out = new BufferedOutputStream(new FileOutputStream(target))) { 110 | Streams.copy(in, out, Math.min(content.readableBytes(), 1024)); 111 | } 112 | } catch (IOException ioe) { 113 | if (target.exists()) { 114 | target.delete(); 115 | } 116 | throw ioe; 117 | } finally { 118 | buf.release(); 119 | } 120 | threadPool.submit(() -> { 121 | if (lastModified != null) { 122 | target.setLastModified(TimeUtil.toUnixTimestamp(lastModified)); 123 | } 124 | }); 125 | return null; 126 | }); 127 | } 128 | } 129 | -------------------------------------------------------------------------------- /tiny-maven-proxy/src/main/java/com/mastfrog/tinymavenproxy/GetActeur.java: -------------------------------------------------------------------------------- 1 | /* 2 | * The MIT License 3 | * 4 | * Copyright 2015 Tim Boudreau. 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in 14 | * all copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 22 | * THE SOFTWARE. 23 | */ 24 | package com.mastfrog.tinymavenproxy; 25 | 26 | import com.google.inject.Inject; 27 | import com.google.inject.name.Named; 28 | import com.mastfrog.acteur.Acteur; 29 | import com.mastfrog.acteur.Closables; 30 | import com.mastfrog.acteur.HttpEvent; 31 | import com.mastfrog.acteur.Response; 32 | import com.mastfrog.acteur.annotations.Concluders; 33 | import com.mastfrog.acteur.annotations.HttpCall; 34 | import com.mastfrog.acteur.errors.Err; 35 | import com.mastfrog.acteur.headers.Headers; 36 | import static com.mastfrog.acteur.headers.Headers.ACCEPT_ENCODING; 37 | import static com.mastfrog.acteur.headers.Headers.LAST_MODIFIED; 38 | import static com.mastfrog.acteur.headers.Method.GET; 39 | import static com.mastfrog.acteur.headers.Method.HEAD; 40 | import com.mastfrog.acteur.preconditions.Description; 41 | import com.mastfrog.acteur.preconditions.Methods; 42 | import static com.mastfrog.acteur.server.ServerModule.X_INTERNAL_COMPRESS_HEADER; 43 | import com.mastfrog.acteur.spi.ApplicationControl; 44 | import com.mastfrog.acteur.header.entities.CacheControl; 45 | import static com.mastfrog.acteur.headers.Headers.CONTENT_LENGTH; 46 | import com.mastfrog.acteur.util.RequestID; 47 | import com.mastfrog.acteurbase.Deferral; 48 | import com.mastfrog.acteurbase.Deferral.Resumer; 49 | import com.mastfrog.bunyan.java.v2.Log; 50 | import com.mastfrog.bunyan.java.v2.Logs; 51 | import com.mastfrog.mime.MimeType; 52 | import com.mastfrog.tinymavenproxy.DownloadReceiver; 53 | import com.mastfrog.tinymavenproxy.GetActeur.ConcludeHttpRequest; 54 | import com.mastfrog.tinymavenproxy.TempFiles.TempFile; 55 | import static com.mastfrog.tinymavenproxy.TinyMavenProxy.ACCESS_LOGGER; 56 | import com.mastfrog.url.Path; 57 | import com.mastfrog.util.streams.Streams; 58 | import com.mastfrog.util.strings.Strings; 59 | import com.mastfrog.util.time.TimeUtil; 60 | import static com.mastfrog.util.time.TimeUtil.GMT; 61 | import io.netty.buffer.ByteBuf; 62 | import io.netty.buffer.Unpooled; 63 | import io.netty.channel.ChannelFuture; 64 | import io.netty.channel.ChannelFutureListener; 65 | import io.netty.channel.DefaultFileRegion; 66 | import io.netty.channel.FileRegion; 67 | import io.netty.handler.codec.http.DefaultHttpContent; 68 | import io.netty.handler.codec.http.DefaultLastHttpContent; 69 | import io.netty.handler.codec.http.HttpHeaderValues; 70 | import io.netty.handler.codec.http.HttpHeaders; 71 | import io.netty.handler.codec.http.HttpResponseStatus; 72 | import static io.netty.handler.codec.http.HttpResponseStatus.NOT_FOUND; 73 | import static io.netty.handler.codec.http.HttpResponseStatus.NOT_MODIFIED; 74 | import io.netty.handler.codec.http.HttpVersion; 75 | import io.netty.handler.codec.http.LastHttpContent; 76 | import io.netty.util.CharsetUtil; 77 | import java.io.BufferedInputStream; 78 | import java.io.BufferedOutputStream; 79 | import java.io.File; 80 | import java.io.FileInputStream; 81 | import java.io.FileNotFoundException; 82 | import java.io.FileOutputStream; 83 | import java.io.IOException; 84 | import java.io.OutputStream; 85 | import java.nio.ByteBuffer; 86 | import java.nio.channels.SeekableByteChannel; 87 | import java.nio.file.Files; 88 | import java.nio.file.StandardOpenOption; 89 | import java.time.ZonedDateTime; 90 | import java.time.temporal.ChronoField; 91 | import java.util.concurrent.CompletableFuture; 92 | import java.util.concurrent.atomic.AtomicBoolean; 93 | import java.util.regex.Pattern; 94 | import java.util.zip.Deflater; 95 | import java.util.zip.GZIPOutputStream; 96 | 97 | /** 98 | * 99 | * @author Tim Boudreau 100 | */ 101 | @HttpCall(order = Integer.MAX_VALUE - 1, scopeTypes = DownloadResult.class) 102 | @Concluders(ConcludeHttpRequest.class) 103 | @Methods({GET, HEAD}) 104 | @Description(category = "Download", value = "Download maven artifacts, fetching them from remote repositories " 105 | + "and caching the results if necessary") 106 | public class GetActeur extends Acteur { 107 | 108 | private static final boolean PREFER_CHUNKED = false; 109 | private final ApplicationControl ctrl; 110 | 111 | @Inject 112 | GetActeur(HttpEvent req, Deferral def, Config config, FileFinder finder, 113 | Closables clos, DownloaderV2A dl, @Named(ACCESS_LOGGER) Logs accessLog, 114 | RequestID id, ApplicationControl ctrl) throws IOException { 115 | this.ctrl = ctrl; 116 | setChunked(PREFER_CHUNKED); 117 | if ("true".equals(req.urlParameter("browse")) || "true".equals(req.urlParameter("index"))) { 118 | reject(); 119 | return; 120 | } 121 | Path path = req.path().normalize(); 122 | if (path.size() == 0) { 123 | reject(); 124 | return; 125 | } 126 | if (!path.isValid()) { 127 | badRequest("Invalid path " + path); 128 | return; 129 | } 130 | if (path.toString().contains("..")) { 131 | setState(new RespondWith(Err.badRequest("Relative paths not allowed"))); 132 | return; 133 | } 134 | File file = finder.find(path.elideEmptyElements()); 135 | if (file != null) { 136 | config.debugLog("send existing file ", file); 137 | try (Log log = accessLog.info("fetch")) { 138 | log.add("path", path).add("id", id).add("cached", true); 139 | add(Headers.LAST_MODIFIED, TimeUtil.fromUnixTimestamp(file.lastModified()).withZoneSameInstant(GMT)); 140 | add(Headers.CONTENT_TYPE, findMimeType(path)); 141 | add(Headers.CACHE_CONTROL, CacheControl.PUBLIC_MUST_REVALIDATE); 142 | ZonedDateTime inm = req.header(Headers.IF_MODIFIED_SINCE); 143 | if (inm != null) { 144 | long theirs = TimeUtil.toUnixTimestamp(inm.with(ChronoField.MILLI_OF_SECOND, 0)); 145 | long ours = TimeUtil.toUnixTimestamp(TimeUtil.fromUnixTimestamp(file.lastModified()).with(ChronoField.MILLI_OF_SECOND, 0)); 146 | if (ours <= theirs) { 147 | reply(NOT_MODIFIED); 148 | return; 149 | } 150 | } 151 | ok(); 152 | if (req.method() != HEAD) { 153 | if (!PREFER_CHUNKED) { 154 | add(Headers.CONTENT_LENGTH, file.length()); 155 | } 156 | setResponseBodyWriter(writerFor(req, file, accessLog, config, ctrl, response())); 157 | } 158 | } 159 | } else { 160 | if (dl.isFailedPath(path)) { 161 | notFound(); 162 | // setChunked(PREFER_CHUNKED); 163 | if (req.request().protocolVersion().equals(HttpVersion.HTTP_1_0)) { 164 | setResponseBodyWriter(ChannelFutureListener.CLOSE); 165 | } 166 | return; 167 | } 168 | String el = path.getLastElement().toString(); 169 | if (el.toString().indexOf('.') < 0) { 170 | config.debugLog("Skip for not having . ", el); 171 | reject(); 172 | return; 173 | } 174 | if (VERSION_PATTERN.matcher(el).find()) { 175 | config.debugLog("Skip for matching pattern", el); 176 | reject(); 177 | return; 178 | } 179 | Path pth = path.elideEmptyElements(); 180 | def.defer((Resumer res) -> { 181 | config.debugLog(" defer and download ", pth); 182 | CompletableFuture l = dl.download(pth, id, new DownloadReceiverImpl(res, config)); 183 | req.channel().closeFuture().addListener(cl -> { 184 | l.cancel(false); 185 | }); 186 | 187 | // ChannelFutureListener l = dl.download(pth, id, new DownloadReceiverImpl(res, config)); 188 | // req.channel().closeFuture().addListener(l); 189 | }); 190 | 191 | next(); 192 | } 193 | } 194 | 195 | private static final Pattern VERSION_PATTERN = Pattern.compile("^\\d+\\.\\d+.*?"); 196 | private static final MimeType ANY_APPLICATION_TYPE = MimeType.create("application", "*"); 197 | 198 | private static MimeType findMimeType(Path path) { 199 | if (path.size() == 0) { 200 | return ANY_APPLICATION_TYPE; 201 | } 202 | String file = path.getLastElement().toString(); 203 | int ix = file.lastIndexOf("."); 204 | if (ix < 0) { 205 | return ANY_APPLICATION_TYPE; 206 | } 207 | String ext = file.substring(ix + 1); 208 | switch (ext) { 209 | case "gz": 210 | return MimeType.GZIP; 211 | case "properties": 212 | return MimeType.JAVA_PROPERTIES; 213 | case "html": 214 | return MimeType.HTML_UTF_8; 215 | case "jar": 216 | return MimeType.create("application", "java-archive"); 217 | case "xml": 218 | case "pom": 219 | return MimeType.XML_UTF_8; 220 | case "sha1": 221 | default: 222 | return MimeType.PLAIN_TEXT_UTF_8; 223 | } 224 | } 225 | 226 | static class ConcludeHttpRequest extends Acteur { 227 | 228 | @Inject 229 | ConcludeHttpRequest(HttpEvent evt, DownloadResult res, @Named(ACCESS_LOGGER) Logs accessLog, 230 | RequestID id, Config config, ApplicationControl ctrl) throws FileNotFoundException, IOException { 231 | 232 | if (!res.isFail()) { 233 | setChunked(PREFER_CHUNKED); 234 | try (Log log = accessLog.info("fetch")) { 235 | ok(); 236 | add(Headers.CONTENT_TYPE, findMimeType(evt.path())); 237 | if (res.headers.contains(LAST_MODIFIED.name())) { 238 | add(LAST_MODIFIED, LAST_MODIFIED.toValue(res.headers.get(LAST_MODIFIED.name()))); 239 | } 240 | if (evt.method() != HEAD) { 241 | add(Headers.CACHE_CONTROL, CacheControl.PUBLIC_MUST_REVALIDATE); 242 | if (res.isFile()) { 243 | log.add("file", res.file.getPath()); 244 | // setResponseBodyWriter(new FW(res.file, accessLog, config, config.bufferSize, true, ctrl)); 245 | setResponseBodyWriter(writerFor(evt, res.file, accessLog, config, ctrl, response())); 246 | } else { 247 | log.add("internalBuffer", true); 248 | setResponseBodyWriter(new Responder2(res.buf, config, PREFER_CHUNKED, ctrl)); 249 | } 250 | } 251 | log.add("path", evt.path()).add("id", id).add("cached", false); 252 | } 253 | } else if (res.isFail() && res.buf != null) { 254 | reply(res.status); 255 | return; 256 | // if (!PREFER_CHUNKED) { 257 | // add(CONTENT_LENGTH, 0); 258 | // return; 259 | // } else { 260 | // setChunked(true); 261 | // } 262 | //// setResponseWriter(new Responder(res.buf, config)); 263 | // setResponseBodyWriter(new Responder2(res.buf, config, false, ctrl)); 264 | } else { 265 | reply(NOT_FOUND); 266 | // reject(); 267 | } 268 | } 269 | } 270 | 271 | static final class FW implements ChannelFutureListener { 272 | 273 | private final File file; 274 | private final Logs logger; 275 | private final Config config; 276 | private SeekableByteChannel channel; 277 | private final ByteBuffer buffer; 278 | private final boolean chunked; 279 | private final ApplicationControl ctrl; 280 | 281 | FW(File file, Logs logger, Config config, int bufferLength, boolean chunked, ApplicationControl ctrl) { 282 | this.file = file; 283 | this.logger = logger; 284 | this.config = config; 285 | this.buffer = ByteBuffer.allocateDirect(bufferLength); 286 | this.chunked = chunked; 287 | this.ctrl = ctrl; 288 | } 289 | 290 | private SeekableByteChannel channel(ChannelFuture fut) throws IOException { 291 | synchronized (this) { 292 | if (channel == null) { 293 | channel = Files.newByteChannel(file.toPath(), StandardOpenOption.READ); 294 | fut.channel().closeFuture().addListener(f -> { 295 | channel.close(); 296 | }); 297 | } 298 | } 299 | return channel; 300 | } 301 | 302 | @Override 303 | public void operationComplete(ChannelFuture f) throws Exception { 304 | if (f.cause() != null) { 305 | if (f.channel().isOpen()) { 306 | f.channel().close(); 307 | } 308 | logger.warn("filewrite").add(f.cause()).close(); 309 | return; 310 | } 311 | try { 312 | SeekableByteChannel channel = channel(f); 313 | if (!channel.isOpen() || channel.position() == channel.size()) { 314 | if (chunked) { 315 | ctrl.logFailure(f.channel().writeAndFlush(LastHttpContent.EMPTY_LAST_CONTENT));//.addListener(CLOSE); 316 | // } else { 317 | // f.addListener(CLOSE); 318 | } 319 | return; 320 | } 321 | buffer.rewind(); 322 | channel.read(buffer); 323 | buffer.flip(); 324 | ByteBuf buf = Unpooled.wrappedBuffer(buffer); 325 | if (chunked) { 326 | ctrl.logFailure(f.channel().writeAndFlush(new DefaultHttpContent(buf)).addListener(this)); 327 | } else { 328 | ctrl.logFailure(f.channel().writeAndFlush(buf).addListener(this)); 329 | } 330 | } catch (Exception ex) { 331 | if (channel != null) { 332 | channel.close(); 333 | } 334 | f.channel().close(); 335 | logger.warn("filewrite").add(ex).close(); 336 | } 337 | } 338 | } 339 | 340 | static final FileWriter writerFor(HttpEvent request, File f, Logs logger, Config config, ApplicationControl ctrl, Response resp) throws IOException { 341 | CharSequence acceptEncoding = request.header(ACCEPT_ENCODING); 342 | 343 | boolean acceptsGzip = acceptEncoding != null 344 | && (acceptEncoding == HttpHeaderValues.GZIP 345 | || acceptEncoding == HttpHeaderValues.GZIP_DEFLATE 346 | || Strings.charSequenceContains(acceptEncoding, HttpHeaderValues.GZIP, true)); 347 | 348 | long uncompressedLength = f.length(); 349 | 350 | resp.chunked(PREFER_CHUNKED); 351 | if (!acceptsGzip) { 352 | resp.add(Headers.CONTENT_ENCODING, HttpHeaderValues.IDENTITY); 353 | if (!PREFER_CHUNKED) { 354 | resp.add(Headers.CONTENT_LENGTH, uncompressedLength); 355 | } 356 | resp.chunked(false); 357 | return new FileWriter(f, logger, config, ctrl); 358 | } else { 359 | if (isGzipCacheFile(f)) { 360 | resp.add(Headers.CONTENT_ENCODING, HttpHeaderValues.IDENTITY); 361 | if (!PREFER_CHUNKED) { 362 | resp.add(Headers.CONTENT_LENGTH, uncompressedLength); 363 | } 364 | resp.chunked(false); 365 | return new FileWriter(f, logger, config, ctrl); 366 | } 367 | File gzippedFile = new File(f.getParentFile(), "_" + f.getName() + ".gz"); 368 | 369 | if (!gzippedFile.exists()) { 370 | int bufferSize = Math.max(2048, (int) f.length()); 371 | if (!gzippedFile.createNewFile()) { 372 | throw new IOException("Could not create " + gzippedFile); 373 | } 374 | try (BufferedOutputStream out = new BufferedOutputStream(new FileOutputStream(gzippedFile), bufferSize)) { 375 | try (GZO gzOut = new GZO(out, 2048, true)) { 376 | try (BufferedInputStream in = new BufferedInputStream(new FileInputStream(f), 2048)) { 377 | Streams.copy(in, gzOut); 378 | } 379 | } 380 | } 381 | if (config.isDebug()) { 382 | resp.add(Headers.header("X-New-Gzip-File"), "1"); 383 | } 384 | gzippedFile.setLastModified(f.lastModified()); 385 | } 386 | long compressedLength = gzippedFile.length(); 387 | if (compressedLength > uncompressedLength) { 388 | resp.add(Headers.CONTENT_ENCODING, HttpHeaderValues.IDENTITY); 389 | if (!PREFER_CHUNKED) { 390 | resp.add(Headers.CONTENT_LENGTH, uncompressedLength); 391 | } 392 | return new FileWriter(f, logger, config, ctrl); 393 | } 394 | if (!PREFER_CHUNKED) { 395 | resp.add(Headers.CONTENT_LENGTH, compressedLength); 396 | } 397 | resp.add(X_INTERNAL_COMPRESS_HEADER, "true"); 398 | resp.add(Headers.CONTENT_ENCODING, HttpHeaderValues.GZIP); 399 | return new FileWriter(gzippedFile, logger, config, ctrl); 400 | } 401 | } 402 | 403 | static boolean isGzipCacheFile(File f) { 404 | return f.getName().charAt(0) == '_' && f.getName().endsWith(".gz"); 405 | } 406 | 407 | static File gzip(File f) throws IOException { 408 | File gzippedFile = new File(f.getParentFile(), "_" + f.getName() + ".gz"); 409 | 410 | if (!gzippedFile.exists()) { 411 | int bufferSize = Math.max(2048, (int) f.length()); 412 | if (!gzippedFile.createNewFile()) { 413 | throw new IOException("Could not create " + gzippedFile); 414 | } 415 | try (BufferedOutputStream out = new BufferedOutputStream(new FileOutputStream(gzippedFile), bufferSize)) { 416 | try (GZO gzOut = new GZO(out, 2048, true)) { 417 | try (BufferedInputStream in = new BufferedInputStream(new FileInputStream(f), 2048)) { 418 | Streams.copy(in, gzOut); 419 | } 420 | } 421 | } 422 | gzippedFile.setLastModified(f.lastModified()); 423 | } 424 | return gzippedFile; 425 | } 426 | 427 | static final class GZO extends GZIPOutputStream { 428 | 429 | public GZO(OutputStream out, int size) throws IOException { 430 | super(out, size); 431 | this.def.setLevel(Deflater.BEST_COMPRESSION); 432 | } 433 | 434 | public GZO(OutputStream out, int size, boolean syncFlush) throws IOException { 435 | super(out, size, syncFlush); 436 | this.def.setLevel(Deflater.BEST_COMPRESSION); 437 | } 438 | } 439 | 440 | static final class FileWriter implements ChannelFutureListener { 441 | 442 | private final File file; 443 | private final Logs logger; 444 | private final Config config; 445 | private final ApplicationControl ctrl; 446 | 447 | FileWriter(File file, Logs logger, Config config, ApplicationControl ctrl) { 448 | this.file = file; 449 | this.logger = logger; 450 | this.config = config; 451 | this.ctrl = ctrl; 452 | } 453 | 454 | @Override 455 | public void operationComplete(ChannelFuture f) throws Exception { 456 | if (!f.isDone() || f.isSuccess()) { 457 | config.debugLog("Send file region for ", file); 458 | FileRegion region = new DefaultFileRegion(file, 0, file.length()); 459 | ctrl.logFailure(f.channel().writeAndFlush(region)).addListener((ChannelFuture f1) -> { 460 | if (!f1.isDone() || f1.isSuccess()) { 461 | ctrl.logFailure(f1.channel().writeAndFlush(DefaultLastHttpContent.EMPTY_LAST_CONTENT)); 462 | } 463 | }); 464 | } else if (f.channel().isOpen()) { 465 | f.channel().close(); 466 | if (f.cause() != null) { 467 | logger.warn("download").add("file", file.getPath()).add(f.cause()); 468 | } 469 | } 470 | } 471 | 472 | @Override 473 | public String toString() { 474 | return "FileWriter-" + file.getName(); 475 | } 476 | } 477 | 478 | static class Responder2 implements ChannelFutureListener { 479 | 480 | private final ByteBuf buf; 481 | private final Config config; 482 | private final boolean chunked; 483 | private final ApplicationControl ctrl; 484 | 485 | public Responder2(ByteBuf buf, Config config, boolean chunked, ApplicationControl ctrl) { 486 | this.buf = buf; 487 | this.config = config; 488 | this.chunked = chunked; 489 | this.ctrl = ctrl; 490 | } 491 | 492 | @Override 493 | public String toString() { 494 | return "Responder2-" + buf.readableBytes(); 495 | } 496 | 497 | @Override 498 | public void operationComplete(ChannelFuture future) throws Exception { 499 | config.debugLog("use responder2 with ", buf.readableBytes()); 500 | if (future.isDone() && !future.isSuccess()) { 501 | if (future.cause() != null) { 502 | future.cause().printStackTrace(); 503 | } 504 | return; 505 | } 506 | if (chunked) { 507 | ctrl.logFailure(future.channel().writeAndFlush(new DefaultLastHttpContent(buf.retain()))); 508 | } else { 509 | ctrl.logFailure(future.channel().writeAndFlush(buf.retain())); 510 | } 511 | } 512 | } 513 | 514 | private static class DownloadReceiverImpl implements DownloadReceiver { 515 | 516 | final Resumer r; 517 | final Config config; 518 | 519 | public DownloadReceiverImpl(Resumer r, Config config) { 520 | this.r = new WrapperResumer(r); 521 | this.config = config; 522 | } 523 | 524 | @Override 525 | public void receive(HttpResponseStatus status, ByteBuf buf, HttpHeaders headers) { 526 | config.debugLog(" resume with ", buf.readableBytes()); 527 | r.resume(new DownloadResult(status, buf, headers)); 528 | } 529 | 530 | @Override 531 | public void failed(final HttpResponseStatus status) { 532 | config.debugLog(" fail ", status); 533 | r.resume(new DownloadResult(status)); 534 | } 535 | 536 | @Override 537 | public void receive(HttpResponseStatus status, File file, HttpHeaders headers) { 538 | config.debugLog("resume with ", file); 539 | r.resume(new DownloadResult(status, file, headers)); 540 | } 541 | 542 | @Override 543 | public void failed(HttpResponseStatus status, String msg) { 544 | config.debugLog(" fail ", status, msg); 545 | if (msg == null) { 546 | msg = "no-message"; 547 | } 548 | ByteBuf buf = Unpooled.buffer(msg.length() + 4); 549 | buf.writeCharSequence(msg, CharsetUtil.UTF_8); 550 | buf.writeChar('\n'); 551 | r.resume(new DownloadResult(status, buf)); 552 | } 553 | } 554 | 555 | private static class WrapperResumer implements Resumer { 556 | 557 | private AtomicBoolean resumed = new AtomicBoolean(); 558 | private final Resumer resumer; 559 | private Exception ex; 560 | 561 | public WrapperResumer(Resumer resumer) { 562 | this.resumer = resumer; 563 | } 564 | 565 | @Override 566 | public void resume(Object... os) { 567 | if (resumed.compareAndSet(false, true)) { 568 | ex = new Exception(); 569 | resumer.resume(os); 570 | } else { 571 | throw new IllegalStateException("Already resumed", ex); 572 | } 573 | } 574 | } 575 | } 576 | -------------------------------------------------------------------------------- /tiny-maven-proxy/src/main/java/com/mastfrog/tinymavenproxy/GetIndex.java: -------------------------------------------------------------------------------- 1 | /* 2 | * The MIT License 3 | * 4 | * Copyright 2015 Tim Boudreau. 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in 14 | * all copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 22 | * THE SOFTWARE. 23 | */ 24 | package com.mastfrog.tinymavenproxy; 25 | 26 | import com.google.inject.name.Named; 27 | import com.mastfrog.acteur.Acteur; 28 | import com.mastfrog.acteur.CheckIfModifiedSinceHeader; 29 | import com.mastfrog.acteur.HttpEvent; 30 | import com.mastfrog.acteur.annotations.HttpCall; 31 | import com.mastfrog.acteur.annotations.Precursors; 32 | import com.mastfrog.acteur.errors.Err; 33 | import com.mastfrog.acteur.headers.Headers; 34 | import static com.mastfrog.acteur.headers.Method.GET; 35 | import static com.mastfrog.acteur.headers.Method.HEAD; 36 | import com.mastfrog.acteur.preconditions.Description; 37 | import com.mastfrog.acteur.preconditions.Methods; 38 | import com.mastfrog.acteur.preconditions.PathRegex; 39 | import com.mastfrog.acteur.header.entities.CacheControl; 40 | import com.mastfrog.bunyan.java.v2.Logs; 41 | import com.mastfrog.tinymavenproxy.GetIndex.FindIndexFile; 42 | import static com.mastfrog.tinymavenproxy.TinyMavenProxy.ACCESS_LOGGER; 43 | import com.mastfrog.util.time.TimeUtil; 44 | import static com.mastfrog.util.time.TimeUtil.GMT; 45 | import io.netty.channel.ChannelFuture; 46 | import io.netty.channel.ChannelFutureListener; 47 | import io.netty.channel.DefaultFileRegion; 48 | import io.netty.channel.FileRegion; 49 | import java.io.File; 50 | import javax.inject.Inject; 51 | 52 | /** 53 | * 54 | * @author Tim Boudreau 55 | */ 56 | @HttpCall(scopeTypes = File.class, order = -1) 57 | @Methods({GET, HEAD}) 58 | @PathRegex("^.index\\/[^\\/].*$") 59 | @Precursors({FindIndexFile.class, CheckIfModifiedSinceHeader.class}) 60 | @Description(category = "Download", value = "Fetch the index for this repository") 61 | public class GetIndex extends Acteur implements ChannelFutureListener { 62 | 63 | private final File target; 64 | private final Logs accessLog; 65 | private final boolean close; 66 | 67 | @Inject 68 | GetIndex(File target, @Named(ACCESS_LOGGER) Logs accessLog, HttpEvent evt) { 69 | this.target = target; 70 | this.accessLog = accessLog; 71 | ok(); 72 | close = !evt.requestsConnectionStayOpen(); 73 | if (evt.method() != HEAD) { 74 | setResponseBodyWriter(this); 75 | } 76 | } 77 | 78 | @Override 79 | public void operationComplete(ChannelFuture f) throws Exception { 80 | if (f.cause() != null) { 81 | accessLog.error("dlIndex").add(f.cause()).add("file", target.getPath()).close(); 82 | return; 83 | } 84 | FileRegion reg = new DefaultFileRegion(target, 0, target.length()); 85 | f = f.channel().writeAndFlush(reg); 86 | if (close) { 87 | f.addListener(CLOSE); 88 | } 89 | } 90 | 91 | static final class FindIndexFile extends Acteur { 92 | 93 | @Inject 94 | FindIndexFile(HttpEvent evt, Config config) { 95 | File target = new File(config.indexDir(), evt.path().getChildPath().toString()); 96 | if (!target.exists() || !target.isFile()) { 97 | reply(Err.gone("No such file: " + target.getAbsolutePath())); 98 | return; 99 | } 100 | add(Headers.LAST_MODIFIED, TimeUtil.fromUnixTimestamp(target.lastModified(), GMT)); 101 | add(Headers.CACHE_CONTROL, CacheControl.PUBLIC_MUST_REVALIDATE); 102 | next(target); 103 | } 104 | } 105 | 106 | } 107 | -------------------------------------------------------------------------------- /tiny-maven-proxy/src/main/java/com/mastfrog/tinymavenproxy/Noop.java: -------------------------------------------------------------------------------- 1 | /* 2 | * The MIT License 3 | * 4 | * Copyright 2022 Mastfrog Technologies. 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in 14 | * all copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 22 | * THE SOFTWARE. 23 | */ 24 | 25 | package com.mastfrog.tinymavenproxy; 26 | 27 | import java.net.http.HttpResponse; 28 | import java.nio.ByteBuffer; 29 | import java.util.List; 30 | import java.util.concurrent.CompletableFuture; 31 | import java.util.concurrent.CompletionStage; 32 | import java.util.concurrent.Flow; 33 | 34 | /** 35 | * 36 | * @author Tim Boudreau 37 | */ 38 | class Noop implements HttpResponse.BodySubscriber { 39 | 40 | @Override 41 | public CompletionStage getBody() { 42 | return CompletableFuture.completedStage(null); 43 | } 44 | 45 | @Override 46 | public void onSubscribe(Flow.Subscription subscription) { 47 | // do nothing 48 | } 49 | 50 | @Override 51 | public void onNext(List item) { 52 | // do nothing 53 | } 54 | 55 | @Override 56 | public void onError(Throwable throwable) { 57 | // do nothing 58 | } 59 | 60 | @Override 61 | public void onComplete() { 62 | // do nothing 63 | } 64 | 65 | } 66 | -------------------------------------------------------------------------------- /tiny-maven-proxy/src/main/java/com/mastfrog/tinymavenproxy/TempFiles.java: -------------------------------------------------------------------------------- 1 | /* 2 | * The MIT License 3 | * 4 | * Copyright 2022 Mastfrog Technologies. 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in 14 | * all copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 22 | * THE SOFTWARE. 23 | */ 24 | package com.mastfrog.tinymavenproxy; 25 | 26 | import com.google.inject.Inject; 27 | import com.google.inject.Singleton; 28 | import com.mastfrog.function.throwing.ThrowingRunnable; 29 | import com.mastfrog.settings.Settings; 30 | import com.mastfrog.shutdown.hooks.ShutdownHooks; 31 | import java.io.IOException; 32 | import static java.lang.System.getProperty; 33 | import static java.lang.Thread.holdsLock; 34 | import java.net.http.HttpResponse; 35 | import java.net.http.HttpResponse.ResponseInfo; 36 | import java.nio.ByteBuffer; 37 | import java.nio.channels.SeekableByteChannel; 38 | import java.nio.file.FileAlreadyExistsException; 39 | import java.nio.file.Files; 40 | import static java.nio.file.Files.deleteIfExists; 41 | import static java.nio.file.Files.move; 42 | import static java.nio.file.Files.createDirectories; 43 | import static java.nio.file.Files.exists; 44 | import static java.nio.file.Files.newByteChannel; 45 | import java.nio.file.Path; 46 | import java.nio.file.Paths; 47 | import static java.nio.file.StandardCopyOption.ATOMIC_MOVE; 48 | import static java.nio.file.StandardCopyOption.REPLACE_EXISTING; 49 | import static java.nio.file.StandardOpenOption.CREATE_NEW; 50 | import static java.nio.file.StandardOpenOption.TRUNCATE_EXISTING; 51 | import static java.nio.file.StandardOpenOption.WRITE; 52 | import java.nio.file.attribute.FileTime; 53 | import java.time.Instant; 54 | import java.time.ZonedDateTime; 55 | import static java.util.Collections.newSetFromMap; 56 | import static java.util.Collections.synchronizedSet; 57 | import java.util.Optional; 58 | import java.util.Set; 59 | import java.util.WeakHashMap; 60 | import java.util.concurrent.atomic.AtomicLong; 61 | 62 | /** 63 | * 64 | * @author Tim Boudreau 65 | */ 66 | @Singleton 67 | final class TempFiles implements ThrowingRunnable { 68 | 69 | private final java.nio.file.Path tmp; 70 | private static final String PREFIX = "m-dl-"; 71 | private static final AtomicLong COUNTER = new AtomicLong(); 72 | private final Set unclosed 73 | = synchronizedSet(newSetFromMap(new WeakHashMap<>())); 74 | 75 | @Inject 76 | @SuppressWarnings("LeakingThisInConstructor") 77 | TempFiles(Settings settings, ShutdownHooks onShutdown) { 78 | this.tmp = Paths.get(settings.getString("download-tmp", 79 | getProperty("java.io.tmpdir"))); 80 | onShutdown.addLastThrowing(this); 81 | } 82 | 83 | private TempFile addTempFile(TempFile file) { 84 | unclosed.add(file); 85 | return file; 86 | } 87 | 88 | public TempFile tempFile(String dlId) { 89 | Path nue = tmp.resolve(PREFIX + "-" + dlId 90 | + "-" + Long.toString(COUNTER.incrementAndGet(), 36)); 91 | return addTempFile(new TempFile(nue)); 92 | } 93 | 94 | @Override 95 | public void run() throws Exception { 96 | for (TempFile file : unclosed) { 97 | file.close(); 98 | } 99 | } 100 | 101 | static final class TempFile implements AutoCloseable { 102 | 103 | static Boolean atomicMoves; 104 | private final Path path; 105 | private SeekableByteChannel channel; 106 | private boolean closed; 107 | private Instant lastModified; 108 | private HttpResponse.ResponseInfo info; 109 | private Path dest; 110 | 111 | public TempFile(Path path) { 112 | this.path = path; 113 | } 114 | 115 | static void atomicMoveBroken() { 116 | atomicMoves = false; 117 | } 118 | 119 | static boolean canAtomicMove() { 120 | Boolean b = atomicMoves; 121 | if (b == null) { 122 | return true; 123 | } 124 | return b; 125 | } 126 | 127 | public Path path() { 128 | synchronized (this) { 129 | if (dest != null) { 130 | return dest; 131 | } 132 | } 133 | return path; 134 | } 135 | 136 | public Optional lastModified() { 137 | return Optional.ofNullable(lastModified); 138 | } 139 | 140 | public boolean isClosed() { 141 | return closed; 142 | } 143 | 144 | public Optional info() { 145 | return Optional.ofNullable(info); 146 | } 147 | 148 | public synchronized TempFile setLastModified(ZonedDateTime zdt) { 149 | this.lastModified = zdt.toInstant(); 150 | return this; 151 | } 152 | 153 | public synchronized TempFile setLastModified(Instant instant) { 154 | this.lastModified = instant; 155 | return this; 156 | } 157 | 158 | public synchronized TempFile withResponseInfo(ResponseInfo info) { 159 | this.info = info; 160 | return this; 161 | } 162 | 163 | private SeekableByteChannel channel() throws IOException { 164 | assert Thread.holdsLock(this); 165 | if (channel == null) { 166 | channel = newByteChannel(path, WRITE, CREATE_NEW, TRUNCATE_EXISTING); 167 | } 168 | return channel; 169 | } 170 | 171 | public synchronized void append(ByteBuffer buf) throws IOException { 172 | if (closed) { 173 | return; 174 | } 175 | channel().write(buf); 176 | } 177 | 178 | private boolean closeChannel() throws IOException { 179 | assert holdsLock(this); 180 | SeekableByteChannel ch = channel; 181 | if (ch != null) { 182 | ch.close(); 183 | return true; 184 | } 185 | return false; 186 | } 187 | 188 | public synchronized boolean close(Path moveTo) throws IOException { 189 | if (!closed) { 190 | closed = true; 191 | closeChannel(); 192 | if (!exists(moveTo.getParent())) { 193 | try { 194 | createDirectories(moveTo.getParent()); 195 | } catch (FileAlreadyExistsException ex) { 196 | // do nothing 197 | } 198 | } 199 | if (canAtomicMove()) { 200 | try { 201 | move(path, moveTo, REPLACE_EXISTING, ATOMIC_MOVE); 202 | } catch (IOException ex) { 203 | atomicMoveBroken(); 204 | move(path, moveTo, REPLACE_EXISTING); 205 | } 206 | } else { 207 | move(path, moveTo, REPLACE_EXISTING); 208 | } 209 | Instant lm = lastModified; 210 | if (lm != null) { 211 | Files.setLastModifiedTime(moveTo, FileTime.from(lm)); 212 | } 213 | dest = moveTo; 214 | return true; 215 | } 216 | return false; 217 | } 218 | 219 | @Override 220 | public synchronized void close() throws IOException { 221 | if (!closed) { 222 | closed = true; 223 | if (closeChannel()) { 224 | deleteIfExists(path); 225 | } 226 | } 227 | } 228 | } 229 | } 230 | -------------------------------------------------------------------------------- /tiny-maven-proxy/src/main/java/com/mastfrog/tinymavenproxy/TinyMavenProxy.java: -------------------------------------------------------------------------------- 1 | /* 2 | * The MIT License 3 | * 4 | * Copyright 2015 Tim Boudreau. 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in 14 | * all copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 22 | * THE SOFTWARE. 23 | */ 24 | package com.mastfrog.tinymavenproxy; 25 | 26 | import com.google.inject.AbstractModule; 27 | import com.google.inject.Inject; 28 | import com.google.inject.Provider; 29 | import com.google.inject.Scopes; 30 | import com.google.inject.Singleton; 31 | import com.google.inject.name.Named; 32 | import com.google.inject.name.Names; 33 | import com.mastfrog.acteur.Acteur; 34 | import com.mastfrog.acteur.annotations.HttpCall; 35 | import com.mastfrog.acteur.bunyan.ActeurBunyanModule; 36 | import com.mastfrog.acteur.header.entities.CacheControl; 37 | import com.mastfrog.acteur.headers.Headers; 38 | import static com.mastfrog.acteur.headers.Method.GET; 39 | import static com.mastfrog.acteur.headers.Method.HEAD; 40 | import com.mastfrog.acteur.preconditions.Description; 41 | import com.mastfrog.acteur.preconditions.Methods; 42 | import com.mastfrog.acteur.preconditions.PathRegex; 43 | import com.mastfrog.acteur.server.ServerBuilder; 44 | import com.mastfrog.acteur.server.ServerModule; 45 | import static com.mastfrog.acteur.server.ServerModule.BYTEBUF_ALLOCATOR_SETTINGS_KEY; 46 | import static com.mastfrog.acteur.server.ServerModule.DIRECT_ALLOCATOR; 47 | import static com.mastfrog.acteur.server.ServerModule.EVENT_THREADS; 48 | import static com.mastfrog.acteur.server.ServerModule.HTTP_COMPRESSION; 49 | import static com.mastfrog.acteur.server.ServerModule.PORT; 50 | import static com.mastfrog.acteur.server.ServerModule.WORKER_THREADS; 51 | import com.mastfrog.acteur.util.ServerControl; 52 | import com.mastfrog.bunyan.java.v2.Log; 53 | import com.mastfrog.bunyan.java.v2.Logs; 54 | import static com.mastfrog.giulius.SettingsBindings.BOOLEAN; 55 | import static com.mastfrog.giulius.SettingsBindings.INT; 56 | import static com.mastfrog.giulius.SettingsBindings.STRING; 57 | import com.mastfrog.giulius.bunyan.java.v2.LoggingModule; 58 | import static com.mastfrog.giulius.bunyan.java.v2.LoggingModule.SETTINGS_KEY_ASYNC_LOGGING; 59 | import static com.mastfrog.giulius.bunyan.java.v2.LoggingModule.SETTINGS_KEY_LOG_LEVEL; 60 | import com.mastfrog.settings.Settings; 61 | import com.mastfrog.settings.SettingsBuilder; 62 | import com.mastfrog.url.URL; 63 | import com.mastfrog.util.libversion.VersionInfo; 64 | import static com.mastfrog.util.preconditions.Checks.notNull; 65 | import com.mastfrog.util.preconditions.ConfigurationError; 66 | import com.mastfrog.util.streams.Streams; 67 | import com.mastfrog.util.strings.AlignedText; 68 | import com.mastfrog.util.strings.UniqueIDs; 69 | import io.netty.buffer.ByteBuf; 70 | import io.netty.buffer.ByteBufAllocator; 71 | import io.netty.buffer.Unpooled; 72 | import static io.netty.handler.codec.http.HttpResponseStatus.GONE; 73 | import java.io.ByteArrayOutputStream; 74 | import java.io.File; 75 | import java.io.IOException; 76 | import java.io.InputStream; 77 | import java.net.http.HttpClient; 78 | import java.security.MessageDigest; 79 | import java.security.NoSuchAlgorithmException; 80 | import java.time.Duration; 81 | import java.time.ZonedDateTime; 82 | import java.util.Base64; 83 | import java.util.concurrent.ExecutorService; 84 | 85 | /** 86 | * 87 | * @author Tim Boudreau 88 | */ 89 | public class TinyMavenProxy extends AbstractModule { 90 | 91 | public static final String APPLICATION_NAME = "tiny-maven-proxy"; 92 | public static final String SETTINGS_KEY_DOWNLOAD_THREADS = "download.threads"; 93 | public static final String DOWNLOAD_LOGGER = "download"; 94 | public static final String ACCESS_LOGGER = ActeurBunyanModule.ACCESS_LOGGER; 95 | public static final String ERROR_LOGGER = ActeurBunyanModule.ERROR_LOGGER; 96 | public static final String SETTINGS_KEY_DOWNLOAD_CHUNK_SIZE = "download.chunk.size"; 97 | public static final String SETTINGS_KEY_HTTP_CLIENT_CONNECT_TIMEOUT_SECONDS 98 | = "http.client.connect.timeout.seconds"; 99 | static final int DEFAULT_DOWNLOAD_CHUNK_SIZE = 1480; 100 | 101 | static SettingsBuilder defaultSettings() { 102 | return new SettingsBuilder(APPLICATION_NAME) 103 | .add("application.name", APPLICATION_NAME) 104 | .add("cors.enabled", false) 105 | .add("download-tmp", System.getProperty("java.io.tmpdir")) 106 | .add(HTTP_COMPRESSION, "false") 107 | .add(SETTINGS_KEY_DOWNLOAD_THREADS, "24") 108 | .add(SETTINGS_KEY_ASYNC_LOGGING, false) 109 | .add(LoggingModule.SETTINGS_KEY_LOG_TO_CONSOLE, true) 110 | .add(WORKER_THREADS, "6") 111 | .add(EVENT_THREADS, "3") 112 | .add(ServerModule.SETTINGS_KEY_SOCKET_WRITE_SPIN_COUNT, 32) 113 | .add(SETTINGS_KEY_LOG_LEVEL, "trace") 114 | // .add(MAX_CONTENT_LENGTH, "128") // we don't accept PUTs, no need for a big buffer 115 | .add(PORT, "5956") 116 | .add(BYTEBUF_ALLOCATOR_SETTINGS_KEY, DIRECT_ALLOCATOR); 117 | } 118 | 119 | public static void main(String[] args) throws IOException, InterruptedException { 120 | 121 | System.setProperty("acteur.debug", "true"); 122 | 123 | Settings settings = defaultSettings() 124 | .add(LoggingModule.SETTINGS_KEY_LOG_TO_CONSOLE, true) 125 | .add(LoggingModule.SETTINGS_KEY_LOG_FILE, "/tmp/tmproxy.log") 126 | .addFilesystemAndClasspathLocations() 127 | .parseCommandLineArguments(args).build(); 128 | ServerControl ctrl = new ServerBuilder(APPLICATION_NAME) 129 | .add(new TinyMavenProxy()) 130 | .add(new ActeurBunyanModule(true) 131 | .bindLogger(DOWNLOAD_LOGGER).bindLogger("startup") 132 | ) 133 | .add(binder -> { 134 | binder.bind(VersionInfo.class) 135 | .toInstance(VersionInfo.find(TinyMavenProxy.class, 136 | "com.mastfrog", "tiny-maven-proxy")); 137 | 138 | }) 139 | .disableCORS() 140 | .enableOnlyBindingsFor(BOOLEAN, INT, STRING) 141 | .add(settings) 142 | .build().start(); 143 | 144 | ctrl.await(); 145 | } 146 | 147 | @Override 148 | protected void configure() { 149 | bind(HttpClient.class).toProvider(JavaHttpClientProvider.class).in(Scopes.SINGLETON); 150 | bind(StartupLogger.class).asEagerSingleton(); 151 | bind(UniqueIDs.class).toProvider(UniqueIDsProvider.class).in(Scopes.SINGLETON); 152 | bind(ByteBuf.class).annotatedWith(Names.named("index")).toProvider(IndexPageProvider.class); 153 | bind(String.class).annotatedWith(Names.named("indexHash")).toProvider(IndexPageHashProvider.class); 154 | bind(String.class).annotatedWith(Names.named("runId")).toProvider(RunIdProvider.class).in(Scopes.SINGLETON); 155 | } 156 | 157 | @Singleton 158 | static final class IndexPageData { 159 | 160 | final ByteBuf buf; 161 | final String hash; 162 | 163 | @Inject 164 | IndexPageData(ByteBufAllocator alloc) throws IOException, NoSuchAlgorithmException { 165 | byte[] bytes; 166 | try (ByteArrayOutputStream out = new ByteArrayOutputStream();) { 167 | try (InputStream in = Browse.IndexPageWriter.class.getResourceAsStream("index.html")) { 168 | if (in == null) { 169 | throw new ConfigurationError("index.html missing from classpath."); 170 | } 171 | Streams.copy(in, out); 172 | } 173 | bytes = out.toByteArray(); 174 | } 175 | ByteBuf buf = alloc.ioBuffer(bytes.length, bytes.length); 176 | buf.writeBytes(bytes); 177 | buf.retain(); 178 | this.buf = Unpooled.unreleasableBuffer(buf); 179 | byte[] digest = MessageDigest.getInstance("SHA-1").digest(bytes); 180 | this.hash = Base64.getEncoder().encodeToString(digest); 181 | } 182 | } 183 | 184 | public static final String SETTINGS_KEY_UIDS_FILE = "uids.base"; 185 | 186 | @Singleton 187 | static final class RunIdProvider implements Provider { 188 | 189 | private final String runId; 190 | 191 | @Inject 192 | RunIdProvider(UniqueIDs ids) { 193 | runId = ids.newId(); 194 | } 195 | 196 | @Override 197 | public String get() { 198 | return runId; 199 | } 200 | } 201 | 202 | @Singleton 203 | static class JavaHttpClientProvider implements Provider { 204 | 205 | private final java.net.http.HttpClient client; 206 | 207 | @Inject 208 | JavaHttpClientProvider(@Named(ServerModule.BACKGROUND_THREAD_POOL_NAME) ExecutorService executor, 209 | Settings settings) { 210 | client = java.net.http.HttpClient.newBuilder() 211 | .connectTimeout(Duration.ofSeconds( 212 | settings.getLong(SETTINGS_KEY_HTTP_CLIENT_CONNECT_TIMEOUT_SECONDS, 20))) 213 | .followRedirects(java.net.http.HttpClient.Redirect.ALWAYS) 214 | .build(); 215 | } 216 | 217 | @Override 218 | public HttpClient get() { 219 | return client; 220 | } 221 | } 222 | 223 | @Singleton 224 | static final class UniqueIDsProvider implements Provider { 225 | 226 | private final UniqueIDs uniqueIds; 227 | 228 | @Inject 229 | UniqueIDsProvider(Settings settings) throws IOException { 230 | uniqueIds = new UniqueIDs(new File(settings.getString("uids.base", ".uids"))); 231 | } 232 | 233 | @Override 234 | public UniqueIDs get() { 235 | return uniqueIds; 236 | } 237 | } 238 | 239 | static final class StartupLogger { 240 | 241 | @Inject 242 | StartupLogger(Settings settings, Config config, @Named("startup") Logs startup) { 243 | StringBuilder sb = new StringBuilder("TinyMavenProxy 1.5 on port\t" + settings.getInt("port") + " serving:\n"); 244 | try (Log log = startup.info("config")) { 245 | log.add(config); 246 | for (URL u : config) { 247 | sb.append("\tRepo:\t").append(u).append('\n'); 248 | } 249 | sb.append("Settings:\n"); 250 | for (String key : new String[]{SETTINGS_KEY_DOWNLOAD_THREADS, WORKER_THREADS, EVENT_THREADS}) { 251 | sb.append(key).append("\t").append(settings.getString(key)).append('\n'); 252 | log.add(key, settings.getString(key)); 253 | } 254 | sb.append("From:\n").append(config.dir); 255 | } 256 | System.out.println(AlignedText.formatTabbed(sb)); 257 | } 258 | } 259 | 260 | @Singleton 261 | static final class IndexPageProvider implements Provider { 262 | 263 | private final IndexPageData data; 264 | 265 | @Inject 266 | IndexPageProvider(IndexPageData data) throws IOException { 267 | this.data = data; 268 | } 269 | 270 | @Override 271 | public ByteBuf get() { 272 | return data.buf.duplicate(); 273 | } 274 | } 275 | 276 | @Singleton 277 | static final class IndexPageHashProvider implements Provider { 278 | 279 | private final IndexPageData data; 280 | 281 | @Inject 282 | IndexPageHashProvider(IndexPageData data) throws IOException { 283 | this.data = data; 284 | notNull("hash", data.hash); 285 | } 286 | 287 | @Override 288 | public String get() { 289 | return data.hash; 290 | } 291 | } 292 | 293 | @HttpCall(order = Integer.MIN_VALUE) 294 | @PathRegex({"^favicon.ico$"}) 295 | @Methods({GET, HEAD}) 296 | @Description("Sends 404 for /favicon.ico") 297 | static class FaviconPage extends Acteur { 298 | 299 | @Inject 300 | FaviconPage() { 301 | add(Headers.CACHE_CONTROL, CacheControl.PUBLIC_MUST_REVALIDATE_MAX_AGE_1_DAY); 302 | add(Headers.EXPIRES, ZonedDateTime.now().plus(Duration.ofDays(1))); 303 | reply(GONE, "Does not exist\n"); 304 | } 305 | } 306 | 307 | } 308 | -------------------------------------------------------------------------------- /tiny-maven-proxy/src/main/java/com/mastfrog/tinymavenproxy/VersionActeur.java: -------------------------------------------------------------------------------- 1 | /* 2 | * The MIT License 3 | * 4 | * Copyright 2015 Tim Boudreau. 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in 14 | * all copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 22 | * THE SOFTWARE. 23 | */ 24 | package com.mastfrog.tinymavenproxy; 25 | 26 | import com.google.inject.Inject; 27 | import com.mastfrog.acteur.Acteur; 28 | import com.mastfrog.acteur.CheckIfModifiedSinceHeader; 29 | import com.mastfrog.acteur.annotations.HttpCall; 30 | import com.mastfrog.acteur.annotations.Precursors; 31 | import com.mastfrog.acteur.headers.Headers; 32 | import static com.mastfrog.acteur.headers.Method.GET; 33 | import com.mastfrog.acteur.preconditions.Methods; 34 | import com.mastfrog.acteur.preconditions.Path; 35 | import com.mastfrog.acteur.header.entities.CacheControl; 36 | import com.mastfrog.mime.MimeType; 37 | import com.mastfrog.tinymavenproxy.VersionActeur.RevLastModifiedActeur; 38 | import static com.mastfrog.util.collections.CollectionUtils.map; 39 | import java.time.ZoneId; 40 | import java.time.ZonedDateTime; 41 | 42 | /** 43 | * 44 | * @author Tim Boudreau 45 | */ 46 | @HttpCall 47 | @Methods(GET) 48 | @Path("/_version") 49 | @Precursors({RevLastModifiedActeur.class, CheckIfModifiedSinceHeader.class}) 50 | public class VersionActeur extends Acteur { 51 | 52 | @Inject 53 | public VersionActeur() { 54 | String revision = com.mastfrog.tinymavenproxy.RevisionInfo.VERSION; 55 | ok(map("version").finallyTo(revision)); 56 | } 57 | 58 | static class RevLastModifiedActeur extends Acteur { 59 | 60 | RevLastModifiedActeur() { 61 | ZonedDateTime when = ZonedDateTime.ofInstant(com.mastfrog.tinymavenproxy.RevisionInfo.COMMIT_TIMESTAMP, ZoneId.systemDefault()); 62 | add(Headers.LAST_MODIFIED, when); 63 | add(Headers.CACHE_CONTROL, CacheControl.PUBLIC_MUST_REVALIDATE); 64 | add(Headers.CONTENT_TYPE, MimeType.JSON_UTF_8); 65 | next(); 66 | } 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /tiny-maven-proxy/src/main/resources/com/mastfrog/tinymavenproxy/index.html: -------------------------------------------------------------------------------- 1 | Maven Repository 2 | 3 | 4 | 5 | 6 | 7 | 8 | 49 | 50 | 51 |
52 |
53 | 87 |
88 |

Tiny Maven Proxy {{revision}}

89 | Fork me on Github 90 |
91 | 92 | ©Copyright 2015-2019, Tim Boudreau, distributed under the MIT license. 93 | 94 |
95 |
96 |
97 |
98 | Loading... 99 |
100 |
101 | 102 | 103 | 104 | 105 | {{folder.name}} 106 | 107 | 108 |
109 |
110 |
111 | 112 | 113 | 114 | 115 | 156 | 157 | 158 | -------------------------------------------------------------------------------- /tiny-maven-proxy/src/test/java/com/mastfrog/tinymavenproxy/FakeMavenServer.java: -------------------------------------------------------------------------------- 1 | /* 2 | * The MIT License 3 | * 4 | * Copyright 2022 Mastfrog Technologies. 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in 14 | * all copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 22 | * THE SOFTWARE. 23 | */ 24 | package com.mastfrog.tinymavenproxy; 25 | 26 | import com.mastfrog.acteur.bunyan.ActeurBunyanModule; 27 | import com.mastfrog.acteur.server.ServerModule; 28 | import com.mastfrog.acteur.util.ServerControl; 29 | import com.mastfrog.function.throwing.ThrowingRunnable; 30 | import com.mastfrog.giulius.bunyan.java.v2.LoggingModule; 31 | import com.mastfrog.settings.Settings; 32 | import com.mastfrog.simple.webserver.FileServer; 33 | import com.mastfrog.util.file.FileUtils; 34 | import com.mastfrog.util.net.PortFinder; 35 | import com.mastfrog.util.preconditions.ConfigurationError; 36 | import com.mastfrog.util.strings.RandomStrings; 37 | import com.mastfrog.util.strings.Strings; 38 | import static com.mastfrog.util.strings.Strings.capitalize; 39 | import com.telenav.cactus.wordy.WordList; 40 | import com.telenav.cactus.wordy.WordLists; 41 | import static com.telenav.cactus.wordy.WordLists.ADJECTIVES; 42 | import static com.telenav.cactus.wordy.WordLists.NOUNS; 43 | import static com.telenav.cactus.wordy.WordLists.POSESSIVES; 44 | import static com.telenav.cactus.wordy.WordLists.PREPOSITIONS; 45 | import static com.telenav.cactus.wordy.WordLists.VERBS; 46 | import java.io.IOException; 47 | import java.io.OutputStream; 48 | import static java.nio.charset.StandardCharsets.UTF_8; 49 | import java.nio.file.Files; 50 | import java.nio.file.Path; 51 | import java.nio.file.Paths; 52 | import static java.nio.file.StandardOpenOption.CREATE; 53 | import static java.nio.file.StandardOpenOption.TRUNCATE_EXISTING; 54 | import static java.nio.file.StandardOpenOption.WRITE; 55 | import java.nio.file.attribute.FileTime; 56 | import java.security.MessageDigest; 57 | import java.security.NoSuchAlgorithmException; 58 | import java.time.Duration; 59 | import java.time.Instant; 60 | import java.util.ArrayList; 61 | import static java.util.Collections.unmodifiableMap; 62 | import java.util.HashMap; 63 | import java.util.List; 64 | import java.util.Map; 65 | import java.util.Random; 66 | import java.util.Set; 67 | import java.util.function.Consumer; 68 | 69 | /** 70 | * 71 | * @author Tim Boudreau 72 | */ 73 | public final class FakeMavenServer { 74 | 75 | public static Path COMMON_PATH = Paths.get("com/mastfrog/fake-maven-server/1.0.0"); 76 | public static Path COMMON_POM = Paths.get("com/mastfrog/fake-maven-server/1.0.0/com.mastfrog.fake-maven-server-1.0.0.pom"); 77 | public static final String COMMON_BODY = "This is served by all fake-maven-servers. All righty, then.\nWhaddaya think about that?\n\nSo there.\n"; 78 | 79 | private final Random rnd; 80 | private final Map relativePaths = new HashMap<>(); 81 | private final Path dir; 82 | private final PortFinder portFinder; 83 | private int port; 84 | private ServerControl ctrl; 85 | 86 | public FakeMavenServer(Random rnd, int count, PortFinder portFinder) throws IOException, NoSuchAlgorithmException { 87 | this.rnd = rnd; 88 | dir = FileUtils.newTempDir("fake-maven"); 89 | RandomStrings rs = new RandomStrings(rnd); 90 | populate(count); 91 | this.portFinder = portFinder; 92 | } 93 | 94 | public static void startN(int n, Random rnd, int filesPer, PortFinder finder, Consumer c) throws IOException, NoSuchAlgorithmException, InterruptedException { 95 | for (int i = 0; i < n; i++) { 96 | FakeMavenServer oneServer = new FakeMavenServer(rnd, filesPer, finder); 97 | oneServer.start(); 98 | c.accept(oneServer); 99 | } 100 | } 101 | 102 | public static void main(String[] args) throws Exception { 103 | Random r = new Random(3298234); 104 | PortFinder pf = new PortFinder(10000, 20000, 1000); 105 | List all = new ArrayList<>(); 106 | startN(3, r, 2, pf, all::add); 107 | all.get(0).await(); 108 | } 109 | 110 | public void await() throws InterruptedException { 111 | ServerControl ctrl; 112 | synchronized (this) { 113 | ctrl = this.ctrl; 114 | } 115 | if (ctrl == null) { 116 | throw new ConfigurationError("Not started"); 117 | } 118 | ctrl.await(); 119 | } 120 | 121 | public synchronized int port() { 122 | return port; 123 | } 124 | 125 | public String baseUri() { 126 | return "http://localhost:" + port + "/"; 127 | } 128 | 129 | public Set paths() { 130 | return relativePaths.keySet(); 131 | } 132 | 133 | public Map contents() { 134 | return unmodifiableMap(relativePaths); 135 | } 136 | 137 | public String content(Path path) { 138 | return relativePaths.get(path); 139 | } 140 | 141 | public ThrowingRunnable shutdown() { 142 | return () -> { 143 | ServerControl ctrl; 144 | synchronized (FakeMavenServer.this) { 145 | ctrl = this.ctrl; 146 | } 147 | try { 148 | if (ctrl != null) { 149 | ctrl.shutdown(true); 150 | } 151 | } finally { 152 | cleanup(); 153 | } 154 | }; 155 | } 156 | 157 | private void cleanup() throws IOException { 158 | FileUtils.deltree(dir); 159 | } 160 | 161 | public ServerControl start() throws IOException, InterruptedException { 162 | if (ctrl == null) { 163 | FileServer server = new FileServer(dir.toFile()); 164 | ActeurBunyanModule mod = new ActeurBunyanModule() 165 | .setRequestLoggerLevel("info"); 166 | server.withModule(mod); 167 | port = portFinder.findAvailableServerPort(); 168 | Settings settings = Settings.builder() 169 | .add(LoggingModule.SETTINGS_KEY_ASYNC_LOGGING, true) 170 | .add(LoggingModule.SETTINGS_KEY_LOG_LEVEL, "info") 171 | .add(ServerModule.PORT, port) 172 | .add(ServerModule.EVENT_THREADS, 3) 173 | .add(ServerModule.WORKER_THREADS, 12) 174 | .build(); 175 | ctrl = server.startServer(settings); 176 | server.latch().await(); 177 | } 178 | return ctrl; 179 | } 180 | 181 | private void populate(int count) throws IOException, NoSuchAlgorithmException { 182 | populateConsistent(); 183 | for (int i = 0; i < count; i++) { 184 | populateOne(); 185 | } 186 | } 187 | 188 | private void populateConsistent() throws IOException, NoSuchAlgorithmException { 189 | Path base = dir.resolve(COMMON_PATH); 190 | String fileBase = "com.mastfrog.fake-maven-server-1.0.0"; 191 | String txt = COMMON_BODY; 192 | Path fl = base.resolve(fileBase + ".pom"); 193 | writeFileAndContent(fl, txt); 194 | String sha = sha1(txt); 195 | Path shaFile = base.resolve(fileBase + ".pom.sha1"); 196 | writeFileAndContent(shaFile, sha); 197 | } 198 | 199 | private void populateOne() throws IOException, NoSuchAlgorithmException { 200 | Path base = dir; 201 | List parts = new ArrayList<>(); 202 | for (int i = 0; i < 4; i++) { 203 | String part = null; 204 | switch (i) { 205 | case 0: 206 | String[] s = new String[]{"com", "org", "edu", "io"}; 207 | String val = s[rnd.nextInt(s.length)]; 208 | base = base.resolve(val); 209 | parts.add(val); 210 | continue; 211 | case 1: 212 | part = randomWord(WordLists.ADVERBS); 213 | break; 214 | case 2: 215 | part = randomWord(WordLists.VERBS); 216 | break; 217 | case 3: 218 | part = randomWord(WordLists.NOUNS); 219 | } 220 | base = base.resolve(part); 221 | parts.add(part); 222 | } 223 | List ver = new ArrayList<>(); 224 | for (int i = 0; i < 3; i++) { 225 | int val = rnd.nextInt(12) + 1; 226 | String vs = Integer.toString(val); 227 | ver.add(vs); 228 | } 229 | 230 | String gid = Strings.join(".", parts.subList(0, parts.size() - 1)); 231 | String aid = parts.get(parts.size() - 1); 232 | String version = Strings.join('.', ver); 233 | String fileBase = gid + "." + aid + "-" + version; 234 | 235 | base = base.resolve(version); 236 | 237 | Path pomFile = base.resolve(fileBase + ".pom"); 238 | String cnt = randomContent(); 239 | writeFileAndContent(pomFile, cnt); 240 | 241 | // System.out.println("--- " + base.relativize(pomFile) + " ---"); 242 | // System.out.println(cnt); 243 | String sha = sha1(cnt); 244 | Path shaFile = base.resolve(fileBase + ".pom.sha1"); 245 | writeFileAndContent(shaFile, sha); 246 | } 247 | 248 | private String sha1(String what) throws NoSuchAlgorithmException { 249 | byte[] bt = what.getBytes(UTF_8); 250 | MessageDigest dig = MessageDigest.getInstance("SHA-1"); 251 | return Strings.toPaddedHex(dig.digest(bt)); 252 | } 253 | 254 | private void writeFileAndContent(Path file, String content) throws IOException { 255 | if (!Files.exists(file.getParent())) { 256 | Files.createDirectories(file.getParent()); 257 | } 258 | try (OutputStream out = Files.newOutputStream(file, CREATE, WRITE, TRUNCATE_EXISTING)) { 259 | out.write(content.getBytes(UTF_8)); 260 | } 261 | relativePaths.put(dir.relativize(file), content); 262 | 263 | // Give them all an unlikely but consistent date 264 | Files.setLastModifiedTime(file, FileTime.from(WHEN)); 265 | } 266 | 267 | public static Instant WHEN = Instant.ofEpochMilli(0) 268 | .plus(Duration.ofDays(365 * 2)) 269 | .plus(Duration.ofDays(128)); 270 | 271 | private String randomContent() { 272 | StringBuilder sents = new StringBuilder(); 273 | for (int i = 0; i < 10; i++) { 274 | if (sents.length() >= 0) { 275 | sents.append(' '); 276 | } 277 | String noun = randomWord(NOUNS); 278 | String adj = rnd.nextBoolean() ? randomWord(ADJECTIVES) : null; 279 | 280 | String firstWord = adj == null ? noun : adj; 281 | String ar = capitalize(articleFor(firstWord)); 282 | 283 | sents.append(ar).append(' '); 284 | 285 | if (adj != null) { 286 | sents.append(adj).append(' '); 287 | } 288 | sents.append(noun).append(' '); 289 | 290 | sents.append(poorMansPresentTensify(randomWord(VERBS))).append(' '); 291 | 292 | adj = rnd.nextBoolean() ? randomWord(ADJECTIVES) : null; 293 | 294 | String nextNoun = randomWord(NOUNS); 295 | 296 | String nextWord = adj == null ? nextNoun : adj; 297 | 298 | if (rnd.nextBoolean()) { 299 | sents.append(randomWord(POSESSIVES)).append(' '); 300 | } else { 301 | sents.append(articleFor(nextWord)).append(' '); 302 | } 303 | if (adj != null) { 304 | sents.append(adj).append(' '); 305 | } 306 | 307 | sents.append(nextNoun).append(", "); 308 | 309 | if (rnd.nextBoolean()) { 310 | switch (rnd.nextInt(3)) { 311 | case 0: 312 | sents.append("that "); 313 | break; 314 | case 1: 315 | sents.append("which "); 316 | break; 317 | case 2: 318 | sents.append("because it "); 319 | } 320 | switch (rnd.nextInt(3)) { 321 | case 0: 322 | sents.append("turned "); 323 | break; 324 | case 1: 325 | sents.append("is "); 326 | break; 327 | case 2: 328 | sents.append("was "); 329 | break; 330 | case 3: 331 | sents.append("became "); 332 | break; 333 | case 4: 334 | sents.append("likes "); 335 | break; 336 | } 337 | sents.append(randomWord(ADJECTIVES)); 338 | sents.append(", "); 339 | } 340 | 341 | sents.append(randomWord(PREPOSITIONS)).append(' '); 342 | sents.append(randomWord(POSESSIVES)).append(' '); 343 | sents.append(randomWord(NOUNS)).append('.'); 344 | } 345 | return sents.toString().trim() + '\n'; 346 | } 347 | 348 | private String poorMansPresentTensify(String verb) { 349 | int last = verb.length() - 1; 350 | switch (verb.charAt(last)) { 351 | case 's': 352 | return verb; 353 | case 'h': 354 | case 'c': 355 | return verb + "es"; 356 | default: 357 | return verb + "s"; 358 | } 359 | } 360 | 361 | private String articleFor(String wd) { 362 | if (rnd.nextBoolean()) { 363 | return "the"; 364 | } else { 365 | if (isVowelStart(wd)) { 366 | return "an"; 367 | } else { 368 | return "a"; 369 | } 370 | } 371 | } 372 | 373 | private boolean isVowelStart(String wd) { 374 | switch (wd.charAt(0)) { 375 | case 'a': 376 | case 'e': 377 | case 'i': 378 | case 'o': 379 | case 'u': 380 | return true; 381 | default: 382 | return false; 383 | } 384 | } 385 | 386 | private String randomWord(WordList list) { 387 | int sz = list.size(); 388 | return list.word(rnd.nextInt(sz)); 389 | } 390 | } 391 | -------------------------------------------------------------------------------- /tiny-maven-proxy/src/test/java/com/mastfrog/tinymavenproxy/FakeMavenServers.java: -------------------------------------------------------------------------------- 1 | /* 2 | * The MIT License 3 | * 4 | * Copyright 2022 Mastfrog Technologies. 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in 14 | * all copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 22 | * THE SOFTWARE. 23 | */ 24 | package com.mastfrog.tinymavenproxy; 25 | 26 | import com.mastfrog.function.throwing.ThrowingBiFunction; 27 | import com.mastfrog.function.throwing.ThrowingRunnable; 28 | import com.mastfrog.url.URL; 29 | import com.mastfrog.util.net.PortFinder; 30 | import com.mastfrog.util.strings.Strings; 31 | import java.io.IOException; 32 | import java.nio.file.Path; 33 | import java.security.NoSuchAlgorithmException; 34 | import java.util.ArrayList; 35 | import java.util.List; 36 | import java.util.Map; 37 | import java.util.Random; 38 | import java.util.Set; 39 | import java.util.TreeMap; 40 | import java.util.TreeSet; 41 | 42 | /** 43 | * 44 | * @author Tim Boudreau 45 | */ 46 | public class FakeMavenServers { 47 | 48 | private final Random rnd = new Random(39244094092049L); 49 | private final List all = new ArrayList<>(); 50 | private final PortFinder finder; 51 | private final ThrowingRunnable shutdown; 52 | 53 | public FakeMavenServers(int servers, int filesPer) throws IOException, NoSuchAlgorithmException, InterruptedException { 54 | this(servers, filesPer, null); 55 | } 56 | 57 | public FakeMavenServers(int servers, int filesPer, PortFinder finder) throws IOException, NoSuchAlgorithmException, InterruptedException { 58 | this.finder = finder == null ? new PortFinder(10000, 22000) : finder; 59 | FakeMavenServer.startN(servers, rnd, filesPer, finder, all::add); 60 | ThrowingRunnable shut = ThrowingRunnable.oneShot(true); 61 | for (FakeMavenServer f : all) { 62 | shut = shut.andAlways(f.shutdown()); 63 | } 64 | this.shutdown = shut; 65 | } 66 | 67 | public T randomPathAndContent(ThrowingBiFunction c) { 68 | List> l = content(); 69 | Map.Entry e = l.get(rnd.nextInt(l.size())); 70 | return c.toNonThrowing().apply(e.getKey(), e.getValue()); 71 | } 72 | 73 | public PortFinder finder() { 74 | return finder; 75 | } 76 | 77 | public String urlsString() { 78 | return Strings.join(',', urls()); 79 | } 80 | 81 | public List urls() { 82 | List result = new ArrayList<>(); 83 | uris().forEach(u -> result.add(URL.parse(u))); 84 | return result; 85 | } 86 | 87 | public List uris() { 88 | Set result = new TreeSet<>(); 89 | for (FakeMavenServer fms : all) { 90 | result.add(fms.baseUri()); 91 | } 92 | return new ArrayList<>(result); 93 | } 94 | 95 | public List> content() { 96 | Map result = new TreeMap<>(); 97 | for (FakeMavenServer f : all) { 98 | result.putAll(f.contents()); 99 | } 100 | return new ArrayList<>(result.entrySet()); 101 | } 102 | 103 | public String contentFor(Path path) { 104 | for (FakeMavenServer f : all) { 105 | String cnt = f.content(path); 106 | if (cnt != null) { 107 | return cnt; 108 | } 109 | } 110 | return null; 111 | } 112 | 113 | public Set allPaths() { 114 | Set result = new TreeSet<>(); 115 | for (FakeMavenServer f : all) { 116 | result.addAll(f.paths()); 117 | } 118 | return result; 119 | } 120 | 121 | public void shutdown() throws Exception { 122 | shutdown.run(); 123 | for (FakeMavenServer fms : all) { 124 | fms.await(); 125 | } 126 | } 127 | 128 | } 129 | -------------------------------------------------------------------------------- /tiny-maven-proxy/src/test/java/com/mastfrog/tinymavenproxy/GeneralProxyingTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * The MIT License 3 | * 4 | * Copyright 2022 Mastfrog Technologies. 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in 14 | * all copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 22 | * THE SOFTWARE. 23 | */ 24 | package com.mastfrog.tinymavenproxy; 25 | 26 | import com.fasterxml.jackson.databind.ObjectMapper; 27 | import com.google.inject.AbstractModule; 28 | import com.google.inject.Inject; 29 | import com.google.inject.Scopes; 30 | import com.google.inject.TypeLiteral; 31 | import com.google.inject.name.Named; 32 | import com.google.inject.name.Names; 33 | import com.mastfrog.acteur.Application; 34 | import com.mastfrog.acteur.annotations.GenericApplicationModule; 35 | import com.mastfrog.acteur.bunyan.ActeurBunyanModule; 36 | import com.mastfrog.acteur.headers.Headers; 37 | import com.mastfrog.acteur.server.ServerLifecycleHook; 38 | import com.mastfrog.acteur.server.ServerModule; 39 | import com.mastfrog.acteur.util.Server; 40 | import com.mastfrog.acteur.util.ServerControl; 41 | import com.mastfrog.function.throwing.ThrowingRunnable; 42 | import com.mastfrog.giulius.tests.GuiceRunner; 43 | import com.mastfrog.giulius.tests.OnInjection; 44 | import com.mastfrog.giulius.tests.TestWith; 45 | import com.mastfrog.http.harness.HarnessLogLevel; 46 | import com.mastfrog.http.harness.HttpTestHarness; 47 | import com.mastfrog.http.harness.TestResults; 48 | import com.mastfrog.settings.Settings; 49 | import com.mastfrog.shutdown.hooks.ShutdownHooks; 50 | import static com.mastfrog.tinymavenproxy.Config.MAVEN_CACHE_DIR; 51 | import static com.mastfrog.tinymavenproxy.Config.SETTINGS_KEY_MIRROR_URLS; 52 | import static com.mastfrog.tinymavenproxy.FakeMavenServer.WHEN; 53 | import com.mastfrog.tinymavenproxy.GeneralProxyingTest.M; 54 | import static com.mastfrog.tinymavenproxy.TinyMavenProxy.DOWNLOAD_LOGGER; 55 | import com.mastfrog.util.file.FileUtils; 56 | import com.mastfrog.util.libversion.VersionInfo; 57 | import com.mastfrog.util.net.PortFinder; 58 | import com.mastfrog.util.preconditions.Exceptions; 59 | import io.netty.channel.Channel; 60 | import java.io.IOException; 61 | import java.net.URI; 62 | import java.net.http.HttpClient; 63 | import java.net.http.HttpRequest; 64 | import java.net.http.HttpResponse; 65 | import java.net.http.HttpResponse.BodyHandlers; 66 | import java.nio.file.Files; 67 | import java.nio.file.Path; 68 | import java.nio.file.Paths; 69 | import java.security.NoSuchAlgorithmException; 70 | import java.time.Duration; 71 | import java.util.LinkedHashMap; 72 | import java.util.Map; 73 | import java.util.concurrent.CountDownLatch; 74 | import javax.inject.Provider; 75 | import org.junit.Test; 76 | import org.junit.runner.RunWith; 77 | 78 | /** 79 | * 80 | * @author Tim Boudreau 81 | */ 82 | @RunWith(GuiceRunner.class) 83 | @TestWith({M.class, TinyMavenProxy.class}) 84 | public class GeneralProxyingTest { 85 | 86 | static final PortFinder FINDER = new PortFinder(); 87 | static final Duration TIMEOUT = Duration.ofSeconds(60); 88 | 89 | @Test(timeout = 40000) 90 | public void testIt(HttpTestHarness harness, 91 | FakeMavenServers servers, Server server) throws Throwable { 92 | 93 | // System.out.println("PORT " + server.getPort()); 94 | // System.out.println("HAVE SERVERS: "); 95 | // System.out.println(servers.urlsString()); 96 | // for (Path p : servers.allPaths()) { 97 | // System.out.println(" * " + p); 98 | // } 99 | Thread.sleep(2000); 100 | Map>> pending = new LinkedHashMap<>(); 101 | 102 | for (Map.Entry e : servers.content()) { 103 | Thread.currentThread().setName(e.getKey().toString()); 104 | Thread.sleep(100); 105 | pending.put(e.getKey(), harness.get(e.getKey()) 106 | .responseStartTimeout(TIMEOUT) 107 | .responseFinishedTimeout(TIMEOUT) 108 | .applyingAssertions(asser -> { 109 | asser.assertBody(e.getValue().trim()) 110 | .assertOk() 111 | .assertHasHeader("last-modified") 112 | .assertHeader("last-modified", lm -> { 113 | if (lm == null) { 114 | return true; 115 | } 116 | return Headers.LAST_MODIFIED.toValue(lm).toInstant() 117 | .equals(WHEN); 118 | }); 119 | })); 120 | } 121 | 122 | for (int i = 0; i < 5; i++) { 123 | String pth = "xx11/yy22/zz33" + i + ".pom"; 124 | Thread.currentThread().setName(pth); 125 | pending.put(Paths.get(pth), harness.get(pth) 126 | .responseFinishedTimeout(TIMEOUT) 127 | .test(assr -> { 128 | assr.assertGone(); 129 | })); 130 | } 131 | 132 | for (Map.Entry>> r : pending.entrySet()) { 133 | r.getValue().await(TIMEOUT); 134 | } 135 | Throwable ae = null; 136 | for (Map.Entry>> r : pending.entrySet()) { 137 | try { 138 | r.getValue().assertAllSucceeded(); 139 | System.out.println("OK: " + r.getKey()); 140 | } catch (Exception | Error e) { 141 | if (ae == null) { 142 | ae = e; 143 | } else { 144 | ae.addSuppressed(e); 145 | } 146 | } 147 | } 148 | 149 | if (ae != null) { 150 | throw ae; 151 | } 152 | } 153 | 154 | @OnInjection 155 | public void initServers(HttpTestHarness harness, 156 | FakeMavenServers servers) throws IOException, InterruptedException { 157 | HttpClient cl = HttpClient.newHttpClient(); 158 | for (String u : servers.uris()) { 159 | HttpRequest req 160 | = HttpRequest.newBuilder(URI.create(u)) 161 | .GET() 162 | .build(); 163 | cl.send(req, BodyHandlers.discarding()); 164 | } 165 | } 166 | 167 | static class M extends AbstractModule { 168 | 169 | @Override 170 | protected void configure() { 171 | try { 172 | Path cacheDir = FileUtils.newTempDir(); 173 | Files.createDirectories(cacheDir); 174 | int port = FINDER.findAvailableServerPort(); 175 | FakeMavenServers fakes = new FakeMavenServers(4, 2, FINDER); 176 | bind(PortFinder.class).toInstance(FINDER); 177 | bind(FakeMavenServers.class).toInstance(fakes); 178 | bind(new TypeLiteral>() { 179 | }).toProvider(HttpTestHarnessProvider.class) 180 | .in(Scopes.SINGLETON); 181 | bind(String.class) 182 | .annotatedWith(Names.named(SETTINGS_KEY_MIRROR_URLS)) 183 | .toProvider(MirrorProvider.class).in(Scopes.SINGLETON); 184 | 185 | install(new ActeurBunyanModule(true) 186 | .bindLogger(DOWNLOAD_LOGGER).bindLogger("startup") 187 | .setRequestLoggerLevel("info") 188 | // .useProbe(true) 189 | ); 190 | 191 | Settings settings = TinyMavenProxy.defaultSettings() 192 | .add(ServerModule.PORT, port) 193 | .add(ServerModule.EVENT_THREADS, 3) 194 | .add(ServerModule.WORKER_THREADS, 12) 195 | .add(ServerModule.BACKGROUND_THREADS, 12) 196 | .add(SETTINGS_KEY_MIRROR_URLS, fakes.urlsString()) 197 | .add("neverKeepAlive", true) 198 | .add(ServerModule.HTTP_COMPRESSION, false) 199 | .add(MAVEN_CACHE_DIR, cacheDir.toString()) 200 | .build(); 201 | 202 | bind(Config.class).toInstance(new Config(settings)); 203 | 204 | bind(VersionInfo.class) 205 | .toInstance(VersionInfo.find(TinyMavenProxy.class, 206 | "com.mastfrog", "tiny-maven-proxy")); 207 | bind(Path.class).annotatedWith(Names.named("theDir")) 208 | .toInstance(cacheDir); 209 | bind(DeleteCacheDir.class).asEagerSingleton(); 210 | bind(Hooks.class).asEagerSingleton(); 211 | install(new GenericApplicationModule(settings)); 212 | bind(Integer.class).annotatedWith(Names.named(ServerModule.PORT)) 213 | .toInstance(port); 214 | bind(CountDownLatch.class).toInstance(new CountDownLatch(1)); 215 | bind(ServersShutdown.class).asEagerSingleton(); 216 | bind(ServerStartStop.class).asEagerSingleton(); 217 | bind(HarnessShutdown.class).asEagerSingleton(); 218 | 219 | } catch (Exception ex) { 220 | Exceptions.chuck(ex); 221 | } 222 | } 223 | } 224 | 225 | static class DeleteCacheDir implements ThrowingRunnable { 226 | 227 | private final Path path; 228 | 229 | @Inject 230 | DeleteCacheDir(@Named("theDir") Path path, ShutdownHooks hooks) { 231 | this.path = path; 232 | } 233 | 234 | @Override 235 | public void run() throws Exception { 236 | FileUtils.deleteIfExists(path); 237 | } 238 | } 239 | 240 | static class MirrorProvider implements Provider { 241 | 242 | private final Provider servers; 243 | 244 | @Inject 245 | MirrorProvider(Provider servers) { 246 | this.servers = servers; 247 | } 248 | 249 | @Override 250 | public String get() { 251 | return servers.get().urlsString(); 252 | } 253 | 254 | } 255 | 256 | static class ServersShutdown { 257 | 258 | @Inject 259 | ServersShutdown(ShutdownHooks hooks, FakeMavenServers servers) throws IOException, NoSuchAlgorithmException { 260 | hooks.addThrowing(() -> servers.shutdown()); 261 | } 262 | } 263 | 264 | static class ServerStartStop implements ThrowingRunnable { 265 | 266 | final ServerControl server; 267 | 268 | @Inject 269 | @SuppressWarnings("LeakingThisInConstructor") 270 | ServerStartStop(Server server, ShutdownHooks hooks, @Named(ServerModule.PORT) int port) throws IOException { 271 | this.server = server.start(port); 272 | hooks.addThrowing(this); 273 | } 274 | 275 | @Override 276 | public void run() throws Exception { 277 | server.shutdown(true); 278 | } 279 | } 280 | 281 | static class HarnessShutdown implements ThrowingRunnable { 282 | 283 | private final HttpTestHarness harn; 284 | 285 | @Inject 286 | HarnessShutdown(HttpTestHarness harn, ShutdownHooks hooks) { 287 | this.harn = harn; 288 | hooks.addThrowing(this); 289 | } 290 | 291 | @Override 292 | public void run() throws Exception { 293 | harn.shutdown(); 294 | } 295 | } 296 | 297 | static class Hooks extends ServerLifecycleHook { 298 | 299 | private final CountDownLatch latch; 300 | 301 | @Inject 302 | Hooks(Registry reg, CountDownLatch latch) { 303 | super(reg); 304 | this.latch = latch; 305 | } 306 | 307 | @Override 308 | protected void onStartup(Application aplctn, Channel chnl) throws Exception { 309 | latch.countDown(); 310 | } 311 | 312 | @Override 313 | protected void onShutdown() throws Exception { 314 | super.onShutdown(); 315 | } 316 | 317 | } 318 | 319 | static class HttpTestHarnessProvider implements Provider> { 320 | 321 | private final HttpTestHarness harn; 322 | 323 | @Inject 324 | HttpTestHarnessProvider(@Named(ServerModule.PORT) int port, ObjectMapper mapper, CountDownLatch latch) { 325 | harn = HttpTestHarness.builder() 326 | // .logToStderr() 327 | .withMapper(mapper) 328 | .withDefaultRequestIdProvider() 329 | .withMinimumLogLevel(HarnessLogLevel.IMPORTANT) 330 | .awaitingReadinessOn(latch) 331 | // .throttlingRequestsWith(new Semaphore(3)) 332 | .withInitialResponseTimeout(TIMEOUT) 333 | .withDefaultResponseTimeout(TIMEOUT) 334 | .withWatchdogInterval(Duration.ofSeconds(1)) 335 | .withHttpVersion(HttpClient.Version.HTTP_1_1) 336 | .build() 337 | .convertingToUrisWith((Object pth) -> { 338 | return URI.create("http://localhost:" + port + "/" + pth); 339 | }); 340 | } 341 | 342 | @Override 343 | public HttpTestHarness get() { 344 | return harn; 345 | } 346 | 347 | } 348 | } 349 | -------------------------------------------------------------------------------- /tiny-maven-proxy/src/test/java/com/mastfrog/tinymavenproxy/TestBug5.java: -------------------------------------------------------------------------------- 1 | /* 2 | * The MIT License 3 | * 4 | * Copyright 2015 Tim Boudreau. 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in 14 | * all copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 22 | * THE SOFTWARE. 23 | */ 24 | package com.mastfrog.tinymavenproxy; 25 | 26 | import com.mastfrog.settings.Settings; 27 | import static com.mastfrog.tinymavenproxy.Config.SETTINGS_KEY_INDEX_DIR; 28 | import static com.mastfrog.tinymavenproxy.Config.SETTINGS_KEY_MIRROR_URLS; 29 | import java.io.File; 30 | import java.io.IOException; 31 | import java.nio.file.Files; 32 | import java.nio.file.Paths; 33 | import java.util.HashSet; 34 | import java.util.Set; 35 | import static org.junit.Assert.assertEquals; 36 | import static org.junit.Assert.assertFalse; 37 | import static org.junit.Assert.assertTrue; 38 | import org.junit.Test; 39 | 40 | /** 41 | * https://github.com/timboudreau/tiny-maven-proxy/issues/5 42 | * 43 | * @author Tim Boudreau 44 | */ 45 | public class TestBug5 { 46 | 47 | private static final String URL = "http://foo.cloud.engineering.mycompany.com/"; 48 | 49 | @Test 50 | public void ensureManyLabelsInURLHost() throws IOException { 51 | String filePath = System.getProperty("java.io.tmpdir") + "/TestBug5"; 52 | try { 53 | Settings s = Settings.builder() 54 | .add(SETTINGS_KEY_MIRROR_URLS, URL) 55 | .add(SETTINGS_KEY_INDEX_DIR, filePath) 56 | .build(); 57 | Config config = new Config(s); 58 | File exp = new File(filePath).getAbsoluteFile(); 59 | assertEquals(exp, config.indexDir.getAbsoluteFile()); 60 | Set urls = new HashSet<>(); 61 | config.forEach(url -> { 62 | System.out.println("URL: " + url); 63 | urls.add(url.toString()); 64 | }); 65 | assertFalse(urls.isEmpty()); 66 | assertTrue(urls.contains(URL)); 67 | } finally { 68 | Files.deleteIfExists(Paths.get(filePath)); 69 | } 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /tiny-maven-proxy/src/test/resources/com/mastfrog/tinymavenproxy/GeneralProxyingTest.properties: -------------------------------------------------------------------------------- 1 | httpCompression=true 2 | http.compression.debug=false 3 | maven.proxy.debug=false 4 | download.threads=12 5 | application.name=gpttest 6 | cors.enabled=false 7 | log.async=true 8 | log.console=true 9 | log.level=info 10 | maxContentLength=8192 11 | acteur.outbound.socket.write.spin.count=32 12 | -------------------------------------------------------------------------------- /tiny-maven-proxy/src/test/resources/com/mastfrog/tinymavenproxy/GetActeurTest.properties: -------------------------------------------------------------------------------- 1 | httpCompression=true 2 | http.compression.debug=true 3 | maven.proxy.debug=true 4 | --------------------------------------------------------------------------------