├── .gitignore ├── README.md ├── README_en.md ├── build.sbt ├── project ├── build.properties └── plugins.sbt └── src ├── main ├── resources │ ├── admin │ │ ├── admin.css │ │ ├── admin.html │ │ ├── admin.js │ │ ├── bower.json │ │ ├── favicon.ico │ │ ├── index.html │ │ ├── partials │ │ │ ├── connectors.html │ │ │ ├── expireRules.html │ │ │ ├── immediate404Rules.html │ │ │ ├── login.html │ │ │ ├── modifyPassword.html │ │ │ ├── parameters.html │ │ │ ├── proxies.html │ │ │ └── upstreams.html │ │ └── repox.config.json │ ├── application.conf │ └── logback.xml └── scala │ └── com │ └── gtan │ └── repox │ ├── ExpirationManager.scala │ ├── ExpirationMigrator.scala │ ├── FileDeleter.scala │ ├── GetAsyncHandler.scala │ ├── GetMaster.scala │ ├── GetQueueWorker.scala │ ├── GetWorker.scala │ ├── Head404Cache.scala │ ├── HeadAsyncHandler.scala │ ├── HeadMaster.scala │ ├── HeadQueueWorker.scala │ ├── HeadWorker.scala │ ├── HttpHelpers.scala │ ├── JsonSerializer.scala │ ├── Main.scala │ ├── Repox.scala │ ├── RequestQueueMaster.scala │ ├── Requests.scala │ ├── TimeoutableFuture.scala │ ├── Timestamp.scala │ ├── admin │ ├── AuthHandler.scala │ ├── ConnectorVO.scala │ ├── ConnectorsHandler.scala │ ├── ExpireRulesHandler.scala │ ├── Immediate404RulesHandler.scala │ ├── ParametersHandler.scala │ ├── ProxiesHandler.scala │ ├── RepoVO.scala │ ├── ResetHandler.scala │ ├── RestHandler.scala │ ├── StaticAssetHandler.scala │ ├── UpstreamsHandler.scala │ └── WebConfigHandler.scala │ ├── config │ ├── Config.scala │ ├── ConfigFormats.scala │ ├── ConfigPersister.scala │ ├── ConfigQuery.scala │ ├── ConnectorPersister.scala │ ├── ExpireRulePersister.scala │ ├── Immediate404RulePersister.scala │ ├── ParameterPersister.scala │ ├── ProxyPersister.scala │ └── RepoPersister.scala │ └── data │ ├── Connector.scala │ ├── DurationFormat.scala │ ├── ExpireRule.scala │ ├── Immediate404Rule.scala │ ├── ProxyServer.scala │ └── Repo.scala └── test └── scala └── com └── gtan └── repox ├── GetMasterSuite.scala └── RepoxSuite.scala /.gitignore: -------------------------------------------------------------------------------- 1 | build/ 2 | .gradle/ 3 | .idea/ 4 | *.iml 5 | *.log 6 | *.swp 7 | .DS_Store 8 | 9 | target/ 10 | bower_components/ 11 | node_modules/ 12 | angular-semantic-ui/ 13 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | :exclamation: 第一个 2.12 版发布，assembly 包减小了近 5M（11.8%） 2 | 3 | :exclamation: 2016年5月27日，Repox 公服出口带宽升至2M 4 | 5 | :exclamation: Repox 公服正式迁移至阿里云, 名称由 "广谈公服" 改为 "社区公服". 正在募捐, [查看详情](http://centaur.github.io/repox/) 6 | 7 | [![Gitter](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/Centaur/repox?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) 8 | 9 | ### Repox是什么 10 | Repox的主要目标是改善sbt解决依赖的速度，但由于它的服务方式与url的格式无关，因此也支持ivy, gradle, maven, leiningen客户端，可以作为nexus/artifactory的替代品来搭建私服。 11 | 12 | 如果对Repox的背景不感兴趣，希望快速脏手，可立即前往[入门指南](https://github.com/Centaur/repox/wiki/入门指南) 13 | 14 | ### 为什么需要Repox 15 | * 每次sbt项目的依赖发生变更，或是sbt版本升级时，sbt会先resolve很长时间，然后download很长时间，甚至有时显示downloading，但你发现事实上没有网络流量产生，sbt假活。 16 | * 为了对已经下载过的依赖进行缓存，保证后续请求能够快速完成，你安装了nexus，并把所有sbt可能用到的仓库以及国内的代理仓库（如oschina等）都设置为nexus的上游，却发现它对于sbt更新依赖慢的问题并没有什么改善。 17 | * 你试图安装typesafe官方使用的Artifactory，却发现其开源版本不支持sbt所使用的自定义url格式。 18 | 19 | ### sbt为什么慢 20 | * sbt默认所使用的仓库都在国外。 21 | * 从某一个版本起，sbt默认使用https协议访问仓库。由于众所周知的原因，连往国外的https连接不稳定，而sbt对这种网络连接的不稳定没有相应的发现和重试机制。 22 | * sbt对依赖的请求分为四个步骤： 23 | 1. resolve : 向所有仓库（内置的如typesafe repo, central maven等，以及build中添加的resolvers）逐个发送HEAD请求询问是否收藏有此文件 24 | 2. download : 向收藏了此文件的仓库发送Get请求下载文件 25 | 3. resolve checksum : 向上游仓库发送HEAD请求询问是否有此文件的sha1 checksum 26 | 4. download checksum : 如果HEAD返回200，则下载 sha1 文件并保证从原文件计算出的sha1与checksum文件的内容一致 27 | * 如果没有组织内私服，那么只在每个开发者的本地`~/.ivy2/cache`目录下有依赖缓存，无法在多个开发者之间共享。 28 | * 如果我们需要源码包（这对使用ide的开发者很正常），sbt连巨大的javadoc包也要一起下载。 29 | * 各种bug，例如 [这个](https://github.com/sbt/sbt/issues/413) 30 | 31 | ### 为什么nexus私服对sbt没什么帮助 32 | * nexus是纯Maven仓库，不代理ivy格式的文件。 33 | * sbt的resolve使用的是HEAD请求，即使是已经缓存过的文件，nexus对HEAD请求每次仍要再到上游仓库去重新逐个询问。 34 | * nexus中的多个上游仓库的次序是设定的。对于尚未缓存的文件，在download环节nexus不会根据HEAD请求的结果剔除掉不包含此文件的仓库。 35 | * 根据我们的试用，nexus开源版本质量不高，在不佳的网络条件下，其应用内部状态很容易就僵死。 36 | 37 | ### Repox的理念 38 | * 代理所有流量 39 | 40 | 这样可以在所有环节进行优化。 41 | 42 | * 要么快速完成，要么快速失败 43 | 44 | * 如果上游仓库中有请求的文件，尽量选取最快的仓库下载。 45 | * 下载过程中如果发现当前连往上游仓库的网络连接长久没有获得数据，则终止重试。 46 | * 如果所有上游仓库都失败，则向sbt返回404，让用户重试，而不是永久等待。 47 | 48 | **重要提示：这一条意味着使用repox时，sbt将有更大的概率更新失败，提示“download failed”错误。这时只需要重试失败的指令即可。我们认为多重试几次是比漫长的等待更友好的用户体验。** 49 | 50 | * 与url的格式无关 51 | 52 | 因此除了改善sbt解决依赖的速度，Repox 还可以为ivy, gradle, maven, leiningen客户端服务。 53 | 54 | * 做代理该做的事 55 | 56 | 对于已经下载过的文件，保证resolve立即成功。 57 | 58 | ### Repox的策略 59 | * 对已下载的文件 60 | 61 | 无论是HEAD还是GET请求都立即响应。 62 | 63 | * 对首次请求的文件 64 | 65 | * HEAD请求：同时向所有上游仓库发出询问，取最早200响应的头信息返回给sbt 66 | * GET请求： 67 | * 首先根据HEAD请求的结果对上游仓库进行过滤，HEAD返回非200的仓库被排除 68 | * 上游仓库被指定不同的优先级，向同一优先级的所有仓库同时发送GET请求，选取第一个返回200的仓库下载，其它仓库的连接被终止 69 | * 当某一优先级的所有仓库均失败（返回非200或超时）时，自动进入下一优先级 70 | * 如果所有仓库都失败，则404 71 | * 支持断点续传 72 | * 有些资源只在某特定的仓库中收藏，而到这个仓库的网络状况比较差。Repox可为某个上游仓库指定HTTP代理 73 | * 可设置全面禁止javadoc包的下载 74 | 75 | 更多的细节请参阅其它Wiki页。 76 | 77 | ### 社区公服 404 了怎么办 78 | 现在越来越多的 artifact 没有被放进 maven 中央仓库, 而是零散地托管在一些小众仓库中。如果社区公服中没有将这样的小众仓库添加为上游仓库, 会导致404. 79 | 这时候你可以临时将 `sbt.override.build.repos` 设置为 `false`, 待依赖解析下载完成后恢复成 `true`; 或者选择一种方式申请向社区公服添加上游仓库: 80 | 81 | * 到 github 上提交 issue 82 | * 发邮件给 43284683 AT qq.com 83 | 84 | ### Repox的优势 85 | * 轻 86 | 87 | 核心代码仅十几个文件，每个文件不超过200行。约一个周末的阅读量。 88 | 89 | * 全异步非阻塞 90 | 91 | undertow + akka + async-http-client 92 | 93 | * 提供了web配置界面来根据用户的网络状况对仓库、代理、参数等进行微调 94 | 95 | * 实用 96 | 97 | Repox是由我们团队自己的需求驱动的，它支持着我们的日常开发任务。 98 | 99 | ### 推荐的使用方式 100 | 1. 作为组织内私服运行 101 | 102 | 这是最推荐的使用方式，这样组织内有一个开发者完成一个项目的依赖更新后，其它开发者能够飞速更新。 103 | 104 | 1. 开发者本机运行 105 | 106 | repox的部署和运行都非常简单，在本机运行也能显著提高生产力。 107 | -------------------------------------------------------------------------------- /README_en.md: -------------------------------------------------------------------------------- 1 | [![Gitter](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/Centaur/repox?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) 2 | 3 | ### What is Repox 4 | The main motivation of Repox is to speedup sbt dependency resolving. But it can be used as ivy/gradle/ maven/leiningen proxy as well，as a replacement of Nexus/Artifactory. 5 | 6 | Currently Repox only do proxying, there is no support for artifact publishing. 7 | 8 | To get your hands dirty immediately, go [Getting Start](https://github.com/Centaur/repox/wiki/Getting-Started) 9 | 10 | ### Why Repox 11 | 12 | We have tried Nexus and Artifactory open source version as private proxy, neither has brought improvements for sbt projects. 13 | 14 | ### Repox Philosophy 15 | 16 | * Proxy Everything 17 | 18 | So that we can introduce improvments at every step. 19 | 20 | * Succeed fast or Fail fast 21 | 22 | * If multiple upstream repos have the request file, choose the fastest one. 23 | * If a connection did not get data for a long time, abort and redownload。 24 | * Remember failed request to make less duplicate. 25 | 26 | **Important Note: This means that repox may have more chance to “download failed”。You just reinvoke the failed command. We believe that more retry brings better experience than waiting forever. ** 27 | 28 | * Do what a proxy is supposed to do 29 | 30 | If a file has ever been downloaded, response immediately, without asking upstreams. 31 | 32 | Read the wiki pages for details. 33 | 34 | ### Repox Pros 35 | 36 | * Lightweight 37 | 38 | Just a dozen of source code files for the core functionality, each less then 200 lines. Just a weekend's reading. 39 | 40 | * Reactive overall 41 | 42 | undertow + akka + async-http-client 43 | 44 | * Adjust configurations by web admin 45 | 46 | * Practical 47 | 48 | We build Repox for our own needs and we are using it everyday. 49 | 50 | ### Suggested Deployment 51 | 1. As Organization Private Proxy 52 | 53 | Recommended. So that once a developer has done `sbt update` a project, others' updating feels like using local repo. 54 | 55 | 1. Run locally 56 | 57 | The deployment and running are very simple，it brings good productivity too when running locally。 58 | -------------------------------------------------------------------------------- /build.sbt: -------------------------------------------------------------------------------- 1 | name := "repox" 2 | 3 | organization := "com.gtan" 4 | 5 | scalaVersion := "2.12.6" 6 | 7 | val akkaVersion = "2.5.14" 8 | 9 | libraryDependencies ++= { 10 | val undertowVer = "2.0.11.Final" 11 | val logbackVer = "1.2.3" 12 | val leveldbVer = "0.7" 13 | val leveldbjniVer = "1.8" 14 | val scalaTestVer = "3.0.5" 15 | val playJsonVer = "2.6.7" 16 | val scalaLoggingVer = "3.9.0" 17 | val ningVer = "1.9.40" 18 | val protobufVer = "3.6.1" 19 | Seq( 20 | "io.undertow" % "undertow-core" % undertowVer, 21 | ("com.ning" % "async-http-client" % ningVer) 22 | .exclude("org.slf4j", "slf4j-api"), 23 | ("com.typesafe.scala-logging" %% "scala-logging" % scalaLoggingVer) 24 | .exclude("org.scala-lang", "scala-library") 25 | .exclude("org.scala-lang", "scala-reflect"), 26 | ("ch.qos.logback" % "logback-classic" % logbackVer) 27 | .exclude("org.slf4j", "slf4j-api"), 28 | ("com.typesafe.akka" %% "akka-actor" % akkaVersion) 29 | .exclude("org.scala-lang", "scala-library") 30 | .exclude("org.slf4j", "slf4j-api"), 31 | ("com.typesafe.akka" %% "akka-slf4j" % akkaVersion) 32 | .exclude("org.scala-lang", "scala-library") 33 | .exclude("org.slf4j", "slf4j-api"), 34 | ("com.typesafe.akka" %% "akka-agent" % akkaVersion) 35 | .exclude("org.scala-lang", "scala-library"), 36 | ("com.typesafe.akka" %% "akka-persistence" % akkaVersion) 37 | .exclude("org.scala-lang", "scala-library"), 38 | ("org.iq80.leveldb" % "leveldb" % leveldbVer) 39 | .exclude("com.google.guava", "guava"), 40 | "org.fusesource.leveldbjni" % "leveldbjni-all" % leveldbjniVer, 41 | ("com.typesafe.akka" %% "akka-persistence-query" % akkaVersion) 42 | .exclude("org.scala-lang", "scala-library") 43 | .exclude("com.typesafe", "config") 44 | .exclude("com.typesafe", "ssl-config-akka_2.12") 45 | .exclude("com.typesafe.akka", "akka-testkit_2.12") 46 | .exclude("com.typesafe.akka", "akka-stream-testkit_2.12"), 47 | ("com.typesafe.play" %% "play-json" % playJsonVer) 48 | .exclude("org.scala-lang", "scala-library"), 49 | "com.google.protobuf" % "protobuf-java" % protobufVer, 50 | "org.scalatest" %% "scalatest" % scalaTestVer % "test" 51 | ) 52 | } 53 | 54 | transitiveClassifiers := Seq("sources") 55 | updateOptions := updateOptions.value.withGigahorse(false) 56 | scalacOptions ++= Seq( 57 | "-feature", 58 | "-deprecation", 59 | "-language:implicitConversions", 60 | // "-language:higherKinds", 61 | // "-language:existentials", 62 | "-language:postfixOps" 63 | ) 64 | 65 | fork := true 66 | 67 | assemblyMergeStrategy in assembly := { 68 | case str@PathList("admin", "bower_components", remains@_*) => remains match { 69 | case Seq("angular", "angular.min.js") => MergeStrategy.deduplicate 70 | case Seq("angular-route", "angular-route.min.js") => MergeStrategy.deduplicate 71 | case Seq("ng-file-upload", "ng-file-upload.min.js") => MergeStrategy.deduplicate 72 | case Seq("underscore", "underscore-min.js") => MergeStrategy.deduplicate 73 | case Seq("jquery", "dist", "jquery.min.js") => MergeStrategy.deduplicate 74 | case Seq("semantic-ui", "dist", "semantic.min.css") => MergeStrategy.deduplicate 75 | case Seq("semantic-ui", "dist", "semantic.min.js") => MergeStrategy.deduplicate 76 | case Seq("semantic-ui", "dist", "themes", "default", "assets", _*) => MergeStrategy.deduplicate 77 | case _ => MergeStrategy.discard 78 | } 79 | case x => 80 | (assemblyMergeStrategy in assembly).value.apply(x) 81 | } 82 | 83 | mainClass in assembly := Some("com.gtan.repox.Main") 84 | 85 | test in assembly := {} 86 | -------------------------------------------------------------------------------- /project/build.properties: -------------------------------------------------------------------------------- 1 | sbt.version=1.2.0 2 | -------------------------------------------------------------------------------- /project/plugins.sbt: -------------------------------------------------------------------------------- 1 | 2 | addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.14.7") 3 | 4 | 5 | -------------------------------------------------------------------------------- /src/main/resources/admin/admin.css: -------------------------------------------------------------------------------- 1 | #page-header { 2 | margin-top: 0.5em ; 3 | margin-left: 0.5em ; 4 | } 5 | 6 | .ui.table thead th, .ui.table td { 7 | text-align: center; 8 | } 9 | 10 | .ui.tiny.icon.button { 11 | padding: .39em; 12 | } 13 | 14 | .ui.modal { 15 | margin-top: -15%; 16 | } 17 | .ui.modal > .header { 18 | padding: 0.6rem 2rem; 19 | } 20 | 21 | .ui.form select { 22 | height: 2.5em; 23 | line-height: 2.5em; 24 | } 25 | 26 | .ui.authForm { 27 | width: 50%; 28 | margin: 0 auto; 29 | } 30 | 31 | #progressBar { 32 | position: fixed; 33 | top: 0; 34 | left: 0; 35 | height: 2px; 36 | background-color: seagreen; 37 | } 38 | -------------------------------------------------------------------------------- /src/main/resources/admin/admin.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 |

11 | 12 | 13 | 14 | Repox 15 |
Make your sbt more responsive.
16 | 17 |

18 |
19 |
20 | 61 |
62 |
63 |
64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | -------------------------------------------------------------------------------- /src/main/resources/admin/bower.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "repox", 3 | "version": "0.0.0", 4 | "homepage": "https://github.com/Centaur/repox", 5 | "authors": [ 6 | "oldpig <43284683@qq.com>" 7 | ], 8 | "keywords": [ 9 | "repox" 10 | ], 11 | "license": "MIT", 12 | "private": true, 13 | "ignore": [ 14 | "**/.*", 15 | "node_modules", 16 | "bower_components", 17 | "test", 18 | "tests" 19 | ], 20 | "dependencies": { 21 | "semantic-ui": "1.12.2", 22 | "angular": "~1.3.6", 23 | "angular-route": "~1.3.6", 24 | "underscore": "*", 25 | "ng-file-upload": "~10.1.9" 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /src/main/resources/admin/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Centaur/repox/714a17dcdcbae13f59963874412a17818525d3e8/src/main/resources/admin/favicon.ico -------------------------------------------------------------------------------- /src/main/resources/admin/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 8 | 9 | 10 | 如果浏览器没有自动跳转, 请点击 这里 11 | 12 | -------------------------------------------------------------------------------- /src/main/resources/admin/partials/connectors.html: -------------------------------------------------------------------------------- 1 |
2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 34 | 35 | 36 | 37 | 38 | 43 | 44 | 45 |
NameconnectionTimeoutidleTimeoutmaxConnectionsmaxConnectionsPerHostCredentialsUse Proxy
18 | 19 | 20 | 21 | 22 | 23 | 24 | {{vo.connector.name}}{{vo.connector.connectionTimeout}}{{vo.connector.connectionIdleTimeout}}{{vo.connector.maxConnections}}{{vo.connector.maxConnectionsPerHost}}{{vo.connector.credentials ? (vo.connector.credentials | displayCredentials) : 'None'}} 32 | {{vo.proxy ? (vo.proxy|displayProxy) : 'No Proxy'}} 33 |
39 | 40 | Add Connector 41 | 42 |
46 |
47 | 48 | 118 | 119 | 189 | -------------------------------------------------------------------------------- /src/main/resources/admin/partials/expireRules.html: -------------------------------------------------------------------------------- 1 |
2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 23 | 24 | 25 | 26 | 27 | 28 | 33 | 34 |
patternduration
13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | {{rule.pattern}}{{rule.duration}}
29 | 30 | Add Expire Rule 31 | 32 |
35 |
36 | 37 | 59 | -------------------------------------------------------------------------------- /src/main/resources/admin/partials/immediate404Rules.html: -------------------------------------------------------------------------------- 1 |
2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 24 | 25 | 26 | 27 | 28 | 29 | 34 | 35 |
includeexclude
13 | 14 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | {{rule.include}}{{rule.exclude}}
30 | 31 | Add Immediate 404 Rule 32 | 33 |
36 |
37 | 38 | 61 | -------------------------------------------------------------------------------- /src/main/resources/admin/partials/login.html: -------------------------------------------------------------------------------- 1 |
2 |
3 |
4 | 5 | 6 |
7 | 8 |
9 |
-------------------------------------------------------------------------------- /src/main/resources/admin/partials/modifyPassword.html: -------------------------------------------------------------------------------- 1 |
2 |
3 |
4 | 5 | 6 |
7 |
8 | 9 | 10 |
11 |
Submit 13 |
14 |
15 |
-------------------------------------------------------------------------------- /src/main/resources/admin/partials/parameters.html: -------------------------------------------------------------------------------- 1 |
2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 17 | 18 | 19 | 20 | 21 |
Namevalue
13 | 14 | 15 | 16 | {{parameter.name}}{{parameter | displayParameter }} {{parameter.unit}}
22 |
23 | 41 | 42 | -------------------------------------------------------------------------------- /src/main/resources/admin/partials/proxies.html: -------------------------------------------------------------------------------- 1 |
2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 38 | 39 |
NameProtocolHostPort
15 | 16 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | {{proxy.name}}{{proxy.protocol}}{{proxy.host}}{{proxy.port}}
34 | 35 | Add Proxy 36 | 37 |
40 |
41 | 42 | 74 | 106 | -------------------------------------------------------------------------------- /src/main/resources/admin/partials/upstreams.html: -------------------------------------------------------------------------------- 1 |
2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 37 | 38 | 39 | 40 | 41 | 44 | 47 | 50 | 51 | 52 | 53 | 54 | 59 | 60 | 61 |
NamePriorityBase UrlParentPure Maven 12 | Get OnlyUse Connector
20 | 21 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | {{upstream.repo.name}}{{upstream.repo.priority}}{{upstream.repo.base}}{{getRepoNameById(upstream.repo.parentId)}} 42 | {{upstream.repo.maven ? 'Yes' : 'No'}} 43 | 45 | {{upstream.repo.getOnly ? 'Yes' : 'No'}} 46 | 48 | {{upstream.connector ? upstream.connector.name : 'default'}} 49 |
55 | 56 | Add Repo 57 | 58 |
62 |
63 | 64 | 134 | 135 | 206 | -------------------------------------------------------------------------------- /src/main/resources/admin/repox.config.json: -------------------------------------------------------------------------------- 1 | {"proxies":[{"id":1,"name":"Lantern","protocol":"HTTP","host":"localhost","port":8787,"disabled":false},{"id":2,"name":"Vultr","protocol":"HTTP","host":"108.61.163.157","port":8888,"disabled":false}],"repos":[{"id":1,"name":"koala","base":"http://nexus.openkoala.org/content/groups/Koala-release","priority":1,"getOnly":true,"maven":true,"disabled":false,"parentId":7},{"id":4,"name":"oschina","base":"http://maven.oschina.net/content/groups/public","priority":1,"getOnly":true,"maven":true,"disabled":false,"parentId":7},{"id":3,"name":"typesafe","base":"http://dl.bintray.com/typesafe/ivy-releases","priority":2,"getOnly":false,"maven":false,"disabled":false},{"id":10,"name":"jcenter","base":"http://jcenter.bintray.com","priority":2,"getOnly":false,"maven":true,"disabled":false},{"id":16,"name":"typesafe-maven","base":"http://dl.bintray.com/typesafe/maven-releases/","priority":2,"getOnly":false,"maven":true,"disabled":false},{"id":5,"name":"sbt-plugin","base":"http://dl.bintray.com/sbt/sbt-plugin-releases","priority":4,"getOnly":false,"maven":false,"disabled":false},{"id":9,"name":"scalajs","base":"http://dl.bintray.com/content/scala-js/scala-js-releases","priority":4,"getOnly":false,"maven":false,"disabled":false},{"id":7,"name":"central","base":"http://repo1.maven.org/maven2","priority":4,"getOnly":false,"maven":true,"disabled":false},{"id":14,"name":"tut-plugin","base":"http://dl.bintray.com/content/tpolecat/sbt-plugin-releases","priority":5,"getOnly":false,"maven":false,"disabled":false},{"id":6,"name":"scalaz","base":"http://dl.bintray.com/scalaz/releases","priority":5,"getOnly":false,"maven":true,"disabled":false},{"id":11,"name":"spray","base":"http://repo.spray.io","priority":5,"getOnly":false,"maven":true,"disabled":false},{"id":2,"name":"sonatype","base":"https://oss.sonatype.org/content/repositories/releases","priority":5,"getOnly":false,"maven":true,"disabled":false},{"id":15,"name":"tut-bintray","base":"http://dl.bintray.com/tpolecat/maven","priority":5,"getOnly":false,"maven":true,"disabled":false},{"id":17,"name":"twitter","base":"https://maven.twttr.com","priority":5,"getOnly":false,"maven":true,"disabled":true},{"id":8,"name":"ibiblio","base":"http://mirrors.ibiblio.org/maven2","priority":6,"getOnly":false,"maven":true,"disabled":true},{"id":12,"name":"scoverage","base":"http://dl.bintray.com/sksamuel/sbt-plugins","priority":5,"getOnly":false,"maven":false,"disabled":false},{"id":13,"name":"gatling-plugin","base":"http://dl.bintray.com/content/gatling/sbt-plugins","priority":5,"getOnly":false,"maven":false,"disabled":false}],"connectorUsage":[{"repo":{"id":1,"name":"koala","base":"http://nexus.openkoala.org/nexus/content/groups/Koala-release","priority":1,"getOnly":true,"maven":true,"disabled":false},"connector":{"id":2,"name":"fast-upstream","connectionTimeout":"5 seconds","connectionIdleTimeout":"5 seconds","maxConnections":3000,"maxConnectionsPerHost":1000}},{"repo":{"id":9,"name":"scalajs","base":"http://dl.bintray.com/content/scala-js/scala-js-releases","priority":4,"getOnly":false,"maven":false,"disabled":false},"connector":{"id":3,"name":"slow-upstream","connectionTimeout":"15 seconds","connectionIdleTimeout":"30 seconds","maxConnections":3000,"maxConnectionsPerHost":1000}},{"repo":{"id":5,"name":"sbt-plugin","base":"http://dl.bintray.com/sbt/sbt-plugin-releases","priority":4,"getOnly":false,"maven":false,"disabled":false},"connector":{"id":3,"name":"slow-upstream","connectionTimeout":"15 seconds","connectionIdleTimeout":"30 seconds","maxConnections":3000,"maxConnectionsPerHost":1000}},{"repo":{"id":1,"name":"koala","base":"http://nexus.openkoala.org/content/groups/Koala-release","priority":1,"getOnly":true,"maven":true,"disabled":false},"connector":{"id":2,"name":"fast-upstream","connectionTimeout":"5 seconds","connectionIdleTimeout":"5 seconds","maxConnections":3000,"maxConnectionsPerHost":1000}},{"repo":{"id":16,"name":"typesafe-maven","base":"https://repo.typesafe.com/typesafe/maven-releases/","priority":2,"getOnly":false,"maven":true,"disabled":false},"connector":{"id":3,"name":"slow-upstream","connectionTimeout":"15 seconds","connectionIdleTimeout":"30 seconds","maxConnections":3000,"maxConnectionsPerHost":1000}},{"repo":{"id":16,"name":"typesafe-maven","base":"http://dl.bintray.com/typesafe/maven-releases/","priority":2,"getOnly":false,"maven":true,"disabled":false},"connector":{"id":3,"name":"slow-upstream","connectionTimeout":"15 seconds","connectionIdleTimeout":"30 seconds","maxConnections":3000,"maxConnectionsPerHost":1000}},{"repo":{"id":1,"name":"koala","base":"http://nexus.openkoala.org/content/groups/Koala-release","priority":1,"getOnly":true,"maven":true,"disabled":false,"parentId":7},"connector":{"id":2,"name":"fast-upstream","connectionTimeout":"5 seconds","connectionIdleTimeout":"5 seconds","maxConnections":3000,"maxConnectionsPerHost":1000}},{"repo":{"id":4,"name":"oschina","base":"http://maven.oschina.net/content/groups/public","priority":1,"getOnly":true,"maven":true,"disabled":false,"parentId":8},"connector":{"id":2,"name":"fast-upstream","connectionTimeout":"5 seconds","connectionIdleTimeout":"5 seconds","maxConnections":3000,"maxConnectionsPerHost":1000}},{"repo":{"id":10,"name":"jcenter","base":"http://jcenter.bintray.com","priority":2,"getOnly":false,"maven":false,"disabled":false},"connector":{"id":3,"name":"slow-upstream","connectionTimeout":"15 seconds","connectionIdleTimeout":"30 seconds","maxConnections":3000,"maxConnectionsPerHost":1000}},{"repo":{"id":2,"name":"sonatype","base":"http://oss.sonatype.org/content/repositories/releases","priority":5,"getOnly":false,"maven":true,"disabled":false},"connector":{"id":3,"name":"slow-upstream","connectionTimeout":"15 seconds","connectionIdleTimeout":"30 seconds","maxConnections":3000,"maxConnectionsPerHost":1000}},{"repo":{"id":2,"name":"sonatype","base":"https://oss.sonatype.org/content/repositories/releases","priority":5,"getOnly":false,"maven":true,"disabled":false},"connector":{"id":3,"name":"slow-upstream","connectionTimeout":"15 seconds","connectionIdleTimeout":"30 seconds","maxConnections":3000,"maxConnectionsPerHost":1000}},{"repo":{"id":4,"name":"oschina","base":"http://maven.oschina.net/content/groups/public","priority":1,"getOnly":true,"maven":true,"disabled":false,"parentId":7},"connector":{"id":2,"name":"fast-upstream","connectionTimeout":"5 seconds","connectionIdleTimeout":"5 seconds","maxConnections":3000,"maxConnectionsPerHost":1000}},{"repo":{"id":2,"name":"sonatype","base":"http://oss.sonatype.org/content/repositories/releases","priority":5,"getOnly":false,"maven":false,"disabled":false},"connector":{"id":3,"name":"slow-upstream","connectionTimeout":"15 seconds","connectionIdleTimeout":"30 seconds","maxConnections":3000,"maxConnectionsPerHost":1000}},{"repo":{"id":3,"name":"typesafe","base":"http://dl.bintray.com/typesafe/ivy-releases","priority":2,"getOnly":false,"maven":false,"disabled":false},"connector":{"id":3,"name":"slow-upstream","connectionTimeout":"15 seconds","connectionIdleTimeout":"30 seconds","maxConnections":3000,"maxConnectionsPerHost":1000}},{"repo":{"id":3,"name":"typesafe","base":"http://repo.typesafe.com/typesafe/releases","priority":2,"getOnly":false,"maven":false,"disabled":false},"connector":{"id":3,"name":"slow-upstream","connectionTimeout":"15 seconds","connectionIdleTimeout":"30 seconds","maxConnections":3000,"maxConnectionsPerHost":1000}},{"repo":{"id":1,"name":"koala","base":"http://nexus.openkoala.org/content/groups/Koala-release","priority":1,"getOnly":true,"maven":true,"disabled":false,"parentId":8},"connector":{"id":2,"name":"fast-upstream","connectionTimeout":"5 seconds","connectionIdleTimeout":"5 seconds","maxConnections":3000,"maxConnectionsPerHost":1000}},{"repo":{"id":4,"name":"oschina","base":"http://maven.oschina.net/content/groups/public","priority":1,"getOnly":true,"maven":true,"disabled":false},"connector":{"id":2,"name":"fast-upstream","connectionTimeout":"5 seconds","connectionIdleTimeout":"5 seconds","maxConnections":3000,"maxConnectionsPerHost":1000}},{"repo":{"id":4,"name":"oschina","base":"http://maven.oschina.net/content/groups/public","priority":2,"getOnly":true,"maven":true,"disabled":false},"connector":{"id":2,"name":"fast-upstream","connectionTimeout":"5 seconds","connectionIdleTimeout":"5 seconds","maxConnections":3000,"maxConnectionsPerHost":1000}},{"repo":{"id":10,"name":"jcenter","base":"http://jcenter.bintray.com","priority":2,"getOnly":false,"maven":true,"disabled":false},"connector":{"id":3,"name":"slow-upstream","connectionTimeout":"15 seconds","connectionIdleTimeout":"30 seconds","maxConnections":3000,"maxConnectionsPerHost":1000}}],"proxyUsage":[{"connector":{"id":4,"name":"use-proxy","connectionTimeout":"6 seconds","connectionIdleTimeout":"20 seconds","maxConnections":50,"maxConnectionsPerHost":25},"proxy":{"id":2,"name":"Vultr","protocol":"HTTP","host":"108.61.163.157","port":8888,"disabled":false}}],"immediate404Rules":[{"id":1,"include":".+-javadoc\\.jar","exclude":"/net/sourceforge/f2j/arpack_combined_all/.*-javadoc.jar","disabled":false},{"id":2,"include":".+-parent.*\\.jar","disabled":false},{"id":3,"include":"(/.+)+/((.+?-project)(_(.+?)(_(.+))?)?)/(.+?)/\\3-\\8(-(.+?))?\\.jar","exclude":"/org/apache/maven/maven-project/(.+?)/maven-project-\\1\\.jar","disabled":false},{"id":4,"include":"(/.+)+/((.+?-pom)(_(.+?)(_(.+))?)?)/(.+?)/\\3-\\8(-(.+?))?\\.jar","disabled":false},{"id":5,"include":"/.+?/(.+?-project)/.+/\\1\\.jar","disabled":false},{"id":6,"include":"/org/jboss/xnio/xnio-all/.+\\.jar","disabled":false},{"id":7,"include":"/org\\.jboss\\.xnio/xnio-all/.+\\.jar","disabled":false},{"id":8,"include":"/org/apache/apache/(\\d+)/.+\\.jar","disabled":false},{"id":9,"include":"/org\\.apache/apache/(\\d+)/.+\\.jar","disabled":false},{"id":10,"include":"/com/google/google/(\\d+)/.+\\.jar","disabled":false},{"id":11,"include":"/com\\.google/google/(\\d+)/.+\\.jar","disabled":false},{"id":12,"include":"/org/ow2/ow2/.+\\.jar","disabled":false},{"id":13,"include":"/org\\.ow2/ow2/.+\\.jar","disabled":false},{"id":14,"include":"(/.+)+/((.+?-site)(_(.+?)(_(.+))?)?)/(.+?)/\\3-\\8(-(.+?))?\\.jar","exclude":"","disabled":false},{"id":15,"include":"/.+?/(.+?-site)/.+/\\1\\.jar","exclude":"/com\\.typesafe\\.sbt/sbt-site/.*","disabled":false},{"id":16,"include":"/org/fusesource/leveldbjni/.+-sources\\.jar","disabled":false},{"id":17,"include":"/org\\.fusesource\\.leveldbjni/.+-sources\\.jar","disabled":false},{"id":18,"include":"/.+?/(.+?-pom)/.+/\\1\\.jar","disabled":false},{"id":19,"include":"/org/webjars/.+-sources.jar","disabled":false},{"id":20,"include":"/org\\.webjars/.+-sources.jar","disabled":false},{"id":21,"include":"/org/scala-sbt/(.+?)/(.+?)/\\1-\\2\\..*","exclude":"/org/scala-sbt/(launcher|test)-interface/(.+?)/\\1-interface-\\2.*","disabled":true}],"expireRules":[{"id":1,"pattern":".+/maven-metadata\\.xml","duration":"1 day","disabled":false}],"connectors":[{"id":1,"name":"default","connectionTimeout":"5 seconds","connectionIdleTimeout":"10 seconds","maxConnections":3000,"maxConnectionsPerHost":1000},{"id":2,"name":"fast-upstream","connectionTimeout":"5 seconds","connectionIdleTimeout":"5 seconds","maxConnections":3000,"maxConnectionsPerHost":1000},{"id":3,"name":"slow-upstream","connectionTimeout":"15 seconds","connectionIdleTimeout":"30 seconds","maxConnections":3000,"maxConnectionsPerHost":1000},{"id":4,"name":"use-proxy","connectionTimeout":"6 seconds","connectionIdleTimeout":"20 seconds","maxConnections":50,"maxConnectionsPerHost":25}],"headTimeout":"10 seconds","headRetryTimes":3,"password":"not exported","extraResources":["/home/xiefei/.m2/repository"]} -------------------------------------------------------------------------------- /src/main/resources/application.conf: -------------------------------------------------------------------------------- 1 | akka { 2 | loggers = ["akka.event.slf4j.Slf4jLogger"] 3 | loglevel = debug 4 | jvm-exit-on-fatal-error = off 5 | actor { 6 | debug { 7 | lifecycle = off 8 | receive = off 9 | unhandled = off 10 | router-misconfiguration = off 11 | } 12 | serializers { 13 | jsonSerializer = "com.gtan.repox.JsonSerializer" 14 | } 15 | serialization-bindings { 16 | "java.io.Serializable" = none 17 | "com.gtan.repox.config.Jsonable" = jsonSerializer 18 | "com.gtan.repox.config.Evt" = jsonSerializer 19 | } 20 | # default-dispatcher { 21 | # #executor = "thread-pool-executor" 22 | # #thread-pool-executor { 23 | # # task-queue-type = "a" 24 | # #} 25 | # fork-join-executor { 26 | # parallelism-min = 2 27 | # parallelism-factor = 8.0 28 | # parallelism-max = 8 29 | # } 30 | # throughput = 20 31 | # } 32 | } 33 | persistence { 34 | journal { 35 | plugin = "akka.persistence.journal.leveldb" 36 | leveldb.dir = ${?HOME}${?HOMEPATH}"/.repox/journal" 37 | } 38 | snapshot-store { 39 | plugin = "akka.persistence.snapshot-store.local" 40 | local.dir = ${?HOME}${?HOMEPATH}"/.repox/snapshots" 41 | } 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /src/main/resources/logback.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 6 | 7 | %d{MM-dd HH:mm:ss.SSS} %highlight(%-5level) %logger{36} %X{akkaSource} %n %blue(%msg) %n %n 8 | 9 | 10 | 11 | 12 | 13 | 14 | ${user.home}/.repox/repox.log 15 | 16 | ${user.home}/.repox/repox.log.%i 17 | 1 18 | 9 19 | 20 | 21 | 100MB 22 | 23 | 25 | 26 | %d{MM-dd HH:mm:ss.SSS} %-5level %logger{36} %X{akkaSource} %n %msg %n %n 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | -------------------------------------------------------------------------------- /src/main/scala/com/gtan/repox/ExpirationManager.scala: -------------------------------------------------------------------------------- 1 | package com.gtan.repox 2 | 3 | import java.time.temporal.ChronoUnit 4 | import java.time.{LocalDateTime, ZoneId} 5 | 6 | import akka.actor.{ActorLogging, Cancellable, Props} 7 | import akka.persistence._ 8 | import com.gtan.repox.config.{Evt, Jsonable} 9 | import play.api.libs.json.{JsValue, Json} 10 | 11 | import scala.concurrent.duration._ 12 | import scala.language.postfixOps 13 | 14 | 15 | object ExpirationManager extends SerializationSupport { 16 | 17 | case class CreateExpiration(uri: String, duration: Duration) 18 | 19 | case class CancelExpiration(uri: String) 20 | 21 | case class PerformExpiration(uri: String) 22 | 23 | case class ExpirationPerformed(uri: String) extends Jsonable with Evt 24 | 25 | case class Expiration(uri: String, timestamp: LocalDateTime) extends Jsonable with Evt 26 | 27 | case class ExpirationSeq(expirations: Seq[Expiration]) extends Jsonable with Evt 28 | 29 | implicit val expirationFormat = Json.format[Expiration] 30 | implicit val expirationPerformedFormat = Json.format[ExpirationPerformed] 31 | implicit val expirationSeqFormat = Json.format[ExpirationSeq] 32 | 33 | val ExpirationClass = classOf[Expiration].getName 34 | val ExpirationPerformedClass = classOf[ExpirationPerformed].getName 35 | val ExpirationSeqClass = classOf[ExpirationSeq].getName 36 | 37 | override val reader: JsValue => PartialFunction[String, Jsonable] = payload => { 38 | case ExpirationClass => payload.as[Expiration] 39 | case ExpirationPerformedClass => payload.as[ExpirationPerformed] 40 | case ExpirationSeqClass => payload.as[ExpirationSeq] 41 | } 42 | 43 | override val writer: PartialFunction[Jsonable, JsValue] = { 44 | case o: Expiration => Json.toJson(o) 45 | case o: ExpirationPerformed => Json.toJson(o) 46 | case o: ExpirationSeq => Json.toJson(o) 47 | } 48 | } 49 | 50 | 51 | /** 52 | * This actor always recover the latest snapshot so that performed or canceled expirations will not be seen in the future. 53 | * To exam the detailed history, use another persistence query to recovery all events. 54 | */ 55 | class ExpirationManager extends PersistentActor with ActorLogging { 56 | 57 | import com.gtan.repox.ExpirationManager._ 58 | 59 | import scala.concurrent.ExecutionContext.Implicits.global 60 | 61 | override def persistenceId: String = "Expiration" 62 | 63 | // The following 2 states are kept in-sync during each repox process. 64 | // memory only 65 | var scheduledExpirations: Map[Expiration, Cancellable] = Map.empty 66 | // persistent 67 | var unperformed: ExpirationSeq = ExpirationSeq(Vector.empty) 68 | 69 | def scheduleFileDelete(expiration: Expiration): Unit = { 70 | if (expiration.timestamp.isAfter(LocalDateTime.now)) { 71 | val delay: Long = expiration.timestamp.atZone(ZoneId.systemDefault).toInstant.toEpochMilli - 72 | LocalDateTime.now.atZone(ZoneId.systemDefault).toInstant.toEpochMilli 73 | log.debug(s"Schedule expiration for ${expiration.uri} at ${expiration.timestamp} in $delay ms") 74 | val cancellable = Repox.system.scheduler.scheduleOnce( 75 | delay.millis, 76 | self, 77 | PerformExpiration(expiration.uri)) 78 | scheduledExpirations = scheduledExpirations.updated(expiration, cancellable) 79 | } else { 80 | log.debug(s"${expiration.uri} expired, trigger FileDelete now.") 81 | context.actorOf(Props(classOf[FileDeleter], expiration.uri, 'ExpirationPersister)) 82 | } 83 | } 84 | 85 | def cancelExpirations(pattern: String): Unit = { 86 | val canceled = scheduledExpirations.collect { 87 | case (expiration, cancellable) if expiration.uri.matches(pattern) => 88 | cancellable.cancel() 89 | expiration 90 | }.toSet 91 | scheduledExpirations = scheduledExpirations.filterKeys(canceled.contains) 92 | unperformed = unperformed.copy(expirations = unperformed.expirations.filterNot(_.uri.matches(pattern))) 93 | } 94 | 95 | override def receiveRecover: Receive = { 96 | case e@Expiration(uri, timestamp) => 97 | unperformed = unperformed.copy(expirations = unperformed.expirations :+ e) 98 | scheduleFileDelete(e) 99 | case CancelExpiration(pattern) => 100 | cancelExpirations(pattern) 101 | case SnapshotOffer(metadata, saved) => 102 | this.unperformed = saved.asInstanceOf[ExpirationSeq] 103 | for(expiration <- unperformed.expirations) { 104 | scheduleFileDelete(expiration) 105 | } 106 | } 107 | 108 | override def receiveCommand: Receive = { 109 | case CreateExpiration(uri, duration) => 110 | if (!scheduledExpirations.exists(_._1.uri == uri)) { 111 | val timestamp = LocalDateTime.now().plus(duration.toMillis.toInt, ChronoUnit.MILLIS) 112 | val expiration = Expiration(uri, timestamp) 113 | persist(expiration) { _ => } 114 | unperformed = unperformed.copy(expirations = unperformed.expirations :+ expiration) 115 | scheduleFileDelete(expiration) 116 | } else { 117 | // this can only happen when there were multiple request in lined in GetQueueWorker stash queue 118 | } 119 | case CancelExpiration(pattern) => 120 | cancelExpirations(pattern) 121 | persist(CancelExpiration(pattern)) { _ => 122 | saveSnapshot(unperformed) 123 | } 124 | case PerformExpiration(uri) => 125 | log.debug(s"$uri expired, trigger FileDelete now.") 126 | context.actorOf(Props(classOf[FileDeleter], uri, 'ExpirationPersister)) 127 | case e@ExpirationPerformed(uri) => 128 | scheduledExpirations = scheduledExpirations.filterKeys(_.uri != uri) 129 | unperformed.expirations.find(_.uri == uri).foreach { performed => 130 | unperformed = unperformed.copy(expirations = unperformed.expirations.filterNot(_ == performed)) 131 | persist(performed) { _ => 132 | saveSnapshot(unperformed) 133 | } 134 | } 135 | } 136 | 137 | } 138 | -------------------------------------------------------------------------------- /src/main/scala/com/gtan/repox/ExpirationMigrator.scala: -------------------------------------------------------------------------------- 1 | package com.gtan.repox 2 | 3 | import java.nio.file.Paths 4 | 5 | import akka.actor.{Props, ActorSystem, ActorLogging} 6 | import akka.persistence.{DeleteMessagesSuccess, RecoveryCompleted, PersistentActor} 7 | import com.gtan.repox.ExpirationManager.Expiration 8 | 9 | import scala.util.Properties._ 10 | 11 | class ExpirationMigrator extends PersistentActor { 12 | 13 | val storagePath = Paths.get(userHome, ".repox", "storage") 14 | 15 | def resolveToPaths(uri: String) = (storagePath.resolve(uri.tail), storagePath.resolve(uri.tail + ".sha1")) 16 | 17 | println("Start migration ...") 18 | override def receiveRecover: Receive = { 19 | case e@Expiration(uri, timestamp) => 20 | val (path, sha1Path) = resolveToPaths(uri) 21 | path.toFile.delete() 22 | sha1Path.toFile.delete() 23 | println(s"$path and $sha1Path deleted.") 24 | case RecoveryCompleted => // This is the last message that receiveRecover would receive 25 | deleteMessages(Long.MaxValue) 26 | } 27 | 28 | override def receiveCommand: Receive = { 29 | case DeleteMessagesSuccess(_) => 30 | println("Migration finished.") 31 | context.system.terminate() 32 | } 33 | 34 | override def persistenceId: String = "Expiration" 35 | 36 | } 37 | 38 | object ExpirationMigrator { 39 | def main(args: Array[String]) { 40 | val system = ActorSystem("ExpirationMigrator") 41 | val migrator = system.actorOf(Props[ExpirationMigrator], "Migrator") 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /src/main/scala/com/gtan/repox/FileDeleter.scala: -------------------------------------------------------------------------------- 1 | package com.gtan.repox 2 | 3 | import akka.actor.{PoisonPill, Actor, ActorLogging} 4 | import com.gtan.repox.ExpirationManager.ExpirationPerformed 5 | 6 | object FileDeleter { 7 | case object Quarantined 8 | } 9 | 10 | /** 11 | * delete file and sha1 file as a whole 12 | * @param uri 13 | */ 14 | class FileDeleter(uri: String, initiator: Symbol) extends Actor with ActorLogging { 15 | import FileDeleter._ 16 | 17 | log.debug(s"$initiator initiated deletion of $uri") 18 | Repox.requestQueueMaster ! RequestQueueMaster.Quarantine(uri: String) 19 | override def receive = { 20 | case Quarantined => 21 | val (path, sha1Path) = Repox.resolveToPaths(uri) 22 | path.toFile.delete() 23 | sha1Path.toFile.delete() 24 | Repox.requestQueueMaster ! RequestQueueMaster.FileDeleted(uri) 25 | log.debug(s"$path and $sha1Path deleted.") 26 | if (initiator == 'ExpirationPersister) { 27 | context.parent ! ExpirationPerformed(uri) 28 | } 29 | self ! PoisonPill 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /src/main/scala/com/gtan/repox/GetAsyncHandler.scala: -------------------------------------------------------------------------------- 1 | package com.gtan.repox 2 | 3 | import java.io.{File, FileOutputStream} 4 | import java.nio.channels.FileChannel 5 | import java.nio.file.{Files, Paths} 6 | import java.security.SecureRandom 7 | 8 | import akka.actor.{ActorRef, PoisonPill} 9 | import com.gtan.repox.GetWorker._ 10 | import com.gtan.repox.Head404Cache.NotFound 11 | import com.gtan.repox.config.Config 12 | import com.gtan.repox.data.Repo 13 | import com.ning.http.client.AsyncHandler.STATE 14 | import com.ning.http.client.{AsyncHandler, HttpResponseBodyPart, HttpResponseHeaders, HttpResponseStatus} 15 | import com.typesafe.scalalogging.LazyLogging 16 | import io.undertow.util.StatusCodes 17 | 18 | class GetAsyncHandler(val uri: String, 19 | val repo: Repo, 20 | val worker: ActorRef, // associated GetWorker 21 | val master: ActorRef, /* associated GetWorker's GetMaster */ 22 | val tempFilePath: Option[String]) extends AsyncHandler[Unit] with LazyLogging { 23 | 24 | private var tempFileOs: FileOutputStream = null 25 | var tempFile: File = null 26 | private var contentType: String = null 27 | 28 | @volatile private var canceled = false 29 | 30 | def cancel(deleteTempFile: Boolean = true) { 31 | canceled = true 32 | cleanup(deleteTempFile) 33 | } 34 | 35 | override def onThrowable(t: Throwable): Unit = { 36 | if (!canceled) 37 | worker ! AsyncHandlerThrows(t) 38 | } 39 | 40 | override def onCompleted(): Unit = { 41 | if (!canceled) { 42 | if (tempFileOs != null) { 43 | logger.debug(s"onCompleted: tempFile.length = ${tempFile.length()}") 44 | tempFileOs.close() 45 | } 46 | if (tempFile != null) { 47 | // completed before parent notify PeerChosen or self cancel 48 | master.!(Completed(tempFile.toPath, repo))(worker) 49 | } 50 | } 51 | } 52 | 53 | override def onBodyPartReceived(bodyPart: HttpResponseBodyPart): STATE = { 54 | if (!canceled) { 55 | bodyPart.writeTo(tempFileOs) 56 | worker ! PartialDataReceived(bodyPart.length()) 57 | STATE.CONTINUE 58 | } else { 59 | cleanup() 60 | STATE.ABORT 61 | } 62 | } 63 | 64 | override def onStatusReceived(responseStatus: HttpResponseStatus): STATE = { 65 | val statusCode: Int = responseStatus.getStatusCode 66 | if (statusCode == StatusCodes.NOT_FOUND) { 67 | Repox.head404Cache ! NotFound(uri, repo) 68 | } 69 | if (canceled) { 70 | cleanup() 71 | STATE.ABORT 72 | } else { 73 | if (statusCode < 200 || statusCode >= 400) { 74 | logger.debug(s"Get ${repo.absolute(uri)} $statusCode") 75 | master.!(UnsuccessResponseStatus(responseStatus))(worker) 76 | cleanup() 77 | STATE.ABORT 78 | } else 79 | STATE.CONTINUE 80 | } 81 | } 82 | 83 | override def onHeadersReceived(headers: HttpResponseHeaders): STATE = { 84 | logger.debug(s"${repo.absolute(uri)} headers ================== \n ${headers.getHeaders}") 85 | if (!canceled) { 86 | val newContentType = headers.getHeaders.getFirstValue("Content-type") 87 | if (contentType == null || contentType == newContentType) { 88 | if (contentType == null) contentType = newContentType 89 | if (tempFile != null) { 90 | logger.debug("Lantern interrupted. Resync data.") 91 | tempFileOs.close() 92 | tempFileOs = new FileOutputStream(tempFile) 93 | worker ! HeadersGot(headers) 94 | } else { 95 | tempFile = newOrReuseTempFile() 96 | tempFileOs = new FileOutputStream(tempFile, true) 97 | worker ! HeadersGot(headers) 98 | master.!(HeadersGot(headers))(worker) 99 | } 100 | STATE.CONTINUE 101 | } else { 102 | worker ! LanternGiveup 103 | cleanup() 104 | STATE.ABORT 105 | } 106 | } else { 107 | cleanup() 108 | STATE.ABORT 109 | } 110 | } 111 | 112 | def cleanup(deleteTempFile: Boolean = true): Unit = { 113 | if (tempFileOs != null) { 114 | tempFileOs.close() 115 | } 116 | if (tempFile != null && deleteTempFile) { 117 | tempFile.delete() 118 | } 119 | } 120 | 121 | private def newOrReuseTempFile(): File = tempFilePath match { 122 | case None => 123 | val tempRoot = Config.storagePath.resolve("temp") 124 | val slot = GetAsyncHandler.secureRandom.nextInt(Int.MaxValue).formatted("%04d").take(4) 125 | val distributedPath = tempRoot.resolve(slot) 126 | Files.createDirectories(distributedPath) 127 | Files.createTempFile(distributedPath, "repox", ".tmp").toFile 128 | case Some(file) => 129 | new File(file) 130 | } 131 | 132 | } 133 | 134 | object GetAsyncHandler { 135 | val secureRandom = new SecureRandom() 136 | } 137 | -------------------------------------------------------------------------------- /src/main/scala/com/gtan/repox/GetMaster.scala: -------------------------------------------------------------------------------- 1 | package com.gtan.repox 2 | 3 | import java.io.FileInputStream 4 | import java.net.URLEncoder 5 | import java.nio.file.StandardCopyOption._ 6 | import java.nio.file.{Files, Path} 7 | import java.security.MessageDigest 8 | 9 | import akka.actor._ 10 | import com.gtan.repox.GetWorker.{Cleanup, PeerChosen} 11 | import com.gtan.repox.Head404Cache.{NotFound, Query} 12 | import com.gtan.repox.data.Repo 13 | import com.typesafe.scalalogging.LazyLogging 14 | 15 | import scala.language.postfixOps 16 | 17 | object GetMaster extends LazyLogging { 18 | 19 | import scala.concurrent.duration._ 20 | 21 | implicit val timeout = new akka.util.Timeout(1 seconds) 22 | 23 | def fileSha1Hex(path: Path): String = { 24 | val sha1 = MessageDigest.getInstance("SHA-1") 25 | val fis = new FileInputStream(path.toFile) 26 | val buffer = new Array[Byte](8192) 27 | var len = fis.read(buffer) 28 | while (len != -1) { 29 | sha1.update(buffer, 0, len) 30 | len = fis.read(buffer) 31 | } 32 | val ba: Array[Byte] = sha1.digest 33 | ba.map(b => "%02x".format(b)).mkString("") 34 | } 35 | } 36 | 37 | /** 38 | * 负责某一个 uri 的 Get, 可能由多个 upstream Repo 生成多个 GetWorker 39 | * 40 | * @param uri 要获取的 uri 41 | * @param from 可能的 Repo 42 | */ 43 | class GetMaster(val uri: String, val from: Seq[Repo]) extends Actor with ActorLogging { 44 | 45 | import scala.concurrent.duration._ 46 | 47 | override val supervisorStrategy = 48 | OneForOneStrategy(maxNrOfRetries = 3, withinTimeRange = 1 minute)(super.supervisorStrategy.decider) 49 | 50 | private[this] def excludeNotFound[T](xss: Seq[Seq[T]], notFoundIn: Set[T]): Seq[Seq[T]] = 51 | xss.map(_.filterNot(notFoundIn)).filter(_.nonEmpty) 52 | 53 | val (resolvedPath, resolvedChecksumPath) = Repox.resolveToPaths(uri) 54 | var downloadedTempFilePath: Path = _ 55 | 56 | var workerChosen = false 57 | var chosenWorker: ActorRef = _ 58 | var chosenRepo: Repo = _ 59 | var childFailCount = 0 60 | var candidateRepos = Repox.orderByPriority( 61 | if (Repox.isIvyUri(uri)) 62 | from.filterNot(_.maven) 63 | else from 64 | ) 65 | 66 | var children: Seq[ActorRef] = _ 67 | 68 | private[this] def waitFor404Cache: Receive = { 69 | case Head404Cache.ExcludeRepos(repos) => 70 | candidateRepos = excludeNotFound(candidateRepos, repos) 71 | if (candidateRepos.isEmpty) { 72 | context.parent ! GetQueueWorker.Get404(uri) 73 | self ! PoisonPill 74 | } else { 75 | log.debug(s"Try ${candidateRepos.map(_.map(_.name))} $uri") 76 | val thisLevel = candidateRepos.head 77 | children = for (upstream <- thisLevel) yield { 78 | startAWorker(upstream, uri) 79 | } 80 | context become gettingFile 81 | } 82 | } 83 | 84 | private[this] def startAWorker(upstream: Repo, target: String): ActorRef = { 85 | context.actorOf( 86 | Props(classOf[GetWorker], upstream, target, None, -1L), 87 | name = s"GetWorker_${URLEncoder.encode(upstream.name, "UTF-8")}_${Repox.nextId}" 88 | ) 89 | } 90 | 91 | private[this] def askHead404Cache(): Unit = { 92 | Repox.head404Cache ! Query(uri) 93 | } 94 | 95 | askHead404Cache() 96 | 97 | def receive = waitFor404Cache 98 | 99 | def gettingFile: Receive = { 100 | case GetWorker.Resume(repo, tempFilePath, totalLength) => 101 | chosenWorker = context.actorOf( 102 | Props(classOf[GetWorker], repo, uri, Some(tempFilePath), totalLength), 103 | name = s"GetWorker_${repo.name}_${Repox.nextId}" 104 | ) 105 | children = chosenWorker :: Nil 106 | case GetWorker.Failed(t) => 107 | if (chosenWorker == sender()) { 108 | log.debug(s"Chosen worker dead. Rechoose") 109 | reset() 110 | } else { 111 | childFailCount += 1 112 | if (childFailCount == children.length) { 113 | candidateRepos.toList match { 114 | case Nil => 115 | log.debug(s"GetMaster all child failed. 404") 116 | context.parent ! GetQueueWorker.Get404(uri) 117 | self ! PoisonPill 118 | case head :: tail => 119 | if (from.length > 1) { 120 | log.debug(s"all child failed. to next level.") 121 | candidateRepos = tail 122 | } else { 123 | log.debug(s"only one repo, no other choose, retry. $uri") 124 | } 125 | reset() 126 | } 127 | } 128 | } 129 | case GetWorker.UnsuccessResponseStatus(status) => 130 | childFailCount += 1 131 | if (childFailCount == children.length) { 132 | candidateRepos.toList match { 133 | case Nil => 134 | log.info(s"GetMaster all child failed. 404") 135 | context.parent ! GetQueueWorker.Get404(uri) 136 | self ! PoisonPill 137 | case head :: tail => 138 | if (from.length > 1) { 139 | log.debug(s"all child failed. to next level.") 140 | candidateRepos = tail 141 | } else { 142 | log.debug(s"only one repo, no other choose, retry. $uri") 143 | } 144 | reset() 145 | } 146 | } 147 | case GetWorker.Completed(path, repo) => 148 | if (sender() == chosenWorker) { 149 | for (child <- children) { 150 | child ! PoisonPill 151 | } 152 | if (uri.endsWith(".sha1")) { 153 | log.debug(s"A standalone .sha1 request, stop trying to retrieve its checksum. $uri") 154 | resolvedPath.getParent.toFile.mkdirs() 155 | java.nio.file.Files.move(path, resolvedPath, REPLACE_EXISTING, ATOMIC_MOVE) 156 | context.parent ! GetQueueWorker.Completed(path, repo, checksumSuccess = true) 157 | self ! PoisonPill 158 | } else { 159 | downloadedTempFilePath = path 160 | chosenRepo = repo 161 | chosenWorker = startAWorker(repo, uri + ".sha1") 162 | children = chosenWorker :: Nil 163 | context become gettingChecksum 164 | } 165 | } else { 166 | sender ! Cleanup 167 | } 168 | case GetWorker.HeadersGot(headers) => 169 | if (children.contains(sender())) { 170 | if (!workerChosen) { 171 | log.debug(s"chose ${sender().path.name}, canceling others. ") 172 | for (others <- children.filterNot(_ == sender())) { 173 | others ! PeerChosen(sender()) 174 | } 175 | chosenWorker = sender() 176 | workerChosen = true 177 | } else if (sender != chosenWorker) { 178 | sender ! PeerChosen(chosenWorker) 179 | } 180 | } else { 181 | log.debug(s"HeadersGot msg from previous level ${sender().path.name} received. Ignore.") 182 | } 183 | } 184 | 185 | def gettingChecksum: Receive = { 186 | case GetWorker.Completed(path, repo) => 187 | val computed = GetMaster.fileSha1Hex(downloadedTempFilePath) 188 | val downloaded = scala.io.Source.fromFile(path.toFile, "UTF-8").mkString.trim 189 | val checksumSuccess: Boolean = computed.equalsIgnoreCase(downloaded) 190 | if (checksumSuccess || candidateRepos.flatten.size == 1) { 191 | Files.createDirectories(resolvedPath.getParent) 192 | log.info(s"GetWorker ${sender().path.name} completed $uri. Checksum ${if (checksumSuccess) "success" else "failed"}") 193 | Files.move(downloadedTempFilePath, resolvedPath, REPLACE_EXISTING, ATOMIC_MOVE) 194 | Files.move(path, resolvedChecksumPath, REPLACE_EXISTING, ATOMIC_MOVE) 195 | context.parent ! GetQueueWorker.Completed(path, repo, checksumSuccess) 196 | children.foreach(child => child ! Cleanup) 197 | self ! PoisonPill 198 | } else { 199 | log.info(s"Checksum failed for $uri. Try other upstreams.") 200 | Repox.head404Cache ! NotFound(uri, repo) 201 | Repox.head404Cache ! NotFound(uri + ".sha1", repo) 202 | Files.deleteIfExists(downloadedTempFilePath) 203 | Files.deleteIfExists(path) 204 | reset() 205 | } 206 | case GetWorker.Failed(t) => 207 | log.debug(s"GetWorker failed in gettingChecksum state: ${t.getMessage}. Restart.") 208 | sender ! PoisonPill 209 | chosenWorker = startAWorker(chosenRepo, uri + ".sha1") 210 | children = chosenWorker :: Nil 211 | case GetWorker.UnsuccessResponseStatus(status) => 212 | if (status.getStatusCode != 404) { 213 | log.debug(s"GetWorker get UnsuccessResponseStatus in gettingChecksum state: $status. Restart.") 214 | sender ! PoisonPill 215 | chosenWorker = startAWorker(chosenRepo, uri + ".sha1") 216 | children = chosenWorker :: Nil 217 | } else { 218 | log.debug(s"Upstream has artifact but not checksum. Response 404 but save downloaded artifact. May generate on repox manually in the future.") 219 | java.nio.file.Files.move(downloadedTempFilePath, resolvedPath, REPLACE_EXISTING, ATOMIC_MOVE) 220 | context.parent ! GetQueueWorker.Get404(uri) 221 | self ! PoisonPill 222 | } 223 | case msg => 224 | log.debug(s"Received message in gettingChecksum state: $msg. Ignore.") 225 | } 226 | 227 | 228 | private[this] def reset() = { 229 | childFailCount = 0 230 | chosenWorker = null 231 | workerChosen = false 232 | for (child <- children) child ! PoisonPill 233 | askHead404Cache() 234 | if (downloadedTempFilePath != null) 235 | Files.deleteIfExists(downloadedTempFilePath) 236 | context become waitFor404Cache 237 | } 238 | } 239 | -------------------------------------------------------------------------------- /src/main/scala/com/gtan/repox/GetQueueWorker.scala: -------------------------------------------------------------------------------- 1 | package com.gtan.repox 2 | 3 | import java.nio.file.Path 4 | 5 | import akka.actor._ 6 | import com.gtan.repox.ExpirationManager.CreateExpiration 7 | import com.gtan.repox.RequestQueueMaster.KillMe 8 | import com.gtan.repox.config.Config 9 | import com.gtan.repox.data.Repo 10 | import io.undertow.server.HttpServerExchange 11 | 12 | import scala.concurrent.duration._ 13 | import scala.language.postfixOps 14 | import scala.util.Random 15 | import scala.util.Random 16 | 17 | object GetQueueWorker { 18 | 19 | case class Get404(uri: String) 20 | 21 | case class Completed(path: Path, repo: Repo, checksumSuccess: Boolean) 22 | 23 | } 24 | 25 | class GetQueueWorker(val uri: String) extends Actor with Stash with ActorLogging { 26 | 27 | import GetQueueWorker._ 28 | 29 | override def receive = start 30 | 31 | def start: Receive = { 32 | case msg@Requests.Get(exchange) => 33 | Repox.downloaded(uri) match { 34 | case Some(Tuple2(resourceManager, resourceHandler)) => 35 | log.info(s"$uri downloaded. Serve immediately.") 36 | Repox.immediateFile(resourceHandler, exchange) 37 | suicide() 38 | case None => 39 | log.info(s"$uri not downloaded. Downloading.") 40 | context.actorOf(Props(classOf[GetMaster], uri, Config.enabledRepos), s"GetMaster_${Repox.nextId}") 41 | self ! msg 42 | context become working 43 | } 44 | } 45 | 46 | var found = false 47 | 48 | var deleteFileAfterResponse = false 49 | 50 | def working: Receive = { 51 | case Requests.Get(_) => 52 | stash() 53 | case result@Completed(path, repo, checksumSuccess) => 54 | log.debug(s"GetQueueWorker completed $uri") 55 | found = true 56 | deleteFileAfterResponse = !checksumSuccess 57 | unstashAll() 58 | context.setReceiveTimeout(1 second) 59 | context become flushWaiting 60 | case Get404(u) => 61 | log.debug(s"GetQueueWorker 404 $u") 62 | found = false 63 | unstashAll() 64 | context.setReceiveTimeout(1 second) 65 | context become flushWaiting 66 | } 67 | 68 | def flushWaiting: Receive = { 69 | case Requests.Get(exchange) => 70 | if (found) { 71 | log.debug(s"flushWaiting $exchange 200. Sending file $uri") 72 | Repox.sendFile(Repox.resourceHandlers.get().apply(Repox.storageManager), exchange) 73 | if (deleteFileAfterResponse) { 74 | context.actorOf(Props(classOf[FileDeleter], uri, 'GetQueueWorker)) 75 | } else Config.enabledExpireRules.find(rule => uri.matches(rule.pattern)).foreach { r => 76 | Repox.expirationPersister ! CreateExpiration(uri, r.duration) 77 | } 78 | } else { 79 | log.debug(s"flushWaiting $exchange 404") 80 | Repox.respond404(exchange) 81 | } 82 | case ReceiveTimeout => 83 | suicide() 84 | } 85 | 86 | private def suicide(): Unit = { 87 | context.parent ! KillMe(Queue('get, uri)) 88 | } 89 | 90 | } 91 | -------------------------------------------------------------------------------- /src/main/scala/com/gtan/repox/GetWorker.scala: -------------------------------------------------------------------------------- 1 | package com.gtan.repox 2 | 3 | import java.io.File 4 | import java.nio.file.Path 5 | 6 | import akka.actor._ 7 | import com.gtan.repox.data.Repo 8 | import com.ning.http.client._ 9 | 10 | import scala.language.postfixOps 11 | 12 | 13 | object GetWorker { 14 | 15 | case class UnsuccessResponseStatus(responseStatus: HttpResponseStatus) 16 | 17 | case class Failed(t: Throwable) 18 | 19 | case class Resume(upstream: Repo, tempFilePath: String, totalLength: Long) 20 | 21 | case class BodyPartGot(bodyPart: HttpResponseBodyPart) 22 | 23 | case class HeadersGot(headers: HttpResponseHeaders) 24 | 25 | case class Completed(path: Path, repo: Repo) 26 | 27 | case class PeerChosen(who: ActorRef) 28 | 29 | case object Cleanup 30 | 31 | // same effect as ReceiveTimeout 32 | case object LanternGiveup 33 | 34 | case class PartialDataReceived(length: Int) 35 | 36 | case class AsyncHandlerThrows(t: Throwable) 37 | 38 | } 39 | 40 | /** 41 | * 负责某一个 uri 在 某一个 upstream Repo 中的获取 42 | * @param upstream 上游 Repo 43 | * @param uri 要获取的uri 44 | * @param tempFilePath 已下载但未完成的临时文件路径,tempFilePath.isDefined时为续传,否则为新的下载任务 45 | * @param totalLength 文件的真实长度, 仅当tempFilePath.isDefined时使用 46 | */ 47 | class GetWorker(val upstream: Repo, 48 | val uri: String, 49 | val tempFilePath: Option[String], 50 | val totalLength: Long) extends Actor with Stash with ActorLogging { 51 | 52 | import com.gtan.repox.GetWorker._ 53 | import scala.concurrent.duration._ 54 | import scala.collection.JavaConverters._ 55 | 56 | val handler = new GetAsyncHandler(uri, upstream, context.self, context.parent, tempFilePath) 57 | var downloaded = 0L 58 | var percentage = 0.0 59 | var contentLength = -1L 60 | var acceptByteRange = false 61 | 62 | val (connector, client) = Repox.clientOf(upstream) 63 | 64 | val requestHeaders = tempFilePath match { 65 | case None => new FluentCaseInsensitiveStringsMap() 66 | .add("Host", List(upstream.host).asJava) 67 | .add("Accept-Encoding", List("identity").asJava) // 禁止压缩, jar文件没有压缩的必要, 其它文件太小不值得. 68 | case Some(file) => 69 | downloaded = new File(file).length() 70 | contentLength = totalLength 71 | new FluentCaseInsensitiveStringsMap() 72 | .add("Host", List(upstream.host).asJava) 73 | .add("Accept-Encoding", List("identity").asJava) // 禁止压缩 74 | .add("Range", List(s"bytes=$downloaded-").asJava) 75 | } 76 | 77 | val future = client.prepareGet(upstream.absolute(uri)) 78 | .setHeaders(requestHeaders) 79 | .execute(handler) 80 | 81 | override def receive = { 82 | case AsyncHandlerThrows(t) => 83 | log.debug(s"AsyncHandler throws -- ${t.getStackTrace.mkString("\n")}") 84 | tempFilePath match { 85 | case None => 86 | context.parent ! Failed(t) 87 | case Some(path) => 88 | log.debug("In a resuming, retry...") 89 | handler.cancel(deleteTempFile = false) 90 | context.parent ! Resume(upstream, path, totalLength) 91 | } 92 | self ! PoisonPill 93 | 94 | case Cleanup => 95 | handler.cancel() 96 | self ! PoisonPill 97 | 98 | case PeerChosen(who) => 99 | handler.cancel() 100 | self ! PoisonPill 101 | 102 | case ReceiveTimeout | LanternGiveup => 103 | log.debug("GetWorker timeout (or lantern giveup).") 104 | handler.cancel(deleteTempFile = false) 105 | self ! PoisonPill 106 | if (acceptByteRange || tempFilePath.isDefined) { 107 | context.parent ! Resume(upstream, 108 | tempFilePath.fold(handler.tempFile.getAbsolutePath)(identity), 109 | tempFilePath.fold(contentLength)(_ => totalLength)) 110 | } else { 111 | context.parent ! Failed(new RuntimeException("Chosen worker timeout or lantern giveup")) 112 | } 113 | 114 | case PartialDataReceived(length) => 115 | downloaded += length 116 | if (contentLength != -1) { 117 | val newPercentage = downloaded * 100.0 / contentLength 118 | if (newPercentage - percentage > 10.0 || downloaded == contentLength) { 119 | log.debug(f"downloaded $downloaded%s bytes. $newPercentage%.2f %%") 120 | percentage = newPercentage 121 | } 122 | } else { 123 | log.debug("PartialDataReceived") 124 | } 125 | 126 | case HeadersGot(headers) => 127 | tempFilePath match { 128 | case None => 129 | val contentLengthHeader = headers.getHeaders.getFirstValue("Content-Length") 130 | val acceptRanges = headers.getHeaders.getFirstValue("Accept-Ranges") 131 | if (contentLengthHeader != null) { 132 | log.debug(s"contentLength=$contentLengthHeader") 133 | contentLength = contentLengthHeader.toLong 134 | } 135 | if (acceptRanges != null) { 136 | log.debug(s"accept-ranges=$acceptRanges") 137 | acceptByteRange = acceptRanges.toLowerCase == "bytes" 138 | } 139 | downloaded = 0 140 | percentage = 0.0 141 | case Some(path) => 142 | downloaded = new File(path).length() 143 | percentage = downloaded * 100.0 / totalLength 144 | log.debug(f"Resuming $uri%s from ${upstream.name}%s, $percentage%.2f %% already downloaded.") 145 | } 146 | context.setReceiveTimeout(connector.connectionIdleTimeout - 1.second) 147 | } 148 | 149 | 150 | } 151 | 152 | 153 | -------------------------------------------------------------------------------- /src/main/scala/com/gtan/repox/Head404Cache.scala: -------------------------------------------------------------------------------- 1 | package com.gtan.repox 2 | 3 | import akka.actor.{Actor, ActorLogging} 4 | import com.gtan.repox.Head404Cache.{NotFound, Query} 5 | import com.gtan.repox.data.Repo 6 | 7 | import scala.language.postfixOps 8 | 9 | /** 10 | * repo that does not have some uri according to previous HEAD request 11 | * hold for 1 day 12 | * User: xf 13 | * Date: 14/11/24 14 | * Time: 下午10:55 15 | */ 16 | object Head404Cache { 17 | 18 | case class Query(uri: String) // answer Set[Repo] 19 | 20 | case class NotFound(uri: String, repo: Repo) 21 | 22 | case class ExcludeRepos(repos: Set[Repo]) 23 | 24 | import scala.concurrent.duration._ 25 | 26 | val idleTimeout = 1 days 27 | } 28 | 29 | class Head404Cache extends Actor with ActorLogging { 30 | import Head404Cache._ 31 | 32 | var data = Map.empty[String, Map[Repo, Timestamp]] 33 | 34 | def expired(timestamp: Timestamp): Boolean = 35 | (timestamp + Head404Cache.idleTimeout).isPast 36 | 37 | override def receive = { 38 | case Query(uri) => 39 | data.get(uri) match { 40 | case None => sender ! ExcludeRepos(Set.empty[Repo]) 41 | case Some(map) => 42 | val notFoundAndNotExpired = map.filter { case (_, timestamp) => !expired(timestamp)} 43 | if(notFoundAndNotExpired.isEmpty){ 44 | data = data - uri 45 | } else { 46 | data = data.updated(uri, notFoundAndNotExpired) 47 | } 48 | sender ! ExcludeRepos(notFoundAndNotExpired.keySet) 49 | } 50 | case NotFound(uri, repo) => 51 | data.get(uri) match { 52 | case None => 53 | data = data.updated(uri, Map(repo -> Timestamp.now)) 54 | case Some(map) => 55 | val notFoundAndNotExpired = map.filter { case (_, timestamp) => !expired(timestamp)} 56 | data = data.updated(uri, notFoundAndNotExpired + (repo -> Timestamp.now)) 57 | } 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /src/main/scala/com/gtan/repox/HeadAsyncHandler.scala: -------------------------------------------------------------------------------- 1 | package com.gtan.repox 2 | 3 | import akka.actor.ActorRef 4 | import com.gtan.repox.config.Config 5 | import com.gtan.repox.data.Repo 6 | import com.ning.http.client.AsyncHandler.STATE 7 | import com.ning.http.client.{AsyncHandler, HttpResponseBodyPart, HttpResponseHeaders, HttpResponseStatus} 8 | import com.typesafe.scalalogging.LazyLogging 9 | import io.undertow.util.StatusCodes 10 | 11 | class HeadAsyncHandler(val worker: ActorRef,val uri: String, val repo: Repo) extends AsyncHandler[Unit] with LazyLogging { 12 | var statusCode = 200 13 | 14 | @volatile private var canceled = false 15 | 16 | def cancel(): Unit = { 17 | canceled = true 18 | } 19 | 20 | override def onThrowable(t: Throwable): Unit = { 21 | if(!canceled) { 22 | logger.debug("HeadAsyncHandler throws ", t) 23 | worker ! HeadWorker.Failed(t) 24 | } 25 | } 26 | 27 | override def onCompleted(): Unit = { 28 | /* we don't get here actually */ 29 | } 30 | 31 | override def onBodyPartReceived(bodyPart: HttpResponseBodyPart): STATE = { 32 | // we don't get here actually 33 | STATE.ABORT 34 | } 35 | 36 | override def onStatusReceived(responseStatus: HttpResponseStatus): STATE = { 37 | if (!canceled) { 38 | statusCode = responseStatus.getStatusCode 39 | statusCode match { 40 | case StatusCodes.NOT_FOUND => 41 | Repox.head404Cache ! Head404Cache.NotFound(uri, repo) 42 | for(child <- Config.repos.filter(_.parentId.exists(repo.id.contains))){ 43 | Repox.head404Cache ! Head404Cache.NotFound(uri, child) 44 | } 45 | case _ => 46 | } 47 | STATE.CONTINUE 48 | } else STATE.ABORT 49 | } 50 | 51 | override def onHeadersReceived(headers: HttpResponseHeaders): STATE = { 52 | if (!canceled) { 53 | import scala.collection.JavaConverters._ 54 | worker ! HeadWorker.Responded(statusCode, mapAsScalaMapConverter(headers.getHeaders).asScala.toMap) 55 | } 56 | STATE.ABORT 57 | } 58 | 59 | } 60 | -------------------------------------------------------------------------------- /src/main/scala/com/gtan/repox/HeadMaster.scala: -------------------------------------------------------------------------------- 1 | package com.gtan.repox 2 | 3 | import java.net.URLEncoder 4 | 5 | import akka.actor.Actor.Receive 6 | import akka.actor._ 7 | import com.gtan.repox.Head404Cache.Query 8 | import com.gtan.repox.Repox.ResponseHeaders 9 | import com.gtan.repox.config.Config 10 | import com.gtan.repox.data.Repo 11 | import com.ning.http.client.FluentCaseInsensitiveStringsMap 12 | import io.undertow.server.HttpServerExchange 13 | import io.undertow.util.{Headers, StatusCodes} 14 | import collection.JavaConverters._ 15 | import scala.collection.Set 16 | import scala.util.Random 17 | 18 | object HeadMaster { 19 | 20 | case class FoundIn(repo: Repo, headers: ResponseHeaders) 21 | 22 | case class NotFound(repo: Repo) 23 | 24 | case class HeadTimeout(repo: Repo) 25 | 26 | } 27 | 28 | class HeadMaster(val exchange: HttpServerExchange) extends Actor with ActorLogging { 29 | 30 | import com.gtan.repox.HeadMaster._ 31 | 32 | val uri = exchange.getRequestURI 33 | val upstreams = { 34 | val candidates = Config.enabledRepos.filterNot(_.getOnly) 35 | if(Repox.isIvyUri(uri)) 36 | candidates.filterNot(_.maven) 37 | else candidates 38 | } 39 | 40 | val requestHeaders = new FluentCaseInsensitiveStringsMap() 41 | for (name <- exchange.getRequestHeaders.getHeaderNames.asScala) { 42 | requestHeaders.add(name.toString, exchange.getRequestHeaders.get(name)) 43 | } 44 | requestHeaders.put(Headers.ACCEPT_ENCODING_STRING, List("identity").asJava) 45 | 46 | 47 | var children: Seq[ActorRef] = _ 48 | 49 | var finishedChildren = 0 50 | var retryTimes = 0 51 | 52 | var candidateRepos: Seq[Repo] = _ 53 | 54 | Repox.head404Cache ! Query(uri) 55 | 56 | def start: Receive = { 57 | case Head404Cache.ExcludeRepos(repos) => 58 | candidateRepos = upstreams.filterNot(repos.contains) 59 | log.debug(s"CandidateRepos: ${candidateRepos.map(_.name)}") 60 | if (candidateRepos.isEmpty) { 61 | context.parent ! HeadQueueWorker.NotFound(exchange) 62 | self ! PoisonPill 63 | } 64 | children = for (upstream <- candidateRepos) yield { 65 | val childName = s"HeadWorker_${URLEncoder.encode(upstream.name, "UTF-8")}_${Repox.nextId}" 66 | context.actorOf( 67 | Props(classOf[HeadWorker], upstream, uri, requestHeaders), 68 | name = childName) 69 | } 70 | context become working 71 | } 72 | 73 | override def receive = start 74 | 75 | def working: Receive = { 76 | case msg@FoundIn(repo, headers) => 77 | finishedChildren += 1 78 | context.parent ! HeadQueueWorker.FoundIn(repo, headers, exchange) 79 | self ! PoisonPill 80 | case msg@NotFound(repo) => 81 | finishedChildren += 1 82 | testAllReturned() 83 | case msg@HeadTimeout(repo) => 84 | finishedChildren += 1 85 | testAllReturned() 86 | } 87 | 88 | def testAllReturned(): Unit = { 89 | if (finishedChildren == children.size) { 90 | // all returned 91 | retryTimes += 1 92 | if (retryTimes == Config.headRetryTimes) { 93 | context.parent ! HeadQueueWorker.NotFound(exchange) 94 | log.debug(s"retried ${Config.headRetryTimes} times, give up.") 95 | self ! PoisonPill 96 | } else { 97 | log.debug("All headworkers return 404. retry.") 98 | finishedChildren = 0 99 | Repox.head404Cache ! Query(uri) 100 | context become start 101 | } 102 | } 103 | } 104 | } 105 | -------------------------------------------------------------------------------- /src/main/scala/com/gtan/repox/HeadQueueWorker.scala: -------------------------------------------------------------------------------- 1 | package com.gtan.repox 2 | 3 | import akka.actor._ 4 | import com.gtan.repox.RequestQueueMaster.KillMe 5 | import com.gtan.repox.config.Config 6 | import com.gtan.repox.data.Repo 7 | import io.undertow.server.HttpServerExchange 8 | import org.w3c.dom.html.HTMLScriptElement 9 | 10 | import scala.concurrent.duration._ 11 | import scala.language.postfixOps 12 | import scala.util.Random 13 | 14 | object HeadQueueWorker { 15 | 16 | case class NotFound(exchange: HttpServerExchange) 17 | 18 | case class FoundIn(repo: Repo, headers: Repox.ResponseHeaders, exchange: HttpServerExchange) 19 | 20 | } 21 | 22 | class HeadQueueWorker(val uri: String) extends Actor with Stash with ActorLogging { 23 | 24 | import HeadQueueWorker._ 25 | 26 | override def receive = start 27 | 28 | var found = false 29 | var resultHeaders: Repox.ResponseHeaders = _ 30 | 31 | def start: Receive = { 32 | case Requests.Head(exchange) => 33 | log.debug(s"Recevied Head request of ${exchange.getRequestURI} in START state") 34 | assert(exchange.getRequestURI == uri) 35 | Repox.downloaded(uri) match { 36 | case Some(Tuple2(resourceManager, _)) => 37 | Repox.immediateHead(resourceManager, exchange) 38 | suicide() 39 | case None => 40 | for (peers <- Repox.peer(uri)) { 41 | peers.find(p => Repox.downloaded(p).isDefined) match { 42 | case Some(peer) => 43 | Repox.smart404(exchange) 44 | suicide() 45 | case _ => 46 | context.actorOf(Props(classOf[HeadMaster], exchange), name = s"HeadMaster_${Repox.nextId}") 47 | context become working 48 | } 49 | } 50 | } 51 | } 52 | 53 | def working: Receive = { 54 | case Requests.Head(exchange) => 55 | log.debug(s"Recevied Head request of ${exchange.getRequestURI} in WORKING state") 56 | assert(exchange.getRequestURI == uri) 57 | stash() 58 | case result@FoundIn(repo, headers, exchange) => 59 | found = true 60 | resultHeaders = headers 61 | Repox.respondHead(exchange, headers) 62 | log.info(s"Request HEAD for $uri respond 200.") 63 | unstashAll() 64 | context.setReceiveTimeout(1 second) 65 | context become flushWaiting 66 | case result@NotFound(exchange) => 67 | found = false 68 | Repox.respond404(exchange) 69 | log.info(s"Tried ${Config.headRetryTimes} times. Give up. Respond with 404. $uri") 70 | unstashAll() 71 | context.setReceiveTimeout(1 second) 72 | context become flushWaiting 73 | } 74 | 75 | def flushWaiting: Receive = { 76 | case Requests.Head(exchange) => 77 | log.debug(s"Recevied Head request of ${exchange.getRequestURI} in FLUSHWAITING state") 78 | if (found) { 79 | log.debug(s"Send found head: $resultHeaders") 80 | Repox.respondHead(exchange, resultHeaders) 81 | } else { 82 | log.debug(s"Send 404") 83 | Repox.respond404(exchange) 84 | } 85 | case ReceiveTimeout => 86 | suicide() 87 | } 88 | 89 | def suicide(): Unit = { 90 | context.parent ! KillMe(Queue('head, uri)) 91 | } 92 | } 93 | -------------------------------------------------------------------------------- /src/main/scala/com/gtan/repox/HeadWorker.scala: -------------------------------------------------------------------------------- 1 | package com.gtan.repox 2 | 3 | import akka.actor._ 4 | import com.gtan.repox.config.Config 5 | import com.gtan.repox.data.Repo 6 | import com.ning.http.client.FluentCaseInsensitiveStringsMap 7 | import io.undertow.util.{Headers, StatusCodes} 8 | 9 | import scala.collection.JavaConverters._ 10 | import scala.language.postfixOps 11 | 12 | object HeadWorker { 13 | 14 | // sent by HeadAsyncHandler 15 | case class Responded(statusCode: Int, headers: Repox.ResponseHeaders) 16 | 17 | case class Failed(t: Throwable) 18 | 19 | } 20 | 21 | class HeadWorker(val repo: Repo, 22 | val uri: String, 23 | val requestHeaders: FluentCaseInsensitiveStringsMap) extends Actor with ActorLogging { 24 | 25 | import HeadWorker._ 26 | 27 | val handler = new HeadAsyncHandler(self, uri, repo) 28 | 29 | context.setReceiveTimeout(Config.headTimeout) 30 | requestHeaders.put(Headers.HOST_STRING, List(repo.host).asJava) 31 | 32 | val (_, client) = Repox.clientOf(repo) 33 | 34 | val requestMethod = if (repo.getOnly) client.prepareGet _ else client.prepareHead _ 35 | requestMethod.apply(repo.absolute(uri)) 36 | .setHeaders(requestHeaders) 37 | .execute(handler) 38 | 39 | override def receive = { 40 | case Responded(statusCode, headers) => 41 | handler.cancel() 42 | statusCode match { 43 | case StatusCodes.OK => 44 | log.debug(s"HeadWorker ${repo.name} 200. $uri") 45 | context.parent ! HeadMaster.FoundIn(repo, headers) 46 | case StatusCodes.NOT_FOUND => 47 | log.debug(s"HeadWorker ${repo.name} got 404. $uri") 48 | context.parent ! HeadMaster.NotFound(repo) 49 | case _ => 50 | // server error? further feature may use failed time information 51 | log.debug(s"HeadWorker ${repo.name} got undetermined result $statusCode for $uri.") 52 | context.parent ! HeadMaster.HeadTimeout(repo) 53 | } 54 | self ! PoisonPill 55 | case Failed(t) => 56 | // further feature may use failed time information 57 | handler.cancel() 58 | log.debug(s"HeadWorker ${repo.name} failed. $uri") 59 | context.parent ! HeadMaster.HeadTimeout(repo) 60 | self ! PoisonPill 61 | case ReceiveTimeout => 62 | handler.cancel() 63 | log.debug(s"HeadWorker ${repo.name} timeout. $uri") 64 | context.parent ! HeadMaster.HeadTimeout(repo) 65 | self ! PoisonPill 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /src/main/scala/com/gtan/repox/HttpHelpers.scala: -------------------------------------------------------------------------------- 1 | package com.gtan.repox 2 | 3 | import com.gtan.repox.data.Repo 4 | import com.typesafe.scalalogging.LazyLogging 5 | import io.undertow.server.HttpServerExchange 6 | import io.undertow.server.handlers.resource.{ResourceManager, ResourceHandler} 7 | import io.undertow.util.{MimeMappings, Headers, HttpString, StatusCodes} 8 | 9 | trait HttpHelpers { self: LazyLogging => 10 | type StatusCode = Int 11 | type ResponseHeaders = Map[String, java.util.List[String]] 12 | type HeaderResponse = (Repo, StatusCode, ResponseHeaders) 13 | 14 | def respond404(exchange: HttpServerExchange): Unit = { 15 | exchange.setStatusCode(StatusCodes.NOT_FOUND) 16 | exchange.getResponseChannel // just to avoid mysterious setting Content-length to 0 in endExchange, ugly 17 | exchange.endExchange() 18 | } 19 | 20 | def immediate404(exchange: HttpServerExchange): Unit = { 21 | logger.info(s"Immediate 404 ${exchange.getRequestURI}.") 22 | respond404(exchange) 23 | } 24 | 25 | def smart404(exchange: HttpServerExchange): Unit = { 26 | logger.info(s"Smart 404 ${exchange.getRequestURI}.") 27 | respond404(exchange) 28 | } 29 | 30 | def sendFile(resourceHandler: ResourceHandler, exchange: HttpServerExchange): Unit = { 31 | resourceHandler.handleRequest(exchange) 32 | } 33 | 34 | def immediateFile(resourceHandler: ResourceHandler, exchange: HttpServerExchange): Unit = { 35 | logger.debug(s"Immediate file ${exchange.getRequestURI}") 36 | sendFile(resourceHandler, exchange) 37 | } 38 | 39 | def respondHead(exchange: HttpServerExchange, headers: ResponseHeaders): Unit = { 40 | exchange.setStatusCode(200) 41 | val target = exchange.getResponseHeaders 42 | for ((k, v) <- headers) 43 | target.putAll(new HttpString(k), v) 44 | exchange.getResponseChannel // just to avoid mysterious setting Content-length to 0 in endExchange, ugly 45 | exchange.endExchange() 46 | } 47 | 48 | def immediateHead(resourceManager: ResourceManager, exchange: HttpServerExchange): Unit = { 49 | val uri = exchange.getRequestURI 50 | val resource = resourceManager.getResource(uri) 51 | exchange.setStatusCode(200) 52 | val headers = exchange.getResponseHeaders 53 | headers.put(Headers.CONTENT_LENGTH, resource.getContentLength) 54 | .put(Headers.SERVER, "repox") 55 | .put(Headers.CONNECTION, Headers.KEEP_ALIVE.toString) 56 | .put(Headers.CONTENT_TYPE, resource.getContentType(MimeMappings.DEFAULT)) 57 | .put(Headers.LAST_MODIFIED, resource.getLastModifiedString) 58 | exchange.getResponseChannel // just to avoid mysterious setting Content-length to 0 in endExchange, ugly 59 | exchange.endExchange() 60 | logger.debug(s"Immediate head $uri. ") 61 | } 62 | 63 | def redirectTo(exchange: HttpServerExchange, targetLocation: String): Unit = { 64 | exchange.setStatusCode(StatusCodes.TEMPORARY_REDIRECT) 65 | exchange.getResponseHeaders.put(Headers.LOCATION, targetLocation) 66 | exchange.endExchange() 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /src/main/scala/com/gtan/repox/JsonSerializer.scala: -------------------------------------------------------------------------------- 1 | package com.gtan.repox 2 | 3 | import java.io.NotSerializableException 4 | 5 | import akka.serialization.Serializer 6 | import com.gtan.repox.ExpirationManager.ExpirationSeq 7 | import com.gtan.repox.config._ 8 | import com.typesafe.scalalogging.LazyLogging 9 | import play.api.libs.json._ 10 | 11 | trait SerializationSupport { 12 | val reader: JsValue => PartialFunction[String, Jsonable] 13 | val writer: PartialFunction[Jsonable, JsValue] 14 | } 15 | 16 | class JsonSerializer extends Serializer with LazyLogging with SerializationSupport { 17 | val ConfigChangedClass = classOf[ConfigChanged].getName 18 | 19 | val serializationSupports: Seq[_ <: SerializationSupport] = Seq(RepoPersister, ProxyPersister, ParameterPersister, Immediate404RulePersister, ExpireRulePersister, ConnectorPersister, ExpirationManager, ConfigPersister) 20 | 21 | override val reader: JsValue => PartialFunction[String, Jsonable] = { jsValue => 22 | serializationSupports.map(_.reader(jsValue)).reduce(_ orElse _) orElse { 23 | case clazzName: String => 24 | throw new NotSerializableException(s"No serialization supported for class $clazzName") 25 | } 26 | } 27 | override val writer: PartialFunction[Jsonable, JsValue] = serializationSupports.map(_.writer).reduce(_ orElse _) orElse { 28 | case jsonable: Jsonable => 29 | throw new NotSerializableException(s"No serialization supported for $jsonable") 30 | } 31 | 32 | override def identifier: Int = 900188 33 | 34 | override def includeManifest: Boolean = false 35 | 36 | override def fromBinary(bytes: Array[Byte], manifest: Option[Class[_]]): AnyRef = manifest match { 37 | case None => 38 | Json.parse(new String(bytes, "UTF-8")) match { 39 | case obj: JsObject => obj.fields match { 40 | case Seq( 41 | ("manifest", JsString(ConfigChangedClass)), 42 | ("config", config: JsValue), 43 | ("cmd", configcmd: JsValue)) => 44 | ConfigChanged(configFromJson(config), jsonableFromJson(configcmd)) 45 | case _ => jsonableFromJson(obj) 46 | } 47 | case JsString("UseDefault") => UseDefault 48 | case other => jsonableFromJson(other) 49 | } 50 | case Some(_) => throw new NotSerializableException("JsonSerializer does not use extra manifest.") 51 | } 52 | 53 | private def configFromJson(config: JsValue): Config = config.as[Config] 54 | 55 | private def jsonableFromJson(evt: JsValue): Jsonable = evt match { 56 | case obj: JsObject => obj.fields match { 57 | case Seq( 58 | ("manifest", JsString(clazzname)), 59 | ("payload", payload: JsValue)) => 60 | reader.apply(payload).apply(clazzname) 61 | } 62 | case _ => throw new NotSerializableException(evt.toString()) 63 | } 64 | 65 | private def toJson(o: AnyRef): JsValue = o match { 66 | case ConfigChanged(config, cmd) => 67 | JsObject( 68 | Seq( 69 | "manifest" -> JsString(ConfigChangedClass), 70 | "config" -> Json.toJson(config), 71 | "cmd" -> toJson(cmd) 72 | ) 73 | ) 74 | case jsonable: Jsonable => 75 | val payload = writer.apply(jsonable) 76 | JsObject(Seq( 77 | "manifest" -> JsString(jsonable.getClass.getName), 78 | "payload" -> payload 79 | )) 80 | case UseDefault => JsString("UseDefault") 81 | } 82 | 83 | override def toBinary(o: AnyRef): Array[Byte] = toJson(o).toString().getBytes("UTF-8") 84 | } 85 | -------------------------------------------------------------------------------- /src/main/scala/com/gtan/repox/Main.scala: -------------------------------------------------------------------------------- 1 | package com.gtan.repox 2 | 3 | 4 | import com.gtan.repox.admin.WebConfigHandler 5 | import io.undertow.predicate.Predicates 6 | import io.undertow.server.handlers.{PredicateContextHandler, PredicateHandler} 7 | import io.undertow.server.{HttpHandler, HttpServerExchange} 8 | import io.undertow.{Undertow, UndertowOptions} 9 | import org.xnio.Options 10 | 11 | object Main { 12 | def httpHandlerBridge(realHandler: HttpServerExchange => Unit): HttpHandler = new HttpHandler() { 13 | override def handleRequest(exchange: HttpServerExchange): Unit = { 14 | exchange.dispatch(scala.concurrent.ExecutionContext.Implicits.global.execute, () => realHandler.apply(exchange)) 15 | } 16 | } 17 | 18 | def main(args: Array[String]) { 19 | Repox.init() 20 | val server: Undertow = Undertow.builder 21 | .addHttpListener(8078, "0.0.0.0") 22 | .setServerOption[Integer](UndertowOptions.IDLE_TIMEOUT, 1000 * 60 * 30) 23 | .setSocketOption[java.lang.Boolean](Options.KEEP_ALIVE, true) 24 | .setHandler( 25 | new PredicateContextHandler( 26 | new PredicateHandler( 27 | Predicates.prefix("/admin/"), 28 | httpHandlerBridge(WebConfigHandler.handle), 29 | httpHandlerBridge(Repox.handle) 30 | )) 31 | ).build 32 | server.start() 33 | } 34 | 35 | } 36 | -------------------------------------------------------------------------------- /src/main/scala/com/gtan/repox/Repox.scala: -------------------------------------------------------------------------------- 1 | package com.gtan.repox 2 | 3 | import java.nio.file.Paths 4 | import java.util.concurrent.atomic.AtomicLong 5 | 6 | import akka.actor.{ActorSystem, Props} 7 | import akka.agent.Agent 8 | import com.gtan.repox.config.{Config, ConfigPersister, ConfigQuery} 9 | import com.gtan.repox.data.{Connector, ExpireRule, Repo} 10 | import com.ning.http.client.{AsyncHttpClient, ProxyServer => JProxyServer} 11 | import com.typesafe.scalalogging.LazyLogging 12 | import io.undertow.Handlers 13 | import io.undertow.server.HttpServerExchange 14 | import io.undertow.server.handlers.resource.{FileResourceManager, ResourceHandler, ResourceManager} 15 | import io.undertow.util._ 16 | 17 | import scala.concurrent.duration._ 18 | import scala.language.postfixOps 19 | import scala.util.{Failure, Success, Try} 20 | 21 | object Repox extends LazyLogging with HttpHelpers { 22 | def lookForExpireRule(uri: String): Option[ExpireRule] = Config.expireRules.find( 23 | rule => !rule.disabled && uri.matches(rule.pattern)) 24 | 25 | 26 | import concurrent.ExecutionContext.Implicits.global 27 | 28 | val system = ActorSystem("repox") 29 | 30 | private[this] val idGenerator = new AtomicLong(1) 31 | 32 | def nextId: Long = idGenerator.getAndIncrement() 33 | 34 | // uncomment his to view config data modification history 35 | // val configQuery = new ConfigQuery(system) 36 | val configPersister = system.actorOf(Props[ConfigPersister], "ConfigPersister") 37 | val expirationPersister = system.actorOf(Props[ExpirationManager], "ExpirationPersister") 38 | val head404Cache = system.actorOf(Props[Head404Cache], "HeaderCache") 39 | val requestQueueMaster = system.actorOf(Props[RequestQueueMaster], "RequestQueueMaster") 40 | 41 | 42 | val clients: Agent[Map[String, AsyncHttpClient]] = Agent(null) 43 | 44 | def clientOf(repo: Repo): (Connector, AsyncHttpClient) = Config.connectorUsage.find { 45 | case (r, connector) => r.id == repo.id 46 | } match { 47 | case None => 48 | Config.connectors.find(_.name == "default").get -> clients.get().apply("default") 49 | case Some(Tuple2(r, connector)) => 50 | println(s"Using client of ${connector.name} for ${repo.name}") 51 | connector -> clients.get().apply(connector.name) 52 | } 53 | 54 | def initClients() = Repox.clients.alter(Config.connectors.map { connector => 55 | connector.name -> connector.createClient 56 | }.toMap) 57 | 58 | def initResourceManagers() = { 59 | val storage = Repox.storageManager -> Handlers.resource(Repox.storageManager) 60 | val (valid, invalid) = Config.resourceBases.partition { base => 61 | Paths.get(base).toFile.exists() 62 | } 63 | if (invalid.nonEmpty) { 64 | logger.debug(s"Excluded invalid base(s) (${invalid.mkString(",")})") 65 | } 66 | val extra = for (rb <- valid) yield { 67 | val resourceManager: FileResourceManager = new FileResourceManager(Paths.get(rb).toFile, 100 * 1024) 68 | val resourceHandler = Handlers.resource(resourceManager) 69 | resourceManager -> resourceHandler 70 | } 71 | 72 | Repox.resourceHandlers.alter((extra :+ storage).toMap) 73 | } 74 | 75 | val storageManager = new FileResourceManager(Config.storagePath.toFile, 100 * 1024) 76 | 77 | implicit val timeout = akka.util.Timeout(1 second) 78 | 79 | def isIvyUri(uri: String) = uri.matches( """/[^/]+?\.[^/]+?/.+""") 80 | 81 | def resolveToPaths(uri: String) = { 82 | val removeSlashesAtBeginning: String = uri.dropWhile('/' ==) 83 | (Config.storagePath.resolve(removeSlashesAtBeginning), Config.storagePath.resolve(removeSlashesAtBeginning + ".sha1")) 84 | } 85 | 86 | def orderByPriority(candidates: Seq[Repo]): Seq[Seq[Repo]] = 87 | candidates.groupBy(_.priority).toSeq.sortBy(_._1).map(_._2) 88 | 89 | 90 | /** 91 | * this is the one and only truth 92 | * 93 | * @param uri resource to get or query 94 | * @return 95 | */ 96 | def downloaded(uri: String): Option[(ResourceManager, ResourceHandler)] = { 97 | resourceHandlers.get().find { case (resourceManager, handler) => 98 | resourceManager.getResource(uri.tail) != null 99 | } 100 | } 101 | 102 | private[repox] val MavenFormat = """(/.+)+/((.+?)(_(.+?)(_(.+))?)?)/(.+?)/(\3-\8(-(.+?))?\.(.+))""".r 103 | private[repox] val IvyFormat = """/(.+?)/(.+?)/(scala_(.+?)/)?(sbt_(.+?)/)?(.+?)/(.+?)s/((.+?)(-(.+))?\.(.+))""".r 104 | private[this] val MetaDataFormat = """.+/maven-metadata\.xml""".r 105 | private[this] val MD5Request = """.+\.md5""".r 106 | private[this] val SHA1Request = """(.+)\.sha1""".r 107 | private[this] val supportedScalaVersion = List("2.10", "2.11", "2.12") 108 | private[this] val supportedSbtVersion = List("0.13") 109 | 110 | /** 111 | * transform between uri formats 112 | * 113 | * @param uri 114 | * @return maven format if is ivy format, or ivy format if is maven format 115 | */ 116 | def peer(uri: String): Try[List[String]] = uri match { 117 | case MD5Request() => 118 | Failure(new RuntimeException("We do not support md5 checksum now.")) 119 | case SHA1Request(prefix) => 120 | peer(prefix).map(_.map(_ + ".sha1")) 121 | case MetaDataFormat() => 122 | if (uri.toUpperCase.endsWith("SNAPSHOT/MAVEN-METADATA.XML")) 123 | Failure(new RuntimeException("SNAPSHOT Request")) 124 | else 125 | Success(Nil) 126 | case MavenFormat(groupIds, _, artifactId, _, scalaVersion, _, sbtVersion, version, fileName, _, classifier, ext) => 127 | if (version.equalsIgnoreCase("unspecified")) { 128 | Failure(new RuntimeException("Gradle Version-Unspecified Request")) 129 | } else if (version.toUpperCase.endsWith("SNAPSHOT")) { 130 | Failure(new RuntimeException("SNAPSHOT Request")) 131 | } else { 132 | val organization = groupIds.split("/").filter(_.nonEmpty).mkString(".") 133 | val typ = ext match { 134 | case "pom" => "ivy" 135 | case _ => "jar" 136 | } 137 | val peerFile = ext match { 138 | case "pom" => "ivy.xml" 139 | case _ => s"$artifactId.$ext" 140 | } 141 | val result = if (scalaVersion != null && sbtVersion != null) { 142 | s"/$organization/$artifactId/scala_$scalaVersion/sbt_$sbtVersion/$version/${typ}s/$peerFile" :: Nil 143 | } else if (scalaVersion == null && sbtVersion == null) { 144 | val guessedMavenArtifacts = for (scala <- supportedScalaVersion; sbt <- supportedSbtVersion) yield 145 | s"$groupIds/${artifactId}_${scala}_$sbt/$version/$fileName" 146 | s"/$organization/$artifactId/$version/${typ}s/$peerFile" :: guessedMavenArtifacts 147 | } else List(s"/$organization/$artifactId/$version/${typ}s/$peerFile") 148 | Success(result) 149 | } 150 | case IvyFormat(organization, module, _, scalaVersion, _, sbtVersion, revision, typ, fileName, artifact, _, classifier, ext) 151 | => 152 | val result = if (scalaVersion == null && sbtVersion == null) { 153 | for (scala <- supportedScalaVersion; sbt <- supportedSbtVersion) yield 154 | s"/${organization.split("\\.").mkString("/")}/${module}_${scala}_$sbt/$revision/$module-$revision.$ext" 155 | } else Nil 156 | if (revision.toUpperCase.endsWith("SNAPSHOT")) { 157 | Failure(new RuntimeException("SNAPSHOT Request")) 158 | } else { 159 | Success(result) 160 | } 161 | case _ => 162 | Failure(new RuntimeException("Invalid Request")) 163 | } 164 | 165 | val resourceHandlers: Agent[Map[FileResourceManager, ResourceHandler]] = Agent(null) 166 | 167 | def handle(exchange: HttpServerExchange): Unit = { 168 | val uri = exchange.getRequestURI 169 | val method = exchange.getRequestMethod 170 | uri match { 171 | case "/" => 172 | redirectTo(exchange, "/admin/index.html") 173 | case "/favicon.ico" => 174 | redirectTo(exchange, "/admin/favicon.ico") 175 | case _ => 176 | Repox.peer(uri) match { 177 | case Success(_) => 178 | method match { 179 | case Methods.HEAD => 180 | requestQueueMaster ! Requests.Head(exchange) 181 | case Methods.GET => 182 | requestQueueMaster ! Requests.Get(exchange) 183 | case _ => 184 | immediate404(exchange) 185 | } 186 | case Failure(_) => 187 | Repox.respond404(exchange) 188 | logger.debug(s"Invalid request $method $uri. 404.") 189 | } 190 | } 191 | } 192 | 193 | def init(): Unit = { 194 | configPersister ! 'LoadConfig // this does nothing but eagerly init Repox 195 | } 196 | } 197 | -------------------------------------------------------------------------------- /src/main/scala/com/gtan/repox/RequestQueueMaster.scala: -------------------------------------------------------------------------------- 1 | package com.gtan.repox 2 | 3 | import java.nio.file.Paths 4 | 5 | import akka.actor._ 6 | import com.gtan.repox.config.{ConfigFormats, Config} 7 | import io.undertow.Handlers 8 | import io.undertow.server.handlers.resource.{FileResourceManager, ResourceManager} 9 | import play.api.libs.json.Json 10 | 11 | import scala.util.Random 12 | 13 | case class Queue(method: Symbol, uri: String) 14 | 15 | object RequestQueueMaster { 16 | 17 | // send by ConfigPersister 18 | case object ConfigLoaded 19 | 20 | // send when mainClient and proxyClients are all initialized 21 | case object ClientsInitialized 22 | 23 | // send by QueueWorker 24 | case class KillMe(queue: Queue) 25 | 26 | // send by FileDeleter 27 | case class Quarantine(uri: String) 28 | 29 | case class FileDeleted(uri: String) 30 | 31 | } 32 | 33 | class RequestQueueMaster extends Actor with Stash with ActorLogging with ConfigFormats { 34 | 35 | import RequestQueueMaster._ 36 | 37 | import concurrent.ExecutionContext.Implicits.global 38 | 39 | var children = Map.empty[Queue, ActorRef] // Queue -> GetHeadQueueWorker 40 | 41 | var quarantined = Map.empty[String, ActorRef] // Uri -> FileDeleter 42 | 43 | override def receive = initializing 44 | 45 | def initializing: Receive = { 46 | case ConfigLoaded => 47 | log.info("Config loaded.") 48 | val fut1 = Repox.initClients() 49 | val fut2 = Repox.initResourceManagers() 50 | for (both <- fut1 zip fut2) { 51 | self ! ClientsInitialized 52 | } 53 | case ClientsInitialized => 54 | log.debug(s"ResourceBases (${Repox.resourceHandlers.get().keys.map(_.getBase).mkString(",")}) initialized.") 55 | log.debug(s"AHC clients (${Repox.clients.get().keys.mkString(",")}) initialized.") 56 | unstashAll() 57 | context become started 58 | case msg => 59 | log.debug("Repox initializing , stash all msgs...") 60 | stash() 61 | } 62 | 63 | def started: Receive = { 64 | case Quarantine(uri) => 65 | quarantined.get(uri) match { 66 | case None => 67 | quarantined = quarantined.updated(uri, sender()) 68 | quarantined = quarantined.updated(uri + ".sha1", sender()) 69 | sender ! FileDeleter.Quarantined 70 | case _ => // already quarantined , ignore 71 | } 72 | case FileDeleted(uri) => 73 | quarantined = quarantined - uri - s"$uri.sha1" 74 | // log.debug(s"Redownloading $uri and $uri.sha1") 75 | // self ! Requests.Download(uri, Config.enabledRepos) 76 | 77 | case KillMe(queue) => 78 | for (worker <- children.get(queue)) { 79 | log.debug(s"RequestQueueMaster stopping worker ${worker.path.name}") 80 | worker ! PoisonPill 81 | } 82 | children = children - queue 83 | 84 | case req@Requests.Get(exchange) => 85 | val uri = exchange.getRequestURI 86 | quarantined.get(uri) match { 87 | case None => 88 | val queue = Queue('get, uri) 89 | if (Config.immediate404Rules.filterNot(_.disabled).exists(_.matches(uri))) { 90 | Repox.immediate404(exchange) 91 | for (worker <- children.get(queue)) { 92 | worker ! PoisonPill 93 | } 94 | } else { 95 | for (peers <- Repox.peer(uri)) { 96 | peers.find(p => Repox.downloaded(p).isDefined) match { 97 | case Some(peer) => 98 | Repox.smart404(exchange) 99 | case None => 100 | Repox.downloaded(uri) match { 101 | case Some(Tuple2(resourceManager, resourceHandler)) => 102 | Repox.immediateFile(resourceHandler, exchange) 103 | for (worker <- children.get(queue)) { 104 | worker ! PoisonPill 105 | } 106 | case None => 107 | children.get(queue) match { 108 | case None => 109 | val childName = s"GetQueueWorker_${Repox.nextId}" 110 | val worker = context.actorOf(Props(classOf[GetQueueWorker], uri), name = childName) 111 | children = children.updated(Queue('get, uri), worker) 112 | worker ! req 113 | case Some(worker) => 114 | worker ! req 115 | } 116 | } 117 | } 118 | } 119 | } 120 | case Some(deleter) => 121 | // file quarantined, 404 122 | Repox.respond404(exchange) 123 | } 124 | case req@Requests.Head(exchange) => 125 | val uri = exchange.getRequestURI 126 | val queue = Queue('head, uri) 127 | if (Config.immediate404Rules.filterNot(_.disabled).exists(_.matches(uri))) { 128 | Repox.immediate404(exchange) 129 | for (worker <- children.get(queue)) { 130 | worker ! PoisonPill 131 | } 132 | } else { 133 | for (peers <- Repox.peer(uri)) { 134 | peers.find(p => Repox.downloaded(p).isDefined) match { 135 | case Some(peer) => 136 | Repox.smart404(exchange) 137 | case _ => 138 | quarantined.get(uri) match { 139 | case None => 140 | Repox.downloaded(uri) match { 141 | case Some(Tuple2(resourceManager, resourceHandler)) => 142 | Repox.immediateHead(resourceManager, exchange) 143 | for (worker <- children.get(queue)) { 144 | worker ! PoisonPill 145 | } 146 | case None => 147 | children.get(queue) match { 148 | case None => 149 | val workerActorName = s"HeadQueueWorker_${Repox.nextId}" 150 | val worker = context.actorOf(Props(classOf[HeadQueueWorker], uri), workerActorName) 151 | log.debug(s"create HeadQueueWorker $workerActorName") 152 | children = children.updated(queue, worker) 153 | worker ! req 154 | case Some(worker) => 155 | log.debug(s"Enqueue to HeadQueueWorker ${worker.path.name} $uri") 156 | worker ! req 157 | } 158 | } 159 | case Some(deleter) => 160 | // file quarantined, 404 161 | Repox.respond404(exchange) 162 | } 163 | } 164 | } 165 | } 166 | } 167 | } 168 | -------------------------------------------------------------------------------- /src/main/scala/com/gtan/repox/Requests.scala: -------------------------------------------------------------------------------- 1 | package com.gtan.repox 2 | 3 | import com.gtan.repox.data.Repo 4 | import io.undertow.server.HttpServerExchange 5 | 6 | object Requests { 7 | trait Request 8 | case class Get(exchange: HttpServerExchange) extends Request 9 | case class Head(exchange: HttpServerExchange) extends Request 10 | } 11 | -------------------------------------------------------------------------------- /src/main/scala/com/gtan/repox/TimeoutableFuture.scala: -------------------------------------------------------------------------------- 1 | package com.gtan.repox 2 | 3 | import java.util.concurrent.TimeUnit 4 | 5 | import com.typesafe.scalalogging.LazyLogging 6 | import org.jboss.netty.handler.timeout 7 | import org.jboss.netty.handler.timeout.TimeoutException 8 | import org.jboss.netty.util.{Timeout, TimerTask, HashedWheelTimer} 9 | 10 | import scala.concurrent.{ExecutionContext, Future, Promise} 11 | import scala.concurrent.duration.Duration 12 | 13 | /** 14 | * Created by xf on 14/11/20. 15 | */ 16 | object TimeoutableFuture extends LazyLogging{ 17 | val timer = new HashedWheelTimer(200, TimeUnit.MILLISECONDS) 18 | 19 | private def scheduleTimeout(name: String, promise: Promise[_], after: Duration) = { 20 | timer.newTimeout(new TimerTask { 21 | def run(timeout: Timeout) { 22 | promise.failure(new TimeoutException(s"future $name timeout (${after.toMillis} millis)")) 23 | } 24 | }, after.toNanos, TimeUnit.NANOSECONDS) 25 | } 26 | 27 | def apply[T](fut: Future[T], after: Duration, name: String)(implicit ec: ExecutionContext) = { 28 | val prom = Promise[T]() 29 | val timeout = scheduleTimeout(name, prom, after) 30 | val combinedFut = Future.firstCompletedOf(List(fut, prom.future)) 31 | fut onComplete { case result => 32 | // if(!timeout.isExpired){ 33 | // logger.debug(s"timeout for $name canceled because of peer future completion") 34 | // } 35 | timeout.cancel() 36 | } 37 | combinedFut 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /src/main/scala/com/gtan/repox/Timestamp.scala: -------------------------------------------------------------------------------- 1 | package com.gtan.repox 2 | 3 | /** 4 | * copied from spray-util 5 | */ 6 | 7 | import concurrent.duration._ 8 | 9 | class Timestamp private(val timestampNanos: Long) extends AnyVal { 10 | def +(period: Duration): Timestamp = 11 | if (isNever) this 12 | else if (!period.isFinite()) Timestamp.never 13 | else new Timestamp(timestampNanos + period.toNanos) 14 | 15 | def -(other: Timestamp): Duration = 16 | if (isNever) Duration.Inf 17 | else if (other.isNever) Duration.MinusInf 18 | else (timestampNanos - other.timestampNanos).nanos 19 | 20 | def isPast: Boolean = System.nanoTime() >= timestampNanos 21 | 22 | def isPast(now: Timestamp): Boolean = now.timestampNanos >= timestampNanos 23 | 24 | def isFuture: Boolean = !isPast 25 | 26 | def isFinite: Boolean = timestampNanos < Long.MaxValue 27 | 28 | def isNever: Boolean = timestampNanos == Long.MaxValue 29 | } 30 | 31 | object Timestamp { 32 | def now: Timestamp = new Timestamp(System.nanoTime()) 33 | 34 | def never: Timestamp = new Timestamp(Long.MaxValue) 35 | 36 | implicit val timestampOrdering: Ordering[Timestamp] = new Ordering[Timestamp] { 37 | def compare(x: Timestamp, y: Timestamp): Int = 38 | if (x.timestampNanos < y.timestampNanos) -1 else if (x.timestampNanos == y.timestampNanos) 0 else 1 39 | } 40 | } -------------------------------------------------------------------------------- /src/main/scala/com/gtan/repox/admin/AuthHandler.scala: -------------------------------------------------------------------------------- 1 | package com.gtan.repox.admin 2 | 3 | import java.io.IOException 4 | import java.nio.charset.Charset 5 | import java.util.Date 6 | 7 | import com.gtan.repox.Repox 8 | import com.gtan.repox.config.ConfigPersister.SaveSnapshot 9 | import com.gtan.repox.config._ 10 | import com.typesafe.scalalogging.LazyLogging 11 | import io.undertow.io.Receiver.{ErrorCallback, FullStringCallback} 12 | import io.undertow.server.HttpServerExchange 13 | import io.undertow.server.handlers.{CookieImpl, Cookie} 14 | import io.undertow.util._ 15 | import play.api.libs.json.Json 16 | import akka.pattern.ask 17 | import concurrent.duration._ 18 | import scala.language.postfixOps 19 | import scala.util.{Failure, Success} 20 | import concurrent.ExecutionContext.Implicits.global 21 | 22 | object AuthHandler extends RestHandler with LazyLogging with ConfigFormats { 23 | 24 | import WebConfigHandler._ 25 | import ParameterPersister._ 26 | 27 | implicit val timeout = akka.util.Timeout(1 second) 28 | 29 | val AUTH_KEY: String = "authenticated" 30 | 31 | private def authenticated(exchange: HttpServerExchange) = { 32 | val cookie = exchange.getRequestCookies.get(AUTH_KEY) 33 | cookie != null && cookie.getValue == "true" 34 | } 35 | 36 | override def route(implicit exchange: HttpServerExchange) = { 37 | val globallyAccessible: PartialFunction[(HttpString, String), Unit] = { 38 | case (Methods.POST, "login") => 39 | val pass = exchange.getQueryParameters.get("v").getFirst 40 | exchange.setStatusCode(StatusCodes.OK) 41 | if (Config.password == pass) { 42 | exchange.setResponseCookie(new CookieImpl(AUTH_KEY, "true").setPath("/admin")) 43 | exchange.getResponseSender.send( """{"success": true}""") 44 | } else { 45 | exchange.getResponseSender.send( """{"success": false}""") 46 | } 47 | case (Methods.POST, "logout") => 48 | exchange.setStatusCode(StatusCodes.OK) 49 | exchange.setResponseCookie(new CookieImpl("authenticated", "true").setPath("/admin").setMaxAge(0)) 50 | exchange.getRequestCookies.remove("authenticated") 51 | exchange.getResponseChannel 52 | exchange.endExchange() 53 | case (Methods.GET, "exportConfig") => 54 | exchange.setStatusCode(StatusCodes.OK) 55 | exchange.getResponseHeaders.add(Headers.CONTENT_TYPE, "application/force-download") 56 | exchange.getResponseHeaders.add(Headers.CONTENT_DISPOSITION, """attachment; filename="repox.config.json""") 57 | exchange.getResponseSender.send(Json.toJson(Config.get.copy(password = "not exported")).toString) 58 | } 59 | val needAuth: PartialFunction[(HttpString, String), Unit] = { 60 | case _ if !authenticated(exchange) => 61 | exchange.setStatusCode(StatusCodes.FORBIDDEN) 62 | exchange.endExchange() 63 | } 64 | globallyAccessible orElse needAuth orElse { 65 | case (Methods.POST, "saveSnapshot") => 66 | (Repox.configPersister ? SaveSnapshot).onComplete { 67 | result => 68 | exchange.getResponseSender.send( s"""{"success": ${result.isSuccess}}""") 69 | } 70 | case (Methods.PUT, "importConfig") => 71 | val contentType = exchange.getRequestHeaders.getFirst(Headers.CONTENT_TYPE) 72 | if (contentType.startsWith("application/json")) { 73 | val splitted = contentType.split("charset=") 74 | val charset = if(splitted.length == 2) Charset.forName(splitted(1)) else Charset.forName("UTF-8") 75 | exchange.getRequestReceiver.receiveFullString( 76 | new FullStringCallback { 77 | override def handle(exchange: HttpServerExchange, message: String): Unit = { 78 | val uploaded = Json.parse(message).as[Config] 79 | setConfigAndRespond(exchange, Repox.configPersister ? ImportConfig(uploaded)) 80 | } 81 | }, new ErrorCallback { 82 | override def error(exchange: HttpServerExchange, e: IOException): Unit = { 83 | exchange.setStatusCode(StatusCodes.INTERNAL_SERVER_ERROR) 84 | exchange.endExchange() 85 | } 86 | }, charset) 87 | } else { 88 | exchange.setStatusCode(StatusCodes.BAD_REQUEST) 89 | exchange.endExchange() 90 | } 91 | case (Methods.PUT, "password") => 92 | val v = exchange.getQueryParameters.get("v").getFirst 93 | val json = Json.parse(v) 94 | val (p1, p2) = ((json \ "p1").as[String], (json \ "p2").as[String]) 95 | if (p1 == p2) { 96 | setConfigAndRespond(exchange, Repox.configPersister ? ModifyPassword(p1)) 97 | } 98 | } 99 | } 100 | } -------------------------------------------------------------------------------- /src/main/scala/com/gtan/repox/admin/ConnectorVO.scala: -------------------------------------------------------------------------------- 1 | package com.gtan.repox.admin 2 | 3 | import com.gtan.repox.config.Config 4 | import com.gtan.repox.data.{Connector, ProxyServer} 5 | import play.api.libs.json.Json 6 | 7 | case class ConnectorVO(connector: Connector, proxy: Option[ProxyServer]) { 8 | } 9 | 10 | object ConnectorVO { 11 | def wrap(connector: Connector) = ConnectorVO(connector, Config.proxyUsage.get(connector)) 12 | 13 | implicit val format = Json.format[ConnectorVO] 14 | } 15 | -------------------------------------------------------------------------------- /src/main/scala/com/gtan/repox/admin/ConnectorsHandler.scala: -------------------------------------------------------------------------------- 1 | package com.gtan.repox.admin 2 | 3 | import java.net.URLDecoder 4 | 5 | import akka.pattern.ask 6 | import com.gtan.repox.Repox 7 | import com.gtan.repox.config.{Config, ConfigPersister} 8 | import com.gtan.repox.data.Connector 9 | import io.undertow.server.HttpServerExchange 10 | import io.undertow.util.{HttpString, Methods} 11 | import play.api.libs.json.{JsObject, Json} 12 | 13 | import scala.concurrent.duration._ 14 | import scala.language.postfixOps 15 | 16 | object ConnectorsHandler extends RestHandler { 17 | 18 | import com.gtan.repox.admin.WebConfigHandler._ 19 | import com.gtan.repox.config.ConnectorPersister._ 20 | 21 | implicit val timeout = akka.util.Timeout(1 second) 22 | 23 | override def route(implicit exchange: HttpServerExchange): PartialFunction[(HttpString, String), Unit] = { 24 | case (Methods.GET, "connectors") => 25 | val config = Config.get 26 | respondJson(exchange, Json.obj( 27 | "connectors" -> config.connectors.map(ConnectorVO.wrap), 28 | "proxies" -> config.proxies) 29 | ) 30 | case (Methods.POST, "connector") => 31 | val newV = exchange.getQueryParameters.get("v").getFirst 32 | val vo = Json.parse(newV).as[ConnectorVO] 33 | setConfigAndRespond(exchange, Repox.configPersister ? NewConnector(vo)) 34 | case (Methods.PUT, "connector") => 35 | val newV = exchange.getQueryParameters.get("v").getFirst 36 | val vo = Json.parse(newV).as[ConnectorVO] 37 | setConfigAndRespond(exchange, Repox.configPersister ? UpdateConnector(vo)) 38 | case (Methods.DELETE, "connector") => 39 | val newV = exchange.getQueryParameters.get("v").getFirst 40 | setConfigAndRespond(exchange, Repox.configPersister ? DeleteConnector(newV.toLong)) 41 | 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /src/main/scala/com/gtan/repox/admin/ExpireRulesHandler.scala: -------------------------------------------------------------------------------- 1 | package com.gtan.repox.admin 2 | 3 | import java.net.URLDecoder 4 | 5 | import com.gtan.repox.Repox 6 | import com.gtan.repox.config.Config 7 | import com.gtan.repox.data.ExpireRule 8 | import io.undertow.server.HttpServerExchange 9 | import io.undertow.util.Methods 10 | import play.api.libs.json.Json 11 | import akka.pattern.ask 12 | import concurrent.duration._ 13 | import com.gtan.repox.config.ExpireRulePersister._ 14 | 15 | import collection.JavaConverters._ 16 | 17 | object ExpireRulesHandler extends RestHandler { 18 | implicit val timeout = akka.util.Timeout(5 seconds) 19 | 20 | import com.gtan.repox.admin.WebConfigHandler._ 21 | 22 | override def route(implicit exchange: HttpServerExchange) = { 23 | case (Methods.GET, "expireRules") => 24 | respondJson(exchange, Config.expireRules) 25 | case (Methods.POST, "expireRule") | (Methods.PUT, "expireRule") => 26 | val newV = exchange.getQueryParameters.get("v").getFirst 27 | val rule = Json.parse(newV).as[ExpireRule] 28 | setConfigAndRespond(exchange, Repox.configPersister ? NewOrUpdateExpireRule(rule)) 29 | case (Methods.PUT, "expireRule/enable") => 30 | val newV = exchange.getQueryParameters.get("v").getFirst 31 | setConfigAndRespond(exchange, Repox.configPersister ? EnableExpireRule(newV.toLong)) 32 | case (Methods.PUT, "expireRule/disable") => 33 | val newV = exchange.getQueryParameters.get("v").getFirst 34 | setConfigAndRespond(exchange, Repox.configPersister ? DisableExpireRule(newV.toLong)) 35 | case (Methods.DELETE, "expireRule") => 36 | val newV = exchange.getQueryParameters.get("v").getFirst 37 | setConfigAndRespond(exchange, Repox.configPersister ? DeleteExpireRule(newV.toLong)) 38 | 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /src/main/scala/com/gtan/repox/admin/Immediate404RulesHandler.scala: -------------------------------------------------------------------------------- 1 | package com.gtan.repox.admin 2 | 3 | import java.net.URLDecoder 4 | 5 | import com.gtan.repox.Repox 6 | import com.gtan.repox.config.Config 7 | import com.gtan.repox.config.Immediate404RulePersister._ 8 | import com.gtan.repox.data.Immediate404Rule 9 | import io.undertow.server.HttpServerExchange 10 | import io.undertow.util.Methods 11 | import play.api.libs.json.Json 12 | import akka.pattern.ask 13 | import concurrent.duration._ 14 | 15 | object Immediate404RulesHandler extends RestHandler { 16 | implicit val timeout = akka.util.Timeout(5 seconds) 17 | 18 | import com.gtan.repox.admin.WebConfigHandler._ 19 | 20 | override def route(implicit exchange: HttpServerExchange) = { 21 | case (Methods.GET, "immediate404Rules") => 22 | respondJson(exchange, Config.immediate404Rules) 23 | case (Methods.POST, "immediate404Rule") | (Methods.PUT, "immediate404Rule") => 24 | val newV = exchange.getQueryParameters.get("v").getFirst 25 | val rule = Json.parse(newV).as[Immediate404Rule] 26 | setConfigAndRespond(exchange, Repox.configPersister ? NewOrUpdateImmediate404Rule(rule)) 27 | case (Methods.PUT, "immediate404Rule/enable") => 28 | val newV = exchange.getQueryParameters.get("v").getFirst 29 | setConfigAndRespond(exchange, Repox.configPersister ? EnableImmediate404Rule(newV.toLong)) 30 | case (Methods.PUT, "immediate404Rule/disable") => 31 | val newV = exchange.getQueryParameters.get("v").getFirst 32 | setConfigAndRespond(exchange, Repox.configPersister ? DisableImmediate404Rule(newV.toLong)) 33 | case (Methods.DELETE, "immediate404Rule") => 34 | val newV = exchange.getQueryParameters.get("v").getFirst 35 | setConfigAndRespond(exchange, Repox.configPersister ? DeleteImmediate404Rule(newV.toLong)) 36 | 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /src/main/scala/com/gtan/repox/admin/ParametersHandler.scala: -------------------------------------------------------------------------------- 1 | package com.gtan.repox.admin 2 | 3 | import com.gtan.repox.Repox 4 | import com.gtan.repox.config.Config 5 | import com.gtan.repox.config.ParameterPersister._ 6 | import io.undertow.server.HttpServerExchange 7 | import io.undertow.util.Methods 8 | import play.api.libs.json.{Json, JsNumber, JsString, JsObject} 9 | import collection.JavaConverters._ 10 | import scala.concurrent.duration._ 11 | import akka.pattern.ask 12 | 13 | import scala.language.postfixOps 14 | 15 | object ParametersHandler extends RestHandler { 16 | 17 | import WebConfigHandler._ 18 | 19 | implicit val timeout = akka.util.Timeout(1 second) 20 | 21 | override def route(implicit exchange: HttpServerExchange) = { 22 | case (Methods.GET, "parameters") => 23 | val config = Config.get 24 | respondJson(exchange, Seq( 25 | Json.obj("name" -> "headRetryTimes", "value" -> config.headRetryTimes), 26 | Json.obj("name" -> "headTimeout", "value" -> config.headTimeout.toSeconds, "unit" -> "s"), 27 | Json.obj("name" -> "extraResources", "value" -> config.extraResources) 28 | )) 29 | case (Methods.PUT, "headTimeout") => 30 | val newV = exchange.getQueryParameters.get("v").getFirst 31 | setConfigAndRespond(exchange, 32 | Repox.configPersister ? SetHeadTimeout(Duration.apply(newV.toInt, SECONDS))) 33 | case (Methods.PUT, "headRetryTimes") => 34 | val newV = exchange.getQueryParameters.get("v").getFirst 35 | setConfigAndRespond(exchange, 36 | Repox.configPersister ? SetHeadRetryTimes(newV.toInt)) 37 | case (Methods.PUT, "extraResources") => 38 | val newV = exchange.getQueryParameters.get("v").getFirst 39 | setConfigAndRespond(exchange, 40 | Repox.configPersister ? SetExtraResources(newV)) 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /src/main/scala/com/gtan/repox/admin/ProxiesHandler.scala: -------------------------------------------------------------------------------- 1 | package com.gtan.repox.admin 2 | 3 | import java.net.URLDecoder 4 | 5 | import com.gtan.repox.Repox 6 | import com.gtan.repox.config.Config 7 | import com.gtan.repox.config.ProxyPersister._ 8 | import com.gtan.repox.data.ProxyServer 9 | import io.undertow.server.HttpServerExchange 10 | import io.undertow.util.{HttpString, Methods} 11 | import play.api.libs.json.Json 12 | import collection.JavaConverters._ 13 | import akka.pattern.ask 14 | import concurrent.duration._ 15 | 16 | object ProxiesHandler extends RestHandler { 17 | implicit val timeout = akka.util.Timeout(5 seconds) 18 | 19 | import WebConfigHandler._ 20 | 21 | override def route(implicit exchange: HttpServerExchange): PartialFunction[(HttpString, String), Unit] = { 22 | case (Methods.GET, "proxies") => 23 | respondJson(exchange, Config.proxies) 24 | case (Methods.POST, "proxy") | (Methods.PUT, "proxy") => 25 | val newV = exchange.getQueryParameters.get("v").getFirst 26 | val proxy = Json.parse(newV).as[ProxyServer] 27 | setConfigAndRespond(exchange, Repox.configPersister ? NewOrUpdateProxy(proxy)) 28 | case (Methods.PUT, "proxy/enable") => 29 | val newV = exchange.getQueryParameters.get("v").getFirst 30 | setConfigAndRespond(exchange, Repox.configPersister ? EnableProxy(newV.toLong)) 31 | case (Methods.PUT, "proxy/disable") => 32 | val newV = exchange.getQueryParameters.get("v").getFirst 33 | setConfigAndRespond(exchange, Repox.configPersister ? DisableProxy(newV.toLong)) 34 | case (Methods.DELETE, "proxy") => 35 | val newV = exchange.getQueryParameters.get("v").getFirst 36 | setConfigAndRespond(exchange, Repox.configPersister ? DeleteProxy(newV.toLong)) 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /src/main/scala/com/gtan/repox/admin/RepoVO.scala: -------------------------------------------------------------------------------- 1 | package com.gtan.repox.admin 2 | 3 | import com.gtan.repox.Repox 4 | import com.gtan.repox.config.Config 5 | import com.gtan.repox.data.{DurationFormat, Connector, ProxyServer, Repo} 6 | import com.ning.http.client.{ProxyServer => JProxyServer} 7 | import play.api.libs.json.Json 8 | 9 | import collection.JavaConverters._ 10 | 11 | case class RepoVO(repo: Repo, connector: Option[Connector]) 12 | 13 | object RepoVO { 14 | def wrap(repo: Repo): RepoVO = RepoVO(repo, Config.connectorUsage.get(repo)) 15 | 16 | implicit val format = Json.format[RepoVO] 17 | } 18 | -------------------------------------------------------------------------------- /src/main/scala/com/gtan/repox/admin/ResetHandler.scala: -------------------------------------------------------------------------------- 1 | package com.gtan.repox.admin 2 | 3 | import com.gtan.repox.Repox 4 | import com.typesafe.scalalogging.LazyLogging 5 | import io.undertow.server.HttpServerExchange 6 | import io.undertow.util.Methods 7 | 8 | object ResetHandler extends RestHandler with LazyLogging{ 9 | import WebConfigHandler._ 10 | 11 | override def route(implicit exchange: HttpServerExchange) = { 12 | case (Methods.POST, "resetMainClient") => 13 | // setConfigAndRespond(exchange, Repox.mainClient.alter(Repox.createMainClient)) 14 | case (Methods.POST, "resetProxyClients") => 15 | // setConfigAndRespond(exchange, Repox.proxyClients.alter(Repox.createProxyClients)) 16 | case _ => 17 | logger.debug(s"Invalid Request. ${exchange.getRequestURI}") 18 | Repox.respond404(exchange) 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /src/main/scala/com/gtan/repox/admin/RestHandler.scala: -------------------------------------------------------------------------------- 1 | package com.gtan.repox.admin 2 | 3 | import io.undertow.server.HttpServerExchange 4 | import io.undertow.util.HttpString 5 | 6 | trait RestHandler { 7 | def route(implicit exchange: HttpServerExchange): PartialFunction[(HttpString, String), Unit] 8 | } 9 | -------------------------------------------------------------------------------- /src/main/scala/com/gtan/repox/admin/StaticAssetHandler.scala: -------------------------------------------------------------------------------- 1 | package com.gtan.repox.admin 2 | 3 | import io.undertow.Handlers 4 | import io.undertow.server.HttpServerExchange 5 | import io.undertow.server.handlers.resource.ClassPathResourceManager 6 | import io.undertow.util.Methods 7 | 8 | object StaticAssetHandler extends RestHandler { 9 | 10 | import WebConfigHandler._ 11 | 12 | override def route(implicit exchange: HttpServerExchange) = { 13 | case (Methods.GET, target) if isStaticRequest(target) => 14 | Handlers 15 | .resource(new ClassPathResourceManager(this.getClass.getClassLoader)) 16 | .handleRequest(exchange) 17 | 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /src/main/scala/com/gtan/repox/admin/UpstreamsHandler.scala: -------------------------------------------------------------------------------- 1 | package com.gtan.repox.admin 2 | 3 | import akka.pattern.ask 4 | import com.gtan.repox.Repox 5 | import com.gtan.repox.config.Config 6 | import com.gtan.repox.config.RepoPersister._ 7 | import io.undertow.server.HttpServerExchange 8 | import io.undertow.util.{HttpString, Methods} 9 | import play.api.libs.json.Json 10 | 11 | import scala.concurrent.duration._ 12 | import scala.language.postfixOps 13 | 14 | object UpstreamsHandler extends RestHandler { 15 | 16 | import WebConfigHandler._ 17 | 18 | implicit val timeout = akka.util.Timeout(1 second) 19 | 20 | override def route(implicit exchange: HttpServerExchange): PartialFunction[(HttpString, String), Unit] = { 21 | case (Methods.GET, "upstreams") => 22 | val config = Config.get 23 | respondJson(exchange, Json.obj( 24 | "upstreams" -> config.repos.sortBy(_.priority).map(RepoVO.wrap), 25 | "connectors" -> config.connectors.filterNot(_.name == "default") 26 | )) 27 | 28 | case (Methods.POST, "upstream") => 29 | val newV = exchange.getQueryParameters.get("v").getFirst 30 | val vo = Json.parse(newV).as[RepoVO] 31 | setConfigAndRespond(exchange, Repox.configPersister ? NewRepo(vo)) 32 | case (Methods.POST, "upstream/up") => 33 | val id = exchange.getQueryParameters.get("v").getFirst.toLong 34 | setConfigAndRespond(exchange, Repox.configPersister ? MoveUpRepo(id)) 35 | case (Methods.POST, "upstream/down") => 36 | val id = exchange.getQueryParameters.get("v").getFirst.toLong 37 | setConfigAndRespond(exchange, Repox.configPersister ? MoveDownRepo(id)) 38 | case (Methods.PUT, "upstream") => 39 | val newV = exchange.getQueryParameters.get("v").getFirst 40 | val vo = Json.parse(newV).as[RepoVO] 41 | setConfigAndRespond(exchange, Repox.configPersister ? UpdateRepo(vo)) 42 | case (Methods.PUT, "upstream/disable") => 43 | val id = exchange.getQueryParameters.get("v").getFirst.toLong 44 | setConfigAndRespond(exchange, Repox.configPersister ? DisableRepo(id)) 45 | case (Methods.PUT, "upstream/enable") => 46 | val id = exchange.getQueryParameters.get("v").getFirst.toLong 47 | setConfigAndRespond(exchange, Repox.configPersister ? EnableRepo(id)) 48 | case (Methods.DELETE, "upstream") => 49 | val newV = exchange.getQueryParameters.get("v").getFirst 50 | setConfigAndRespond(exchange, Repox.configPersister ? DeleteRepo(newV.toLong)) 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /src/main/scala/com/gtan/repox/admin/WebConfigHandler.scala: -------------------------------------------------------------------------------- 1 | package com.gtan.repox.admin 2 | 3 | import java.nio.ByteBuffer 4 | import java.nio.charset.Charset 5 | 6 | import io.undertow.server.HttpServerExchange 7 | import io.undertow.util.{Headers, StatusCodes} 8 | import play.api.libs.json.Format 9 | 10 | import scala.concurrent.ExecutionContext.Implicits.global 11 | import scala.concurrent.Future 12 | import scala.language.postfixOps 13 | import scala.util.{Failure, Success} 14 | 15 | 16 | object WebConfigHandler { 17 | def handle(httpServerExchange: HttpServerExchange) = { 18 | implicit val exchange = httpServerExchange 19 | val (method, uriUnprefixed) = ( 20 | httpServerExchange.getRequestMethod, 21 | httpServerExchange.getRequestURI.drop("/admin/".length)) 22 | restHandlers.map(_.route).reduce(_ orElse _).apply(method -> uriUnprefixed) 23 | } 24 | 25 | val restHandlers: Seq[RestHandler] = List( 26 | StaticAssetHandler, 27 | AuthHandler, 28 | UpstreamsHandler, 29 | ConnectorsHandler, 30 | ProxiesHandler, 31 | Immediate404RulesHandler, 32 | ExpireRulesHandler, 33 | ParametersHandler, 34 | ResetHandler 35 | ) 36 | 37 | def setConfigAndRespond(exchange: HttpServerExchange, result: Future[Any]): Unit = { 38 | result.onComplete { 39 | case Success(_) => 40 | respondEmptyOK(exchange) 41 | case Failure(t) => 42 | respondError(exchange, t) 43 | } 44 | } 45 | 46 | def isStaticRequest(target: String) = Set(".html", ".css", ".js", ".json", ".ico", ".ttf", ".map", "woff", "woff2", ".svg", "otf", "png", "jpg", "gif").exists(target.endsWith) 47 | 48 | def respondJson[T: Format](exchange: HttpServerExchange, data: T): Unit = { 49 | exchange.setStatusCode(StatusCodes.OK) 50 | val respondHeaders = exchange.getResponseHeaders 51 | respondHeaders.put(Headers.CONTENT_TYPE, "application/json") 52 | val json = implicitly[Format[T]].writes(data) 53 | exchange.getResponseChannel.writeFinal(ByteBuffer.wrap(json.toString().getBytes(Charset.forName("UTF-8")))) 54 | exchange.endExchange() 55 | } 56 | 57 | def respondError(exchange: HttpServerExchange, t: Throwable): Unit = { 58 | exchange.setStatusCode(StatusCodes.INTERNAL_SERVER_ERROR) 59 | exchange.getResponseChannel 60 | exchange.endExchange() 61 | } 62 | 63 | def respondEmptyOK(exchange: HttpServerExchange): Unit = { 64 | exchange.setStatusCode(StatusCodes.OK) 65 | exchange.getResponseChannel 66 | exchange.endExchange() 67 | } 68 | 69 | } -------------------------------------------------------------------------------- /src/main/scala/com/gtan/repox/config/Config.scala: -------------------------------------------------------------------------------- 1 | package com.gtan.repox.config 2 | 3 | import java.nio.file.Paths 4 | 5 | import akka.agent.Agent 6 | import com.gtan.repox.data._ 7 | import com.ning.http.client.{ProxyServer => JProxyServer} 8 | import com.typesafe.scalalogging.LazyLogging 9 | import play.api.libs.json.Json 10 | 11 | import scala.concurrent.ExecutionContext.Implicits.global 12 | import scala.concurrent.Future 13 | import scala.concurrent.duration._ 14 | import scala.language.postfixOps 15 | import scala.util.Properties.userHome 16 | 17 | case class Config(proxies: Seq[ProxyServer], 18 | repos: IndexedSeq[Repo], 19 | connectorUsage: Map[Repo, Connector], 20 | proxyUsage: Map[Connector, ProxyServer], 21 | immediate404Rules: Seq[Immediate404Rule], 22 | expireRules: Seq[ExpireRule], 23 | connectors: Set[Connector], 24 | headTimeout: Duration, 25 | headRetryTimes: Int, 26 | password: String, 27 | extraResources: Seq[String]) extends Jsonable 28 | 29 | object Config extends LazyLogging with ConfigFormats { 30 | 31 | val defaultProxies = List( 32 | ProxyServer(id = Some(1), name = "Lantern", protocol = JProxyServer.Protocol.HTTP, host = "localhost", port = 8787) 33 | ) 34 | val defaultConnectors = Set( 35 | Connector(id = Some(1), 36 | name = "default", 37 | connectionTimeout = 5 seconds, 38 | connectionIdleTimeout = 10 seconds, 39 | maxConnections = 40, 40 | maxConnectionsPerHost = 20), 41 | Connector(id = Some(2), 42 | name = "fast-upstream", 43 | connectionTimeout = 5 seconds, 44 | connectionIdleTimeout = 5 seconds, 45 | maxConnections = 40, 46 | maxConnectionsPerHost = 20 47 | ), 48 | Connector(id = Some(3), 49 | name = "slow-upstream", 50 | connectionTimeout = 5 seconds, 51 | connectionIdleTimeout = 40 seconds, 52 | maxConnections = 40, 53 | maxConnectionsPerHost = 20 54 | ) 55 | ) 56 | val defaultRepos: IndexedSeq[Repo] = IndexedSeq( 57 | Repo(Some(1), "koala", "http://nexus.openkoala.org/content/groups/Koala-release", 58 | priority = 1, getOnly = true, maven = true), 59 | Repo(Some(3), "sonatype", "http://oss.sonatype.org/content/repositories/releases", priority = 2), 60 | Repo(Some(4), "typesafe", "http://repo.typesafe.com/typesafe/releases", priority = 2), 61 | Repo(Some(2), "oschina", "http://maven.oschina.net/content/groups/public", 62 | priority = 2, getOnly = true, maven = true, parentId = Some(8)), 63 | Repo(Some(5), "sbt-plugin", "http://dl.bintray.com/sbt/sbt-plugin-releases", priority = 4), 64 | Repo(Some(6), "scalaz", "http://dl.bintray.com/scalaz/releases", priority = 4), 65 | Repo(Some(9), "scalajs", "http://dl.bintray.com/content/scala-js/scala-js-releases", priority = 4), 66 | Repo(Some(7), "central", "http://repo1.maven.org/maven2", priority = 4, maven = true), 67 | Repo(Some(8), "ibiblio", "http://mirrors.ibiblio.org/maven2", priority = 5, maven = true, disabled = true) 68 | ) 69 | 70 | 71 | val defaultImmediate404Rules: Seq[Immediate404Rule] = Vector( 72 | Immediate404Rule(Some(1), """.+-javadoc\.jar"""), // we don't want javadoc 73 | Immediate404Rule(Some(2), """.+-parent.*\.jar"""), // parent have no jar 74 | Immediate404Rule(Some(3), """(/.+)+/((.+?-project)(_(.+?)(_(.+))?)?)/(.+?)/\3-\8(-(.+?))?\.jar"""), // maven x-project have no jar 75 | Immediate404Rule(Some(4), """(/.+)+/((.+?-pom)(_(.+?)(_(.+))?)?)/(.+?)/\3-\8(-(.+?))?\.jar"""), // maven x-pom have no jar 76 | Immediate404Rule(Some(5), """/.+?/(.+?-project)/.+/\1\.jar"""), // ivy x-project have no jar 77 | Immediate404Rule(Some(6), """/org/jboss/xnio/xnio-all/.+\.jar"""), 78 | Immediate404Rule(Some(7), """/org\.jboss\.xnio/xnio-all/.+\.jar"""), 79 | Immediate404Rule(Some(8), """/org/apache/apache/(\d+)/.+\.jar"""), 80 | Immediate404Rule(Some(9), """/org\.apache/apache/(\d+)/.+\.jar"""), 81 | Immediate404Rule(Some(10), """/com/google/google/(\d+)/.+\.jar"""), 82 | Immediate404Rule(Some(11), """/com\.google/google/(\d+)/.+\.jar"""), 83 | Immediate404Rule(Some(12), """/org/ow2/ow2/.+\.jar"""), 84 | Immediate404Rule(Some(13), """/org\.ow2/ow2/.+\.jar"""), 85 | Immediate404Rule(Some(14), """(/.+)+/((.+?-site)(_(.+?)(_(.+))?)?)/(.+?)/\3-\8(-(.+?))?\.jar"""), // maven x-site have no jar 86 | Immediate404Rule(Some(15), 87 | """/.+?/(.+?-site)/.+/\1\.jar""", // ivy x-site have no jar 88 | Some( """/com\.typesafe\.sbt/sbt-site/.*""")), // except 'sbt-site', being used by akka 89 | Immediate404Rule(Some(16), """/org/fusesource/leveldbjni/.+-sources\.jar"""), 90 | Immediate404Rule(Some(17), """/org\.fusesource\.leveldbjni/.+-sources\.jar"""), 91 | Immediate404Rule(Some(18), """/.+?/(.+?-pom)/.+/\1\.jar"""), // ivy x-pom have no jar 92 | Immediate404Rule(Some(19), """/org/webjars/.+-sources.jar"""), // webjars have no sources 93 | Immediate404Rule(Some(20), """/org\.webjars/.+-sources.jar""") 94 | ) 95 | 96 | def defaultExpireRules = Seq( 97 | ExpireRule(Some(1), ".+/maven-metadata.xml", 1 day) 98 | ) 99 | 100 | implicit class string2Repo(repoName: String) { 101 | def use(connectorName: String) = 102 | defaultRepos.find(_.name == repoName).get -> defaultConnectors.find(_.name == connectorName).get 103 | } 104 | 105 | val default = Config( 106 | proxies = defaultProxies, 107 | repos = defaultRepos, 108 | connectorUsage = Map( 109 | "koala" use "fast-upstream", 110 | "typesafe" use "slow-upstream", 111 | "oschina" use "fast-upstream", 112 | "scalajs" use "slow-upstream" 113 | ), 114 | proxyUsage = Map(), 115 | immediate404Rules = defaultImmediate404Rules, 116 | expireRules = defaultExpireRules, 117 | connectors = defaultConnectors, 118 | headTimeout = 3 seconds, 119 | headRetryTimes = 3, 120 | password = "zhimakaimen", 121 | extraResources = Seq(Paths.get(userHome, ".m2", "repository").toString) 122 | ) 123 | 124 | private[this] val instance: Agent[Config] = Agent[Config](null) 125 | 126 | def set(data: Config): Future[Config] = instance.alter(data) 127 | 128 | def get = instance.get() 129 | 130 | val storagePath = Paths.get(userHome, ".repox", "storage") 131 | 132 | def repos: Seq[Repo] = instance.get().repos 133 | 134 | def enabledRepos: Seq[Repo] = repos.filterNot(_.disabled) 135 | 136 | def proxies: Seq[ProxyServer] = instance.get().proxies 137 | 138 | def enabledProxies: Seq[ProxyServer] = proxies.filterNot(_.disabled) 139 | 140 | def connectorUsage: Map[Repo, Connector] = instance.get().connectorUsage 141 | 142 | def proxyUsage: Map[Connector, ProxyServer] = instance.get().proxyUsage 143 | 144 | def immediate404Rules: Seq[Immediate404Rule] = instance.get().immediate404Rules 145 | 146 | def enabledImmediate404Rules: Seq[Immediate404Rule] = immediate404Rules.filterNot(_.disabled) 147 | 148 | def expireRules: Seq[ExpireRule] = instance.get().expireRules 149 | 150 | def enabledExpireRules: Seq[ExpireRule] = expireRules.filterNot(_.disabled) 151 | 152 | def password: String = instance.get().password 153 | 154 | def headRetryTimes: Int = instance.get().headRetryTimes 155 | 156 | def headTimeout: Duration = instance.get().headTimeout 157 | 158 | def connectors: Set[Connector] = instance.get().connectors 159 | 160 | def extraResources: Seq[String] = instance.get().extraResources 161 | 162 | def resourceBases: Seq[String] = extraResources :+ storagePath.toString 163 | } 164 | -------------------------------------------------------------------------------- /src/main/scala/com/gtan/repox/config/ConfigFormats.scala: -------------------------------------------------------------------------------- 1 | package com.gtan.repox.config 2 | 3 | import com.gtan.repox.data.{Connector, ProxyServer, Repo} 4 | import play.api.libs.json._ 5 | 6 | trait ConfigFormats { 7 | implicit val connectorUsageFormat = new Format[Map[Repo, Connector]] { 8 | override def writes(o: Map[Repo, Connector]) = JsArray(o.map { 9 | case (repo, connector) => JsObject( 10 | Seq( 11 | "repo" -> Json.toJson(repo), 12 | "connector" -> Json.toJson(connector) 13 | ) 14 | ) 15 | }.toSeq) 16 | 17 | override def reads(json: JsValue): JsResult[Map[Repo, Connector]] = try { 18 | (json: @unchecked) match { 19 | case JsArray(values) => 20 | JsSuccess(values.map { value => 21 | (value: @unchecked) match { 22 | case obj: JsObject => (obj.fields: @unchecked) match { 23 | case Seq( 24 | ("repo", repoJsVal: JsValue), 25 | ("connector", connectorJsVal: JsValue)) => 26 | repoJsVal.as[Repo] -> connectorJsVal.as[Connector] 27 | } 28 | } 29 | } toMap) 30 | } 31 | } catch { 32 | case e: MatchError => JsError(s"Config.connectorUsage deserialize from json failed. $e") 33 | } 34 | } 35 | 36 | implicit val proxyUsageFormat = new Format[Map[Connector, ProxyServer]] { 37 | override def writes(o: Map[Connector, ProxyServer]) = JsArray(o.map { 38 | case (connector, proxy) => JsObject( 39 | Seq( 40 | "connector" -> Json.toJson(connector), 41 | "proxy" -> Json.toJson(proxy) 42 | ) 43 | ) 44 | }.toSeq) 45 | 46 | override def reads(json: JsValue): JsResult[Map[Connector, ProxyServer]] = try { 47 | (json: @unchecked) match { 48 | case JsArray(values) => 49 | JsSuccess(values.map { value => 50 | (value: @unchecked) match { 51 | case obj: JsObject => (obj.fields: @unchecked) match { 52 | case Seq( 53 | ("connector", connectorJsVal: JsValue), 54 | ("proxy", proxyJsVal: JsValue)) => 55 | connectorJsVal.as[Connector] -> proxyJsVal.as[ProxyServer] 56 | } 57 | } 58 | } toMap) 59 | } 60 | } catch { 61 | case e: MatchError => JsError(s"Config.proxyUsage deserialize from json failed. $e") 62 | } 63 | } 64 | 65 | import com.gtan.repox.data.DurationFormat._ 66 | 67 | implicit val format = Json.format[Config] 68 | 69 | } 70 | -------------------------------------------------------------------------------- /src/main/scala/com/gtan/repox/config/ConfigPersister.scala: -------------------------------------------------------------------------------- 1 | package com.gtan.repox.config 2 | 3 | import java.nio.file.Paths 4 | 5 | import akka.actor.{Status, ActorLogging, ActorRef} 6 | import akka.persistence._ 7 | import com.gtan.repox.config.ConfigPersister.SaveSnapshot 8 | import com.gtan.repox.{SerializationSupport, Repox, RequestQueueMaster} 9 | import com.ning.http.client.{AsyncHttpClient, ProxyServer => JProxyServer} 10 | import io.undertow.Handlers 11 | import io.undertow.server.handlers.resource.{FileResourceManager, ResourceManager} 12 | import io.undertow.util.StatusCodes 13 | import play.api.libs.json.{Json, JsValue} 14 | 15 | import scala.concurrent.ExecutionContext.Implicits.global 16 | import scala.concurrent.Future 17 | 18 | trait Jsonable 19 | 20 | trait ConfigCmd extends Jsonable { 21 | def transform(old: Config): Config = old 22 | } 23 | 24 | case class ImportConfig(uploaded: Config) extends ConfigCmd { 25 | override def transform(old: Config): Config = uploaded.copy(password = old.password) 26 | } 27 | 28 | object ImportConfig { 29 | implicit val formats = Json.format[ImportConfig] 30 | } 31 | 32 | // Everything can be command or Jsonable, but only Evt will be persisted. 33 | trait Evt 34 | 35 | case class ConfigChanged(config: Config, configCmd: Jsonable) extends Evt 36 | 37 | case object UseDefault extends Evt 38 | 39 | object ConfigPersister extends SerializationSupport { 40 | 41 | case object SaveSnapshot 42 | 43 | val ConfigClass = classOf[Config].getName 44 | val ImportConfigClass = classOf[ImportConfig].getName 45 | 46 | override val reader: (JsValue) => PartialFunction[String, Jsonable] = payload => { 47 | case ConfigClass => payload.as[Config] 48 | case ImportConfigClass => payload.as[ImportConfig] 49 | } 50 | 51 | override val writer: PartialFunction[Jsonable, JsValue] = { 52 | case o: Config => Json.toJson(o) 53 | case o: ImportConfig => Json.toJson(o) 54 | } 55 | } 56 | 57 | class ConfigPersister extends PersistentActor with ActorLogging { 58 | 59 | import com.gtan.repox.config.ConnectorPersister._ 60 | import com.gtan.repox.config.ParameterPersister._ 61 | 62 | 63 | override def persistenceId = "Config" 64 | 65 | var config: Config = _ 66 | var saveSnapshotRequester: Option[ActorRef] = None 67 | 68 | def onConfigSaved(sender: ActorRef, c: ConfigChanged) = { 69 | log.debug(s"event caused by cmd: ${c.configCmd}") 70 | config = c.config 71 | for { 72 | _ <- Config.set(config) 73 | _ <- c.configCmd match { 74 | case NewConnector(vo) => 75 | Repox.clients.alter { clients => 76 | clients.updated(vo.connector.name, vo.connector.createClient) 77 | } 78 | case DeleteConnector(id) => 79 | Repox.clients.alter { clients => 80 | Config.connectors.find(_.id.contains(id)).fold(clients) { connector => 81 | for (client <- clients.get(connector.name)) { 82 | client.closeAsynchronously() 83 | } 84 | clients - connector.name 85 | } 86 | } 87 | case UpdateConnector(vo) => 88 | Repox.clients.alter { clients => 89 | for (client <- clients.get(vo.connector.name)) { 90 | client.closeAsynchronously() 91 | } 92 | clients.updated(vo.connector.name, vo.connector.createClient) 93 | } 94 | case SetExtraResources(_) => 95 | Repox.resourceHandlers.alter((for (er <- Config.resourceBases) yield { 96 | val resourceManager: FileResourceManager = new FileResourceManager(Paths.get(er).toFile, 100 * 1024) 97 | val resourceHandler = Handlers.resource(resourceManager) 98 | resourceManager -> resourceHandler 99 | }).toMap) 100 | Future { 101 | Map.empty[String, AsyncHttpClient] 102 | } 103 | case ImportConfig(_) => 104 | for(client <- Repox.clients.get.valuesIterator) { 105 | client.closeAsynchronously() 106 | } 107 | for(manager <- Repox.resourceHandlers.get.keysIterator) { 108 | manager.close() 109 | } 110 | val fut1 = Repox.initClients() 111 | val fut2 = Repox.initResourceManagers() 112 | for (both <- fut1 zip fut2) yield { 113 | log.debug(s"ResourceBases (${Repox.resourceHandlers.get().keys.map(_.getBase).mkString(",")}) initialized.") 114 | log.debug(s"AHC clients (${Repox.clients.get().keys.mkString(",")}) initialized.") 115 | Map.empty[String, AsyncHttpClient] 116 | } 117 | case _ => Future { 118 | Map.empty[String, AsyncHttpClient] 119 | } 120 | } 121 | } { 122 | sender ! StatusCodes.OK 123 | } 124 | } 125 | 126 | val receiveCommand: Receive = { 127 | case cmd: ConfigCmd => 128 | val newConfig = cmd.transform(config) 129 | if (newConfig == config) { 130 | // no change 131 | sender ! StatusCodes.OK 132 | } else { 133 | persist(ConfigChanged(newConfig, cmd))(onConfigSaved(sender(), _)) 134 | } 135 | case UseDefault => 136 | persist(UseDefault) { _ => 137 | config = Config.default 138 | Config.set(config).foreach { _ => 139 | Repox.requestQueueMaster ! RequestQueueMaster.ConfigLoaded 140 | } 141 | } 142 | case SaveSnapshot => 143 | saveSnapshot(config) 144 | saveSnapshotRequester = Some(sender()) 145 | case SaveSnapshotSuccess(metadata) => 146 | log.debug(s"Config snapshot saved. Delete old ones.") 147 | deleteSnapshots(SnapshotSelectionCriteria(maxSequenceNr = metadata.sequenceNr - 1)) 148 | case f@SaveSnapshotFailure(metadata, cause) => 149 | log.debug(f.toString) 150 | for (requester <- saveSnapshotRequester) { 151 | requester ! Status.Failure(cause) 152 | saveSnapshotRequester = None 153 | } 154 | case DeleteSnapshotsSuccess(criteria) => 155 | for (requester <- saveSnapshotRequester) { 156 | requester ! criteria.maxTimestamp 157 | saveSnapshotRequester = None 158 | } 159 | case DeleteSnapshotsFailure(criteria, cause) => 160 | for (requester <- saveSnapshotRequester) { 161 | requester ! Status.Failure(cause) 162 | saveSnapshotRequester = None 163 | } 164 | } 165 | 166 | val receiveRecover: Receive = { 167 | case ConfigChanged(data, cmd) => 168 | log.debug(s"Config changed, cmd=$cmd") 169 | config = data 170 | 171 | case UseDefault => 172 | config = Config.default 173 | 174 | case SnapshotOffer(metadata, offeredSnapshot) => 175 | config = offeredSnapshot.asInstanceOf[Config] 176 | 177 | case RecoveryCompleted => 178 | if (config == null) { 179 | // no config history, save default data as snapshot 180 | self ! UseDefault 181 | } else { 182 | Config.set(config).foreach { _ => 183 | Repox.requestQueueMaster ! RequestQueueMaster.ConfigLoaded 184 | } 185 | } 186 | } 187 | } 188 | -------------------------------------------------------------------------------- /src/main/scala/com/gtan/repox/config/ConfigQuery.scala: -------------------------------------------------------------------------------- 1 | package com.gtan.repox.config 2 | 3 | import akka.actor.ActorSystem 4 | import akka.persistence.query.{EventEnvelope, PersistenceQuery} 5 | import akka.persistence.query.journal.leveldb.scaladsl.LeveldbReadJournal 6 | import akka.stream.ActorMaterializer 7 | import akka.stream.scaladsl.Source 8 | import com.typesafe.scalalogging.LazyLogging 9 | 10 | class ConfigQuery(val system: ActorSystem) extends LazyLogging { 11 | implicit val mat = ActorMaterializer()(system) 12 | 13 | val queries = PersistenceQuery(system).readJournalFor[LeveldbReadJournal]( 14 | LeveldbReadJournal.Identifier) 15 | 16 | val src = 17 | queries.eventsByPersistenceId("Config", 0L, Long.MaxValue) 18 | 19 | val events = src.map(_.event) 20 | events.runForeach { 21 | case ConfigChanged(_, cmd) => logger.debug(s"ConfigView received event caused by cmd: $cmd") 22 | case UseDefault => logger.debug(s"ConfigView received UseDefault evt") 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /src/main/scala/com/gtan/repox/config/ConnectorPersister.scala: -------------------------------------------------------------------------------- 1 | package com.gtan.repox.config 2 | 3 | import com.gtan.repox.SerializationSupport 4 | import com.gtan.repox.admin.{ConnectorVO, RepoVO} 5 | import com.gtan.repox.data.{Connector, Repo} 6 | import play.api.libs.json.{JsValue, Json} 7 | 8 | object ConnectorPersister extends SerializationSupport { 9 | 10 | case class NewConnector(vo: ConnectorVO) extends ConfigCmd { 11 | override def transform(old: Config) = { 12 | val oldConnectors = old.connectors 13 | val oldProxyUsage = old.proxyUsage 14 | // ToDo: validation 15 | val voWithId = vo.copy(connector = vo.connector.copy(id = Some(Connector.nextId.incrementAndGet()))) 16 | val newConfig = old.copy(connectors = oldConnectors + voWithId.connector) 17 | vo.proxy match { 18 | case None => newConfig 19 | case Some(p) => newConfig.copy(proxyUsage = oldProxyUsage.updated(voWithId.connector, p)) 20 | } 21 | } 22 | } 23 | 24 | implicit val NewConnectorFormat = Json.format[NewConnector] 25 | 26 | 27 | case class UpdateConnector(vo: ConnectorVO) extends ConfigCmd { 28 | override def transform(old: Config) = { 29 | val oldConnectors = old.connectors 30 | val oldProxyUsage = old.proxyUsage 31 | val id = vo.connector.id 32 | val newConfig = old.copy( 33 | connectors = oldConnectors.map { 34 | case Connector(`id`, _, _, _, _, _, _) => vo.connector 35 | case c => c 36 | }, 37 | connectorUsage = old.connectorUsage.map { 38 | case (repo, Connector(`id`, _, _, _, _, _, _)) => (repo, vo.connector) 39 | case p => p 40 | } 41 | ) 42 | vo.proxy.fold(newConfig) { p => 43 | newConfig.copy(proxyUsage = oldProxyUsage.updated(vo.connector, p)) 44 | } 45 | } 46 | } 47 | 48 | implicit val updateConnectorFormat = Json.format[UpdateConnector] 49 | 50 | case class DeleteConnector(id: Long) extends ConfigCmd { 51 | override def transform(old: Config) = { 52 | val oldConnectors = old.connectors 53 | val oldConnectorUsage = old.connectorUsage 54 | old.copy( 55 | connectors = oldConnectors.filterNot(_.id.contains(id)), 56 | connectorUsage = oldConnectorUsage.filterNot { case (repo, connector) => repo.id.contains(id) } 57 | ) 58 | } 59 | } 60 | 61 | implicit val DeleteConnectorFormat = Json.format[DeleteConnector] 62 | 63 | val NewConnectorClass = classOf[NewConnector].getName 64 | val UpdateConnectorClass = classOf[UpdateConnector].getName 65 | val DeleteConnectorClass = classOf[DeleteConnector].getName 66 | 67 | override val reader: (JsValue) => PartialFunction[String, Jsonable] = payload => { 68 | case NewConnectorClass => payload.as[NewConnector] 69 | case UpdateConnectorClass => payload.as[UpdateConnector] 70 | case DeleteConnectorClass => payload.as[DeleteConnector] 71 | } 72 | override val writer: PartialFunction[Jsonable, JsValue] = { 73 | case o: NewConnector => Json.toJson(o) 74 | case o: UpdateConnector => Json.toJson(o) 75 | case o: DeleteConnector => Json.toJson(o) 76 | } 77 | } -------------------------------------------------------------------------------- /src/main/scala/com/gtan/repox/config/ExpireRulePersister.scala: -------------------------------------------------------------------------------- 1 | package com.gtan.repox.config 2 | 3 | import javax.sql.rowset.serial.SerialStruct 4 | 5 | import com.gtan.repox.SerializationSupport 6 | import com.gtan.repox.data.ExpireRule 7 | import play.api.libs.json.{JsValue, Json} 8 | 9 | object ExpireRulePersister extends SerializationSupport { 10 | 11 | case class NewOrUpdateExpireRule(rule: ExpireRule) extends ConfigCmd { 12 | override def transform(old: Config) = { 13 | val oldRules = old.expireRules 14 | old.copy(expireRules = rule.id.fold(oldRules :+ rule.copy(id = Some(ExpireRule.nextId.incrementAndGet()))) { _id => 15 | oldRules.map { 16 | case r@ExpireRule(Some(`_id`), _, _, _) => rule 17 | case r => r 18 | } 19 | }) 20 | } 21 | } 22 | 23 | implicit val NewOrUpdateExpireRuleFormat = Json.format[NewOrUpdateExpireRule] 24 | 25 | case class EnableExpireRule(id: Long) extends ConfigCmd { 26 | override def transform(old: Config) = { 27 | val oldRules = old.expireRules 28 | old.copy(expireRules = oldRules.map { 29 | case p@ExpireRule(Some(`id`), _, _, _) => p.copy(disabled = false) 30 | case p => p 31 | }) 32 | } 33 | } 34 | 35 | implicit val EnableExpireRuleFormat = Json.format[EnableExpireRule] 36 | 37 | case class DisableExpireRule(id: Long) extends ConfigCmd { 38 | override def transform(old: Config) = { 39 | val oldRules = old.expireRules 40 | old.copy(expireRules = oldRules.map { 41 | case p@ExpireRule(Some(`id`), _, _, _) => p.copy(disabled = true) 42 | case p => p 43 | }) 44 | } 45 | } 46 | 47 | implicit val DisalbExpireRuleFormat = Json.format[DisableExpireRule] 48 | 49 | case class DeleteExpireRule(id: Long) extends ConfigCmd { 50 | override def transform(old: Config) = { 51 | val oldRules = old.expireRules 52 | old.copy( 53 | expireRules = oldRules.filterNot(_.id.contains(id)) 54 | ) 55 | } 56 | } 57 | 58 | implicit val DeleteExpireRuleFormat = Json.format[DeleteExpireRule] 59 | 60 | val NewOrUpdateExpireRuleClass = classOf[NewOrUpdateExpireRule].getName 61 | val EnableExpireRuleClass = classOf[EnableExpireRule].getName 62 | val DisableExpireRuleClass = classOf[DisableExpireRule].getName 63 | val DeleteExpireRuleClass = classOf[DeleteExpireRule].getName 64 | 65 | override val reader: (JsValue) => PartialFunction[String, Jsonable] = payload => { 66 | case NewOrUpdateExpireRuleClass => payload.as[NewOrUpdateExpireRule] 67 | case EnableExpireRuleClass => payload.as[EnableExpireRule] 68 | case DisableExpireRuleClass => payload.as[DisableExpireRule] 69 | case DeleteExpireRuleClass => payload.as[DeleteExpireRule] 70 | 71 | } 72 | override val writer: PartialFunction[Jsonable, JsValue] = { 73 | case o: NewOrUpdateExpireRule => Json.toJson(o) 74 | case o: EnableExpireRule => Json.toJson(o) 75 | case o: DisableExpireRule => Json.toJson(o) 76 | case o: DeleteExpireRule => Json.toJson(o) 77 | } 78 | } 79 | -------------------------------------------------------------------------------- /src/main/scala/com/gtan/repox/config/Immediate404RulePersister.scala: -------------------------------------------------------------------------------- 1 | package com.gtan.repox.config 2 | 3 | import com.gtan.repox.SerializationSupport 4 | import com.gtan.repox.data.Immediate404Rule 5 | import play.api.libs.json.{JsValue, Json} 6 | 7 | object Immediate404RulePersister extends SerializationSupport { 8 | 9 | case class NewOrUpdateImmediate404Rule(rule: Immediate404Rule) extends ConfigCmd { 10 | override def transform(old: Config) = { 11 | val oldRules = old.immediate404Rules 12 | old.copy(immediate404Rules = rule.id.fold(oldRules :+ rule.copy(id = Some(Immediate404Rule.nextId.incrementAndGet()))) { _id => 13 | oldRules.map { 14 | case r@Immediate404Rule(Some(`_id`), _, _, _) => rule 15 | case r => r 16 | } 17 | }) 18 | } 19 | } 20 | 21 | implicit val newOrUpdateImmediate404RuleFormat = Json.format[NewOrUpdateImmediate404Rule] 22 | 23 | case class EnableImmediate404Rule(id: Long) extends ConfigCmd { 24 | override def transform(old: Config) = { 25 | val oldRules = old.immediate404Rules 26 | old.copy(immediate404Rules = oldRules.map { 27 | case p@Immediate404Rule(Some(`id`), _, _, _) => p.copy(disabled = false) 28 | case p => p 29 | }) 30 | } 31 | } 32 | 33 | implicit val enableImmediate404RuleFormat = Json.format[EnableImmediate404Rule] 34 | 35 | case class DisableImmediate404Rule(id: Long) extends ConfigCmd { 36 | override def transform(old: Config) = { 37 | val oldRules = old.immediate404Rules 38 | old.copy(immediate404Rules = oldRules.map { 39 | case p@Immediate404Rule(Some(`id`), _, _, _) => p.copy(disabled = true) 40 | case p => p 41 | }) 42 | } 43 | } 44 | 45 | implicit val DisableImmediate404RuleFormat = Json.format[DisableImmediate404Rule] 46 | 47 | case class DeleteImmediate404Rule(id: Long) extends ConfigCmd { 48 | override def transform(old: Config) = { 49 | val oldRules = old.immediate404Rules 50 | old.copy( 51 | immediate404Rules = oldRules.filterNot(_.id.contains(id)) 52 | ) 53 | } 54 | } 55 | 56 | implicit val DeleteImmediat404RuleFormat = Json.format[DeleteImmediate404Rule] 57 | 58 | val NewOrUpdateImmediate404RuleClass = classOf[NewOrUpdateImmediate404Rule].getName 59 | val EnableImmediate404RuleClass = classOf[EnableImmediate404Rule].getName 60 | val DisableImmediate404RuleClass = classOf[DisableImmediate404Rule].getName 61 | val DeleteImmediate404RuleClass = classOf[DeleteImmediate404Rule].getName 62 | 63 | override val reader: (JsValue) => PartialFunction[String, Jsonable] = payload => { 64 | case NewOrUpdateImmediate404RuleClass => payload.as[NewOrUpdateImmediate404Rule] 65 | case EnableImmediate404RuleClass => payload.as[EnableImmediate404Rule] 66 | case DisableImmediate404RuleClass => payload.as[DisableImmediate404Rule] 67 | case DeleteImmediate404RuleClass => payload.as[DeleteImmediate404Rule] 68 | } 69 | override val writer: PartialFunction[Jsonable, JsValue] = { 70 | case o: NewOrUpdateImmediate404Rule => Json.toJson(o) 71 | case o: EnableImmediate404Rule => Json.toJson(o) 72 | case o: DisableImmediate404Rule => Json.toJson(o) 73 | case o: DeleteImmediate404Rule => Json.toJson(o) 74 | } 75 | } 76 | -------------------------------------------------------------------------------- /src/main/scala/com/gtan/repox/config/ParameterPersister.scala: -------------------------------------------------------------------------------- 1 | package com.gtan.repox.config 2 | 3 | import com.gtan.repox.SerializationSupport 4 | import com.gtan.repox.data.DurationFormat 5 | import play.api.libs.json.{JsValue, Json} 6 | 7 | import scala.concurrent.duration.Duration 8 | 9 | 10 | object ParameterPersister extends SerializationSupport { 11 | 12 | case class SetHeadTimeout(m: Duration) extends ConfigCmd { 13 | override def transform(old: Config) = { 14 | old.copy(headTimeout = m) 15 | } 16 | } 17 | 18 | 19 | import DurationFormat._ 20 | implicit val SetHeadTimeoutFormat = Json.format[SetHeadTimeout] 21 | 22 | case class SetHeadRetryTimes(m: Int) extends ConfigCmd { 23 | override def transform(old: Config) = { 24 | old.copy(headRetryTimes = m) 25 | } 26 | } 27 | 28 | implicit val SetHeadRetryTimesFormat = Json.format[SetHeadRetryTimes] 29 | 30 | case class ModifyPassword(newPassword: String) extends ConfigCmd { 31 | override def transform(old: Config): Config = { 32 | old.copy(password = newPassword) 33 | } 34 | } 35 | 36 | implicit val ModifyPasswordFormat = Json.format[ModifyPassword] 37 | 38 | case class SetExtraResources(value: String) extends ConfigCmd { 39 | override def transform(old: Config): Config = { 40 | old.copy(extraResources = value.split(":")) 41 | } 42 | } 43 | 44 | implicit val SetExtraResourcesFormat = Json.format[SetExtraResources] 45 | 46 | val SetHeadTimeoutClass = classOf[SetHeadTimeout].getName 47 | val SetHeadRetryTimesClass = classOf[SetHeadRetryTimes].getName 48 | val ModifyPasswordClass = classOf[ModifyPassword].getName 49 | val SetExtraResourcesClass = classOf[SetExtraResources].getName 50 | 51 | override val reader: (JsValue) => PartialFunction[String, Jsonable] = payload => { 52 | case SetHeadTimeoutClass => payload.as[SetHeadTimeout] 53 | case SetHeadRetryTimesClass => payload.as[SetHeadRetryTimes] 54 | case ModifyPasswordClass => payload.as[ModifyPassword] 55 | case SetExtraResourcesClass => payload.as[SetExtraResources] 56 | } 57 | override val writer: PartialFunction[Jsonable, JsValue] = { 58 | case o: SetHeadTimeout => Json.toJson(o) 59 | case o: SetHeadRetryTimes => Json.toJson(o) 60 | case o: ModifyPassword => Json.toJson(o) 61 | case o: SetExtraResources => Json.toJson(o) 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /src/main/scala/com/gtan/repox/config/ProxyPersister.scala: -------------------------------------------------------------------------------- 1 | package com.gtan.repox.config 2 | 3 | import com.gtan.repox.SerializationSupport 4 | import com.gtan.repox.data.{Connector, ProxyServer, Repo} 5 | import play.api.libs.json.{JsValue, Json} 6 | 7 | object ProxyPersister extends SerializationSupport { 8 | 9 | case class NewOrUpdateProxy(proxy: ProxyServer) extends ConfigCmd { 10 | override def transform(old: Config) = { 11 | val oldProxies = old.proxies 12 | val oldProxyUsages: Map[Connector, ProxyServer] = old.proxyUsage 13 | old.copy(proxies = proxy.id.fold(oldProxies :+ proxy.copy(id = Some(ProxyServer.nextId.incrementAndGet()))) { _id => 14 | oldProxies.map { 15 | case ProxyServer(Some(`_id`), _, _, _, _, _) => proxy 16 | case p => p 17 | } 18 | }, proxyUsage = proxy.id.fold(oldProxyUsages) { _id => 19 | oldProxyUsages.map { 20 | case (connector, ProxyServer(Some(`_id`), _, _, _, _, _)) => connector -> proxy 21 | case u => u 22 | } 23 | }) 24 | } 25 | } 26 | 27 | implicit val newOrUpdateProxyformat = Json.format[NewOrUpdateProxy] 28 | 29 | case class EnableProxy(id: Long) extends ConfigCmd { 30 | override def transform(old: Config) = { 31 | old.copy(proxies = old.proxies.map { 32 | case p@ProxyServer(Some(`id`), _, _, _, _, _) => p.copy(disabled = false) 33 | case p => p 34 | }) 35 | } 36 | } 37 | 38 | implicit val enableProxyFormat = Json.format[EnableProxy] 39 | 40 | case class DisableProxy(id: Long) extends ConfigCmd { 41 | override def transform(old: Config) = { 42 | old.copy(proxies = old.proxies.map { 43 | case p@ProxyServer(Some(`id`), _, _, _, _, _) => p.copy(disabled = true) 44 | case p => p 45 | }, proxyUsage = old.proxyUsage.filterNot { 46 | case (connector, proxy) => proxy.id.contains(id) 47 | }) 48 | } 49 | } 50 | 51 | implicit val disableProxyFormat = Json.format[DisableProxy] 52 | 53 | case class DeleteProxy(id: Long) extends ConfigCmd { 54 | override def transform(old: Config) = { 55 | old.copy( 56 | proxies = old.proxies.filterNot(_.id.contains(id)), 57 | proxyUsage = old.proxyUsage.filterNot { case (connector, proxy) => proxy.id.contains(id) } 58 | ) 59 | } 60 | } 61 | 62 | implicit val deleteProxyFormat = Json.format[DeleteProxy] 63 | 64 | val NewOrUpdateProxyClass = classOf[NewOrUpdateProxy].getName 65 | val EnableProxyClass = classOf[EnableProxy].getName 66 | val DisableProxyClass = classOf[DisableProxy].getName 67 | val DeleteProxyClass = classOf[DeleteProxy].getName 68 | 69 | override val reader: (JsValue) => PartialFunction[String, Jsonable] = payload => { 70 | case NewOrUpdateProxyClass => payload.as[NewOrUpdateProxy] 71 | case DisableProxyClass => payload.as[DisableProxy] 72 | case EnableProxyClass => payload.as[EnableProxy] 73 | case DeleteProxyClass => payload.as[DeleteProxy] 74 | } 75 | 76 | override val writer: PartialFunction[Jsonable, JsValue] = { 77 | case o: NewOrUpdateProxy => Json.toJson(o) 78 | case o: DisableProxy => Json.toJson(o) 79 | case o: EnableProxy => Json.toJson(o) 80 | case o: DeleteProxy => Json.toJson(o) 81 | } 82 | } 83 | -------------------------------------------------------------------------------- /src/main/scala/com/gtan/repox/config/RepoPersister.scala: -------------------------------------------------------------------------------- 1 | package com.gtan.repox.config 2 | 3 | import com.gtan.repox.SerializationSupport 4 | import com.gtan.repox.admin.RepoVO 5 | import com.gtan.repox.data.Repo 6 | import play.api.libs.json.{JsValue, Json} 7 | 8 | object RepoPersister extends SerializationSupport { 9 | 10 | case class NewRepo(vo: RepoVO) extends ConfigCmd { 11 | override def transform(old: Config) = { 12 | val oldRepos = old.repos 13 | val oldConnectorUsage = old.connectorUsage 14 | // ToDo: validation 15 | val voWithId = vo.copy(repo = vo.repo.copy(id = Some(Repo.nextId.incrementAndGet()))) 16 | val insertPoint = oldRepos.indexWhere(_.priority > vo.repo.priority) 17 | val newRepos = if (insertPoint == -1) { 18 | // put to the last 19 | old.copy(repos = oldRepos :+ voWithId.repo) 20 | } else { 21 | val (before, after) = oldRepos.splitAt(insertPoint) 22 | old.copy(repos = (before :+ voWithId.repo) ++ after) 23 | } 24 | vo.connector match { 25 | case None => newRepos 26 | case Some(p) => newRepos.copy(connectorUsage = oldConnectorUsage.updated(voWithId.repo, p)) 27 | } 28 | } 29 | } 30 | 31 | implicit val newRepoFormat = Json.format[NewRepo] 32 | 33 | case class DisableRepo(id: Long) extends ConfigCmd { 34 | override def transform(old: Config) = { 35 | val oldRepos = old.repos 36 | val oldConnectorUsage = old.connectorUsage 37 | old.copy(repos = oldRepos.map { 38 | case o@Repo(Some(`id`), _, _, _, _, _, _, _) => o.copy(disabled = true) 39 | case o => o 40 | }, connectorUsage = oldConnectorUsage.map { 41 | case (o@Repo(Some(`id`), _, _, _, _, _, _, _), connector) => o.copy(disabled = true) -> connector 42 | case u => u 43 | }) 44 | } 45 | } 46 | 47 | implicit val disableRepoFormat = Json.format[DisableRepo] 48 | 49 | case class EnableRepo(id: Long) extends ConfigCmd { 50 | override def transform(old: Config) = { 51 | val oldRepos = old.repos 52 | val oldConnectorUsage = old.connectorUsage 53 | old.copy(repos = oldRepos.map { 54 | case o@Repo(Some(`id`), _, _, _, _, _, _, _) => o.copy(disabled = false) 55 | case o => o 56 | }, connectorUsage = oldConnectorUsage.map { 57 | case (o@Repo(Some(`id`), _, _, _, _, _, _, _), connector) => o.copy(disabled = false) -> connector 58 | case u => u 59 | }) 60 | } 61 | } 62 | 63 | implicit val enableRepoFormat = Json.format[EnableRepo] 64 | 65 | case class DeleteRepo(id: Long) extends ConfigCmd { 66 | override def transform(old: Config) = { 67 | val oldRepos = old.repos 68 | val oldProxyUsage = old.connectorUsage 69 | old.copy( 70 | repos = oldRepos.filterNot(_.id.contains(id)), 71 | connectorUsage = oldProxyUsage.filterNot { case (repo, proxy) => repo.id.contains(id) } 72 | ) 73 | } 74 | } 75 | 76 | implicit val deleteRepoFormat = Json.format[DeleteRepo] 77 | 78 | case class UpdateRepo(vo: RepoVO) extends ConfigCmd { 79 | override def transform(old: Config) = { 80 | val oldRepos = old.repos 81 | val oldConnectorUsage = old.connectorUsage 82 | 83 | val newConfig = for (found <- oldRepos.find(_.id == vo.repo.id)) yield { 84 | val indexOfTarget = oldRepos.indexOf(found) 85 | val repoUpdated: Config = old.copy(repos = oldRepos.updated(indexOfTarget, vo.repo)) 86 | (oldConnectorUsage.get(vo.repo), vo.connector) match { 87 | case (None, None) => repoUpdated 88 | case (None, Some(p)) => repoUpdated.copy(connectorUsage = oldConnectorUsage.updated(vo.repo, p)) 89 | case (Some(p), None) => repoUpdated.copy(connectorUsage = oldConnectorUsage - vo.repo) 90 | case (Some(o), Some(n)) if o == n => repoUpdated 91 | case (Some(o), Some(n)) => repoUpdated.copy(connectorUsage = oldConnectorUsage.updated(vo.repo, n)) 92 | } 93 | } 94 | newConfig.getOrElse(old) 95 | } 96 | } 97 | 98 | implicit val updateRepoFormat = Json.format[UpdateRepo] 99 | 100 | case class MoveUpRepo(id: Long) extends ConfigCmd { 101 | override def transform(old: Config) = { 102 | val oldRepos = old.repos 103 | val oldConnectorUsage = old.connectorUsage 104 | 105 | val repo = oldRepos.find(_.id.contains(id)) 106 | repo.fold(old) { _repo => 107 | val index = oldRepos.indexOf(_repo) 108 | if (index == 0) { 109 | if (_repo.priority == 1) old // no higher level 110 | else old.copy( 111 | repos = oldRepos.map { 112 | case `_repo` => _repo.copy(priority = _repo.priority - 1) 113 | case r => r 114 | }, connectorUsage = oldConnectorUsage.map { 115 | case (o@Repo(Some(`id`), _, _, oldPriority, _, _, _, _), connector) => o.copy(priority = oldPriority - 1) -> connector 116 | case u => u 117 | }) 118 | } else { 119 | val previous = oldRepos(index - 1) 120 | if (previous.priority == _repo.priority) { 121 | // swap this two 122 | old.copy( 123 | repos = oldRepos.map { 124 | case `previous` => _repo 125 | case `_repo` => previous 126 | case r => r 127 | } 128 | ) 129 | } else { 130 | // if(previous.priority == _repo.priority - 1) uplevel as last 131 | // if(previous.priority < _repo.priority - 1) uplevel as the only one 132 | old.copy( 133 | repos = oldRepos.map { 134 | case `_repo` => _repo.copy(priority = _repo.priority - 1) 135 | case r => r 136 | }, connectorUsage = oldConnectorUsage.map { 137 | case (o@Repo(Some(`id`), _, _, oldPriority, _, _, _, _), connector) => o.copy(priority = oldPriority - 1) -> connector 138 | case u => u 139 | }) 140 | } 141 | } 142 | } 143 | } 144 | } 145 | 146 | implicit val moveUpRepoFormat = Json.format[MoveUpRepo] 147 | 148 | case class MoveDownRepo(id: Long) extends ConfigCmd { 149 | override def transform(old: Config) = { 150 | val oldRepos = old.repos 151 | val repo = oldRepos.find(_.id.contains(id)) 152 | repo.fold(old) { _repo => 153 | val index = oldRepos.indexOf(_repo) 154 | if (index == oldRepos.length - 1) { 155 | if (_repo.priority == 10) old // no lower priority 156 | else old.copy( 157 | repos = oldRepos.map { 158 | case `_repo` => _repo.copy(priority = _repo.priority + 1) 159 | case r => r 160 | } 161 | ) 162 | } else { 163 | val next = oldRepos(index + 1) 164 | if (next.priority == _repo.priority) { 165 | // swap this two 166 | old.copy( 167 | repos = oldRepos.map { 168 | case `next` => _repo 169 | case `_repo` => next 170 | case r => r 171 | } 172 | ) 173 | } else { 174 | // if(next.priority == _repo.priority + 1) downlevel as first 175 | // if(next.priority > _repo.priority + 1) downlevel as the only one 176 | old.copy( 177 | repos = oldRepos.map { 178 | case `_repo` => _repo.copy(priority = _repo.priority + 1) 179 | case r => r 180 | } 181 | ) 182 | } 183 | } 184 | } 185 | } 186 | } 187 | 188 | implicit val moveDownRepoFormat = Json.format[MoveDownRepo] 189 | 190 | val NewRepoClass = classOf[NewRepo].getName 191 | val DisableRepoClass = classOf[DisableRepo].getName 192 | val EnableRepoClass = classOf[EnableRepo].getName 193 | val DeleteRepoClass = classOf[DeleteRepo].getName 194 | val UpdateRepoClass = classOf[UpdateRepo].getName 195 | val MoveUpRepoClass = classOf[MoveUpRepo].getName 196 | val MoveDownRepoClass = classOf[MoveDownRepo].getName 197 | 198 | override val reader: JsValue => PartialFunction[String, Jsonable] = payload => { 199 | case NewRepoClass => payload.as[NewRepo] 200 | case DisableRepoClass => payload.as[DisableRepo] 201 | case EnableRepoClass => payload.as[EnableRepo] 202 | case DeleteRepoClass => payload.as[DeleteRepo] 203 | case UpdateRepoClass => payload.as[UpdateRepo] 204 | case MoveUpRepoClass => payload.as[MoveUpRepo] 205 | case MoveDownRepoClass => payload.as[MoveDownRepo] 206 | } 207 | 208 | override val writer: PartialFunction[Jsonable, JsValue] = { 209 | case o: NewRepo => Json.toJson(o) 210 | case o: DisableRepo => Json.toJson(o) 211 | case o: EnableRepo => Json.toJson(o) 212 | case o: DeleteRepo => Json.toJson(o) 213 | case o: UpdateRepo => Json.toJson(o) 214 | case o: MoveUpRepo => Json.toJson(o) 215 | case o: MoveDownRepo => Json.toJson(o) 216 | } 217 | } -------------------------------------------------------------------------------- /src/main/scala/com/gtan/repox/data/Connector.scala: -------------------------------------------------------------------------------- 1 | package com.gtan.repox.data 2 | 3 | import java.util.concurrent.atomic.AtomicLong 4 | 5 | import com.gtan.repox.config.Config 6 | import com.ning.http.client.Realm.{AuthScheme, RealmBuilder} 7 | import com.ning.http.client.{AsyncHttpClientConfig, AsyncHttpClient} 8 | import play.api.libs.json.Json 9 | 10 | import scala.concurrent.duration.Duration 11 | import com.ning.http.client.{Realm => JRealm} 12 | 13 | case class Realm(user: String, password: String, scheme: String) { 14 | def toJava: JRealm = new RealmBuilder().setPrincipal(user) 15 | .setPassword(password) 16 | .setUsePreemptiveAuth(true) 17 | .setScheme(AuthScheme.valueOf(scheme)) 18 | .build() 19 | } 20 | 21 | case class Connector(id: Option[Long], 22 | name: String, 23 | connectionTimeout: Duration, 24 | connectionIdleTimeout: Duration, 25 | maxConnections: Int, 26 | maxConnectionsPerHost: Int, 27 | credentials: Option[Realm] = None) { 28 | 29 | 30 | def createClient = { 31 | val configBuilder = new AsyncHttpClientConfig.Builder() 32 | .setRequestTimeout(Int.MaxValue) 33 | .setReadTimeout(connectionIdleTimeout.toMillis.toInt) 34 | .setConnectTimeout(connectionTimeout.toMillis.toInt) 35 | .setPooledConnectionIdleTimeout(connectionIdleTimeout.toMillis.toInt) 36 | .setAllowPoolingConnections(true) 37 | .setAllowPoolingSslConnections(true) 38 | .setMaxConnectionsPerHost(maxConnections) 39 | .setMaxConnections(maxConnectionsPerHost) 40 | .setFollowRedirect(true) 41 | val withCredentials = this.credentials.fold(configBuilder) { x => configBuilder.setRealm(x.toJava) } 42 | val builder = Config.proxyUsage.get(this).fold(withCredentials) { x => withCredentials.setProxyServer(x.toJava) } 43 | 44 | new AsyncHttpClient(builder.build()) 45 | } 46 | } 47 | 48 | object Connector { 49 | lazy val nextId: AtomicLong = new AtomicLong(Config.connectors.flatMap(_.id).reduceOption[Long](math.max).getOrElse(1L)) 50 | 51 | import DurationFormat._ 52 | 53 | implicit val realmFormat = Json.format[Realm] 54 | implicit val format = Json.format[Connector] 55 | } 56 | -------------------------------------------------------------------------------- /src/main/scala/com/gtan/repox/data/DurationFormat.scala: -------------------------------------------------------------------------------- 1 | package com.gtan.repox.data 2 | 3 | import play.api.libs.json._ 4 | 5 | import scala.concurrent.duration.Duration 6 | 7 | object DurationFormat { 8 | implicit val durationFormat = new Format[Duration] { 9 | override def reads(json: JsValue) = json match { 10 | case JsString(str) => 11 | JsSuccess(Duration(str)) 12 | case _ => 13 | JsError("duration json format need string") 14 | } 15 | 16 | override def writes(o: Duration) = JsString(o.toString) 17 | } 18 | 19 | } 20 | -------------------------------------------------------------------------------- /src/main/scala/com/gtan/repox/data/ExpireRule.scala: -------------------------------------------------------------------------------- 1 | package com.gtan.repox.data 2 | 3 | import java.util.concurrent.atomic.AtomicLong 4 | 5 | import com.gtan.repox.Repox 6 | import com.gtan.repox.config.Config 7 | import play.api.libs.json._ 8 | 9 | import scala.concurrent.duration.Duration 10 | 11 | import collection.JavaConverters._ 12 | 13 | case class ExpireRule(id: Option[Long], pattern: String, duration: Duration, disabled: Boolean = false) 14 | 15 | object ExpireRule { 16 | 17 | import DurationFormat._ 18 | 19 | lazy val nextId: AtomicLong = new AtomicLong(Config.expireRules.flatMap(_.id).reduceOption[Long](math.max).getOrElse(1)) 20 | 21 | implicit val format = Json.format[ExpireRule] 22 | } 23 | -------------------------------------------------------------------------------- /src/main/scala/com/gtan/repox/data/Immediate404Rule.scala: -------------------------------------------------------------------------------- 1 | package com.gtan.repox.data 2 | 3 | import java.util.concurrent.atomic.AtomicLong 4 | 5 | import com.gtan.repox.Repox 6 | import com.gtan.repox.config.Config 7 | import play.api.libs.json.Json 8 | 9 | import scala.collection.JavaConverters._ 10 | 11 | /** 12 | * Created by IntelliJ IDEA. 13 | * User: xf 14 | * Date: 14/11/23 15 | * Time: 下午12:15 16 | */ 17 | case class Immediate404Rule(id: Option[Long], include: String, exclude: Option[String] = None, disabled: Boolean = false) { 18 | def matches(uri: String): Boolean = { 19 | val included = uri.matches(include) 20 | exclude match { 21 | case None => included 22 | case Some(regex) => 23 | val excluded = uri.matches(regex) 24 | included && !excluded 25 | } 26 | } 27 | 28 | } 29 | 30 | object Immediate404Rule { 31 | lazy val nextId: AtomicLong = new AtomicLong(Config.immediate404Rules.flatMap(_.id).reduceOption[Long](math.max).getOrElse(1)) 32 | implicit val format = Json.format[Immediate404Rule] 33 | } 34 | 35 | case class BlacklistRule(pattern: String, repoName: String) 36 | -------------------------------------------------------------------------------- /src/main/scala/com/gtan/repox/data/ProxyServer.scala: -------------------------------------------------------------------------------- 1 | package com.gtan.repox.data 2 | 3 | import java.util.concurrent.atomic.AtomicLong 4 | 5 | import com.gtan.repox.Repox 6 | import com.gtan.repox.config.Config 7 | import com.ning.http.client.ProxyServer.Protocol 8 | import com.ning.http.client.{ProxyServer => JProxyServer} 9 | import play.api.libs.json._ 10 | 11 | import scala.collection.JavaConverters._ 12 | 13 | case class ProxyServer(id: Option[Long], name: String, protocol: JProxyServer.Protocol, host: String, port: Int, disabled: Boolean = false) { 14 | def toJava: JProxyServer = new JProxyServer(protocol, host, port) 15 | 16 | } 17 | 18 | object ProxyServer { 19 | lazy val nextId: AtomicLong = new AtomicLong(Config.proxies.flatMap(_.id).reduceOption[Long](math.max).getOrElse(1)) 20 | 21 | implicit val protocolFormat = new Format[JProxyServer.Protocol] { 22 | override def reads(json: JsValue):JsResult[JProxyServer.Protocol] = json match { 23 | case JsString(str) => 24 | JsSuccess(JProxyServer.Protocol.valueOf(str)) 25 | case _ => 26 | JsError("not a valid protocol") 27 | } 28 | 29 | override def writes(o: Protocol) = JsString(o.name()) 30 | } 31 | 32 | implicit val format = Json.format[ProxyServer] 33 | } 34 | -------------------------------------------------------------------------------- /src/main/scala/com/gtan/repox/data/Repo.scala: -------------------------------------------------------------------------------- 1 | package com.gtan.repox.data 2 | 3 | import java.net.URL 4 | import java.util.concurrent.atomic.AtomicLong 5 | 6 | import com.gtan.repox.config.Config 7 | import play.api.libs.json.Json 8 | 9 | /** 10 | * Created by IntelliJ IDEA. 11 | * User: xf 12 | * Date: 14/11/23 13 | * Time: 下午3:55 14 | */ 15 | case class Repo(id: Option[Long], name: String, base: String, priority: Int, getOnly: Boolean = false, maven: Boolean = false, disabled: Boolean = false, parentId: Option[Long] = None) { 16 | def absolute(uri: String): String = base + uri 17 | lazy val host = new URL(base).getHost 18 | } 19 | 20 | object Repo { 21 | lazy val nextId: AtomicLong = new AtomicLong(Config.repos.flatMap(_.id).reduceOption[Long](math.max).getOrElse(1)) 22 | 23 | implicit val format = Json.format[Repo] 24 | } 25 | 26 | -------------------------------------------------------------------------------- /src/test/scala/com/gtan/repox/GetMasterSuite.scala: -------------------------------------------------------------------------------- 1 | package com.gtan.repox 2 | 3 | import java.net.URLEncoder 4 | 5 | import akka.actor.ActorPath 6 | import org.scalatest._ 7 | 8 | class GetMasterSuite extends FunSuite{ 9 | val src = Seq("ascii", "中文Path", "带 空 格", ".其$它*特@殊!字#符~:+") 10 | /** 11 | * URLEncoder is used to generate part of GetWorker/HeadWorker names 12 | */ 13 | test("normalized actor paths are valid") { 14 | assert(src.forall{ path => 15 | ActorPath.isValidPathElement(URLEncoder.encode(path, "UTF-8")) 16 | }) 17 | } 18 | 19 | } 20 | -------------------------------------------------------------------------------- /src/test/scala/com/gtan/repox/RepoxSuite.scala: -------------------------------------------------------------------------------- 1 | package com.gtan.repox 2 | 3 | import org.scalatest._ 4 | 5 | class RepoxSuite extends FunSuite { 6 | val mavenData = Map( 7 | "/org/scala-lang/modules/scalajs/scalajs-sbt-plugin_2.10_0.13/0.5.6/scalajs-sbt-plugin-0.5.6.pom" -> "/org.scala-lang.modules.scalajs/scalajs-sbt-plugin/scala_2.10/sbt_0.13/0.5.6/ivys/ivy.xml", 8 | "/io/spray/sbt-revolver_2.10_0.13/0.7.2/sbt-revolver-0.7.2.pom" -> "/io.spray/sbt-revolver/scala_2.10/sbt_0.13/0.7.2/ivys/ivy.xml", 9 | "/org/scala-lang/modules/scalajs/scalajs-tools_2.10/0.5.6/scalajs-tools_2.10-0.5.6.pom" -> "/org.scala-lang.modules.scalajs/scalajs-tools_2.10/0.5.6/ivys/ivy.xml", 10 | "/com/eed3si9n/sbt-assembly_2.10_0.13/0.12.0/sbt-assembly-0.12.0.pom" -> "/com.eed3si9n/sbt-assembly/scala_2.10/sbt_0.13/0.12.0/ivys/ivy.xml", 11 | "/org/scala-sbt/sbt/0.13.7/sbt-0.13.7.pom" -> "/org.scala-sbt/sbt/0.13.7/ivys/ivy.xml", 12 | "/org/scala-sbt/main/0.13.7/main-0.13.7.pom" -> "/org.scala-sbt/main/0.13.7/ivys/ivy.xml", 13 | "/org/scala-sbt/actions/0.13.7/actions-0.13.7.pom" -> "/org.scala-sbt/actions/0.13.7/ivys/ivy.xml" 14 | , "/net/virtual-void/sbt-dependency-graph/0.7.4/sbt-dependency-graph-0.7.4.jar" -> "/net/virtual-void/sbt-dependency-graph_2.10_0.13/0.7.4/sbt-dependency-graph-0.7.4.jar" 15 | , "/com/github/mpeltonen/sbt-idea/1.6.0/sbt-idea-1.6.0.jar" -> "/com/github/mpeltonen/sbt-idea_2.10_0.13/1.6.0/sbt-idea-1.6.0.jar" 16 | ) 17 | 18 | val ivyData = Map( 19 | "/net.virtual-void/sbt-dependency-graph/0.7.4/jars/sbt-dependency-graph.jar" -> "/net/virtual-void/sbt-dependency-graph_2.10_0.13/0.7.4/sbt-dependency-graph-0.7.4.jar" 20 | , "/com.github.mpeltonen/sbt-idea/1.6.0/jars/sbt-idea.jar" -> "/com/github/mpeltonen/sbt-idea_2.10_0.13/1.6.0/sbt-idea-1.6.0.jar" 21 | ) 22 | 23 | val invalidData = Seq( 24 | "/joda-time/joda-time/", 25 | "/joda-time/joda-time" 26 | ) 27 | 28 | test("MavenFormat should match keys of mavenData") { 29 | assert(mavenData.keys.forall(_.matches(Repox.MavenFormat.regex))) 30 | } 31 | 32 | test("IvyFormat should match keys of ivyData") { 33 | assert(ivyData.keys.forall(_.matches(Repox.IvyFormat.regex))) 34 | } 35 | 36 | test("keys in all data map should have the value as peer") { 37 | assert((mavenData ++ ivyData).forall { case (k, v) => 38 | val peers = Repox.peer(k) 39 | peers.isSuccess && peers.get.contains(v) 40 | }) 41 | } 42 | 43 | test("requests in invalidData are invalid request") { 44 | assert(invalidData.forall { uri => 45 | Repox.peer(uri).isFailure 46 | }) 47 | } 48 | } 49 | --------------------------------------------------------------------------------