├── .gitignore ├── Cargo.toml ├── LICENSE ├── README.md └── src ├── config.rs ├── errors.rs ├── main.rs ├── modules ├── authentication │ ├── mod.rs │ └── openid.rs ├── mod.rs └── storage │ ├── artifactory.rs │ ├── file.rs │ └── mod.rs ├── parser.rs └── registry.rs /.gitignore: -------------------------------------------------------------------------------- 1 | # Generated by Cargo 2 | # will have compiled files and executables 3 | /target/ 4 | 5 | # Remove Cargo.lock from gitignore if creating an executable, leave it for libraries 6 | # More information here http://doc.crates.io/guide.html#cargotoml-vs-cargolock 7 | Cargo.lock 8 | 9 | # These are backup files generated by rustfmt 10 | **/*.rs.bk 11 | 12 | registry.toml 13 | -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | authors = ["Chris Swindle "] 3 | name = "caesium" 4 | version = "0.1.0" 5 | 6 | [dependencies] 7 | error-chain = "0.11.0" 8 | futures = "0.1.17" 9 | git2 = "0.6.8" 10 | hyper = "0.11.7" 11 | hyper-tls = "0.1.2" 12 | nom = "3.2.1" 13 | router = "0.5.1" 14 | rust-crypto = "0.2.36" 15 | serde = "1.0.21" 16 | serde_derive = "1.0.21" 17 | serde_json = "1.0.6" 18 | tokio-core = "0.1.10" 19 | toml = "0.4.5" 20 | url = "1.6.0" 21 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2017 cswindle 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Caesium 2 | Caesium is an alternative crate registry for Rust, allowing crates to be 3 | publshed directly from Cargo. Below are the key features of Caesium: 4 | 5 | - it is designed to fit into different enterprise systems 6 | - it has a number of areas that modules can implement support for different systems, these are: 7 | - storage, where the crates can be stored. This currently includes support for the following: 8 | - local file system 9 | - upload to Artifactory 10 | 11 | The key areas which it does not have compared to crates.io are: 12 | 13 | - Web UI for querying crates 14 | - Caesium only supports the publish API 15 | 16 | ## How to use it 17 | There are two parts that are required for Caesium, these are: 18 | 19 | - git repository for storing information which Cargo uses 20 | - the actual Caesium server (this application) 21 | 22 | The following sections will go through how to set each of these up. 23 | 24 | ### Setting up git index 25 | You will need a new Git repository which needs to contain a config.json file at 26 | the base of the repository. This needs to contain the following information 27 | (note this is subject to change as the interface is still evolving): 28 | 29 | ``` 30 | { 31 | "dl": "file:///path/to/my/crates/store", 32 | "api": "http://127.0.0.1:3000" 33 | } 34 | 35 | ``` 36 | 37 | The `dl` field is the URL that uploaded crates can be downloaded from, note that 38 | using a file based URL is only sensible if all machines that need to download 39 | crates have access to the same location. 40 | 41 | The `api` field provides details of the URL to access the Caesium server. 42 | 43 | ### Setting up Caesium configuration 44 | Caesium loads registry.toml from the current directory for the configuration, an 45 | example of the configuration is shown below: 46 | 47 | ``` 48 | [registry] 49 | index = "ssh://git@git.server/index.git" 50 | 51 | [storage.file] 52 | location = "/path/to/my/crates/store" 53 | ``` 54 | 55 | The `index` field is the URL for the Git index that was setup in the previous 56 | step. 57 | 58 | ## Configuration guide 59 | Below are the key areas of config, the items in bold are mandatory: 60 | 61 | - **[registry](#registry-config---mandatory)** 62 | - **[storage](#registry-config---mandatory)** 63 | - [server](#server-config) 64 | 65 | ### Registry Config - MANDATORY 66 | This just has a single entry for the index, which is mandatory, below is an example: 67 | 68 | ``` 69 | [registry] 70 | index = "ssh://git@git.server/index.git" 71 | ``` 72 | 73 | ### Storage Config - MANDATORY 74 | The storage config contains the following options (one of which must be set): 75 | 76 | - [storage.file] 77 | - [storage.artifactory] 78 | 79 | #### File based storage 80 | There is only one key for file based storage, that is the `location` of where 81 | to store the crates. Below is an example: 82 | 83 | ``` 84 | [storage.file] 85 | location = "/crates/storage/path" 86 | ``` 87 | 88 | #### Artifactory based storage 89 | Artifactory includes the following configuration: 90 | 91 | - base_url 92 | - api_key 93 | 94 | Below is an example: 95 | 96 | ``` 97 | [storage.artifactory] 98 | base_url = "https://artifactory.server/caesium" 99 | api_key = "ABSSJKDNAKSNCNUuansiasncsMKA..." 100 | ``` 101 | 102 | 103 | ### Server config 104 | The server config just has one optional field, this allows setting the port that 105 | Caesium sets the server up on (by default this is 3000). Below is an example: 106 | 107 | ``` 108 | [server] 109 | port = 3000 110 | ``` 111 | -------------------------------------------------------------------------------- /src/config.rs: -------------------------------------------------------------------------------- 1 | 2 | use ::modules; 3 | 4 | use std; 5 | use std::io::Read; 6 | 7 | use toml; 8 | 9 | #[derive(Debug, Deserialize)] 10 | pub struct CaesiumConfig { 11 | pub registry: CeasiumRegistryConfig, 12 | pub storage: CaesiumStorageConfig, 13 | pub authentication: Option, 14 | pub server: Option, 15 | } 16 | 17 | #[derive(Debug, Deserialize)] 18 | pub struct CeasiumRegistryConfig { 19 | pub index: String, 20 | } 21 | 22 | #[derive(Debug, Deserialize)] 23 | pub struct CaesiumStorageConfig { 24 | pub file: Option, 25 | pub artifactory: Option, 26 | } 27 | 28 | #[derive(Debug, Deserialize)] 29 | pub struct CaesiumFileStorageConfig { 30 | pub location: String, 31 | } 32 | 33 | #[derive(Debug, Deserialize)] 34 | pub struct CaesiumArtifactoryStorageConfig { 35 | pub base_url: String, 36 | pub api_key: String, 37 | } 38 | 39 | #[derive(Debug, Deserialize)] 40 | pub struct CaesiumAuthenticationConfig { 41 | pub openid: Option, 42 | pub oauth2: Option, 43 | } 44 | 45 | #[derive(Debug, Deserialize)] 46 | pub struct CaesiumOpenIdConfig { 47 | pub openid_configuration_url: String, 48 | } 49 | 50 | #[derive(Debug, Deserialize)] 51 | pub struct CaesiumOAuth2Config { 52 | pub client_id: String, 53 | pub client_secret: String, 54 | pub authorization_url: String, 55 | pub token_url: String, 56 | pub scope: Vec, 57 | } 58 | 59 | #[derive(Debug, Deserialize)] 60 | pub struct CaesiumServerConfig { 61 | pub port: Option, 62 | } 63 | 64 | impl CaesiumConfig { 65 | pub fn new(config_file: &str) -> CaesiumConfig { 66 | let mut toml = String::new(); 67 | std::fs::File::open(config_file).and_then(|mut f| f.read_to_string(&mut toml)).expect("Failed to read file"); 68 | 69 | toml::from_str(&toml).unwrap() 70 | } 71 | 72 | pub fn create_storage_module(&self) -> Box { 73 | if let Some(ref file) = self.storage.file { 74 | Box::new(modules::storage::file::FileCrateStorage::new(&file.location)) 75 | } else if let Some(ref artifactory) = self.storage.artifactory { 76 | Box::new(modules::storage::artifactory::ArtifactoryCrateStorage::new(&artifactory.base_url, &artifactory.api_key)) 77 | } else { 78 | panic!("No storage config present"); 79 | } 80 | } 81 | 82 | pub fn create_authentication_module(&self) -> Option> { 83 | if let Some(ref auth) = self.authentication { 84 | if let Some(ref openid) = auth.openid { 85 | Some(Box::new(modules::authentication::openid::OpenIdAuthentication::new(&openid.openid_configuration_url))) 86 | // } else if let Some(ref oauth) = auth.oauth2 { 87 | // Some(Box::new(modules::authentication::oauth2::OAuth2Authentication::new())) 88 | } else { 89 | None 90 | } 91 | } else { 92 | None 93 | } 94 | } 95 | } 96 | -------------------------------------------------------------------------------- /src/errors.rs: -------------------------------------------------------------------------------- 1 | 2 | error_chain! { 3 | // Bindings to types implementing std::error::Error. 4 | foreign_links { 5 | Io(::std::io::Error); 6 | UrlParse(::url::ParseError); 7 | Git(::git2::Error); 8 | Serde(::serde_json::Error); 9 | Hyper(::hyper::Error); 10 | UriError(::hyper::error::UriError); 11 | } 12 | 13 | errors { 14 | AuthenticationError(v: String) { 15 | description("Failed to authenticate"), 16 | display("Failed to authenticate: '{}'", v), 17 | } 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /src/main.rs: -------------------------------------------------------------------------------- 1 | #[macro_use] 2 | extern crate error_chain; 3 | #[macro_use] 4 | extern crate nom; 5 | // extern crate router; 6 | extern crate url; 7 | #[macro_use] 8 | extern crate hyper; 9 | extern crate hyper_tls; 10 | extern crate futures; 11 | #[macro_use] 12 | extern crate serde_derive; 13 | extern crate serde; 14 | extern crate serde_json; 15 | extern crate crypto; 16 | extern crate git2; 17 | extern crate toml; 18 | extern crate tokio_core; 19 | 20 | mod config; 21 | mod errors; 22 | mod parser; 23 | mod modules; 24 | mod registry; 25 | 26 | use errors::*; 27 | 28 | use hyper::{Put, StatusCode}; 29 | use hyper::server::{Http, Service, Request, Response}; 30 | use hyper::header::Authorization; 31 | 32 | use futures::Stream; 33 | use futures::Future; 34 | 35 | use std::sync::Arc; 36 | 37 | 38 | struct Caesium { 39 | registry: registry::Registry, 40 | 41 | // Config 42 | config: config::CaesiumConfig, 43 | 44 | authentication: Option>, 45 | 46 | // authorization: Option, 47 | 48 | storage: Box, 49 | } 50 | 51 | impl Caesium { 52 | pub fn new() -> Caesium { 53 | 54 | let config = config::CaesiumConfig::new("registry.toml"); 55 | let storage = config.create_storage_module(); 56 | let authentication = config.create_authentication_module(); 57 | 58 | Caesium { 59 | registry: registry::Registry::new(&config.registry.index.clone()), 60 | config: config, 61 | storage: storage, 62 | authentication: authentication, 63 | } 64 | } 65 | 66 | fn publish(&self, manifest: &str, crate_tar: &[u8], token: &str) -> Result<()> { 67 | 68 | let manifest: registry::CargoManifest = serde_json::from_str(&manifest).unwrap(); 69 | 70 | // Authenticate 71 | let userinfo = if let Some(ref authentication) = self.authentication { 72 | Some(authentication.authenticate(token)?) 73 | } else { 74 | None 75 | }; 76 | 77 | // Authorize 78 | 79 | // Now call into the storage driver to store the crate 80 | self.storage.upload(&manifest, crate_tar)?; 81 | 82 | // Now that everything is stored, we need to update the index file so 83 | // that the crate is available. 84 | self.registry.add_crate(&manifest, crate_tar)?; 85 | 86 | let username = userinfo.and_then(|userinfo| userinfo.name) 87 | .unwrap_or("an anonymous user".to_string()); 88 | 89 | println!("Crate {} v{} was uploaded by {}", manifest.name, manifest.vers, username); 90 | 91 | Ok(()) 92 | } 93 | } 94 | 95 | struct CaesiumService { 96 | caesium: Arc 97 | } 98 | 99 | impl CaesiumService { 100 | pub fn new(caesium: Arc) -> CaesiumService { 101 | CaesiumService { 102 | caesium: caesium, 103 | } 104 | } 105 | } 106 | 107 | impl Service for CaesiumService { 108 | type Request = Request; 109 | type Response = Response; 110 | type Error = hyper::Error; 111 | type Future = Box>; 112 | 113 | fn call(&self, req: Request) -> Self::Future { 114 | match (req.method(), req.path()) { 115 | (&Put, "/api/v1/crates/new") => { 116 | 117 | println!("Handling new upload request"); 118 | 119 | let mut caesium = self.caesium.clone(); 120 | 121 | let token = match req.headers().get::>() { 122 | Some(auth_header) => auth_header.0.clone(), 123 | None => panic!("No authorization header found") 124 | }; 125 | 126 | Box::new(req.body() 127 | .fold(Vec::new(), |mut acc, chunk| { 128 | acc.extend_from_slice(&*chunk); 129 | futures::future::ok::<_, Self::Error>(acc) 130 | }) 131 | .map(move |body| { 132 | let (manifest, tar) = parser::parse_crate_upload(body.as_slice()).unwrap(); 133 | 134 | match caesium.publish(manifest, tar, &token) { 135 | Ok(_) => Response::new().with_status(StatusCode::Ok), 136 | Err(e) => { 137 | match *e.kind() { 138 | ErrorKind::AuthenticationError(_) => Response::new().with_status(StatusCode::Forbidden), 139 | _ => Response::new().with_status(StatusCode::InternalServerError), 140 | } 141 | } 142 | } 143 | })) 144 | }, 145 | _ => { 146 | Box::new(futures::future::ok(Response::new().with_status(StatusCode::NotFound))) 147 | } 148 | } 149 | } 150 | } 151 | 152 | fn main() { 153 | let caesium = Arc::new(Caesium::new()); 154 | 155 | let port = match caesium.config.server { 156 | Some(ref server) => server.port, 157 | None => None 158 | }; 159 | 160 | let port = if let Some(port) = port { 161 | port 162 | } else { 163 | 3000 164 | }; 165 | 166 | let addr = format!("0.0.0.0:{}", port).parse().unwrap(); 167 | let mut server = Http::new().bind(&addr, move || Ok(CaesiumService::new(caesium.clone()))).unwrap(); 168 | server.no_proto(); 169 | println!("Listening on http://{} with 1 thread.", server.local_addr().unwrap()); 170 | server.run().unwrap(); 171 | } 172 | -------------------------------------------------------------------------------- /src/modules/authentication/mod.rs: -------------------------------------------------------------------------------- 1 | 2 | use errors::*; 3 | 4 | #[derive(Debug, Deserialize)] 5 | pub struct AuthenticationUserInfo { 6 | pub sub: String, 7 | pub name: Option, 8 | } 9 | 10 | pub trait Authentication { 11 | // Authenticate using the token provided by cargo publish. 12 | fn authenticate(&self, token: &str) -> Result; 13 | } 14 | 15 | pub mod openid; 16 | -------------------------------------------------------------------------------- /src/modules/authentication/openid.rs: -------------------------------------------------------------------------------- 1 | 2 | use errors::*; 3 | use modules::*; 4 | 5 | use tokio_core; 6 | use hyper; 7 | use hyper::Client; 8 | use hyper_tls::HttpsConnector; 9 | use hyper::header::{Authorization, Bearer}; 10 | use futures::Future; 11 | use serde_json; 12 | use serde_json::Value; 13 | 14 | use futures::Stream; 15 | use std::str::FromStr; 16 | use std::io; 17 | 18 | #[derive(Debug)] 19 | pub struct OpenIdAuthentication { 20 | pub openid_configuration: hyper::Uri, 21 | 22 | // Information retrieved from openid configuration 23 | pub authorization_endpoint: hyper::Uri, 24 | pub token_endpoint: hyper::Uri, 25 | pub userinfo_endpoint: hyper::Uri, 26 | } 27 | 28 | impl OpenIdAuthentication { 29 | pub fn new(openid_configuration: &str) -> OpenIdAuthentication { 30 | 31 | // Get the endpoints that should be used with OAuth2 32 | let mut core = tokio_core::reactor::Core::new().unwrap(); 33 | let handle = core.handle(); 34 | 35 | let openid_config_url = hyper::Uri::from_str(openid_configuration).expect("Invalid OpenID URL"); 36 | 37 | let client = Client::configure() 38 | .connector(HttpsConnector::new(4, &handle).expect("Failed to setup HTTPS")) 39 | .build(&handle); 40 | let work = client.get(openid_config_url.clone()) 41 | .and_then(|res| { 42 | res.body().concat2().and_then(move |body| { 43 | let json: Value = serde_json::from_slice(&body).map_err(|e| { 44 | io::Error::new( 45 | io::ErrorKind::Other, 46 | e 47 | ) 48 | })?; 49 | Ok(json) 50 | }) 51 | }); 52 | let openid_config = core.run(work).expect("Failed to get openid configuration"); 53 | 54 | let auth_endpoint = hyper::Uri::from_str(openid_config["authorization_endpoint"].as_str().unwrap()).expect("Invalid authorization_endpoint received"); 55 | let token_endpoint = hyper::Uri::from_str(openid_config["token_endpoint"].as_str().unwrap()).expect("Invalid token_endpoint received"); 56 | let userinfo_endpoint = hyper::Uri::from_str(openid_config["userinfo_endpoint"].as_str().unwrap()).expect("Invalid userinfo_endpoint received"); 57 | 58 | OpenIdAuthentication { 59 | openid_configuration: openid_config_url, 60 | authorization_endpoint: auth_endpoint, 61 | token_endpoint: token_endpoint, 62 | userinfo_endpoint: userinfo_endpoint, 63 | } 64 | } 65 | } 66 | 67 | impl authentication::Authentication for OpenIdAuthentication { 68 | // Authenticate using the token provided by cargo publish. 69 | fn authenticate(&self, token: &str) -> Result { 70 | 71 | // We need to try and get the userinfo_data here, if it fails then we 72 | // are not authenticated. 73 | let mut core = tokio_core::reactor::Core::new().unwrap(); 74 | let handle = core.handle(); 75 | 76 | let client = Client::configure() 77 | .connector(HttpsConnector::new(4, &handle).unwrap()) 78 | .build(&handle); 79 | 80 | let mut request = hyper::Request::new(hyper::Method::Get, self.userinfo_endpoint.clone()); 81 | request.headers_mut().set(Authorization( 82 | Bearer { 83 | token: token.to_string(), 84 | } 85 | )); 86 | 87 | let work = client.request(request) 88 | .and_then(|res| { 89 | if res.status() != hyper::StatusCode::Ok { 90 | bail!(io::Error::new( 91 | io::ErrorKind::Other, 92 | format!("Invalid status code: {}", res.status()), 93 | )); 94 | } 95 | Ok(res) 96 | }).and_then(|res| { 97 | res.body().concat2().and_then(move |body| { 98 | let json: authentication::AuthenticationUserInfo = serde_json::from_slice(&body).map_err(|e| { 99 | io::Error::new( 100 | io::ErrorKind::Other, 101 | e 102 | ) 103 | })?; 104 | Ok(json) 105 | }) 106 | }); 107 | let user_info = core.run(work).map_err(|_| ErrorKind::AuthenticationError("Failed to authenticate".to_string()))?; 108 | 109 | Ok(user_info) 110 | } 111 | } 112 | -------------------------------------------------------------------------------- /src/modules/mod.rs: -------------------------------------------------------------------------------- 1 | 2 | pub mod authentication; 3 | pub mod storage; 4 | -------------------------------------------------------------------------------- /src/modules/storage/artifactory.rs: -------------------------------------------------------------------------------- 1 | 2 | use errors::*; 3 | use modules::*; 4 | 5 | use std::str::FromStr; 6 | 7 | use tokio_core; 8 | use hyper; 9 | use hyper::Client; 10 | use futures::Future; 11 | use url; 12 | 13 | header! { (XJFrogArtApi, "X-JFrog-Art-Api") => [String] } 14 | 15 | pub struct ArtifactoryCrateStorage { 16 | base_url: url::Url, 17 | api_key: String, 18 | } 19 | 20 | impl ArtifactoryCrateStorage { 21 | pub fn new(url: &String, api_key: &String) -> ArtifactoryCrateStorage { 22 | 23 | // May be worth ensuring that we can authenticate using the provided 24 | // credentials. 25 | 26 | ArtifactoryCrateStorage { 27 | base_url: url::Url::parse(url).expect("Invalid Artifactory URL in config"), 28 | api_key: api_key.clone(), 29 | } 30 | } 31 | } 32 | 33 | /// Artifactory based storage 34 | impl storage::CrateStorage for ArtifactoryCrateStorage { 35 | fn upload(&self, manifest: &::registry::CargoManifest, tar: &[u8]) -> Result<()> { 36 | 37 | let mut core = tokio_core::reactor::Core::new().unwrap(); 38 | let handle = core.handle(); 39 | 40 | let client = Client::new(&handle); 41 | let body = tar.to_vec(); 42 | 43 | let mut url = self.base_url.clone(); 44 | url.path_segments_mut().unwrap() 45 | .push(&manifest.name) 46 | .push(&manifest.vers) 47 | .push("download"); 48 | let hyper_uri = hyper::Uri::from_str(url.as_str())?; 49 | 50 | let mut request = hyper::Request::new(hyper::Method::Put, hyper_uri); 51 | request.set_body(body); 52 | request.headers_mut().set(XJFrogArtApi(self.api_key.clone())); 53 | 54 | let work = client.request(request).and_then(|res| { 55 | if res.status() != hyper::StatusCode::Created { 56 | panic!("Received invalid error code: {}", res.status()); 57 | } 58 | 59 | Ok(()) 60 | }); 61 | 62 | core.run(work)?; 63 | 64 | Ok(()) 65 | } 66 | } 67 | -------------------------------------------------------------------------------- /src/modules/storage/file.rs: -------------------------------------------------------------------------------- 1 | 2 | use errors::*; 3 | use modules::*; 4 | 5 | use std; 6 | use std::fs::File; 7 | use std::io::Write; 8 | use std::path::PathBuf; 9 | 10 | pub struct FileCrateStorage { 11 | pub location: PathBuf, 12 | } 13 | 14 | impl FileCrateStorage { 15 | pub fn new(location: &String) -> FileCrateStorage { 16 | FileCrateStorage { 17 | location: PathBuf::from(location), 18 | } 19 | } 20 | } 21 | 22 | /// File based storage 23 | impl storage::CrateStorage for FileCrateStorage { 24 | fn upload(&self, manifest: &::registry::CargoManifest, tar: &[u8]) -> Result<()> { 25 | let upload_file = self.location.join(manifest.name.clone()).join(manifest.vers.clone()).join("download"); 26 | 27 | std::fs::create_dir_all(upload_file.parent().unwrap()).expect("Failed to create dir"); 28 | 29 | let mut uploaded_crate = File::create(upload_file.clone()).unwrap(); 30 | uploaded_crate.write_all(tar).unwrap(); 31 | 32 | Ok(()) 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /src/modules/storage/mod.rs: -------------------------------------------------------------------------------- 1 | 2 | use errors::*; 3 | 4 | pub trait CrateStorage { 5 | // Uploads the crate of the tar file and returns the URL that it is 6 | // available at. 7 | fn upload(&self, manifest: &::registry::CargoManifest, tar: &[u8]) -> Result<()>; 8 | } 9 | 10 | pub mod file; 11 | pub mod artifactory; 12 | -------------------------------------------------------------------------------- /src/parser.rs: -------------------------------------------------------------------------------- 1 | use std; 2 | use nom; 3 | use nom::{le_u32}; 4 | 5 | use errors::*; 6 | 7 | named!(crate_parser<&[u8], (&str, &[u8])>, 8 | do_parse!( 9 | manifest_len: le_u32 >> 10 | manifest: map_res!( 11 | take!(manifest_len), 12 | std::str::from_utf8 13 | ) >> 14 | tar_len: le_u32 >> 15 | tar: take!(tar_len) >> 16 | 17 | (manifest, tar) 18 | ) 19 | ); 20 | 21 | pub fn parse_crate_upload(upload: &[u8]) -> Result<(&str, &[u8])> { 22 | match crate_parser(upload) { 23 | nom::IResult::Done(_,(manifest, tar)) => Ok((manifest, tar)), 24 | _ => bail!("Failed to parse binary"), 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /src/registry.rs: -------------------------------------------------------------------------------- 1 | use std; 2 | use std::collections::HashMap; 3 | use std::path::{Path, PathBuf}; 4 | use std::io::{Read, Write}; 5 | 6 | use crypto::digest::Digest; 7 | use crypto::sha2::Sha256; 8 | use serde_json; 9 | use git2; 10 | use git2::build::RepoBuilder; 11 | use git2::{FetchOptions, Repository}; 12 | 13 | use errors::*; 14 | 15 | #[derive(Debug, Deserialize)] 16 | pub struct CargoManifest { 17 | pub name: String, 18 | pub vers: String, 19 | pub deps: Vec, 20 | pub features: HashMap>, 21 | pub authors: Vec, 22 | pub description: Option, 23 | pub documentation: Option, 24 | pub homepage: Option, 25 | pub readme: Option, 26 | pub keywords: Vec, 27 | pub categories: Vec, 28 | pub license: Option, 29 | pub license_file: Option, 30 | pub repository: Option, 31 | pub badges: HashMap>, 32 | } 33 | 34 | #[derive(Debug, Deserialize)] 35 | pub struct CargoManifestDependency { 36 | pub optional: bool, 37 | pub default_features: bool, 38 | pub name: String, 39 | pub features: Vec, 40 | pub version_req: String, 41 | pub target: Option, 42 | pub kind: String, 43 | pub registry: Option, 44 | } 45 | 46 | #[derive(Debug, Serialize)] 47 | pub struct RegistryIndexEntry { 48 | pub name: String, 49 | pub vers: String, 50 | pub deps: Vec, 51 | pub cksum: String, 52 | pub features: HashMap>, 53 | pub yanked: bool, 54 | } 55 | 56 | #[derive(Debug, Serialize)] 57 | pub struct RegistryIndexDependency { 58 | pub name: String, 59 | pub vers: String, 60 | pub registry: Option, 61 | pub features: Vec, 62 | pub optional: bool, 63 | pub default_features: bool, 64 | pub target: Option, 65 | pub kind: String, 66 | } 67 | 68 | impl RegistryIndexEntry { 69 | pub fn new(manifest: &CargoManifest, cksum: String) -> RegistryIndexEntry { 70 | RegistryIndexEntry { 71 | name: manifest.name.clone(), 72 | vers: manifest.vers.clone(), 73 | deps: manifest.deps.iter().map(|dep| RegistryIndexDependency::new(dep)).collect(), 74 | cksum: cksum, 75 | features: manifest.features.clone(), 76 | yanked: false, 77 | } 78 | } 79 | } 80 | 81 | impl RegistryIndexDependency { 82 | pub fn new(dep: &CargoManifestDependency) -> RegistryIndexDependency { 83 | RegistryIndexDependency { 84 | name: dep.name.clone(), 85 | vers: dep.version_req.clone(), 86 | registry: dep.registry.clone(), 87 | features: dep.features.clone(), 88 | optional: dep.optional, 89 | default_features: dep.default_features, 90 | target: dep.target.clone(), 91 | kind: dep.kind.clone(), 92 | } 93 | } 94 | } 95 | 96 | impl From for RegistryIndexDependency { 97 | fn from(dep: CargoManifestDependency) -> Self { 98 | RegistryIndexDependency { 99 | name: dep.name.clone(), 100 | vers: dep.version_req.clone(), 101 | registry: dep.registry.clone(), 102 | features: dep.features.clone(), 103 | optional: dep.optional, 104 | default_features: dep.default_features, 105 | target: dep.target.clone(), 106 | kind: dep.kind.clone(), 107 | } 108 | } 109 | } 110 | 111 | pub struct Registry { 112 | index_repo: Repository, 113 | } 114 | 115 | impl Registry { 116 | pub fn new(registry_index: &str) -> Registry { 117 | let mut cb = git2::RemoteCallbacks::new(); 118 | cb.credentials(|_url, username, _allowed| { 119 | git2::Cred::ssh_key_from_agent(username.unwrap()) 120 | }); 121 | 122 | let mut fo = FetchOptions::new(); 123 | fo.remote_callbacks(cb); 124 | 125 | let registry_path = Path::new("./repo"); 126 | 127 | // Try and remove the repo directory before we clone 128 | let _ = std::fs::remove_dir_all(registry_path); 129 | 130 | let repo = match RepoBuilder::new().fetch_options(fo) 131 | .clone(registry_index, registry_path) { 132 | Ok(repo) => repo, 133 | Err(e) => panic!("failed to clone: {}", e), 134 | }; 135 | 136 | Registry { 137 | index_repo: repo, 138 | } 139 | } 140 | 141 | fn index_file(&self, name: &str) -> PathBuf { 142 | let base = self.index_repo.workdir().unwrap(); 143 | 144 | let name = name.chars() 145 | .flat_map(|c| c.to_lowercase()) 146 | .collect::(); 147 | match name.len() { 148 | 1 => base.join("1").join(&name), 149 | 2 => base.join("2").join(&name), 150 | 3 => base.join("3").join(&name[..1]).join(&name), 151 | _ => base.join(&name[0..2]).join(&name[2..4]).join(&name), 152 | } 153 | } 154 | 155 | fn update_crate_index(&self, dst: &PathBuf, entry: &RegistryIndexEntry) -> Result<()> { 156 | std::fs::create_dir_all(dst.parent().unwrap())?; 157 | let mut prev = String::new(); 158 | if std::fs::metadata(&dst).is_ok() { 159 | std::fs::File::open(&dst).and_then(|mut f| f.read_to_string(&mut prev))?; 160 | } 161 | let s = serde_json::to_string(&entry)?; 162 | let new = prev + &s; 163 | let mut f = std::fs::File::create(&dst)?; 164 | f.write_all(new.as_bytes())?; 165 | f.write_all(b"\n")?; 166 | 167 | Ok(()) 168 | } 169 | 170 | fn commit(&self, index_file: &PathBuf, message: String) -> Result<()> { 171 | let mut index = self.index_repo.index()?; 172 | let mut repo_path = self.index_repo.workdir().unwrap().iter(); 173 | let dst = index_file.iter() 174 | .skip_while(|s| Some(*s) == repo_path.next()) 175 | .collect::(); 176 | index.add_path(&dst)?; 177 | index.write().unwrap(); 178 | let tree_id = index.write_tree()?; 179 | let tree = self.index_repo.find_tree(tree_id)?; 180 | 181 | let head = self.index_repo.head()?; 182 | let parent = self.index_repo.find_commit(head.target().unwrap())?; 183 | let signature = self.index_repo.signature()?; 184 | 185 | self.index_repo.commit(Some("HEAD"), // point HEAD to our new commit 186 | &signature, // author 187 | &signature, // committer 188 | &message, // commit message 189 | &tree, // tree 190 | &[&parent])?; // parents 191 | 192 | Ok(()) 193 | } 194 | 195 | fn push(&self) -> Result<()> { 196 | let mut ref_status = None; 197 | let mut origin = self.index_repo.find_remote("origin")?; 198 | let res = { 199 | let mut callbacks = git2::RemoteCallbacks::new(); 200 | callbacks.credentials(|_url, username, _allowed| { 201 | git2::Cred::ssh_key_from_agent(username.unwrap()) 202 | }); 203 | callbacks.push_update_reference(|refname, status| { 204 | assert_eq!(refname, "refs/heads/master"); 205 | ref_status = status.map(|s| s.to_string()); 206 | Ok(()) 207 | }); 208 | let mut opts = git2::PushOptions::new(); 209 | opts.remote_callbacks(callbacks); 210 | origin.push(&["refs/heads/master"], Some(&mut opts)) 211 | }; 212 | match res { 213 | Ok(()) if ref_status.is_none() => Ok(()), 214 | Ok(()) => bail!("failed to push a ref: {:?}", ref_status), 215 | Err(e) => bail!("failure to push: {}", e), 216 | } 217 | } 218 | 219 | pub fn add_crate(&self, manifest: &CargoManifest, crate_tar: &[u8]) -> Result<()> { 220 | 221 | let mut sha = Sha256::new(); 222 | sha.input(crate_tar); 223 | 224 | // Convert the manifest into the registry index 225 | let entry = RegistryIndexEntry::new(manifest, sha.result_str()); 226 | 227 | let index_file = self.index_file(&entry.name); 228 | 229 | self.update_crate_index(&index_file, &entry)?; 230 | 231 | self.commit(&index_file, format!("Adding {} {}", manifest.name, manifest.vers))?; 232 | 233 | self.push()?; 234 | 235 | Ok(()) 236 | } 237 | } 238 | --------------------------------------------------------------------------------