├── .gitignore ├── CHANGELOG.txt ├── LICENSE ├── README.md ├── cache.php ├── caches └── .gitkeep ├── config.php.sample └── mirror.php /.gitignore: -------------------------------------------------------------------------------- 1 | p/* 2 | packagist/* 3 | composer.phar 4 | packages.json 5 | config.php 6 | vendor/ 7 | caches/ -------------------------------------------------------------------------------- /CHANGELOG.txt: -------------------------------------------------------------------------------- 1 | Changelog: 2 | 3 | - [2014-08-05] Fix some warnings 4 | - Add option to include source and dist array from mirroring 5 | - Add support for local cache 6 | - Add support for new hashed file names -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License 2 | 3 | Copyright (c) 2010 thomas.rabaix@ekino.com 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in 13 | all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | THE SOFTWARE. -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ⚠️ Deprecated Project ⚠️ 2 | ===================== 3 | 4 | Please note, the current implementation is not suitable anymore and does not work anymore with composer if you are using an ``ext3`` partition. Also, the mirroring solution requires a dedicated git host to server repository which is not an easy and simple task on the long run. 5 | 6 | You should now use: [https://github.com/rande/pkgmirror](https://github.com/rande/pkgmirror) 7 | 8 | 9 | Packagist and Github mirroring 10 | ============================== 11 | 12 | Experiencing Github issues ? You want a local cache to speed up tests and deployment ? 13 | 14 | 15 | Mirroring Packagist 16 | ------------------- 17 | 18 | 1. Set up a new host on your webserver : packagist.mycompany.com 19 | 20 | 2. Go to the document root for the new host 21 | 22 | 3. run `git clone git://github.com/ekino/php-mirroring.git .` 23 | 24 | 4. run `cp config.php.sample config.php` 25 | 26 | 5. edit rules: `vim config.php` 27 | 28 | 6. run `php mirror.php` - this can take a while for the first one 29 | 30 | 7. add a new cron job `sudo crontab -u www-data -e`: 31 | 32 | 0 */1 * * * cd /PATH_TO_DOCUMENT_ROOT && php mirror.php 33 | 34 | Mirroring Github 35 | ---------------- 36 | 37 | 1. Create a new mirror directory 38 | 39 | mkdir mirrors/github.com 40 | 41 | 2. Add new mirror 42 | 43 | cd mirrors/github.com 44 | git clone --mirror git://github.com/ekino/php-mirroring.git ekino/php-mirroring.git 45 | 46 | Of course you need to repeat this operation for each mirror. 47 | 48 | 3. Setup a cron to update the mirror: 49 | 50 | 0 */1 * * * /git/repositories/mirrors/update-mirrors.sh 51 | 52 | 4. Add the file : `vim /git/repositories/mirrors/update-mirrors.sh` 53 | 54 | #!/bin/sh 55 | cd /git/repositories/mirrors/github.com 56 | 57 | for i in */*.git; do 58 | cd $i 59 | 60 | url=`echo $i | sed "s/.git//"` 61 | 62 | echo " > check 404 on https://github.com/${url}" 63 | curl -ILs https://github.com/${url} | head -n 1 | grep "HTTP/1.1 200 OK" > /dev/null 64 | 65 | if [ $? -eq 0 ] ; then 66 | echo " > fetching git changes" 67 | git fetch 68 | fi 69 | 70 | #echo "end ..." 71 | cd ../.. 72 | done 73 | 74 | 75 | Usages 76 | ------ 77 | 78 | ### Using source mode with gitolite 79 | 80 | 1. Setup a Gitolite 81 | 82 | 2. Add a new entry 83 | 84 | repo mirrors/..* 85 | R = @team 86 | 87 | 3. set the source mode to true 88 | 89 | function include_source() 90 | { 91 | return true; 92 | } 93 | 94 | ### Using dist mode with a http server 95 | 96 | 1. setup a new php instance with a dedicated vhost 97 | 98 | 2. make sure the ``replace_dist_host`` function point to the correct vhost 99 | 100 | function replace_dist_host(array $metadata) 101 | { 102 | list($vendor, $name) = explode("/", $metadata['name']); 103 | 104 | if (!preg_match('@(https://api.github.com/repos/|https://github.com/|https://bitbucket.org)([a-zA-Z0-9_\-\.]*)/([a-zA-Z0-9_\-\.]*)/(zipball|archive|get)/([a-zA-Z0-9\.\-]*)(|.zip)@', $metadata['dist']['url'], $matches)) { 105 | return ''; 106 | } 107 | 108 | $host = sprintf('http://packagist.mycompany.com/cache.php/github.com/%s/%s/%s.zip', 109 | $matches[2], 110 | $matches[3], 111 | $metadata['dist']['reference'] 112 | ); 113 | 114 | return $host; 115 | } 116 | 117 | function include_dist() 118 | { 119 | return true; 120 | } 121 | 122 | function include_source() 123 | { 124 | return true; // set to false if you only want to expose distribution 125 | } 126 | 127 | ### Composer.json 128 | 129 | 130 | 1. Make sure you have a clean project 131 | 132 | rm -rf composer.lock vendor 133 | 134 | 2. update the ``composer.json`` file to disable packagist and add the new one 135 | 136 | { 137 | "repositories":[ 138 | { "packagist": false }, 139 | { "type": "composer", "url": "http://packagist.mycompany.com"} 140 | ], 141 | 142 | // ... 143 | } 144 | 145 | If you are using non-composer packages, use the correct source url when adding them. 146 | 147 | { 148 | "repositories":[ 149 | { 150 | "type": "package", 151 | "package": { 152 | "name": "documentcloud/underscore", 153 | "version": "dev-master", 154 | "source": { 155 | "url": "git@git.mycompany.com:/mirrors/github.com/documentcloud/underscore.git", 156 | "type": "git", 157 | "reference": "master" 158 | } 159 | } 160 | } 161 | 162 | // ... 163 | ], 164 | 165 | // ... 166 | } 167 | 168 | 169 | 3. Install the dependencies 170 | 171 | php composer.phar install 172 | 173 | -------------------------------------------------------------------------------- /cache.php: -------------------------------------------------------------------------------- 1 | 7 | * 8 | * For the full copyright and license information, please view the LICENSE 9 | * file that was distributed with this source code. 10 | */ 11 | 12 | // error_reporting(E_ALL); 13 | 14 | if (!is_file('config.php')) { 15 | die('Please create a config.php file'); 16 | } 17 | 18 | include __DIR__.'/config.php'; 19 | 20 | if (!isset($_SERVER['PATH_INFO'])) { 21 | die('invalid package'); 22 | } 23 | 24 | $data = build_parameters($_SERVER['PATH_INFO']); 25 | $path = sprintf('caches/%s/%s', $data['vendor'], $data['name']); 26 | 27 | if (!is_dir($path)) { 28 | mkdir($path, 0755, true); 29 | } 30 | 31 | $file = sprintf('%s/%s.bin', $path, hash('sha256', serialize($data))); 32 | 33 | if (!is_file($file)) { 34 | rename(create_archive($data), $file); 35 | } 36 | 37 | if (!is_file($file)) { 38 | die('invalid package'); 39 | } 40 | 41 | send_file(sprintf('%s-%s-%s.zip', $data['vendor'], $data['name'], $data['version']), $file); 42 | 43 | $tag = preg_match('/[0-9]{1,2}\.[0-9]{1,2}\.[0-9]{1,2}/', $data['version']) === 1; 44 | 45 | // not a sha1 or version (X.X.X) ... 46 | if (strlen($data['version']) != 40 || !$tag) { 47 | unlink($file); 48 | } 49 | 50 | /** 51 | * Create the archive and return the localtion path 52 | * The created filed is temporary, and will be deleted by 53 | * the calling function 54 | * 55 | * @return string 56 | */ 57 | function create_archive(array $parameters) 58 | { 59 | $out = sprintf('/tmp/php-mirroring-%s', md5(serialize($parameters).uniqid())); 60 | 61 | $path = sprintf('%s/%s.git', get_repository_path(), $parameters['package']); 62 | 63 | if (!$path || substr($path, 0, strlen(get_repository_path())) !== get_repository_path()) { 64 | throw new Exception("Unable to parse the request"); 65 | } 66 | 67 | if (!is_dir($path) || !is_file($path.'/config')) { 68 | throw new Exception("Unable to parse the request"); 69 | } 70 | 71 | $cmd = sprintf('cd %s && /usr/bin/git archive %s --format=zip -o %s', 72 | escapeshellcmd($path), 73 | escapeshellcmd($parameters['version']), 74 | $out 75 | ); 76 | 77 | system($cmd); 78 | 79 | if (!is_file($out)) { 80 | die('unable to create the file'); 81 | } 82 | 83 | return $out; 84 | } 85 | 86 | function send_file($name, $file) { 87 | header('Content-Description: File Transfer'); 88 | header('Content-Type: application/octet-stream'); 89 | header(sprintf('Content-Disposition: attachment; filename=%s', $name)); 90 | header('Content-Transfer-Encoding: binary'); 91 | header('Expires: 0'); 92 | header('Cache-Control: must-revalidate'); 93 | header('Pragma: public'); 94 | header('Content-Length: ' . filesize($file)); 95 | 96 | readfile($file); 97 | } 98 | -------------------------------------------------------------------------------- /caches/.gitkeep: -------------------------------------------------------------------------------- 1 | .gitkeep -------------------------------------------------------------------------------- /config.php.sample: -------------------------------------------------------------------------------- 1 | vendor / package / version 73 | */ 74 | function build_parameters($url) 75 | { 76 | if (preg_match('@/github.com/([a-zA-Z0-9\-\.]*)/([a-zA-Z0-9\-\.]*)/([a-zA-Z0-9\-\.]*).zip@', $url, $matches)) { 77 | return array( 78 | 'package' => 'github.com/'.$matches[1].'/'.$matches[2], 79 | 'version' => $matches[3], 80 | 'vendor' => $matches[1], 81 | 'name' => $matches[2] 82 | ); 83 | } 84 | 85 | if (preg_match('@/drupal.org/([a-zA-Z0-9\-\.]*)/([a-zA-Z0-9\-\.]*).zip@', $url, $matches)) { 86 | return array( 87 | 'package' => 'drupal.org/'.$matches[1], 88 | 'version' => $matches[2], 89 | 'vendor' => 'drupal', 90 | 'name' => $matches[1] 91 | ); 92 | } 93 | 94 | die('Unable to parse the request'); 95 | } 96 | -------------------------------------------------------------------------------- /mirror.php: -------------------------------------------------------------------------------- 1 | 7 | * 8 | * For the full copyright and license information, please view the LICENSE 9 | * file that was distributed with this source code. 10 | */ 11 | 12 | error_reporting(E_ALL); 13 | 14 | if (php_sapi_name() !== 'cli') { 15 | die('This script must run from the command line'); 16 | } 17 | 18 | if (!is_file(__DIR__.'/config.php')) { 19 | die('Please create a config.php file'); 20 | } 21 | 22 | ini_set('memory_limit','256M'); 23 | 24 | include __DIR__.'/config.php'; 25 | 26 | 27 | echo "step 1 : retrieving composer.phar\n"; 28 | file_put_contents('composer.phar', file_get_contents('https://getcomposer.org/composer.phar')); 29 | 30 | echo "step 2: retrieving packages.json - updated packages\n"; 31 | list($packages, $algo, $target, $mode) = download_file('packages.json', array()); 32 | 33 | echo "step 3: retrieving packages definition \n"; 34 | 35 | foreach ($packages['provider-includes'] as $file => &$options) { 36 | list($content, $algo, $target, $mode) = download_file($file, $options); 37 | 38 | if ($content === false) { 39 | continue; // fail to download 40 | } 41 | 42 | if (isset($content['packages'])) { 43 | $content['packages'] = update_packages($content['packages']); 44 | } else { 45 | // legacy repo handling 46 | $content['providers'] = update_providers($content['providers']); 47 | } 48 | 49 | $options[$algo] = store_content($file, $content, $algo); 50 | } 51 | 52 | store_content('packages.json', $packages, 'sha256'); 53 | 54 | echo "step 4: have a beer!\n"; 55 | 56 | /** 57 | * Iterate over package and fix references 58 | * 59 | * @param array $packages 60 | * 61 | * @return array the altered packages array 62 | */ 63 | function update_packages(array $packages) 64 | { 65 | foreach ($packages as $package => &$versions) { 66 | foreach ($versions as $version => &$metadata) { 67 | if (include_source()) { 68 | if (isset($metadata['source'])) { 69 | $metadata['source']['url'] = replace_source_host($metadata['source']['url']); 70 | } 71 | } else { 72 | unset($metadata['source']); 73 | } 74 | 75 | if (include_dist()) { 76 | if (isset($metadata['dist'])) { 77 | $metadata['dist']['url'] = replace_dist_host($metadata); 78 | } 79 | } else { 80 | unset($metadata['dist']); 81 | } 82 | } 83 | } 84 | 85 | return $packages; 86 | } 87 | 88 | /** 89 | * Loop providers to retrieve the different packages, and update the hash value 90 | * 91 | * @param array $providers 92 | * 93 | * @return array the altered providers array 94 | */ 95 | function update_providers(array $providers) 96 | { 97 | foreach ($providers as $provider => &$options) { 98 | list($content, $algo, $target, $mode) = download_file($provider, $options); 99 | 100 | if (!$content) { 101 | continue; // fail to download, ignore the package 102 | } 103 | 104 | if (is_array($content['packages'])) { 105 | $content['packages'] = update_packages($content['packages']); 106 | } 107 | 108 | $options[$algo] = store_content($provider, $content, $algo); 109 | } 110 | 111 | return $providers; 112 | } 113 | 114 | /** 115 | * @param string $file file name 116 | * @param array $content the content to store 117 | * @param string $algo 118 | * 119 | * @return string 120 | */ 121 | function store_content($file, array $content, $algo) 122 | { 123 | if (substr($file, -5) != '.json') { 124 | $file = 'p/'.$file . ".json"; 125 | } 126 | 127 | $content = json_encode($content); 128 | $hash = hash($algo, $content); 129 | 130 | $path = dirname($file); 131 | 132 | if (!is_dir($path)) { 133 | mkdir($path, 0755, true); 134 | } 135 | 136 | $isPackage = strpos($file, '%hash%') === false; 137 | $link = sprintf("%s$%s.json", str_replace('.json', '', $file), $hash); 138 | 139 | if ($isPackage && file_exists($link)) { // file already exist, return 140 | return $hash; 141 | } 142 | 143 | if ($isPackage) { 144 | $glob = sprintf('%s\$*.json', str_replace('.json', '', $file)); 145 | } else { 146 | $glob = str_replace('$%hash%', '\$*', $file); 147 | } 148 | 149 | foreach(glob($glob) as $f) { 150 | if ($isPackage && $f == $link) { 151 | continue; 152 | } 153 | 154 | echo sprintf(" delete: %s\n", $f); 155 | unlink($f); 156 | } 157 | 158 | if (!$isPackage) { 159 | $file = str_replace('%hash%', $hash, $file); 160 | } 161 | 162 | file_put_contents($file, $content); 163 | echo sprintf(" write: %s\n", $file); 164 | 165 | if ($isPackage) { 166 | @unlink($link); 167 | file_put_contents($link, $content); 168 | 169 | echo sprintf(" write: %s (package)\n", $link); 170 | } 171 | 172 | return $hash; 173 | } 174 | 175 | /** 176 | * Download a json file from packagist, the code also a use case where the package name is 177 | * provided. 178 | * 179 | * @param string $file the file to download 180 | * @param array $hash the information 181 | */ 182 | function download_file($file, array $hash) 183 | { 184 | if (substr($file, -5) != '.json') { 185 | $file = 'p/'.$file . ".json"; 186 | } 187 | 188 | $t = $hash; 189 | $algo = isset($hash['sha1']) ? 'sha1' : 'sha256'; 190 | $hash = isset($hash[$algo]) ? $hash[$algo] : null; 191 | 192 | if ($algo == 'sha256') { 193 | $file = str_replace('%hash%', $hash, $file); 194 | } 195 | 196 | $target = 'packagist/'.$file; 197 | 198 | $path = dirname($file); 199 | 200 | if (!is_dir('packagist/'.$path)) { 201 | mkdir('packagist/'.$path, 0755, true); 202 | } 203 | 204 | if (!is_file($target) || hash_file($algo, $target) != $hash) { 205 | echo sprintf(" > Retrieving %- 70s => %s \n", 'http://packagist.org/'.$file, $target); 206 | 207 | $content = @file_get_contents('http://packagist.org/'.$file); 208 | 209 | if (!$content) { 210 | echo " ERR: Unable to retrieve the file http://packagist.org/$file\n"; 211 | 212 | return array(false, false, false, false); 213 | } 214 | 215 | file_put_contents($target, $content); 216 | 217 | $mode = "DL"; 218 | } else { 219 | // echo sprintf(" > SKIP %s \n", $target); 220 | $mode = "SKIP"; 221 | } 222 | 223 | return array(json_decode(file_get_contents($target), true), $algo, 'packagist/'.$file, $mode); 224 | } 225 | 226 | if (!function_exists('include_dist')) { 227 | /** 228 | * Return true if you want to include the dist array 229 | * The dist array is used to download the archive version (zip) 230 | * 231 | * @return boolean 232 | */ 233 | function include_dist() { 234 | return true; 235 | } 236 | 237 | } 238 | 239 | if (!function_exists('include_source')) { 240 | /** 241 | * Return true if you want to include the source array 242 | * The source array is used to download from git/subversion 243 | * 244 | * @return boolean 245 | */ 246 | function include_source() { 247 | return true; 248 | } 249 | } 250 | --------------------------------------------------------------------------------