├── .circleci └── config.yml ├── .env ├── .github └── workflows │ └── up.yml ├── .gitignore ├── LICENSE ├── README.md ├── bin ├── index-cpan.sh ├── metacpan-docker └── partial-cpan-mirror.sh ├── compose └── web.yml ├── configs └── metacpan-ingest │ └── metacpan_ingest_local.conf ├── docker-compose.yml ├── github-meets-cpan └── environment.json ├── grep.env ├── localapi_test.env ├── logging.env ├── orepan2 ├── Dockerfile └── README.md └── pg ├── Dockerfile ├── docker-entrypoint-initdb.d ├── 100-roles.sql └── 200-minion.sql └── healthcheck.sh /.circleci/config.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Use the latest 2.1 version of CircleCI pipeline process engine. See: 3 | # https://circleci.com/docs/2.0/configuration-reference 4 | version: 2.1 5 | # Orchestrate or schedule a set of jobs 6 | workflows: 7 | docker-compose: 8 | jobs: 9 | - build-and-test 10 | jobs: 11 | build-and-test: 12 | machine: true 13 | resource_class: large 14 | steps: 15 | - checkout 16 | - run: 17 | command: | 18 | ./bin/metacpan-docker init 19 | name: clone missing repositories 20 | - run: 21 | command: | 22 | docker-compose --verbose --profile test up -d 23 | name: compose up 24 | # Since we're running docker-compose -d, we don't actually know if 25 | # Elasticsearch is available at the time this build step begins. We 26 | # probably need to wait for it here, so we'll add our own check. 27 | - run: 28 | command: | 29 | ./src/metacpan-api/wait-for-es.sh http://localhost:9200 elasticsearch_test -- 30 | name: wait for ES 31 | - run: 32 | name: Run complete MetaCPAN API Test Suite 33 | command: | 34 | docker-compose --profile test exec -T api_test prove -lr --jobs 4 t 35 | docker-compose --profile test down 36 | - run: 37 | name: Show Docker container logs on Error 38 | command: | 39 | docker-compose --profile test logs 40 | docker stats --no-stream 41 | docker ps -a | head 42 | when: on_fail 43 | -------------------------------------------------------------------------------- /.env: -------------------------------------------------------------------------------- 1 | API_SERVER="morbo -l http://*:5000 -w app.psgi -w bin -w lib -w templates --verbose" 2 | COLUMNS=80 3 | ES=elasticsearch:9200 4 | ES_TEST=elasticsearch_test:9200 5 | MINICPAN=/CPAN 6 | MOJO_MODE=development 7 | NET_ASYNC_HTTP_MAXCONNS=1 8 | PERL_CARTON_PATH=/carton 9 | PG_DATABASE=metacpan 10 | PG_HOST=pghost 11 | PG_PASSWORD=t00lchain 12 | PG_PORT=5432 13 | PG_USER=metacpan 14 | PLACK_ENV=development 15 | -------------------------------------------------------------------------------- /.github/workflows/up.yml: -------------------------------------------------------------------------------- 1 | --- 2 | name: docker-compose up 3 | 4 | on: 5 | push: 6 | schedule: 7 | # Every Sunday morning 8 | - cron: "15 4 * * *" 9 | jobs: 10 | build: 11 | name: init and up 12 | runs-on: ubuntu-latest 13 | strategy: 14 | max-parallel: 1 15 | steps: 16 | - uses: actions/checkout@v3 17 | 18 | - name: init 19 | run: bin/metacpan-docker init 20 | 21 | - name: up api 22 | run: docker-compose up -d api 23 | - name: down 24 | run: docker-compose down 25 | 26 | - name: up api_test 27 | run: docker-compose up -d api_test 28 | - name: down 29 | run: docker-compose down 30 | 31 | - name: up web 32 | run: docker-compose up -d web 33 | - name: down 34 | run: docker-compose down 35 | 36 | - name: up grep 37 | run: docker-compose up -d grep 38 | - name: down 39 | run: docker-compose down 40 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /blib/ 2 | /.build/ 3 | _build/ 4 | cover_db/ 5 | inc/ 6 | Build 7 | !Build/ 8 | Build.bat 9 | .last_cover_stats 10 | /Makefile 11 | /Makefile.old 12 | /MANIFEST.bak 13 | /META.yml 14 | /META.json 15 | /MYMETA.* 16 | nytprof.out 17 | /pm_to_blib 18 | *.o 19 | *.bs 20 | src/ 21 | .DS_Store 22 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The Artistic License 2.0 2 | 3 | Copyright (c) 2014 MetaCPAN 4 | 5 | Everyone is permitted to copy and distribute verbatim copies 6 | of this license document, but changing it is not allowed. 7 | 8 | Preamble 9 | 10 | This license establishes the terms under which a given free software 11 | Package may be copied, modified, distributed, and/or redistributed. 12 | The intent is that the Copyright Holder maintains some artistic 13 | control over the development of that Package while still keeping the 14 | Package available as open source and free software. 15 | 16 | You are always permitted to make arrangements wholly outside of this 17 | license directly with the Copyright Holder of a given Package. If the 18 | terms of this license do not permit the full use that you propose to 19 | make of the Package, you should contact the Copyright Holder and seek 20 | a different licensing arrangement. 21 | 22 | Definitions 23 | 24 | "Copyright Holder" means the individual(s) or organization(s) 25 | named in the copyright notice for the entire Package. 26 | 27 | "Contributor" means any party that has contributed code or other 28 | material to the Package, in accordance with the Copyright Holder's 29 | procedures. 30 | 31 | "You" and "your" means any person who would like to copy, 32 | distribute, or modify the Package. 33 | 34 | "Package" means the collection of files distributed by the 35 | Copyright Holder, and derivatives of that collection and/or of 36 | those files. A given Package may consist of either the Standard 37 | Version, or a Modified Version. 38 | 39 | "Distribute" means providing a copy of the Package or making it 40 | accessible to anyone else, or in the case of a company or 41 | organization, to others outside of your company or organization. 42 | 43 | "Distributor Fee" means any fee that you charge for Distributing 44 | this Package or providing support for this Package to another 45 | party. It does not mean licensing fees. 46 | 47 | "Standard Version" refers to the Package if it has not been 48 | modified, or has been modified only in ways explicitly requested 49 | by the Copyright Holder. 50 | 51 | "Modified Version" means the Package, if it has been changed, and 52 | such changes were not explicitly requested by the Copyright 53 | Holder. 54 | 55 | "Original License" means this Artistic License as Distributed with 56 | the Standard Version of the Package, in its current version or as 57 | it may be modified by The Perl Foundation in the future. 58 | 59 | "Source" form means the source code, documentation source, and 60 | configuration files for the Package. 61 | 62 | "Compiled" form means the compiled bytecode, object code, binary, 63 | or any other form resulting from mechanical transformation or 64 | translation of the Source form. 65 | 66 | 67 | Permission for Use and Modification Without Distribution 68 | 69 | (1) You are permitted to use the Standard Version and create and use 70 | Modified Versions for any purpose without restriction, provided that 71 | you do not Distribute the Modified Version. 72 | 73 | 74 | Permissions for Redistribution of the Standard Version 75 | 76 | (2) You may Distribute verbatim copies of the Source form of the 77 | Standard Version of this Package in any medium without restriction, 78 | either gratis or for a Distributor Fee, provided that you duplicate 79 | all of the original copyright notices and associated disclaimers. At 80 | your discretion, such verbatim copies may or may not include a 81 | Compiled form of the Package. 82 | 83 | (3) You may apply any bug fixes, portability changes, and other 84 | modifications made available from the Copyright Holder. The resulting 85 | Package will still be considered the Standard Version, and as such 86 | will be subject to the Original License. 87 | 88 | 89 | Distribution of Modified Versions of the Package as Source 90 | 91 | (4) You may Distribute your Modified Version as Source (either gratis 92 | or for a Distributor Fee, and with or without a Compiled form of the 93 | Modified Version) provided that you clearly document how it differs 94 | from the Standard Version, including, but not limited to, documenting 95 | any non-standard features, executables, or modules, and provided that 96 | you do at least ONE of the following: 97 | 98 | (a) make the Modified Version available to the Copyright Holder 99 | of the Standard Version, under the Original License, so that the 100 | Copyright Holder may include your modifications in the Standard 101 | Version. 102 | 103 | (b) ensure that installation of your Modified Version does not 104 | prevent the user installing or running the Standard Version. In 105 | addition, the Modified Version must bear a name that is different 106 | from the name of the Standard Version. 107 | 108 | (c) allow anyone who receives a copy of the Modified Version to 109 | make the Source form of the Modified Version available to others 110 | under 111 | 112 | (i) the Original License or 113 | 114 | (ii) a license that permits the licensee to freely copy, 115 | modify and redistribute the Modified Version using the same 116 | licensing terms that apply to the copy that the licensee 117 | received, and requires that the Source form of the Modified 118 | Version, and of any works derived from it, be made freely 119 | available in that license fees are prohibited but Distributor 120 | Fees are allowed. 121 | 122 | 123 | Distribution of Compiled Forms of the Standard Version 124 | or Modified Versions without the Source 125 | 126 | (5) You may Distribute Compiled forms of the Standard Version without 127 | the Source, provided that you include complete instructions on how to 128 | get the Source of the Standard Version. Such instructions must be 129 | valid at the time of your distribution. If these instructions, at any 130 | time while you are carrying out such distribution, become invalid, you 131 | must provide new instructions on demand or cease further distribution. 132 | If you provide valid instructions or cease distribution within thirty 133 | days after you become aware that the instructions are invalid, then 134 | you do not forfeit any of your rights under this license. 135 | 136 | (6) You may Distribute a Modified Version in Compiled form without 137 | the Source, provided that you comply with Section 4 with respect to 138 | the Source of the Modified Version. 139 | 140 | 141 | Aggregating or Linking the Package 142 | 143 | (7) You may aggregate the Package (either the Standard Version or 144 | Modified Version) with other packages and Distribute the resulting 145 | aggregation provided that you do not charge a licensing fee for the 146 | Package. Distributor Fees are permitted, and licensing fees for other 147 | components in the aggregation are permitted. The terms of this license 148 | apply to the use and Distribution of the Standard or Modified Versions 149 | as included in the aggregation. 150 | 151 | (8) You are permitted to link Modified and Standard Versions with 152 | other works, to embed the Package in a larger work of your own, or to 153 | build stand-alone binary or bytecode versions of applications that 154 | include the Package, and Distribute the result without restriction, 155 | provided the result does not expose a direct interface to the Package. 156 | 157 | 158 | Items That are Not Considered Part of a Modified Version 159 | 160 | (9) Works (including, but not limited to, modules and scripts) that 161 | merely extend or make use of the Package, do not, by themselves, cause 162 | the Package to be a Modified Version. In addition, such works are not 163 | considered parts of the Package itself, and are not subject to the 164 | terms of this license. 165 | 166 | 167 | General Provisions 168 | 169 | (10) Any use, modification, and distribution of the Standard or 170 | Modified Versions is governed by this Artistic License. By using, 171 | modifying or distributing the Package, you accept this license. Do not 172 | use, modify, or distribute the Package, if you do not accept this 173 | license. 174 | 175 | (11) If your Modified Version has been derived from a Modified 176 | Version made by someone other than you, you are nevertheless required 177 | to ensure that your Modified Version complies with the requirements of 178 | this license. 179 | 180 | (12) This license does not grant you the right to use any trademark, 181 | service mark, tradename, or logo of the Copyright Holder. 182 | 183 | (13) This license includes the non-exclusive, worldwide, 184 | free-of-charge patent license to make, have made, use, offer to sell, 185 | sell, import and otherwise transfer the Package with respect to any 186 | patent claims licensable by the Copyright Holder that are necessarily 187 | infringed by the Package. If you institute patent litigation 188 | (including a cross-claim or counterclaim) against any party alleging 189 | that the Package constitutes direct or contributory patent 190 | infringement, then this Artistic License to you shall terminate on the 191 | date that such litigation is filed. 192 | 193 | (14) Disclaimer of Warranty: 194 | THE PACKAGE IS PROVIDED BY THE COPYRIGHT HOLDER AND CONTRIBUTORS "AS 195 | IS' AND WITHOUT ANY EXPRESS OR IMPLIED WARRANTIES. THE IMPLIED 196 | WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, OR 197 | NON-INFRINGEMENT ARE DISCLAIMED TO THE EXTENT PERMITTED BY YOUR LOCAL 198 | LAW. UNLESS REQUIRED BY LAW, NO COPYRIGHT HOLDER OR CONTRIBUTOR WILL 199 | BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, OR CONSEQUENTIAL 200 | DAMAGES ARISING IN ANY WAY OUT OF THE USE OF THE PACKAGE, EVEN IF 201 | ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 202 | 203 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # MetaCPAN Docker 2 | 3 | [![CircleCI](https://circleci.com/gh/metacpan/metacpan-docker.svg?style=svg)](https://circleci.com/gh/metacpan/metacpan-docker) 4 | 5 | ![docker compose up](https://github.com/metacpan/metacpan-docker/workflows/docker-compose%20up/badge.svg?branch=master) 6 | 7 | 8 | 9 | - [Running the MetaCPAN stack with Docker (via Docker Compose)](#running-the-metacpan-stack-with-docker-via-docker-compose) 10 | - [Quick Start](#quick-start) 11 | - [Working with Containers](#working-with-containers) 12 | - [Building Containers](#building-containers) 13 | - [Accessing Containers](#accessing-containers) 14 | - [Accessing Services](#accessing-services) 15 | - [`web`](#web) 16 | - [`api`](#api) 17 | - [`Elasticsearch`](#elasticsearch) 18 | - [`PostgreSQL`](#postgresql) 19 | - [`grep`](#grep) 20 | - [System architecture](#system-architecture) 21 | - [The `bin/metacpan-docker` script](#the-binmetacpan-docker-script) 22 | - [`bin/metacpan init`](#binmetacpan-init) 23 | - [`bin/metacpan localapi`](#binmetacpan-localapi) 24 | - [`bin/metacpan-docker pull`](#binmetacpan-docker-pull) 25 | - [`bin/metacpan-docker reset`](#binmetacpan-docker-reset) 26 | - [`bin/metacpan-docker` build/up/down/start/stop/run/ps/top...](#binmetacpan-docker-buildupdownstartstoprunpstop) 27 | - [Services](#services) 28 | - [`web`](#web-1) 29 | - [`api`](#api-1) 30 | - [`grep`](#grep-1) 31 | - [Setting up a partial CPAN in the `api` service](#setting-up-a-partial-cpan-in-the-api-service) 32 | - [Bootstrapping the `elasticsearch` indices](#bootstrapping-the-elasticsearch-indices) 33 | - [Putting the above all together](#putting-the-above-all-together) 34 | - [elasticsearch and elasticsearch_test](#elasticsearch-and-elasticsearch_test) 35 | - [Tips and tricks](#tips-and-tricks) 36 | - [Running your own miniCPAN inside metacpan-docker](#running-your-own-minicpan-inside-metacpan-docker) 37 | - [Running tests](#running-tests) 38 | - [Updating Carton dependencies](#updating-carton-dependencies) 39 | - [Updating the git repositories](#updating-the-git-repositories) 40 | - [Running Kibana to peek into Elasticsearch data](#running-kibana-to-peek-into-elasticsearch-data) 41 | - [Peeking Inside the Container](#peeking-inside-the-container) 42 | - [To Do](#to-do) 43 | - [See also](#see-also) 44 | 45 | 46 | 47 | ## Running the MetaCPAN stack with Docker (via Docker Compose) 48 | 49 | **Notice**: This project is in experimental stage. It works, but there are a lot 50 | of things to be done better. Please use it and create Issues with your problems. 51 | 52 | ## Quick Start 53 | 54 | Install [Docker][0] and [Docker Compose][1] (v2+) for your platform. [Docker for 55 | Mac][2] or [Docker for Windows][3] will install both tools for you, if you are 56 | on either of these environments. 57 | 58 | [0]: https://docs.docker.com/installation 59 | [1]: https://docs.docker.com/compose/install 60 | [2]: https://docs.docker.com/docker-for-mac/ 61 | [3]: https://docs.docker.com/docker-for-windows/ 62 | 63 | On Debian / Ubuntu, install using: 64 | 65 | apt install docker-compose-v2 66 | 67 | On Linux, Docker's default implementation only allows `root` user access to 68 | Docker commands and to control containers. In order to allow a regular user to 69 | access docker follow the 70 | [post-installation instructions](https://docs.docker.com/install/linux/linux-postinstall/). 71 | This document assumes the post-installation steps have been followed for the 72 | current user. 73 | 74 | You will also need Docker buildx, and to enable Docker BuildKit. They should be 75 | set up by default when using Docker Desktop, but on Linux you may need to 76 | install them. buildx is the `docker-buildx` package on Debian based systems. 77 | Docker BuildKit can be enabled by following the 78 | [Getting Started](https://docs.docker.com/build/buildkit/#getting-started) 79 | instructions. 80 | 81 | On Debian / Ubuntu, install using: 82 | 83 | apt install docker-buildx 84 | 85 | If you are running a Mac ARM64 system, you will need to manually tell docker to 86 | use the x86_64 version of Elasticsearch 2.4. This can be done by running the 87 | command: 88 | 89 | docker pull elasticsearch:2.4 --platform=linux/x86_64 90 | 91 | It is highly recommended that you alias `docker compose` to `fig` (its original 92 | name) and use it wherever `docker compose` is used. You are going to have to 93 | type this command a lot. 94 | 95 | Then, clone this repo and set up the environment: 96 | 97 | git clone https://github.com/metacpan/metacpan-docker.git 98 | cd metacpan-docker 99 | bin/metacpan-docker init 100 | 101 | The `bin/metacpan-docker init` command clones the source repositories for: 102 | 103 | - `metacpan-web` 104 | - `metacpan-api` 105 | - `metacpan-grep-front-end` 106 | - `metacpan-cpan-extracted-lite` 107 | 108 | These repositories are automatically mounted into the appropriate docker 109 | containers allowing the developer to use their preferred tools to work with the 110 | source code. 111 | 112 | The `docker compose --profile dev up` command on its own will bring up the 113 | entire stack in the foreground (logs will be displayed). 114 | 115 | The `docker compose --profile dev up` command will also fetch the official 116 | container images from 117 | [MetaCPAN Docker Hub](https://cloud.docker.com/u/metacpan/repository/list) 118 | repositories. 119 | 120 | This will build the Docker containers for MetaCPAN, PostgreSQL and Elasticsearch 121 | services (which will take a while, especially on a fresh first time install of 122 | Docker) and run the services. 123 | 124 | Don't forget to seed the local `metacpan-api` with a partial CPAN; run the 125 | following command in a separate terminal to get yourself up to speed: 126 | 127 | docker compose exec api index-cpan.sh 128 | 129 | This will prompt you to confirm removing old indices and setting up mappings on 130 | the Elasticsearch service (say `YES`). It will then proceed to rsync a partial 131 | CPAN in `/CPAN` for its metadata to be imported. 132 | 133 | Once the above is done, you should be able to see your local partial CPAN data 134 | in e.g. [http://localhost:5001/recent](http://localhost:5001/recent) and 135 | elsewhere. 136 | 137 | Alternatively, if you just want to hack on the web frontend, you can run this 138 | instead of all the above: 139 | 140 | docker compose up web-server 141 | 142 | From here, you can proceed and hack on the MetaCPAN code at `src/metacpan-api` 143 | and/or `src/metacpan-web` directories, and saving edits will reload the 144 | corresponding apps automatically! 145 | 146 | When done hacking (or, more likely, when you need to rebuild/refresh your Docker 147 | environment) you can then run 148 | 149 | docker compose --profile dev down 150 | 151 | in another terminal to stop all MetaCPAN services and remove the containers. 152 | 153 | For further details, read on! 154 | 155 | ## Working with Containers 156 | 157 | ### Building Containers 158 | 159 | You can (re)build arbitrary containers. For instance, if you want to rebuild the 160 | `api` container: 161 | 162 | docker compose build api 163 | 164 | ### Accessing Containers 165 | 166 | Containers are accessible via the `docker compose exec` command followed by the 167 | container and then the command to execute. For example, to start a shell prompt 168 | in the `api` container: 169 | 170 | docker compose exec api /bin/bash 171 | 172 | Executing tests via `prove` inside the API container: 173 | 174 | docker compose exec api_test prove -lvr \ 175 | t/00_setup.t \ 176 | t/01_darkpan.t \ 177 | t/api/controller/cover.t 178 | 179 | To access the `psql` command line client in the PostgreSQL container: 180 | 181 | docker compose exec pgdb psql 182 | 183 | ### Accessing Services 184 | 185 | Each container is responsible for a different service. Some of these services 186 | are available in the developer environment via ports on the host system. 187 | 188 | We are using [traefik][13] to manage the traffic between services. The current 189 | configuration is: 190 | 191 | - api: [http://api.metacpan.localhost](http://api.metacpan.localhost) 192 | - web: [http://web.metacpan.localhost](http://web.metacpan.localhost) 193 | - grep: [http://grep.metacpan.localhost](http://grep.metacpan.localhost) 194 | 195 | In order to access to the localhost subdomains, you probably have to manually 196 | add these entries in you `/etc/hosts` file. 197 | 198 | # add to /etc/hosts 199 | 127.0.0.1 api.metacpan.localhost 200 | 127.0.0.1 gh.metacpan.localhost 201 | 127.0.0.1 grep.metacpan.localhost 202 | 127.0.0.1 metacpan.localhost 203 | 127.0.0.1 web.metacpan.localhost 204 | 205 | You can access the dashboard configuration via: 206 | [http://metacpan.localhost:8080](http://metacpan.localhost:8080) 207 | 208 | [0]: https://docs.traefik.io/providers/docker/ 209 | 210 | #### `web` 211 | 212 | The local instance of the web front end is accessible via: 213 | 214 | - [http://localhost:5001](http://localhost:5001) 215 | - [http://web.metacpan.localhost](http://web.metacpan.localhost) 216 | 217 | #### `api` 218 | 219 | - [http://localhost:5000](http://localhost:5000) 220 | - [http://api.metacpan.localhost](http://api.metacpan.localhost) 221 | 222 | #### `Elasticsearch` 223 | 224 | The `elasticsearch` and `elasticsearch_test` containers are not exposed 225 | directly. They are available via the `api` and `api_test` containers. 226 | 227 | You can query the `elasticsearch` container via: 228 | 229 | docker compose exec elasticsearch curl http://localhost:9200 230 | 231 | You can query the `elasticsearch_test` container via: 232 | 233 | docker compose exec elasticsearch_test curl http://localhost:9200 234 | 235 | #### `PostgreSQL` 236 | 237 | The PostgreSQL service by default is only accessible from other containers. 238 | 239 | #### `grep` 240 | 241 | The grep metacpan front end is accessible via: 242 | 243 | - [http://grep.metacpan.localhost](http://grep.metacpan.localhost) 244 | 245 | Note: this is using a smaller, frozen version of `metacpan-cpan-extracted` via 246 | [metacpan-cpan-extracted-lite](https://github.com/metacpan/metacpan-cpan-extracted-lite). 247 | 248 | ## System architecture 249 | 250 | The system consists of several services that live in docker containers: 251 | 252 | - `web` — the web interface on [http://localhost:5001](http://localhost:5001) 253 | - `api` — the main server on [http://localhost:5000](http://localhost:5000) 254 | - `api_test` — the api server for running tests via `prove` 255 | - `elasticsearch` — database for `api` 256 | - `elasticsearch_test` — database for `api_test` 257 | - `pgdb` - PostgreSQL database container 258 | - `logspout` - Docker log interface to [honeycomb.io](https://honeycomb.io) 259 | - `grep` - the web interface for grep.metacpan on 260 | [http://localhost:3001](http://localhost:3001) 261 | 262 | These services use one or more Docker volumes: 263 | 264 | - `metacpan_cpan`: holds the CPAN archive, mounted in `/CPAN` 265 | - `metacpan_elasticsearch`: holds the Elasticsearch database files 266 | - `metacpan_elasticsearch_test`: holds the Elasticsearch test database files 267 | - `metacpan_api_carton` and `metacpan_web_carton`: holds the dependencies 268 | installed by [Carton][4] for the `api` and `web` services, respectively; 269 | mounted on `/carton` instead of `local`, to prevent clashing with the host 270 | user's Carton 271 | - `metacpan_git_shared`: points to the git repo containing all extracted CPAN 272 | versions. This is mounted in `/shared/metacpan_git`. This can be either 273 | `metacpan-cpan-extracted` or `metacpan-cpan-extracted-lite`. The volume is 274 | bound to the local repo at `${PWD}/src/metacpan-cpan-extracted`. 275 | 276 | [4]: https://metacpan.org/pod/Carton 277 | 278 | Docker Compose is used to, uh, _compose_ them all together into one system. 279 | Using `docker compose` directly is a mouthful, however, so putting this all 280 | together is done via the `bin/metacpan-docker` script to simplify setup and 281 | usage (and to get you started hacking on the MetaCPAN sooner!) 282 | 283 | ### The `bin/metacpan-docker` script 284 | 285 | `bin/metacpan-docker` is a thin wrapper around the `docker compose` command, 286 | providing the environment variables necessary to run a basic MetaCPAN 287 | environment. It provides these subcommands: 288 | 289 | #### `bin/metacpan init` 290 | 291 | The `init` subcommand basically clones the [metacpan-api][5] and 292 | [metacpan-web][6] repositories, and sets up the git commit hooks for each of 293 | them, in preparation for future `docker compose` or 294 | `bin/metacpan-docker localapi` commands. 295 | 296 | It also clones the `metacpan-grep-front-end` and `metacpan-cpan-extracted-lite` 297 | repositories. 298 | 299 | [5]: https://github.com/metacpan/metacpan-api 300 | [6]: https://github.com/metacpan/metacpan-web 301 | 302 | #### `bin/metacpan localapi` 303 | 304 | The `localapi` subcommand adds the necessary configuration for `docker compose` 305 | to run both the `metacpan-web` and `metacpan-api` services, along with 306 | `elasticsearch` and Docker volumes. Under the hood, it customizes the 307 | `COMPOSE_FILE` and `COMPOSE_PROJECT_NAME` environment variables used by 308 | `docker compose` to use additional YAML configuration files aside from the 309 | default `docker compose.yml`. 310 | 311 | #### `bin/metacpan-docker pull` 312 | 313 | This is used to update all the git repository in `src/*`. This will stay on your 314 | current local branch. 315 | 316 | #### `bin/metacpan-docker reset` 317 | 318 | This is used to reset all the git repositories in `src/*` to their latest 319 | version on `upstream/master`. This will fail if you have some uncommitted local 320 | changes. You should then commit or cancel your changes before re-running the 321 | command. 322 | 323 | #### `bin/metacpan-docker` build/up/down/start/stop/run/ps/top... 324 | 325 | As noted earlier, `bin/metacpan-docker` is a thin wrapper around 326 | `docker compose`, so commands like `up`, `down`, and `run` will work as expected 327 | from `docker compose`. See the [docker compose docs][7] for an overview of 328 | available commands. 329 | 330 | [7]: 331 | https://docs.docker.com/compose/reference/overview/#command-options-overview-and-help 332 | 333 | ### Services 334 | 335 | #### `web` 336 | 337 | The `web` service is a checkout of `metacpan-web`, built as a Docker image. 338 | Running this service alone is enough if you want to just hack on the frontend, 339 | since by default the service is configured to talk to 340 | [https://fastapi.metacpan.org](https://fastapi.metacpan.org) for its backend; if 341 | this is what you want, then you can simply invoke `docker compose up` or 342 | `docker compose up web`. 343 | 344 | #### `api` 345 | 346 | The `api` service is a checkout of `metacpan-api`, built as a Docker image, just 347 | like the `web` service. 348 | 349 | If using this service to run a local backend, you will need to run some 350 | additional commands in a separate terminal once 351 | `bin/metacpan-docker localapi up` runs. 352 | 353 | #### `grep` 354 | 355 | The `grep` service is a checkout of `metacpan-grep-front-end`, built as a Docker 356 | image. Note that this is using the `metacpan_git_shared` volume, which requires 357 | the git repo for `metacpan-cpan-extracted` which can be initialized by running: 358 | 359 | ./bin/metacpan-docker init 360 | 361 | ##### Setting up a partial CPAN in the `api` service 362 | 363 | Running 364 | 365 | bin/metacpan-docker localapi exec api partial-cpan-mirror.sh 366 | 367 | will `rsync` modules from selected CPAN authors, plus the package and author 368 | indices, into the `api` service's `/CPAN` directory. This is nearly equivalent 369 | to the same script in the (now deprecated) [metacpan-developer][8] repository. 370 | [8]: https://github.com/metacpan/metacpan-developer 371 | 372 | ##### Bootstrapping the `elasticsearch` indices 373 | 374 | Running 375 | 376 | bin/metacpan-docker localapi exec api bin/run bin/metacpan mapping --delete 377 | bin/metacpan-docker localapi exec api bin/run bin/metacpan release /CPAN/authors/id 378 | bin/metacpan-docker localapi exec api bin/run bin/metacpan latest 379 | bin/metacpan-docker localapi exec api bin/run bin/metacpan author 380 | 381 | in sequence will create the indices and mappings in the `elasticsearch` service, 382 | and import the `/CPAN` data into `elasticsearch`. 383 | 384 | ##### Putting the above all together 385 | 386 | If you're impatient or too lazy to do all the above, just running 387 | 388 | bin/metacpan-docker localapi exec api index-cpan.sh 389 | 390 | instead will set it all up for you. 391 | 392 | #### elasticsearch and elasticsearch_test 393 | 394 | The `elasticsearch` and `elasticsearch_test` services use the official 395 | [Elasticsearch Docker image][9], configured with settings and scripts taken from 396 | the [metacpan-puppet][10] repository. The `api` service depends on the 397 | `elasticsearch` service and the `api_test` service depends on the 398 | `elasticsearch_test` services. 399 | 400 | [9]: https://store.docker.com/images/elasticsearch 401 | [10]: https://github.com/metacpan/metacpan-puppet 402 | 403 | ## Tips and tricks 404 | 405 | ### Running your own miniCPAN inside metacpan-docker 406 | 407 | Suppose you have a local minicpan in `/home/ftp/pub/CPAN`. If you would like to 408 | use this in `metacpan-docker`, then edit the `docker compose.localapi.yml` to 409 | change the `api` service's volume mounts to use your local minicpan as `/CPAN`, 410 | e.g.: 411 | 412 | ```yaml 413 | services: 414 | api: 415 | volumes: 416 | - /home/ftp/pub/CPAN:/CPAN 417 | ... 418 | ``` 419 | 420 | Note that if you want CPAN author data indexed into Elasticsearch, your minicpan 421 | should include `authors/00whois.xml`. Full indexing would take a better part of 422 | a day or two, depending on your hardware. 423 | 424 | ### Running tests 425 | 426 | Use `bin/metacpan-docker run` and similar: 427 | 428 | # Run tests for metacpan-web against fastapi.metacpan.org 429 | bin/metacpan-docker exec web bin/prove 430 | 431 | # Run tests for metacpan-web against a local api 432 | bin/metacpan-docker localapi exec web bin/prove 433 | 434 | # Run tests for metacpan-api against a local elasticsearch_test 435 | bin/metacpan-docker localapi exec api bin/prove 436 | 437 | ### Updating Carton dependencies 438 | 439 | Because both the `api` and `web` services are running inside clean [Perl][11] 440 | containers, it is possible to maintain a clean set of Carton dependencies 441 | independent of your host machine's perl. Just update the `cpanfile` of the 442 | project, and run 443 | 444 | bin/metacpan-docker exec web carton install 445 | # or 446 | bin/metacpan-docker exec api carton install 447 | 448 | Due to the way the Compose services are configured, these commands will update 449 | the corresponding `cpanfile.snapshot` safely, even if you do _or_ don't have a 450 | `local` directory (internally, the containers' `local` directory is placed in 451 | `/carton` instead, to prevent interfering with the host user's own `local` 452 | Carton directory.) 453 | 454 | ### Updating the git repositories 455 | 456 | You can use `bin/metacpan-docker pull` to update all `src/*` directories. 457 | 458 | [11]: https://github.com/Perl/docker-perl 459 | 460 | ### Running Kibana to peek into Elasticsearch data 461 | 462 | By default, the `docker compose.localapi.yml` configures the `elasticsearch` 463 | service to listen on the Docker host at 464 | [http://localhost:9200](http://localhost:9200), and is also accessible via the 465 | Docker `default` network address of 466 | [http://172.17.0.1:9200](http://172.17.0.1:9200); you can inspect it via simple 467 | `curl` or `wget` requests, or use a [Kibana][12] container, e.g. 468 | 469 | docker run --rm -p 5601:5601 -e ELASTICSEARCH_URL=http://172.17.0.1:9200 -it kibana:4.6 470 | 471 | Running the above will provide a Kibana container at 472 | [http://localhost:5601](http://localhost:5601), which you can configure to have 473 | it read the `cpan*` index in the `elasticsearch` service. 474 | 475 | It is also certainly possible to run Kibana as part of the compose setup, by 476 | configuring e.g. a `kibana` service. 477 | 478 | [12]: https://hub.docker.com/_/kibana/ 479 | 480 | ## Peeking Inside the Container 481 | 482 | If you run `docker ps` you'll see the containers. You might see something like: 483 | 484 | ```bash 485 | $ docker ps 486 | CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES 487 | 2efb9c475c83 metacpan-web:latest "carton exec plackup…" 12 hours ago Up 12 hours 0.0.0.0:5001->5001/tcp metacpan_web_1 488 | 8850110e06d8 metacpan-api:latest "/wait-for-it.sh db:…" 12 hours ago Up 12 hours 0.0.0.0:5000->5000/tcp metacpan_api_1 489 | 7686d7ea03c6 postgres:9.6-alpine "docker-entrypoint.s…" 12 hours ago Up 12 hours (healthy) 0.0.0.0:5432->5432/tcp metacpan_pgdb_1 490 | c7de256d29b2 elasticsearch:2.4 "/docker-entrypoint.…" 5 months ago Up 26 hours 0.0.0.0:9200->9200/tcp, 9300/tcp metacpan_elasticsearch_1 491 | f1e04fe53598 elasticsearch:2.4 "/docker-entrypoint.…" 5 months ago Up 26 hours 9300/tcp, 0.0.0.0:9900->9200/tcp metacpan_elasticsearch_test_1 492 | ``` 493 | 494 | You can then use the container name to get shell access. For instance, to log in 495 | to the API container: 496 | 497 | `docker exec -it metacpan_api_1 /bin/bash` 498 | 499 | ## To Do 500 | 501 | - Integrate all other MetaCPAN services 502 | - Add more Tips and tricks (as we continue hacking MetaCPAN in Docker) 503 | - Provide a "near-production" Docker Compose configuration, suitable for Docker 504 | Swarm, and/or 505 | - Refactor configuration to be suitable for Kubernetes (Google Cloud) 506 | deployments 507 | 508 | ## See also 509 | 510 | - [Docker Compose documentation](https://docs.docker.com/compose/overview) 511 | - [metacpan-developer][7] and [metacpan-puppet][9] from which much information 512 | about the architecture is based on 513 | -------------------------------------------------------------------------------- /bin/index-cpan.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | ./bin/run bin/metacpan mapping --delete 4 | 5 | /bin/partial-cpan-mirror.sh 6 | 7 | ./bin/run bin/metacpan release /CPAN/authors/id/ 8 | ./bin/run bin/metacpan latest 9 | ./bin/run bin/metacpan author 10 | ./bin/run bin/metacpan permission 11 | -------------------------------------------------------------------------------- /bin/metacpan-docker: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # metacpan-docker: simple wrapper for docker compose running MetaCPAN 3 | 4 | set -e 5 | 6 | GitRepos=("metacpan-api" "metacpan-web" "metacpan-grep-front-end" "metacpan-cpan-extracted-lite" "metacpan-ingest") 7 | 8 | # sanity check 9 | type "docker" > /dev/null 10 | 11 | git_clone_and_setup_hooks() { 12 | local repo=$1 13 | ( 14 | cd src 15 | [ -d "$repo" ] || git clone "https://github.com/metacpan/$repo.git" 16 | cd "$repo" 17 | [ -e git/hooks/pre-commit ] && chmod +x git/hooks/pre-commit 18 | [ -d .git/hooks ] || mkdir .git/hooks 19 | cd .git/hooks 20 | ln -sf ../../git/hooks/pre-commit 21 | ) 22 | echo "Repository $repo ready." 23 | } 24 | 25 | init() { 26 | echo "Initializing metacpan-docker repositories:" 27 | mkdir -p src 28 | for repo in ${GitRepos[@]}; do 29 | git_clone_and_setup_hooks "$repo" 30 | done 31 | 32 | [ -e src/metacpan-cpan-extracted ] || ln -s metacpan-cpan-extracted-lite src/metacpan-cpan-extracted 33 | 34 | docker volume create \ 35 | --opt type=none \ 36 | --opt device="$PWD/src/metacpan-cpan-extracted" \ 37 | --opt o=bind \ 38 | metacpan_git_shared 39 | 40 | echo "metacpan-docker ready! Run 'bin/metacpan-docker localapi up' to start." 41 | } 42 | 43 | git_update_repo() { 44 | local repo=$1 45 | ( 46 | cd "src/$repo" 47 | git fetch origin 48 | git pull origin master 49 | ) 50 | echo "Repository $repo updated." 51 | } 52 | 53 | git_reset_repo() { 54 | local repo=$1 55 | echo "Updating repository $repo" 56 | ( 57 | cd "src/$repo" 58 | git fetch origin 59 | git checkout master 60 | git pull origin master 61 | ) 62 | echo "Repository $repo updated." 63 | } 64 | 65 | update() { 66 | echo "Updating metacpan-docker repositories from upstream" 67 | 68 | git fetch origin 69 | git pull origin master 70 | 71 | for repo in ${GitRepos[@]}; do 72 | git_update_repo "$repo" 73 | done 74 | } 75 | 76 | reset_repo() { 77 | echo "Resetting metacpan-docker repositories:" 78 | 79 | for repo in ${GitRepos[@]}; do 80 | git_reset_repo "$repo" 81 | done 82 | } 83 | 84 | case "x$1" in 85 | 'xinit') 86 | init 87 | exit 88 | ;; 89 | 'xreset') 90 | reset_repo 91 | exit 92 | ;; 93 | 'xpull') 94 | update 95 | exit 96 | ;; 97 | 'xlocalapi') 98 | shift 99 | ;; 100 | 'x') 101 | init 102 | update 103 | exit 104 | ;; 105 | *) ;; 106 | esac 107 | 108 | exec docker compose "$@" 109 | -------------------------------------------------------------------------------- /bin/partial-cpan-mirror.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | MINICPAN=${MINICPAN:-$HOME/CPAN} 4 | mkdir -p $MINICPAN 5 | 6 | RSYNC='/usr/bin/rsync -av --delete --relative' 7 | PATH='cpan-rsync.perl.org::CPAN' 8 | 9 | $RSYNC $PATH/authors/id/L/LL/LLAP $MINICPAN/ 10 | $RSYNC $PATH/authors/id/N/NE/NEILB $MINICPAN/ 11 | $RSYNC $PATH/authors/id/O/OA/OALDERS $MINICPAN/ 12 | $RSYNC $PATH/authors/id/P/PE/PERLER $MINICPAN/ 13 | $RSYNC $PATH/authors/id/R/RW/RWSTAUNER $MINICPAN/ 14 | 15 | $RSYNC $PATH/authors/0* $MINICPAN/ 16 | $RSYNC $PATH/modules/0* $MINICPAN/ 17 | 18 | $RSYNC $PATH/indices/mirrors.json $MINICPAN/ 19 | $RSYNC $PATH/indices/find-ls.gz $MINICPAN/ 20 | -------------------------------------------------------------------------------- /compose/web.yml: -------------------------------------------------------------------------------- 1 | --- 2 | services: 3 | web-server: 4 | profiles: 5 | - cloud-es 6 | ports: 7 | - "5001:80" 8 | networks: 9 | - web-network 10 | -------------------------------------------------------------------------------- /configs/metacpan-ingest/metacpan_ingest_local.conf: -------------------------------------------------------------------------------- 1 | cpan = /CPAN 2 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | --- 2 | x-project: 3 | name: metacpan 4 | # ____ _____ ______ _____ ____ _____ ____ 5 | # / ___|| ____| _ \ \ / /_ _/ ___| ____/ ___| 6 | # \___ \| _| | |_) \ \ / / | | | | _| \___ \ 7 | # ___) | |___| _ < \ V / | | |___| |___ ___) | 8 | # |____/|_____|_| \_\ \_/ |___\____|_____|____/ 9 | # 10 | 11 | include: 12 | - path: 13 | - src/metacpan-web/docker-compose.yml 14 | - compose/web.yml 15 | 16 | services: 17 | # _ 18 | # __ _ _ __ (_) 19 | # / _` | '_ \| | 20 | # | (_| | |_) | | 21 | # \__,_| .__/|_| 22 | # |_| 23 | # 24 | 25 | cloud_api: 26 | profiles: 27 | - cloud-es 28 | depends_on: 29 | pghost: 30 | condition: service_healthy 31 | image: metacpan/metacpan-api:latest 32 | build: 33 | context: ./src/metacpan-api 34 | # put variables for compose inside a .env file 35 | # use env_file for variables to be set inside the container 36 | env_file: 37 | - .env 38 | command: > 39 | ${API_SERVER} ./bin/api.pl 40 | volumes: 41 | - type: volume 42 | source: cpan 43 | target: /CPAN 44 | - type: bind 45 | source: ./src/metacpan-api 46 | target: /metacpan-api 47 | - type: bind 48 | source: ./bin/index-cpan.sh 49 | target: /bin/index-cpan.sh 50 | read_only: true 51 | - type: bind 52 | source: ./bin/partial-cpan-mirror.sh 53 | target: /bin/partial-cpan-mirror.sh 54 | read_only: true 55 | ports: 56 | - "5000:5000" 57 | networks: 58 | - database 59 | - web-network 60 | 61 | api: 62 | profiles: 63 | - dev 64 | depends_on: 65 | elasticsearch: 66 | condition: service_healthy 67 | pghost: 68 | condition: service_healthy 69 | image: metacpan/metacpan-api:latest 70 | build: 71 | context: ./src/metacpan-api 72 | # put variables for compose inside a .env file 73 | # use env_file for variables to be set inside the container 74 | env_file: 75 | - .env 76 | command: > 77 | ${API_SERVER} ./bin/api.pl 78 | volumes: 79 | - type: volume 80 | source: cpan 81 | target: /CPAN 82 | - type: bind 83 | source: ./src/metacpan-api 84 | target: /metacpan-api 85 | - type: bind 86 | source: ./bin/index-cpan.sh 87 | target: /bin/index-cpan.sh 88 | read_only: true 89 | - type: bind 90 | source: ./bin/partial-cpan-mirror.sh 91 | target: /bin/partial-cpan-mirror.sh 92 | read_only: true 93 | ports: 94 | - "5000:5000" 95 | networks: 96 | - database 97 | - elasticsearch 98 | - web-network 99 | 100 | api_test: 101 | profiles: 102 | - test 103 | - ingest-test 104 | depends_on: 105 | elasticsearch_test: 106 | condition: service_healthy 107 | pghost: 108 | condition: service_healthy 109 | image: metacpan/metacpan-api:latest 110 | build: 111 | context: ./src/metacpan-api 112 | env_file: 113 | - localapi_test.env 114 | command: > 115 | ${API_SERVER} ./bin/api.pl 116 | volumes: 117 | - type: volume 118 | source: cpan 119 | target: /CPAN 120 | - type: bind 121 | source: ./src/metacpan-api 122 | target: /metacpan-api 123 | - type: bind 124 | source: ./bin/index-cpan.sh 125 | target: /bin/index-cpan.sh 126 | read_only: true 127 | - type: bind 128 | source: ./bin/partial-cpan-mirror.sh 129 | target: /bin/partial-cpan-mirror.sh 130 | read_only: true 131 | ports: 132 | - "5000" 133 | networks: 134 | - database 135 | - elasticsearch 136 | 137 | ingest: 138 | profiles: 139 | - dev 140 | - ingest 141 | image: metacpan/metacpan-ingest:latest 142 | volumes: 143 | - type: volume 144 | source: cpan 145 | target: /CPAN 146 | - type: bind 147 | source: ./configs/metacpan-ingest/metacpan_ingest_local.conf 148 | target: /metacpan-ingest/metacpan_ingest_local.conf 149 | read_only: true 150 | depends_on: 151 | - elasticsearch 152 | networks: 153 | - elasticsearch 154 | 155 | ingest-test: 156 | profiles: 157 | - ingest-test 158 | image: metacpan/metacpan-ingest:latest 159 | environment: 160 | PLACK_ENV: development 161 | volumes: 162 | - type: volume 163 | source: cpan 164 | target: /CPAN 165 | - type: bind 166 | source: ./configs/metacpan-ingest/metacpan_ingest_local.conf 167 | target: /metacpan-ingest/metacpan_ingest_local.conf 168 | read_only: true 169 | depends_on: 170 | - elasticsearch_test 171 | networks: 172 | - elasticsearch 173 | 174 | # __ _ _ __ ___ _ __ 175 | # / _` | '__/ _ \ '_ \ 176 | # | (_| | | | __/ |_) | 177 | # \__, |_| \___| .__/ 178 | # __/ | | | 179 | # |___/ |_| 180 | 181 | grep: 182 | profiles: 183 | - grep 184 | image: metacpan/metacpan-grep-front-end:latest 185 | build: 186 | context: ./src/metacpan-grep-front-end 187 | volumes: 188 | - type: volume 189 | source: metacpan_git_shared 190 | target: /shared/metacpan_git 191 | read_only: true 192 | - type: bind 193 | source: ./src/metacpan-grep-front-end 194 | target: /metacpan-grep-front-end 195 | read_only: true 196 | env_file: 197 | - grep.env 198 | 199 | # ____ _ _____ _ ____ _ ____ _____ ____ 200 | # | _ \ / \|_ _|/ \ | __ ) / \ / ___|| ____/ ___| 201 | # | | | |/ _ \ | | / _ \ | _ \ / _ \ \___ \| _| \___ \ 202 | # | |_| / ___ \| |/ ___ \| |_) / ___ \ ___) | |___ ___) | 203 | # |____/_/ \_\_/_/ \_\____/_/ \_\____/|_____|____/ 204 | # 205 | 206 | # _ _ _ _ 207 | # ___| | __ _ ___| |_(_) ___ ___ ___ __ _ _ __ ___| |__ 208 | # / _ \ |/ _` / __| __| |/ __/ __|/ _ \/ _` | '__/ __| '_ \ 209 | # | __/ | (_| \__ \ |_| | (__\__ \ __/ (_| | | | (__| | | | 210 | # \___|_|\__,_|___/\__|_|\___|___/\___|\__,_|_| \___|_| |_| 211 | # 212 | 213 | elasticsearch: 214 | profiles: 215 | - dev 216 | - ingest 217 | image: elasticsearch:2.4 218 | environment: 219 | - discovery.type=single-node 220 | volumes: 221 | - type: volume 222 | source: elasticsearch 223 | target: /usr/share/elasticsearch/data 224 | healthcheck: 225 | timeout: 5s 226 | start_period: 60s 227 | test: ["CMD", "curl", "--fail", "http://localhost:9200/_cluster/health?wait_for_status=yellow&timeout=5s"] 228 | ports: 229 | - "9200" 230 | networks: 231 | - elasticsearch 232 | 233 | # _ _ _ _ 234 | # ___| | __ _ ___| |_(_) ___ ___ ___ __ _ _ __ ___| |__ 235 | # / _ \ |/ _` / __| __| |/ __/ __|/ _ \/ _` | '__/ __| '_ \ 236 | # | __/ | (_| \__ \ |_| | (__\__ \ __/ (_| | | | (__| | | | 237 | # \___|_|\__,_|___/\__|_|\___|___/\___|\__,_|_| \___|_| |_| 238 | # 239 | # _ _ 240 | # | |_ ___ ___| |_ 241 | # | __/ _ \/ __| __| 242 | # | || __/\__ \ |_ 243 | # \__\___||___/\__| 244 | # 245 | 246 | elasticsearch_test: 247 | profiles: 248 | - test 249 | - ingest-test 250 | image: elasticsearch:2.4 251 | environment: 252 | - discovery.type=single-node 253 | volumes: 254 | - type: volume 255 | source: elasticsearch_test 256 | target: /usr/share/elasticsearch/data 257 | healthcheck: 258 | timeout: 5s 259 | start_period: 60s 260 | test: ["CMD", "curl", "--fail", "http://localhost:9200/_cluster/health?wait_for_status=yellow&timeout=5s"] 261 | ports: 262 | - "9200" 263 | networks: 264 | - elasticsearch 265 | # _ _ 266 | # _ __ ___ ___| |_ __ _ _ __ ___ ___ __ _| | 267 | # | '_ \ / _ \/ __| __/ _` | '__/ _ \/ __|/ _` | | 268 | # | |_) | (_) \__ \ || (_| | | | __/\__ \ (_| | | 269 | # | .__/ \___/|___/\__\__, |_| \___||___/\__, |_| 270 | # |_| |___/ |_| 271 | # 272 | 273 | pghost: 274 | profiles: 275 | - cloud-es 276 | - dev 277 | - test 278 | - ingest-test 279 | hostname: pghost 280 | image: "postgres:${PG_VERSION_TAG:-9.6-alpine}" 281 | build: 282 | context: "./pg" 283 | args: 284 | PG_TAG: "${PG_VERSION_TAG:-9.6-alpine}" 285 | environment: 286 | POSTGRES_PASSWORD: metacpan 287 | POSTGRES_USER: metacpan123 288 | POSTGRES_DB: metacpan 289 | networks: 290 | - database 291 | healthcheck: 292 | interval: 10s 293 | timeout: 1s 294 | retries: 0 295 | start_period: 480s 296 | test: ["CMD", "/healthcheck.sh"] 297 | volumes: 298 | - type: volume 299 | source: pghost-data 300 | target: /var/lib/postgresql/data 301 | - type: bind 302 | source: ./pg/docker-entrypoint-initdb.d 303 | target: /docker-entrypoint-initdb.d 304 | read_only: true 305 | - type: bind 306 | source: ./pg/healthcheck.sh 307 | target: /healthcheck.sh 308 | read_only: true 309 | 310 | # _ _ _____ _______ _____ ____ _ ______ 311 | # | \ | | ____|_ _\ \ / / _ \| _ \| |/ / ___| 312 | # | \| | _| | | \ \ /\ / / | | | |_) | ' /\___ \ 313 | # | |\ | |___ | | \ V V /| |_| | _ <| . \ ___) | 314 | # |_| \_|_____| |_| \_/\_/ \___/|_| \_\_|\_\____/ 315 | # 316 | 317 | networks: 318 | database: 319 | elasticsearch: 320 | web-network: 321 | 322 | # __ _____ _ _ _ __ __ _____ ____ 323 | # \ \ / / _ \| | | | | | \/ | ____/ ___| 324 | # \ \ / / | | | | | | | | |\/| | _| \___ \ 325 | # \ V /| |_| | |__| |_| | | | | |___ ___) | 326 | # \_/ \___/|_____\___/|_| |_|_____|____/ 327 | # 328 | 329 | volumes: 330 | api_carton: 331 | cpan: 332 | elasticsearch: 333 | elasticsearch_test: 334 | pghost-data: 335 | metacpan_git_shared: 336 | external: true 337 | -------------------------------------------------------------------------------- /github-meets-cpan/environment.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/metacpan/metacpan-docker/858a06a05ea06639f64ed4ce5c74b9b73743ab52/github-meets-cpan/environment.json -------------------------------------------------------------------------------- /grep.env: -------------------------------------------------------------------------------- 1 | # grep service - development environment 2 | 3 | GREP_SITE_PORT=3001 4 | GREP_PLACKUP_SERVER_ARGS="-E development -R lib,bin" 5 | -------------------------------------------------------------------------------- /localapi_test.env: -------------------------------------------------------------------------------- 1 | NET_ASYNC_HTTP_MAXCONNS=1 2 | COLUMNS=80 3 | ES=elasticsearch_test:9200 4 | ES_TEST=elasticsearch_test:9200 5 | HARNESS_ACTIVE=1 6 | # Instantiate Catalyst models using metacpan_server_testing.conf 7 | METACPAN_SERVER_CONFIG_LOCAL_SUFFIX=testing 8 | MINICPAN=/CPAN 9 | PERL_CARTON_PATH=/carton 10 | MOJO_MODE=testing 11 | -------------------------------------------------------------------------------- /logging.env: -------------------------------------------------------------------------------- 1 | ## The API KEY 2 | HONEYCOMB_WRITE_KEY=XXX 3 | 4 | ## HONEYCOMB_DATASET: 'dev' or 'production' 5 | HONEYCOMB_DATASET=YYY 6 | 7 | ## This is honeycomb magic 8 | ROUTE_URIS=honeycomb://localhost -------------------------------------------------------------------------------- /orepan2/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM ubuntu:14.04 2 | 3 | ENV UPDATED_AT 2014-11-22 4 | 5 | RUN apt-get update 6 | 7 | RUN apt-get install -y \ 8 | curl \ 9 | gcc \ 10 | libcurl4-openssl-dev \ 11 | make 12 | 13 | RUN curl -L http://cpanmin.us | perl - App::cpanminus 14 | 15 | # This is a fix, until this ticket is solved: https://github.com/tokuhirom/OrePAN2/pull/31 16 | RUN cpanm IO::Socket::SSL 17 | 18 | RUN cpanm OrePAN2 19 | -------------------------------------------------------------------------------- /orepan2/README.md: -------------------------------------------------------------------------------- 1 | # OrePAN2 2 | 3 | This is a dockerized version of perl module 4 | [OrePAN2](https://metacpan.org/release/OrePAN2). 5 | 6 | First you need to build docker image: 7 | 8 | docker build --tag orepan2 . 9 | 10 | And then you can use that image to create dakpan structure. 11 | 12 | docker run \ 13 | --rm \ 14 | --volume $(pwd)/darkpan:/darkpan \ 15 | orepan2 \ 16 | orepan2-inject --author LOGIN Test::Whitespaces /darkpan 17 | 18 | This command will download module Test::Whitespaces from the big CPAN and 19 | place in at `$(pwd)/darkpan` on your host machine. Here the sample of what 20 | will be created: 21 | 22 | $ find $(pwd)/darkpan 23 | /Users/bessarabov/darkpan 24 | /Users/bessarabov/darkpan/authors 25 | /Users/bessarabov/darkpan/authors/id 26 | /Users/bessarabov/darkpan/authors/id/L 27 | /Users/bessarabov/darkpan/authors/id/L/LO 28 | /Users/bessarabov/darkpan/authors/id/L/LO/LOGIN 29 | /Users/bessarabov/darkpan/authors/id/L/LO/LOGIN/Test-Whitespaces-1.2.1.tar.gz 30 | /Users/bessarabov/darkpan/modules 31 | /Users/bessarabov/darkpan/modules/02packages.details.txt.gz 32 | /Users/bessarabov/darkpan/orepan2-cache.json 33 | 34 | For more examples of using orepan2-inject see the 35 | [docs](https://metacpan.org/pod/distribution/OrePAN2/script/orepan2-inject). 36 | -------------------------------------------------------------------------------- /pg/Dockerfile: -------------------------------------------------------------------------------- 1 | ARG PG_TAG=9.6-alpine 2 | 3 | FROM postgres:${PG_TAG} 4 | 5 | VOLUME /logs 6 | -------------------------------------------------------------------------------- /pg/docker-entrypoint-initdb.d/100-roles.sql: -------------------------------------------------------------------------------- 1 | CREATE ROLE metacpan WITH LOGIN PASSWORD 't00lchain'; 2 | CREATE ROLE "metacpan-api" WITH LOGIN; 3 | 4 | -- make things easier for when we're poking around from inside the container 5 | CREATE USER root; 6 | -------------------------------------------------------------------------------- /pg/docker-entrypoint-initdb.d/200-minion.sql: -------------------------------------------------------------------------------- 1 | CREATE DATABASE minion_queue OWNER metacpan 2 | -------------------------------------------------------------------------------- /pg/healthcheck.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -eo pipefail 3 | 4 | host="$(hostname || echo '127.0.0.1')" 5 | user="${POSTGRES_USER:-postgres}" 6 | db="${POSTGRES_DB:-$POSTGRES_USER}" 7 | export PGPASSWORD="${POSTGRES_PASSWORD:-}" 8 | 9 | args=( 10 | # force postgres to not use the local unix socket (test "external" connectibility) 11 | --host "$host" 12 | --username "$user" 13 | --dbname "$db" 14 | --quiet --no-align --tuples-only 15 | ) 16 | 17 | if select="$(echo 'SELECT 1' | psql "${args[@]}")" && [ "$select" = '1' ]; then 18 | exit 0 19 | fi 20 | 21 | exit 1 22 | --------------------------------------------------------------------------------