├── .gitignore ├── .ruby-gemset ├── .ruby-version ├── Gemfile ├── Gemfile.lock ├── LICENCE ├── README.mdown ├── TODO.mdown ├── build_and_inspect_tester ├── build_and_inspect_ui ├── build_and_run_tester ├── build_and_run_ui ├── config.ru ├── dev ├── .gitignore ├── blobs │ └── .gitkeep ├── conf.d │ └── .gitkeep ├── custom │ ├── engines │ │ └── .gitkeep │ └── transports │ │ └── .gitkeep ├── logs │ └── .gitkeep ├── tests │ └── .gitkeep └── uphold.example.yml ├── dockerfiles ├── tester │ └── Dockerfile └── ui │ └── Dockerfile ├── docs └── screenshot_ui.png ├── environment.rb ├── lib ├── config.rb ├── engine.rb ├── engines │ ├── mongodb.rb │ ├── mysql.rb │ └── postgresql.rb ├── helpers │ ├── command.rb │ ├── compression.rb │ ├── logger.rb │ └── sockets.rb ├── runner.rb ├── tests.rb ├── transport.rb └── transports │ ├── local.rb │ └── s3.rb ├── public ├── font │ ├── Roboto-Regular-webfont.eot │ ├── Roboto-Regular-webfont.svg │ ├── Roboto-Regular-webfont.ttf │ └── Roboto-Regular-webfont.woff ├── images │ ├── bad.png │ ├── bad_engine.png │ ├── bad_tests.png │ ├── bad_transport.png │ ├── ok.png │ ├── ok_no_test.png │ ├── run.png │ └── running.gif └── stylesheets │ ├── normalize.min.css │ └── uphold.css ├── tester.rb ├── ui.rb └── views ├── index.erb ├── layout.erb └── log.erb /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/forward3d/uphold/40a252df58de35e78e4173beabd4f3decd94c67c/.gitignore -------------------------------------------------------------------------------- /.ruby-gemset: -------------------------------------------------------------------------------- 1 | uphold 2 | -------------------------------------------------------------------------------- /.ruby-version: -------------------------------------------------------------------------------- 1 | 2.3.0 2 | -------------------------------------------------------------------------------- /Gemfile: -------------------------------------------------------------------------------- 1 | source 'https://rubygems.org' 2 | 3 | gem 'sequel', '~> 4.30' 4 | gem 'docker-api', '~> 1.33.2' 5 | 6 | group :tester do 7 | # core 8 | gem 'rubyzip', '~> 1.1', '>= 1.1.7' 9 | gem 'minitest', '~> 5.8', '>= 5.8.3' 10 | 11 | # engines 12 | gem 'pg', '~> 0.18.4' 13 | gem 'mysql2', '~> 0.4.6' 14 | gem 'mongoid', '~> 5.0', '>= 5.0.2' 15 | gem 'rethinkdb', '~> 2.2', '>= 2.2.0.2' 16 | gem 'sqlite3', '~> 1.3', '>= 1.3.11' 17 | 18 | # transports 19 | gem 'aws-sdk', '~> 2.2', '>= 2.2.12' 20 | end 21 | 22 | group :ui do 23 | gem 'sinatra', '~> 1.4', '>= 1.4.6' 24 | gem 'thin', '~> 1.6', '>= 1.6.4' 25 | end 26 | 27 | group :development do 28 | gem 'git-up' 29 | end 30 | -------------------------------------------------------------------------------- /Gemfile.lock: -------------------------------------------------------------------------------- 1 | GEM 2 | remote: https://rubygems.org/ 3 | specs: 4 | activemodel (4.2.5) 5 | activesupport (= 4.2.5) 6 | builder (~> 3.1) 7 | activesupport (4.2.5) 8 | i18n (~> 0.7) 9 | json (~> 1.7, >= 1.7.7) 10 | minitest (~> 5.1) 11 | thread_safe (~> 0.3, >= 0.3.4) 12 | tzinfo (~> 1.1) 13 | aws-sdk (2.2.12) 14 | aws-sdk-resources (= 2.2.12) 15 | aws-sdk-core (2.2.12) 16 | jmespath (~> 1.0) 17 | aws-sdk-resources (2.2.12) 18 | aws-sdk-core (= 2.2.12) 19 | bson (4.0.0) 20 | builder (3.2.2) 21 | colored (1.2) 22 | daemons (1.2.3) 23 | diff-lcs (1.2.5) 24 | docker-api (1.33.2) 25 | excon (>= 0.38.0) 26 | json 27 | eventmachine (1.0.9.1) 28 | excon (0.55.0) 29 | git-up (0.5.12) 30 | colored (>= 1.2) 31 | grit 32 | grit (2.5.0) 33 | diff-lcs (~> 1.1) 34 | mime-types (~> 1.15) 35 | posix-spawn (~> 0.3.6) 36 | i18n (0.7.0) 37 | jmespath (1.1.3) 38 | json (1.8.3) 39 | mime-types (1.25.1) 40 | minitest (5.8.3) 41 | mongo (2.2.1) 42 | bson (~> 4.0) 43 | mongoid (5.0.2) 44 | activemodel (~> 4.0) 45 | mongo (~> 2.1) 46 | origin (~> 2.1) 47 | tzinfo (>= 0.3.37) 48 | mysql2 (0.4.2) 49 | origin (2.1.1) 50 | pg (0.18.4) 51 | posix-spawn (0.3.11) 52 | rack (1.6.4) 53 | rack-protection (1.5.3) 54 | rack 55 | rethinkdb (2.2.0.2) 56 | json 57 | rubyzip (1.1.7) 58 | sequel (4.30.0) 59 | sinatra (1.4.6) 60 | rack (~> 1.4) 61 | rack-protection (~> 1.4) 62 | tilt (>= 1.3, < 3) 63 | sqlite3 (1.3.11) 64 | thin (1.6.4) 65 | daemons (~> 1.0, >= 1.0.9) 66 | eventmachine (~> 1.0, >= 1.0.4) 67 | rack (~> 1.0) 68 | thread_safe (0.3.5) 69 | tilt (2.0.2) 70 | tzinfo (1.2.2) 71 | thread_safe (~> 0.1) 72 | 73 | PLATFORMS 74 | ruby 75 | 76 | DEPENDENCIES 77 | aws-sdk (~> 2.2, >= 2.2.12) 78 | docker-api (~> 1.25) 79 | git-up 80 | minitest (~> 5.8, >= 5.8.3) 81 | mongoid (~> 5.0, >= 5.0.2) 82 | mysql2 (~> 0.4.2) 83 | pg (~> 0.18.4) 84 | rethinkdb (~> 2.2, >= 2.2.0.2) 85 | rubyzip (~> 1.1, >= 1.1.7) 86 | sequel (~> 4.30) 87 | sinatra (~> 1.4, >= 1.4.6) 88 | sqlite3 (~> 1.3, >= 1.3.11) 89 | thin (~> 1.6, >= 1.6.4) 90 | 91 | BUNDLED WITH 92 | 1.11.2 93 | -------------------------------------------------------------------------------- /LICENCE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2016 Lloyd Pick 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.mdown: -------------------------------------------------------------------------------- 1 | # Uphold 2 | 3 | Schrödinger's Backup: *"The condition of any backup is unknown until a restore is attempted"* 4 | 5 | So you're backing up your databases, but are you regularly checking that the backups are actually useable? Uphold will help you automatically test them by downloading the backup, decompressing, loading and then running programmatic tests against it that you define to make sure they really have what you need. 6 | 7 | ![Uphold UI](https://raw.githubusercontent.com/forward3d/uphold/master/docs/screenshot_ui.png) 8 | 9 | ## Table of Contents 10 | - [Preface](#preface) 11 | - [Prerequisites](#prerequisites) 12 | - [How does it work?](#how-does-it-work) 13 | - [Installation](#installation) 14 | - [Configuration](#configuration) 15 | - [`uphold.yml` Example](#upholdyml-example) 16 | - [`/etc/uphold/conf.d` Example](#etcupholdconfd-example) 17 | - [Transports](#transports) 18 | - [Generic Transport Parameters](#generic-transport-parameters) 19 | - [S3 (type: `s3`)](#s3-type-s3) 20 | - [S3 Transport Example](#s3-transport-example) 21 | - [Local File (type: `local`)](#local-file-type-local) 22 | - [Local File Example](#local-file-example) 23 | - [Engines](#engines) 24 | - [Generic Parameters](#generic-parameters) 25 | - [MongoDB (type: `mongodb`)](#mongodb-type-mongodb) 26 | - [Full `mongodb` Engine Example](#full-mongodb-engine-example) 27 | - [MySQL (type: `mysql`)](#mysql-type-mysql) 28 | - [Full `mysql` Engine Example](#full-mysql-engine-example) 29 | - [PostgreSQL (type: `postgresql`)](#postgresql-type-postgresql) 30 | - [Full `postgresql` Engine Example](#full-postgresql-engine-example) 31 | - [Tests](#tests) 32 | - [Example Test](#example-test) 33 | - [Running](#running) 34 | - [Scheduling](#scheduling) 35 | - [API](#api) 36 | - [`GET /api/1.0/backups/config-name-here`](#get-api10backupsconfig-name-here) 37 | - [`GET /api/1.0/backups/config-name-here/latest`](#get-api10backupsconfig-name-herelatest) 38 | - [`POST /api/1.0/backup`](#post-api10backup) 39 | - [Development](#development) 40 | 41 | ### Preface 42 | 43 | This project is very new and subsequently very beta so contributions and pulls are very much welcomed. We have a [TODO](TODO.mdown) file with things that we know about that would be awesome if worked on. 44 | 45 | ### Prerequisites 46 | 47 | * Backups 48 | * Docker (>= v1.3.*) with the ability to talk to the Docker API 49 | 50 | ### How does it work? 51 | 52 | In order to make the processes are repeatable as possible all the code and databases are run inside single process Docker containers. There are currently three types of container, the `ui`, the `tester` and the database itself. Each triggers the next... 53 | 54 | uphold-ui 55 | \ 56 | -> uphold-tester 57 | \ 58 | -> engine-container 59 | 60 | This way each time the process is run, the containers are fresh and new, they hold no state. So each time the database is imported into a cold database. 61 | 62 | The output of each process run is a log and a state file and these are stored in `/var/log/uphold` by default. The UI reads these files to display the state of the runs occurring, no other state is stored in the system. 63 | 64 | /var/log/uphold 65 | /var/log/uphold/1453489253_my_db_backup.log 66 | /var/log/uphold/1453489253_my_db_backup_ok 67 | 68 | This is the output of a backup run for 'my_db_backup' that was started at `1453489253` unix epoch time. The log file contains the full output of the run, and the state file is an empty file, it's name shows the status of the run... 69 | 70 | * `ok` Backup was declared good, was transported, loaded and tested successfully 71 | * `ok_no_test` Backup was successfully transported and loaded into the DB, but there were no tests to run 72 | * `bad_transport` Transport failed 73 | * `bad_engine` Container did not open it's port in a timely manner 74 | * `bad_tests` At least one of the programmatic tests failed 75 | * `bad` An error occurred either in transport or loading into the db engine 76 | 77 | Logs are not automatically rotated or removed, it is left up to you to decide how long you want to keep them. Once they become compressed, they will disappear from the UI. The same goes for the exited Docker containers of 'uphold-tester', they are left on the system incase you wish to inspect them. The database containers however are wiped after they are used. 78 | 79 | ### Installation 80 | 81 | Most of the installation goes around configuring the tool, you must create the following directory structure on the machine you want to run Uphold on... 82 | 83 | /etc/uphold/ 84 | /etc/uphold/conf.d/ 85 | /etc/uphold/engines/ 86 | /etc/uphold/transports/ 87 | /etc/uphold/tests/ 88 | /var/log/uphold 89 | 90 | ### Configuration 91 | 92 | Create a global config in `/etc/uphold/uphold.yml` (even if you leave it empty), the settings inside are... 93 | 94 | * `log_level` (default: `DEBUG`) 95 | * You can decrease the verbosity of the logging by changing this to `INFO`, but not recommended 96 | * `config_path` (default: `/etc/uphold`) 97 | * Generally only overridden in development on OSX when you need to mount your own src directory 98 | * `docker_log_path` (default: `/var/log/uphold`) 99 | * Generally only overridden in development on OSX when you need to mount your own src directory 100 | * `docker_url` (default: `unix:///var/run/docker.sock`) 101 | * If you connect to Docker via a TCP socket instead of a Unix one, then you would supply `tcp://example.com:5422` instead (untested) 102 | * `docker_container` (default: `forward3d/uphold-tester`) 103 | * If you need to customize the docker container and use a different one, you can override it here 104 | * `docker_tag` (default: `latest`) 105 | * Can override the Docker container tag if you want to run from a specific version 106 | * `docker_mounts` (default: `none`) 107 | * If your backups exist on the host machine and you want to use the `local` transport, the folders they exist in need to be mounted into the container. You can specify them here as a YAML array of directories. They will be mounted at the same location inside the container 108 | * `ui_datetime` (default: `%F %T %Z`) 109 | * Overrides the strftime used by the UI to display the outcomes, useful if you want to make it smaller or add info 110 | 111 | If you change the global config you will need to restart the UI docker container, as some settings are only read at launch time. 112 | 113 | #### `uphold.yml` Example 114 | 115 | log_level: DEBUG 116 | config_path: /etc/uphold 117 | docker_log_path: /var/log/uphold 118 | docker_url: unix:///var/run/docker.sock 119 | docker_container: forward3d/uphold-tester 120 | docker_tag: latest 121 | docker_mounts: 122 | - /var/my_backups 123 | - /var/my_other_backups 124 | 125 | #### `/etc/uphold/conf.d` Example 126 | 127 | Each config is in YAML format, and is constructed of a transport, an engine and tests. In your `/etc/uphold/conf.d` directory simply create as many YAML files as you need, one per backup. Configs in this directory are re-read, so you don't need to restart the UI container if you add new ones. 128 | 129 | enabled: true 130 | name: s3-mongo 131 | engine: 132 | type: mongodb 133 | settings: 134 | timeout: 10 135 | database: your_db_name 136 | transport: 137 | type: s3 138 | settings: 139 | region: us-west-2 140 | access_key_id: your-access-key-id 141 | secret_access_key: your-secret-access-key 142 | bucket: your-backups 143 | path: mongodb/systemx/{date} 144 | filename: mongodb.tar 145 | date_format: '%Y.%m.%d' 146 | date_offset: 0 147 | folder_within: mongodb/databases/MongoDB 148 | tests: 149 | - test_structure.rb 150 | - test_data_integrity.rb 151 | 152 | * `enabled` 153 | * `true` or `false`, allows you to disable a config if needs be 154 | * `name` 155 | * Just so that if it's referenced anywhere, you have a nicer name 156 | 157 | See the sections below for how to configure Engines, Transports and Tests. 158 | 159 | #### Transports 160 | 161 | Transports are how you retrieve the backup file itself. They are also responsible for decompressing the file, the code supports nested compression (compressed files within compressed files). Currently implemented transports are... 162 | 163 | * S3 164 | * Local file 165 | 166 | Custom transports can also be loaded at runtime if they are placed in `/etc/uphold/transports`. If you need extra rubygems installed you will need to create a new Dockerfile with the base set to `uphold-tester` and then override the Gemfile and re-bundle. Then adjust your `uphold.yml` to use your new container. 167 | 168 | ##### Generic Transport Parameters 169 | 170 | Transports all inherit these generic parameters... 171 | 172 | * `path` 173 | * This is the path to the folder that the backup is inside, if it contains a date replace it with `{date}`, eg. `/var/backups/2016-01-21` would be `/var/backups/{date}` 174 | * `filename` 175 | * The filename of the backup file, if it contains a date replace it with `{date}`, eg. `mongodb-2016-01-21.tar` would be `mongodb-{date}.tar` 176 | * `date_format` (default: `%Y-%m-%d`) 177 | * If your filename or path contains a date, supply it's format here 178 | * `date_offset` (default: `0`) 179 | * When using dates the code starts at `Date.today` and then subtracts this number, so for checking a backup that exists for yesterday, you would enter `1` 180 | * `folder_within` 181 | * Once your backup has been decompressed it may have folders inside, if so, you need to provide where the last directory is, this generally can't be programmatically found as some database backups may contain folders in their own structure. 182 | 183 | ##### S3 (type: `s3`) 184 | 185 | The S3 transport allows you to pull your backup files from a bucket in S3. It has it's own extra settings... 186 | 187 | * `region` 188 | * Provide the region that your S3 bucket resides in (eg. `us-west-2`) 189 | * `access_key_id` 190 | * AWS access key that has privileges to read from the specified bucket 191 | * `secret_access_key` 192 | * AWS secret access key that has privileges to read from the specified bucket 193 | 194 | Paths do not need to be complete with S3, as it provides globbing capability. So if you had a path like this... 195 | 196 | my-service-backups/mongodb/2016.01.21.00.36.03/mongodb.tar 197 | 198 | Theres no realistic way for us to re-create that date, so you would do this instead... 199 | 200 | path: my-service-backups/mongodb/{date} 201 | filename: mongodb.tar 202 | date_format: '%Y.%m.%d' 203 | 204 | As the `path` is sent to the S3 API as a prefix, it will match all folders, the code then picks the first one it matches correctly. So be aware that not being specific enough with the `date_format` could cause the wrong backup to be tested. 205 | 206 | ###### S3 Transport Example 207 | 208 | transport: 209 | type: s3 210 | settings: 211 | region: us-west-2 212 | access_key_id: your-access-key-id 213 | secret_access_key: your-secret-access-key 214 | bucket: your-backups 215 | path: mongodb/systemx/{date} 216 | filename: mongodb.tar 217 | date_format: '%Y.%m.%d' 218 | date_offset: 0 219 | folder_within: mongodb/databases/MongoDB 220 | 221 | ##### Local File (type: `local`) 222 | 223 | The local transport allows you to pull your backup files from the same machine that is running the Docker container. Be aware, since this code runs within a container you will need to add the volume that contains the backup when starting up. We auto-mount `/var/uphold` to the same place within the container to reduce confusion. 224 | 225 | It has no extra parameters and only uses the generic ones, `filename`, `path` and `folder_within` 226 | 227 | ###### Local File Example 228 | 229 | transport: 230 | type: local 231 | settings: 232 | path: /var/uphold/mongodb 233 | filename: mongodb.tar 234 | folder_within: mongodb/databases/MongoDB 235 | 236 | #### Engines 237 | 238 | Engines are used to load the backup that was retrieved by the transport into the database. Databases are started inside fresh docker containers each time so no installation is required. Currently supported databases are... 239 | 240 | * MongoDB 241 | * MySQL 242 | * PostgreSQL 243 | 244 | Custom engines can also be loaded at runtime if they are placed in `/etc/uphold/engines` 245 | 246 | ##### Generic Parameters 247 | 248 | Engines all inherit these generic parameters, but are usually significantly easier to configure when compared to transports... 249 | 250 | * `type` 251 | * The name of the engine class you want to use (eg. `mongodb`) 252 | * `database` 253 | * The name of the database you want to recover, as your backup may contain multiple 254 | * `port` 255 | * The port number that the database will run on (engine will provide a sane default) 256 | * `docker_image` 257 | * The name of the Docker container (engine will provide a sane default) 258 | * `docker_tag` 259 | * The tag of the Docker container (engine will provide a sane default) 260 | * `timeout` (default: `10`) 261 | * The number of seconds you will give the container to respond on it's TCP port. You may need to increase this if you start many backup tests at the same time. 262 | 263 | ##### MongoDB (type: `mongodb`) 264 | 265 | Unless you need to change any of the defaults, a standard configuration for MongoDB will look quite small. 266 | 267 | engine: 268 | type: mongodb 269 | settings: 270 | database: your_db_name 271 | 272 | ###### Full `mongodb` Engine Example 273 | 274 | engine: 275 | type: mongodb 276 | settings: 277 | database: your_db_name 278 | docker_image: mongo 279 | docker_tag: 3.2.1 280 | port: 27017 281 | 282 | ##### MySQL (type: `mysql`) 283 | 284 | engine: 285 | type: mysql 286 | settings: 287 | database: your_database_name 288 | sql_file: your_sql_file.sql 289 | 290 | ###### Full `mysql` Engine Example 291 | 292 | engine: 293 | type: mysql 294 | settings: 295 | database: your_database_name 296 | docker_image: mariadb 297 | docker_tag: 5.5.42 298 | port: 3306 299 | sql_file: MySQL.sql 300 | 301 | ##### PostgreSQL (type: `postgresql`) 302 | 303 | engine: 304 | type: postgresql 305 | settings: 306 | database: your_database_name 307 | sql_file: your_sql_file.sql 308 | 309 | ###### Full `postgresql` Engine Example 310 | 311 | The `database` also becomes your username for when you run the tests. 312 | 313 | engine: 314 | type: postgresql 315 | settings: 316 | database: your_database_name 317 | docker_image: postgres 318 | docker_tag: 9.5.0 319 | port: 5432 320 | sql_file: PostgreSQL.sql 321 | 322 | #### Tests 323 | 324 | Tests are the final step in configuration. They are how you validate that the data contained within your backup is really what you want, and that your backup is operating correctly. Tests are written in Ruby using Minitest, this gives you the most flexibility in writing tests programmatically as it supports both Unit & Spec tests. To configure a test you simply provide an array of ruby files you want to run... 325 | 326 | tests: 327 | - test_structure.rb 328 | - test_data_integrity.rb 329 | 330 | Tests should be placed within the `/etc/uphold/tests` directory, all files inside will be volume mounted into the container so if you need extra files they are available to you. 331 | 332 | ##### Example Test 333 | 334 | We need to establish a connection to the database, and the values will not be known in advance. So they will be provided to you by environmental variables `UPHOLD_IP`, `UPHOLD_PORT` and `UPHOLD_DB`. You must use these when connecting to your database. 335 | 336 | require 'minitest/autorun' 337 | require 'mongoid' 338 | 339 | class TestClients < Minitest::Test 340 | Mongo::Logger.logger.level = Logger::FATAL 341 | @@mongo = Mongo::Client.new("mongodb://#{ENV['UPHOLD_IP']}:#{ENV['UPHOLD_PORT']}/#{ENV['UPHOLD_DB']}") 342 | 343 | def test_that_we_can_talk_to_mongo 344 | assert_equal 1, @@mongo.collections.count 345 | end 346 | end 347 | 348 | Obviously this is just a simple test, but you can write any number of tests you like. All must pass in order for the backup to be considered 'good'. 349 | 350 | ### Running 351 | 352 | Once you have finished your configuration, to get the system running you only need to start the Docker container called 'uphold-ui'. 353 | 354 | docker pull forward3d/uphold-ui:latest 355 | docker run \ 356 | --rm \ 357 | -p 8079:8079 \ 358 | -v /var/run/docker.sock:/var/run/docker.sock \ 359 | -v /etc/uphold:/etc/uphold \ 360 | -v /var/log/uphold:/var/log/uphold \ 361 | forward3d/uphold-ui:latest 362 | 363 | * You must make sure that you mount in the `docker.sock` from wherever it resides. 364 | * Feel free to change the port number if you don't want it to start up on port 8079. 365 | * Mount in the config and log directories as otherwise it can't read your configuration. 366 | 367 | Once the container is live you can browse to it, see all previous available runs for a backup and the states the ended in. You can manually start a backup test from here if you want to. 368 | 369 | ### Scheduling 370 | 371 | No option to schedule backup runs exists at present. Until one exists you can use the API to trigger backup runs to start. This way you can schedule however you like, crontab, notifier or any other service capable of sending a POST. 372 | 373 | ### API 374 | 375 | #### `GET /api/1.0/backups/config-name-here` 376 | 377 | This will return all the available backup runs for the config name provided in JSON format... 378 | 379 | [ 380 | { 381 | "epoch": 1453921377, 382 | "state": "ok", 383 | "filename": "1453921377_s3-mongo.log" 384 | }, 385 | { 386 | "epoch": 1453909916, 387 | "state": "ok", 388 | "filename": "1453909916_s3-mongo.log" 389 | } 390 | ] 391 | 392 | #### `GET /api/1.0/backups/config-name-here/latest` 393 | 394 | This will return a plain text string of the state of the last backup run for the config name provided. If no runs were available, it will return `none` 395 | 396 | #### `POST /api/1.0/backup` 397 | 398 | You must pass the name of the config you want to trigger in a form field called `name`. It will then start the run and return `200`. An example of how to trigger a backup run for the config named `s3-mongo`... 399 | 400 | curl --data "name=s3-mongo" http://ip.of.your.container/api/1.0/backup 401 | 402 | ### Development 403 | 404 | To aid with development there is a helper script in the root directory called `build_and_run` and `build_and_inspect` which will build or inspect the Dockerfile and then run it using some default options. Since otherwise testing is a bit of a nightmare when trying to talk to containers on your local machine. Various folders from within the project directory will be auto-mounted into the container... 405 | 406 | * `dev/uphold.yml` -> `/etc/uphold/uphold.yml` 407 | * `dev/conf.d` -> `/etc/uphold/conf.d` 408 | * `dev/tests` -> `/etc/uphold/tests` 409 | * `dev/custom/engines` -> `/etc/uphold/engines` 410 | * `dev/custom/transports` -> `/etc/uphold/transports` 411 | * `dev/blobs` -> `/var/backups` 412 | 413 | Remember to place a `uphold.yml` config of your own in the `dev/config` directory. 414 | -------------------------------------------------------------------------------- /TODO.mdown: -------------------------------------------------------------------------------- 1 | # TODO 2 | 3 | * Tests, tests and more tests 4 | * Create deb for easier installation 5 | * Example log rotation script 6 | * Proper error handling for S3 transport 7 | * Auto detect the SQL file for the MySQL engine 8 | * Allow the user to specify the collections to import through the MongoDB engine 9 | * Move all the code to start containers into a helper to DRY it up 10 | * API calls should actually catch some errors and respond appropriately 11 | * Config names should be checked that they are safe to be used in URLs 12 | * Refactor the way that the UI collects up the logs, it's slow and cumbersome 13 | * Create an upstart/systemd script to start the uphold-ui container with the correct volume mounts 14 | * Some sort of authentication 15 | * SQLite support 16 | -------------------------------------------------------------------------------- /build_and_inspect_tester: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | docker build --tag="uphold-tester:dockerfile" --file="dockerfiles/tester/Dockerfile" . 4 | docker run \ 5 | -it \ 6 | --entrypoint=/bin/bash \ 7 | -v /var/run/docker.sock:/var/run/docker.sock \ 8 | -v `pwd`/dev:/etc/uphold \ 9 | -v `pwd`/dev/logs:/var/log/uphold \ 10 | uphold-tester:dockerfile 11 | -------------------------------------------------------------------------------- /build_and_inspect_ui: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | docker build --tag="uphold-ui:dockerfile" --file="dockerfiles/ui/Dockerfile" . 4 | docker run \ 5 | -it \ 6 | --entrypoint=/bin/bash \ 7 | -v /var/run/docker.sock:/var/run/docker.sock \ 8 | -v `pwd`/dev:/etc/uphold \ 9 | -v `pwd`/dev/logs:/var/log/uphold \ 10 | uphold-ui:dockerfile 11 | -------------------------------------------------------------------------------- /build_and_run_tester: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | docker build --tag="uphold-tester:dockerfile" --file="dockerfiles/tester/Dockerfile" . 4 | docker run \ 5 | -it \ 6 | -v /var/run/docker.sock:/var/run/docker.sock \ 7 | -v `pwd`/dev/logs:/var/log/uphold \ 8 | -v `pwd`/dev:/etc/uphold \ 9 | uphold-tester:dockerfile \ 10 | s3-mongo.yml 11 | 12 | if [ $? -eq 0 ]; then 13 | echo Success 14 | else 15 | echo Failed 16 | fi 17 | -------------------------------------------------------------------------------- /build_and_run_ui: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | docker build --tag="uphold-tester:dockerfile" --file="dockerfiles/tester/Dockerfile" . 4 | docker build --tag="uphold-ui:dockerfile" --file="dockerfiles/ui/Dockerfile" . 5 | 6 | docker run \ 7 | -it \ 8 | --rm \ 9 | -p 8079:8079 \ 10 | -v /var/run/docker.sock:/var/run/docker.sock \ 11 | -v `pwd`/dev:/etc/uphold \ 12 | -v `pwd`/dev/logs:/var/log/uphold \ 13 | uphold-ui:dockerfile 14 | -------------------------------------------------------------------------------- /config.ru: -------------------------------------------------------------------------------- 1 | $:.unshift(File.dirname(__FILE__)) 2 | 3 | require 'ui' 4 | run Uphold::Ui 5 | -------------------------------------------------------------------------------- /dev/.gitignore: -------------------------------------------------------------------------------- 1 | blobs/*.tar 2 | blobs/*.gz 3 | blobs/*.zip 4 | conf.d/*.yml 5 | custom/engines/*.rb 6 | custom/transports/*.rb 7 | tests/*.rb 8 | logs/*.log 9 | logs/*_ok 10 | logs/*_ok* 11 | logs/*_bad* 12 | uphold.yml 13 | -------------------------------------------------------------------------------- /dev/blobs/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/forward3d/uphold/40a252df58de35e78e4173beabd4f3decd94c67c/dev/blobs/.gitkeep -------------------------------------------------------------------------------- /dev/conf.d/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/forward3d/uphold/40a252df58de35e78e4173beabd4f3decd94c67c/dev/conf.d/.gitkeep -------------------------------------------------------------------------------- /dev/custom/engines/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/forward3d/uphold/40a252df58de35e78e4173beabd4f3decd94c67c/dev/custom/engines/.gitkeep -------------------------------------------------------------------------------- /dev/custom/transports/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/forward3d/uphold/40a252df58de35e78e4173beabd4f3decd94c67c/dev/custom/transports/.gitkeep -------------------------------------------------------------------------------- /dev/logs/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/forward3d/uphold/40a252df58de35e78e4173beabd4f3decd94c67c/dev/logs/.gitkeep -------------------------------------------------------------------------------- /dev/tests/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/forward3d/uphold/40a252df58de35e78e4173beabd4f3decd94c67c/dev/tests/.gitkeep -------------------------------------------------------------------------------- /dev/uphold.example.yml: -------------------------------------------------------------------------------- 1 | log_level: DEBUG 2 | config_path: /Users/lloyd/code/f3d/uphold/dev 3 | docker_url: unix:///var/run/docker.sock 4 | docker_container: uphold-tester 5 | docker_tag: dockerfile 6 | docker_log_path: /Users/lloyd/code/f3d/uphold/dev/logs 7 | ui_datetime: '%F %T' 8 | docker_mounts: 9 | - /var/backups 10 | - /Users/lloyd/code/f3d/uphold/dev/blobs 11 | -------------------------------------------------------------------------------- /dockerfiles/tester/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM ruby:2.3-slim 2 | 3 | RUN apt-get update 4 | RUN apt-get -y install libmysqlclient-dev mysql-client libpq-dev libsqlite3-dev mongodb-clients postgresql-client 5 | 6 | WORKDIR /opt/uphold 7 | COPY Gemfile /opt/uphold/Gemfile 8 | COPY Gemfile.lock /opt/uphold/Gemfile.lock 9 | 10 | RUN \ 11 | apt-get install -y build-essential && \ 12 | bundle install --without ui development && \ 13 | apt-get remove -y build-essential && \ 14 | apt-get autoremove -y && apt-get clean && \ 15 | rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* 16 | 17 | COPY lib /opt/uphold/lib 18 | COPY environment.rb tester.rb /opt/uphold/ 19 | 20 | ENTRYPOINT ["ruby", "tester.rb"] 21 | -------------------------------------------------------------------------------- /dockerfiles/ui/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM ruby:2.3-slim 2 | 3 | RUN apt-get update 4 | 5 | WORKDIR /opt/uphold 6 | COPY Gemfile /opt/uphold/Gemfile 7 | COPY Gemfile.lock /opt/uphold/Gemfile.lock 8 | 9 | RUN \ 10 | apt-get install -y build-essential && \ 11 | bundle install --without tester development && \ 12 | apt-get remove -y build-essential && \ 13 | apt-get autoremove -y && apt-get clean && \ 14 | rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* 15 | 16 | COPY lib /opt/uphold/lib 17 | COPY public /opt/uphold/public 18 | COPY views /opt/uphold/views 19 | COPY config.ru environment.rb ui.rb /opt/uphold/ 20 | 21 | EXPOSE 8079 22 | CMD ["bundle", "exec", "rackup", "config.ru", "-p", "8079", "-s", "thin", "-o", "0.0.0.0"] 23 | -------------------------------------------------------------------------------- /docs/screenshot_ui.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/forward3d/uphold/40a252df58de35e78e4173beabd4f3decd94c67c/docs/screenshot_ui.png -------------------------------------------------------------------------------- /environment.rb: -------------------------------------------------------------------------------- 1 | module Uphold 2 | require 'rubygems' 3 | require 'rubygems/package' 4 | require 'bundler/setup' 5 | Bundler.require(:default) 6 | 7 | ROOT = File.dirname(File.expand_path(__FILE__)) 8 | Dir["#{ROOT}/lib/helpers/*.rb"].sort.each { |file| require file } 9 | Dir["#{ROOT}/lib/*.rb"].sort.each { |file| require file } 10 | 11 | UPHOLD = Config.load_global 12 | 13 | include Logging 14 | logger.level = Logger.const_get(UPHOLD[:log_level]) 15 | logger.info 'Starting Uphold' 16 | 17 | Docker.url = UPHOLD[:docker_url] 18 | logger.debug "Docker URL - '#{Docker.url}'" 19 | end 20 | -------------------------------------------------------------------------------- /lib/config.rb: -------------------------------------------------------------------------------- 1 | module Uphold 2 | class Config 3 | require 'yaml' 4 | include Logging 5 | PREFIX = '/etc/uphold' 6 | 7 | attr_reader :yaml 8 | 9 | def initialize(config) 10 | fail unless config 11 | yaml = YAML.load_file(File.join(PREFIX, 'conf.d', config)) 12 | yaml.merge!(file: File.basename(config, '.yml')) 13 | @yaml = Config.deep_convert(yaml) 14 | fail unless valid? 15 | logger.debug "Loaded config '#{@yaml[:name]}' from '#{config}'" 16 | @yaml[:tests] ||= [] 17 | @yaml = supplement 18 | end 19 | 20 | def valid? 21 | valid = true 22 | valid = false unless Config.engines.any? { |e| e[:name] == @yaml[:engine][:type] } 23 | valid = false unless Config.transports.any? { |e| e[:name] == @yaml[:transport][:type] } 24 | valid 25 | end 26 | 27 | def supplement 28 | @yaml[:engine][:klass] = Config.engines.find { |e| e[:name] == @yaml[:engine][:type] }[:klass] 29 | @yaml[:transport][:klass] = Config.transports.find { |e| e[:name] == @yaml[:transport][:type] }[:klass] 30 | @yaml 31 | end 32 | 33 | def self.load_configs 34 | Dir[File.join(PREFIX, 'conf.d', '*.yml')].sort.map do |file| 35 | new(File.basename(file)).yaml 36 | end 37 | end 38 | 39 | def self.load_global 40 | yaml = YAML.load_file(File.join(PREFIX, 'uphold.yml')) 41 | yaml = deep_convert(yaml) 42 | yaml[:log_level] ||= 'DEBUG' 43 | yaml[:docker_url] ||= 'unix:///var/run/docker.sock' 44 | yaml[:docker_container] ||= 'forward3d/uphold-tester' 45 | yaml[:docker_tag] ||= 'latest' 46 | yaml[:docker_mounts] ||= [] 47 | yaml[:config_path] ||= '/etc/uphold' 48 | yaml[:docker_log_path] ||= '/var/log/uphold' 49 | yaml[:ui_datetime] ||= '%F %T %Z' 50 | yaml 51 | end 52 | 53 | def self.load_engines 54 | [Dir["#{ROOT}/lib/engines/*.rb"], Dir[File.join(PREFIX, 'engines', '*.rb')]].flatten.uniq.sort.each do |file| 55 | require file 56 | basename = File.basename(file, '.rb') 57 | add_engine name: basename, klass: Object.const_get("Uphold::Engines::#{File.basename(file, '.rb').capitalize}") 58 | end 59 | end 60 | 61 | def self.engines 62 | @engines ||= [] 63 | end 64 | 65 | def self.add_engine(engine) 66 | list = engines 67 | list << engine 68 | logger.debug "Loaded engine #{engine[:klass]}" 69 | list.uniq! { |e| e[:name] } 70 | end 71 | 72 | def self.load_transports 73 | [Dir["#{ROOT}/lib/transports/*.rb"], Dir[File.join(PREFIX, 'transports', '*.rb')]].flatten.uniq.sort.each do |file| 74 | require file 75 | basename = File.basename(file, '.rb') 76 | add_transport name: basename, klass: Object.const_get("Uphold::Transports::#{File.basename(file, '.rb').capitalize}") 77 | end 78 | end 79 | 80 | def self.add_transport(transport) 81 | list = transports 82 | list << transport 83 | logger.debug "Loaded transport #{transport[:klass]}" 84 | list.uniq! { |e| e[:name] } 85 | end 86 | 87 | def self.transports 88 | @transports ||= [] 89 | end 90 | 91 | private 92 | 93 | def self.deep_convert(element) 94 | return element.collect { |e| deep_convert(e) } if element.is_a?(Array) 95 | return element.inject({}) { |sh,(k,v)| sh[k.to_sym] = deep_convert(v); sh } if element.is_a?(Hash) 96 | element 97 | end 98 | end 99 | end 100 | -------------------------------------------------------------------------------- /lib/engine.rb: -------------------------------------------------------------------------------- 1 | module Uphold 2 | class Engine 3 | include Logging 4 | include Command 5 | include Sockets 6 | 7 | attr_reader :port, :database 8 | 9 | def initialize(params) 10 | @database = params[:database] 11 | @docker_image = params[:docker_image] 12 | @docker_tag = params[:docker_tag] 13 | @docker_env = params[:docker_env] 14 | @timeout = params[:timeout] || 10 15 | @container = nil 16 | @port = nil 17 | end 18 | 19 | def load(path:) 20 | logger.info "Engine starting #{self.class}" 21 | t1 = Time.now 22 | process = load_backup(path) 23 | t2 = Time.now 24 | delta = t2 - t1 25 | if process.success? 26 | logger.info "Engine finished successfully (#{format('%.2f', delta)}s)" 27 | true 28 | else 29 | logger.error "Engine failed! (#{format('%.2f', delta)}s)" 30 | false 31 | end 32 | rescue => e 33 | touch_state_file('bad_engine') 34 | raise e 35 | end 36 | 37 | def load_backup 38 | fail "Your engine must implement the 'load_backup' method" 39 | end 40 | 41 | def start_container 42 | if Docker::Image.exist?("#{@docker_image}:#{@docker_tag}") 43 | logger.debug "Docker image '#{@docker_image}' with tag '#{@docker_tag}' available" 44 | Docker::Image.get("#{@docker_image}:#{@docker_tag}") 45 | else 46 | logger.debug "Docker image '#{@docker_image}' with tag '#{@docker_tag}' does not exist locally, fetching" 47 | Docker::Image.create('fromImage' => @docker_image, 'tag' => @docker_tag) 48 | end 49 | 50 | @container = Docker::Container.create( 51 | 'Image' => "#{@docker_image}:#{@docker_tag}", 52 | 'Env' => @docker_env 53 | ) 54 | @container.start 55 | logger.debug "Docker container '#{container_name}' starting" 56 | wait_for_container_to_be_ready 57 | rescue => e 58 | touch_state_file('bad_engine') 59 | logger.info 'Backup is BAD' 60 | raise e 61 | end 62 | 63 | def wait_for_container_to_be_ready 64 | logger.debug "Waiting for Docker container '#{container_name}' to be ready" 65 | tcp_port_open?(container_name, container_ip_address, port, @timeout) 66 | end 67 | 68 | def container_ip_address 69 | @container.json['NetworkSettings']['IPAddress'] 70 | end 71 | 72 | def container_id 73 | @container.id[0..11] 74 | end 75 | 76 | def container_name 77 | File.basename @container.json['Name'] 78 | end 79 | 80 | def stop_container 81 | logger.debug "Docker container '#{container_name}' stopping" 82 | @container.stop 83 | @container.delete 84 | end 85 | 86 | end 87 | end 88 | -------------------------------------------------------------------------------- /lib/engines/mongodb.rb: -------------------------------------------------------------------------------- 1 | module Uphold 2 | module Engines 3 | class Mongodb < Engine 4 | 5 | def initialize(params) 6 | super(params) 7 | @docker_image ||= 'mongo' 8 | @docker_tag ||= '3.2.1' 9 | @port ||= 27_017 10 | end 11 | 12 | def load_backup(path) 13 | Dir.chdir(path) do 14 | run_command("mongorestore --verbose --host #{container_ip_address} --port #{@port} --drop --db #{@database} #{@database}") 15 | end 16 | end 17 | end 18 | end 19 | end 20 | -------------------------------------------------------------------------------- /lib/engines/mysql.rb: -------------------------------------------------------------------------------- 1 | module Uphold 2 | module Engines 3 | class Mysql < Engine 4 | 5 | def initialize(params) 6 | super(params) 7 | @docker_image ||= 'mysql' 8 | @docker_tag ||= '5.7.10' 9 | @docker_env ||= ['MYSQL_ALLOW_EMPTY_PASSWORD=yes', "MYSQL_DATABASE=#{@database}"] 10 | @port ||= 3306 11 | @sql_file = params[:sql_file] || 'MySQL.sql' 12 | end 13 | 14 | def load_backup(path) 15 | Dir.chdir(path) do 16 | run_command("mysql -u root --host=#{container_ip_address} --port=#{@port} #{@database} < #{@sql_file}") 17 | end 18 | end 19 | end 20 | end 21 | end 22 | -------------------------------------------------------------------------------- /lib/engines/postgresql.rb: -------------------------------------------------------------------------------- 1 | module Uphold 2 | module Engines 3 | class Postgresql < Engine 4 | 5 | def initialize(params) 6 | super(params) 7 | @docker_image ||= 'postgres' 8 | @docker_tag ||= '9.5.0' 9 | @docker_env ||= ["POSTGRES_USER=#{@database}", "POSTGRES_DB=#{@database}"] 10 | @port ||= 5432 11 | @sql_file = params[:sql_file] || 'PostgreSQL.sql' 12 | end 13 | 14 | def load_backup(path) 15 | Dir.chdir(path) do 16 | run_command("psql --no-password --set ON_ERROR_STOP=on --username=#{@database} --host=#{container_ip_address} --port=#{@port} --dbname=#{@database} < #{@sql_file}") 17 | end 18 | end 19 | end 20 | end 21 | end 22 | -------------------------------------------------------------------------------- /lib/helpers/command.rb: -------------------------------------------------------------------------------- 1 | module Uphold 2 | module Command 3 | module_function 4 | 5 | require 'open3' 6 | 7 | def run_command(cmd, log_command = nil) 8 | logger.debug "Running command '#{cmd}'" 9 | log_command ||= "#{cmd.split(' ')[0]}" 10 | Open3.popen3(cmd) do |_stdin, stdout, stderr, thread| 11 | # read each stream from a new thread 12 | { out: stdout, err: stderr }.each do |key, stream| 13 | Thread.new do 14 | until (line = stream.gets).nil? do 15 | # yield the block depending on the stream 16 | if key == :out 17 | logger.debug(log_command) { line.chomp } unless line.nil? 18 | else 19 | logger.error(log_command) { line.chomp } unless line.nil? 20 | end 21 | end 22 | end 23 | end 24 | 25 | thread.join # don't exit until the external process is done 26 | return thread.value 27 | end 28 | end 29 | end 30 | end 31 | -------------------------------------------------------------------------------- /lib/helpers/compression.rb: -------------------------------------------------------------------------------- 1 | module Uphold 2 | module Compression 3 | module_function 4 | 5 | BUFFER_SIZE = 4_194_304 6 | TYPES = { 7 | 'gzip' => 'gzip', 8 | 'gz' => 'gzip', 9 | 'tar' => 'tar', 10 | 'zip' => 'zip' 11 | } 12 | 13 | def decompress(file, &blk) 14 | if compressed?(file) 15 | logger.debug "Decompressing '#{File.basename(file)}'" 16 | extract(file).each do |decompressed_file| 17 | compressed?(decompressed_file) ? decompress(decompressed_file, &blk) : blk.call(decompressed_file) 18 | end 19 | else 20 | blk.call(file) 21 | end 22 | end 23 | 24 | def compressed?(file) 25 | identify(file) 26 | end 27 | 28 | private 29 | 30 | def extract(file, opts = {}) 31 | type = opts.delete(:type) || opts.delete('type') || identify(file) 32 | fail("Could not decompress #{File.basename(file)}. Please add a handler to handle files of type: #{type}.") unless respond_to?("#{type}_decompress", true) 33 | files = send("#{type}_decompress", file, opts) 34 | [files].flatten 35 | end 36 | 37 | def identify(file) 38 | type = File.extname(file)[1..-1] 39 | TYPES[type] 40 | end 41 | 42 | def zip_decompress(input_file, opts = {}) 43 | files = [] 44 | Zip::InputStream.open(input_file) do |zip| 45 | while (entry = zip.get_next_entry) 46 | files << save_zip_entry(zip, entry, File.dirname(input_file), opts) 47 | end 48 | end 49 | files 50 | end 51 | 52 | def gzip_decompress(input_file, _opts = {}) 53 | File.join(File.dirname(input_file), File.basename(input_file, '.*')).tap do |output_file| 54 | open(output_file, 'w:binary') do |output| 55 | Zlib::GzipReader.open(input_file) do |gz| 56 | output.write gz.read(BUFFER_SIZE) until gz.eof? 57 | end 58 | end 59 | end 60 | end 61 | 62 | # be wary of directories and tar long links 63 | def tar_decompress(input_file, _opts = {}) 64 | files = [] 65 | File.open(input_file) do |input_file_io| 66 | Gem::Package::TarReader.new(input_file_io) do |tar| 67 | tar.rewind 68 | tar.each do |entry| 69 | files << save_tar_entry(entry, File.dirname(input_file)) if entry.file? 70 | end 71 | end 72 | end 73 | files 74 | end 75 | 76 | def save_zip_entry(zip, entry, dir, opts) 77 | File.join(dir, entry.name).tap do |filename| 78 | open(filename, 'w') do |output_file| 79 | if opts[:zip_encoding] 80 | output_file.write zip.read(BUFFER_SIZE).encode(opts[:zip_encoding], undef: :replace, replace: '') until zip.eof? 81 | else 82 | output_file.write zip.read(BUFFER_SIZE) until zip.eof? 83 | end 84 | end 85 | end 86 | end 87 | 88 | def save_tar_entry(entry, dir) 89 | File.join(dir, entry.full_name).tap do |filename| 90 | FileUtils.mkdir_p(File.dirname(filename)) 91 | open(filename, 'w') do |output_file| 92 | output_file.write entry.read(BUFFER_SIZE) until entry.eof? 93 | end 94 | end 95 | end 96 | 97 | end 98 | end 99 | -------------------------------------------------------------------------------- /lib/helpers/logger.rb: -------------------------------------------------------------------------------- 1 | module Uphold 2 | require 'logger' 3 | 4 | module Logging 5 | class << self 6 | def logger 7 | @logger ||= Logger.new("| tee /var/log/uphold/#{ENV['UPHOLD_LOG_FILENAME'].nil? ? 'uphold' : ENV['UPHOLD_LOG_FILENAME']}.log") 8 | end 9 | 10 | def logger=(logger) 11 | @logger = logger 12 | end 13 | end 14 | 15 | # Addition 16 | def self.included(base) 17 | class << base 18 | def logger 19 | Logging.logger 20 | end 21 | end 22 | end 23 | 24 | def logger 25 | Logging.logger 26 | end 27 | 28 | def touch_state_file(state) 29 | FileUtils.touch(File.join('/var/log/uphold', ENV['UPHOLD_LOG_FILENAME'] + '_' + state)) unless ENV['UPHOLD_LOG_FILENAME'].nil? 30 | end 31 | end 32 | end 33 | -------------------------------------------------------------------------------- /lib/helpers/sockets.rb: -------------------------------------------------------------------------------- 1 | module Uphold 2 | module Sockets 3 | module_function 4 | 5 | require 'socket' 6 | require 'timeout' 7 | 8 | def tcp_port_open?(name, host, port, timeout = 10, sleep_period = 2.0) 9 | Timeout.timeout(timeout) do 10 | begin 11 | s = TCPSocket.new(host, port) 12 | s.close 13 | logger.info "Docker container '#{name}' ready!" 14 | return true 15 | rescue Errno::ECONNREFUSED, Errno::EHOSTUNREACH 16 | logger.warn "Docker container '#{name}' port is #{port} not open yet" 17 | sleep(sleep_period) 18 | retry 19 | end 20 | end 21 | rescue Timeout::Error 22 | logger.error "Docker container '#{name}' port #{port} did not open port in a timely manner" 23 | return false 24 | end 25 | end 26 | end 27 | -------------------------------------------------------------------------------- /lib/runner.rb: -------------------------------------------------------------------------------- 1 | module Uphold 2 | class Runner 3 | include Logging 4 | include Command 5 | 6 | def initialize(config:) 7 | @name = config.yaml[:name] 8 | @engine = config.yaml[:engine][:klass] 9 | @transport = config.yaml[:transport][:klass] 10 | @config = config.yaml 11 | end 12 | 13 | def start 14 | t1 = Time.now 15 | transport = @transport.new(@config[:transport][:settings]) 16 | engine = @engine.new(@config[:engine][:settings]) 17 | 18 | begin 19 | working_path = transport.fetch 20 | unless engine.start_container 21 | touch_state_file('bad_engine') 22 | logger.info 'Backup is BAD' 23 | exit 0 24 | end 25 | 26 | if engine.load(path: working_path) 27 | if @config[:tests].any? 28 | tests = Tests.new(tests: @config[:tests], ip_address: engine.container_ip_address, port: engine.port, database: engine.database) 29 | if tests.run 30 | touch_state_file('ok') 31 | logger.info 'Backup is OK' 32 | exit 0 33 | else 34 | logger.fatal "Backup for #{@config[:name]} is BAD" 35 | touch_state_file('bad_tests') 36 | exit 1 37 | end 38 | else 39 | logger.info 'No tests found, but OK' 40 | touch_state_file('ok_no_test') 41 | exit 0 42 | end 43 | else 44 | logger.fatal "Backup for #{@config[:name]} is BAD" 45 | touch_state_file('bad') 46 | exit 1 47 | end 48 | rescue => e 49 | logger.error e 50 | raise e 51 | ensure 52 | engine.stop_container 53 | logger.debug "Removing tmpdir '#{transport.tmpdir}'" 54 | FileUtils.remove_entry_secure(transport.tmpdir) 55 | 56 | t2 = Time.now 57 | delta = t2 - t1 58 | logger.info "Done! (#{format('%.2f', delta)}s)" 59 | end 60 | end 61 | end 62 | end 63 | -------------------------------------------------------------------------------- /lib/tests.rb: -------------------------------------------------------------------------------- 1 | module Uphold 2 | class Tests 3 | include Logging 4 | include Command 5 | 6 | def initialize(ip_address:, port:, database:, tests:) 7 | @ip_address = ip_address 8 | @port = port 9 | @database = database 10 | @tests = tests 11 | end 12 | 13 | def run 14 | logger.info 'Tests starting' 15 | t1 = Time.now 16 | 17 | outcomes = @tests.collect do |t| 18 | process = run_command("UPHOLD_IP=#{@ip_address} UPHOLD_PORT=#{@port} UPHOLD_DB=#{@database} ruby /etc/uphold/tests/#{t}", 'ruby') 19 | if process.success? 20 | logger.info "Test #{t} finished successfully" 21 | true 22 | else 23 | logger.error "Test #{t} did NOT finish successfully" 24 | false 25 | end 26 | end 27 | 28 | t2 = Time.now 29 | delta = t2 - t1 30 | logger.info "Tests finished (#{format('%.2f', delta)}s)" 31 | !outcomes.include?(false) 32 | end 33 | end 34 | end 35 | -------------------------------------------------------------------------------- /lib/transport.rb: -------------------------------------------------------------------------------- 1 | module Uphold 2 | class Transport 3 | include Logging 4 | include Compression 5 | 6 | attr_reader :tmpdir 7 | 8 | def initialize(params) 9 | @tmpdir = Dir.mktmpdir('uphold') 10 | @path = params[:path] 11 | @filename = params[:filename] 12 | @folder_within = params[:folder_within] 13 | 14 | @date_format = params[:date_format] || '%Y-%m-%d' 15 | @date_offset = params[:date_offset] || 0 16 | @path.gsub!('{date}', (Date.today - @date_offset).strftime(@date_format)) 17 | @filename.gsub!('{date}', (Date.today - @date_offset).strftime(@date_format)) 18 | end 19 | 20 | def fetch 21 | logger.info "Transport starting #{self.class}" 22 | logger.debug "Temporary directory '#{@tmpdir}'" 23 | 24 | t1 = Time.now 25 | path = fetch_backup 26 | t2 = Time.now 27 | delta = t2 - t1 28 | if path.nil? 29 | logger.fatal "Transport failed! (#{format('%.2f', delta)}s)" 30 | touch_state_file('bad_transport') 31 | exit 1 32 | else 33 | logger.info "Transport finished successfully (#{format('%.2f', delta)}s)" 34 | path 35 | end 36 | rescue => e 37 | touch_state_file('bad_transport') 38 | raise e 39 | end 40 | 41 | def fetch_backup 42 | fail "Your transport must implement the 'fetch' method" 43 | end 44 | end 45 | end 46 | -------------------------------------------------------------------------------- /lib/transports/local.rb: -------------------------------------------------------------------------------- 1 | module Uphold 2 | module Transports 3 | class Local < Transport 4 | def initialize(params) 5 | super(params) 6 | end 7 | 8 | def fetch_backup 9 | file_path = File.join(@path, @filename) 10 | if File.file?(file_path) 11 | tmp_path = File.join(@tmpdir, File.basename(file_path)) 12 | logger.info "Copying '#{file_path}' to '#{tmp_path}'" 13 | FileUtils.cp(file_path, tmp_path) 14 | decompress(tmp_path) do |_b| 15 | end 16 | File.join(@tmpdir, @folder_within) 17 | else 18 | logger.fatal "No file exists at '#{file_path}'" 19 | end 20 | end 21 | 22 | end 23 | end 24 | end 25 | -------------------------------------------------------------------------------- /lib/transports/s3.rb: -------------------------------------------------------------------------------- 1 | module Uphold 2 | module Transports 3 | class S3 < Transport 4 | def initialize(params) 5 | super(params) 6 | @region = params[:region] 7 | @access_key_id = params[:access_key_id] 8 | @secret_access_key = params[:secret_access_key] 9 | @bucket = params[:bucket] 10 | end 11 | 12 | def fetch_backup 13 | s3 = Aws::S3::Client.new(region: @region, access_key_id: @access_key_id, secret_access_key: @secret_access_key) 14 | matching_prefix = s3.list_objects(bucket: @bucket, max_keys: 10, prefix: @path).contents.collect(&:key) 15 | matching_file = matching_prefix.find { |s3_file| File.fnmatch(@filename, File.basename(s3_file)) } 16 | 17 | File.open(File.join(@tmpdir, File.basename(matching_file)), 'wb') do |file| 18 | logger.info "Downloading '#{matching_file}' from S3 bucket #{@bucket}" 19 | s3.get_object({ bucket: @bucket, key: matching_file }, target: file) 20 | decompress(file) do |_b| 21 | end 22 | end 23 | File.join(@tmpdir, @folder_within) 24 | end 25 | end 26 | end 27 | end 28 | -------------------------------------------------------------------------------- /public/font/Roboto-Regular-webfont.eot: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/forward3d/uphold/40a252df58de35e78e4173beabd4f3decd94c67c/public/font/Roboto-Regular-webfont.eot -------------------------------------------------------------------------------- /public/font/Roboto-Regular-webfont.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | 75 | 76 | 77 | 78 | 79 | 80 | 81 | 82 | 83 | 84 | 85 | 86 | 87 | 88 | 89 | 90 | 91 | 92 | 93 | 94 | 95 | 96 | 97 | 98 | 99 | 100 | 101 | 102 | 103 | 104 | 105 | 106 | 107 | 108 | 109 | 110 | 111 | 112 | 113 | 114 | 115 | 116 | 117 | 118 | 119 | 120 | 121 | 122 | 123 | 124 | 125 | 126 | 127 | 128 | 129 | 130 | 131 | 132 | 133 | 134 | 135 | 136 | 137 | 138 | 139 | 140 | 141 | 142 | 143 | 144 | 145 | 146 | 147 | 148 | 149 | 150 | 151 | 152 | 153 | 154 | 155 | 156 | 157 | 158 | 159 | 160 | 161 | 162 | 163 | 164 | 165 | 166 | 167 | 168 | 169 | 170 | 171 | 172 | 173 | 174 | 175 | 176 | 177 | 178 | 179 | 180 | 181 | 182 | 183 | 184 | 185 | 186 | 187 | 188 | 189 | 190 | 191 | 192 | 193 | 194 | 195 | 196 | 197 | 198 | 199 | 200 | 201 | 202 | 203 | 204 | 205 | 206 | 207 | 208 | 209 | 210 | 211 | 212 | 213 | 214 | 215 | 216 | 217 | 218 | 219 | 220 | 221 | 222 | 223 | 224 | 225 | 226 | 227 | 228 | 229 | 230 | 231 | 232 | 233 | 234 | 235 | 236 | 237 | 238 | 239 | 240 | 241 | 242 | 243 | 244 | 245 | 246 | 247 | 248 | 249 | 250 | 251 | 252 | 253 | 254 | 255 | 256 | 257 | 258 | 259 | 260 | 261 | 262 | 263 | 264 | 265 | 266 | 267 | 268 | 269 | 270 | 271 | 272 | 273 | 274 | 275 | 276 | 277 | 278 | 279 | 280 | 281 | 282 | 283 | 284 | 285 | 286 | 287 | 288 | 289 | 290 | 291 | 292 | 293 | 294 | 295 | 296 | 297 | 298 | 299 | 300 | 301 | 302 | 303 | 304 | 305 | 306 | 307 | 308 | 309 | 310 | 311 | 312 | 313 | 314 | 315 | 316 | 317 | 318 | 319 | 320 | 321 | 322 | 323 | 324 | 325 | 326 | 327 | 328 | 329 | 330 | 331 | 332 | 333 | 334 | 335 | 336 | 337 | 338 | 339 | 340 | 341 | 342 | 343 | 344 | 345 | 346 | 347 | 348 | 349 | 350 | 351 | 352 | 353 | 354 | 355 | 356 | 357 | 358 | 359 | 360 | 361 | 362 | 363 | 364 | 365 | 366 | 367 | 368 | 369 | 370 | 371 | 372 | 373 | 374 | 375 | 376 | 377 | 378 | 379 | 380 | 381 | 382 | 383 | 384 | 385 | 386 | 387 | 388 | 389 | 390 | 391 | 392 | 393 | 394 | 395 | 396 | 397 | 398 | 399 | 400 | 401 | 402 | 403 | 404 | 405 | 406 | 407 | 408 | 409 | 410 | 411 | 412 | 413 | 414 | 415 | 416 | 417 | 418 | 419 | 420 | 421 | 422 | 423 | 424 | 425 | 426 | 427 | 428 | 429 | 430 | 431 | 432 | 433 | 434 | 435 | 436 | 437 | 438 | 439 | 440 | 441 | 442 | 443 | 444 | 445 | 446 | 447 | 448 | 449 | 450 | 451 | 452 | 453 | 454 | 455 | 456 | 457 | 458 | 459 | 460 | 461 | 462 | 463 | 464 | 465 | 466 | 467 | 468 | 469 | 470 | 471 | 472 | 473 | 474 | 475 | 476 | 477 | 478 | 479 | 480 | 481 | 482 | 483 | 484 | 485 | 486 | 487 | 488 | 489 | 490 | 491 | 492 | 493 | 494 | 495 | 496 | 497 | 498 | 499 | 500 | 501 | 502 | 503 | 504 | 505 | 506 | 507 | 508 | 509 | 510 | 511 | 512 | 513 | 514 | 515 | 516 | 517 | 518 | 519 | 520 | 521 | 522 | 523 | 524 | 525 | 526 | 527 | 528 | 529 | 530 | 531 | 532 | 533 | 534 | 535 | 536 | 537 | 538 | 539 | 540 | 541 | 542 | 543 | 544 | 545 | 546 | 547 | 548 | 549 | 550 | 551 | 552 | 553 | 554 | 555 | 556 | 557 | 558 | 559 | 560 | 561 | 562 | 563 | 564 | 565 | 566 | 567 | 568 | 569 | 570 | 571 | 572 | 573 | 574 | 575 | 576 | 577 | 578 | 579 | 580 | 581 | 582 | 583 | 584 | 585 | 586 | 587 | 588 | 589 | 590 | 591 | 592 | 593 | 594 | 595 | 596 | 597 | 598 | 599 | 600 | 601 | 602 | 603 | 604 | 605 | 606 | 607 | 608 | 609 | 610 | 611 | 612 | 613 | 614 | 615 | 616 | 617 | 618 | 619 | 620 | 621 | -------------------------------------------------------------------------------- /public/font/Roboto-Regular-webfont.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/forward3d/uphold/40a252df58de35e78e4173beabd4f3decd94c67c/public/font/Roboto-Regular-webfont.ttf -------------------------------------------------------------------------------- /public/font/Roboto-Regular-webfont.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/forward3d/uphold/40a252df58de35e78e4173beabd4f3decd94c67c/public/font/Roboto-Regular-webfont.woff -------------------------------------------------------------------------------- /public/images/bad.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/forward3d/uphold/40a252df58de35e78e4173beabd4f3decd94c67c/public/images/bad.png -------------------------------------------------------------------------------- /public/images/bad_engine.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/forward3d/uphold/40a252df58de35e78e4173beabd4f3decd94c67c/public/images/bad_engine.png -------------------------------------------------------------------------------- /public/images/bad_tests.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/forward3d/uphold/40a252df58de35e78e4173beabd4f3decd94c67c/public/images/bad_tests.png -------------------------------------------------------------------------------- /public/images/bad_transport.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/forward3d/uphold/40a252df58de35e78e4173beabd4f3decd94c67c/public/images/bad_transport.png -------------------------------------------------------------------------------- /public/images/ok.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/forward3d/uphold/40a252df58de35e78e4173beabd4f3decd94c67c/public/images/ok.png -------------------------------------------------------------------------------- /public/images/ok_no_test.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/forward3d/uphold/40a252df58de35e78e4173beabd4f3decd94c67c/public/images/ok_no_test.png -------------------------------------------------------------------------------- /public/images/run.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/forward3d/uphold/40a252df58de35e78e4173beabd4f3decd94c67c/public/images/run.png -------------------------------------------------------------------------------- /public/images/running.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/forward3d/uphold/40a252df58de35e78e4173beabd4f3decd94c67c/public/images/running.gif -------------------------------------------------------------------------------- /public/stylesheets/normalize.min.css: -------------------------------------------------------------------------------- 1 | /*! normalize.css v3.0.2 | MIT License | git.io/normalize */html{font-family:sans-serif;-ms-text-size-adjust:100%;-webkit-text-size-adjust:100%}body{margin:0}article,aside,details,figcaption,figure,footer,header,hgroup,main,menu,nav,section,summary{display:block}audio,canvas,progress,video{display:inline-block;vertical-align:baseline}audio:not([controls]){display:none;height:0}[hidden],template{display:none}a{background-color:transparent}a:active,a:hover{outline:0}abbr[title]{border-bottom:1px dotted}b,strong{font-weight:700}dfn{font-style:italic}h1{font-size:2em;margin:.67em 0}mark{background:#ff0;color:#000}small{font-size:80%}sub,sup{font-size:75%;line-height:0;position:relative;vertical-align:baseline}sup{top:-.5em}sub{bottom:-.25em}img{border:0}svg:not(:root){overflow:hidden}figure{margin:1em 40px}hr{-moz-box-sizing:content-box;box-sizing:content-box;height:0}pre{overflow:auto}code,kbd,pre,samp{font-family:monospace,monospace;font-size:1em}button,input,optgroup,select,textarea{color:inherit;font:inherit;margin:0}button{overflow:visible}button,select{text-transform:none}button,html input[type=button],input[type=reset],input[type=submit]{-webkit-appearance:button;cursor:pointer}button[disabled],html input[disabled]{cursor:default}button::-moz-focus-inner,input::-moz-focus-inner{border:0;padding:0}input{line-height:normal}input[type=checkbox],input[type=radio]{box-sizing:border-box;padding:0}input[type=number]::-webkit-inner-spin-button,input[type=number]::-webkit-outer-spin-button{height:auto}input[type=search]{-webkit-appearance:textfield;-moz-box-sizing:content-box;-webkit-box-sizing:content-box;box-sizing:content-box}input[type=search]::-webkit-search-cancel-button,input[type=search]::-webkit-search-decoration{-webkit-appearance:none}fieldset{border:1px solid silver;margin:0 2px;padding:.35em .625em .75em}legend{border:0;padding:0}textarea{overflow:auto}optgroup{font-weight:700}table{border-collapse:collapse;border-spacing:0}td,th{padding:0} 2 | -------------------------------------------------------------------------------- /public/stylesheets/uphold.css: -------------------------------------------------------------------------------- 1 | @font-face { 2 | font-family: 'robotoregular'; 3 | src: url('font/Roboto-Regular-webfont.eot'); 4 | src: url('font/Roboto-Regular-webfont.eot?#iefix') format('embedded-opentype'), 5 | url('font/Roboto-Regular-webfont.woff') format('woff'), 6 | url('font/Roboto-Regular-webfont.ttf') format('truetype'), 7 | url('font/Roboto-Regular-webfont.svg#robotoregular') format('svg'); 8 | font-weight: normal; 9 | font-style: normal; 10 | 11 | } 12 | 13 | body { 14 | padding: 40px; 15 | font-family: 'robotoregular', sans-serif; 16 | } 17 | 18 | table { 19 | border-collapse: collapse 20 | } 21 | 22 | table, th, td { 23 | border: 1px solid #e6ece8 24 | } 25 | 26 | th { 27 | font-size: larger; 28 | text-align: left; 29 | padding-left: 20px 30 | } 31 | 32 | th.config_name { 33 | min-width: 140px; 34 | padding: 20px; 35 | } 36 | 37 | td.config_name { 38 | padding: 20px; 39 | vertical-align: top; 40 | } 41 | 42 | td a { 43 | display: inline-block; 44 | height: 20px; 45 | width: 20px; 46 | background-size: 20px; 47 | margin-right: 5px; 48 | vertical-align: top; 49 | } 50 | 51 | td a.run { 52 | background-image: url('../images/run.png'); 53 | margin-right: 0; 54 | margin-left: 10px; 55 | float: right; 56 | } 57 | 58 | td a.running { background-image: url('../images/running.gif'); } 59 | td a.ok { background-image: url('../images/ok.png'); } 60 | td a.ok_no_test { background-image: url('../images/ok_no_test.png'); } 61 | td a.bad { background-image: url('../images/bad.png'); } 62 | td a.bad_tests { background-image: url('../images/bad_tests.png'); } 63 | td a.bad_transport { background-image: url('../images/bad_transport.png'); } 64 | td a.bad_engine { background-image: url('../images/bad_engine.png'); } 65 | 66 | ul { 67 | padding-top: 10px; 68 | padding-bottom: 10px; 69 | padding-left: 20px; 70 | margin: 0; 71 | } 72 | 73 | li { 74 | display: inline-block; 75 | margin-top: 10px; 76 | margin-bottom: 10px; 77 | margin-right: 20px; 78 | } 79 | -------------------------------------------------------------------------------- /tester.rb: -------------------------------------------------------------------------------- 1 | module Uphold 2 | require 'rubygems' 3 | require 'rubygems/package' 4 | require 'bundler/setup' 5 | Bundler.require(:default, :tester) 6 | load 'environment.rb' 7 | 8 | Config.load_engines 9 | Config.load_transports 10 | 11 | run = Runner.new(config: Uphold::Config.new(ARGV[0])) 12 | run.start 13 | end 14 | -------------------------------------------------------------------------------- /ui.rb: -------------------------------------------------------------------------------- 1 | require 'rubygems' 2 | require 'rubygems/package' 3 | require 'bundler/setup' 4 | Bundler.require(:default, :ui) 5 | load 'environment.rb' 6 | 7 | module Uphold 8 | class Ui < ::Sinatra::Base 9 | include Logging 10 | set :views, settings.root + '/views' 11 | set :public_folder, settings.root + '/public' 12 | 13 | helpers do 14 | def h(text) 15 | Rack::Utils.escape_html(text) 16 | end 17 | 18 | def epoch_to_datetime(epoch) 19 | Time.at(epoch).utc.to_datetime.strftime(UPHOLD[:ui_datetime]) 20 | end 21 | end 22 | 23 | before do 24 | Config.load_engines 25 | Config.load_transports 26 | @configs = Config.load_configs 27 | end 28 | 29 | get '/' do 30 | @logs = logs 31 | erb :index 32 | end 33 | 34 | get '/run/:slug' do 35 | start_docker_container(params[:slug]) 36 | redirect '/' 37 | end 38 | 39 | get '/logs/:filename' do 40 | @log = File.join('/var/log/uphold', params[:filename]) 41 | erb :log 42 | end 43 | 44 | post '/api/1.0/backup' do 45 | start_docker_container(params[:name]) 46 | 200 47 | end 48 | 49 | get '/api/1.0/backups/:name' do 50 | # get all the runs for the named config 51 | content_type :json 52 | @logs = logs[params[:name]] 53 | if @logs.nil? 54 | [].to_json 55 | else 56 | @logs.to_json 57 | end 58 | end 59 | 60 | get '/api/1.0/backups/:name/latest' do 61 | # get the latest state for the named config 62 | @logs = logs[params[:name]] 63 | if @logs.nil? 64 | 'none' 65 | else 66 | @logs.first[:state] 67 | end 68 | end 69 | 70 | private 71 | 72 | def start_docker_container(slug) 73 | if Docker::Image.exist?("#{UPHOLD[:docker_container]}:#{UPHOLD[:docker_tag]}") 74 | Docker::Image.get("#{UPHOLD[:docker_container]}:#{UPHOLD[:docker_tag]}") 75 | else 76 | Docker::Image.create('fromImage' => UPHOLD[:docker_container], 'tag' => UPHOLD[:docker_tag]) 77 | end 78 | 79 | volumes = {} 80 | UPHOLD[:docker_mounts].flatten.each { |m| volumes[m] = { "#{m}" => 'ro' } } 81 | 82 | # this is a hack for when you're working in development on osx 83 | volumes[UPHOLD[:config_path]] = { '/etc/uphold' => 'ro' } 84 | volumes[UPHOLD[:docker_log_path]] = { '/var/log/uphold' => 'rw' } 85 | 86 | # Unix sockets when mounted can't have the protocol at the start 87 | if UPHOLD[:docker_url].include?('unix://') 88 | without_protocol = UPHOLD[:docker_url].split('unix://')[1] 89 | volumes[without_protocol] = { "#{without_protocol}" => 'rw' } 90 | end 91 | 92 | @container = Docker::Container.create( 93 | 'Image' => "#{UPHOLD[:docker_container]}:#{UPHOLD[:docker_tag]}", 94 | 'Cmd' => [slug + '.yml'], 95 | 'Volumes' => volumes, 96 | 'Env' => ["UPHOLD_LOG_FILENAME=#{Time.now.to_i}_#{slug}"] 97 | ) 98 | 99 | @container.start('Binds' => volumes.map { |v, h| "#{v}:#{h.keys.first}" }) 100 | end 101 | 102 | def logs 103 | logs = {} 104 | raw_test_logs.each do |log| 105 | epoch = log.split('_')[0] 106 | config = log.split('_')[1].gsub!('.log', '') 107 | state = raw_state_files.find { |s| s.include?("#{epoch}_#{config}") } 108 | if state 109 | state = state.gsub("#{epoch}_#{config}", '')[1..-1] 110 | else 111 | state = 'running' 112 | end 113 | logs[config] ||= [] 114 | logs[config] << { epoch: epoch.to_i, state: state, filename: log } 115 | logs[config].sort_by! { |h| h[:epoch].to_i }.reverse! 116 | end 117 | logs 118 | end 119 | 120 | def raw_test_logs 121 | raw_files.select { |file| File.extname(file) == '.log' } 122 | end 123 | 124 | def raw_state_files 125 | raw_files.select { |file| File.extname(file) == '' } 126 | end 127 | 128 | def raw_files 129 | Dir[File.join('/var/log/uphold', '*')].select { |log| File.basename(log) =~ /^[0-9]{10}/ }.map { |file| File.basename(file) } 130 | end 131 | end 132 | end 133 | -------------------------------------------------------------------------------- /views/index.erb: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | <% @configs.sort_by { |c| c[:name] }.each do |c| %> 8 | 9 | 10 | 23 | 24 | <% end %> 25 | 26 |
Config NameOutcomes
<%= c[:name] %> 11 | <% unless @logs[c[:name]].nil? %> 12 |
    13 | <% @logs[c[:name]].each do |log| %> 14 |
  • 15 | <%= epoch_to_datetime(log[:epoch]) %> 16 |
    • 17 | <% end %> 18 |
19 | <% else %> 20 |
  • No logs
21 | <% end %> 22 |
27 | -------------------------------------------------------------------------------- /views/layout.erb: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | Uphold 7 | 8 | 9 | 10 | 11 | 12 | 13 | <%= yield %> 14 | 15 | 16 | -------------------------------------------------------------------------------- /views/log.erb: -------------------------------------------------------------------------------- 1 | 2 | <% IO.foreach(@log) do |line| %> 3 | <%= line %>
4 | <% end %> 5 | 6 | --------------------------------------------------------------------------------