├── .gitignore ├── .travis.yml ├── AUTHORS ├── Jenkinsfile ├── LICENSE ├── README.md ├── debian ├── .gitignore ├── changelog ├── compat ├── control ├── copyright ├── docs ├── prebuild.sh ├── rules ├── source │ └── format └── tarantool-dump.install ├── docs └── truck.png ├── dump-scm-1.rockspec ├── dump └── init.lua ├── rpm ├── prebuild.sh └── tarantool-dump.spec └── test └── dump.test.lua /.gitignore: -------------------------------------------------------------------------------- 1 | CMakeFiles/ 2 | obj-*/ 3 | CMakeCache.txt 4 | Makefile 5 | cmake_*.cmake 6 | install_manifest.txt 7 | *.a 8 | *.cbp 9 | *.d 10 | *.dylib 11 | *.gcno 12 | *.gcda 13 | *.user 14 | *.o 15 | *.reject 16 | *.so 17 | *.snap* 18 | *.xlog* 19 | *~ 20 | .gdb_history 21 | ./build 22 | /build 23 | VERSION 24 | CTestTestfile.cmake 25 | Testing 26 | CTestTestfile.cmake 27 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | sudo: false 2 | language: C 3 | services: 4 | - docker 5 | 6 | cache: 7 | directories: 8 | - $HOME/.cache 9 | 10 | git: 11 | depth: 100500 12 | 13 | env: 14 | global: 15 | - PRODUCT=tarantool-dump 16 | matrix: 17 | - OS=el DIST=6 18 | - OS=el DIST=7 19 | - OS=fedora DIST=27 20 | - OS=fedora DIST=28 21 | - OS=ubuntu DIST=bionic 22 | - OS=ubuntu DIST=cosmic 23 | - OS=ubuntu DIST=trusty 24 | - OS=ubuntu DIST=xenial 25 | - OS=debian DIST=jessie 26 | - OS=debian DIST=stretch 27 | 28 | script: 29 | - git describe --long 30 | - git clone https://github.com/packpack/packpack.git packpack 31 | - packpack/packpack 32 | 33 | before_deploy: 34 | - ls -l build/ 35 | 36 | deploy: 37 | # Deploy packages to PackageCloud 38 | - provider: packagecloud 39 | username: tarantool 40 | repository: "1_10" 41 | token: ${PACKAGECLOUD_TOKEN} 42 | dist: ${OS}/${DIST} 43 | package_glob: build/*.{deb,rpm,dsc} 44 | skip_cleanup: true 45 | on: 46 | branch: master 47 | condition: -n "${OS}" && -n "${DIST}" && -n "${PACKAGECLOUD_TOKEN}" 48 | - provider: packagecloud 49 | username: tarantool 50 | repository: "2x" 51 | token: ${PACKAGECLOUD_TOKEN} 52 | dist: ${OS}/${DIST} 53 | package_glob: build/*.{deb,rpm,dsc} 54 | skip_cleanup: true 55 | on: 56 | branch: master 57 | condition: -n "${OS}" && -n "${DIST}" && -n "${PACKAGECLOUD_TOKEN}" 58 | 59 | notifications: 60 | email: 61 | recipients: 62 | - build@tarantool.org 63 | on_success: change 64 | on_failure: always 65 | -------------------------------------------------------------------------------- /AUTHORS: -------------------------------------------------------------------------------- 1 | Konstantin Osipov 2 | Andrey Drozdov 3 | -------------------------------------------------------------------------------- /Jenkinsfile: -------------------------------------------------------------------------------- 1 | stage('Build'){ 2 | packpack = new org.tarantool.packpack() 3 | node { 4 | checkout scm 5 | packpack.prepareSources() 6 | } 7 | packpack.packpackBuildMatrix('result') 8 | } 9 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (C) 2010-2017 Tarantool AUTHORS: please see AUTHORS file. 2 | 3 | Redistribution and use in source and binary forms, with or 4 | without modification, are permitted provided that the following 5 | conditions are met: 6 | 7 | 1. Redistributions of source code must retain the above 8 | copyright notice, this list of conditions and the 9 | following disclaimer. 10 | 11 | 2. Redistributions in binary form must reproduce the above 12 | copyright notice, this list of conditions and the following 13 | disclaimer in the documentation and/or other materials 14 | provided with the distribution. 15 | 16 | THIS SOFTWARE IS PROVIDED BY AUTHORS AND CONTRIBUTORS ``AS IS'' AND 17 | ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 18 | TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 19 | A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL 20 | AUTHORS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, 21 | INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 23 | SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR 24 | BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 25 | LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF 27 | THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 28 | SUCH DAMAGE. 29 | 30 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 |
2 | 3 | # dump 4 | 5 | Logical backup and restore of a Tarantool instance. 6 | 7 | ## Why you need a logical backup 8 | 9 | Tarantool provides a physical backup with 10 | [box.backup](https://tarantool.org/en/doc/1.7/book/admin/backups.html#hot-backup-vinyl-memtx) 11 | module. But you may still want a logical one. 12 | 13 | Here's why: 14 | 15 | * A logical backup contains tuples, not files. By making a logical backup, you 16 | ensure that the database server can read the contents of the database. 17 | A logical backup sees the database through the same lense as your application. 18 | * If Tarantool binary file layout changes, you still can restore data from a 19 | logical backup. 20 | * You could use a logical backup to export data into another database (although 21 | we recommend using Tarantool's [MySQL](http://github.com/tarantool/mysql) or 22 | [PostgreSQL](http://github.com/tarantool/pg) connectors for this). 23 | 24 | ## How to use 25 | 26 | This module takes each space in Tarantool and dumps it into a file in a specified directory. 27 | This includes data for system spaces _space and _index, which are dumped first. The restore 28 | is performed in the opposite order, when files for restore are sorted according to their id 29 | (system spaces have id < 512, so their data is naturally restored first). Restoring the system 30 | spaces first ensures that all *definitions* for user-defined spaces are already in place when a 31 | user-defined space dump file is restored. 32 | 33 | ### Preparation 34 | 35 | Ensure that the database is not being changed while dump or restore is in progress. 36 | 37 | ### Execution 38 | 39 | ```local status, error = require('dump').dump('/path/to/logical/backup')``` 40 | 41 | The path should not exist, or be an empty directory. It is created if it does 42 | not exist. The command then dumps all space and index definitions, users, roles 43 | and privileges, and space data. Each space is dumped into a file in the path 44 | named `.dump`. 45 | 46 | ```local status, error = require('dump').restore('/path/to/logical/backup')``` 47 | 48 | Please note that this module does not throw exceptions, and uses Lua conventions for 49 | errors: check the return value explicitly to see if your dump or restore has succeeded. 50 | 51 | This command restores a logical dump. 52 | 53 | ### Advanced usage 54 | 55 | You can use a filter function as an additional argument to dump and restore. 56 | A filter is a function that takes a space and a tuple and returns a tuple. 57 | This function is called for each dumped/restored tuple. It can be used to overwrite 58 | what is written to the dump file or restored. If it returns nil the tuple is skipped. 59 | 60 | For example, 'filter' option can be used to convert memtx spaces to vinyl as 61 | shown below: 62 | ``` 63 | dump.restore('dump', { 64 | filter = function(space, tuple) 65 | if space.id == box.schema.SPACE_ID then 66 | return tuple:update{{'=', 4, 'vinyl'}} 67 | else 68 | return tuple 69 | end 70 | end 71 | }) 72 | ``` 73 | 74 | ### Details 75 | 76 | The backup utility creates a file for each dumped space, using space id for file name. The dump skips spaces with id < 512 (the system spaces), with the exception of tuples which contain metadata of user-defined spaces, to ensure smooth restore on an empty instance. If you want to restore data into an existing space, delete files with ids < 512 from the dump directory and create the destination space manually with Lua during restore. Alternatively, you can keep all files with id < 512, this will restore or space definitions, and your particular space file, and pass this to dump. For more intellectual filtering, use dump/restore filtering. 77 | -------------------------------------------------------------------------------- /debian/.gitignore: -------------------------------------------------------------------------------- 1 | tarantool-dump/ 2 | files 3 | stamp-* 4 | *.substvars 5 | *.log 6 | -------------------------------------------------------------------------------- /debian/changelog: -------------------------------------------------------------------------------- 1 | tarantool-dump (1.0.0-1) unstable; urgency=medium 2 | 3 | * Initial release 4 | 5 | -- Konstantin Osipov Fri, 16 Feb 2017 15:42:33 +0300 6 | -------------------------------------------------------------------------------- /debian/compat: -------------------------------------------------------------------------------- 1 | 9 2 | -------------------------------------------------------------------------------- /debian/control: -------------------------------------------------------------------------------- 1 | Source: tarantool-dump 2 | Priority: optional 3 | Section: database 4 | Maintainer: Konstantin Osipov 5 | Build-Depends: debhelper (>= 9), tarantool (>= 1.9.0) 6 | Standards-Version: 3.9.8 7 | Homepage: https://github.com/tarantool/dump 8 | Vcs-Git: git://github.com/tarantool/dump.git 9 | Vcs-Browser: https://github.com/tarantool/dump 10 | 11 | Package: tarantool-dump 12 | Architecture: all 13 | Depends: tarantool (>= 1.9.0), ${shlibs:Depends}, ${misc:Depends} 14 | Pre-Depends: ${misc:Pre-Depends} 15 | Description: Logical backup and restore for Tarantool 16 | Backup all spaces into a set of files, one for each space. Restore from the 17 | files at any time. 18 | -------------------------------------------------------------------------------- /debian/copyright: -------------------------------------------------------------------------------- 1 | Format: http://www.debian.org/doc/packaging-manuals/copyright-format/1.0/ 2 | Debianized-By: Konstantin Osipov 3 | Upstream-Name: tarantool-dump 4 | Upstream-Contact: kostja@tarantool.org 5 | Source: https://github.com/tarantool/dump 6 | 7 | Files: * 8 | Copyright: 2017 Konstantin Osipov 9 | License: BSD-2-Clause 10 | Redistribution and use in source and binary forms, with or 11 | without modification, are permitted provided that the following 12 | conditions are met: 13 | . 14 | 1. Redistributions of source code must retain the above 15 | copyright notice, this list of conditions and the 16 | following disclaimer. 17 | . 18 | 2. Redistributions in binary form must reproduce the above 19 | copyright notice, this list of conditions and the following 20 | disclaimer in the documentation and/or other materials 21 | provided with the distribution. 22 | . 23 | THIS SOFTWARE IS PROVIDED BY AUTHORS AND CONTRIBUTORS ``AS IS'' AND 24 | ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 25 | TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 26 | A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL 27 | AUTHORS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, 28 | INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 29 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 30 | SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR 31 | BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 32 | LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 33 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF 34 | THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 35 | SUCH DAMAGE. 36 | -------------------------------------------------------------------------------- /debian/docs: -------------------------------------------------------------------------------- 1 | README.md 2 | AUTHORS 3 | -------------------------------------------------------------------------------- /debian/prebuild.sh: -------------------------------------------------------------------------------- 1 | curl -s https://packagecloud.io/install/repositories/tarantool/1_10/script.deb.sh | sudo bash 2 | -------------------------------------------------------------------------------- /debian/rules: -------------------------------------------------------------------------------- 1 | #!/usr/bin/make -f 2 | 3 | %: 4 | dh $@ 5 | 6 | override_dh_auto_test: 7 | prove -v ./test/dump.test.lua 8 | -------------------------------------------------------------------------------- /debian/source/format: -------------------------------------------------------------------------------- 1 | 3.0 (quilt) 2 | -------------------------------------------------------------------------------- /debian/tarantool-dump.install: -------------------------------------------------------------------------------- 1 | dump/init.lua usr/share/tarantool/dump/ 2 | -------------------------------------------------------------------------------- /docs/truck.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tarantool/dump/886de611441cc04ec2c429ecb0aa3a648007acda/docs/truck.png -------------------------------------------------------------------------------- /dump-scm-1.rockspec: -------------------------------------------------------------------------------- 1 | -- name of the package to be published 2 | package = 'dump' 3 | 4 | -- version of the package; it's mandatory, but we don't use it in Tarantool; 5 | -- instead, provide below a specific branch in the package's repository at 6 | -- GitHub and set version to some stub value, e.g. 'scm-1' 7 | version = 'scm-1' 8 | 9 | -- url and branch of the package's repository at GitHub 10 | source = { 11 | url = 'git+https://github.com/tarantool/dump.git'; 12 | branch = 'master'; 13 | } 14 | 15 | -- general information about the package; 16 | -- for a Tarantool package, we require three fields (summary, homepage, license) 17 | -- and more package information is always welcome 18 | description = { 19 | summary = "Logical dump and restore for Tarantool"; 20 | detailed = [[ 21 | Logical backups are the only true backups. 22 | ]]; 23 | homepage = 'https://github.com/tarantool/dump.git'; 24 | maintainer = "Konstantin Osipov "; 25 | license = 'BSD2'; 26 | } 27 | 28 | -- Lua version and other packages on which this one depends; 29 | -- Tarantool currently supports strictly Lua 5.1 30 | dependencies = { 31 | 'lua == 5.1'; 32 | } 33 | 34 | -- build options and paths for the package; 35 | -- this package distributes modules in pure Lua, so the build type = 'builtin'; 36 | -- also, specify here paths to all Lua modules within the package 37 | -- (this package contains just one Lua module named 'dump') 38 | build = { 39 | type = 'builtin'; 40 | modules = { 41 | ['dump'] = 'dump/init.lua'; 42 | } 43 | } 44 | -- vim: syntax=lua ts=4 sts=4 sw=4 et 45 | -------------------------------------------------------------------------------- /dump/init.lua: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- 2 | --- Logical dump and restore for Tarantool 3 | -------------------------------------------------------------------------------- 4 | 5 | -- 6 | -- Dependencies 7 | -- 8 | 9 | local log = require('log') 10 | local fio = require('fio') 11 | local json = require('json') 12 | local msgpack = require('msgpack') 13 | local errno = require('errno') 14 | local fun = require('fun') 15 | local buffer = require('buffer') 16 | local ffi = require('ffi') 17 | 18 | -- Constants 19 | 20 | local BUFSIZ = 1024*1024 21 | 22 | -- Utility functions {{{ 23 | -- 24 | 25 | -- Return true if a space is a system one so we need to skip dumping 26 | -- it 27 | local function space_is_system(space_id) 28 | return space_id <= box.schema.SYSTEM_ID_MAX 29 | end 30 | 31 | -- Create a directory at a given path if it doesn't exist yet. 32 | -- If it does exist, check that it's empty. 33 | local function mkpath(path, interim) 34 | local stat = fio.stat(path) 35 | if stat then 36 | if not stat:is_dir() then 37 | return nil, string.format("File %s exists and is not a directory", 38 | path) 39 | end 40 | -- 41 | -- It's OK for interim dirs to contain files. 42 | -- 43 | if interim then 44 | return true 45 | end 46 | local files = fio.glob(fio.pathjoin(path, "*")) 47 | if not files then 48 | return nil, string.format("Failed to read directory %s, errno %d (%s)", 49 | path, errno(), errno.strerror()) 50 | end 51 | -- 52 | -- The leaf directory must be empty. 53 | -- 54 | if #files > 0 then 55 | return nil, string.format("Directory %s is not empty", path) 56 | end 57 | return true 58 | end 59 | local dir = fio.dirname(path) 60 | if not dir then 61 | return nil, string.format("Incorrect path: %s", path) 62 | end 63 | local stat = fio.stat(dir) 64 | if not stat then 65 | local status, msg = mkpath(dir, true) 66 | if not status then 67 | return nil, msg 68 | end 69 | end 70 | -- mkdir expects an octal mask :-/, pass in 0750 71 | if not fio.mkdir(path, 488) then 72 | return nil, string.format("Failed to create directory %s, errno %d (%s)", 73 | path, errno(), errno.strerror()); 74 | end 75 | return true 76 | end 77 | 78 | local croak = type(log.verbose) == 'function' and log.verbose or log.info 79 | 80 | local function box_is_configured() 81 | if type(box.cfg) == "function" then 82 | return nil, "Please start the database with box.cfg{} first" 83 | end 84 | return true 85 | end 86 | 87 | local function tuple_extract_key(tuple, space) 88 | return fun.map(function(x) return tuple[x.fieldno] end, 89 | space.index[0].parts):totable() 90 | end 91 | 92 | -- }}} Utility functions 93 | 94 | -- Dump stream {{{ 95 | -- 96 | 97 | -- Create a file in the dump directory 98 | local function begin_dump_space(stream, space_id) 99 | croak("Started dumping space %s", box.space[space_id].name) 100 | local path = fio.pathjoin(stream.path, string.format("%d.dump", space_id)) 101 | local fh = fio.open(path, {'O_APPEND', 'O_CREAT', 'O_WRONLY'}, {'S_IRUSR', 'S_IRGRP'}) 102 | if not fh then 103 | return nil, string.format("Can't open file %s, errno %d (%s)", 104 | path, errno(), errno.strerror()) 105 | end 106 | stream.files[space_id] = { 107 | path = path; 108 | fh = fh; 109 | rows = 0; 110 | buf = buffer.ibuf(BUFSIZ) 111 | } 112 | return true 113 | end 114 | 115 | -- Flush dump buffer 116 | local function flush_dump_stream(stream, space_id) 117 | local buf = stream.files[space_id].buf 118 | local fh = stream.files[space_id].fh 119 | local res = fh:write(buf.rpos, buf:size()) 120 | if not res then 121 | return nil, string.format("Failed to write to file %s, errno %d (%s)", 122 | stream.files[space_id].path, errno(), errno.strerror()) 123 | end 124 | buf:reset() 125 | return true 126 | end 127 | 128 | -- Close and sync a space dump file 129 | -- If the file is empty, don't clutter the dump and silently delete it 130 | -- instead. 131 | -- Update dump stats. 132 | local function end_dump_space(stream, space_id) 133 | local status, msg = flush_dump_stream(stream, space_id) 134 | if not status then 135 | return nil, msg 136 | end 137 | croak("Ended dumping space %s", box.space[space_id].name) 138 | local fh = stream.files[space_id].fh 139 | fh:fsync() 140 | if not fh:close() then 141 | return nil, string.format("Failed to close file %s, errno %d (%s)", 142 | stream.files[space_id].path, errno(), errno.strerror()) 143 | end 144 | local rows = stream.files[space_id].rows 145 | if rows == 0 then 146 | croak("Space %s was empty", box.space[space_id].name) 147 | fio.unlink(stream.files[space_id].path) 148 | else 149 | croak("Space %s had %d rows", box.space[space_id].name, rows) 150 | stream.spaces = stream.spaces + 1 151 | end 152 | stream.rows = stream.rows + rows 153 | stream.files[space_id] = nil 154 | return true 155 | end 156 | 157 | -- Write tuple data (in msgpack) to a file 158 | local function dump_tuple(stream, space_id, tuple) 159 | local buf = stream.files[space_id].buf 160 | msgpack.encode(tuple, buf) 161 | if buf:size() > BUFSIZ/2 then 162 | local status, msg = flush_dump_stream(stream, space_id) 163 | if not status then 164 | return nil, msg 165 | end 166 | end 167 | stream.files[space_id].rows = stream.files[space_id].rows + 1 168 | return true 169 | end 170 | 171 | -- Create a dump stream object for path 172 | -- Creates the path if it doesn't exist. 173 | local function dump_stream_new(dump_stream, path, opts) 174 | local dump_stream_vtab = { 175 | begin_dump_space = begin_dump_space; 176 | end_dump_space = end_dump_space; 177 | dump_tuple = dump_tuple; 178 | } 179 | local status, msg = mkpath(path) 180 | if not status then 181 | return nil, msg 182 | end 183 | local dump_object = { 184 | path = path; 185 | opts = opts; 186 | files = {}; 187 | spaces = 0; 188 | rows = 0; 189 | } 190 | setmetatable(dump_object, { __index = dump_stream_vtab; }) 191 | return dump_object 192 | end 193 | 194 | -- Dump stream module 195 | local dump_stream = 196 | { 197 | new = dump_stream_new 198 | } 199 | 200 | -- }}} Dump stream 201 | 202 | -- Restore stream {{{ 203 | -- 204 | 205 | -- Create a restore stream object for a path 206 | -- Scans the path, finds all dump files and prepares 207 | -- them for restore. 208 | local function restore_stream_new(restore_stream, path, opts) 209 | croak("Reading contents of %s", path) 210 | local stat = fio.stat(path) 211 | if not stat then 212 | return nil, string.format("Path %s does not exist", path) 213 | end 214 | if not stat:is_dir() then 215 | return nil, string.format("Path %s is not a directory", path) 216 | end 217 | local files = fio.glob(fio.pathjoin(path, "*.dump")) 218 | if not files then 219 | return nil, string.format("Failed to read %s, errno %d (%s)", path, 220 | errno(), errno.strerror()) 221 | end 222 | -- Convert "280.dump" -> 280 223 | local function to_id(file) 224 | return tonumber(string.match(fio.basename(file), "^%d+")) 225 | end 226 | files = fun.map(to_id, files):totable() 227 | -- Ensure restore processes spaces in the same order 228 | -- as checkpoint recovery 229 | table.sort(files) 230 | croak("Found %d files", #files) 231 | local restore_object = { 232 | path = path; 233 | opts = opts; 234 | files = files; 235 | spaces = 0; 236 | rows = 0; 237 | } 238 | return restore_object 239 | end 240 | 241 | -- Decode the next tuple stored in the stream buffer and 242 | -- advance the buffer position accordingly. Return nil if 243 | -- the decoder failed. 244 | local function space_stream_next_tuple(stream) 245 | local status, tuple, rpos = pcall(msgpack.decode, 246 | stream.buf.rpos, stream.buf:size()) 247 | if not status then 248 | return nil 249 | end 250 | stream.buf.rpos = rpos 251 | return box.tuple.new(tuple) 252 | end 253 | 254 | -- Read in more data from the dump file to the stream buffer. 255 | -- Return the number of bytes read on success, nil on failure. 256 | local function space_stream_read(stream) 257 | if stream.buf.rpos ~= stream.buf.buf then 258 | -- Move whatever is left in the buffer 259 | -- to the beginning. 260 | local buf = buffer.ibuf(BUFSIZ) 261 | ffi.copy(buf:alloc(stream.buf:size()), 262 | stream.buf.rpos, stream.buf:size()) 263 | stream.buf:recycle() 264 | stream.buf = buf 265 | end 266 | local len = stream.fh:read(stream.buf:reserve(BUFSIZ), BUFSIZ) 267 | if not len or len == 0 then 268 | stream.fh:close() 269 | end 270 | if not len then 271 | return nil, string.format("Failed to read file %s, errno %d, (%s)", 272 | stream.path, errno(), errno.strerror()) 273 | end 274 | stream.buf:alloc(len) 275 | return len 276 | end 277 | 278 | -- Restore data in a single space 279 | local function space_stream_restore(stream) 280 | croak("Started restoring space %s", stream.space.name) 281 | -- System spaces do not support multi-statement transactions. 282 | local TXN_ROWS = space_is_system(stream.space.id) and 1 or 200 283 | if TXN_ROWS > 1 then 284 | box.begin() 285 | end 286 | while true do 287 | local tuple = space_stream_next_tuple(stream) 288 | if not tuple then 289 | -- Commit the current transaction, because read() yields. 290 | if TXN_ROWS > 1 then 291 | box.commit() 292 | end 293 | local len, err = space_stream_read(stream) 294 | if not len then 295 | return false, err -- read error 296 | end 297 | if len == 0 and stream.buf:size() > 0 then 298 | return false, string.format("Failed to decode tuple: " .. 299 | "trailing bytes in the input stream %s", stream.path) 300 | end 301 | if TXN_ROWS > 1 then 302 | box.begin() 303 | end 304 | if len == 0 then 305 | break -- eof 306 | end 307 | else 308 | if stream.rows % TXN_ROWS == 1 then 309 | box.commit() 310 | box.begin() 311 | end 312 | tuple = stream.opts.filter(stream.space, tuple) 313 | if tuple ~= nil then 314 | stream.space:replace(tuple) 315 | stream.rows = stream.rows + 1 316 | end 317 | end 318 | end 319 | if TXN_ROWS > 1 then 320 | box.commit() 321 | end 322 | croak("Loaded %d rows in space %s", stream.rows, stream.space.name) 323 | return true 324 | end 325 | 326 | -- Create a new stream to restore a single space 327 | local function space_stream_new(dir, space_id, opts) 328 | local space = box.space[space_id] 329 | if space == nil then 330 | return nil, string.format("The dump directory is missing metadata for space %d", 331 | space_id) 332 | end 333 | local path = fio.pathjoin(dir, string.format("%d.dump", space_id)) 334 | local fh = fio.open(path, {'O_RDONLY'}) 335 | if fh == nil then 336 | return nil, string.format("Can't open file '%s', errno %d (%s)", 337 | path, errno(), errno.strerror()) 338 | end 339 | local space_stream = { 340 | space = space; 341 | path = path; 342 | opts = opts; 343 | fh = fh; 344 | rows = 0; 345 | buf = buffer.ibuf(BUFSIZ); 346 | } 347 | local space_stream_vtab = { 348 | restore = space_stream_restore; 349 | } 350 | setmetatable(space_stream, { __index = space_stream_vtab }) 351 | return space_stream 352 | end 353 | 354 | -- Restore stream module 355 | local restore_stream = { 356 | new = restore_stream_new 357 | } 358 | 359 | -- }}} Restore stream 360 | 361 | -- {{{ Database-wide dump and restore 362 | 363 | local common_opts_template = { 364 | filter = function(space, tuple) return tuple end, 365 | } 366 | 367 | local dump_opts_template = common_opts_template 368 | local restore_opts_template = common_opts_template 369 | 370 | local function check_opts(opts, template) 371 | opts = opts or {} 372 | for k, v in pairs(opts) do 373 | local default = template[k] 374 | if default == nil then 375 | return nil, string.format("Invalid option '%s'", k) 376 | end 377 | if type(default) ~= type(v) then 378 | return nil, string.format("Invalid value for option '%s': expected %s, got %s", 379 | k, type(default), type(v)) 380 | end 381 | end 382 | for k, v in pairs(template) do 383 | if opts[k] == nil then 384 | opts[k] = v 385 | end 386 | end 387 | return opts 388 | end 389 | 390 | -- 391 | -- Dump data from a single space into a stream. 392 | -- Apply filter. 393 | -- 394 | local function dump_space(stream, space, filter) 395 | local space_id = space.id 396 | local status, msg = stream:begin_dump_space(space_id) 397 | if not status then 398 | return nil, msg 399 | end 400 | 401 | -- iterate in batches using GT iterator 402 | -- HASH index supports GT iterator and, in absence 403 | -- of data changes, re-positions the iterator 404 | -- to the bucket where the key resides, if the key 405 | -- is specified 406 | local options = {iterator = 'GT', limit = 200} 407 | local last_key 408 | local batch 409 | if not space.index[0] then 410 | batch = {} 411 | else 412 | batch = space:select({}, options) 413 | end 414 | while #batch > 0 do 415 | for _, v in ipairs(batch) do 416 | if filter and filter(v) then 417 | goto continue 418 | end 419 | v = stream.opts.filter(space, v) 420 | if v == nil then 421 | goto continue 422 | end 423 | local status, msg = stream:dump_tuple(space_id, v) 424 | if not status then 425 | stream:end_dump_space(space_id) 426 | return nil, msg 427 | end 428 | ::continue:: 429 | end 430 | last_key = tuple_extract_key(batch[#batch], space) 431 | batch = space:select(last_key, options) 432 | end 433 | 434 | -- end of dump 435 | local status, msg = stream:end_dump_space(space_id) 436 | if not status then 437 | return nil, msg 438 | end 439 | return true 440 | end 441 | 442 | -- Filter out all system spaces 443 | local function space__space_filter(tuple) 444 | return space_is_system(tuple[1]) 445 | end 446 | 447 | -- 448 | -- Dump system spaces, but skip metadata of system spaces, 449 | -- system functions, users, roles and grants. 450 | -- Then dump all other spaces. 451 | -- 452 | local function dump(path, opts) 453 | local opts, msg = check_opts(opts, dump_opts_template) 454 | if not opts then 455 | return nil, msg 456 | end 457 | local status, msg = box_is_configured() 458 | if not status then 459 | return nil, msg 460 | end 461 | local stream, msg = dump_stream:new(path, opts) 462 | if not stream then 463 | return nil, msg 464 | end 465 | -- 466 | -- Dump system spaces: apply a filter to not dump 467 | -- system data, which is also stored in system spaces. 468 | -- This data will already exist at restore. 469 | -- 470 | local status, msg = dump_space(stream, box.space._space, space__space_filter) 471 | if not status then 472 | return nil, msg 473 | end 474 | local status, msg = dump_space(stream, box.space._index, space__space_filter) 475 | if not status then 476 | return nil, msg 477 | end 478 | 479 | -- dump all other spaces 480 | for k, v in box.space._space:pairs() do 481 | local space_id = v[1] 482 | if space_is_system(space_id) then 483 | goto continue 484 | end 485 | local status, msg = dump_space(stream, box.space[space_id]) 486 | if not status then 487 | return nil, msg 488 | end 489 | ::continue:: 490 | end 491 | return { spaces = stream.spaces; rows = stream.rows; } 492 | end 493 | 494 | -- 495 | -- Restore all spaces from the backup stored at the given path. 496 | -- 497 | local function restore(path, opts) 498 | local opts, msg = check_opts(opts, restore_opts_template) 499 | if not opts then 500 | return nil, msg 501 | end 502 | local status, msg = box_is_configured() 503 | if not status then 504 | return nil, msg 505 | end 506 | local stream, msg = restore_stream:new(path, opts) 507 | if not stream then 508 | return nil, msg 509 | end 510 | -- 511 | -- Iterate over all spaces in the path, and restore data 512 | for k, space_id in pairs(stream.files) do 513 | -- 514 | -- The restore stream iterates over system spaces first, 515 | -- so all user defined spaces should be created by the time 516 | -- they are restored 517 | -- 518 | local space_stream, msg = space_stream_new(stream.path, space_id, opts) 519 | if not space_stream then 520 | return nil, msg 521 | end 522 | local status, msg = space_stream:restore() 523 | if not status then 524 | return nil, msg 525 | end 526 | stream.spaces = stream.spaces + 1 527 | stream.rows = stream.rows + space_stream.rows 528 | end 529 | return { spaces = stream.spaces; rows = stream.rows } 530 | end 531 | 532 | -- }}} Database-wide dump and restore 533 | 534 | -- Exported functions 535 | -- 536 | 537 | -- result returned from require('dump') 538 | return { 539 | dump = dump; 540 | restore = restore; 541 | } 542 | -- vim: ts=4 sts=4 sw=4 et 543 | -------------------------------------------------------------------------------- /rpm/prebuild.sh: -------------------------------------------------------------------------------- 1 | curl -s https://packagecloud.io/install/repositories/tarantool/1_10/script.rpm.sh | sudo bash 2 | -------------------------------------------------------------------------------- /rpm/tarantool-dump.spec: -------------------------------------------------------------------------------- 1 | Name: tarantool-dump 2 | Version: 1.0.0 3 | Release: 1%{?dist} 4 | Summary: Logical backup and restore for Tarantool 5 | Group: Applications/Databases 6 | License: BSD 7 | URL: https://github.com/tarantool/dump 8 | Source0: dump-%{version}.tar.gz 9 | BuildArch: noarch 10 | BuildRequires: tarantool >= 1.9.0 11 | Requires: tarantool >= 1.9.0 12 | 13 | %description 14 | This package provides logical dump and restore for Tarantool. 15 | 16 | %prep 17 | %setup -q -n dump-%{version} 18 | 19 | %check 20 | ./test/dump.test.lua 21 | 22 | %install 23 | # Create /usr/share/tarantool/dump 24 | mkdir -p %{buildroot}%{_datadir}/tarantool/dump 25 | # Copy init.lua to /usr/share/tarantool/dump/init.lua 26 | cp -p dump/*.lua %{buildroot}%{_datadir}/tarantool/dump 27 | 28 | %files 29 | %dir %{_datadir}/tarantool/dump 30 | %{_datadir}/tarantool/dump/ 31 | %doc README.md 32 | %{!?_licensedir:%global license %doc} 33 | %license LICENSE AUTHORS 34 | 35 | %changelog 36 | * Fri Jun 16 2017 Konstantin Osipov 1.0.0-1 37 | - Initial release 38 | -------------------------------------------------------------------------------- /test/dump.test.lua: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env tarantool 2 | 3 | local dump = require('dump') 4 | local tap = require('tap') 5 | local fio = require('fio') 6 | local fiber = require('fiber') 7 | local json = require('json') 8 | 9 | local test = tap.test('dump tests') 10 | 11 | local function rmpath(path) 12 | local stat = fio.stat(path) 13 | if not stat then 14 | return 15 | end 16 | if stat:is_dir() then 17 | for k, file in pairs(fio.glob(fio.pathjoin(path, "*"))) do 18 | rmpath(file) 19 | end 20 | fio.rmdir(path) 21 | else 22 | fio.unlink(path) 23 | end 24 | end 25 | 26 | function basic(test) 27 | test:plan(10) 28 | test:is(type(dump.dump), "function", "Dump function is present") 29 | test:is(type(dump.restore), "function", "Restore function is present") 30 | local status, msg = dump.dump('dir', {invalid = true}) 31 | test:is(status, nil, "Unknown dump option - status") 32 | test:is(msg, "Invalid option 'invalid'", "Unknown dump option - message") 33 | local status, msg = dump.restore('dir', {invalid = true}) 34 | test:is(status, nil, "Unknown restore option - status") 35 | test:is(msg, "Invalid option 'invalid'", "Unknown restore option - message") 36 | local status, msg = dump.dump('dir', {filter = true}) 37 | test:is(status, nil, "Invalid value of dump option - status") 38 | test:is(msg, "Invalid value for option 'filter': expected function, got boolean", 39 | "Invalid value of dump option - message") 40 | local status, msg = dump.restore('dir', {filter = true}) 41 | test:is(status, nil, "Invalid value of restore option - status") 42 | test:is(msg, "Invalid value for option 'filter': expected function, got boolean", 43 | "Invalid value of restore option - message") 44 | end 45 | 46 | function box_is_configured(test) 47 | test:plan(4) 48 | local status, msg = dump.dump("/tmp/1") 49 | test:is(status, nil, "Dump before box.cfg{} status") 50 | test:like(msg, "box.cfg", "Dump before box.cfg{} message") 51 | local status, msg = dump.restore("/tmp/1") 52 | test:is(status, nil, "Restore before box.cfg{} status") 53 | test:like(msg, "box.cfg", "Restore before box.cfg{} message") 54 | end 55 | 56 | function dump_and_restore(test) 57 | test:plan(1) 58 | local ROWS = 2000 59 | local TXN_ROWS = 100 60 | box.schema.space.create('memtx') 61 | box.space.memtx:create_index('pk') 62 | box.schema.space.create('vinyl', {engine = 'vinyl'}) 63 | box.space.vinyl:create_index('pk') 64 | for i = 0, ROWS/TXN_ROWS - 1 do 65 | box.begin() 66 | for j = 1, TXN_ROWS do 67 | box.space.memtx:insert{i*TXN_ROWS + j, fiber.time()} 68 | end 69 | box.commit() 70 | end 71 | for i = 0, ROWS/TXN_ROWS - 1 do 72 | box.begin() 73 | for j = 1, TXN_ROWS do 74 | box.space.vinyl:insert{i*TXN_ROWS + j, fiber.time()} 75 | end 76 | box.commit() 77 | end 78 | local dir = fio.tempdir() 79 | dump.dump(dir) 80 | box.space.memtx:drop() 81 | box.space.vinyl:drop() 82 | dump.restore(dir) 83 | local rows = 0 84 | for i = 1, ROWS do 85 | local i1 = box.space.memtx:get{i} 86 | local i2 = box.space.vinyl:get{i} 87 | if i1 and i2 and i1[1] == i2[1] and i1[1] == i then 88 | rows = rows + 1 89 | end 90 | end 91 | test:is(rows, ROWS, "The number of rows is correct after restore") 92 | box.space.memtx:drop() 93 | box.space.vinyl:drop() 94 | rmpath(dir) 95 | end 96 | 97 | local function dump_access_denied(test) 98 | test:plan(2) 99 | local dir = fio.tempdir() 100 | fio.chmod(dir, 0) 101 | local status, msg = dump.dump(dir) 102 | test:is(status, nil, "Dump access denied returns error") 103 | test:diag(msg) 104 | test:is(type(msg), "string", "Dump access denied provides error message") 105 | rmpath(dir) 106 | end 107 | 108 | local function dump_after_dump(test) 109 | test:plan(2) 110 | local dir = fio.tempdir() 111 | box.schema.space.create('test') 112 | local status, msg = dump.dump(dir) 113 | test:is(not status, false, "First dump is successful") 114 | local status, msg = dump.dump(dir) 115 | test:is(not status, true, "Second dump fails") 116 | box.space.test:drop() 117 | rmpath(dir) 118 | end 119 | 120 | local function restore_no_such_path(test) 121 | test:plan(2) 122 | local dir = fio.tempdir() 123 | rmpath(dir) 124 | local status, msg = dump.restore(dir) 125 | test:is(type(status), "nil", "Restore of a non-existent dir returns error") 126 | test:is(type(msg), "string", "Restore of a non-existent dir provides error message") 127 | end 128 | 129 | local function dump_hash_index(test) 130 | test:plan(2) 131 | local ROWS = 2000 132 | local TXN_ROWS = 100 133 | box.schema.space.create('hash') 134 | box.space.hash:create_index('pk', {type='hash', parts = {1, 'str'}}) 135 | for i = 0, ROWS/TXN_ROWS - 1 do 136 | box.begin() 137 | for j = 1, TXN_ROWS do 138 | box.space.hash:insert{tostring(i*TXN_ROWS + j), fiber.time()} 139 | end 140 | box.commit() 141 | end 142 | local dir = fio.tempdir() 143 | dump.dump(dir) 144 | box.space.hash:drop() 145 | dump.restore(dir) 146 | test:is(box.space.hash.index[0].type, 'HASH', 147 | "Restored space has HASH primary key") 148 | local rows = 0 149 | for i = 1, ROWS do 150 | local key = tostring(i) 151 | local i1 = box.space.hash:get{key} 152 | if i1 and i1[1] == key then 153 | rows = rows + 1 154 | end 155 | end 156 | test:is(rows, ROWS, "The number of rows is correct after restore") 157 | box.space.hash:drop() 158 | rmpath(dir) 159 | end 160 | 161 | local function dump_filter(test) 162 | test:plan(3) 163 | local space = box.schema.space.create('test') 164 | local space_id = space.id 165 | space:create_index('pk') 166 | space:insert{1, 'ignore'} 167 | space:insert{2, 'update'} 168 | space:insert{3, 'filter'} 169 | local dir = fio.tempdir() 170 | dump.dump(dir, { 171 | filter = function(space, tuple) 172 | if space.id ~= space_id then return tuple end 173 | if tuple[2] == 'ignore' then return tuple end 174 | if tuple[2] == 'filter' then return nil end 175 | if tuple[2] == 'update' then 176 | return tuple:update{{'=', 2, 'updated'}} 177 | end 178 | end, 179 | }) 180 | space:drop() 181 | dump.restore(dir) 182 | space = box.space.test 183 | test:is(space:get(1)[2], 'ignore', "Dump filter can ignore a tuple") 184 | test:is(space:get(2)[2], 'updated', "Dump filter can update a tuples") 185 | test:is(space:get(3), nil, "Dump filter can filter out a tuple") 186 | space:drop() 187 | rmpath(dir) 188 | end 189 | 190 | local function restore_filter(test) 191 | test:plan(3) 192 | local space = box.schema.space.create('test') 193 | local space_id = space.id 194 | space:create_index('pk') 195 | space:insert{1, 'ignore'} 196 | space:insert{2, 'update'} 197 | space:insert{3, 'filter'} 198 | local dir = fio.tempdir() 199 | dump.dump(dir) 200 | space:drop() 201 | dump.restore(dir, { 202 | filter = function(space, tuple) 203 | if space.id ~= space_id then return tuple end 204 | if tuple[2] == 'ignore' then return tuple end 205 | if tuple[2] == 'filter' then return nil end 206 | if tuple[2] == 'update' then 207 | return tuple:update{{'=', 2, 'updated'}} 208 | end 209 | end, 210 | }) 211 | space = box.space.test 212 | test:is(space:get(1)[2], 'ignore', "Restore filter can ignore a tuple") 213 | test:is(space:get(2)[2], 'updated', "Restore filter can update a tuple") 214 | test:is(space:get(3), nil, "Restore filter can filter out a tuple") 215 | space:drop() 216 | rmpath(dir) 217 | end 218 | 219 | test:plan(9) 220 | 221 | test:test('Basics', basic) 222 | test:test('Using the rock without calling box.cfg{}', box_is_configured) 223 | 224 | box.cfg{log_level=6} 225 | 226 | test:test('Dump and restore', dump_and_restore) 227 | test:test('Dump into a non-writable directory', dump_access_denied) 228 | test:test('Dump into a non-empty directory', dump_after_dump) 229 | test:test('Restore of a non-existent path', restore_no_such_path) 230 | test:test('Dump and restore of a space with HASH primary key', dump_hash_index) 231 | test:test('Dump filter', dump_filter) 232 | test:test('Restore filter', restore_filter) 233 | 234 | os.exit(test:check() == true and 0 or -1) 235 | --------------------------------------------------------------------------------