├── crud ├── version.lua ├── common │ ├── dev_checks.lua │ ├── const.lua │ ├── call_cache.lua │ ├── sharding_key.lua │ ├── sharding │ │ ├── utils.lua │ │ ├── router_metadata_cache.lua │ │ └── storage_metadata_cache.lua │ ├── sharding_func.lua │ ├── roles.lua │ ├── batching_utils.lua │ ├── collations.lua │ ├── compat.lua │ ├── map_call_cases │ │ ├── base_iter.lua │ │ ├── batch_postprocessor.lua │ │ ├── base_postprocessor.lua │ │ ├── batch_insert_iter.lua │ │ └── batch_upsert_iter.lua │ └── stash.lua ├── select │ └── compat │ │ └── common.lua ├── stats │ ├── operation.lua │ └── registry_utils.lua ├── truncate.lua ├── compare │ ├── keydef.lua │ ├── type_comparators.lua │ └── conditions.lua ├── len.lua ├── ratelimit.lua ├── schema.lua ├── storage_info.lua └── storage.lua ├── .github ├── pull_request_template.md └── workflows │ ├── check_on_push.yaml │ ├── reusable_test.yml │ └── push_rockspec.yaml ├── .luacheckrc ├── .luacov ├── cmake ├── FindLuaCheck.cmake ├── FindLuaCov.cmake ├── FindLuaTest.cmake └── FindLuaCovCoveralls.cmake ├── test ├── entrypoint │ ├── srv_ddl │ │ ├── router.lua │ │ └── cartridge_init.lua │ ├── srv_ddl_reload │ │ ├── router.lua │ │ └── cartridge_init.lua │ ├── srv_migration │ │ ├── storage.lua │ │ └── cartridge_init.lua │ ├── srv_read_calls_strategies │ │ ├── storage.lua │ │ ├── all.lua │ │ └── cartridge_init.lua │ ├── srv_not_initialized │ │ ├── storage.lua │ │ └── cartridge_init.lua │ ├── srv_say_hi │ │ ├── cartridge_init.lua │ │ └── all.lua │ ├── srv_schema │ │ ├── cartridge_init.lua │ │ └── storage.lua │ ├── srv_batch_operations │ │ ├── cartridge_init.lua │ │ └── storage.lua │ ├── srv_bucket_id_pk │ │ ├── cartridge_init.lua │ │ └── storage.lua │ ├── srv_update_schema │ │ ├── cartridge_init.lua │ │ └── storage.lua │ ├── srv_simple_operations │ │ └── cartridge_init.lua │ ├── srv_select │ │ └── cartridge_init.lua │ ├── srv_stats │ │ ├── storage.lua │ │ └── cartridge_init.lua │ └── srv_reload │ │ └── cartridge_init.lua ├── path.lua ├── unit │ ├── utils_append_array_test.lua │ ├── privileges_test.lua │ ├── validate_bucket_id_test.lua │ ├── select_dropped_indexes_test.lua │ ├── select_plan_bad_indexes_test.lua │ ├── parse_conditions_test.lua │ ├── cut_result_test.lua │ ├── serialization_test.lua │ └── not_initialized_test.lua ├── vshard_helpers │ ├── instances │ │ ├── utils.lua │ │ ├── router.lua │ │ └── storage.lua │ ├── vclock.lua │ └── cluster.lua ├── integration │ ├── readview_not_supported_test.lua │ ├── write_scenario.lua │ ├── len_test.lua │ ├── migration_test.lua │ ├── async_bootstrap_test.lua │ ├── truncate_test.lua │ ├── role_test.lua │ ├── read_calls_strategies_test.lua │ └── cartridge_reload_test.lua ├── luacov-merger.lua ├── tarantool3_helpers │ ├── utils.lua │ └── treegen.lua └── doc │ └── playground_test.lua ├── .gitignore ├── cartridge └── roles │ ├── crud-storage.lua │ └── crud-router.lua ├── crud-scm-1.rockspec ├── AUTHORS ├── luarocks.patch ├── roles ├── crud-storage.lua └── crud-router.lua ├── deps.sh ├── LICENSE ├── Makefile ├── CMakeLists.txt └── doc └── playground.lua /crud/version.lua: -------------------------------------------------------------------------------- 1 | -- Сontains the module version. 2 | -- Requires manual update in case of release commit. 3 | 4 | return '1.6.1' 5 | -------------------------------------------------------------------------------- /.github/pull_request_template.md: -------------------------------------------------------------------------------- 1 | What has been done? Why? What problem is being solved? 2 | 3 | I didn't forget about 4 | 5 | - [ ] Tests 6 | - [ ] Changelog 7 | - [ ] Documentation 8 | 9 | Closes #??? 10 | -------------------------------------------------------------------------------- /crud/common/dev_checks.lua: -------------------------------------------------------------------------------- 1 | local checks = require('checks') 2 | 3 | local dev_checks = function() end 4 | 5 | if os.getenv('TARANTOOL_CRUD_ENABLE_INTERNAL_CHECKS') == 'ON' then 6 | dev_checks = checks 7 | end 8 | 9 | return dev_checks 10 | -------------------------------------------------------------------------------- /.luacheckrc: -------------------------------------------------------------------------------- 1 | redefined = false 2 | globals = {'box', 'utf8', 'checkers', '_TARANTOOL'} 3 | include_files = {'**/*.lua', '*.luacheckrc', '*.rockspec'} 4 | exclude_files = {'**/*.rocks/', 'tmp/', 'sdk-*'} 5 | max_line_length = 120 6 | max_comment_line_length = 150 7 | -------------------------------------------------------------------------------- /.luacov: -------------------------------------------------------------------------------- 1 | local ci_node_index = os.getenv("CI_NODE_INDEX") or "" 2 | 3 | statsfile = "luacov.stats" .. ci_node_index .. ".out" 4 | exclude = { 5 | "test" 6 | } 7 | include = { 8 | "crud%/.+$", 9 | "crud.lua", 10 | "roles%/.+$", 11 | "cartridge%/.+$", 12 | } 13 | -------------------------------------------------------------------------------- /cmake/FindLuaCheck.cmake: -------------------------------------------------------------------------------- 1 | find_program(LUACHECK luacheck 2 | HINTS .rocks/ 3 | PATH_SUFFIXES bin 4 | DOC "Lua linter" 5 | ) 6 | 7 | include(FindPackageHandleStandardArgs) 8 | find_package_handle_standard_args(LuaCheck 9 | REQUIRED_VARS LUACHECK 10 | ) 11 | 12 | mark_as_advanced(LUACHECK) 13 | -------------------------------------------------------------------------------- /cmake/FindLuaCov.cmake: -------------------------------------------------------------------------------- 1 | find_program(LUACOV luacov 2 | HINTS .rocks/ 3 | PATH_SUFFIXES bin 4 | DOC "Lua test coverage analysis" 5 | ) 6 | 7 | include(FindPackageHandleStandardArgs) 8 | find_package_handle_standard_args(LuaCov 9 | REQUIRED_VARS LUACOV 10 | ) 11 | 12 | mark_as_advanced(LUACOV) 13 | -------------------------------------------------------------------------------- /cmake/FindLuaTest.cmake: -------------------------------------------------------------------------------- 1 | find_program(LUATEST luatest 2 | HINTS .rocks/ 3 | PATH_SUFFIXES bin 4 | DOC "Lua testing framework" 5 | ) 6 | 7 | include(FindPackageHandleStandardArgs) 8 | find_package_handle_standard_args(LuaTest 9 | REQUIRED_VARS LUATEST 10 | ) 11 | 12 | mark_as_advanced(LUATEST) 13 | -------------------------------------------------------------------------------- /cmake/FindLuaCovCoveralls.cmake: -------------------------------------------------------------------------------- 1 | find_program(LUACOVCOVERALLS luacov-coveralls 2 | HINTS .rocks/ 3 | PATH_SUFFIXES bin 4 | DOC "LuaCov reporter for coveralls.io service" 5 | ) 6 | 7 | include(FindPackageHandleStandardArgs) 8 | find_package_handle_standard_args(LuaCovCoveralls 9 | REQUIRED_VARS LUACOVCOVERALLS 10 | ) 11 | 12 | mark_as_advanced(LUACOVCOVERALLS) 13 | -------------------------------------------------------------------------------- /test/entrypoint/srv_ddl/router.lua: -------------------------------------------------------------------------------- 1 | return { 2 | init = function() 3 | local some_module = { 4 | sharding_func = function(key) 5 | if key ~= nil and key[1] ~= nil then 6 | return key[1] % 10 7 | end 8 | end 9 | } 10 | 11 | rawset(_G, 'some_module', some_module) 12 | end, 13 | } 14 | -------------------------------------------------------------------------------- /test/path.lua: -------------------------------------------------------------------------------- 1 | local fio = require('fio') 2 | 3 | local ROOT = fio.dirname(fio.dirname(fio.abspath(package.search('test.path')))) 4 | 5 | local LUA_PATH = ROOT .. '/?.lua;' .. 6 | ROOT .. '/?/init.lua;' .. 7 | ROOT .. '/.rocks/share/tarantool/?.lua;' .. 8 | ROOT .. '/.rocks/share/tarantool/?/init.lua' 9 | 10 | return { 11 | ROOT = ROOT, 12 | LUA_PATH = LUA_PATH, 13 | } 14 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | build.luarocks/ 2 | .rocks/ 3 | myapp/ 4 | **.snap 5 | **.xlog 6 | .tarantool.cookie 7 | CMakeCache.txt 8 | CMakeFiles/ 9 | luacov.report.out 10 | luacov.stats.out 11 | build/*.cmake 12 | build/Makefile 13 | .idea 14 | sdk 15 | sdk-2 16 | sdk-3 17 | 18 | # Vim Swap files. 19 | .*.s[a-w][a-z] 20 | 21 | # tt files. 22 | tt.yaml 23 | bin/ 24 | distfiles/ 25 | include/ 26 | instances.enabled/ 27 | modules/ 28 | templates/ 29 | -------------------------------------------------------------------------------- /crud/common/const.lua: -------------------------------------------------------------------------------- 1 | local const = {} 2 | 3 | const.RELOAD_RETRIES_NUM = 1 4 | const.RELOAD_SCHEMA_TIMEOUT = 3 -- 3 seconds 5 | const.FETCH_SHARDING_METADATA_TIMEOUT = 3 -- 3 seconds 6 | const.SHARDING_RELOAD_RETRIES_NUM = 1 7 | 8 | const.NEED_SCHEMA_RELOAD = 0x0001000 9 | const.NEED_SHARDING_RELOAD = 0x0001001 10 | 11 | const.DEFAULT_VSHARD_CALL_TIMEOUT = 2 12 | 13 | const.DEFAULT_YIELD_EVERY = 1000 14 | 15 | return const 16 | -------------------------------------------------------------------------------- /test/unit/utils_append_array_test.lua: -------------------------------------------------------------------------------- 1 | local t = require("luatest") 2 | local g = t.group() 3 | 4 | local utils = require("crud.common.utils") 5 | 6 | g.test_append_void = function() 7 | local res = utils.append_array({"too, foo"}) 8 | t.assert_equals(res, {"too, foo"}) 9 | end 10 | 11 | g.test_concat = function() 12 | local res = utils.append_array({"too, foo"}, {"bar, baz, buzz"}) 13 | t.assert_equals(res, {"too, foo", "bar, baz, buzz"}) 14 | end 15 | -------------------------------------------------------------------------------- /cartridge/roles/crud-storage.lua: -------------------------------------------------------------------------------- 1 | local crud = require('crud') 2 | local stash = require('crud.common.stash') 3 | 4 | local function init() 5 | crud.init_storage() 6 | stash.setup_cartridge_reload() 7 | end 8 | 9 | local function stop() 10 | crud.stop_storage() 11 | end 12 | 13 | return { 14 | role_name = 'crud-storage', 15 | init = init, 16 | stop = stop, 17 | implies_storage = true, 18 | dependencies = {'cartridge.roles.vshard-storage'}, 19 | } 20 | -------------------------------------------------------------------------------- /crud-scm-1.rockspec: -------------------------------------------------------------------------------- 1 | package = 'crud' 2 | version = 'scm-1' 3 | source = { 4 | url = 'git+https://github.com/tarantool/crud.git', 5 | branch = 'master', 6 | } 7 | 8 | description = { 9 | license = 'BSD', 10 | } 11 | 12 | dependencies = { 13 | 'lua ~> 5.1', 14 | 'checks >= 3.3.0-1', 15 | 'errors >= 2.2.1-1', 16 | 'vshard >= 0.1.36-1', 17 | } 18 | 19 | build = { 20 | type = 'cmake', 21 | variables = { 22 | version = 'scm-1', 23 | TARANTOOL_INSTALL_LUADIR = '$(LUADIR)', 24 | }, 25 | } 26 | -------------------------------------------------------------------------------- /test/entrypoint/srv_ddl_reload/router.lua: -------------------------------------------------------------------------------- 1 | return { 2 | init = function() 3 | local customers_module = { 4 | sharding_func_default = function(key) 5 | local id = key[1] 6 | assert(id ~= nil) 7 | 8 | return id % 3000 + 1 9 | end, 10 | sharding_func_new = function(key) 11 | local id = key[1] 12 | assert(id ~= nil) 13 | 14 | return (id + 42) % 3000 + 1 15 | end, 16 | } 17 | rawset(_G, 'customers_module', customers_module) 18 | end, 19 | } 20 | -------------------------------------------------------------------------------- /AUTHORS: -------------------------------------------------------------------------------- 1 | In the order of a first contribution. 2 | 3 | Elizaveta Dokshina 4 | Elena Shebunyaeva 5 | Oleg Babin 6 | Sergey Volgin 7 | Konstantin Nazarov 8 | Andreev Semen 9 | Alexey Kuzin 10 | Alexander Turenko 11 | Alexey Romanov 12 | Anastasia Neklepaeva 13 | Yaroslav Dynnikov 14 | Vladimir Rogach 15 | Sergey Bronnikov 16 | Georgy Moiseev 17 | -------------------------------------------------------------------------------- /test/vshard_helpers/instances/utils.lua: -------------------------------------------------------------------------------- 1 | local fun = require('fun') 2 | local json = require('json') 3 | 4 | local utils = {} 5 | 6 | local function default_cfg() 7 | return { 8 | work_dir = os.getenv('TARANTOOL_WORKDIR'), 9 | listen = os.getenv('TARANTOOL_LISTEN'), 10 | } 11 | end 12 | 13 | local function env_cfg() 14 | local src = os.getenv('TARANTOOL_BOX_CFG') 15 | if src == nil then 16 | return {} 17 | end 18 | local res = json.decode(src) 19 | assert(type(res) == 'table') 20 | return res 21 | end 22 | 23 | function utils.box_cfg(cfg) 24 | return fun.chain(default_cfg(), env_cfg(), cfg or {}):tomap() 25 | end 26 | 27 | return utils 28 | -------------------------------------------------------------------------------- /test/entrypoint/srv_migration/storage.lua: -------------------------------------------------------------------------------- 1 | local helper = require('test.helper') 2 | 3 | return { 4 | init = helper.wrap_schema_init(function() 5 | box.schema.space.create('customers', {if_not_exists = true}) 6 | 7 | box.space['customers']:format{ 8 | {name = 'id', is_nullable = false, type = 'unsigned'}, 9 | {name = 'bucket_id', is_nullable = false, type = 'unsigned'}, 10 | {name = 'sharding_key', is_nullable = false, type = 'unsigned'}, 11 | } 12 | 13 | box.space['customers']:create_index('pk', {parts = { 'id' }, if_not_exists = true}) 14 | box.space['customers']:create_index('bucket_id', {parts = { 'bucket_id' }, if_not_exists = true}) 15 | end), 16 | wait_until_ready = helper.wait_schema_init, 17 | } 18 | -------------------------------------------------------------------------------- /test/entrypoint/srv_read_calls_strategies/storage.lua: -------------------------------------------------------------------------------- 1 | local helper = require('test.helper') 2 | 3 | return { 4 | init = helper.wrap_schema_init(function() 5 | local engine = os.getenv('ENGINE') or 'memtx' 6 | local customers_space = box.schema.space.create('customers', { 7 | format = { 8 | {name = 'id', type = 'unsigned'}, 9 | {name = 'bucket_id', type = 'unsigned'}, 10 | {name = 'name', type = 'string'}, 11 | {name = 'age', type = 'number'}, 12 | }, 13 | if_not_exists = true, 14 | engine = engine, 15 | }) 16 | customers_space:create_index('id', { 17 | parts = { {field = 'id'} }, 18 | if_not_exists = true, 19 | }) 20 | end), 21 | wait_until_ready = helper.wait_schema_init, 22 | } 23 | -------------------------------------------------------------------------------- /crud/common/call_cache.lua: -------------------------------------------------------------------------------- 1 | local func_name_to_func_cache = {} 2 | 3 | local function func_name_to_func(func_name) 4 | if func_name_to_func_cache[func_name] then 5 | return func_name_to_func_cache[func_name] 6 | end 7 | 8 | local current = _G 9 | for part in string.gmatch(func_name, "[^%.]+") do 10 | current = rawget(current, part) 11 | if current == nil then 12 | error(("Function '%s' is not registered"):format(func_name)) 13 | end 14 | end 15 | 16 | if type(current) ~= "function" then 17 | error(func_name .. " is not a function") 18 | end 19 | 20 | func_name_to_func_cache[func_name] = current 21 | return current 22 | end 23 | 24 | local function reset() 25 | func_name_to_func_cache = {} 26 | end 27 | 28 | return { 29 | func_name_to_func = func_name_to_func, 30 | reset = reset, 31 | } -------------------------------------------------------------------------------- /.github/workflows/check_on_push.yaml: -------------------------------------------------------------------------------- 1 | name: Run static analysis 2 | 3 | on: 4 | push: 5 | pull_request: 6 | 7 | jobs: 8 | run-static-analysis: 9 | if: | 10 | github.event_name == 'push' || 11 | github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository 12 | runs-on: ubuntu-22.04 13 | steps: 14 | - uses: actions/checkout@v4 15 | 16 | - name: Setup Tarantool CE 17 | uses: tarantool/setup-tarantool@v4 18 | with: 19 | tarantool-version: '2.11' 20 | 21 | - name: Setup tt 22 | run: | 23 | curl -L https://tarantool.io/release/2/installer.sh | sudo bash 24 | sudo apt install -y tt 25 | tt version 26 | 27 | - name: Setup luacheck 28 | run: | 29 | tt init 30 | tt rocks install luacheck 0.25.0 31 | 32 | - run: cmake -S . -B build 33 | 34 | - name: Run luacheck 35 | run: make -C build luacheck 36 | -------------------------------------------------------------------------------- /test/entrypoint/srv_not_initialized/storage.lua: -------------------------------------------------------------------------------- 1 | local helper = require('test.helper') 2 | 3 | return { 4 | init = helper.wrap_schema_init(function() 5 | local customers_space = box.schema.space.create('customers', { 6 | format = { 7 | {name = 'id', type = 'unsigned'}, 8 | {name = 'bucket_id', type = 'unsigned'}, 9 | {name = 'name', type = 'string'}, 10 | {name = 'age', type = 'number'}, 11 | }, 12 | if_not_exists = true, 13 | }) 14 | customers_space:create_index('id', { 15 | parts = { {field = 'id' } }, 16 | if_not_exists = true, 17 | }) 18 | customers_space:create_index('bucket_id', { 19 | parts = { {field = 'bucket_id' } }, 20 | unique = false, 21 | if_not_exists = true, 22 | }) 23 | end), 24 | wait_until_ready = helper.wait_schema_init, 25 | } 26 | -------------------------------------------------------------------------------- /crud/common/sharding_key.lua: -------------------------------------------------------------------------------- 1 | local log = require('log') 2 | 3 | local sharding_metadata_module = require('crud.common.sharding.sharding_metadata') 4 | local utils = require('crud.common.utils') 5 | 6 | local sharding_key_cache = {} 7 | 8 | -- This method is exported here because 9 | -- we already have customers using old API 10 | -- for updating sharding key cache in their 11 | -- projects like `require('crud.common.sharding_key').update_cache()` 12 | function sharding_key_cache.update_cache(space_name, vshard_router) 13 | log.warn("require('crud.common.sharding_key').update_cache()" .. 14 | "is deprecated and will be removed in future releases") 15 | 16 | local vshard_router, err = utils.get_vshard_router_instance(vshard_router) 17 | if err ~= nil then 18 | return nil, err 19 | end 20 | 21 | return sharding_metadata_module.update_sharding_key_cache(vshard_router, space_name) 22 | end 23 | 24 | return sharding_key_cache 25 | -------------------------------------------------------------------------------- /crud/select/compat/common.lua: -------------------------------------------------------------------------------- 1 | local ratelimit = require('crud.ratelimit') 2 | local utils = require('crud.common.utils') 3 | local check_select_safety_rl = ratelimit.new() 4 | 5 | local common = {} 6 | 7 | common.SELECT_FUNC_NAME = utils.get_storage_call('select_on_storage') 8 | common.READVIEW_SELECT_FUNC_NAME = utils.get_storage_call('select_readview_on_storage') 9 | common.DEFAULT_BATCH_SIZE = 100 10 | 11 | common.check_select_safety = function(space_name, plan, opts) 12 | if opts.fullscan == true then 13 | return 14 | end 15 | 16 | if opts.first ~= nil and math.abs(opts.first) <= 1000 then 17 | return 18 | end 19 | 20 | local iter = plan.tarantool_iter 21 | if iter == box.index.EQ or iter == box.index.REQ then 22 | return 23 | end 24 | 25 | local rl = check_select_safety_rl 26 | local traceback = debug.traceback() 27 | rl:log_crit("Potentially long select from space '%s'\n %s", space_name, traceback) 28 | end 29 | 30 | return common 31 | -------------------------------------------------------------------------------- /luarocks.patch: -------------------------------------------------------------------------------- 1 | diff --git a/src/luarocks/manif.lua b/src/luarocks/manif.lua 2 | index 34ae02da5..79a427819 100644 3 | --- a/src/luarocks/manif.lua 4 | +++ b/src/luarocks/manif.lua 5 | @@ -444,11 +444,10 @@ function manif.add_to_manifest(name, version, repo, deps_mode) 6 | 7 | local manifest, err = manif_core.load_local_manifest(rocks_dir) 8 | if not manifest then 9 | - util.printerr("No existing manifest. Attempting to rebuild...") 10 | - -- Manifest built by `manif.make_manifest` should already 11 | - -- include information about given name and version, 12 | - -- no need to update it. 13 | - return manif.make_manifest(rocks_dir, deps_mode) 14 | + util.printerr("No existing manifest. Creating an empty one...") 15 | + -- Create an empty manifest. 16 | + manifest, err = { repository = {}, modules = {}, commands = {} }, nil 17 | + manif_core.cache_manifest(rocks_dir, nil, manifest) 18 | end 19 | 20 | local results = {[name] = {[version] = {{arch = "installed", repo = rocks_dir}}}} 21 | -------------------------------------------------------------------------------- /crud/common/sharding/utils.lua: -------------------------------------------------------------------------------- 1 | local digest = require('digest') 2 | local errors = require('errors') 3 | local msgpack = require('msgpack') 4 | 5 | local utils = {} 6 | 7 | utils.SPACE_NAME_FIELDNO = 1 8 | utils.SPACE_SHARDING_KEY_FIELDNO = 2 9 | utils.SPACE_SHARDING_FUNC_NAME_FIELDNO = 2 10 | utils.SPACE_SHARDING_FUNC_BODY_FIELDNO = 3 11 | 12 | utils.ShardingHashMismatchError = errors.new_class("ShardingHashMismatchError", {capture_stack = false}) 13 | 14 | function utils.extract_sharding_func_def(tuple) 15 | if not tuple then 16 | return nil 17 | end 18 | 19 | if tuple[utils.SPACE_SHARDING_FUNC_BODY_FIELDNO] ~= nil then 20 | return {body = tuple[utils.SPACE_SHARDING_FUNC_BODY_FIELDNO]} 21 | end 22 | 23 | if tuple[utils.SPACE_SHARDING_FUNC_NAME_FIELDNO] ~= nil then 24 | return tuple[utils.SPACE_SHARDING_FUNC_NAME_FIELDNO] 25 | end 26 | 27 | return nil 28 | end 29 | 30 | function utils.compute_hash(val) 31 | return digest.murmur(msgpack.encode(val)) 32 | end 33 | 34 | return utils 35 | -------------------------------------------------------------------------------- /crud/common/sharding_func.lua: -------------------------------------------------------------------------------- 1 | local log = require('log') 2 | 3 | local sharding_metadata_module = require('crud.common.sharding.sharding_metadata') 4 | local utils = require('crud.common.utils') 5 | 6 | local sharding_func_cache = {} 7 | 8 | -- This method is exported here because 9 | -- we already have customers using old API 10 | -- for updating sharding key cache in their 11 | -- projects like `require('crud.common.sharding_key').update_cache()` 12 | -- This method provides similar behavior for 13 | -- sharding function cache. 14 | function sharding_func_cache.update_cache(space_name, vshard_router) 15 | log.warn("require('crud.common.sharding_func').update_cache()" .. 16 | "is deprecated and will be removed in future releases") 17 | 18 | local vshard_router, err = utils.get_vshard_router_instance(vshard_router) 19 | if err ~= nil then 20 | return nil, err 21 | end 22 | 23 | return sharding_metadata_module.update_sharding_func_cache(vshard_router, space_name) 24 | end 25 | 26 | return sharding_func_cache 27 | -------------------------------------------------------------------------------- /test/integration/readview_not_supported_test.lua: -------------------------------------------------------------------------------- 1 | local t = require('luatest') 2 | 3 | local helpers = require('test.helper') 4 | 5 | local pgroup = t.group('readview_not_supported', helpers.backend_matrix({ 6 | {engine = 'memtx'}, 7 | })) 8 | 9 | 10 | pgroup.before_all(function(g) 11 | if helpers.tarantool_version_at_least(2, 11, 0) 12 | and require('luatest.tarantool').is_enterprise_package() then 13 | t.skip("Readview is supported") 14 | end 15 | 16 | helpers.start_default_cluster(g, 'srv_select') 17 | 18 | g.space_format = g.cluster.servers[2].net_box.space.customers:format() 19 | end) 20 | 21 | pgroup.after_all(function(g) 22 | helpers.stop_cluster(g.cluster, g.params.backend) 23 | end) 24 | 25 | pgroup.test_open = function(g) 26 | local obj, err = g.router:eval([[ 27 | local crud = require('crud') 28 | local foo, err = crud.readview({name = 'foo'}) 29 | return foo, err 30 | ]]) 31 | 32 | t.assert_equals(obj, nil) 33 | t.assert_str_contains(err.str, 'Tarantool does not support readview') 34 | 35 | end 36 | -------------------------------------------------------------------------------- /roles/crud-storage.lua: -------------------------------------------------------------------------------- 1 | local errors = require('errors') 2 | 3 | local crud = require('crud') 4 | local common_role_utils = require('crud.common.roles') 5 | local common_utils = require('crud.common.utils') 6 | 7 | local TarantoolRoleConfigurationError = errors.new_class('TarantoolRoleConfigurationError', {capture_stack = false}) 8 | 9 | local tarantool_version = rawget(_G, '_TARANTOOL') 10 | TarantoolRoleConfigurationError:assert( 11 | common_utils.tarantool_supports_config_get_inside_roles() 12 | and common_utils.tarantool_role_privileges_not_revoked(), 13 | ('Tarantool 3 role is not supported for Tarantool %s, use 3.0.2 or newer'):format(tarantool_version) 14 | ) 15 | 16 | local function validate() 17 | TarantoolRoleConfigurationError:assert( 18 | common_role_utils.is_sharding_role_enabled('storage'), 19 | 'instance must be a sharding storage to enable roles.crud-storage' 20 | ) 21 | end 22 | 23 | local function apply() 24 | crud.init_storage{async = true} 25 | end 26 | 27 | local function stop() 28 | crud.stop_storage() 29 | end 30 | 31 | return { 32 | validate = validate, 33 | apply = apply, 34 | stop = stop, 35 | } 36 | -------------------------------------------------------------------------------- /crud/common/roles.lua: -------------------------------------------------------------------------------- 1 | local config = require('config') 2 | 3 | local function is_sharding_role_enabled(expected_sharding_role) 4 | -- Works only for versions newer than 3.0.1-10 (3.0.2 and following) 5 | -- https://github.com/tarantool/tarantool/commit/e0e1358cb60d6749c34daf508e05586e0959bf89 6 | -- and newer than 3.1.0-entrypoint-77 (3.1.0 and following). 7 | -- https://github.com/tarantool/tarantool/commit/ebb170cb8cf2b9c4634bcf0178665909f578c335 8 | -- Corresponding EE releases: 3.0.1-10 (works with 3.0.2 and following) 9 | -- https://github.com/tarantool/tarantool-ee/commit/1dea81bed4cbe4856a0fc77dcc548849a2dabf45 10 | -- and 3.1.0-entrypoint-44 (works with 3.1.0 and following) 11 | -- https://github.com/tarantool/tarantool-ee/commit/368cc4007727af30ae3ca3a3cdfc7065f34e02aa 12 | local actual_sharding_roles = config:get('sharding.roles') 13 | 14 | for _, actual_sharding_role in ipairs(actual_sharding_roles or {}) do 15 | if actual_sharding_role == expected_sharding_role then 16 | return true 17 | end 18 | end 19 | 20 | return false 21 | end 22 | 23 | return { 24 | is_sharding_role_enabled = is_sharding_role_enabled, 25 | } 26 | -------------------------------------------------------------------------------- /crud/stats/operation.lua: -------------------------------------------------------------------------------- 1 | -- It is not clear how to describe modules 2 | -- with constants for ldoc. ldoc-styled description 3 | -- for this module is available at `crud.stats.init`. 4 | -- See https://github.com/lunarmodules/LDoc/issues/369 5 | -- for possible updates. 6 | return { 7 | -- INSERT identifies both `insert` and `insert_object`. 8 | INSERT = 'insert', 9 | -- INSERT_MANY identifies both `insert_many` and `insert_object_many`. 10 | INSERT_MANY = 'insert_many', 11 | GET = 'get', 12 | -- REPLACE identifies both `replace` and `replace_object`. 13 | REPLACE = 'replace', 14 | -- REPLACE_MANY identifies both `replace_many` and `replace_object_many`. 15 | REPLACE_MANY = 'replace_many', 16 | UPDATE = 'update', 17 | -- UPSERT identifies both `upsert` and `upsert_object`. 18 | UPSERT = 'upsert', 19 | -- UPSERT_MANY identifies both `upsert_many` and `upsert_object_many`. 20 | UPSERT_MANY = 'upsert_many', 21 | DELETE = 'delete', 22 | -- SELECT identifies both `pairs` and `select`. 23 | SELECT = 'select', 24 | TRUNCATE = 'truncate', 25 | LEN = 'len', 26 | COUNT = 'count', 27 | -- BORDERS identifies both `min` and `max`. 28 | BORDERS = 'borders', 29 | } 30 | -------------------------------------------------------------------------------- /test/entrypoint/srv_say_hi/cartridge_init.lua: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env tarantool 2 | 3 | require('strict').on() 4 | _G.is_initialized = function() return false end 5 | 6 | local fio = require('fio') 7 | local log = require('log') 8 | local errors = require('errors') 9 | local cartridge = require('cartridge') 10 | 11 | if package.setsearchroot ~= nil then 12 | package.setsearchroot() 13 | else 14 | package.path = package.path .. debug.sourcedir() .. "/?.lua;" 15 | end 16 | 17 | local root = fio.dirname(fio.dirname(fio.dirname(debug.sourcedir()))) 18 | package.path = package.path .. root .. "/?.lua;" 19 | 20 | package.preload['customers-storage'] = function() 21 | return { 22 | role_name = 'customers-storage', 23 | } 24 | end 25 | 26 | local ok, err = errors.pcall('CartridgeCfgError', cartridge.cfg, { 27 | advertise_uri = 'localhost:3301', 28 | http_port = 8081, 29 | bucket_count = 3000, 30 | roles = { 31 | 'cartridge.roles.crud-storage', 32 | 'cartridge.roles.crud-router', 33 | 'customers-storage', 34 | }, 35 | }) 36 | 37 | if not ok then 38 | log.error('%s', err) 39 | os.exit(1) 40 | end 41 | 42 | require('all').init() 43 | 44 | _G.is_initialized = cartridge.is_healthy 45 | -------------------------------------------------------------------------------- /test/vshard_helpers/instances/router.lua: -------------------------------------------------------------------------------- 1 | local fio = require('fio') 2 | 3 | local appdir = fio.abspath(debug.sourcedir() .. '/../../../') 4 | if package.setsearchroot ~= nil then 5 | package.setsearchroot(appdir) 6 | else 7 | package.path = package.path .. appdir .. '/?.lua;' 8 | package.path = package.path .. appdir .. '/?/init.lua;' 9 | package.path = package.path .. appdir .. '/.rocks/share/tarantool/?.lua;' 10 | package.path = package.path .. appdir .. '/.rocks/share/tarantool/?/init.lua;' 11 | package.cpath = package.cpath .. appdir .. '/?.so;' 12 | package.cpath = package.cpath .. appdir .. '/?.dylib;' 13 | package.cpath = package.cpath .. appdir .. '/.rocks/lib/tarantool/?.so;' 14 | package.cpath = package.cpath .. appdir .. '/.rocks/lib/tarantool/?.dylib;' 15 | end 16 | 17 | local utils = require('test.vshard_helpers.instances.utils') 18 | 19 | -- Somewhy shutdown hangs on new Tarantools even though the nodes do not seem to 20 | -- have any long requests running. 21 | if box.ctl.set_on_shutdown_timeout then 22 | box.ctl.set_on_shutdown_timeout(0.001) 23 | end 24 | 25 | box.cfg(utils.box_cfg()) 26 | box.schema.user.grant('guest', 'super', nil, nil, {if_not_exists = true}) 27 | 28 | _G.ready = true 29 | -------------------------------------------------------------------------------- /test/entrypoint/srv_schema/cartridge_init.lua: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env tarantool 2 | 3 | require('strict').on() 4 | _G.is_initialized = function() return false end 5 | 6 | local fio = require('fio') 7 | local log = require('log') 8 | local errors = require('errors') 9 | local cartridge = require('cartridge') 10 | 11 | if package.setsearchroot ~= nil then 12 | package.setsearchroot() 13 | else 14 | package.path = package.path .. debug.sourcedir() .. "/?.lua;" 15 | end 16 | 17 | local root = fio.dirname(fio.dirname(fio.dirname(debug.sourcedir()))) 18 | package.path = package.path .. root .. "/?.lua;" 19 | 20 | package.preload['customers-storage'] = function() 21 | return { 22 | role_name = 'customers-storage', 23 | init = require('storage').init, 24 | } 25 | end 26 | 27 | local ok, err = errors.pcall('CartridgeCfgError', cartridge.cfg, { 28 | advertise_uri = 'localhost:3301', 29 | http_port = 8081, 30 | bucket_count = 3000, 31 | roles = { 32 | 'cartridge.roles.crud-router', 33 | 'cartridge.roles.crud-storage', 34 | 'customers-storage', 35 | }} 36 | ) 37 | 38 | if not ok then 39 | log.error('%s', err) 40 | os.exit(1) 41 | end 42 | 43 | _G.is_initialized = cartridge.is_healthy 44 | -------------------------------------------------------------------------------- /test/entrypoint/srv_migration/cartridge_init.lua: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env tarantool 2 | 3 | require('strict').on() 4 | _G.is_initialized = function() return false end 5 | 6 | local fio = require('fio') 7 | local log = require('log') 8 | local errors = require('errors') 9 | local cartridge = require('cartridge') 10 | 11 | if package.setsearchroot ~= nil then 12 | package.setsearchroot() 13 | else 14 | package.path = package.path .. debug.sourcedir() .. "/?.lua;" 15 | end 16 | 17 | local root = fio.dirname(fio.dirname(fio.dirname(debug.sourcedir()))) 18 | package.path = package.path .. root .. "/?.lua;" 19 | 20 | package.preload['customers-storage'] = function() 21 | return { 22 | role_name = 'customers-storage', 23 | init = require('storage').init, 24 | } 25 | end 26 | 27 | local ok, err = errors.pcall('CartridgeCfgError', cartridge.cfg, { 28 | advertise_uri = 'localhost:3301', 29 | http_port = 8081, 30 | bucket_count = 3000, 31 | roles = { 32 | 'customers-storage', 33 | 'cartridge.roles.crud-router', 34 | 'cartridge.roles.crud-storage', 35 | }} 36 | ) 37 | 38 | if not ok then 39 | log.error('%s', err) 40 | os.exit(1) 41 | end 42 | 43 | _G.is_initialized = cartridge.is_healthy 44 | -------------------------------------------------------------------------------- /test/entrypoint/srv_batch_operations/cartridge_init.lua: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env tarantool 2 | 3 | require('strict').on() 4 | _G.is_initialized = function() return false end 5 | 6 | local fio = require('fio') 7 | local log = require('log') 8 | local errors = require('errors') 9 | local cartridge = require('cartridge') 10 | 11 | if package.setsearchroot ~= nil then 12 | package.setsearchroot() 13 | else 14 | package.path = package.path .. debug.sourcedir() .. "/?.lua;" 15 | end 16 | 17 | local root = fio.dirname(fio.dirname(fio.dirname(debug.sourcedir()))) 18 | package.path = package.path .. root .. "/?.lua;" 19 | 20 | package.preload['customers-storage'] = function() 21 | return { 22 | role_name = 'customers-storage', 23 | init = require('storage').init, 24 | } 25 | end 26 | 27 | local ok, err = errors.pcall('CartridgeCfgError', cartridge.cfg, { 28 | advertise_uri = 'localhost:3301', 29 | http_port = 8081, 30 | bucket_count = 3000, 31 | roles = { 32 | 'customers-storage', 33 | 'cartridge.roles.crud-router', 34 | 'cartridge.roles.crud-storage', 35 | }, 36 | }) 37 | 38 | if not ok then 39 | log.error('%s', err) 40 | os.exit(1) 41 | end 42 | 43 | _G.is_initialized = cartridge.is_healthy 44 | -------------------------------------------------------------------------------- /test/entrypoint/srv_bucket_id_pk/cartridge_init.lua: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env tarantool 2 | 3 | require('strict').on() 4 | _G.is_initialized = function() return false end 5 | 6 | local fio = require('fio') 7 | local log = require('log') 8 | local errors = require('errors') 9 | local cartridge = require('cartridge') 10 | 11 | if package.setsearchroot ~= nil then 12 | package.setsearchroot() 13 | else 14 | package.path = package.path .. debug.sourcedir() .. "/?.lua;" 15 | end 16 | 17 | local root = fio.dirname(fio.dirname(fio.dirname(debug.sourcedir()))) 18 | package.path = package.path .. root .. "/?.lua;" 19 | 20 | package.preload['customers-storage'] = function() 21 | return { 22 | role_name = 'customers-storage', 23 | init = require('storage').init, 24 | } 25 | end 26 | 27 | local ok, err = errors.pcall('CartridgeCfgError', cartridge.cfg, { 28 | advertise_uri = 'localhost:3301', 29 | http_port = 8081, 30 | bucket_count = 3000, 31 | roles = { 32 | 'customers-storage', 33 | 'cartridge.roles.crud-router', 34 | 'cartridge.roles.crud-storage', 35 | }, 36 | }) 37 | 38 | if not ok then 39 | log.error('%s', err) 40 | os.exit(1) 41 | end 42 | 43 | _G.is_initialized = cartridge.is_healthy 44 | -------------------------------------------------------------------------------- /test/entrypoint/srv_update_schema/cartridge_init.lua: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env tarantool 2 | 3 | require('strict').on() 4 | _G.is_initialized = function() return false end 5 | 6 | local fio = require('fio') 7 | local log = require('log') 8 | local errors = require('errors') 9 | local cartridge = require('cartridge') 10 | 11 | if package.setsearchroot ~= nil then 12 | package.setsearchroot() 13 | else 14 | package.path = package.path .. debug.sourcedir() .. "/?.lua;" 15 | end 16 | 17 | local root = fio.dirname(fio.dirname(fio.dirname(debug.sourcedir()))) 18 | package.path = package.path .. root .. "/?.lua;" 19 | 20 | package.preload['customers-storage'] = function() 21 | return { 22 | role_name = 'customers-storage', 23 | init = require('storage').init, 24 | } 25 | end 26 | 27 | local ok, err = errors.pcall('CartridgeCfgError', cartridge.cfg, { 28 | advertise_uri = 'localhost:3301', 29 | http_port = 8081, 30 | bucket_count = 3000, 31 | roles = { 32 | 'cartridge.roles.crud-router', 33 | 'cartridge.roles.crud-storage', 34 | 'customers-storage', 35 | }, 36 | }) 37 | 38 | if not ok then 39 | log.error('%s', err) 40 | os.exit(1) 41 | end 42 | 43 | _G.is_initialized = cartridge.is_healthy 44 | -------------------------------------------------------------------------------- /test/entrypoint/srv_read_calls_strategies/all.lua: -------------------------------------------------------------------------------- 1 | return { 2 | init = function() 3 | rawset(_G, 'vshard_calls', {}) 4 | 5 | rawset(_G, 'clear_vshard_calls', function() 6 | table.clear(_G.vshard_calls) 7 | end) 8 | 9 | rawset(_G, 'patch_vshard_calls', function(vshard_call_names) 10 | local vshard = require('vshard') 11 | 12 | local replicasets = vshard.router.routeall() 13 | 14 | local _, replicaset = next(replicasets) 15 | local replicaset_mt = getmetatable(replicaset) 16 | 17 | for _, vshard_call_name in ipairs(vshard_call_names) do 18 | local old_func = replicaset_mt.__index[vshard_call_name] 19 | assert(old_func ~= nil, vshard_call_name) 20 | 21 | replicaset_mt.__index[vshard_call_name] = function(...) 22 | local func_name = select(2, ...) 23 | if not string.startswith(func_name, 'vshard.') or func_name == 'vshard.storage.call' then 24 | table.insert(_G.vshard_calls, vshard_call_name) 25 | end 26 | return old_func(...) 27 | end 28 | end 29 | end) 30 | end, 31 | } 32 | -------------------------------------------------------------------------------- /test/entrypoint/srv_simple_operations/cartridge_init.lua: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env tarantool 2 | 3 | require('strict').on() 4 | _G.is_initialized = function() return false end 5 | 6 | local fio = require('fio') 7 | local log = require('log') 8 | local errors = require('errors') 9 | local cartridge = require('cartridge') 10 | 11 | if package.setsearchroot ~= nil then 12 | package.setsearchroot() 13 | else 14 | package.path = package.path .. debug.sourcedir() .. "/?.lua;" 15 | end 16 | 17 | local root = fio.dirname(fio.dirname(fio.dirname(debug.sourcedir()))) 18 | package.path = package.path .. root .. "/?.lua;" 19 | 20 | package.preload['customers-storage'] = function() 21 | return { 22 | role_name = 'customers-storage', 23 | init = require('storage').init, 24 | } 25 | end 26 | 27 | local ok, err = errors.pcall('CartridgeCfgError', cartridge.cfg, { 28 | advertise_uri = 'localhost:3301', 29 | http_port = 8081, 30 | bucket_count = 3000, 31 | roles = { 32 | 'customers-storage', 33 | 'cartridge.roles.crud-router', 34 | 'cartridge.roles.crud-storage', 35 | }, 36 | }) 37 | 38 | if not ok then 39 | log.error('%s', err) 40 | os.exit(1) 41 | end 42 | 43 | _G.is_initialized = cartridge.is_healthy 44 | -------------------------------------------------------------------------------- /deps.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # Call this script to install test dependencies 3 | # Usage examples: 4 | # CARTRIDGE_VERSION=2.16.3 ./deps.sh 5 | # VSHARD_VERSION=0.1.36 ./deps.sh 6 | 7 | set -e 8 | 9 | TTCTL=tt 10 | if ! [ -x "$(command -v tt)" ]; 11 | then 12 | echo "tt not found" 13 | exit 1 14 | fi 15 | 16 | # Test dependencies: 17 | ${TTCTL} rocks install luatest 1.0.1 18 | ${TTCTL} rocks install luacheck 0.26.0 19 | 20 | if [[ -n "${CARTRIDGE_VERSION}" ]] 21 | then 22 | ${TTCTL} rocks install https://raw.githubusercontent.com/luarocks/cluacov/master/cluacov-dev-1.rockspec 23 | ${TTCTL} rocks install https://luarocks.org/manifests/dhkolf/dkjson-2.8-1.rockspec 24 | ${TTCTL} rocks install https://raw.githubusercontent.com/keplerproject/luafilesystem/master/luafilesystem-scm-1.rockspec 25 | ${TTCTL} rocks install https://raw.githubusercontent.com/moteus/lua-path/master/rockspecs/lua-path-scm-0.rockspec 26 | ${TTCTL} rocks install luacov-coveralls --only-server=https://luarocks.org/ 27 | 28 | ${TTCTL} rocks install cartridge "${CARTRIDGE_VERSION}" 29 | ${TTCTL} rocks install migrations 1.1.0 30 | else 31 | ${TTCTL} rocks install vshard "${VSHARD_VERSION:-0.1.36}" 32 | fi 33 | 34 | ${TTCTL} rocks install ddl 1.7.1 35 | 36 | ${TTCTL} rocks make -------------------------------------------------------------------------------- /crud/common/batching_utils.lua: -------------------------------------------------------------------------------- 1 | local errors = require('errors') 2 | local dev_checks = require('crud.common.dev_checks') 3 | local sharding_utils = require('crud.common.sharding.utils') 4 | 5 | local NotPerformedError = errors.new_class('NotPerformedError', {capture_stack = false}) 6 | 7 | local batching_utils = {} 8 | 9 | batching_utils.stop_on_error_msg = "Operation with tuple was not performed" 10 | batching_utils.rollback_on_error_msg = "Operation with tuple was rollback" 11 | 12 | function batching_utils.construct_sharding_hash_mismatch_errors(err_msg, tuples) 13 | dev_checks('string', 'table') 14 | 15 | local errs = {} 16 | 17 | for _, tuple in ipairs(tuples) do 18 | local err_obj = sharding_utils.ShardingHashMismatchError:new(err_msg) 19 | err_obj.operation_data = tuple 20 | table.insert(errs, err_obj) 21 | end 22 | 23 | return errs 24 | end 25 | 26 | function batching_utils.complement_batching_errors(errs, err_msg, tuples) 27 | dev_checks('table', 'string', 'table') 28 | 29 | for _, tuple in ipairs(tuples) do 30 | local err_obj = NotPerformedError:new(err_msg) 31 | err_obj.operation_data = tuple 32 | table.insert(errs, err_obj) 33 | end 34 | 35 | return errs 36 | end 37 | 38 | return batching_utils 39 | -------------------------------------------------------------------------------- /test/entrypoint/srv_read_calls_strategies/cartridge_init.lua: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env tarantool 2 | 3 | require('strict').on() 4 | _G.is_initialized = function() return false end 5 | 6 | local fio = require('fio') 7 | local log = require('log') 8 | local errors = require('errors') 9 | local cartridge = require('cartridge') 10 | 11 | if package.setsearchroot ~= nil then 12 | package.setsearchroot() 13 | else 14 | package.path = package.path .. debug.sourcedir() .. "/?.lua;" 15 | end 16 | 17 | local root = fio.dirname(fio.dirname(fio.dirname(debug.sourcedir()))) 18 | package.path = package.path .. root .. "/?.lua;" 19 | 20 | package.preload['customers-storage'] = function() 21 | return { 22 | role_name = 'customers-storage', 23 | init = require('storage').init, 24 | } 25 | end 26 | 27 | local ok, err = errors.pcall('CartridgeCfgError', cartridge.cfg, { 28 | advertise_uri = 'localhost:3301', 29 | http_port = 8081, 30 | bucket_count = 3000, 31 | roles = { 32 | 'customers-storage', 33 | 'cartridge.roles.crud-router', 34 | 'cartridge.roles.crud-storage', 35 | }, 36 | }) 37 | 38 | if not ok then 39 | log.error('%s', err) 40 | os.exit(1) 41 | end 42 | 43 | require('all').init() 44 | 45 | _G.is_initialized = cartridge.is_healthy 46 | -------------------------------------------------------------------------------- /crud/common/collations.lua: -------------------------------------------------------------------------------- 1 | local json = require('json') 2 | 3 | local COLLATION_NAME_FN = 2 4 | 5 | local collations = {} 6 | 7 | collations.NONE = 'none' 8 | collations.BINARY = 'binary' 9 | collations.UNICODE = 'unicode' 10 | collations.UNICODE_CI = 'unicode_ci' 11 | 12 | function collations.get(index_part) 13 | if index_part.collation ~= nil then 14 | return index_part.collation 15 | end 16 | 17 | if index_part.collation_id == nil then 18 | return collations.NONE 19 | end 20 | 21 | local collation_tuple = box.space._collation:get(index_part.collation_id) 22 | assert(collation_tuple ~= nil, "Unknown collation_id: " .. json.encode(index_part.collation_id)) 23 | 24 | local collation = collation_tuple[COLLATION_NAME_FN] 25 | return collation 26 | end 27 | 28 | function collations.is_default(collation) 29 | if collation == nil then 30 | return true 31 | end 32 | 33 | if collation == collations.NONE or collation == collations.BINARY then 34 | return true 35 | end 36 | 37 | return false 38 | end 39 | 40 | function collations.is_unicode(collation) 41 | if collation == nil then 42 | return false 43 | end 44 | 45 | return string.startswith(collation, 'unicode_') 46 | end 47 | 48 | return collations 49 | -------------------------------------------------------------------------------- /test/entrypoint/srv_select/cartridge_init.lua: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env tarantool 2 | 3 | require('strict').on() 4 | _G.is_initialized = function() return false end 5 | 6 | local fio = require('fio') 7 | local log = require('log') 8 | local errors = require('errors') 9 | local cartridge = require('cartridge') 10 | 11 | if package.setsearchroot ~= nil then 12 | package.setsearchroot() 13 | else 14 | package.path = package.path .. debug.sourcedir() .. "/?.lua;" 15 | end 16 | 17 | local root = fio.dirname(fio.dirname(fio.dirname(debug.sourcedir()))) 18 | package.path = package.path .. root .. "/?.lua;" 19 | 20 | package.preload['customers-storage'] = function() 21 | return { 22 | role_name = 'customers-storage', 23 | init = require('storage').init, 24 | } 25 | end 26 | 27 | local box_opts = { 28 | readahead = 10 * 1024 * 1024, 29 | } 30 | local ok, err = errors.pcall('CartridgeCfgError', cartridge.cfg, { 31 | advertise_uri = 'localhost:3301', 32 | http_port = 8081, 33 | bucket_count = 3000, 34 | roles = { 35 | 'cartridge.roles.crud-router', 36 | 'cartridge.roles.crud-storage', 37 | 'customers-storage', 38 | }}, 39 | box_opts 40 | ) 41 | 42 | if not ok then 43 | log.error('%s', err) 44 | os.exit(1) 45 | end 46 | 47 | _G.is_initialized = cartridge.is_healthy 48 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright 2020-2025 crud AUTHORS: please see the AUTHORS file. 2 | 3 | Redistribution and use in source and binary forms, with or without modification, 4 | are permitted provided that the following conditions are met: 5 | 6 | 1. Redistributions of source code must retain the above copyright notice, this 7 | list of conditions and the following disclaimer. 8 | 9 | 2. Redistributions in binary form must reproduce the above copyright notice, 10 | this list of conditions and the following disclaimer in the documentation 11 | and/or other materials provided with the distribution. 12 | 13 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 14 | ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 15 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 16 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR 17 | ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 18 | (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 19 | LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON 20 | ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 21 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 22 | SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 23 | -------------------------------------------------------------------------------- /test/vshard_helpers/instances/storage.lua: -------------------------------------------------------------------------------- 1 | local fio = require('fio') 2 | 3 | local appdir = fio.abspath(debug.sourcedir() .. '/../../../') 4 | if package.setsearchroot ~= nil then 5 | package.setsearchroot(appdir) 6 | else 7 | package.path = package.path .. appdir .. '/?.lua;' 8 | package.path = package.path .. appdir .. '/?/init.lua;' 9 | package.path = package.path .. appdir .. '/.rocks/share/tarantool/?.lua;' 10 | package.path = package.path .. appdir .. '/.rocks/share/tarantool/?/init.lua;' 11 | package.cpath = package.cpath .. appdir .. '/?.so;' 12 | package.cpath = package.cpath .. appdir .. '/?.dylib;' 13 | package.cpath = package.cpath .. appdir .. '/.rocks/lib/tarantool/?.so;' 14 | package.cpath = package.cpath .. appdir .. '/.rocks/lib/tarantool/?.dylib;' 15 | end 16 | 17 | local utils = require('test.vshard_helpers.instances.utils') 18 | 19 | -- It is not necessary in fact, but simplify `callrw` calls in tests. 20 | _G.vshard = { 21 | storage = require('vshard.storage'), 22 | } 23 | 24 | -- Somewhy shutdown hangs on new Tarantools even though the nodes do not seem to 25 | -- have any long requests running. 26 | if box.ctl.set_on_shutdown_timeout then 27 | box.ctl.set_on_shutdown_timeout(0.001) 28 | end 29 | 30 | box.cfg(utils.box_cfg()) 31 | box.schema.user.grant('guest', 'super', nil, nil, {if_not_exists = true}) 32 | 33 | _G.ready = true 34 | -------------------------------------------------------------------------------- /test/entrypoint/srv_stats/storage.lua: -------------------------------------------------------------------------------- 1 | local helper = require('test.helper') 2 | 3 | return { 4 | init = helper.wrap_schema_init(function() 5 | local engine = os.getenv('ENGINE') or 'memtx' 6 | local customers_space = box.schema.space.create('customers', { 7 | format = { 8 | {name = 'id', type = 'unsigned'}, 9 | {name = 'bucket_id', type = 'unsigned'}, 10 | {name = 'name', type = 'string'}, 11 | {name = 'last_name', type = 'string'}, 12 | {name = 'age', type = 'number'}, 13 | {name = 'city', type = 'string'}, 14 | }, 15 | if_not_exists = true, 16 | engine = engine, 17 | id = 542, 18 | }) 19 | -- primary index 20 | customers_space:create_index('id_index', { 21 | parts = { {field = 'id'} }, 22 | if_not_exists = true, 23 | }) 24 | customers_space:create_index('bucket_id', { 25 | parts = { {field = 'bucket_id'} }, 26 | unique = false, 27 | if_not_exists = true, 28 | }) 29 | customers_space:create_index('age_index', { 30 | parts = { {field = 'age'} }, 31 | unique = false, 32 | if_not_exists = true, 33 | }) 34 | end), 35 | wait_until_ready = helper.wait_schema_init, 36 | } 37 | -------------------------------------------------------------------------------- /crud/common/compat.lua: -------------------------------------------------------------------------------- 1 | local log = require('log') 2 | 3 | local compat = {} 4 | 5 | function compat.require(module_name, builtin_module_name) 6 | local module_cached_name = string.format('__crud_%s_cached', module_name) 7 | 8 | local module 9 | 10 | local module_cached = rawget(_G, module_cached_name) 11 | if module_cached ~= nil then 12 | module = module_cached 13 | elseif package.search(module_name) then 14 | -- we don't use pcall(require, module_name) here because it 15 | -- leads to ignoring errors other than 'No LuaRocks module found' 16 | log.info('%q module is used', module_name) 17 | module = require(module_name) 18 | else 19 | log.info('%q module is not found. Built-in %q is used', module_name, builtin_module_name) 20 | module = require(builtin_module_name) 21 | end 22 | 23 | rawset(_G, module_cached_name, module) 24 | 25 | return module 26 | end 27 | 28 | function compat.exists(module_name, builtin_module_name) 29 | local module_cached = rawget(_G, string.format('__crud_%s_cached', module_name)) 30 | if module_cached ~= nil then 31 | return true 32 | end 33 | 34 | if package.search(module_name) then 35 | return true 36 | end 37 | 38 | if package.loaded[builtin_module_name] ~= nil then 39 | return true 40 | end 41 | 42 | return false 43 | end 44 | 45 | return compat 46 | -------------------------------------------------------------------------------- /test/entrypoint/srv_ddl/cartridge_init.lua: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env tarantool 2 | 3 | require('strict').on() 4 | _G.is_initialized = function() return false end 5 | 6 | local fio = require('fio') 7 | local log = require('log') 8 | local errors = require('errors') 9 | local cartridge = require('cartridge') 10 | 11 | if package.setsearchroot ~= nil then 12 | package.setsearchroot() 13 | else 14 | package.path = package.path .. debug.sourcedir() .. "/?.lua;" 15 | end 16 | 17 | local root = fio.dirname(fio.dirname(fio.dirname(debug.sourcedir()))) 18 | package.path = package.path .. root .. "/?.lua;" 19 | 20 | package.preload['customers-storage'] = function() 21 | return { 22 | role_name = 'customers-storage', 23 | init = require('storage').init, 24 | } 25 | end 26 | 27 | local ok, err = errors.pcall('CartridgeCfgError', cartridge.cfg, { 28 | advertise_uri = 'localhost:3301', 29 | http_port = 8081, 30 | bucket_count = 3000, 31 | roles = { 32 | 'customers-storage', 33 | 'cartridge.roles.crud-router', 34 | 'cartridge.roles.crud-storage', 35 | }}, 36 | -- Increase readahead for performance tests. 37 | -- Performance tests on HP ProBook 440 G5 16 Gb 38 | -- bump into default readahead limit and thus not 39 | -- give a full picture. 40 | { readahead = 20 * 1024 * 1024 } 41 | ) 42 | 43 | if not ok then 44 | log.error('%s', err) 45 | os.exit(1) 46 | end 47 | 48 | _G.is_initialized = cartridge.is_healthy 49 | -------------------------------------------------------------------------------- /test/unit/privileges_test.lua: -------------------------------------------------------------------------------- 1 | local t = require("luatest") 2 | local g = t.group() 3 | 4 | local helper = require("test.helper") 5 | local call = require("crud.common.call") 6 | 7 | g.before_all(function() 8 | helper.box_cfg({listen = 3401}) 9 | 10 | box.schema.user.create("unittestuser", {password = "secret", if_not_exists = true}) 11 | box.schema.user.grant("unittestuser", "read,write,execute,create,alter,drop", "universe", nil, 12 | {if_not_exists = true}) 13 | 14 | rawset(_G, "unittestfunc", function(...) 15 | return ... 16 | end) 17 | end) 18 | 19 | g.test_prepend_current_user_smoke = function() 20 | local res = call.storage_api.call_on_storage(box.session.effective_user(), "unittestfunc", {"too", "foo"}) 21 | t.assert_equals(res, {"too", "foo"}) 22 | end 23 | 24 | g.test_non_existent_user = function() 25 | t.assert_error_msg_contains("User 'non_existent_user' is not found", 26 | call.storage_api.call_on_storage, "non_existent_user", "unittestfunc") 27 | end 28 | 29 | g.test_that_the_session_switches_back = function() 30 | rawset(_G, "unittestfunc2", function() 31 | return box.session.effective_user() 32 | end) 33 | 34 | local reference_user = box.session.effective_user() 35 | t.assert_not_equals(reference_user, "unittestuser") 36 | 37 | local res = call.storage_api.call_on_storage("unittestuser", "unittestfunc2") 38 | t.assert_equals(res, "unittestuser") 39 | t.assert_equals(box.session.effective_user(), reference_user) 40 | end 41 | -------------------------------------------------------------------------------- /test/entrypoint/srv_not_initialized/cartridge_init.lua: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env tarantool 2 | 3 | require('strict').on() 4 | _G.is_initialized = function() return false end 5 | 6 | local fio = require('fio') 7 | local log = require('log') 8 | local errors = require('errors') 9 | local cartridge = require('cartridge') 10 | 11 | if package.setsearchroot ~= nil then 12 | package.setsearchroot() 13 | else 14 | package.path = package.path .. debug.sourcedir() .. "/?.lua;" .. debug.sourcedir() .. "/?/init.lua;" 15 | end 16 | 17 | local root = fio.dirname(fio.dirname(fio.dirname(debug.sourcedir()))) 18 | package.path = package.path .. root .. "/?.lua;" .. root .. "/?/init.lua;" 19 | 20 | local root = fio.dirname(fio.dirname(fio.dirname(debug.sourcedir()))) 21 | package.path = package.path .. root .. "/?.lua;" 22 | 23 | package.preload['customers-storage'] = function() 24 | return { 25 | role_name = 'customers-storage', 26 | init = require('storage').init, 27 | dependencies = {'cartridge.roles.vshard-storage'}, 28 | } 29 | end 30 | 31 | local ok, err = errors.pcall('CartridgeCfgError', cartridge.cfg, { 32 | advertise_uri = 'localhost:3301', 33 | http_port = 8081, 34 | bucket_count = 3000, 35 | roles = { 36 | 'cartridge.roles.vshard-router', 37 | 'customers-storage', 38 | }, 39 | }) 40 | 41 | if not ok then 42 | log.error('%s', err) 43 | os.exit(1) 44 | end 45 | 46 | -- crud.init_storage() isn't called 47 | 48 | rawset(_G, 'say_hi', function() return "Hi" end) 49 | 50 | _G.is_initialized = cartridge.is_healthy 51 | -------------------------------------------------------------------------------- /crud/common/sharding/router_metadata_cache.lua: -------------------------------------------------------------------------------- 1 | local fiber = require('fiber') 2 | 3 | local router_metadata_cache = {} 4 | 5 | router_metadata_cache.SHARDING_KEY_MAP_NAME = "sharding_key_as_index_obj_map" 6 | router_metadata_cache.SHARDING_FUNC_MAP_NAME = "sharding_func_map" 7 | router_metadata_cache.META_HASH_MAP_NAME = "sharding_meta_hash_map" 8 | 9 | local internal_storage = {} 10 | 11 | function router_metadata_cache.get_instance(vshard_router) 12 | local name = vshard_router.name 13 | 14 | if internal_storage[name] ~= nil then 15 | return internal_storage[name] 16 | end 17 | 18 | internal_storage[name] = { 19 | [router_metadata_cache.SHARDING_KEY_MAP_NAME] = nil, 20 | [router_metadata_cache.SHARDING_FUNC_MAP_NAME] = nil, 21 | [router_metadata_cache.META_HASH_MAP_NAME] = {}, 22 | fetch_lock = fiber.channel(1), 23 | is_part_of_pk = {} 24 | } 25 | 26 | return internal_storage[name] 27 | end 28 | 29 | function router_metadata_cache.drop_instance(vshard_router) 30 | local name = vshard_router.name 31 | 32 | if internal_storage[name] == nil then 33 | return 34 | end 35 | 36 | if internal_storage[name].fetch_lock ~= nil then 37 | internal_storage[name].fetch_lock:close() 38 | end 39 | 40 | internal_storage[name] = nil 41 | end 42 | 43 | function router_metadata_cache.drop_caches() 44 | for name, _ in pairs(internal_storage) do 45 | router_metadata_cache.drop_instance(name) 46 | end 47 | 48 | internal_storage = {} 49 | end 50 | 51 | return router_metadata_cache -------------------------------------------------------------------------------- /.github/workflows/reusable_test.yml: -------------------------------------------------------------------------------- 1 | name: Reusable test 2 | 3 | on: 4 | workflow_call: 5 | inputs: 6 | artifact_name: 7 | description: The name of the Tarantool build artifact 8 | default: ubuntu-focal 9 | required: false 10 | type: string 11 | 12 | jobs: 13 | run_tests: 14 | runs-on: ubuntu-22.04 15 | 16 | steps: 17 | - name: Clone the crud module 18 | uses: actions/checkout@v4 19 | with: 20 | repository: ${{ github.repository_owner }}/crud 21 | 22 | - name: Download the Tarantool build artifact 23 | uses: actions/download-artifact@v4 24 | with: 25 | name: ${{ inputs.artifact_name }} 26 | 27 | # All dependencies for tarantool are already installed in Ubuntu 20.04. 28 | # Check package dependencies when migrating to other OS version. 29 | - name: Install Tarantool 30 | run: | 31 | sudo dpkg -i tarantool_*.deb tarantool-common_*.deb tarantool-dev_*.deb 32 | tarantool --version 33 | 34 | - name: Setup tt 35 | run: | 36 | curl -L https://tarantool.io/release/2/installer.sh | sudo bash 37 | sudo apt install -y tt 38 | tt version 39 | 40 | - name: Install requirements 41 | run: ./deps.sh 42 | 43 | # This server starts and listen on 8084 port that is used for tests 44 | - name: Stop Mono server 45 | run: sudo kill -9 $(sudo lsof -t -i tcp:8084) || true 46 | 47 | - run: cmake -S . -B build 48 | 49 | - name: Run regression tests 50 | run: make -C build luatest-no-coverage 51 | -------------------------------------------------------------------------------- /test/vshard_helpers/vclock.lua: -------------------------------------------------------------------------------- 1 | local fiber = require('fiber') 2 | local log = require('log') 3 | local yaml = require('yaml') 4 | 5 | -- Simple implementation without metatables. 6 | local function extend_with(t, new_fields) 7 | for k, v in pairs(new_fields) do 8 | t[k] = v 9 | end 10 | 11 | return t 12 | end 13 | 14 | local function get_vclock(self) 15 | return self:exec(function() return box.info.vclock end) 16 | end 17 | 18 | local function wait_vclock(self, to_vclock) 19 | while true do 20 | local vclock = self:get_vclock() 21 | local ok = true 22 | 23 | for server_id, to_lsn in pairs(to_vclock) do 24 | local lsn = vclock[server_id] 25 | if lsn == nil or lsn < to_lsn then 26 | ok = false 27 | break 28 | end 29 | end 30 | 31 | if ok then 32 | return 33 | end 34 | 35 | log.info("wait vclock: %s to %s", 36 | yaml.encode(vclock), yaml.encode(to_vclock)) 37 | fiber.sleep(0.001) 38 | end 39 | end 40 | 41 | local function wait_vclock_of(self, other_server) 42 | local vclock = other_server:get_vclock() 43 | -- First component is for local changes. 44 | vclock[0] = nil 45 | return self:wait_vclock(vclock) 46 | end 47 | 48 | local function extend_with_vclock_methods(server) 49 | return extend_with(server, { 50 | get_vclock = get_vclock, 51 | wait_vclock = wait_vclock, 52 | wait_vclock_of = wait_vclock_of, 53 | }) 54 | end 55 | 56 | return { 57 | extend_with_vclock_methods = extend_with_vclock_methods, 58 | } 59 | -------------------------------------------------------------------------------- /test/entrypoint/srv_stats/cartridge_init.lua: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env tarantool 2 | 3 | require('strict').on() 4 | _G.is_initialized = function() return false end 5 | 6 | local fio = require('fio') 7 | local log = require('log') 8 | local errors = require('errors') 9 | local cartridge = require('cartridge') 10 | 11 | local crud_utils = require('crud.common.utils') 12 | 13 | if package.setsearchroot ~= nil then 14 | package.setsearchroot() 15 | else 16 | package.path = package.path .. debug.sourcedir() .. "/?.lua;" 17 | end 18 | 19 | local root = fio.dirname(fio.dirname(fio.dirname(debug.sourcedir()))) 20 | package.path = package.path .. root .. "/?.lua;" 21 | 22 | package.preload['customers-storage'] = function() 23 | return { 24 | role_name = 'customers-storage', 25 | init = require('storage').init, 26 | } 27 | end 28 | 29 | local roles_reload_allowed = nil 30 | if crud_utils.is_cartridge_hotreload_supported() then 31 | roles_reload_allowed = true 32 | end 33 | 34 | local is_metrics = pcall(require, 'metrics') 35 | local roles = { 36 | 'cartridge.roles.crud-router', 37 | 'cartridge.roles.crud-storage', 38 | 'customers-storage', 39 | } 40 | if is_metrics then 41 | table.insert(roles, 'cartridge.roles.metrics') 42 | end 43 | 44 | local ok, err = errors.pcall('CartridgeCfgError', cartridge.cfg, { 45 | advertise_uri = 'localhost:3301', 46 | http_port = 8081, 47 | bucket_count = 3000, 48 | roles = roles, 49 | roles_reload_allowed = roles_reload_allowed, 50 | }) 51 | 52 | if not ok then 53 | log.error('%s', err) 54 | os.exit(1) 55 | end 56 | 57 | _G.is_initialized = cartridge.is_healthy 58 | -------------------------------------------------------------------------------- /test/unit/validate_bucket_id_test.lua: -------------------------------------------------------------------------------- 1 | local t = require('luatest') 2 | local g = t.group() 3 | 4 | local sharding = require('crud.common.sharding') 5 | 6 | local ffi = require('ffi') 7 | 8 | local cases = { 9 | positive_number = {value = 1, should_fail = false}, 10 | large_number = {value = 100000, should_fail = false}, 11 | zero = {value = 0, should_fail = true}, 12 | negative_number = {value = -1, should_fail = true}, 13 | non_integer_number = {value = 123.45, should_fail = true}, 14 | string_value = {value = 'abc', should_fail = true}, 15 | boolean_value = {value = true, should_fail = true}, 16 | table_value = {value = {}, should_fail = true}, 17 | nil_value = {value = nil, should_fail = true}, 18 | box_null = {value = box.NULL, should_fail = true}, 19 | ffi_uint64 = {value = ffi.new('uint64_t', 1), should_fail = false}, 20 | ffi_uint64_zero = {value = ffi.new('uint64_t', 0), should_fail = true}, 21 | ffi_int64_negative = {value = ffi.new('int64_t', -1), should_fail = true}, 22 | } 23 | 24 | for name, case in pairs(cases) do 25 | g["test_validate_bucket_id_" .. name] = function() 26 | local err = sharding.validate_bucket_id(case.value) 27 | 28 | if case.should_fail then 29 | t.assert(err, ('%s should be rejected'):format(name)) 30 | t.assert_equals(err.class_name, 'BucketIDError') 31 | t.assert_str_contains(err.err, 'expected unsigned') 32 | else 33 | t.assert_equals(err, nil, ('%s should be accepted'):format(name)) 34 | end 35 | end 36 | end 37 | -------------------------------------------------------------------------------- /test/integration/write_scenario.lua: -------------------------------------------------------------------------------- 1 | local t = require('luatest') 2 | local checks = require('checks') 3 | 4 | -- Scenario is for 'srv_batch_operations' entrypoint. 5 | local function gh_437_many_explicit_bucket_ids(cg, operation, opts) 6 | checks('table', 'string', { 7 | objects = '?boolean', 8 | upsert = '?boolean', 9 | partial_explicit_bucket_ids = '?boolean', 10 | }) 11 | 12 | opts = opts or {} 13 | 14 | local rows = { 15 | {1, 1, 'Kumiko', 18}, 16 | {2, 2, 'Reina', 19}, 17 | {3, 3, 'Shuuichi', 18}, 18 | } 19 | 20 | if opts.partial_explicit_bucket_ids then 21 | rows[2][2] = box.NULL 22 | end 23 | 24 | local objects = {} 25 | for k, v in ipairs(rows) do 26 | objects[k] = {id = v[1], bucket_id = v[2], name = v[3], age = v[4]} 27 | end 28 | 29 | local data 30 | if opts.objects then 31 | data = objects 32 | else 33 | data = rows 34 | end 35 | 36 | if opts.upsert then 37 | local update_operations = {} 38 | for k, v in ipairs(data) do 39 | data[k] = {v, update_operations} 40 | end 41 | end 42 | 43 | local args = {'customers_sharded_by_age', data} 44 | 45 | local result, errs = cg.router:call('crud.' .. operation, args) 46 | t.assert_equals(errs, nil) 47 | 48 | local result_rows = table.deepcopy(rows) 49 | if opts.partial_explicit_bucket_ids then 50 | result_rows[2][2] = 1325 51 | end 52 | if opts.upsert then 53 | -- upsert never return anything. 54 | t.assert_equals(result.rows, nil) 55 | else 56 | t.assert_items_equals(result.rows, result_rows) 57 | end 58 | end 59 | 60 | return { 61 | gh_437_many_explicit_bucket_ids = gh_437_many_explicit_bucket_ids, 62 | } 63 | -------------------------------------------------------------------------------- /.github/workflows/push_rockspec.yaml: -------------------------------------------------------------------------------- 1 | name: Push rockspec 2 | 3 | on: 4 | push: 5 | branches: 6 | - master 7 | tags: 8 | - '*' 9 | 10 | env: 11 | ROCK_NAME: "crud" 12 | 13 | jobs: 14 | version-check: 15 | # We need this job to run only on push with tag. 16 | if: ${{ github.event_name == 'push' && startsWith(github.ref, 'refs/tags/') }} 17 | runs-on: ubuntu-22.04 18 | steps: 19 | - name: Check module version 20 | uses: tarantool/actions/check-module-version@master 21 | with: 22 | module-name: 'crud' 23 | 24 | push-scm-rockspec: 25 | runs-on: ubuntu-22.04 26 | if: github.ref == 'refs/heads/master' 27 | steps: 28 | - uses: actions/checkout@v4 29 | 30 | - uses: tarantool/rocks.tarantool.org/github-action@master 31 | with: 32 | auth: ${{ secrets.ROCKS_AUTH }} 33 | files: ${{ env.ROCK_NAME }}-scm-1.rockspec 34 | 35 | push-tagged-rockspec: 36 | runs-on: ubuntu-22.04 37 | if: startsWith(github.ref, 'refs/tags') 38 | needs: version-check 39 | steps: 40 | - uses: actions/checkout@v4 41 | 42 | # https://stackoverflow.com/questions/58177786/get-the-current-pushed-tag-in-github-actions 43 | - name: Set env 44 | run: echo "GIT_TAG=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV 45 | 46 | - name: Create release rockspec 47 | run: | 48 | sed \ 49 | -e "s/branch = '.\+'/tag = '${GIT_TAG}'/g" \ 50 | -e "s/version = '.\+'/version = '${GIT_TAG}-1'/g" \ 51 | ${{ env.ROCK_NAME }}-scm-1.rockspec > ${{ env.ROCK_NAME }}-${GIT_TAG}-1.rockspec 52 | 53 | - uses: tarantool/rocks.tarantool.org/github-action@master 54 | with: 55 | auth: ${{ secrets.ROCKS_AUTH }} 56 | files: ${{ env.ROCK_NAME }}-${GIT_TAG}-1.rockspec 57 | -------------------------------------------------------------------------------- /test/entrypoint/srv_bucket_id_pk/storage.lua: -------------------------------------------------------------------------------- 1 | local helper = require('test.helper') 2 | 3 | return { 4 | init = helper.wrap_schema_init(function() 5 | local engine = os.getenv('ENGINE') or 'memtx' 6 | local customers_space = box.schema.space.create('customers', { 7 | format = { 8 | {name = 'id', type = 'unsigned'}, 9 | {name = 'bucket_id', type = 'unsigned'}, 10 | {name = 'name', type = 'string'}, 11 | {name = 'age', type = 'number'}, 12 | }, 13 | if_not_exists = true, 14 | engine = engine, 15 | }) 16 | customers_space:create_index('bucket_id', { 17 | parts = { {field = 'bucket_id'}, {field = 'id'} }, 18 | if_not_exists = true, 19 | }) 20 | 21 | -- https://github.com/tarantool/migrations/blob/a7c31a17f6ac02d4498b4203c23e495856861444/migrator/utils.lua#L35-L53 22 | if box.space._ddl_sharding_key == nil then 23 | local sharding_space = box.schema.space.create('_ddl_sharding_key', { 24 | format = { 25 | {name = 'space_name', type = 'string', is_nullable = false}, 26 | {name = 'sharding_key', type = 'array', is_nullable = false} 27 | }, 28 | if_not_exists = true, 29 | }) 30 | sharding_space:create_index( 31 | 'space_name', { 32 | type = 'TREE', 33 | unique = true, 34 | parts = {{'space_name', 'string', is_nullable = false}}, 35 | if_not_exists = true, 36 | } 37 | ) 38 | end 39 | box.space._ddl_sharding_key:replace{'customers', {'id'}} 40 | end), 41 | wait_until_ready = helper.wait_schema_init, 42 | } 43 | -------------------------------------------------------------------------------- /test/entrypoint/srv_ddl_reload/cartridge_init.lua: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env tarantool 2 | 3 | require('strict').on() 4 | _G.is_initialized = function() return false end 5 | 6 | local fio = require('fio') 7 | local log = require('log') 8 | local errors = require('errors') 9 | local cartridge = require('cartridge') 10 | 11 | local crud_utils = require('crud.common.utils') 12 | 13 | if package.setsearchroot ~= nil then 14 | package.setsearchroot() 15 | else 16 | package.path = package.path .. debug.sourcedir() .. "/?.lua;" 17 | end 18 | 19 | local root = fio.dirname(fio.dirname(fio.dirname(debug.sourcedir()))) 20 | package.path = package.path .. root .. "/?.lua;" 21 | 22 | package.preload['customers-storage'] = function() 23 | local customers_module = { 24 | sharding_func_default = function(key) 25 | local id = key[1] 26 | assert(id ~= nil) 27 | 28 | return id % 3000 + 1 29 | end, 30 | sharding_func_new = function(key) 31 | local id = key[1] 32 | assert(id ~= nil) 33 | 34 | return (id + 42) % 3000 + 1 35 | end, 36 | } 37 | rawset(_G, 'customers_module', customers_module) 38 | 39 | return { 40 | role_name = 'customers-storage', 41 | init = require('storage').init, 42 | } 43 | end 44 | 45 | local roles_reload_allowed = nil 46 | if crud_utils.is_cartridge_hotreload_supported() then 47 | roles_reload_allowed = true 48 | end 49 | 50 | local ok, err = errors.pcall('CartridgeCfgError', cartridge.cfg, { 51 | advertise_uri = 'localhost:3301', 52 | http_port = 8081, 53 | bucket_count = 3000, 54 | roles = { 55 | 'customers-storage', 56 | 'cartridge.roles.crud-router', 57 | 'cartridge.roles.crud-storage', 58 | }, 59 | roles_reload_allowed = roles_reload_allowed, 60 | }) 61 | 62 | if not ok then 63 | log.error('%s', err) 64 | os.exit(1) 65 | end 66 | 67 | _G.is_initialized = cartridge.is_healthy 68 | -------------------------------------------------------------------------------- /test/entrypoint/srv_reload/cartridge_init.lua: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env tarantool 2 | 3 | require('strict').on() 4 | _G.is_initialized = function() return false end 5 | 6 | local log = require('log') 7 | local errors = require('errors') 8 | local cartridge = require('cartridge') 9 | 10 | local crud_utils = require('crud.common.utils') 11 | 12 | package.preload['customers-storage'] = function() 13 | return { 14 | role_name = 'customers-storage', 15 | init = function() 16 | local engine = os.getenv('ENGINE') or 'memtx' 17 | local customers_space = box.schema.space.create('customers', { 18 | format = { 19 | {name = 'id', type = 'unsigned'}, 20 | {name = 'bucket_id', type = 'unsigned'}, 21 | }, 22 | if_not_exists = true, 23 | engine = engine, 24 | }) 25 | 26 | customers_space:create_index('id', { 27 | parts = { {field = 'id'} }, 28 | unique = true, 29 | type = 'TREE', 30 | if_not_exists = true, 31 | }) 32 | 33 | customers_space:create_index('bucket_id', { 34 | parts = { {field = 'bucket_id'} }, 35 | unique = false, 36 | type = 'TREE', 37 | if_not_exists = true, 38 | }) 39 | end, 40 | dependencies = {'cartridge.roles.crud-storage'} 41 | } 42 | end 43 | 44 | local roles_reload_allowed = nil 45 | if crud_utils.is_cartridge_hotreload_supported() then 46 | roles_reload_allowed = true 47 | end 48 | 49 | local ok, err = errors.pcall('CartridgeCfgError', cartridge.cfg, { 50 | advertise_uri = 'localhost:3301', 51 | http_port = 8081, 52 | bucket_count = 3000, 53 | roles = { 54 | 'customers-storage', 55 | 'cartridge.roles.crud-router', 56 | 'cartridge.roles.crud-storage' 57 | }, 58 | roles_reload_allowed = roles_reload_allowed, 59 | }) 60 | 61 | if not ok then 62 | log.error('%s', err) 63 | os.exit(1) 64 | end 65 | 66 | _G.is_initialized = cartridge.is_healthy 67 | -------------------------------------------------------------------------------- /test/integration/len_test.lua: -------------------------------------------------------------------------------- 1 | local t = require('luatest') 2 | 3 | local helpers = require('test.helper') 4 | 5 | local pgroup = t.group('len', helpers.backend_matrix({ 6 | {engine = 'memtx'}, 7 | })) 8 | 9 | pgroup.before_all(function(g) 10 | helpers.start_default_cluster(g, 'srv_select') 11 | end) 12 | 13 | pgroup.after_all(function(g) 14 | helpers.stop_cluster(g.cluster, g.params.backend) 15 | end) 16 | 17 | pgroup.before_each(function(g) 18 | helpers.truncate_space_on_cluster(g.cluster, 'customers') 19 | end) 20 | 21 | pgroup.test_len_non_existent_space = function(g) 22 | local result, err = g.router:call('crud.len', {'non_existent_space'}) 23 | 24 | t.assert_equals(result, nil) 25 | t.assert_str_contains(err.err, "Space \"non_existent_space\" doesn't exist") 26 | end 27 | 28 | pgroup.test_len = function(g) 29 | local customers = {} 30 | local expected_len = 100 31 | 32 | -- let's insert a large number of tuples in a simple loop that gives 33 | -- really high probability that there is at least one tuple on each storage 34 | for i = 1, expected_len do 35 | table.insert(customers, { 36 | id = i, name = tostring(i), last_name = tostring(i), 37 | age = i, city = tostring(i), 38 | }) 39 | end 40 | 41 | helpers.insert_objects(g, 'customers', customers) 42 | 43 | local result, err = g.router:call('crud.len', {'customers'}) 44 | 45 | t.assert_equals(err, nil) 46 | t.assert_equals(result, expected_len) 47 | end 48 | 49 | pgroup.test_len_empty_space = function(g) 50 | local result, err = g.router:call('crud.len', {'customers'}) 51 | 52 | t.assert_equals(err, nil) 53 | t.assert_equals(result, 0) 54 | end 55 | 56 | pgroup.test_opts_not_damaged = function(g) 57 | local len_opts = {timeout = 1} 58 | local new_len_opts, err = g.router:eval([[ 59 | local crud = require('crud') 60 | 61 | local len_opts = ... 62 | 63 | local _, err = crud.len('customers', len_opts) 64 | 65 | return len_opts, err 66 | ]], {len_opts}) 67 | 68 | t.assert_equals(err, nil) 69 | t.assert_equals(new_len_opts, len_opts) 70 | end 71 | -------------------------------------------------------------------------------- /test/integration/migration_test.lua: -------------------------------------------------------------------------------- 1 | local t = require('luatest') 2 | 3 | local helpers = require('test.helper') 4 | 5 | -- The migrations package requires cartridge as a dependency. 6 | local pgroup = t.group('migration', { 7 | {backend = helpers.backend.CARTRIDGE, engine = 'memtx'}, 8 | }) 9 | 10 | pgroup.before_all(function(g) 11 | helpers.start_default_cluster(g, 'srv_migration') 12 | end) 13 | 14 | pgroup.after_all(function(g) 15 | helpers.stop_cluster(g.cluster, g.params.backend) 16 | end) 17 | 18 | pgroup.test_gh_308_select_after_improper_ddl_space_drop = function(g) 19 | -- Create a space sharded by key with ddl tools. 20 | helpers.call_on_storages(g.cluster, function(server) 21 | server.net_box:eval([[ 22 | local migrator_utils = require('migrator.utils') 23 | 24 | if not box.info.ro then 25 | box.schema.space.create('customers_v2') 26 | 27 | box.space['customers_v2']:format{ 28 | {name = 'id_v2', is_nullable = false, type = 'unsigned'}, 29 | {name = 'bucket_id', is_nullable = false, type = 'unsigned'}, 30 | {name = 'sharding_key', is_nullable = false, type = 'unsigned'}, 31 | } 32 | 33 | box.space['customers_v2']:create_index('pk', {parts = { 'id_v2' }}) 34 | box.space['customers_v2']:create_index('bucket_id', {parts = { 'bucket_id' }}) 35 | 36 | migrator_utils.register_sharding_key('customers_v2', {'sharding_key'}) 37 | end 38 | ]]) 39 | end) 40 | 41 | -- Do not do any requests to refresh sharding metadata. 42 | 43 | -- Drop space, but do not clean up ddl sharding data. 44 | helpers.call_on_storages(g.cluster, function(server) 45 | server.net_box:eval([[ 46 | if not box.info.ro then 47 | box.space['customers_v2']:drop() 48 | end 49 | ]]) 50 | end) 51 | 52 | -- Ensure that crud request for existing space is ok. 53 | local _, err = g.router:call('crud.select', { 54 | 'customers', nil, {first = 1, mode = 'write'}, 55 | }) 56 | t.assert_equals(err, nil) 57 | end 58 | -------------------------------------------------------------------------------- /crud/common/map_call_cases/base_iter.lua: -------------------------------------------------------------------------------- 1 | local errors = require('errors') 2 | 3 | local dev_checks = require('crud.common.dev_checks') 4 | local GetReplicasetsError = errors.new_class('GetReplicasetsError') 5 | 6 | local BaseIterator = {} 7 | 8 | --- Create new base iterator for map call 9 | -- 10 | -- @function new 11 | -- 12 | -- @tparam[opt] table opts 13 | -- Options of BaseIterator:new 14 | -- @tparam[opt] table opts.func_args 15 | -- Function arguments to call 16 | -- @tparam[opt] table opts.replicasets 17 | -- Replicasets to call 18 | -- 19 | -- @return[1] table iterator 20 | -- @treturn[2] nil 21 | -- @treturn[2] table of tables Error description 22 | function BaseIterator:new(opts) 23 | dev_checks('table', { 24 | func_args = '?table', 25 | replicasets = '?table', 26 | vshard_router = 'table', 27 | }) 28 | 29 | local replicasets, err 30 | if opts.replicasets ~= nil then 31 | replicasets = opts.replicasets 32 | else 33 | replicasets, err = opts.vshard_router:routeall() 34 | if err ~= nil then 35 | return nil, GetReplicasetsError:new("Failed to get all replicasets: %s", err.err) 36 | end 37 | end 38 | 39 | local next_index, next_replicaset = next(replicasets) 40 | 41 | local iter = { 42 | func_args = opts.func_args, 43 | replicasets = replicasets, 44 | next_replicaset = next_replicaset, 45 | next_index = next_index 46 | } 47 | 48 | setmetatable(iter, self) 49 | self.__index = self 50 | 51 | return iter 52 | end 53 | 54 | --- Check there is next replicaset to call 55 | -- 56 | -- @function has_next 57 | -- 58 | -- @return[1] boolean 59 | function BaseIterator:has_next() 60 | return self.next_index ~= nil 61 | end 62 | 63 | --- Get function arguments and next replicaset 64 | -- 65 | -- @function get 66 | -- 67 | -- @return[1] table func_args 68 | -- @return[2] table replicaset 69 | -- @return[3] string replicaset_id 70 | function BaseIterator:get() 71 | local replicaset_id = self.next_index 72 | local replicaset = self.next_replicaset 73 | self.next_index, self.next_replicaset = next(self.replicasets, self.next_index) 74 | 75 | return self.func_args, replicaset, replicaset_id 76 | end 77 | 78 | return BaseIterator 79 | -------------------------------------------------------------------------------- /test/luacov-merger.lua: -------------------------------------------------------------------------------- 1 | -- Utility merges luacov coverage statistic files by coverage percentage into a single result 2 | -- USAGE: ... 3 | 4 | local function read_file(filename) 5 | local file = io.open(filename, "r") 6 | if not file then 7 | error("Failed to open file: " .. filename) 8 | end 9 | 10 | local data = {} 11 | for line in file:lines() do 12 | table.insert(data, line) 13 | end 14 | 15 | file:close() 16 | return data 17 | end 18 | 19 | local function merge_coverage_data(files) 20 | local coverage_data = {} 21 | 22 | for _, filename in ipairs(files) do 23 | local data = read_file(filename) 24 | 25 | for i = 1, #data, 2 do 26 | local header = data[i] 27 | local counts = data[i + 1] 28 | 29 | local file_path = header:match(":(.+)") 30 | local line_counts = {} 31 | 32 | for count in counts:gmatch("%d+") do 33 | table.insert(line_counts, tonumber(count)) 34 | end 35 | 36 | if not coverage_data[file_path] then 37 | coverage_data[file_path] = line_counts 38 | else 39 | for j = 1, #line_counts do 40 | coverage_data[file_path][j] = (coverage_data[file_path][j] or 0) + line_counts[j] 41 | end 42 | end 43 | end 44 | end 45 | 46 | return coverage_data 47 | end 48 | 49 | local function write_merged_data_to_file(coverage_data, output_filename) 50 | local file = io.open(output_filename, "w") 51 | if not file then 52 | error("Failed to open file for writing: " .. output_filename) 53 | end 54 | 55 | for file_path, counts in pairs(coverage_data) do 56 | file:write(#counts .. ":" .. file_path .. "\n") 57 | file:write(table.concat(counts, " ") .. "\n") 58 | end 59 | 60 | file:close() 61 | end 62 | 63 | local files_list = table.deepcopy(arg) 64 | files_list[-1] = nil 65 | files_list[0] = nil 66 | local output_filename = files_list[1] 67 | table.remove(files_list, 1) 68 | 69 | local merged_data = merge_coverage_data(files_list) 70 | write_merged_data_to_file(merged_data, output_filename) 71 | print("Luacovs merge: Done") 72 | -------------------------------------------------------------------------------- /crud/truncate.lua: -------------------------------------------------------------------------------- 1 | local checks = require('checks') 2 | local errors = require('errors') 3 | 4 | local dev_checks = require('crud.common.dev_checks') 5 | local call = require('crud.common.call') 6 | local utils = require('crud.common.utils') 7 | 8 | local TruncateError = errors.new_class('TruncateError', {capture_stack = false}) 9 | 10 | local truncate = {} 11 | 12 | local TRUNCATE_FUNC_NAME = 'truncate_on_storage' 13 | local CRUD_TRUNCATE_FUNC_NAME = utils.get_storage_call(TRUNCATE_FUNC_NAME) 14 | 15 | local function truncate_on_storage(space_name) 16 | dev_checks('string') 17 | 18 | local space = box.space[space_name] 19 | if space == nil then 20 | return nil, TruncateError:new("Space %q doesn't exist", space_name) 21 | end 22 | 23 | return space:truncate() 24 | end 25 | 26 | truncate.storage_api = {[TRUNCATE_FUNC_NAME] = truncate_on_storage} 27 | 28 | --- Truncates specified space 29 | -- 30 | -- @function call 31 | -- 32 | -- @param string space_name 33 | -- A space name 34 | -- 35 | -- @tparam ?number opts.timeout 36 | -- Function call timeout 37 | -- 38 | -- @tparam ?string|table opts.vshard_router 39 | -- Cartridge vshard group name or vshard router instance. 40 | -- Set this parameter if your space is not a part of the 41 | -- default vshard cluster. 42 | -- 43 | -- @return[1] boolean true 44 | -- @treturn[2] nil 45 | -- @treturn[2] table Error description 46 | -- 47 | function truncate.call(space_name, opts) 48 | checks('string', { 49 | timeout = '?number', 50 | vshard_router = '?string|table', 51 | }) 52 | 53 | opts = opts or {} 54 | 55 | local vshard_router, err = utils.get_vshard_router_instance(opts.vshard_router) 56 | if err ~= nil then 57 | return nil, TruncateError:new(err) 58 | end 59 | 60 | local replicasets, err = vshard_router:routeall() 61 | if err ~= nil then 62 | return nil, TruncateError:new("Failed to get router replicasets: %s", err) 63 | end 64 | 65 | local _, err = call.map(vshard_router, CRUD_TRUNCATE_FUNC_NAME, {space_name}, { 66 | mode = 'write', 67 | replicasets = replicasets, 68 | timeout = opts.timeout, 69 | }) 70 | 71 | if err ~= nil then 72 | return nil, TruncateError:new("Failed to truncate: %s", err) 73 | end 74 | 75 | return true 76 | end 77 | 78 | return truncate 79 | -------------------------------------------------------------------------------- /crud/compare/keydef.lua: -------------------------------------------------------------------------------- 1 | local comparators = require('crud.compare.comparators') 2 | local collations = require('crud.common.collations') 3 | 4 | local compat = require('crud.common.compat') 5 | local keydef_lib = compat.require('tuple.keydef', 'key_def') 6 | 7 | -- As "tuple.key_def" doesn't support collation_id 8 | -- we manually change it to collation 9 | local function normalize_parts(index_parts) 10 | local result = {} 11 | 12 | for _, part in ipairs(index_parts) do 13 | if part.collation_id == nil then 14 | table.insert(result, part) 15 | else 16 | local part_copy = table.copy(part) 17 | part_copy.collation = collations.get(part) 18 | part_copy.collation_id = nil 19 | table.insert(result, part_copy) 20 | end 21 | end 22 | 23 | return result 24 | end 25 | 26 | local keydef_cache = {} 27 | setmetatable(keydef_cache, {__mode = 'k'}) 28 | 29 | local function new(space, field_names, index_id) 30 | -- Get requested and primary index metainfo. 31 | local index = space.index[index_id] 32 | 33 | -- We use "index" as key here (not some string or something else) 34 | -- since cache should be invalidated on schema update. 35 | -- It will be done automatically because fetch_schema 36 | -- rewrites "index" table in space object. 37 | -- Later lua garbage collector will drop old 38 | -- value from "keydef_cache" table. Since it's a weak table with "k" mode. 39 | if field_names == nil and keydef_cache[index] ~= nil then 40 | return keydef_cache[index] 41 | end 42 | 43 | -- Create a key def 44 | local primary_index = space.index[0] 45 | local space_format = space:format() 46 | local updated_parts = comparators.update_key_parts_by_field_names( 47 | space_format, field_names, index.parts 48 | ) 49 | 50 | local keydef = keydef_lib.new(normalize_parts(updated_parts)) 51 | if not index.unique then 52 | updated_parts = comparators.update_key_parts_by_field_names( 53 | space_format, field_names, primary_index.parts 54 | ) 55 | keydef = keydef:merge(keydef_lib.new(normalize_parts(updated_parts))) 56 | end 57 | 58 | if field_names == nil then 59 | keydef_cache[index] = keydef 60 | end 61 | 62 | return keydef 63 | end 64 | 65 | return { 66 | new = new, 67 | } 68 | -------------------------------------------------------------------------------- /crud/common/stash.lua: -------------------------------------------------------------------------------- 1 | ---- Module for preserving data between reloads. 2 | -- @module crud.common.stash 3 | -- 4 | local dev_checks = require('crud.common.dev_checks') 5 | local utils = require('crud.common.utils') 6 | 7 | local stash = {} 8 | 9 | --- Available stashes list. 10 | -- 11 | -- @tfield string cfg 12 | -- Stash for CRUD module configuration. 13 | -- 14 | -- @tfield string stats_internal 15 | -- Stash for main stats module. 16 | -- 17 | -- @tfield string stats_local_registry 18 | -- Stash for local metrics registry. 19 | -- 20 | -- @tfield string stats_metrics_registry 21 | -- Stash for metrics rocks statistics registry. 22 | -- 23 | -- @tfield string select_module_compat_info 24 | -- Stash for select compatability version registry. 25 | -- 26 | -- @tfield string storage_init 27 | -- Stash for storage initializing tools. 28 | -- 29 | stash.name = { 30 | cfg = '__crud_cfg', 31 | stats_internal = '__crud_stats_internal', 32 | stats_local_registry = '__crud_stats_local_registry', 33 | stats_metrics_registry = '__crud_stats_metrics_registry', 34 | ddl_triggers = '__crud_ddl_spaces_triggers', 35 | select_module_compat_info = '__select_module_compat_info', 36 | storage_readview = '__crud_storage_readview', 37 | storage_init = '__crud_storage_init', 38 | } 39 | 40 | --- Setup Tarantool Cartridge reload. 41 | -- 42 | -- Call on Tarantool Cartridge roles that are expected 43 | -- to use stashes. 44 | -- 45 | -- @function setup_cartridge_reload 46 | -- 47 | -- @return Returns 48 | -- 49 | function stash.setup_cartridge_reload() 50 | local hotreload_supported, hotreload = utils.is_cartridge_hotreload_supported() 51 | if not hotreload_supported then 52 | return 53 | end 54 | 55 | for _, name in pairs(stash.name) do 56 | hotreload.whitelist_globals({ name }) 57 | end 58 | end 59 | 60 | --- Get a stash instance, initialize if needed. 61 | -- 62 | -- Stashes are persistent to package reload. 63 | -- To use them with Cartridge roles reload, 64 | -- call `stash.setup_cartridge_reload` in role. 65 | -- 66 | -- @function get 67 | -- 68 | -- @string name 69 | -- Stash identifier. Use one from `stash.name` table. 70 | -- 71 | -- @treturn table A stash instance. 72 | -- 73 | function stash.get(name) 74 | dev_checks('string') 75 | 76 | local instance = rawget(_G, name) or {} 77 | rawset(_G, name, instance) 78 | 79 | return instance 80 | end 81 | 82 | return stash 83 | -------------------------------------------------------------------------------- /test/tarantool3_helpers/utils.lua: -------------------------------------------------------------------------------- 1 | local function has_role(object, role) 2 | if object == nil then 3 | return false 4 | end 5 | 6 | for _, v in ipairs(object.roles or {}) do 7 | if v == role then 8 | return true 9 | end 10 | end 11 | 12 | return false 13 | end 14 | 15 | local function is_group_has_sharding_role(group, role) 16 | return has_role(group.sharding, role) 17 | end 18 | 19 | local function is_replicaset_has_sharding_role(group, replicaset, role) 20 | if is_group_has_sharding_role(group, role) then 21 | return true 22 | end 23 | 24 | return has_role(replicaset.sharding, role) 25 | end 26 | 27 | local function is_replicaset_a_sharding_router(group, replicaset) 28 | return is_replicaset_has_sharding_role(group, replicaset, 'router') 29 | end 30 | 31 | local function is_replicaset_a_sharding_storage(group, replicaset) 32 | return is_replicaset_has_sharding_role(group, replicaset, 'storage') 33 | end 34 | 35 | local function is_group_a_sharding_router(group) 36 | return is_group_has_sharding_role(group, 'router') 37 | end 38 | 39 | local function is_group_a_sharding_storage(group) 40 | return is_group_has_sharding_role(group, 'storage') 41 | end 42 | 43 | local function is_group_a_crud_router(group) 44 | return has_role(group, 'roles.crud-router') 45 | end 46 | 47 | local function is_group_a_crud_storage(group) 48 | return has_role(group, 'roles.crud-storage') 49 | end 50 | 51 | local function is_replicaset_has_role(group, replicaset, role) 52 | return has_role(group, role) or has_role(replicaset, role) 53 | end 54 | 55 | local function is_replicaset_a_crud_router(group, replicaset) 56 | return is_replicaset_has_role(group, replicaset, 'roles.crud-router') 57 | end 58 | 59 | local function is_replicaset_a_crud_storage(group, replicaset) 60 | return is_replicaset_has_role(group, replicaset, 'roles.crud-storage') 61 | end 62 | 63 | return { 64 | is_group_a_sharding_router = is_group_a_sharding_router, 65 | is_group_a_sharding_storage = is_group_a_sharding_storage, 66 | is_replicaset_a_sharding_router = is_replicaset_a_sharding_router, 67 | is_replicaset_a_sharding_storage = is_replicaset_a_sharding_storage, 68 | 69 | is_group_a_crud_router = is_group_a_crud_router, 70 | is_group_a_crud_storage = is_group_a_crud_storage, 71 | is_replicaset_a_crud_router = is_replicaset_a_crud_router, 72 | is_replicaset_a_crud_storage = is_replicaset_a_crud_storage, 73 | } 74 | -------------------------------------------------------------------------------- /test/integration/async_bootstrap_test.lua: -------------------------------------------------------------------------------- 1 | local t = require('luatest') 2 | 3 | local helpers = require('test.helper') 4 | local vtest = require('test.vshard_helpers.vtest') 5 | 6 | local g = t.group('async_bootstrap') 7 | 8 | local function prepare_clean_cluster(cg) 9 | local cfg = { 10 | sharding = { 11 | ['s-1'] = { 12 | replicas = { 13 | ['s1-master'] = { 14 | instance_uuid = helpers.uuid('b', 1), 15 | master = true, 16 | }, 17 | }, 18 | }, 19 | }, 20 | bucket_count = 3000, 21 | storage_entrypoint = nil, 22 | router_entrypoint = nil, 23 | all_entrypoint = nil, 24 | crud_init = false, 25 | } 26 | 27 | cg.cfg = vtest.config_new(cfg) 28 | vtest.cluster_new(cg, cg.cfg) 29 | end 30 | 31 | 32 | g.test_async_storage_bootstrap = function(cg) 33 | helpers.skip_if_box_watch_unsupported() 34 | 35 | -- Prepare a clean vshard cluster with 1 router and 1 storage. 36 | prepare_clean_cluster(cg) 37 | 38 | -- Sync bootstrap router. 39 | cg.cluster:server('router'):exec(function() 40 | require('crud').init_router() 41 | end) 42 | 43 | -- Async bootstrap storage. 44 | cg.cluster:server('s1-master'):exec(function() 45 | require('crud').init_storage{async = true} 46 | end) 47 | 48 | -- Assert storage is ready after some time. 49 | cg.router = cg.cluster:server('router') 50 | helpers.wait_crud_is_ready_on_cluster(cg, {backend = helpers.backend.VSHARD}) 51 | end 52 | 53 | g.after_test('test_async_storage_bootstrap', function(cg) 54 | if cg.cluster ~= nil then 55 | cg.cluster:drop() 56 | end 57 | end) 58 | 59 | 60 | g.test_async_storage_bootstrap_unsupported = function(cg) 61 | helpers.skip_if_box_watch_supported() 62 | 63 | -- Prepare a clean vshard cluster with 1 router and 1 storage. 64 | prepare_clean_cluster(cg) 65 | 66 | -- Async bootstrap storage (fails). 67 | cg.cluster:server('s1-master'):exec(function() 68 | t.assert_error_msg_contains( 69 | 'async start is supported only for Tarantool versions with box.watch support', 70 | function() 71 | require('crud').init_storage{async = true} 72 | end 73 | ) 74 | end) 75 | end 76 | 77 | g.after_test('test_async_storage_bootstrap_unsupported', function(cg) 78 | if cg.cluster ~= nil then 79 | cg.cluster:drop() 80 | end 81 | end) 82 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | SHELL := /bin/bash 2 | 3 | S3_TARANTOOL_SDK_3_PATH := s3://packages/enterprise/release/linux/x86_64/3.5/tarantool-enterprise-sdk-gc64-3.5.0-0-r70.linux.x86_64.tar.gz 4 | S3_TARANTOOL_SDK_2_PATH := s3://packages/enterprise/release/linux/x86_64/2.11/tarantool-enterprise-sdk-gc64-2.11.7-0-r691.linux.x86_64.tar.gz 5 | S3_ENDPOINT_URL := $(if $(S3_ENDPOINT_URL),$(S3_ENDPOINT_URL),https://hb.vkcs.cloud) 6 | 7 | .rocks: sdk 8 | source ./sdk-2/env.sh && \ 9 | tt rocks install luacheck 0.26.0 --only-server=sdk-2/rocks && \ 10 | tt rocks install luacov 0.13.0 --only-server=sdk-2/rocks && \ 11 | tt rocks install luacov-reporters 0.1.0 --only-server=sdk-2/rocks && \ 12 | tt rocks install metrics 1.5.0 && \ 13 | tt rocks install cartridge 2.16.3 && \ 14 | tt rocks install migrations 1.1.0 && \ 15 | tt rocks make 16 | 17 | sdk-2: 18 | aws --endpoint-url "$(S3_ENDPOINT_URL)" s3 cp "$(S3_TARANTOOL_SDK_2_PATH)" . 19 | mkdir sdk-2 && tar -xvzf tarantool-enterprise-*.tar.gz -C ./sdk-2 --strip-components=1 && rm tarantool-enterprise-*.tar.gz 20 | 21 | sdk-3: 22 | aws --endpoint-url "$(S3_ENDPOINT_URL)" s3 cp "$(S3_TARANTOOL_SDK_3_PATH)" . 23 | mkdir sdk-3 && tar -xvzf tarantool-enterprise-*.tar.gz -C ./sdk-3 --strip-components=1 && rm tarantool-enterprise-*.tar.gz 24 | 25 | sdk: sdk-2 sdk-3 26 | source sdk-3/env.sh && \ 27 | cp sdk-2/rocks/luatest-1.0.1-1.all.rock sdk-3/rocks/ && \ 28 | chmod 644 sdk-3/rocks/* && \ 29 | tt rocks make_manifest sdk-3/rocks 30 | 31 | lint: 32 | source sdk-2/env.sh && .rocks/bin/luacheck . 33 | 34 | .PHONY: test 35 | test: 36 | @if [ -z "$(SDK_TEST)" ]; then \ 37 | echo "Select SDK:"; \ 38 | echo "1) SDK with Tarantool 2.x"; \ 39 | echo "2) SDK with Tarantool 3.x"; \ 40 | read -p "Enter number (1 or 2): " choice; \ 41 | case $$choice in \ 42 | 1) SDK_TEST=sdk-2; SDK_LABEL="SDK with Tarantool 2.x" ;; \ 43 | 2) SDK_TEST=sdk-3; SDK_LABEL="SDK with Tarantool 3.x" ;; \ 44 | *) echo "Invalid selection" >&2; exit 1 ;; \ 45 | esac; \ 46 | else \ 47 | if [ "$(SDK_TEST)" = "sdk-2" ]; then \ 48 | SDK_LABEL="SDK with Tarantool 2.x"; \ 49 | elif [ "$(SDK_TEST)" = "sdk-3" ]; then \ 50 | SDK_LABEL="SDK with Tarantool 3.x"; \ 51 | else \ 52 | SDK_LABEL="Custom SDK ($(SDK_TEST))"; \ 53 | fi; \ 54 | fi; \ 55 | echo "Running tests with $$SDK_LABEL..."; \ 56 | source $$SDK_TEST/env.sh && \ 57 | tt rocks install luatest 1.0.1 --only-server=$$SDK_TEST/rocks && \ 58 | .rocks/bin/luatest -v --coverage test/ 59 | 60 | coverage: 61 | source sdk-2/env.sh && ./.rocks/bin/luacov -r summary && cat luacov.report.out 62 | -------------------------------------------------------------------------------- /cartridge/roles/crud-router.lua: -------------------------------------------------------------------------------- 1 | local errors = require('errors') 2 | 3 | local crud = require('crud') 4 | local stash = require('crud.common.stash') 5 | local stats = require('crud.stats') 6 | 7 | local RoleConfigurationError = errors.new_class('RoleConfigurationError', {capture_stack = false}) 8 | 9 | local function init() 10 | crud.init_router() 11 | stash.setup_cartridge_reload() 12 | end 13 | 14 | local function stop() 15 | crud.stop_router() 16 | end 17 | 18 | local cfg_types = { 19 | stats = 'boolean', 20 | stats_driver = 'string', 21 | stats_quantiles = 'boolean', 22 | stats_quantile_tolerated_error = 'number', 23 | stats_quantile_age_buckets_count = 'number', 24 | stats_quantile_max_age_time = 'number', 25 | } 26 | 27 | local cfg_values = { 28 | stats_driver = function(value) 29 | RoleConfigurationError:assert( 30 | stats.is_driver_supported(value), 31 | 'Invalid crud configuration field "stats_driver" value: %q is not supported', 32 | value 33 | ) 34 | end, 35 | } 36 | 37 | local function validate_config(conf_new, _) 38 | local crud_cfg = conf_new['crud'] 39 | 40 | if crud_cfg == nil then 41 | return true 42 | end 43 | 44 | RoleConfigurationError:assert( 45 | type(crud_cfg) == 'table', 46 | 'Configuration "crud" section must be a table' 47 | ) 48 | 49 | RoleConfigurationError:assert( 50 | crud_cfg.crud == nil, 51 | '"crud" section is already presented as a name of "crud.yml", ' .. 52 | 'do not use it as a top-level section name' 53 | ) 54 | 55 | for name, value in pairs(crud_cfg) do 56 | RoleConfigurationError:assert( 57 | cfg_types[name] ~= nil, 58 | 'Unknown crud configuration field %q', name 59 | ) 60 | 61 | RoleConfigurationError:assert( 62 | type(value) == cfg_types[name], 63 | 'Invalid crud configuration field %q type: expected %s, got %s', 64 | name, cfg_types[name], type(value) 65 | ) 66 | 67 | if cfg_values[name] ~= nil then 68 | cfg_values[name](value) 69 | end 70 | end 71 | 72 | return true 73 | end 74 | 75 | local function apply_config(conf) 76 | crud.cfg(conf['crud']) 77 | end 78 | 79 | return { 80 | role_name = 'crud-router', 81 | init = init, 82 | stop = stop, 83 | validate_config = validate_config, 84 | apply_config = apply_config, 85 | implies_router = true, 86 | dependencies = {'cartridge.roles.vshard-router'}, 87 | } 88 | -------------------------------------------------------------------------------- /crud/stats/registry_utils.lua: -------------------------------------------------------------------------------- 1 | ---- Internal module used by statistics registries. 2 | -- @module crud.stats.registry_utils 3 | -- 4 | 5 | local dev_checks = require('crud.common.dev_checks') 6 | local op_module = require('crud.stats.operation') 7 | 8 | local registry_utils = {} 9 | 10 | --- Build collectors for local registry. 11 | -- 12 | -- @function build_collectors 13 | -- 14 | -- @string op 15 | -- Label of registry collectors. 16 | -- Use `require('crud.stats').op` to pick one. 17 | -- 18 | -- @treturn table Returns collectors for success and error requests. 19 | -- Collectors store 'count', 'latency', 'latency_average', 20 | -- 'latency_quantile_recent' and 'time' values. Also 21 | -- returns additional collectors for select operation. 22 | -- 23 | function registry_utils.build_collectors(op) 24 | dev_checks('string') 25 | 26 | local collectors = { 27 | ok = { 28 | count = 0, 29 | latency = 0, 30 | latency_average = 0, 31 | -- latency_quantile_recent presents only if driver 32 | -- is 'metrics' and quantiles enabled. 33 | latency_quantile_recent = nil, 34 | time = 0, 35 | }, 36 | error = { 37 | count = 0, 38 | latency = 0, 39 | latency_average = 0, 40 | -- latency_quantile_recent presents only if driver 41 | -- is 'metrics' and quantiles enabled. 42 | latency_quantile_recent = nil, 43 | time = 0, 44 | }, 45 | } 46 | 47 | if op == op_module.SELECT then 48 | collectors.details = { 49 | tuples_fetched = 0, 50 | tuples_lookup = 0, 51 | map_reduces = 0, 52 | } 53 | end 54 | 55 | return collectors 56 | end 57 | 58 | --- Initialize all statistic collectors for a space operation. 59 | -- 60 | -- @function init_collectors_if_required 61 | -- 62 | -- @tab spaces 63 | -- `spaces` section of registry. 64 | -- 65 | -- @string space_name 66 | -- Name of space. 67 | -- 68 | -- @string op 69 | -- Label of registry collectors. 70 | -- Use `require('crud.stats').op` to pick one. 71 | -- 72 | function registry_utils.init_collectors_if_required(spaces, space_name, op) 73 | dev_checks('table', 'string', 'string') 74 | 75 | if spaces[space_name] == nil then 76 | spaces[space_name] = {} 77 | end 78 | 79 | local space_collectors = spaces[space_name] 80 | if space_collectors[op] == nil then 81 | space_collectors[op] = registry_utils.build_collectors(op) 82 | end 83 | end 84 | 85 | return registry_utils 86 | -------------------------------------------------------------------------------- /crud/common/map_call_cases/batch_postprocessor.lua: -------------------------------------------------------------------------------- 1 | local dev_checks = require('crud.common.dev_checks') 2 | local utils = require('crud.common.utils') 3 | local sharding_utils = require('crud.common.sharding.utils') 4 | 5 | local BasePostprocessor = require('crud.common.map_call_cases.base_postprocessor') 6 | 7 | local BatchPostprocessor = {} 8 | -- inheritance from BasePostprocessor 9 | setmetatable(BatchPostprocessor, {__index = BasePostprocessor}) 10 | 11 | --- Collect data after call 12 | -- 13 | -- @function collect 14 | -- 15 | -- @tparam[opt] table result_info 16 | -- Data of function call result 17 | -- @tparam[opt] result_info.key 18 | -- Key for collecting result 19 | -- @tparam[opt] result_info.value 20 | -- Value for collecting result by result_info.key 21 | -- 22 | -- @tparam[opt] table err_info 23 | -- Data of function call error 24 | -- @tparam[opt] function|table err_info.err_wrapper 25 | -- Wrapper for error formatting 26 | -- @tparam[opt] table|cdata err_info.err 27 | -- Err of function call 28 | -- @tparam[opt] table err_info.wrapper_args 29 | -- Additional args for error wrapper 30 | -- 31 | -- @return[1] boolean early_exit 32 | function BatchPostprocessor:collect(result_info, err_info) 33 | dev_checks('table', { 34 | key = '?', 35 | value = '?', 36 | },{ 37 | err_wrapper = 'function|table', 38 | err = '?table|cdata', 39 | wrapper_args = '?table', 40 | }) 41 | 42 | if result_info.value ~= nil then 43 | self.storage_info[result_info.key] = {replica_schema_version = result_info.value[3]} 44 | end 45 | 46 | local errs = {err_info.err} 47 | if err_info.err == nil then 48 | errs = result_info.value[2] 49 | end 50 | 51 | if errs ~= nil then 52 | for _, err in pairs(errs) do 53 | local err_to_wrap = err 54 | if err.class_name ~= sharding_utils.ShardingHashMismatchError.name and err.err then 55 | err_to_wrap = err.err 56 | end 57 | 58 | local err_obj = err_info.err_wrapper(self.vshard_router, err_to_wrap, unpack(err_info.wrapper_args)) 59 | err_obj.operation_data = err.operation_data 60 | err_obj.space_schema_hash = err.space_schema_hash 61 | 62 | self.errs = self.errs or {} 63 | table.insert(self.errs, err_obj) 64 | end 65 | end 66 | 67 | if result_info.value ~= nil and result_info.value[1] ~= nil then 68 | self.results = utils.list_extend(self.results, result_info.value[1]) 69 | end 70 | 71 | return self.early_exit 72 | end 73 | 74 | return BatchPostprocessor 75 | -------------------------------------------------------------------------------- /crud/len.lua: -------------------------------------------------------------------------------- 1 | local checks = require('checks') 2 | local errors = require('errors') 3 | 4 | local call = require('crud.common.call') 5 | local utils = require('crud.common.utils') 6 | local dev_checks = require('crud.common.dev_checks') 7 | 8 | local LenError = errors.new_class('LenError', {capture_stack = false}) 9 | 10 | local len = {} 11 | 12 | local LEN_FUNC_NAME = 'len_on_storage' 13 | local CRUD_LEN_FUNC_NAME = utils.get_storage_call(LEN_FUNC_NAME) 14 | 15 | local function len_on_storage(space_name) 16 | dev_checks('string|number') 17 | 18 | return box.space[space_name]:len() 19 | end 20 | 21 | len.storage_api = {[LEN_FUNC_NAME] = len_on_storage} 22 | 23 | --- Calculates the number of tuples in the space for memtx engine 24 | --- Calculates the maximum approximate number of tuples in the space for vinyl engine 25 | -- 26 | -- @function call 27 | -- 28 | -- @param string|number space_name 29 | -- A space name as well as numerical id 30 | -- 31 | -- @tparam ?number opts.timeout 32 | -- Function call timeout 33 | -- 34 | -- @tparam ?string|table opts.vshard_router 35 | -- Cartridge vshard group name or vshard router instance. 36 | -- Set this parameter if your space is not a part of the 37 | -- default vshard cluster. 38 | -- 39 | -- @return[1] number 40 | -- @treturn[2] nil 41 | -- @treturn[2] table Error description 42 | -- 43 | function len.call(space_name, opts) 44 | checks('string', { 45 | timeout = '?number', 46 | vshard_router = '?string|table', 47 | }) 48 | 49 | opts = opts or {} 50 | 51 | local vshard_router, err = utils.get_vshard_router_instance(opts.vshard_router) 52 | if err ~= nil then 53 | return nil, LenError:new(err) 54 | end 55 | 56 | local space, err = utils.get_space(space_name, vshard_router, opts.timeout) 57 | if err ~= nil then 58 | return nil, LenError:new("An error occurred during the operation: %s", err) 59 | end 60 | if space == nil then 61 | return nil, LenError:new("Space %q doesn't exist", space_name) 62 | end 63 | 64 | local results, err = call.map(vshard_router, CRUD_LEN_FUNC_NAME, {space_name}, { 65 | mode = 'write', 66 | timeout = opts.timeout, 67 | }) 68 | 69 | if err ~= nil then 70 | return nil, LenError:new("Failed to call len on storage-side: %s", err) 71 | end 72 | 73 | local total_len = 0 74 | for _, replicaset_results in pairs(results) do 75 | if replicaset_results[1] ~= nil then 76 | total_len = total_len + replicaset_results[1] 77 | end 78 | end 79 | 80 | return total_len 81 | end 82 | 83 | return len 84 | -------------------------------------------------------------------------------- /roles/crud-router.lua: -------------------------------------------------------------------------------- 1 | local errors = require('errors') 2 | 3 | local crud = require('crud') 4 | local common_role_utils = require('crud.common.roles') 5 | local common_utils = require('crud.common.utils') 6 | local stats = require('crud.stats') 7 | 8 | local TarantoolRoleConfigurationError = errors.new_class('TarantoolRoleConfigurationError') 9 | 10 | local tarantool_version = rawget(_G, '_TARANTOOL') 11 | TarantoolRoleConfigurationError:assert( 12 | common_utils.tarantool_supports_config_get_inside_roles(), 13 | ('Tarantool 3 role is not supported for Tarantool %s, use 3.0.2 or newer'):format(tarantool_version) 14 | ) 15 | 16 | 17 | local function validate_enabled_on_sharding_router() 18 | TarantoolRoleConfigurationError:assert( 19 | common_role_utils.is_sharding_role_enabled('router'), 20 | 'Instance must be a sharding router to enable roles.crud-router' 21 | ) 22 | end 23 | 24 | local cfg_types = { 25 | stats = 'boolean', 26 | stats_driver = 'string', 27 | stats_quantiles = 'boolean', 28 | stats_quantile_tolerated_error = 'number', 29 | stats_quantile_age_buckets_count = 'number', 30 | stats_quantile_max_age_time = 'number', 31 | } 32 | 33 | local cfg_values = { 34 | stats_driver = function(value) 35 | TarantoolRoleConfigurationError:assert( 36 | stats.is_driver_supported(value), 37 | 'Invalid "stats_driver" field value: %q is not supported', 38 | value 39 | ) 40 | end, 41 | } 42 | 43 | local function validate_roles_cfg(roles_cfg) 44 | if roles_cfg == nil then 45 | return 46 | end 47 | 48 | TarantoolRoleConfigurationError:assert( 49 | type(roles_cfg) == 'table', 50 | 'roles_cfg must be a table' 51 | ) 52 | 53 | for name, value in pairs(roles_cfg) do 54 | TarantoolRoleConfigurationError:assert( 55 | cfg_types[name] ~= nil, 56 | 'Unknown field %q', name 57 | ) 58 | 59 | TarantoolRoleConfigurationError:assert( 60 | type(value) == cfg_types[name], 61 | 'Invalid %q field type: expected %s, got %s', 62 | name, cfg_types[name], type(value) 63 | ) 64 | 65 | if cfg_values[name] ~= nil then 66 | cfg_values[name](value) 67 | end 68 | end 69 | end 70 | 71 | local function validate(roles_cfg) 72 | validate_enabled_on_sharding_router() 73 | 74 | validate_roles_cfg(roles_cfg) 75 | end 76 | 77 | local function apply(roles_cfg) 78 | crud.init_router() 79 | 80 | crud.cfg(roles_cfg) 81 | end 82 | 83 | local function stop() 84 | crud.stop_router() 85 | end 86 | 87 | return { 88 | validate = validate, 89 | apply = apply, 90 | stop = stop, 91 | } 92 | -------------------------------------------------------------------------------- /test/entrypoint/srv_say_hi/all.lua: -------------------------------------------------------------------------------- 1 | local fiber = require('fiber') 2 | local helper = require('test.helper') 3 | 4 | -- Adds execution rights to a function for a vshard storage user. 5 | local function add_storage_execute(func_name) 6 | if box.schema.user.exists('storage') then 7 | box.schema.func.create(func_name, {setuid = true, if_not_exists = true}) 8 | box.schema.user.grant('storage', 'execute', 'function', func_name, {if_not_exists = true}) 9 | end 10 | end 11 | 12 | return { 13 | init = function() 14 | rawset(_G, 'say_hi_politely', function (to_name) 15 | to_name = to_name or "handsome" 16 | local my_alias = box.info.id 17 | return string.format("HI, %s! I am %s", to_name, my_alias) 18 | end) 19 | 20 | rawset(_G, 'say_hi_sleepily', function (time_to_sleep) 21 | if time_to_sleep ~= nil then 22 | fiber.sleep(time_to_sleep) 23 | end 24 | 25 | return "HI" 26 | end) 27 | 28 | rawset(_G, 'vshard_calls', {}) 29 | 30 | rawset(_G, 'clear_vshard_calls', function() 31 | table.clear(_G.vshard_calls) 32 | end) 33 | 34 | rawset(_G, 'patch_vshard_calls', function(vshard_call_names) 35 | local vshard = require('vshard') 36 | 37 | local replicasets = vshard.router.routeall() 38 | 39 | local _, replicaset = next(replicasets) 40 | local replicaset_mt = getmetatable(replicaset) 41 | 42 | for _, vshard_call_name in ipairs(vshard_call_names) do 43 | local old_func = replicaset_mt.__index[vshard_call_name] 44 | assert(old_func ~= nil, vshard_call_name) 45 | 46 | replicaset_mt.__index[vshard_call_name] = function(...) 47 | local func_name = select(2, ...) 48 | if not string.startswith(func_name, 'vshard.') or func_name == 'vshard.storage.call' then 49 | table.insert(_G.vshard_calls, vshard_call_name) 50 | end 51 | return old_func(...) 52 | end 53 | end 54 | end) 55 | 56 | if type(box.cfg) ~= 'table' then 57 | -- Cartridge, unit tests. 58 | return 59 | else 60 | helper.wrap_schema_init(function() 61 | add_storage_execute('clear_vshard_calls') 62 | add_storage_execute('say_hi_sleepily') 63 | add_storage_execute('say_hi_politely') 64 | add_storage_execute('patch_vshard_calls') 65 | add_storage_execute('non_existent_func') 66 | end)() 67 | end 68 | end, 69 | wait_until_ready = helper.wait_schema_init, 70 | } 71 | -------------------------------------------------------------------------------- /test/integration/truncate_test.lua: -------------------------------------------------------------------------------- 1 | local t = require('luatest') 2 | 3 | local helpers = require('test.helper') 4 | 5 | local pgroup = t.group('truncate', helpers.backend_matrix({ 6 | {engine = 'memtx'}, 7 | })) 8 | 9 | pgroup.before_all(function(g) 10 | helpers.start_default_cluster(g, 'srv_select') 11 | 12 | g.space_format = g.cluster:server('s1-master').net_box.space.customers:format() 13 | end) 14 | 15 | pgroup.after_all(function(g) 16 | helpers.stop_cluster(g.cluster, g.params.backend) 17 | end) 18 | 19 | pgroup.before_each(function(g) 20 | helpers.truncate_space_on_cluster(g.cluster, 'customers') 21 | end) 22 | 23 | 24 | pgroup.test_non_existent_space = function(g) 25 | -- insert 26 | local obj, err = g.router:call( 27 | 'crud.truncate', {'non_existent_space'} 28 | ) 29 | 30 | t.assert_equals(obj, nil) 31 | t.assert_str_contains(err.err, "Space \"non_existent_space\" doesn't exist") 32 | end 33 | 34 | pgroup.test_truncate = function(g) 35 | local customers = helpers.insert_objects(g, 'customers', { 36 | { 37 | id = 1, name = "Elizabeth", last_name = "Jackson", 38 | age = 12, city = "New York", 39 | }, { 40 | id = 2, name = "Mary", last_name = "Brown", 41 | age = 46, city = "Los Angeles", 42 | }, { 43 | id = 3, name = "David", last_name = "Smith", 44 | age = 33, city = "Los Angeles", 45 | }, { 46 | id = 4, name = "William", last_name = "White", 47 | age = 81, city = "Chicago", 48 | }, 49 | }) 50 | 51 | table.sort(customers, function(obj1, obj2) return obj1.id < obj2.id end) 52 | 53 | local result, err = g.router:call('crud.select', { 54 | 'customers', 55 | nil, 56 | {fullscan = true, mode = 'write'}, 57 | }) 58 | t.assert_equals(err, nil) 59 | t.assert_gt(#result.rows, 0) 60 | 61 | local result, err = g.router:call('crud.truncate', {'customers'}) 62 | t.assert_equals(err, nil) 63 | t.assert_equals(result, true) 64 | 65 | local result, err = g.router:call('crud.select', { 66 | 'customers', 67 | nil, 68 | {fullscan = true, mode = 'write'}, 69 | }) 70 | t.assert_equals(err, nil) 71 | t.assert_equals(#result.rows, 0) 72 | end 73 | 74 | pgroup.test_opts_not_damaged = function(g) 75 | local truncate_opts = {timeout = 1} 76 | local new_truncate_opts, err = g.router:eval([[ 77 | local crud = require('crud') 78 | 79 | local truncate_opts = ... 80 | 81 | local _, err = crud.truncate('customers', truncate_opts) 82 | 83 | return truncate_opts, err 84 | ]], {truncate_opts}) 85 | 86 | t.assert_equals(err, nil) 87 | t.assert_equals(new_truncate_opts, truncate_opts) 88 | end 89 | -------------------------------------------------------------------------------- /crud/common/map_call_cases/base_postprocessor.lua: -------------------------------------------------------------------------------- 1 | local dev_checks = require('crud.common.dev_checks') 2 | 3 | local BasePostprocessor = {} 4 | 5 | --- Create new base postprocessor for map call 6 | -- 7 | -- @function new 8 | -- 9 | -- @return[1] table postprocessor 10 | function BasePostprocessor:new(vshard_router) 11 | local postprocessor = { 12 | results = {}, 13 | early_exit = false, 14 | errs = nil, 15 | vshard_router = vshard_router, 16 | storage_info = {}, 17 | } 18 | 19 | setmetatable(postprocessor, self) 20 | self.__index = self 21 | 22 | return postprocessor 23 | end 24 | 25 | --- Collect data after call 26 | -- 27 | -- @function collect 28 | -- 29 | -- @tparam[opt] table result_info 30 | -- Data of function call result 31 | -- @tparam[opt] result_info.key 32 | -- Key for collecting result 33 | -- @tparam[opt] result_info.value 34 | -- Value for collecting result by result_info.key 35 | -- 36 | -- @tparam[opt] table err_info 37 | -- Data of function call error 38 | -- @tparam[opt] function|table err_info.err_wrapper 39 | -- Wrapper for error formatting 40 | -- @tparam[opt] table|cdata err_info.err 41 | -- Err of function call 42 | -- @tparam[opt] table err_info.wrapper_args 43 | -- Additional args for error wrapper 44 | -- 45 | -- @return[1] boolean early_exit 46 | function BasePostprocessor:collect(result_info, err_info) 47 | dev_checks('table', { 48 | key = '?', 49 | value = '?', 50 | },{ 51 | err_wrapper = 'function|table', 52 | err = '?table|cdata', 53 | wrapper_args = '?table', 54 | }) 55 | 56 | if result_info.value ~= nil and type(result_info.value[1]) == 'table' then 57 | if result_info.value[1].storage_info ~= nil then 58 | self.storage_info[result_info.key] = { 59 | replica_schema_version = result_info.value[1].storage_info.replica_schema_version 60 | } 61 | end 62 | end 63 | 64 | local err = err_info.err 65 | if err == nil and result_info.value[1] == nil then 66 | err = result_info.value[2] 67 | end 68 | 69 | if err ~= nil then 70 | self.results = nil 71 | self.errs = err_info.err_wrapper(self.vshard_router, err, unpack(err_info.wrapper_args)) 72 | self.early_exit = true 73 | 74 | return self.early_exit 75 | end 76 | 77 | if self.early_exit ~= true then 78 | self.results[result_info.key] = result_info.value 79 | end 80 | 81 | return self.early_exit 82 | end 83 | 84 | --- Get collected data 85 | -- 86 | -- @function get 87 | -- 88 | -- @return[1] table results 89 | -- @return[2] table errs 90 | -- @return[3] table storage_info 91 | function BasePostprocessor:get() 92 | return self.results, self.errs, self.storage_info 93 | end 94 | 95 | return BasePostprocessor 96 | -------------------------------------------------------------------------------- /test/entrypoint/srv_update_schema/storage.lua: -------------------------------------------------------------------------------- 1 | return { 2 | init = function() 3 | local engine = os.getenv('ENGINE') or 'memtx' 4 | rawset(_G, 'create_space', function() 5 | local customers_space = box.schema.space.create('customers', { 6 | format = { 7 | {name = 'id', type = 'unsigned'}, 8 | {name = 'bucket_id', type = 'unsigned'}, 9 | {name = 'value', type = 'string'}, 10 | {name = 'number', type = 'integer', is_nullable = true}, 11 | }, 12 | if_not_exists = true, 13 | engine = engine, 14 | }) 15 | 16 | -- primary index 17 | customers_space:create_index('id_index', { 18 | parts = { {field = 'id'} }, 19 | if_not_exists = true, 20 | }) 21 | end) 22 | 23 | rawset(_G, 'create_bucket_id_index', function() 24 | box.space.customers:create_index('bucket_id', { 25 | parts = { {field = 'bucket_id'} }, 26 | if_not_exists = true, 27 | unique = false, 28 | }) 29 | end) 30 | 31 | rawset(_G, 'set_value_type_to_unsigned', function() 32 | local new_format = {} 33 | 34 | for _, field_format in ipairs(box.space.customers:format()) do 35 | if field_format.name == 'value' then 36 | field_format.type = 'unsigned' 37 | end 38 | table.insert(new_format, field_format) 39 | end 40 | 41 | box.space.customers:format(new_format) 42 | end) 43 | 44 | rawset(_G, 'add_extra_field', function() 45 | local new_format = box.space.customers:format() 46 | table.insert(new_format, {name = 'extra', type = 'string', is_nullable = true}) 47 | box.space.customers:format(new_format) 48 | end) 49 | 50 | rawset(_G, 'add_value_index', function() 51 | box.space.customers:create_index('value_index', { 52 | parts = { {field = 'value'} }, 53 | if_not_exists = true, 54 | unique = false, 55 | }) 56 | end) 57 | 58 | rawset(_G, 'create_number_value_index', function() 59 | box.space.customers:create_index('number_value_index', { 60 | parts = { {field = 'number'}, {field = 'value'} }, 61 | if_not_exists = true, 62 | unique = false, 63 | }) 64 | end) 65 | 66 | rawset(_G, 'alter_number_value_index', function() 67 | box.space.customers.index.number_value_index:alter({ 68 | parts = { {field = 'value'}, {field = 'number'} }, 69 | }) 70 | end) 71 | end, 72 | } 73 | -------------------------------------------------------------------------------- /crud/common/map_call_cases/batch_insert_iter.lua: -------------------------------------------------------------------------------- 1 | local errors = require('errors') 2 | 3 | local dev_checks = require('crud.common.dev_checks') 4 | local sharding = require('crud.common.sharding') 5 | 6 | local BaseIterator = require('crud.common.map_call_cases.base_iter') 7 | 8 | local SplitTuplesError = errors.new_class('SplitTuplesError') 9 | 10 | local BatchInsertIterator = {} 11 | -- inheritance from BaseIterator 12 | setmetatable(BatchInsertIterator, {__index = BaseIterator}) 13 | 14 | --- Create new batch insert iterator for map call 15 | -- 16 | -- @function new 17 | -- 18 | -- @tparam[opt] table opts 19 | -- Options of BatchInsertIterator:new 20 | -- @tparam[opt] table opts.tuples 21 | -- Tuples to be inserted 22 | -- @tparam[opt] table opts.space 23 | -- Space to be inserted into 24 | -- @tparam[opt] table opts.execute_on_storage_opts 25 | -- Additional opts for call on storage 26 | -- 27 | -- @return[1] table iterator 28 | -- @treturn[2] nil 29 | -- @treturn[2] table of tables Error description 30 | function BatchInsertIterator:new(opts) 31 | dev_checks('table', { 32 | tuples = 'table', 33 | space = 'table', 34 | execute_on_storage_opts = 'table', 35 | vshard_router = 'table', 36 | }) 37 | 38 | local sharding_data, err = sharding.split_tuples_by_replicaset(opts.vshard_router, opts.tuples, opts.space) 39 | if err ~= nil then 40 | return nil, SplitTuplesError:new("Failed to split tuples by replicaset: %s", err.err) 41 | end 42 | 43 | local next_index, next_batch = next(sharding_data.batches) 44 | 45 | local execute_on_storage_opts = opts.execute_on_storage_opts 46 | execute_on_storage_opts.sharding_func_hash = sharding_data.sharding_func_hash 47 | execute_on_storage_opts.sharding_key_hash = sharding_data.sharding_key_hash 48 | execute_on_storage_opts.skip_sharding_hash_check = sharding_data.skip_sharding_hash_check 49 | 50 | local iter = { 51 | space_name = opts.space.name, 52 | opts = execute_on_storage_opts, 53 | batches_by_replicasets = sharding_data.batches, 54 | next_index = next_index, 55 | next_batch = next_batch, 56 | } 57 | 58 | setmetatable(iter, self) 59 | self.__index = self 60 | 61 | return iter 62 | end 63 | 64 | --- Get function arguments and next replicaset 65 | -- 66 | -- @function get 67 | -- 68 | -- @return[1] table func_args 69 | -- @return[2] table replicaset 70 | -- @return[3] string replicaset_id 71 | function BatchInsertIterator:get() 72 | local replicaset_id = self.next_index 73 | local replicaset = self.next_batch.replicaset 74 | local func_args = { 75 | self.space_name, 76 | self.next_batch.tuples, 77 | self.opts, 78 | } 79 | 80 | self.next_index, self.next_batch = next(self.batches_by_replicasets, self.next_index) 81 | 82 | return func_args, replicaset, replicaset_id 83 | end 84 | 85 | return BatchInsertIterator 86 | -------------------------------------------------------------------------------- /crud/compare/type_comparators.lua: -------------------------------------------------------------------------------- 1 | local errors = require('errors') 2 | 3 | local collations = require('crud.common.collations') 4 | local TypeMismatchError = errors.new_class('TypeMismatchError') 5 | local UnsupportedCollationError = errors.new_class('UnsupportedCollationError') 6 | 7 | local types = {} 8 | 9 | local function lt_nullable(cmp) 10 | return function(lhs, rhs) 11 | if lhs == nil and rhs ~= nil then 12 | return true 13 | elseif rhs == nil then 14 | return false 15 | end 16 | 17 | return cmp(lhs, rhs) 18 | end 19 | end 20 | 21 | local function lt_default(lhs, rhs) 22 | return lhs < rhs 23 | end 24 | 25 | local lt = lt_nullable(lt_default) 26 | 27 | local function eq(lhs, rhs) 28 | return lhs == rhs 29 | end 30 | 31 | local function lt_boolean_default(lhs, rhs) 32 | local lhs_is_boolean = type(lhs) == 'boolean' 33 | local rhs_is_boolean = type(rhs) == 'boolean' 34 | 35 | if lhs_is_boolean and rhs_is_boolean then 36 | return (not lhs) and rhs 37 | elseif lhs_is_boolean or rhs_is_boolean then 38 | TypeMismatchError:assert(false, 'Could not compare boolean and not boolean') 39 | end 40 | end 41 | 42 | local lt_boolean = lt_nullable(lt_boolean_default) 43 | 44 | local function lt_unicode(lhs, rhs) 45 | if type(lhs) == 'string' and type(rhs) == 'string' then 46 | return utf8.cmp(lhs, rhs) == -1 47 | end 48 | 49 | return lt(lhs, rhs) 50 | end 51 | 52 | local function lt_unicode_ci(lhs, rhs) 53 | if type(lhs) == 'string' and type(rhs) == 'string' then 54 | return utf8.casecmp(lhs, rhs) == -1 55 | end 56 | 57 | return lt(lhs, rhs) 58 | end 59 | 60 | local function eq_unicode(lhs, rhs) 61 | if type(lhs) == 'string' and type(rhs) == 'string' then 62 | return utf8.cmp(lhs, rhs) == 0 63 | end 64 | 65 | return lhs == rhs 66 | end 67 | 68 | local function eq_unicode_ci(lhs, rhs) 69 | if type(lhs) == 'string' and type(rhs) == 'string' then 70 | return utf8.casecmp(lhs, rhs) == 0 71 | end 72 | 73 | return lhs == rhs 74 | end 75 | 76 | local comparators_by_type = { 77 | boolean = function () 78 | return lt_boolean, eq 79 | end, 80 | string = function (key_part) 81 | local collation = collations.get(key_part) 82 | if collations.is_default(collation) then 83 | return lt, eq 84 | elseif collation == collations.UNICODE then 85 | return lt_unicode, eq_unicode 86 | elseif collation == collations.UNICODE_CI then 87 | return lt_unicode_ci, eq_unicode_ci 88 | else 89 | UnsupportedCollationError:assert(false, 'Unsupported Tarantool collation %q', collation) 90 | end 91 | end, 92 | } 93 | 94 | function types.get_comparators_by_type(key_part) 95 | if comparators_by_type[key_part.type] then 96 | return comparators_by_type[key_part.type](key_part) 97 | else 98 | return lt, eq 99 | end 100 | end 101 | 102 | return types 103 | -------------------------------------------------------------------------------- /crud/common/map_call_cases/batch_upsert_iter.lua: -------------------------------------------------------------------------------- 1 | local errors = require('errors') 2 | 3 | local dev_checks = require('crud.common.dev_checks') 4 | local sharding = require('crud.common.sharding') 5 | 6 | local BaseIterator = require('crud.common.map_call_cases.base_iter') 7 | 8 | local SplitTuplesError = errors.new_class('SplitTuplesError') 9 | 10 | local BatchUpsertIterator = {} 11 | -- inheritance from BaseIterator 12 | setmetatable(BatchUpsertIterator, {__index = BaseIterator}) 13 | 14 | --- Create new batch upsert iterator for map call 15 | -- 16 | -- @function new 17 | -- 18 | -- @tparam[opt] table opts 19 | -- Options of BatchUpsertIterator:new 20 | -- @tparam[opt] table opts.tuples 21 | -- Tuples to be upserted 22 | -- @tparam[opt] table opts.space 23 | -- Space to be upserted into 24 | -- @tparam[opt] table opts.operations 25 | -- Operations to be performed on tuples 26 | -- @tparam[opt] table opts.execute_on_storage_opts 27 | -- Additional opts for call on storage 28 | -- 29 | -- @return[1] table iterator 30 | -- @treturn[2] nil 31 | -- @treturn[2] table of tables Error description 32 | function BatchUpsertIterator:new(opts) 33 | dev_checks('table', { 34 | tuples = 'table', 35 | space = 'table', 36 | operations = 'table', 37 | execute_on_storage_opts = 'table', 38 | vshard_router = 'table', 39 | }) 40 | 41 | local sharding_data, err = sharding.split_tuples_by_replicaset( 42 | opts.vshard_router, 43 | opts.tuples, 44 | opts.space, 45 | {operations = opts.operations}) 46 | 47 | if err ~= nil then 48 | return nil, SplitTuplesError:new("Failed to split tuples by replicaset: %s", err.err) 49 | end 50 | 51 | local next_index, next_batch = next(sharding_data.batches) 52 | 53 | local execute_on_storage_opts = opts.execute_on_storage_opts 54 | execute_on_storage_opts.sharding_func_hash = sharding_data.sharding_func_hash 55 | execute_on_storage_opts.sharding_key_hash = sharding_data.sharding_key_hash 56 | execute_on_storage_opts.skip_sharding_hash_check = sharding_data.skip_sharding_hash_check 57 | 58 | local iter = { 59 | space_name = opts.space.name, 60 | opts = execute_on_storage_opts, 61 | batches_by_replicasets = sharding_data.batches, 62 | next_index = next_index, 63 | next_batch = next_batch, 64 | } 65 | 66 | setmetatable(iter, self) 67 | self.__index = self 68 | 69 | return iter 70 | end 71 | 72 | --- Get function arguments and next replicaset 73 | -- 74 | -- @function get 75 | -- 76 | -- @return[1] table func_args 77 | -- @return[2] table replicaset 78 | -- @return[3] string replicaset_id 79 | function BatchUpsertIterator:get() 80 | local replicaset_id = self.next_index 81 | local replicaset = self.next_batch.replicaset 82 | local func_args = { 83 | self.space_name, 84 | self.next_batch.tuples, 85 | self.next_batch.operations, 86 | self.opts, 87 | } 88 | 89 | self.next_index, self.next_batch = next(self.batches_by_replicasets, self.next_index) 90 | 91 | return func_args, replicaset, replicaset_id 92 | end 93 | 94 | return BatchUpsertIterator 95 | -------------------------------------------------------------------------------- /test/integration/role_test.lua: -------------------------------------------------------------------------------- 1 | local t = require('luatest') 2 | 3 | local helpers = require('test.helper') 4 | 5 | local g = t.group() 6 | 7 | g.before_all(function(cg) 8 | helpers.skip_if_tarantool3_crud_roles_unsupported() 9 | 10 | cg.template_cfg = helpers.build_default_tarantool3_cluster_cfg('srv_select') 11 | end) 12 | 13 | g.before_each(function(cg) 14 | -- Tests are rather dangerous and may break the cluster, 15 | -- so it's safer to restart for each case. 16 | helpers.start_cluster(cg, nil, nil, cg.template_cfg, { 17 | backend = helpers.backend.CONFIG, 18 | }) 19 | end) 20 | 21 | g.after_each(function(cg) 22 | cg.cluster:drop() 23 | end) 24 | 25 | -- Use autoincrement id so one test would be able to call the helper multiple times. 26 | local last_id = 0 27 | local function basic_insert_get_object(cluster) 28 | last_id = last_id + 1 29 | 30 | cluster:server('router'):exec(function(id) 31 | local crud = require('crud') 32 | 33 | local _, err = crud.insert_object('customers', 34 | { 35 | id = id, 36 | name = 'Vincent', 37 | last_name = 'Brooks', 38 | age = 32, 39 | city = 'Babel', 40 | }, 41 | {noreturn = true} 42 | ) 43 | t.assert_equals(err, nil) 44 | 45 | local result, err = crud.get('customers', id, {mode = 'write'}) 46 | t.assert_equals(err, nil) 47 | t.assert_equals(#result.rows, 1, 'Tuple found') 48 | 49 | local objects, err = crud.unflatten_rows(result.rows, result.metadata) 50 | t.assert_equals(err, nil) 51 | t.assert_equals(objects[1].id, id) 52 | t.assert_equals(objects[1].name, 'Vincent') 53 | t.assert_equals(objects[1].last_name, 'Brooks') 54 | t.assert_equals(objects[1].age, 32) 55 | t.assert_equals(objects[1].city, 'Babel') 56 | end, {last_id}) 57 | end 58 | 59 | g.test_cluster_works_if_roles_enabled = function(cg) 60 | basic_insert_get_object(cg.cluster) 61 | end 62 | 63 | g.test_cluster_works_after_vshard_user_password_alter = function(cg) 64 | -- Alter the cluster. 65 | local cfg = cg.cluster:cfg() 66 | 67 | local old_password = cfg.credentials.users['storage'].password 68 | cfg.credentials.users['storage'].password = old_password .. '_new_suffix' 69 | 70 | cg.cluster:reload_config(cfg) 71 | 72 | -- Wait until ready. 73 | helpers.wait_crud_is_ready_on_cluster(cg, {backend = helpers.backend.CONFIG}) 74 | 75 | -- Check everything is fine. 76 | basic_insert_get_object(cg.cluster) 77 | end 78 | 79 | g.test_cluster_works_after_vshard_user_alter = function(cg) 80 | -- Alter the cluster. 81 | local cfg = cg.cluster:cfg() 82 | 83 | cfg.credentials.users['storage'] = nil 84 | cfg.credentials.users['new_storage'] = { 85 | password = 'storing-buckets-instead-of-storage', 86 | roles = {'sharding'}, 87 | } 88 | 89 | cfg.iproto.advertise.sharding.login = 'new_storage' 90 | 91 | cg.cluster:reload_config(cfg) 92 | 93 | -- Wait until ready. 94 | helpers.wait_crud_is_ready_on_cluster(cg, {backend = helpers.backend.CONFIG}) 95 | 96 | -- Check everything is fine. 97 | basic_insert_get_object(cg.cluster) 98 | end 99 | -------------------------------------------------------------------------------- /test/vshard_helpers/cluster.lua: -------------------------------------------------------------------------------- 1 | local fio = require('fio') 2 | local Server = require('test.vshard_helpers.server') 3 | 4 | local root = os.environ()['SOURCEDIR'] or '.' 5 | 6 | local Cluster = {} 7 | 8 | function Cluster:new(object) 9 | self:inherit(object) 10 | object:initialize() 11 | self.servers = object.servers 12 | self.built_servers = object.built_servers 13 | return object 14 | end 15 | 16 | function Cluster:inherit(object) 17 | object = object or {} 18 | setmetatable(object, self) 19 | self.__index = self 20 | self.servers = {} 21 | self.built_servers = {} 22 | return object 23 | end 24 | 25 | function Cluster:initialize() 26 | self.servers = {} 27 | end 28 | 29 | function Cluster:server(alias) 30 | for _, server in ipairs(self.servers) do 31 | if server.alias == alias then 32 | return server 33 | end 34 | end 35 | return nil 36 | end 37 | 38 | function Cluster:drop() 39 | for _, server in ipairs(self.servers) do 40 | if server ~= nil then 41 | server:stop() 42 | server:cleanup() 43 | end 44 | end 45 | end 46 | 47 | function Cluster:get_index(server) 48 | local index = nil 49 | for i, v in ipairs(self.servers) do 50 | if (v.id == server) then 51 | index = i 52 | end 53 | end 54 | return index 55 | end 56 | 57 | function Cluster:delete_server(server) 58 | local idx = self:get_index(server) 59 | if idx == nil then 60 | print("Key does not exist") 61 | else 62 | table.remove(self.servers, idx) 63 | end 64 | end 65 | 66 | function Cluster:stop() 67 | for _, server in ipairs(self.servers) do 68 | if server ~= nil then 69 | server:stop() 70 | end 71 | end 72 | end 73 | 74 | function Cluster:start(opts) 75 | for _, server in ipairs(self.servers) do 76 | if not server.process then 77 | server:start({wait_for_readiness = false}) 78 | end 79 | end 80 | 81 | -- The option is true by default. 82 | local wait_for_readiness = true 83 | if opts ~= nil and opts.wait_for_readiness ~= nil then 84 | wait_for_readiness = opts.wait_for_readiness 85 | end 86 | 87 | if wait_for_readiness then 88 | for _, server in ipairs(self.servers) do 89 | server:wait_for_readiness() 90 | end 91 | end 92 | end 93 | 94 | function Cluster:build_server(server_config, instance_file) 95 | if instance_file == nil then 96 | error('instance_file must be set') 97 | end 98 | server_config = table.deepcopy(server_config) 99 | server_config.command = fio.pathjoin(root, 'test/vshard_helpers/instances/', instance_file) 100 | assert(server_config.alias, 'Either replicaset.alias or server.alias must be given') 101 | local server = Server:new(server_config) 102 | table.insert(self.built_servers, server) 103 | return server 104 | end 105 | 106 | function Cluster:add_server(server) 107 | if self:server(server.alias) ~= nil then 108 | error('Alias is not provided') 109 | end 110 | table.insert(self.servers, server) 111 | end 112 | 113 | return Cluster 114 | -------------------------------------------------------------------------------- /test/entrypoint/srv_schema/storage.lua: -------------------------------------------------------------------------------- 1 | local schema = require('crud.schema') 2 | local helper = require('test.helper') 3 | 4 | return { 5 | init = function() 6 | local engine = os.getenv('ENGINE') or 'memtx' 7 | 8 | rawset(_G, 'reload_schema', function() 9 | for name, space in pairs(box.space) do 10 | -- Can be indexed by space id and space name, 11 | -- so we need to be careful with duplicates. 12 | if type(name) == 'string' and schema.system_spaces[name] == nil then 13 | space:drop() 14 | end 15 | end 16 | 17 | local customers_space = box.schema.space.create('customers', { 18 | format = { 19 | {name = 'id', type = 'unsigned'}, 20 | {name = 'bucket_id', type = 'unsigned'}, 21 | {name = 'name', type = 'string'}, 22 | {name = 'age', type = 'number'}, 23 | }, 24 | if_not_exists = true, 25 | engine = engine, 26 | }) 27 | customers_space:create_index('id', { 28 | parts = { {field = 'id'} }, 29 | if_not_exists = true, 30 | }) 31 | customers_space:create_index('bucket_id', { 32 | parts = { {field = 'bucket_id'} }, 33 | unique = false, 34 | if_not_exists = true, 35 | }) 36 | 37 | local shops_space = box.schema.space.create('shops', { 38 | format = { 39 | {name = 'registry_id', type = 'unsigned'}, 40 | {name = 'bucket_id', type = 'unsigned'}, 41 | {name = 'name', type = 'string'}, 42 | {name = 'address', type = 'string'}, 43 | {name = 'owner', type = 'string', is_nullable = true}, 44 | }, 45 | if_not_exists = true, 46 | engine = engine, 47 | }) 48 | shops_space:create_index('registry', { 49 | parts = { {field = 'registry_id'} }, 50 | if_not_exists = true, 51 | }) 52 | shops_space:create_index('bucket_id', { 53 | parts = { {field = 'bucket_id'} }, 54 | unique = false, 55 | if_not_exists = true, 56 | }) 57 | shops_space:create_index('address', { 58 | parts = { {field = 'address'} }, 59 | unique = true, 60 | if_not_exists = true, 61 | }) 62 | end) 63 | 64 | rawset(_G, 'alter_schema', function() 65 | box.space['customers']:create_index('age', { 66 | parts = { {field = 'age'} }, 67 | unique = false, 68 | if_not_exists = true, 69 | }) 70 | 71 | box.space['shops']:format({ 72 | {name = 'registry_id', type = 'unsigned'}, 73 | {name = 'bucket_id', type = 'unsigned'}, 74 | {name = 'name', type = 'string'}, 75 | {name = 'address', type = 'string'}, 76 | {name = 'owner', type = 'string', is_nullable = true}, 77 | {name = 'salary', type = 'unsigned', is_nullable = true}, 78 | }) 79 | end) 80 | 81 | helper.wrap_schema_init( 82 | rawget(_G, 'reload_schema') 83 | )() 84 | end, 85 | wait_until_ready = helper.wait_schema_init, 86 | } 87 | -------------------------------------------------------------------------------- /test/unit/select_dropped_indexes_test.lua: -------------------------------------------------------------------------------- 1 | local select_plan = require('crud.compare.plan') 2 | 3 | local compare_conditions = require('crud.compare.conditions') 4 | local cond_funcs = compare_conditions.funcs 5 | 6 | local t = require('luatest') 7 | local g = t.group('select_dropped_indexes') 8 | 9 | local helpers = require('test.helper') 10 | 11 | g.before_all = function() 12 | helpers.box_cfg() 13 | 14 | local customers = box.schema.space.create('customers', { 15 | format = { 16 | {'id', 'unsigned'}, 17 | {'bucket_id', 'unsigned'}, 18 | {'name', 'string'}, 19 | {'age', 'unsigned'}, 20 | {'number_of_pets', 'unsigned'}, 21 | {'cars', 'array'}, 22 | }, 23 | if_not_exists = true, 24 | }) 25 | 26 | customers:create_index('index1', { 27 | type = 'TREE', 28 | parts = {'id'}, 29 | if_not_exists = true, 30 | }) 31 | 32 | customers:create_index('index2', { 33 | parts = {'bucket_id'}, 34 | unique = false, 35 | if_not_exists = true, 36 | }) 37 | 38 | customers:create_index('index3', { 39 | type = 'TREE', 40 | parts = {'age'}, 41 | unique = false, 42 | if_not_exists = true, 43 | }) 44 | 45 | customers:create_index('index4_dropped', { 46 | type = 'HASH', 47 | parts = {'name'}, 48 | if_not_exists = true, 49 | }) 50 | 51 | customers:create_index('index5_dropped', { 52 | type = 'HASH', 53 | parts = {'age'}, 54 | if_not_exists = true, 55 | }) 56 | 57 | customers:create_index('index6', { 58 | type = 'TREE', 59 | parts = {'name'}, 60 | unique = false, 61 | if_not_exists = true, 62 | }) 63 | 64 | customers.index.index4_dropped:drop() 65 | customers.index.index5_dropped:drop() 66 | 67 | -- We need this check to make sure that test actually covers a problem. 68 | t.assert_not_equals(#box.space.customers.index, table.maxn(box.space.customers.index)) 69 | end 70 | 71 | g.after_all = function() 72 | box.space.customers:drop() 73 | end 74 | 75 | g.test_before_dropped_index_field = function() 76 | local conditions = { cond_funcs.eq('index3', 20) } 77 | local plan, err = select_plan.new(box.space.customers, conditions) 78 | 79 | t.assert_equals(err, nil) 80 | t.assert_type(plan, 'table') 81 | 82 | t.assert_equals(plan.conditions, conditions) 83 | t.assert_equals(plan.space_name, 'customers') 84 | t.assert_equals(plan.index_id, 2) 85 | end 86 | 87 | g.test_after_dropped_index_field = function() 88 | local conditions = { cond_funcs.eq('index6', 'Alexey') } 89 | local plan, err = select_plan.new(box.space.customers, conditions) 90 | 91 | t.assert_equals(err, nil) 92 | t.assert_type(plan, 'table') 93 | 94 | t.assert_equals(plan.conditions, conditions) 95 | t.assert_equals(plan.space_name, 'customers') 96 | t.assert_equals(plan.index_id, 5) 97 | end 98 | 99 | g.test_non_indexed_field = function() 100 | local conditions = { cond_funcs.eq('number_of_pets', 2) } 101 | local plan, err = select_plan.new(box.space.customers, conditions) 102 | 103 | t.assert_equals(err, nil) 104 | t.assert_type(plan, 'table') 105 | 106 | t.assert_equals(plan.conditions, conditions) 107 | t.assert_equals(plan.space_name, 'customers') 108 | t.assert_equals(plan.index_id, 0) -- PK 109 | end 110 | -------------------------------------------------------------------------------- /crud/common/sharding/storage_metadata_cache.lua: -------------------------------------------------------------------------------- 1 | local stash = require('crud.common.stash') 2 | local utils = require('crud.common.sharding.utils') 3 | 4 | local storage_metadata_cache = {} 5 | 6 | local FUNC = 1 7 | local KEY = 2 8 | 9 | local cache_data = { 10 | [FUNC] = nil, 11 | [KEY] = nil, 12 | } 13 | 14 | local ddl_space = { 15 | [FUNC] = '_ddl_sharding_func', 16 | [KEY] = '_ddl_sharding_key', 17 | } 18 | 19 | local trigger_stash = stash.get(stash.name.ddl_triggers) 20 | 21 | local function update_sharding_func_hash(old, new) 22 | if new ~= nil then 23 | local space_name = new[utils.SPACE_NAME_FIELDNO] 24 | local sharding_func_def = utils.extract_sharding_func_def(new) 25 | cache_data[FUNC][space_name] = utils.compute_hash(sharding_func_def) 26 | else 27 | local space_name = old[utils.SPACE_NAME_FIELDNO] 28 | cache_data[FUNC][space_name] = nil 29 | end 30 | end 31 | 32 | local function update_sharding_key_hash(old, new) 33 | if new ~= nil then 34 | local space_name = new[utils.SPACE_NAME_FIELDNO] 35 | local sharding_key_def = new[utils.SPACE_SHARDING_KEY_FIELDNO] 36 | cache_data[KEY][space_name] = utils.compute_hash(sharding_key_def) 37 | else 38 | local space_name = old[utils.SPACE_NAME_FIELDNO] 39 | cache_data[KEY][space_name] = nil 40 | end 41 | end 42 | 43 | local update_hash = { 44 | [FUNC] = update_sharding_func_hash, 45 | [KEY] = update_sharding_key_hash, 46 | } 47 | 48 | local function init_cache(section) 49 | cache_data[section] = {} 50 | 51 | local space = box.space[ddl_space[section]] 52 | 53 | local update_hash_func = update_hash[section] 54 | 55 | -- Remove old trigger if there was some code reload. 56 | -- It is possible that ddl space was dropped and created again, 57 | -- so removing non-existing trigger will cause fail; 58 | -- thus we use pcall. 59 | pcall(space.on_replace, space, nil, trigger_stash[section]) 60 | 61 | trigger_stash[section] = space:on_replace( 62 | function(old, new) 63 | return update_hash_func(old, new) 64 | end 65 | ) 66 | 67 | for _, tuple in space:pairs() do 68 | local space_name = tuple[utils.SPACE_NAME_FIELDNO] 69 | -- If the cache record for a space is not nil, it means 70 | -- that it was already set to up-to-date value with trigger. 71 | -- It is more like an overcautiousness since the cycle 72 | -- isn't expected to yield, but let it be here. 73 | if cache_data[section][space_name] == nil then 74 | update_hash_func(nil, tuple) 75 | end 76 | end 77 | end 78 | 79 | local function get_sharding_hash(space_name, section) 80 | if box.space[ddl_space[section]] == nil then 81 | return nil 82 | end 83 | 84 | -- If one would drop and rebuild ddl spaces fom scratch manually, 85 | -- caching is likely to break. 86 | if cache_data[section] == nil then 87 | init_cache(section) 88 | end 89 | 90 | return cache_data[section][space_name] 91 | end 92 | 93 | function storage_metadata_cache.get_sharding_func_hash(space_name) 94 | return get_sharding_hash(space_name, FUNC) 95 | end 96 | 97 | function storage_metadata_cache.get_sharding_key_hash(space_name) 98 | return get_sharding_hash(space_name, KEY) 99 | end 100 | 101 | function storage_metadata_cache.drop_caches() 102 | cache_data = {} 103 | end 104 | 105 | return storage_metadata_cache 106 | -------------------------------------------------------------------------------- /CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 3.5 FATAL_ERROR) 2 | 3 | project(crud NONE) 4 | 5 | set(CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake" ${CMAKE_MODULE_PATH}) 6 | set(CMAKE_SKIP_INSTALL_ALL_DEPENDENCY TRUE) 7 | 8 | file(GLOB_RECURSE LUA_FILES 9 | "${CMAKE_CURRENT_SOURCE_DIR}/crud.lua" 10 | "${CMAKE_CURRENT_SOURCE_DIR}/crud/*.lua" 11 | "${CMAKE_CURRENT_SOURCE_DIR}/cartridge/roles/*.lua" 12 | "${CMAKE_CURRENT_SOURCE_DIR}/roles/*.lua" 13 | ) 14 | 15 | ## Testing #################################################################### 16 | ############################################################################### 17 | 18 | enable_testing() 19 | 20 | find_package(LuaCheck) 21 | add_custom_target(luacheck 22 | COMMAND ${LUACHECK} ${PROJECT_SOURCE_DIR} 23 | WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} 24 | ) 25 | 26 | find_package(LuaTest) 27 | find_package(LuaCov) 28 | find_package(LuaCovCoveralls) 29 | 30 | set(CODE_COVERAGE_REPORT "${PROJECT_SOURCE_DIR}/luacov.report.out") 31 | set(CODE_COVERAGE_STATS "${PROJECT_SOURCE_DIR}/luacov.stats.out") 32 | 33 | add_custom_target(luatest 34 | COMMAND ${LUATEST} -v --coverage 35 | BYPRODUCTS ${CODE_COVERAGE_STATS} 36 | WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} 37 | COMMENT "Run regression tests" 38 | ) 39 | 40 | add_custom_target(luatest-no-coverage 41 | COMMAND ${LUATEST} -v 42 | WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} 43 | COMMENT "Run regression tests without coverage" 44 | ) 45 | 46 | set(PERFORMANCE_TESTS_SUBDIR "test/performance") 47 | 48 | add_custom_target(performance 49 | COMMAND PERF_MODE_ON=true ${LUATEST} -v -c ${PERFORMANCE_TESTS_SUBDIR} 50 | WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} 51 | COMMENT "Run performance tests" 52 | ) 53 | 54 | add_custom_target(coverage 55 | COMMAND ${LUACOV} ${PROJECT_SOURCE_DIR} && grep -A999 '^Summary' ${CODE_COVERAGE_REPORT} 56 | DEPENDS ${CODE_COVERAGE_STATS} 57 | BYPRODUCTS ${CODE_COVERAGE_REPORT} 58 | WORKING_DIRECTORY ${PROJECT_SOURCE_DIR} 59 | COMMENT "Generate code coverage stats" 60 | ) 61 | 62 | if(DEFINED ENV{GITHUB_TOKEN}) 63 | set(COVERALLS_COMMAND ${LUACOVCOVERALLS} -v -r ${PROJECT_SOURCE_DIR} --repo-token $ENV{GITHUB_TOKEN}) 64 | else() 65 | set(COVERALLS_COMMAND ${CMAKE_COMMAND} -E echo "Skipped uploading to coveralls.io: no token.") 66 | endif() 67 | 68 | add_custom_target(coveralls 69 | # Replace absolute paths with relative ones. 70 | # In command line: sed -i -e 's@'"$(realpath .)"'/@@'. 71 | COMMAND sed -i -e "\"s@\"'${PROJECT_SOURCE_DIR}'\"/@@\"" ${CODE_COVERAGE_STATS} 72 | COMMAND ${COVERALLS_COMMAND} 73 | DEPENDS ${CODE_COVERAGE_STATS} 74 | WORKING_DIRECTORY ${PROJECT_SOURCE_DIR} 75 | COMMENT "Send code coverage data to the coveralls.io service" 76 | ) 77 | 78 | ## Install #################################################################### 79 | ############################################################################### 80 | 81 | if(NOT DEFINED TARANTOOL_INSTALL_LUADIR) 82 | set(TARANTOOL_INSTALL_LUADIR "${PROJECT_SOURCE_DIR}/.rocks/share/tarantool") 83 | endif() 84 | 85 | install( 86 | DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/${PROJECT_NAME} 87 | DESTINATION ${TARANTOOL_INSTALL_LUADIR} 88 | ) 89 | 90 | install( 91 | FILES ${CMAKE_CURRENT_SOURCE_DIR}/crud.lua 92 | DESTINATION ${TARANTOOL_INSTALL_LUADIR} 93 | ) 94 | 95 | install( 96 | DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/cartridge 97 | DESTINATION ${TARANTOOL_INSTALL_LUADIR} 98 | ) 99 | 100 | install( 101 | DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/roles 102 | DESTINATION ${TARANTOOL_INSTALL_LUADIR} 103 | ) 104 | -------------------------------------------------------------------------------- /test/entrypoint/srv_batch_operations/storage.lua: -------------------------------------------------------------------------------- 1 | local helper = require('test.helper') 2 | 3 | return { 4 | init = helper.wrap_schema_init(function() 5 | local engine = os.getenv('ENGINE') or 'memtx' 6 | local customers_space = box.schema.space.create('customers', { 7 | format = { 8 | {name = 'id', type = 'unsigned'}, 9 | {name = 'bucket_id', type = 'unsigned'}, 10 | {name = 'name', type = 'string'}, 11 | {name = 'age', type = 'number'}, 12 | }, 13 | if_not_exists = true, 14 | engine = engine, 15 | }) 16 | customers_space:create_index('id', { 17 | parts = { {field = 'id'} }, 18 | if_not_exists = true, 19 | }) 20 | customers_space:create_index('bucket_id', { 21 | parts = { {field = 'bucket_id'} }, 22 | unique = false, 23 | if_not_exists = true, 24 | }) 25 | 26 | local developers_space = box.schema.space.create('developers', { 27 | format = { 28 | {name = 'id', type = 'unsigned'}, 29 | {name = 'bucket_id', type = 'unsigned'}, 30 | {name = 'name', type = 'string'}, 31 | {name = 'login', type = 'string'}, 32 | }, 33 | if_not_exists = true, 34 | engine = engine, 35 | }) 36 | developers_space:create_index('id', { 37 | parts = { {field = 'id'} }, 38 | if_not_exists = true, 39 | }) 40 | developers_space:create_index('bucket_id', { 41 | parts = { {field = 'bucket_id'} }, 42 | unique = false, 43 | if_not_exists = true, 44 | }) 45 | developers_space:create_index('login', { 46 | parts = { {field = 'login'} }, 47 | unique = true, 48 | if_not_exists = true, 49 | }) 50 | 51 | local customers_sharded_by_age_space = box.schema.space.create('customers_sharded_by_age', { 52 | format = { 53 | {name = 'id', type = 'unsigned'}, 54 | {name = 'bucket_id', type = 'unsigned'}, 55 | {name = 'name', type = 'string'}, 56 | {name = 'age', type = 'number'}, 57 | }, 58 | if_not_exists = true, 59 | engine = engine, 60 | }) 61 | customers_sharded_by_age_space:create_index('id', { 62 | parts = { {field = 'id'} }, 63 | if_not_exists = true, 64 | }) 65 | customers_sharded_by_age_space:create_index('bucket_id', { 66 | parts = { {field = 'bucket_id'} }, 67 | unique = false, 68 | if_not_exists = true, 69 | }) 70 | 71 | -- https://github.com/tarantool/migrations/blob/a7c31a17f6ac02d4498b4203c23e495856861444/migrator/utils.lua#L35-L53 72 | if box.space._ddl_sharding_key == nil then 73 | local sharding_space = box.schema.space.create('_ddl_sharding_key', { 74 | format = { 75 | {name = 'space_name', type = 'string', is_nullable = false}, 76 | {name = 'sharding_key', type = 'array', is_nullable = false} 77 | }, 78 | if_not_exists = true, 79 | }) 80 | sharding_space:create_index( 81 | 'space_name', { 82 | type = 'TREE', 83 | unique = true, 84 | parts = {{'space_name', 'string', is_nullable = false}}, 85 | if_not_exists = true, 86 | } 87 | ) 88 | end 89 | box.space._ddl_sharding_key:replace{'customers_sharded_by_age', {'age'}} 90 | end), 91 | wait_until_ready = helper.wait_schema_init, 92 | } 93 | -------------------------------------------------------------------------------- /crud/ratelimit.lua: -------------------------------------------------------------------------------- 1 | -- Mostly it's a copy-paste from tarantool/tarantool log.lua: 2 | -- https://github.com/tarantool/tarantool/blob/29654ffe3638e5a218dd32f1788830ff05c1c05c/src/lua/log.lua 3 | -- 4 | -- We have three reasons for the copy-paste: 5 | -- 1. Tarantool has not log.crit() (a function for logging with CRIT level). 6 | -- 2. Only new versions of Tarantool have Ratelimit type. 7 | -- 3. We want own copy of Ratelimit in case the implementation in Tarantool 8 | -- changes. Less pain between Tarantool versions. 9 | local ffi = require('ffi') 10 | 11 | local S_CRIT = ffi.C.S_CRIT 12 | local S_WARN = ffi.C.S_WARN 13 | 14 | local function say(level, fmt, ...) 15 | if ffi.C.log_level < level then 16 | -- don't waste cycles on debug.getinfo() 17 | return 18 | end 19 | local type_fmt = type(fmt) 20 | local format = "%s" 21 | if select('#', ...) ~= 0 then 22 | local stat 23 | stat, fmt = pcall(string.format, fmt, ...) 24 | if not stat then 25 | error(fmt, 3) 26 | end 27 | elseif type_fmt == 'table' then 28 | -- An implementation in tarantool/tarantool supports encoding a table in 29 | -- JSON, but it requires more dependencies from FFI. So we just deleted 30 | -- it because we don't need such encoding in the module. 31 | error("table not supported", 3) 32 | elseif type_fmt ~= 'string' then 33 | fmt = tostring(fmt) 34 | end 35 | 36 | local debug = require('debug') 37 | local frame = debug.getinfo(3, "Sl") 38 | local line, file = 0, 'eval' 39 | if type(frame) == 'table' then 40 | line = frame.currentline or 0 41 | file = frame.short_src or frame.src or 'eval' 42 | end 43 | 44 | ffi.C._say(level, file, line, nil, format, fmt) 45 | end 46 | 47 | local ratelimit_enabled = true 48 | 49 | local function ratelimit_enable() 50 | ratelimit_enabled = true 51 | end 52 | 53 | local function ratelimit_disable() 54 | ratelimit_enabled = false 55 | end 56 | 57 | local Ratelimit = { 58 | interval = 60, 59 | burst = 10, 60 | emitted = 0, 61 | suppressed = 0, 62 | start = 0, 63 | } 64 | 65 | local function ratelimit_new(object) 66 | return Ratelimit:new(object) 67 | end 68 | 69 | function Ratelimit:new(object) 70 | object = object or {} 71 | setmetatable(object, self) 72 | self.__index = self 73 | return object 74 | end 75 | 76 | function Ratelimit:check() 77 | if not ratelimit_enabled then 78 | return 0, true 79 | end 80 | 81 | local clock = require('clock') 82 | local now = clock.monotonic() 83 | local saved_suppressed = 0 84 | if now > self.start + self.interval then 85 | saved_suppressed = self.suppressed 86 | self.suppressed = 0 87 | self.emitted = 0 88 | self.start = now 89 | end 90 | 91 | if self.emitted < self.burst then 92 | self.emitted = self.emitted + 1 93 | return saved_suppressed, true 94 | end 95 | self.suppressed = self.suppressed + 1 96 | return saved_suppressed, false 97 | end 98 | 99 | function Ratelimit:log_check(lvl) 100 | local suppressed, ok = self:check() 101 | if lvl >= S_WARN and suppressed > 0 then 102 | say(S_WARN, '%d messages suppressed due to rate limiting', suppressed) 103 | end 104 | return ok 105 | end 106 | 107 | function Ratelimit:log(lvl, fmt, ...) 108 | if self:log_check(lvl) then 109 | say(lvl, fmt, ...) 110 | end 111 | end 112 | 113 | local function log_ratelimited_closure(lvl) 114 | return function(self, fmt, ...) 115 | self:log(lvl, fmt, ...) 116 | end 117 | end 118 | 119 | Ratelimit.log_crit = log_ratelimited_closure(S_CRIT) 120 | 121 | return { 122 | new = ratelimit_new, 123 | enable = ratelimit_enable, 124 | disable = ratelimit_disable, 125 | } 126 | -------------------------------------------------------------------------------- /test/unit/select_plan_bad_indexes_test.lua: -------------------------------------------------------------------------------- 1 | local select_plan = require('crud.compare.plan') 2 | 3 | local compare_conditions = require('crud.compare.conditions') 4 | local cond_funcs = compare_conditions.funcs 5 | 6 | local t = require('luatest') 7 | local g = t.group('select_plan_bad_indexes') 8 | 9 | local helpers = require('test.helper') 10 | 11 | local NOT_FOUND_INDEX_ERR_MSG = 'An index that matches specified conditions was not found' 12 | 13 | g.before_all = function() 14 | helpers.box_cfg() 15 | 16 | local customers_space = box.schema.space.create('customers', { 17 | format = { 18 | {'id', 'unsigned'}, 19 | {'bucket_id', 'unsigned'}, 20 | {'name', 'string'}, 21 | {'last_name', 'string'}, 22 | {'age', 'unsigned'}, 23 | {'cars', 'array'}, 24 | }, 25 | if_not_exists = true, 26 | }) 27 | customers_space:create_index('id', { -- primary index is HASH 28 | type = 'HASH', 29 | parts = {'id'}, 30 | if_not_exists = true, 31 | }) 32 | customers_space:create_index('bucket_id', { 33 | parts = {'bucket_id'}, 34 | unique = false, 35 | if_not_exists = true, 36 | }) 37 | customers_space:create_index('age_tree', { 38 | type = 'TREE', 39 | parts = {'age'}, 40 | unique = false, 41 | if_not_exists = true, 42 | }) 43 | customers_space:create_index('age_hash', { 44 | type = 'HASH', 45 | parts = {'age'}, 46 | if_not_exists = true, 47 | }) 48 | customers_space:create_index('age_bitset', { 49 | type = 'BITSET', 50 | parts = {'age'}, 51 | unique = false, 52 | if_not_exists = true, 53 | }) 54 | customers_space:create_index('full_name_hash', { 55 | type = 'HASH', 56 | parts = { 57 | {field = 'name', collation = 'unicode_ci'}, 58 | {field = 'last_name'}, 59 | }, 60 | if_not_exists = true, 61 | }) 62 | customers_space:create_index('cars_rtree', { 63 | type = 'RTREE', 64 | parts = {'cars'}, 65 | unique = false, 66 | if_not_exists = true, 67 | }) 68 | end 69 | 70 | g.after_all = function() 71 | box.space.customers:drop() 72 | end 73 | 74 | g.test_select_all_bad_primary = function() 75 | local plan, err = select_plan.new(box.space.customers) 76 | 77 | t.assert_equals(plan, nil) 78 | t.assert_str_contains(err.err, NOT_FOUND_INDEX_ERR_MSG) 79 | end 80 | 81 | g.test_cond_with_good_index = function() 82 | -- check that conditions with bad indexes are just skipped 83 | local plan, err = select_plan.new(box.space.customers, { 84 | cond_funcs.lt('age_hash', 30), 85 | cond_funcs.lt('age_tree', 30), 86 | }) 87 | 88 | t.assert_equals(err, nil) 89 | local index = box.space.customers.index[plan.index_id] 90 | t.assert_equals(index.name, 'age_tree') 91 | end 92 | 93 | g.test_cond_with_hash_index = function() 94 | local plan, err = select_plan.new(box.space.customers, { 95 | cond_funcs.lt('age_hash', 30), 96 | }) 97 | 98 | t.assert_equals(plan, nil) 99 | t.assert_str_contains(err.err, NOT_FOUND_INDEX_ERR_MSG) 100 | end 101 | 102 | g.test_cond_with_hash_index = function() 103 | local plan, err = select_plan.new(box.space.customers, { 104 | cond_funcs.lt('age_bitset', 30), 105 | }) 106 | 107 | t.assert_equals(plan, nil) 108 | t.assert_str_contains(err.err, NOT_FOUND_INDEX_ERR_MSG) 109 | end 110 | 111 | g.test_cond_with_bad_composite_index = function() 112 | local plan, err = select_plan.new(box.space.customers, { 113 | cond_funcs.lt('name', {'John', 'Doe'}), 114 | }) 115 | 116 | t.assert_equals(plan, nil) 117 | t.assert_str_contains(err.err, NOT_FOUND_INDEX_ERR_MSG) 118 | end 119 | 120 | g.test_cond_with_rtree_index = function() 121 | local plan, err = select_plan.new(box.space.customers, { 122 | cond_funcs.eq('cars', {'Porshe', 'Mercedes', 'Range Rover'}), 123 | }) 124 | 125 | t.assert_equals(plan, nil) 126 | t.assert_str_contains(err.err, NOT_FOUND_INDEX_ERR_MSG) 127 | end 128 | -------------------------------------------------------------------------------- /test/unit/parse_conditions_test.lua: -------------------------------------------------------------------------------- 1 | local t = require('luatest') 2 | local g = t.group('parse_conditions') 3 | 4 | local compare_conditions = require('crud.compare.conditions') 5 | local cond_funcs = compare_conditions.funcs 6 | 7 | g.test_parse = function() 8 | local user_conditions = { 9 | {'==', 'aaaa', nil}, 10 | {'=', 'bbb', {12, 'aaaa'}}, 11 | {'<', 'ccccc', 11}, 12 | {'<=', 'dd', {1, 2, 3}}, 13 | {'>', 'eeeeee', 666}, 14 | {'>=', 'f', {3, 3, 4}}, 15 | } 16 | 17 | local conditions, err = compare_conditions.parse(user_conditions) 18 | t.assert_equals(err, nil) 19 | t.assert_equals(conditions, { 20 | cond_funcs.eq('aaaa', nil), 21 | cond_funcs.eq('bbb', {12, 'aaaa'}), 22 | cond_funcs.lt('ccccc', 11), 23 | cond_funcs.le('dd', {1, 2, 3}), 24 | cond_funcs.gt('eeeeee', 666), 25 | cond_funcs.ge('f', {3, 3, 4}), 26 | }) 27 | end 28 | 29 | g.test_parse_errors = function() 30 | -- conditions are no table 31 | local user_conditions = 'bbb = {12}' 32 | 33 | local _, err = compare_conditions.parse(user_conditions) 34 | t.assert_str_contains(err.err, 'Conditions should be table, got "string"') 35 | 36 | -- condition is no table 37 | local user_conditions = { 38 | {'==', 'aaaa', nil}, 39 | 'bbb = {12}', 40 | } 41 | 42 | local _, err = compare_conditions.parse(user_conditions) 43 | t.assert_str_contains(err.err, 'Each condition should be table, got "string" (condition 2)') 44 | 45 | -- condition len is wrong 46 | local user_conditions = { 47 | {'==', 'aaaa', nil}, 48 | {'='}, 49 | } 50 | 51 | local _, err = compare_conditions.parse(user_conditions) 52 | t.assert_str_contains( 53 | err.err, 54 | 'Each condition should be {"", "", } (condition 2)' 55 | ) 56 | 57 | local user_conditions = { 58 | {'==', 'aaaa', nil}, 59 | {'=', 'bb', 1, 2}, 60 | } 61 | 62 | local _, err = compare_conditions.parse(user_conditions) 63 | t.assert_str_contains( 64 | err.err, 65 | 'Each condition should be {"", "", } (condition 2)' 66 | ) 67 | 68 | -- bad operator type 69 | local user_conditions = { 70 | {'==', 'aaaa', nil}, 71 | {3, 'bb', 1}, 72 | } 73 | 74 | local _, err = compare_conditions.parse(user_conditions) 75 | t.assert_str_contains( 76 | err.err, 77 | 'condition[1] should be string, got "number" (condition 2)' 78 | ) 79 | 80 | -- bad operator 81 | local user_conditions = { 82 | {'==', 'aaaa', nil}, 83 | {'===', 'bb', 1}, 84 | } 85 | 86 | local _, err = compare_conditions.parse(user_conditions) 87 | t.assert_str_contains( 88 | err.err, 89 | 'condition[1] "===" isn\'t a valid condition operator, (condition 2)' 90 | ) 91 | 92 | -- bad operand 93 | local user_conditions = { 94 | {'==', 'aaaa', nil}, 95 | {'=', 3, 1}, 96 | } 97 | 98 | local _, err = compare_conditions.parse(user_conditions) 99 | t.assert_str_contains( 100 | err.err, 101 | 'condition[2] should be string, got "number" (condition 2)' 102 | ) 103 | end 104 | 105 | g.test_jsonpath_parse = function() 106 | local user_conditions = { 107 | {'==', '[\'name\']', 'Alexey'}, 108 | {'=', '["name"].a.b', 'Sergey'}, 109 | {'<', '["year"]["field_1"][\'field_2\']', 2021}, 110 | {'<=', '[2].a', {1, 2, 3}}, 111 | {'>', '[2]', 'Jackson'}, 112 | {'>=', '[\'year\'].a["f2"][\'f3\']', 2017}, 113 | } 114 | 115 | local conditions, err = compare_conditions.parse(user_conditions) 116 | t.assert_equals(err, nil) 117 | t.assert_equals(conditions, { 118 | cond_funcs.eq('[\'name\']', 'Alexey'), 119 | cond_funcs.eq('["name"].a.b', 'Sergey'), 120 | cond_funcs.lt('["year"]["field_1"][\'field_2\']', 2021), 121 | cond_funcs.le('[2].a', {1, 2, 3}), 122 | cond_funcs.gt('[2]', 'Jackson'), 123 | cond_funcs.ge('[\'year\'].a["f2"][\'f3\']', 2017), 124 | }) 125 | end 126 | -------------------------------------------------------------------------------- /crud/schema.lua: -------------------------------------------------------------------------------- 1 | local checks = require('checks') 2 | local errors = require('errors') 3 | 4 | local SchemaError = errors.new_class('SchemaError', {capture_stack = false}) 5 | 6 | local schema_module = require('crud.common.schema') 7 | local utils = require('crud.common.utils') 8 | 9 | local schema = {} 10 | 11 | schema.system_spaces = { 12 | -- https://github.com/tarantool/tarantool/blob/3240201a2f5bac3bddf8a74015db9b351954e0b5/src/box/schema_def.h#L77-L127 13 | ['_vinyl_deferred_delete'] = true, 14 | ['_schema'] = true, 15 | ['_collation'] = true, 16 | ['_vcollation'] = true, 17 | ['_space'] = true, 18 | ['_vspace'] = true, 19 | ['_sequence'] = true, 20 | ['_sequence_data'] = true, 21 | ['_vsequence'] = true, 22 | ['_index'] = true, 23 | ['_vindex'] = true, 24 | ['_func'] = true, 25 | ['_vfunc'] = true, 26 | ['_user'] = true, 27 | ['_vuser'] = true, 28 | ['_priv'] = true, 29 | ['_vpriv'] = true, 30 | ['_cluster'] = true, 31 | ['_trigger'] = true, 32 | ['_truncate'] = true, 33 | ['_space_sequence'] = true, 34 | ['_vspace_sequence'] = true, 35 | ['_fk_constraint'] = true, 36 | ['_ck_constraint'] = true, 37 | ['_func_index'] = true, 38 | ['_session_settings'] = true, 39 | ['_gc_consumers'] = true, 40 | -- https://github.com/tarantool/vshard/blob/b3c27b32637863e9a03503e641bb7c8c69779a00/vshard/storage/init.lua#L752 41 | ['_bucket'] = true, 42 | -- https://github.com/tarantool/ddl/blob/b55d0ff7409f32e4d527e2d25444d883bce4163b/test/set_sharding_metadata_test.lua#L92-L98 43 | ['_ddl_sharding_key'] = true, 44 | ['_ddl_sharding_func'] = true, 45 | -- https://github.com/tarantool/tt-ee/blob/6045cd6f4f9b10fbba7e2c6abeecb8f856fee9b0/lib/migrations/internal/eval/body/lua/status_api.lua 46 | ['_tt_migrations'] = true, 47 | -- https://github.com/tarantool/cluster-federation/blob/01738cafa0dc7a3138e64f93c4e84cb323653257/src/internal/utils/utils.go#L17 48 | ['_cdc_state'] = true, 49 | } 50 | 51 | local function get_crud_schema(space) 52 | local sch = schema_module.get_normalized_space_schema(space) 53 | 54 | -- bucket_id is not nullable for a storage, yet 55 | -- it is optional for a crud user. 56 | for _, v in ipairs(sch.format) do 57 | if v.name == 'bucket_id' then 58 | v.is_nullable = true 59 | end 60 | end 61 | 62 | for id, v in pairs(sch.indexes) do 63 | -- There is no reason for a user to know about 64 | -- bucket_id index. 65 | if v.name == 'bucket_id' then 66 | sch.indexes[id] = nil 67 | end 68 | end 69 | 70 | return sch 71 | end 72 | 73 | schema.call = function(space_name, opts) 74 | checks('?string', { 75 | vshard_router = '?string|table', 76 | timeout = '?number', 77 | cached = '?boolean', 78 | }) 79 | 80 | opts = opts or {} 81 | 82 | local vshard_router, err = utils.get_vshard_router_instance(opts.vshard_router) 83 | if err ~= nil then 84 | return nil, SchemaError:new(err) 85 | end 86 | 87 | if opts.cached ~= true then 88 | local _, err = schema_module.reload_schema(vshard_router) 89 | if err ~= nil then 90 | return nil, SchemaError:new(err) 91 | end 92 | end 93 | 94 | local spaces, err = utils.get_spaces(vshard_router, opts.timeout) 95 | if err ~= nil then 96 | return nil, SchemaError:new(err) 97 | end 98 | 99 | if space_name ~= nil then 100 | local space = spaces[space_name] 101 | if space == nil then 102 | return nil, SchemaError:new("Space %q doesn't exist", space_name) 103 | end 104 | return get_crud_schema(space) 105 | else 106 | local resp = {} 107 | 108 | for name, space in pairs(spaces) do 109 | -- Can be indexed by space id and space name, 110 | -- so we need to be careful with duplicates. 111 | if type(name) == 'string' and schema.system_spaces[name] == nil then 112 | resp[name] = get_crud_schema(space) 113 | end 114 | end 115 | 116 | return resp 117 | end 118 | end 119 | 120 | return schema 121 | -------------------------------------------------------------------------------- /crud/compare/conditions.lua: -------------------------------------------------------------------------------- 1 | local errors = require('errors') 2 | 3 | local dev_checks = require('crud.common.dev_checks') 4 | 5 | local ParseConditionError = errors.new_class('ParseConditionError') 6 | 7 | local conditions = {} 8 | conditions.funcs = {} 9 | 10 | conditions.operators = { 11 | EQ = '==', 12 | LT = '<', 13 | LE = '<=', 14 | GT = '>', 15 | GE = '>=', 16 | } 17 | 18 | local tarantool_iter_by_cond_operators = { 19 | [conditions.operators.EQ] = box.index.EQ, 20 | [conditions.operators.LT] = box.index.LT, 21 | [conditions.operators.LE] = box.index.LE, 22 | [conditions.operators.GT] = box.index.GT, 23 | [conditions.operators.GE] = box.index.GE, 24 | } 25 | 26 | local function new_condition(opts) 27 | dev_checks({ 28 | operator = 'string', 29 | operand = 'string|table', 30 | values = '?', 31 | }) 32 | 33 | local values = opts.values 34 | if type(values) ~= 'table' then 35 | values = { values } 36 | end 37 | 38 | local obj = { 39 | operator = opts.operator, 40 | operand = opts.operand, 41 | values = values, 42 | } 43 | 44 | return obj 45 | end 46 | 47 | function conditions.get_tarantool_iter(condition) 48 | return tarantool_iter_by_cond_operators[condition.operator] 49 | end 50 | 51 | local cond_operators_by_func_names = { 52 | eq = conditions.operators.EQ, 53 | lt = conditions.operators.LT, 54 | le = conditions.operators.LE, 55 | gt = conditions.operators.GT, 56 | ge = conditions.operators.GE, 57 | } 58 | 59 | for func_name, operator in pairs(cond_operators_by_func_names) do 60 | assert(operator ~= nil) 61 | conditions.funcs[func_name] = function(operand, values) 62 | dev_checks('string|table', '?') 63 | return new_condition({ 64 | operator = operator, 65 | operand = operand, 66 | values = values, 67 | }) 68 | end 69 | end 70 | 71 | local funcs_by_symbols = { 72 | ['=='] = conditions.funcs.eq, 73 | ['='] = conditions.funcs.eq, 74 | ['<'] = conditions.funcs.lt, 75 | ['<='] = conditions.funcs.le, 76 | ['>'] = conditions.funcs.gt, 77 | ['>='] = conditions.funcs.ge, 78 | } 79 | 80 | function conditions.parse(user_conditions) 81 | if user_conditions == nil then 82 | return nil 83 | end 84 | 85 | if type(user_conditions) ~= 'table' then 86 | return nil, ParseConditionError:new("Conditions should be table, got %q", type(user_conditions)) 87 | end 88 | 89 | local parsed_conditions = {} 90 | 91 | for i, user_condition in ipairs(user_conditions) do 92 | if type(user_condition) ~= 'table' then 93 | return nil, ParseConditionError:new( 94 | "Each condition should be table, got %q (condition %s)", 95 | type(user_condition), i 96 | ) 97 | end 98 | 99 | if #user_condition > 3 or #user_condition < 2 then 100 | return nil, ParseConditionError:new( 101 | 'Each condition should be {"", "", } (condition %s)', i 102 | ) 103 | end 104 | 105 | -- operator 106 | local operator_symbol = user_condition[1] 107 | if type(operator_symbol) ~= 'string' then 108 | return nil, ParseConditionError:new( 109 | "condition[1] should be string, got %q (condition %s)", type(operator_symbol), i 110 | ) 111 | end 112 | 113 | local cond_func = funcs_by_symbols[operator_symbol] 114 | if cond_func == nil then 115 | return nil, ParseConditionError:new( 116 | "condition[1] %q isn't a valid condition operator, (condition %s)", operator_symbol, i 117 | ) 118 | end 119 | 120 | -- operand 121 | local operand = user_condition[2] 122 | if type(operand) ~= 'string' then 123 | return nil, ParseConditionError:new( 124 | "condition[2] should be string, got %q (condition %s)", type(operand), i 125 | ) 126 | end 127 | 128 | local value = user_condition[3] 129 | 130 | table.insert(parsed_conditions, cond_func(operand, value)) 131 | end 132 | 133 | return parsed_conditions 134 | end 135 | 136 | return conditions 137 | -------------------------------------------------------------------------------- /crud/storage_info.lua: -------------------------------------------------------------------------------- 1 | local checks = require('checks') 2 | local errors = require('errors') 3 | local fiber = require('fiber') 4 | local log = require('log') 5 | 6 | local const = require('crud.common.const') 7 | local utils = require('crud.common.utils') 8 | 9 | local StorageInfoError = errors.new_class('StorageInfoError') 10 | 11 | local storage_info = {} 12 | 13 | local STORAGE_INFO_FUNC_NAME = 'storage_info_on_storage' 14 | local CRUD_STORAGE_INFO_FUNC_NAME = utils.get_storage_call(STORAGE_INFO_FUNC_NAME) 15 | 16 | --- Storage status information. 17 | -- 18 | -- @function storage_info_on_storage 19 | -- 20 | -- @return a table with storage status. 21 | local function storage_info_on_storage() 22 | return {status = "running"} 23 | end 24 | 25 | storage_info.storage_api = {[STORAGE_INFO_FUNC_NAME] = storage_info_on_storage} 26 | 27 | --- Polls replicas for storage state 28 | -- 29 | -- @function call 30 | -- 31 | -- @tparam ?number opts.timeout 32 | -- Function call timeout 33 | -- 34 | -- @tparam ?string|table opts.vshard_router 35 | -- Cartridge vshard group name or vshard router instance. 36 | -- 37 | -- @return a table of storage states by replica id. 38 | function storage_info.call(opts) 39 | checks({ 40 | timeout = '?number', 41 | vshard_router = '?string|table', 42 | }) 43 | 44 | opts = opts or {} 45 | 46 | local vshard_router, err = utils.get_vshard_router_instance(opts.vshard_router) 47 | if err ~= nil then 48 | return nil, StorageInfoError:new(err) 49 | end 50 | 51 | local replicasets, err = vshard_router:routeall() 52 | if err ~= nil then 53 | return nil, StorageInfoError:new("Failed to get router replicasets: %s", err.err) 54 | end 55 | 56 | local futures_by_replicas = {} 57 | local replica_state_by_id = {} 58 | local async_opts = {is_async = true} 59 | local timeout = opts.timeout or const.DEFAULT_VSHARD_CALL_TIMEOUT 60 | 61 | for _, replicaset in pairs(replicasets) do 62 | local master = utils.get_replicaset_master(replicaset, {cached = false}) 63 | 64 | for replica_id, replica in pairs(replicaset.replicas) do 65 | 66 | replica_state_by_id[replica_id] = { 67 | status = "error", 68 | is_master = master == replica 69 | } 70 | 71 | if replica.backoff_err ~= nil then 72 | replica_state_by_id[replica_id].message = tostring(replica.backoff_err) 73 | else 74 | local ok, res = pcall(replica.conn.call, replica.conn, CRUD_STORAGE_INFO_FUNC_NAME, 75 | {}, async_opts) 76 | if ok then 77 | futures_by_replicas[replica_id] = res 78 | else 79 | local err_msg = string.format("Error getting storage info for %s", replica_id) 80 | if res ~= nil then 81 | log.error("%s: %s", err_msg, res) 82 | replica_state_by_id[replica_id].message = tostring(res) 83 | else 84 | log.error(err_msg) 85 | replica_state_by_id[replica_id].message = err_msg 86 | end 87 | end 88 | end 89 | end 90 | end 91 | 92 | local deadline = fiber.clock() + timeout 93 | for replica_id, future in pairs(futures_by_replicas) do 94 | local wait_timeout = deadline - fiber.clock() 95 | if wait_timeout < 0 then 96 | wait_timeout = 0 97 | end 98 | 99 | local result, err = future:wait_result(wait_timeout) 100 | if result == nil then 101 | future:discard() 102 | local err_msg = string.format("Error getting storage info for %s", replica_id) 103 | if err ~= nil then 104 | if err.type == 'ClientError' and err.code == box.error.NO_SUCH_PROC then 105 | replica_state_by_id[replica_id].status = "uninitialized" 106 | else 107 | log.error("%s: %s", err_msg, err) 108 | replica_state_by_id[replica_id].message = tostring(err) 109 | end 110 | else 111 | log.error(err_msg) 112 | replica_state_by_id[replica_id].message = err_msg 113 | end 114 | else 115 | replica_state_by_id[replica_id].status = result[1].status or "uninitialized" 116 | end 117 | end 118 | 119 | return replica_state_by_id 120 | end 121 | 122 | return storage_info 123 | -------------------------------------------------------------------------------- /test/integration/read_calls_strategies_test.lua: -------------------------------------------------------------------------------- 1 | local t = require('luatest') 2 | 3 | local helpers = require('test.helper') 4 | 5 | local pgroup = t.group('read_calls_strategies', helpers.backend_matrix({ 6 | -- mode: write 7 | {exp_vshard_call = 'callrw', mode = 'write'}, 8 | 9 | -- mode: read 10 | 11 | -- no params -> callro 12 | {exp_vshard_call = 'callro'}, 13 | 14 | -- not prefer_replica, not balance -> callro 15 | {exp_vshard_call = 'callro', prefer_replica = false, balance = false}, 16 | 17 | -- not prefer_replica, balance -> callbro 18 | {exp_vshard_call = 'callbro', prefer_replica = false, balance = true}, 19 | {exp_vshard_call = 'callbro', balance = true}, 20 | 21 | -- prefer_replica, not balance -> callre 22 | {exp_vshard_call = 'callre', prefer_replica = true, balance = false}, 23 | {exp_vshard_call = 'callre', prefer_replica = true}, 24 | 25 | -- prefer_replica, balance -> callbre 26 | {exp_vshard_call = 'callbre', prefer_replica = true, balance = true}, 27 | })) 28 | 29 | pgroup.before_all(function(g) 30 | helpers.start_default_cluster(g, 'srv_read_calls_strategies') 31 | 32 | g.space_format = g.cluster:server('s1-master').net_box.space.customers:format() 33 | 34 | g.clear_vshard_calls = function() 35 | g.router:call('clear_vshard_calls') 36 | end 37 | 38 | g.get_vshard_call_strategies = function() 39 | -- Retries are possible, especially in CI, so we don't assert 40 | -- the quantity of calls, only strategies used. 41 | local vshard_calls = g.router:eval('return _G.vshard_calls') 42 | 43 | local vshard_call_strategies_map = {} 44 | for _, v in ipairs(vshard_calls) do 45 | vshard_call_strategies_map[v] = true 46 | end 47 | 48 | local vshard_call_strategies = {} 49 | for k, _ in pairs(vshard_call_strategies_map) do 50 | table.insert(vshard_call_strategies, k) 51 | end 52 | return vshard_call_strategies 53 | end 54 | 55 | -- patch vshard.router.call* functions 56 | local vshard_call_names = {'callro', 'callbro', 'callre', 'callbre', 'callrw'} 57 | g.router:call('patch_vshard_calls', {vshard_call_names}) 58 | 59 | helpers.wait_cluster_replication_finished(g) 60 | end) 61 | 62 | pgroup.after_all(function(g) 63 | helpers.stop_cluster(g.cluster, g.params.backend) 64 | end) 65 | 66 | pgroup.test_get = function(g) 67 | g.clear_vshard_calls() 68 | local _, err = g.router:call('crud.get', {'customers', 1, { 69 | mode = g.params.mode, 70 | balance = g.params.balance, 71 | prefer_replica = g.params.prefer_replica 72 | }}) 73 | t.assert_equals(err, nil) 74 | local vshard_call_strategies = g.get_vshard_call_strategies('call_single_impl') 75 | t.assert_equals(vshard_call_strategies, {g.params.exp_vshard_call}) 76 | end 77 | 78 | pgroup.test_select = function(g) 79 | g.clear_vshard_calls() 80 | local _, err = g.router:call('crud.select', {'customers', nil, { 81 | mode = g.params.mode, 82 | balance = g.params.balance, 83 | prefer_replica = g.params.prefer_replica, 84 | fullscan = true 85 | }}) 86 | t.assert_equals(err, nil) 87 | local vshard_call_strategies = g.get_vshard_call_strategies('call_impl') 88 | t.assert_equals(vshard_call_strategies, {g.params.exp_vshard_call}) 89 | end 90 | 91 | pgroup.test_pairs = function(g) 92 | g.clear_vshard_calls() 93 | 94 | local opts = { 95 | mode = g.params.mode, 96 | balance = g.params.balance, 97 | prefer_replica = g.params.prefer_replica 98 | } 99 | 100 | local _, err = g.router:eval([[ 101 | local crud = require('crud') 102 | 103 | local opts = ... 104 | for _, _ in crud.pairs('customers', nil, opts) do end 105 | ]], {opts}) 106 | t.assert_equals(err, nil) 107 | local vshard_call_strategies = g.get_vshard_call_strategies('call_impl') 108 | t.assert_equals(vshard_call_strategies, {g.params.exp_vshard_call}) 109 | end 110 | 111 | pgroup.test_count = function(g) 112 | g.clear_vshard_calls() 113 | local _, err = g.router:call('crud.count', {'customers', nil, { 114 | mode = g.params.mode, 115 | balance = g.params.balance, 116 | prefer_replica = g.params.prefer_replica, 117 | fullscan = true 118 | }}) 119 | t.assert_equals(err, nil) 120 | local vshard_call_strategies = g.get_vshard_call_strategies('call_impl') 121 | t.assert_equals(vshard_call_strategies, {g.params.exp_vshard_call}) 122 | end 123 | -------------------------------------------------------------------------------- /crud/storage.lua: -------------------------------------------------------------------------------- 1 | local checks = require('checks') 2 | 3 | local dev_checks = require('crud.common.dev_checks') 4 | local stash = require('crud.common.stash') 5 | local utils = require('crud.common.utils') 6 | 7 | local call = require('crud.common.call') 8 | local sharding_metadata = require('crud.common.sharding.sharding_metadata') 9 | local insert = require('crud.insert') 10 | local insert_many = require('crud.insert_many') 11 | local replace = require('crud.replace') 12 | local replace_many = require('crud.replace_many') 13 | local get = require('crud.get') 14 | local update = require('crud.update') 15 | local upsert = require('crud.upsert') 16 | local upsert_many = require('crud.upsert_many') 17 | local delete = require('crud.delete') 18 | local select = require('crud.select') 19 | local truncate = require('crud.truncate') 20 | local len = require('crud.len') 21 | local count = require('crud.count') 22 | local borders = require('crud.borders') 23 | local readview = require('crud.readview') 24 | local storage_info = require('crud.storage_info') 25 | 26 | local storage = {} 27 | 28 | local internal_stash = stash.get(stash.name.storage_init) 29 | 30 | local function init_local_part(_, name, func) 31 | rawset(_G[utils.STORAGE_NAMESPACE], name, func) 32 | end 33 | 34 | local function init_persistent_part(user, name, _) 35 | name = utils.get_storage_call(name) 36 | box.schema.func.create(name, {setuid = true, if_not_exists = true}) 37 | box.schema.user.grant(user, 'execute', 'function', name, {if_not_exists = true}) 38 | end 39 | 40 | --- Initializes a storage function by its name. 41 | -- 42 | -- It adds the function into the global scope by its name and required 43 | -- access to a vshard storage user. 44 | -- 45 | -- @function init_storage_call 46 | -- 47 | -- @param string name of a user. 48 | -- @param string name a name of the function. 49 | -- @param function func the function. 50 | -- 51 | -- @return nil 52 | local function init_storage_call(user, storage_api) 53 | dev_checks('?string', 'table') 54 | 55 | for name, func in pairs(storage_api) do 56 | init_local_part(user, name, func) 57 | 58 | if user ~= nil then 59 | init_persistent_part(user, name, func) 60 | end 61 | end 62 | end 63 | 64 | local modules_with_storage_api = { 65 | call, 66 | sharding_metadata, 67 | insert, 68 | insert_many, 69 | get, 70 | replace, 71 | replace_many, 72 | update, 73 | upsert, 74 | upsert_many, 75 | delete, 76 | select, 77 | truncate, 78 | len, 79 | count, 80 | borders, 81 | readview, 82 | -- Must be initialized last: properly working storage info is the flag 83 | -- of initialization success. 84 | storage_info, 85 | } 86 | 87 | local DEFAULT_ASYNC 88 | if utils.is_tarantool_3() then 89 | DEFAULT_ASYNC = true 90 | else 91 | DEFAULT_ASYNC = false 92 | end 93 | 94 | local function init_impl() 95 | rawset(_G, utils.STORAGE_NAMESPACE, {}) 96 | 97 | -- User is required only for persistent part of the init. 98 | -- vshard may not yet be properly set up in cartridge on replicas, 99 | -- see [1] CI fails 100 | -- https://github.com/tarantool/crud/actions/runs/8432361330/job/23091298092?pr=417 101 | local user = nil 102 | if not box.info.ro then 103 | user = utils.get_this_replica_user() or 'guest' 104 | end 105 | 106 | for _, module in ipairs(modules_with_storage_api) do 107 | init_storage_call(user, module.storage_api) 108 | end 109 | end 110 | 111 | function storage.init(opts) 112 | checks({async = '?boolean'}) 113 | 114 | opts = opts or {} 115 | 116 | if opts.async == nil then 117 | opts.async = DEFAULT_ASYNC 118 | end 119 | 120 | if type(box.cfg) ~= 'table' then 121 | error('box.cfg() must be called first') 122 | end 123 | 124 | if internal_stash.watcher ~= nil then 125 | internal_stash.watcher:unregister() 126 | internal_stash.watcher = nil 127 | end 128 | 129 | if opts.async then 130 | assert(utils.tarantool_supports_box_watch(), 131 | 'async start is supported only for Tarantool versions with box.watch support') 132 | 133 | internal_stash.watcher = box.watch('box.status', init_impl) 134 | else 135 | init_impl() 136 | end 137 | end 138 | 139 | function storage.stop() 140 | if internal_stash.watcher ~= nil then 141 | internal_stash.watcher:unregister() 142 | internal_stash.watcher = nil 143 | end 144 | 145 | rawset(_G, utils.STORAGE_NAMESPACE, nil) 146 | end 147 | 148 | return storage 149 | -------------------------------------------------------------------------------- /test/unit/cut_result_test.lua: -------------------------------------------------------------------------------- 1 | local t = require('luatest') 2 | local g = t.group('cut_rows') 3 | 4 | local utils = require('crud.common.utils') 5 | 6 | g.test_cut_rows = function() 7 | local rows = { 8 | {3, 'Pavel', 27}, 9 | {6, 'Alexey', 31}, 10 | {4, 'Mikhail', 51}, 11 | } 12 | 13 | local expected_rows = { 14 | {3, 'Pavel'}, 15 | {6, 'Alexey'}, 16 | {4, 'Mikhail'}, 17 | } 18 | 19 | local metadata = { 20 | {name = 'id', type = 'unsigned'}, 21 | {name = 'name', type = 'string'}, 22 | {name = 'age', type = 'number'}, 23 | } 24 | 25 | local expected_metadata = { 26 | {name = 'id', type = 'unsigned'}, 27 | {name = 'name', type = 'string'}, 28 | } 29 | 30 | local fields = {'id', 'name'} 31 | 32 | local result, err = utils.cut_rows(rows, metadata, fields) 33 | 34 | t.assert_equals(err, nil) 35 | t.assert_equals(result.metadata, expected_metadata) 36 | t.assert_equals(result.rows, expected_rows) 37 | 38 | -- using box.tuple 39 | rows = { 40 | box.tuple.new({3, 'Pavel', 27}), 41 | box.tuple.new({6, 'Alexey', 31}), 42 | box.tuple.new({4, 'Mikhail', 51}), 43 | } 44 | 45 | metadata = { 46 | {name = 'id', type = 'unsigned'}, 47 | {name = 'name', type = 'string'}, 48 | {name = 'age', type = 'number'}, 49 | } 50 | 51 | result, err = utils.cut_rows(rows, metadata, fields) 52 | 53 | t.assert_equals(err, nil) 54 | t.assert_equals(result.metadata, expected_metadata) 55 | t.assert_equals(result.rows, expected_rows) 56 | 57 | -- without metadata 58 | 59 | local rows = { 60 | {3, 'Pavel', 27}, 61 | {6, 'Alexey', 31}, 62 | {4, 'Mikhail', 51}, 63 | } 64 | 65 | result, err = utils.cut_rows(rows, nil, fields) 66 | 67 | t.assert_equals(err, nil) 68 | t.assert_equals(result.metadata, nil) 69 | t.assert_equals(result.rows, expected_rows) 70 | end 71 | 72 | g.test_cut_objects = function() 73 | local objs = { 74 | {id = 3, name = 'Pavel', age = 27}, 75 | {id = 6, name = 'Alexey', age = 31}, 76 | {id = 4, name = 'Mikhail', age = 51}, 77 | } 78 | 79 | local expected_objs = { 80 | {id = 3, name = 'Pavel'}, 81 | {id = 6, name = 'Alexey'}, 82 | {id = 4, name = 'Mikhail'}, 83 | } 84 | 85 | local fields = {'id', 'name'} 86 | 87 | local result = utils.cut_objects(objs, fields) 88 | 89 | t.assert_equals(result, expected_objs) 90 | 91 | -- with nullable field 92 | local objs = { 93 | {id = 3, name = box.NULL, lastname = 'Smith', age = 27}, 94 | {id = 6, name = 'Alexey', lastname = 'Black', age = 31}, 95 | {id = 4, name = 'Mikhail', lastname = 'Smith', age = 51}, 96 | } 97 | 98 | fields = {'id', 'name', 'lastname'} 99 | 100 | local expected_objs = { 101 | {id = 3, name = box.NULL, lastname = 'Smith'}, 102 | {id = 6, name = 'Alexey', lastname = 'Black'}, 103 | {id = 4, name = 'Mikhail', lastname = 'Smith'}, 104 | } 105 | 106 | result = utils.cut_objects(objs, fields) 107 | 108 | t.assert_equals(result, expected_objs) 109 | 110 | fields = {'id', 'surname', 'name'} 111 | 112 | objs = { 113 | {id = 3, name = 'Pavel', lastname = 'Smith', age = 27}, 114 | {id = 6, name = 'Alexey', lastname = 'Black', age = 31}, 115 | {id = 4, name = 'Mikhail', lastname = 'Smith', age = 51}, 116 | } 117 | 118 | expected_objs = { 119 | {id = 3, name = 'Pavel'}, 120 | {id = 6, name = 'Alexey'}, 121 | {id = 4, name = 'Mikhail'}, 122 | } 123 | 124 | result = utils.cut_objects(objs, fields) 125 | 126 | t.assert_equals(result, expected_objs) 127 | end 128 | 129 | g.test_cut_rows_errors = function() 130 | local rows = { 131 | {3, 'Pavel', 27}, 132 | {6, 'Alexey', 31}, 133 | {4, 'Mikhail', 51}, 134 | } 135 | 136 | local metadata = { 137 | {name = 'id', type = 'unsigned'}, 138 | {name = 'name', type = 'string'}, 139 | {name = 'age', type = 'number'}, 140 | } 141 | 142 | local fields = {'name', 'id', 'age'} 143 | 144 | local result, err = utils.cut_rows(rows, metadata, fields) 145 | 146 | t.assert_equals(result, nil) 147 | t.assert_str_contains(err.err, 'Field names don\'t match to tuple metadata') 148 | 149 | fields = {'id', 'lastname'} 150 | 151 | result, err = utils.cut_rows(rows, metadata, fields) 152 | 153 | t.assert_equals(result, nil) 154 | t.assert_str_contains(err.err, 'Field names don\'t match to tuple metadata') 155 | end 156 | -------------------------------------------------------------------------------- /test/integration/cartridge_reload_test.lua: -------------------------------------------------------------------------------- 1 | local fio = require('fio') 2 | local log = require('log') 3 | local fiber = require('fiber') 4 | local errors = require('errors') 5 | 6 | local t = require('luatest') 7 | local g = t.group() 8 | 9 | local helpers = require('test.helper') 10 | 11 | g.before_all(function() 12 | helpers.skip_cartridge_unsupported() 13 | 14 | g.cfg = { 15 | datadir = fio.tempdir(), 16 | use_vshard = true, 17 | server_command = helpers.entrypoint_cartridge('srv_reload'), 18 | replicasets = helpers.get_test_cartridge_replicasets(), 19 | } 20 | 21 | g.cluster = helpers.Cluster:new(g.cfg) 22 | g.cluster:start() 23 | 24 | g.router = assert(g.cluster:server('router')) 25 | g.s1_master = assert(g.cluster:server('s1-master')) 26 | g.s1_replica = assert(g.cluster:server('s1-replica')) 27 | 28 | helpers.wait_crud_is_ready_on_cluster(g, {backend = helpers.backend.CARTRIDGE}) 29 | 30 | g.insertions_passed = {} 31 | g.insertions_failed = {} 32 | end) 33 | 34 | g.after_all(function() 35 | g.cluster:stop() 36 | fio.rmtree(g.cluster.datadir) 37 | end) 38 | 39 | local function _insert(cnt, label) 40 | local result, err = g.router.net_box:call('crud.insert', {'customers', {cnt, 1, label}}) 41 | 42 | if result == nil then 43 | log.error('CNT %d: %s', cnt, err) 44 | table.insert(g.insertions_failed, {cnt = cnt, err = err}) 45 | else 46 | table.insert(g.insertions_passed, result.rows[1]) 47 | end 48 | 49 | return true 50 | end 51 | 52 | local highload_cnt = 0 53 | local function highload_loop(label) 54 | fiber.name('test.highload') 55 | log.warn('Highload started ----------') 56 | 57 | while true do 58 | highload_cnt = highload_cnt + 1 59 | local ok, err = errors.pcall('E', _insert, highload_cnt, label) 60 | 61 | if ok == nil then 62 | log.error('CNT %d: %s', highload_cnt, err) 63 | end 64 | 65 | fiber.sleep(0.002) 66 | end 67 | end 68 | 69 | g.after_each(function() 70 | if g.highload_fiber ~= nil and g.highload_fiber:status() == 'suspended' then 71 | g.highload_fiber:cancel() 72 | end 73 | 74 | log.warn( 75 | 'Total insertions: %d (%d good, %d failed)', 76 | highload_cnt, #g.insertions_passed, #g.insertions_failed 77 | ) 78 | 79 | for _, e in ipairs(g.insertions_failed) do 80 | log.error('#%d: %s', e.cnt, e.err) 81 | end 82 | end) 83 | 84 | function g.test_router() 85 | t.skip_if(not helpers.is_cartridge_hotreload_supported(), 86 | "Cartridge roles reload is not supported") 87 | helpers.skip_old_tarantool_cartridge_hotreload() 88 | 89 | g.highload_fiber = fiber.new(highload_loop, 'A') 90 | 91 | g.cluster:retrying({}, function() 92 | local last_insert = g.insertions_passed[#g.insertions_passed] 93 | t.assert_equals(last_insert[3], 'A', 'No workload for label A') 94 | end) 95 | 96 | helpers.reload_roles(g.router) 97 | 98 | local cnt = #g.insertions_passed 99 | g.cluster:retrying({}, function() 100 | assert(#g.insertions_passed > cnt) 101 | end) 102 | 103 | g.highload_fiber:cancel() 104 | 105 | local result, err = g.router.net_box:call('crud.select', { 106 | 'customers', nil, {fullscan = true, mode = 'write'}, 107 | }) 108 | t.assert_equals(err, nil) 109 | t.assert_items_include(result.rows, g.insertions_passed) 110 | end 111 | 112 | function g.test_storage() 113 | t.skip_if(not helpers.is_cartridge_hotreload_supported(), 114 | "Cartridge roles reload is not supported") 115 | helpers.skip_old_tarantool_cartridge_hotreload() 116 | 117 | g.highload_fiber = fiber.new(highload_loop, 'B') 118 | 119 | g.cluster:retrying({}, function() 120 | local last_insert = g.insertions_passed[#g.insertions_passed] 121 | t.assert_equals(last_insert[3], 'B', 'No workload for label B') 122 | end) 123 | 124 | -- snapshot with a signal 125 | g.s1_master.process:kill('USR1') 126 | 127 | helpers.reload_roles(g.s1_master) 128 | 129 | g.cluster:retrying({}, function() 130 | g.s1_master.net_box:call('box.snapshot') 131 | end) 132 | 133 | local cnt = #g.insertions_passed 134 | g.cluster:retrying({timeout = 2}, function() 135 | helpers.assert_ge(#g.insertions_passed, cnt+1) 136 | end) 137 | 138 | g.highload_fiber:cancel() 139 | 140 | local result, err = g.router.net_box:call('crud.select', { 141 | 'customers', nil, {fullscan = true, mode = 'write'}, 142 | }) 143 | t.assert_equals(err, nil) 144 | t.assert_items_include(result.rows, g.insertions_passed) 145 | end 146 | -------------------------------------------------------------------------------- /test/doc/playground_test.lua: -------------------------------------------------------------------------------- 1 | local yaml = require('yaml') 2 | local t = require('luatest') 3 | local g = t.group() 4 | 5 | local popen_ok, popen = pcall(require, 'popen') 6 | 7 | g.before_all(function() 8 | t.skip_if(not popen_ok, 'no built-in popen module') 9 | t.skip_if(jit.os == 'OSX', 'popen is broken on Mac OS: ' .. 10 | 'https://github.com/tarantool/tarantool/issues/6674') 11 | end) 12 | 13 | -- Run ./doc/playground.lua, execute a request and compare the 14 | -- output with reference return values. 15 | -- 16 | -- The first arguments is the request string. All the following 17 | -- arguments are expected return values (as Lua values). 18 | -- 19 | -- The function ignores trailing `null` values in the YAML 20 | -- output. 21 | local function check_request(request, ...) 22 | local ph, err = popen.new({'./doc/playground.lua'}, { 23 | stdin = popen.opts.PIPE, 24 | stdout = popen.opts.PIPE, 25 | stderr = popen.opts.DEVNULL, 26 | }) 27 | if ph == nil then 28 | error('popen.new: ' .. tostring(err)) 29 | end 30 | 31 | local ok, err = ph:write(request, {timeout = 1}) 32 | if not ok then 33 | ph:close() 34 | error('ph:write: ' .. tostring(err)) 35 | end 36 | ph:shutdown({stdin = true}) 37 | 38 | -- Read everything until EOF. 39 | local chunks = {} 40 | while true do 41 | local chunk, err = ph:read() 42 | if chunk == nil then 43 | ph:close() 44 | error('ph:read: ' .. tostring(err)) 45 | end 46 | if chunk == '' then break end -- EOF 47 | table.insert(chunks, chunk) 48 | end 49 | 50 | local status = ph:wait() 51 | assert(status.state == popen.state.EXITED) 52 | 53 | -- Glue all chunks, parse response. 54 | local stdout = table.concat(chunks) 55 | local response_yaml = string.match(stdout, '%-%-%-.-%.%.%.') 56 | local response = yaml.decode(response_yaml) 57 | 58 | -- NB: This call does NOT differentiate `nil` and `box.NULL`. 59 | t.assert_equals(response, {...}) 60 | end 61 | 62 | local cases = { 63 | test_select_customers = { 64 | request = "crud.select('customers', {{'<=', 'age', 35}}, {first = 10, mode = 'write'})", 65 | retval_1 = { 66 | metadata = { 67 | {name = 'id', type = 'unsigned'}, 68 | {name = 'bucket_id', type = 'unsigned'}, 69 | {name = 'name', type = 'string'}, 70 | {name = 'age', type = 'number'}, 71 | }, 72 | rows = { 73 | {5, 1172, 'Jack', 35}, 74 | {3, 2804, 'David', 33}, 75 | {6, 1064, 'William', 25}, 76 | {7, 693, 'Elizabeth', 18}, 77 | {1, 477, 'Elizabeth', 12}, 78 | }, 79 | } 80 | }, 81 | test_select_developers = { 82 | request = "crud.select('developers', nil, {first = 6, mode = 'write'})", 83 | retval_1 = { 84 | metadata = { 85 | {name = 'id', type = 'unsigned'}, 86 | {name = 'bucket_id', type = 'unsigned'}, 87 | {name = 'name', type = 'string'}, 88 | {name = 'surname', type = 'string'}, 89 | {name = 'age', type = 'number'}, 90 | }, 91 | rows = { 92 | {1, 477, 'Alexey', 'Adams', 20}, 93 | {2, 401, 'Sergey', 'Allred', 21}, 94 | {3, 2804, 'Pavel', 'Adams', 27}, 95 | {4, 1161, 'Mikhail', 'Liston', 51}, 96 | {5, 1172, 'Dmitry', 'Jacobi', 16}, 97 | {6, 1064, 'Alexey', 'Sidorov', 31}, 98 | }, 99 | }, 100 | }, 101 | test_insert = { 102 | request = ("crud.insert('developers', %s)"):format( 103 | "{100, nil, 'Alfred', 'Hitchcock', 123}"), 104 | retval_1 = { 105 | metadata = { 106 | {name = 'id', type = 'unsigned'}, 107 | {name = 'bucket_id', type = 'unsigned'}, 108 | {name = 'name', type = 'string'}, 109 | {name = 'surname', type = 'string'}, 110 | {name = 'age', type = 'number'}, 111 | }, 112 | rows = { 113 | {100, 2976, 'Alfred', 'Hitchcock', 123}, 114 | }, 115 | } 116 | }, 117 | test_error = { 118 | request = [[ 119 | do 120 | local res, err = crud.select('non_existent', nil, {first = 10, mode = 'write'}) 121 | return res, err and err.err or nil 122 | end 123 | ]], 124 | retval_1 = box.NULL, 125 | retval_2 = 'Space "non_existent" doesn\'t exist', 126 | }, 127 | } 128 | 129 | for case_name, case in pairs(cases) do 130 | g[case_name] = function() 131 | check_request(case.request, case.retval_1, case.retval_2) 132 | end 133 | end 134 | -------------------------------------------------------------------------------- /doc/playground.lua: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env tarantool 2 | 3 | -- How to run: 4 | -- 5 | -- $ ./doc/playground.lua 6 | -- 7 | -- Or 8 | -- 9 | -- $ KEEP_DATA=1 ./doc/playground.lua 10 | -- 11 | -- What to do next: 12 | -- 13 | -- Choose an example from README.md, doc/select.md or doc/pairs.md 14 | -- and run. For example: 15 | -- 16 | -- tarantool> crud.select('customers', {{'<=', 'age', 35}}, {first = 10}) 17 | -- tarantool> crud.select('developers', nil, {first = 6}) 18 | 19 | local fio = require('fio') 20 | local console = require('console') 21 | local vshard = require('vshard') 22 | local crud = require('crud') 23 | 24 | -- Trick to don't leave *.snap, *.xlog files. See 25 | -- test/tuple_keydef.test.lua in the tuple-keydef module. 26 | if os.getenv('KEEP_DATA') ~= nil then 27 | box.cfg() 28 | else 29 | local tempdir = fio.tempdir() 30 | box.cfg({ 31 | memtx_dir = tempdir, 32 | wal_mode = 'none', 33 | }) 34 | fio.rmtree(tempdir) 35 | end 36 | 37 | local replicaset_uuid 38 | if box.info().replicaset ~= nil then 39 | replicaset_uuid = box.info().replicaset.uuid 40 | else 41 | replicaset_uuid = box.info().cluster.uuid 42 | end 43 | 44 | -- Setup vshard. 45 | _G.vshard = vshard 46 | local uri = 'guest@localhost:3301' 47 | local cfg = { 48 | bucket_count = 3000, 49 | sharding = { 50 | [replicaset_uuid] = { 51 | replicas = { 52 | [box.info().uuid] = { 53 | uri = uri, 54 | name = 'storage', 55 | master = true, 56 | }, 57 | }, 58 | }, 59 | }, 60 | } 61 | vshard.storage.cfg(cfg, box.info().uuid) 62 | vshard.router.cfg(cfg) 63 | vshard.router.bootstrap() 64 | 65 | -- Create the 'customers' space. 66 | box.once('customers', function() 67 | box.schema.create_space('customers', { 68 | format = { 69 | {name = 'id', type = 'unsigned'}, 70 | {name = 'bucket_id', type = 'unsigned'}, 71 | {name = 'name', type = 'string'}, 72 | {name = 'age', type = 'number'}, 73 | } 74 | }) 75 | box.space.customers:create_index('primary_index', { 76 | parts = { 77 | {field = 1, type = 'unsigned'}, 78 | }, 79 | }) 80 | box.space.customers:create_index('bucket_id', { 81 | parts = { 82 | {field = 2, type = 'unsigned'}, 83 | }, 84 | unique = false, 85 | }) 86 | box.space.customers:create_index('age', { 87 | parts = { 88 | {field = 4, type = 'number'}, 89 | }, 90 | unique = false, 91 | }) 92 | 93 | -- Fill the space. 94 | box.space.customers:insert({1, 477, 'Elizabeth', 12}) 95 | box.space.customers:insert({2, 401, 'Mary', 46}) 96 | box.space.customers:insert({3, 2804, 'David', 33}) 97 | box.space.customers:insert({4, 1161, 'William', 81}) 98 | box.space.customers:insert({5, 1172, 'Jack', 35}) 99 | box.space.customers:insert({6, 1064, 'William', 25}) 100 | box.space.customers:insert({7, 693, 'Elizabeth', 18}) 101 | end) 102 | 103 | -- Create the developers space. 104 | box.once('developers', function() 105 | box.schema.create_space('developers', { 106 | format = { 107 | {name = 'id', type = 'unsigned'}, 108 | {name = 'bucket_id', type = 'unsigned'}, 109 | {name = 'name', type = 'string'}, 110 | {name = 'surname', type = 'string'}, 111 | {name = 'age', type = 'number'}, 112 | } 113 | }) 114 | box.space.developers:create_index('primary_index', { 115 | parts = { 116 | {field = 1, type = 'unsigned'}, 117 | }, 118 | }) 119 | box.space.developers:create_index('bucket_id', { 120 | parts = { 121 | {field = 2, type = 'unsigned'}, 122 | }, 123 | unique = false, 124 | }) 125 | box.space.developers:create_index('age_index', { 126 | parts = { 127 | {field = 5, type = 'number'}, 128 | }, 129 | unique = false, 130 | }) 131 | box.space.developers:create_index('full_name', { 132 | parts = { 133 | {field = 3, type = 'string'}, 134 | {field = 4, type = 'string'}, 135 | }, 136 | unique = false, 137 | }) 138 | 139 | -- Fill the space. 140 | box.space.developers:insert({1, 477, 'Alexey', 'Adams', 20}) 141 | box.space.developers:insert({2, 401, 'Sergey', 'Allred', 21}) 142 | box.space.developers:insert({3, 2804, 'Pavel', 'Adams', 27}) 143 | box.space.developers:insert({4, 1161, 'Mikhail', 'Liston', 51}) 144 | box.space.developers:insert({5, 1172, 'Dmitry', 'Jacobi', 16}) 145 | box.space.developers:insert({6, 1064, 'Alexey', 'Sidorov', 31}) 146 | end) 147 | 148 | -- Initialize crud. 149 | crud.init_storage() 150 | crud.init_router() 151 | 152 | -- Start a console. 153 | console.start() 154 | os.exit() 155 | -------------------------------------------------------------------------------- /test/unit/serialization_test.lua: -------------------------------------------------------------------------------- 1 | local utils = require('crud.common.utils') 2 | 3 | local t = require('luatest') 4 | local g = t.group('serialization') 5 | 6 | local helpers = require('test.helper') 7 | 8 | g.before_all = function() 9 | helpers.box_cfg() 10 | 11 | local customers_space = box.schema.space.create('customers', { 12 | format = { 13 | {name = 'id', type = 'unsigned'}, 14 | {name = 'bucket_id', type = 'unsigned'}, 15 | {name = 'name', type = 'string'}, 16 | {name = 'age', type = 'number', is_nullable = true}, 17 | }, 18 | if_not_exists = true, 19 | }) 20 | customers_space:create_index('id', { 21 | parts = {'id'}, 22 | if_not_exists = true, 23 | }) 24 | end 25 | 26 | g.after_all(function() 27 | box.space.customers:drop() 28 | end) 29 | 30 | g.test_flatten = function() 31 | local space_format = box.space.customers:format() 32 | 33 | -- ok 34 | local object = { 35 | id = 1, 36 | bucket_id = 1024, 37 | name = 'Marilyn', 38 | age = 50, 39 | } 40 | 41 | local tuple, err = utils.flatten(object, space_format) 42 | t.assert_equals(err, nil) 43 | t.assert_equals(tuple, {1, 1024, 'Marilyn', 50}) 44 | 45 | 46 | -- set bucket_id 47 | local object = { 48 | id = 1, 49 | name = 'Marilyn', 50 | age = 50, 51 | } 52 | 53 | local tuple, err = utils.flatten(object, space_format, 1025) 54 | t.assert_equals(err, nil) 55 | t.assert_equals(tuple, {1, 1025, 'Marilyn', 50}) 56 | 57 | -- non-nullable field name is nil 58 | local object = { 59 | id = 1, 60 | bucket_id = 1024, 61 | name = nil, 62 | age = 50, 63 | } 64 | 65 | local tuple, err = utils.flatten(object, space_format) 66 | t.assert_equals(tuple, nil) 67 | t.assert_not_equals(err, nil) 68 | t.assert_str_contains(err.err, 'Field "name" isn\'t nullable') 69 | 70 | -- system field bucket_id is nil 71 | local object = { 72 | id = 1, 73 | bucket_id = nil, 74 | name = 'Marilyn', 75 | age = 50, 76 | } 77 | 78 | local tuple, err = utils.flatten(object, space_format) 79 | t.assert_equals(err, nil) 80 | t.assert_equals(tuple, {1, nil, 'Marilyn', 50}) 81 | 82 | -- nullable field is nil 83 | local object = { 84 | id = 1, 85 | bucket_id = 1024, 86 | name = 'Marilyn', 87 | age = nil, 88 | } 89 | 90 | local tuple, err = utils.flatten(object, space_format) 91 | t.assert_equals(err, nil) 92 | t.assert_equals(tuple, {1, 1024, 'Marilyn', nil}) 93 | 94 | -- unknown field is specififed 95 | local object = { 96 | id = 1, 97 | bucket_id = 1024, 98 | name = 'Marilyn', 99 | age = 50, 100 | some_unknown_field = 'XXX', 101 | } 102 | 103 | local tuple, err = utils.flatten(object, space_format) 104 | t.assert_is(tuple, nil) 105 | t.assert_is_not(err, nil) 106 | t.assert_str_contains(err.err, 'Unknown field \"some_unknown_field\" is specified') 107 | end 108 | 109 | g.test_unflatten = function() 110 | local space_format = box.space.customers:format() 111 | 112 | -- ok 113 | local tuple = {1, 1024, 'Marilyn', 50} 114 | local object, err = utils.unflatten(tuple, space_format) 115 | t.assert_equals(err, nil) 116 | t.assert_equals(object, { 117 | id = 1, 118 | bucket_id = 1024, 119 | name = 'Marilyn', 120 | age = 50, 121 | }) 122 | 123 | -- non-nullable field id nil 124 | local tuple = {1, nil, 'Marilyn', 50} 125 | local object, err = utils.unflatten(tuple, space_format) 126 | t.assert_equals(object, nil) 127 | t.assert_not_equals(err, nil) 128 | t.assert_str_contains(err.err, "Field 2 isn't nullable") 129 | 130 | -- nullable field is nil 131 | local tuple = {1, 1024, 'Marilyn', nil} 132 | local object, err = utils.unflatten(tuple, space_format) 133 | t.assert_equals(err, nil) 134 | t.assert_equals(object, { 135 | id = 1, 136 | bucket_id = 1024, 137 | name = 'Marilyn', 138 | age = nil, 139 | }) 140 | 141 | -- extra field 142 | local tuple = {1, 1024, 'Marilyn', 50, 'one-bad-value'} 143 | local object, err = utils.unflatten(tuple, space_format) 144 | t.assert_equals(err, nil) 145 | t.assert_equals(object, { 146 | id = 1, 147 | bucket_id = 1024, 148 | name = 'Marilyn', 149 | age = 50, 150 | }) 151 | end 152 | 153 | g.test_extract_key = function() 154 | local tuple = {1, nil, 'Marilyn', 50} 155 | 156 | local key = utils.extract_key(tuple, {{fieldno = 1}}) 157 | t.assert_equals(key, {1}) 158 | 159 | local key = utils.extract_key(tuple, { 160 | {fieldno = 3}, {fieldno = 2}, {fieldno = 1}, 161 | }) 162 | t.assert_equals(key, {'Marilyn', nil, 1}) 163 | end 164 | -------------------------------------------------------------------------------- /test/tarantool3_helpers/treegen.lua: -------------------------------------------------------------------------------- 1 | -- Borrowed from https://github.com/tarantool/tarantool/blob/b5864c40a0bfc8f26cc65189f3a5c76e441a9396/test/treegen.lua 2 | 3 | -- Working tree generator. 4 | -- 5 | -- Generates a tree of Lua files using provided templates and 6 | -- filenames. Reworked to be used inside the Cluster. 7 | 8 | local fio = require('fio') 9 | local log = require('log') 10 | local fun = require('fun') 11 | 12 | local treegen = {} 13 | 14 | local function find_template(storage, script) 15 | for _, template_def in ipairs(storage.templates) do 16 | if script:match(template_def.pattern) then 17 | return template_def.template 18 | end 19 | end 20 | error(("treegen: can't find a template for script %q"):format(script)) 21 | end 22 | 23 | -- Write provided script into the given directory. 24 | function treegen.write_script(dir, script, body) 25 | local script_abspath = fio.pathjoin(dir, script) 26 | local flags = {'O_CREAT', 'O_WRONLY', 'O_TRUNC'} 27 | local mode = tonumber('644', 8) 28 | 29 | local scriptdir_abspath = fio.dirname(script_abspath) 30 | log.info(('Creating a directory: %s'):format(scriptdir_abspath)) 31 | fio.mktree(scriptdir_abspath) 32 | 33 | log.info(('Writing a script: %s'):format(script_abspath)) 34 | local fh = fio.open(script_abspath, flags, mode) 35 | fh:write(body) 36 | fh:close() 37 | return script_abspath 38 | end 39 | 40 | -- Generate a script that follows a template and write it at the 41 | -- given path in the given directory. 42 | local function gen_script(storage, dir, script, replacements) 43 | local template = find_template(storage, script) 44 | local replacements = fun.chain({script = script}, replacements):tomap() 45 | local body = template:gsub('<(.-)>', replacements) 46 | treegen.write_script(dir, script, body) 47 | end 48 | 49 | function treegen.init(storage) 50 | storage.tempdirs = {} 51 | storage.templates = {} 52 | end 53 | 54 | -- Remove all temporary directories created by the test 55 | -- unless KEEP_DATA environment variable is set to a 56 | -- non-empty value. 57 | function treegen.clean(storage) 58 | local dirs = table.copy(storage.tempdirs) or {} 59 | storage.tempdirs = nil 60 | 61 | local keep_data = (os.getenv('KEEP_DATA') or '') ~= '' 62 | 63 | for _, dir in ipairs(dirs) do 64 | if keep_data then 65 | log.info(('Left intact due to KEEP_DATA env var: %s'):format(dir)) 66 | else 67 | log.info(('Recursively removing: %s'):format(dir)) 68 | fio.rmtree(dir) 69 | end 70 | end 71 | 72 | storage.templates = nil 73 | end 74 | 75 | function treegen.add_template(storage, pattern, template) 76 | table.insert(storage.templates, { 77 | pattern = pattern, 78 | template = template, 79 | }) 80 | end 81 | 82 | -- Create a temporary directory with given scripts. 83 | -- 84 | -- The scripts are generated using templates added by 85 | -- treegen.add_template(). 86 | -- 87 | -- Example for {'foo/bar.lua', 'baz.lua'}: 88 | -- 89 | -- / 90 | -- + tmp/ 91 | -- + rfbWOJ/ 92 | -- + foo/ 93 | -- | + bar.lua 94 | -- + baz.lua 95 | -- 96 | -- The return value is '/tmp/rfbWOJ' for this example. 97 | function treegen.prepare_directory(storage, scripts, replacements) 98 | local replacements = replacements or {} 99 | 100 | assert(type(scripts) == 'table') 101 | assert(type(replacements) == 'table') 102 | 103 | local dir = fio.tempdir() 104 | 105 | -- fio.tempdir() follows the TMPDIR environment variable. 106 | -- If it ends with a slash, the return value contains a double 107 | -- slash in the middle: for example, if TMPDIR=/tmp/, the 108 | -- result is like `/tmp//rfbWOJ`. 109 | -- 110 | -- It looks harmless on the first glance, but this directory 111 | -- path may be used later to form an URI for a Unix domain 112 | -- socket. As result the URI looks like 113 | -- `unix/:/tmp//rfbWOJ/instance-001.iproto`. 114 | -- 115 | -- It confuses net_box.connect(): it reports EAI_NONAME error 116 | -- from getaddrinfo(). 117 | -- 118 | -- It seems, the reason is a peculiar of the URI parsing: 119 | -- 120 | -- tarantool> uri.parse('unix/:/foo/bar.iproto') 121 | -- --- 122 | -- - host: unix/ 123 | -- service: /foo/bar.iproto 124 | -- unix: /foo/bar.iproto 125 | -- ... 126 | -- 127 | -- tarantool> uri.parse('unix/:/foo//bar.iproto') 128 | -- --- 129 | -- - host: unix 130 | -- path: /foo//bar.iproto 131 | -- ... 132 | -- 133 | -- Let's normalize the path using fio.abspath(), which 134 | -- eliminates the double slashes. 135 | dir = fio.abspath(dir) 136 | 137 | table.insert(storage.tempdirs, dir) 138 | 139 | for _, script in ipairs(scripts) do 140 | gen_script(storage, dir, script, replacements) 141 | end 142 | 143 | return dir 144 | end 145 | 146 | return treegen 147 | -------------------------------------------------------------------------------- /test/unit/not_initialized_test.lua: -------------------------------------------------------------------------------- 1 | local fio = require('fio') 2 | local helpers = require('test.helper') 3 | local t = require('luatest') 4 | local server = require('luatest.server') 5 | 6 | local pgroup = t.group('not-initialized', helpers.backend_matrix({ 7 | {}, 8 | })) 9 | 10 | local vshard_cfg_template = { 11 | sharding = { 12 | storages = { 13 | replicas = { 14 | storage = { 15 | master = true, 16 | }, 17 | }, 18 | }, 19 | }, 20 | bucket_count = 20, 21 | storage_entrypoint = helpers.entrypoint_vshard_storage('srv_not_initialized'), 22 | } 23 | 24 | local cartridge_cfg_template = { 25 | datadir = fio.tempdir(), 26 | server_command = helpers.entrypoint_cartridge('srv_not_initialized'), 27 | use_vshard = true, 28 | replicasets = { 29 | { 30 | uuid = helpers.uuid('a'), 31 | alias = 'router', 32 | roles = { 'vshard-router' }, 33 | servers = { 34 | { instance_uuid = helpers.uuid('a', 1), alias = 'router' }, 35 | }, 36 | }, 37 | { 38 | uuid = helpers.uuid('b'), 39 | alias = 's-1', 40 | roles = { 'customers-storage' }, 41 | servers = { 42 | { instance_uuid = helpers.uuid('b', 1), alias = 's1-master' }, 43 | }, 44 | }, 45 | }, 46 | } 47 | 48 | local tarantool3_cluster_cfg_template = { 49 | groups = { 50 | routers = { 51 | sharding = { 52 | roles = {'router'}, 53 | }, 54 | replicasets = { 55 | ['router'] = { 56 | leader = 'router', 57 | instances = { 58 | ['router'] = {}, 59 | }, 60 | }, 61 | }, 62 | }, 63 | storages = { 64 | sharding = { 65 | roles = {'storage'}, 66 | }, 67 | replicasets = { 68 | ['s-1'] = { 69 | leader = 's1-master', 70 | instances = { 71 | ['s1-master'] = {}, 72 | }, 73 | }, 74 | }, 75 | }, 76 | }, 77 | bucket_count = 20, 78 | storage_entrypoint = helpers.entrypoint_vshard_storage('srv_not_initialized'), 79 | } 80 | 81 | pgroup.before_all(function(g) 82 | helpers.start_cluster(g, 83 | cartridge_cfg_template, 84 | vshard_cfg_template, 85 | tarantool3_cluster_cfg_template, 86 | {wait_crud_is_ready = false, retries = 1} 87 | ) 88 | 89 | g.router = g.cluster:server('router') 90 | end) 91 | 92 | pgroup.after_all(function(g) 93 | helpers.stop_cluster(g.cluster, g.params.backend) 94 | end) 95 | 96 | pgroup.test_insert = function(g) 97 | local results, err = g.router:eval([[ 98 | local crud = require('crud') 99 | return crud.insert('customers', {id = 1, name = 'Fedor', age = 15}) 100 | ]]) 101 | 102 | t.assert_equals(results, nil) 103 | helpers.assert_str_contains_pattern_with_replicaset_id(err.err, "Failed for [replicaset_id]") 104 | t.assert_str_contains(err.err, "crud isn't initialized on replicaset") 105 | end 106 | 107 | pgroup.test_no_box_cfg = function() 108 | t.assert_error_msg_contains('box.cfg() must be called first', function() 109 | require('crud').init_storage() 110 | end) 111 | end 112 | 113 | pgroup.before_test('test_no_vshard_storage_cfg', function(g) 114 | g.test_server = server:new({alias = 'master'}) 115 | g.test_server:start({wait_until_ready = true}) 116 | 117 | local appdir = fio.abspath(debug.sourcedir() .. '/../../') 118 | g.test_server:exec(function(appdir) 119 | if package.setsearchroot ~= nil then 120 | package.setsearchroot(appdir) 121 | else 122 | package.path = package.path .. appdir .. '/?.lua;' 123 | package.path = package.path .. appdir .. '/?/init.lua;' 124 | package.path = package.path .. appdir .. '/.rocks/share/tarantool/?.lua;' 125 | package.path = package.path .. appdir .. '/.rocks/share/tarantool/?/init.lua;' 126 | package.cpath = package.cpath .. appdir .. '/?.so;' 127 | package.cpath = package.cpath .. appdir .. '/?.dylib;' 128 | package.cpath = package.cpath .. appdir .. '/.rocks/lib/tarantool/?.so;' 129 | package.cpath = package.cpath .. appdir .. '/.rocks/lib/tarantool/?.dylib;' 130 | end 131 | end, {appdir}) 132 | end) 133 | 134 | pgroup.test_no_vshard_storage_cfg = function(g) 135 | t.assert_error_msg_contains('vshard.storage.cfg() must be called first', function() 136 | g.test_server:exec(function() 137 | require('crud').init_storage{async = false} 138 | end) 139 | end) 140 | end 141 | 142 | pgroup.after_test('test_no_vshard_storage_cfg', function(g) 143 | g.test_server:stop() 144 | g.test_server = nil 145 | end) 146 | --------------------------------------------------------------------------------