├── .github └── workflows │ ├── fast_testing.yml │ ├── packaging.yml │ ├── publish.yml │ └── reusable_testing.yml ├── .gitignore ├── CHANGELOG.md ├── CMakeLists.txt ├── LICENSE ├── README.md ├── cmake └── FindTarantool.cmake ├── debian ├── .gitignore ├── changelog ├── compat ├── control ├── copyright ├── docs ├── prebuild.sh ├── rules ├── source │ └── format └── tarantool-queue.install ├── queue-scm-1.rockspec ├── queue ├── CMakeLists.txt ├── abstract.lua ├── abstract │ ├── driver │ │ ├── fifo.lua │ │ ├── fifottl.lua │ │ ├── limfifottl.lua │ │ ├── utube.lua │ │ └── utubettl.lua │ ├── queue_session.lua │ ├── queue_state.lua │ └── state.lua ├── compat.lua ├── init.lua ├── util.lua └── version.lua ├── rpm ├── prebuild.sh └── tarantool-queue.spec └── t ├── 000-init.t ├── 001-tube-init.t ├── 010-fifo.t ├── 020-fifottl.t ├── 030-utube.t ├── 040-utubettl.t ├── 050-ttl.t ├── 060-async.t ├── 070-compat.t ├── 080-otc-cb.t ├── 090-grant-check.t ├── 100-limfifottl.t ├── 110-disconnect-trigger-check.t ├── 120-take-task-after-reconnect.t ├── 130-release-all-tasks-on-start.t ├── 140-register-driver-after-cfg.t ├── 150-lazy-start.t ├── 160-validate-space.t ├── 170-register-driver-after-reload.t ├── 180-work-with-uuid.t ├── 190-work-with-ttl-buried-task.t ├── 200-master-replica.t ├── 210-cfg-in-replicaset.t ├── 220-mvcc.t ├── 230-orphan-not-stalling-init.t ├── benchmark ├── async_python.lua ├── async_python.py ├── busy_utubes.lua ├── many_utubes.lua └── multi_consumer_work.lua └── tnt └── init.lua /.github/workflows/fast_testing.yml: -------------------------------------------------------------------------------- 1 | name: fast_testing 2 | 3 | on: 4 | workflow_dispatch: 5 | pull_request: 6 | push: 7 | branches: 8 | - 'master' 9 | tags: 10 | - '*' 11 | 12 | jobs: 13 | run_tests: 14 | runs-on: ubuntu-22.04 15 | 16 | strategy: 17 | fail-fast: false 18 | matrix: 19 | tarantool: 20 | - '1.10' 21 | - '2.11' 22 | - '3.4' 23 | 24 | steps: 25 | - name: Clone the module 26 | uses: actions/checkout@v3 27 | 28 | - name: Setup tarantool ${{ matrix.tarantool }} 29 | uses: tarantool/setup-tarantool@v3 30 | with: 31 | tarantool-version: ${{ matrix.tarantool }} 32 | 33 | - run: cmake . && make check 34 | -------------------------------------------------------------------------------- /.github/workflows/packaging.yml: -------------------------------------------------------------------------------- 1 | name: packaging 2 | 3 | on: [push, pull_request] 4 | 5 | jobs: 6 | # Run not only on tags, otherwise dependent job will skip. 7 | version-check: 8 | # Skip pull request job when the source branch is in the same 9 | # repository. 10 | if: | 11 | github.event_name == 'push' || 12 | github.event.pull_request.head.repo.full_name != github.repository 13 | runs-on: ubuntu-24.04 14 | steps: 15 | - name: Check module version 16 | # We need this step to run only on push with tag. 17 | if: ${{ github.event_name == 'push' && startsWith(github.ref, 'refs/tags/') }} 18 | uses: tarantool/actions/check-module-version@master 19 | with: 20 | module-name: 'queue' 21 | 22 | package: 23 | # Skip pull request jobs when the source branch is in the same 24 | # repository. 25 | if: | 26 | github.event_name == 'push' || 27 | github.event.pull_request.head.repo.full_name != github.repository 28 | # Packaging for CentOS 7 does not work with other versions, see: 29 | # https://github.com/packpack/packpack/issues/145 30 | runs-on: ubuntu-24.04 31 | needs: version-check 32 | 33 | strategy: 34 | fail-fast: false 35 | matrix: 36 | platform: 37 | - { os: 'debian', dist: 'stretch' } 38 | - { os: 'debian', dist: 'buster' } 39 | - { os: 'debian', dist: 'bullseye' } 40 | - { os: 'el', dist: '7' } 41 | - { os: 'el', dist: '8' } 42 | - { os: 'fedora', dist: '30' } 43 | - { os: 'fedora', dist: '31' } 44 | - { os: 'fedora', dist: '32' } 45 | - { os: 'fedora', dist: '33' } 46 | - { os: 'fedora', dist: '34' } 47 | - { os: 'fedora', dist: '35' } 48 | - { os: 'fedora', dist: '36' } 49 | - { os: 'ubuntu', dist: 'xenial' } 50 | - { os: 'ubuntu', dist: 'bionic' } 51 | - { os: 'ubuntu', dist: 'focal' } 52 | - { os: 'ubuntu', dist: 'groovy' } 53 | - { os: 'ubuntu', dist: 'jammy' } 54 | 55 | env: 56 | OS: ${{ matrix.platform.os }} 57 | DIST: ${{ matrix.platform.dist }} 58 | 59 | steps: 60 | - name: Clone the module 61 | uses: actions/checkout@v2 62 | with: 63 | fetch-depth: 0 64 | 65 | - name: Clone the packpack tool 66 | uses: actions/checkout@v2 67 | with: 68 | repository: packpack/packpack 69 | path: packpack 70 | 71 | - name: Fetch tags 72 | # Found that Github checkout Actions pulls all the tags, but 73 | # right it deannotates the testing tag, check: 74 | # https://github.com/actions/checkout/issues/290 75 | # But we use 'git describe ..' calls w/o '--tags' flag and it 76 | # prevents us from getting the needed tag for packages version 77 | # setup. To avoid of it, let's fetch it manually, to be sure 78 | # that all tags will exists always. 79 | run: git fetch --tags -f 80 | 81 | - name: Create packages 82 | run: ./packpack/packpack 83 | 84 | - name: Deploy packages 85 | # We need this step to run only on push with tag. 86 | if: ${{ github.event_name == 'push' && startsWith(github.ref, 'refs/tags/') }} 87 | env: 88 | RWS_URL_PART: https://rws.tarantool.org/tarantool-modules 89 | RWS_AUTH: ${{ secrets.RWS_AUTH }} 90 | PRODUCT_NAME: tarantool-queue 91 | run: | 92 | CURL_CMD="curl -LfsS \ 93 | -X PUT ${RWS_URL_PART}/${OS}/${DIST} \ 94 | -u ${RWS_AUTH} \ 95 | -F product=${PRODUCT_NAME}" 96 | 97 | # We don't want to try to print secrets to the log, but we want 98 | # to print a "curl" command to see what's going on. 99 | CURL_CMD_ECHO="curl -LfsS \ 100 | -X PUT ${RWS_URL_PART}/${OS}/${DIST} \ 101 | -u '***' \ 102 | -F product=${PRODUCT_NAME}" 103 | 104 | for f in $(ls -I '*build*' -I '*.changes' ./build); do 105 | CURL_CMD+=" -F $(basename ${f})=@./build/${f}" 106 | CURL_CMD_ECHO+=" -F $(basename ${f})=@./build/${f}" 107 | done 108 | 109 | echo ${CURL_CMD_ECHO} 110 | 111 | ${CURL_CMD} 112 | -------------------------------------------------------------------------------- /.github/workflows/publish.yml: -------------------------------------------------------------------------------- 1 | name: publish 2 | 3 | on: 4 | push: 5 | branches: [master] 6 | tags: ['*'] 7 | 8 | jobs: 9 | version-check: 10 | # We need this job to run only on push with tag. 11 | if: ${{ github.event_name == 'push' && startsWith(github.ref, 'refs/tags/') }} 12 | runs-on: ubuntu-24.04 13 | steps: 14 | - name: Check module version 15 | uses: tarantool/actions/check-module-version@master 16 | with: 17 | module-name: 'queue' 18 | 19 | publish-rockspec-scm-1: 20 | if: github.ref == 'refs/heads/master' 21 | runs-on: ubuntu-latest 22 | steps: 23 | - uses: actions/checkout@v3 24 | - uses: tarantool/rocks.tarantool.org/github-action@master 25 | with: 26 | auth: ${{ secrets.ROCKS_AUTH }} 27 | files: queue-scm-1.rockspec 28 | 29 | publish-rockspec-tag: 30 | if: startsWith(github.ref, 'refs/tags/') 31 | runs-on: ubuntu-22.04 32 | needs: version-check 33 | steps: 34 | - uses: actions/checkout@v3 35 | 36 | # Create a rockspec for the release. 37 | - run: printf '%s=%s\n' TAG "${GITHUB_REF##*/}" >> "${GITHUB_ENV}" 38 | - run: sed -E 39 | -e "s/branch = '.+'/tag = '${{ env.TAG }}'/g" 40 | -e "s/version = '.+'/version = '${{ env.TAG }}-1'/g" 41 | queue-scm-1.rockspec > queue-${{ env.TAG }}-1.rockspec 42 | 43 | # Create a rock for the release (.all.rock). 44 | # 45 | # `tarantoolctl rocks pack ` creates 46 | # .all.rock tarball. It speeds up 47 | # `tarantoolctl rocks install ` and 48 | # frees it from dependency on git. 49 | # 50 | # Don't confuse this command with 51 | # `tarantoolctl rocks pack `, which creates a 52 | # source tarball (.src.rock). 53 | # 54 | # Important: Don't upload binary rocks to 55 | # rocks.tarantool.org. Lua/C modules should be packed into 56 | # .src.rock instead. See [1] for description of rock types. 57 | # 58 | # [1]: https://github.com/luarocks/luarocks/wiki/Types-of-rocks 59 | - uses: tarantool/setup-tarantool@v3 60 | with: 61 | tarantool-version: '2.10' 62 | - run: tarantoolctl rocks install queue-${{ env.TAG }}-1.rockspec 63 | - run: tarantoolctl rocks pack queue ${{ env.TAG }} 64 | 65 | # Upload .rockspec and .all.rock. 66 | - uses: tarantool/rocks.tarantool.org/github-action@master 67 | with: 68 | auth: ${{ secrets.ROCKS_AUTH }} 69 | files: | 70 | queue-${{ env.TAG }}-1.rockspec 71 | queue-${{ env.TAG }}-1.all.rock 72 | -------------------------------------------------------------------------------- /.github/workflows/reusable_testing.yml: -------------------------------------------------------------------------------- 1 | name: reusable_testing 2 | 3 | on: 4 | workflow_call: 5 | inputs: 6 | artifact_name: 7 | description: 'The name of the tarantool build artifact' 8 | default: ubuntu-focal 9 | required: false 10 | type: string 11 | 12 | jobs: 13 | run_tests: 14 | runs-on: ubuntu-24.04 15 | steps: 16 | - name: 'Clone the queue module' 17 | uses: actions/checkout@v4 18 | with: 19 | repository: ${{ github.repository_owner }}/queue 20 | 21 | - name: 'Download the tarantool build artifact' 22 | uses: actions/download-artifact@v4 23 | with: 24 | name: ${{ inputs.artifact_name }} 25 | 26 | - name: 'Install tarantool' 27 | # Now we're lucky: all dependencies are already installed. Check package 28 | # dependencies when migrating to other OS version. 29 | run: sudo dpkg -i tarantool*.deb 30 | 31 | - run: cmake . && make check 32 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | t/var 2 | VERSION 3 | # cmake 4 | CMakeFiles 5 | cmake_install.cmake 6 | install_manifest.txt 7 | CMakeCache.txt 8 | Makefile 9 | # ctest 10 | Testing 11 | CTestTestfile.cmake 12 | # tarantool garbage 13 | *.snap 14 | *.xlog 15 | *.vylog 16 | # vim 17 | *~ 18 | .*.swp 19 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | 3 | All notable changes to this project will be documented in this file. 4 | 5 | The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) 6 | and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html). 7 | 8 | ## [Unreleased] 9 | 10 | ### Added 11 | 12 | ### Changed 13 | 14 | ### Fixed 15 | 16 | ## [1.4.4] - 2025-05-26 17 | 18 | The patch release fixes incorrect behavior of the utubettl driver with enabled 19 | `STORAGE_MODE_READY_BUFFER` mode. 20 | 21 | ### Fixed 22 | 23 | - Incorrect choice by priority of task with utubettl driver + ready buffer 24 | mode (#244). 25 | - Unable to take task with utubettl driver + ready buffer mode (#244). 26 | 27 | ## [1.4.3] - 2025-03-05 28 | 29 | The release fixes start of a queue on instances with gaps inside 30 | the box.info.replication array. 31 | 32 | ### Fixed 33 | 34 | - Grant method was added for `*_ready_buffer` spaces (#237). 35 | - Attempt to index a nil value if box.info.replication array has gaps. 36 | 37 | ## [1.4.2] - 2024-08-10 38 | 39 | The release re-publish packages. 40 | 41 | ### Fixed 42 | 43 | - Package publishing. 44 | 45 | ## [1.4.1] - 2024-05-21 46 | 47 | The release fixes too long release time by a `ttr` value or a `delay` timeout. 48 | 49 | ### Fixed 50 | 51 | - Too long timings in the `utubettl` driver (#223). 52 | 53 | ## [1.4.0] - 2024-05-20 54 | 55 | The release introduces an experimental `storage_mode` option for the `utube` 56 | and `utubettl` drivers with the `memtx` engine. It could be used to create a 57 | `utube` or `utubettl` queue with an additional buffer space that stores only 58 | ready to take tasks. 59 | 60 | ```Lua 61 | local tube = queue.create_tube('utube_with_ready_buffer', 'utube', 62 | {storage_mode = queue.driver.utube.STORAGE_MODE_READY_BUFFER}) 63 | ``` 64 | ```Lua 65 | local tube = queue.create_tube('utubettl_with_ready_buffer', 'utubettl', 66 | {storage_mode = queue.driver.utubettl.STORAGE_MODE_READY_BUFFER}) 67 | ``` 68 | 69 | The storage mode slower in general cases, but a much faster in cases when 70 | you have utubes with many tasks (see README.md for the performance comparison). 71 | So you should make your choice carefully. 72 | 73 | ### Added 74 | - Experimental `storage_mode` option for creating a `utube` and `utubettl` 75 | tube (#228). 76 | It enables the workaround for slow takes while working with busy tubes. 77 | 78 | ### Fixed 79 | 80 | - Stuck in `INIT` state if an instance failed to enter the `running` mode 81 | in time (#226). This fix works only for Tarantool versions >= 2.10.0. 82 | - Slow takes on busy `utube` and `utubettl` tubes (#228). The workaround could 83 | be enabled by passing the `storage_mode = "ready_buffer"` option while 84 | creating the tube. 85 | 86 | ## [1.3.3] - 2023-09-13 87 | 88 | ### Fixed 89 | 90 | - In replicaset mode, the behavior of the public API is reduced to the same behavior 91 | in all queue states, including INIT. Previously, in the INIT state, an ambiguous 92 | error was thrown when trying to access a public method on a replica and the script 93 | was interrupted by an error. 94 | 95 | Old behavior (call `create_tube` on replica, queue is in INIT state): 96 | ``` 97 | 2023-09-04 14:01:11.000 [5990] main/103/replica.lua/box.load_cfg I> set 'read_only' configuration option to true 98 | stack traceback: 99 | /home/void/tmp/cluster/repl/queue/init.lua:44: in function '__index' 100 | replica.lua:13: in main chunk 101 | 2023-09-04 14:01:11.004 [5990] main/105/checkpoint_daemon I> scheduled next checkpoint for Mon Sep 4 15:11:32 2023 102 | 2023-09-04 14:01:11.004 [5990] main utils.c:610 E> LuajitError: /home/void/tmp/cluster/repl/queue/init.lua:45: Please configure box.cfg{} in read/write mode first 103 | ``` 104 | After this fix: 105 | ``` 106 | 2023-09-11 10:24:31.463 [19773] main/103/replica.lua abstract.lua:93 E> create_tube: queue is in INIT state 107 | ``` 108 | 109 | ## [1.3.2] - 2023-08-24 110 | 111 | ### Fixed 112 | 113 | - Duplicate id error with mvvc on put and take (#207). 114 | 115 | ## [1.3.1] - 2023-07-31 116 | 117 | ### Fixed 118 | 119 | - Yield in the fifottl/utubettl queue drivers. 120 | 121 | ## [1.3.0] - 2023-03-13 122 | 123 | ### Added 124 | 125 | - Possibility to get the module version. 126 | 127 | ### Fixed 128 | 129 | - Bug when working with the replicaset (#202). 130 | 131 | ## [1.2.5] - 2023-02-28 132 | 133 | This is a technical release that should fix several problems in the 134 | repositories with packages that were caused by the mistaken creation 135 | of several tags (for this reason there are no 1.2.3 and 1.2.4 tags 136 | and corresponding packages in the repositories). 137 | 138 | ## [1.2.2] - 2022-11-03 139 | 140 | ### Fixed 141 | 142 | - Excessive CPU consumption of the state fiber. 143 | 144 | ## [1.2.1] - 2022-09-12 145 | 146 | ### Added 147 | 148 | - Rockspec publishing to CD. 149 | 150 | ### Fixed 151 | 152 | - "replication" mode switching on tarantool < 2.2.1 153 | 154 | ## [1.2.0] - 2022-06-06 155 | 156 | ### Added 157 | 158 | - Master-replica switching support. 159 | Added the ability to use a queue in the master replica scheme. 160 | More information is available: 161 | https://github.com/tarantool/queue#queue-state-diagram 162 | https://github.com/tarantool/queue#queue-and-replication 163 | - Granting privilege to `statistics`, `put` and `truncate`. 164 | 165 | ### Fixed 166 | 167 | - Work with "ttl" of buried task. 168 | Previously, if a task was "buried" after it was "taken" (and the task has a 169 | "ttr"), then the time when the "release" should occur (the "ttr" timer) will 170 | be interpreted as the end of the "ttl" timer and the task will be deleted. 171 | 172 | ## [1.1.0] - 2020-12-25 173 | 174 | ### Added 175 | 176 | - "Shared sessions" was added to the queue. Previously, a connection to server 177 | was synonym of the queue session. Now the session has a unique UUID (returned 178 | by the "queue.identify()" method), and one session can have many connections. 179 | To connect to an existing session, call "queue.identify(uuid)" with the 180 | previously obtained UUID. 181 | - Possibility to work with tasks after reconnect. The queue module now provides 182 | the ability to set the `ttr` setting for sessions by 183 | `queue.cfg({ ttr = ttr_in_sec})`, which characterizes how long the logical 184 | session will exist after all active connections are closed. 185 | 186 | ### Fixed 187 | 188 | - Custom driver registration after reboot. Previously, if a custom driver is 189 | registered after calling box.cfg() it causes a problem when the instance will 190 | be restarted. 191 | 192 | ## [1.0.8] - 2020-10-17 193 | 194 | ### Added 195 | 196 | - The ability to start an instance with a loaded queue module in read-only 197 | mode. In this case, a start of the module will be delayed until the instance 198 | will be configured with read_only = false. Previously, when trying to 199 | initialize the queue module on an instance configured in ro mode, an error 200 | occurred: "ER_READONLY: Can't modify data because this instance is in 201 | read-only mode." See https://github.com/tarantool/queue#initialization for 202 | more details. 203 | 204 | ## [1.0.7] - 2020-09-03 205 | 206 | ### Breaking changes 207 | 208 | - A check for a driver API implementation was added. Now, the consumer will be 209 | informed about the missing methods in the driver implementation (an error will 210 | be thrown). 211 | 212 | ### Added 213 | 214 | - Notification about missing methods in the driver implementation (#126). 215 | 216 | ### Fixed 217 | 218 | - The tasks releases on start for some drivers (utubettl, fifottl, 219 | limfifottl). Before, an attempt to release "taken" tasks on start lead to an 220 | error: "attempt to index local 'opts' (a nil value)" (#121). 221 | 222 | ### Changed 223 | 224 | - Updated the requirements for the "delete" driver method. Now, the method 225 | should change the state of a task to "done". Before, it was duplicated by 226 | external code. 227 | 228 | ## [1.0.6] - 2020-02-29 229 | 230 | ### Breaking changes 231 | 232 | External drivers should be updated with the new `tasks_by_state()` method. 233 | Consider the example from the `fifo` driver: 234 | 235 | ```lua 236 | -- get iterator to tasks in a certain state 237 | function method.tasks_by_state(self, task_state) 238 | return self.space.index.status:pairs(task_state) 239 | end 240 | ``` 241 | 242 | The example uses 'status' secondary index, which is built on top of 'status' 243 | and 'task_id' fields. 244 | 245 | This new method is necessary to correctly manage state of tasks: when tarantool 246 | instance is restarted we should release all taken tasks (otherwise they would 247 | stuck in the taken state forever). See #66 and #126 for more information. 248 | 249 | ### Fixed 250 | 251 | - Releasing tasks at a client disconnection (#103). 252 | - Optimize statistics build (-15% in some cases) (#92). 253 | - Release all taken tasks at start (#66). 254 | - Don't take tasks during a client disconnection (#104). 255 | -------------------------------------------------------------------------------- /CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 2.8 FATAL_ERROR) 2 | 3 | project(queue) 4 | 5 | if(NOT CMAKE_BUILD_TYPE) 6 | set(CMAKE_BUILD_TYPE RelWithDebInfo) 7 | endif() 8 | set(CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake" ${CMAKE_MODULE_PATH}) 9 | 10 | set(TARANTOOL_FIND_REQUIRED ON) 11 | find_package(Tarantool) 12 | 13 | add_subdirectory(queue) 14 | 15 | enable_testing() 16 | 17 | set (LUA_PATH "LUA_PATH=${PROJECT_SOURCE_DIR}/?.lua\\;${PROJECT_SOURCE_DIR}/?/init.lua\\;") 18 | 19 | file(GLOB TEST_LIST ${CMAKE_SOURCE_DIR}/t/*.t) 20 | 21 | foreach(TEST ${TEST_LIST}) 22 | get_filename_component(TEST_NAME ${TEST} NAME) 23 | add_test (memtx-${TEST_NAME} tarantool ${TEST}) 24 | set_tests_properties(memtx-${TEST_NAME} PROPERTIES ENVIRONMENT "ENGINE=memtx;${LUA_PATH}") 25 | add_test (vinyl-${TEST_NAME} tarantool ${TEST}) 26 | set_tests_properties(vinyl-${TEST_NAME} PROPERTIES ENVIRONMENT "ENGINE=vinyl;${LUA_PATH}") 27 | endforeach() 28 | 29 | add_custom_target(check 30 | WORKING_DIRECTORY ${PROJECT_BUILD_DIR} 31 | COMMAND ctest -V) 32 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (C) 2014-2016 Tarantool AUTHORS: 2 | please see AUTHORS file in tarantool/tarantool repository. 3 | 4 | /* 5 | * Redistribution and use in source and binary forms, with or 6 | * without modification, are permitted provided that the following 7 | * conditions are met: 8 | * 9 | * 1. Redistributions of source code must retain the above 10 | * copyright notice, this list of conditions and the 11 | * following disclaimer. 12 | * 13 | * 2. Redistributions in binary form must reproduce the above 14 | * copyright notice, this list of conditions and the following 15 | * disclaimer in the documentation and/or other materials 16 | * provided with the distribution. 17 | * 18 | * THIS SOFTWARE IS PROVIDED BY AUTHORS ``AS IS'' AND 19 | * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 20 | * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 21 | * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL 22 | * AUTHORS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, 23 | * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 24 | * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 | * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR 26 | * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 27 | * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 28 | * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF 29 | * THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 30 | * SUCH DAMAGE. 31 | */ 32 | -------------------------------------------------------------------------------- /cmake/FindTarantool.cmake: -------------------------------------------------------------------------------- 1 | # Define GNU standard installation directories 2 | include(GNUInstallDirs) 3 | 4 | macro(extract_definition name output input) 5 | string(REGEX MATCH "#define[\t ]+${name}[\t ]+\"([^\"]*)\"" 6 | _t "${input}") 7 | string(REGEX REPLACE "#define[\t ]+${name}[\t ]+\"(.*)\"" "\\1" 8 | ${output} "${_t}") 9 | endmacro() 10 | 11 | find_path(TARANTOOL_INCLUDE_DIR tarantool/module.h 12 | HINTS ENV TARANTOOL_DIR 13 | ) 14 | 15 | if(TARANTOOL_INCLUDE_DIR) 16 | set(_config "-") 17 | file(READ "${TARANTOOL_INCLUDE_DIR}/tarantool/module.h" _config0) 18 | string(REPLACE "\\" "\\\\" _config ${_config0}) 19 | unset(_config0) 20 | extract_definition(PACKAGE_VERSION TARANTOOL_VERSION ${_config}) 21 | extract_definition(INSTALL_PREFIX _install_prefix ${_config}) 22 | unset(_config) 23 | endif() 24 | 25 | include(FindPackageHandleStandardArgs) 26 | find_package_handle_standard_args(TARANTOOL 27 | REQUIRED_VARS TARANTOOL_INCLUDE_DIR VERSION_VAR TARANTOOL_VERSION) 28 | if(TARANTOOL_FOUND) 29 | set(TARANTOOL_INSTALL_LIBDIR "${CMAKE_INSTALL_LIBDIR}/tarantool") 30 | set(TARANTOOL_INSTALL_LUADIR "${CMAKE_INSTALL_DATADIR}/tarantool") 31 | set(TARANTOOL_INCLUDE_DIRS "${TARANTOOL_INCLUDE_DIR}" 32 | "${TARANTOOL_INCLUDE_DIR}/tarantool/") 33 | 34 | if (NOT "${CMAKE_INSTALL_PREFIX}" STREQUAL "/usr/local" AND 35 | NOT "${CMAKE_INSTALL_PREFIX}" STREQUAL "${_install_prefix}") 36 | message(WARNING "Provided CMAKE_INSTALL_PREFIX is different from " 37 | "CMAKE_INSTALL_PREFIX of Tarantool. You might need to set " 38 | "corrent package.path/package.cpath to load this module or " 39 | "change your build prefix:" 40 | "\n" 41 | "cmake . -DCMAKE_INSTALL_PREFIX=${_install_prefix}" 42 | "\n" 43 | ) 44 | endif () 45 | if (NOT TARANTOOL_FIND_QUIETLY AND NOT FIND_TARANTOOL_DETAILS) 46 | set(FIND_TARANTOOL_DETAILS ON CACHE INTERNAL "Details about TARANTOOL") 47 | message(STATUS "Tarantool LUADIR is ${TARANTOOL_INSTALL_LUADIR}") 48 | message(STATUS "Tarantool LIBDIR is ${TARANTOOL_INSTALL_LIBDIR}") 49 | endif () 50 | endif() 51 | mark_as_advanced(TARANTOOL_INCLUDE_DIRS TARANTOOL_INSTALL_LIBDIR 52 | TARANTOOL_INSTALL_LUADIR) 53 | -------------------------------------------------------------------------------- /debian/.gitignore: -------------------------------------------------------------------------------- 1 | tarantool-queue/ 2 | files 3 | stamp-* 4 | *.substvars 5 | *.log 6 | -------------------------------------------------------------------------------- /debian/changelog: -------------------------------------------------------------------------------- 1 | tarantool-queue (1.0.0) unstable; urgency=medium 2 | 3 | * Initial release 4 | 5 | -- Roman Tsisyk Thu, 18 Feb 2016 19:58:35 +0300 6 | -------------------------------------------------------------------------------- /debian/compat: -------------------------------------------------------------------------------- 1 | 9 2 | -------------------------------------------------------------------------------- /debian/control: -------------------------------------------------------------------------------- 1 | Source: tarantool-queue 2 | Priority: optional 3 | Section: database 4 | Maintainer: Roman Tsisyk 5 | Build-Depends: debhelper (>= 9), 6 | tarantool-dev (>= 1.7), 7 | # For /usr/bin/prove 8 | perl (>= 5.10.0) 9 | Standards-Version: 3.9.6 10 | Homepage: https://github.com/tarantool/queue 11 | Vcs-Git: git://github.com/tarantool/queue.git 12 | Vcs-Browser: https://github.com/tarantool/queue 13 | 14 | Package: tarantool-queue 15 | Architecture: all 16 | Depends: tarantool (>= 1.7), ${misc:Depends} 17 | Description: Persistent in-memory queues for Tarantool 18 | A collection of persistent queue implementations for Tarantool. 19 | -------------------------------------------------------------------------------- /debian/copyright: -------------------------------------------------------------------------------- 1 | Format: http://www.debian.org/doc/packaging-manuals/copyright-format/1.0/ 2 | Debianized-By: Roman Tsisyk 3 | Upstream-Name: tarantool-queue 4 | Upstream-Contact: support@tarantool.org 5 | Source: https://github.com/tarantool/queue 6 | 7 | Files: * 8 | Copyright: 2014-2016 Tarantool AUTHORS 9 | License: BSD-2-Clause 10 | Redistribution and use in source and binary forms, with or without 11 | modification, are permitted provided that the following conditions 12 | are met: 13 | 1. Redistributions of source code must retain the above copyright 14 | notice, this list of conditions and the following disclaimer. 15 | 2. Redistributions in binary form must reproduce the above copyright 16 | notice, this list of conditions and the following disclaimer in the 17 | documentation and/or other materials provided with the distribution. 18 | . 19 | THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 20 | ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 21 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 22 | ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 23 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 24 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 25 | OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 26 | HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 27 | LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 28 | OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 29 | SUCH DAMAGE. 30 | -------------------------------------------------------------------------------- /debian/docs: -------------------------------------------------------------------------------- 1 | README.md 2 | -------------------------------------------------------------------------------- /debian/prebuild.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -exu # Strict shell (w/o -o pipefail) 4 | 5 | # At the time of adding the changes, tarantool 1.10 is absent in the 6 | # repositories Ubuntu impish and jammy. 7 | if [[ $DIST == "impish" ]] || [[ $DIST == "jammy" ]]; then 8 | curl -LsSf https://www.tarantool.io/release/2/installer.sh | sudo bash 9 | else 10 | curl -LsSf https://www.tarantool.io/release/1.10/installer.sh | sudo bash 11 | fi 12 | -------------------------------------------------------------------------------- /debian/rules: -------------------------------------------------------------------------------- 1 | #!/usr/bin/make -f 2 | 3 | %: 4 | dh $@ 5 | -------------------------------------------------------------------------------- /debian/source/format: -------------------------------------------------------------------------------- 1 | 3.0 (quilt) 2 | -------------------------------------------------------------------------------- /debian/tarantool-queue.install: -------------------------------------------------------------------------------- 1 | queue/*.lua usr/share/tarantool/queue/ 2 | queue/abstract/*.lua usr/share/tarantool/queue/abstract/ 3 | queue/abstract/driver/*.lua usr/share/tarantool/queue/abstract/driver/ 4 | -------------------------------------------------------------------------------- /queue-scm-1.rockspec: -------------------------------------------------------------------------------- 1 | package = 'queue' 2 | version = 'scm-1' 3 | source = { 4 | url = 'git+https://github.com/tarantool/queue.git', 5 | branch = 'master', 6 | } 7 | description = { 8 | summary = "A set of persistent in-memory queues", 9 | homepage = 'https://github.com/tarantool/queue.git', 10 | license = 'BSD', 11 | } 12 | dependencies = { 13 | 'lua >= 5.1' 14 | } 15 | build = { 16 | type = 'builtin', 17 | 18 | modules = { 19 | ['queue.abstract'] = 'queue/abstract.lua', 20 | ['queue.abstract.state'] = 'queue/abstract/state.lua', 21 | ['queue.abstract.queue_session'] = 'queue/abstract/queue_session.lua', 22 | ['queue.abstract.queue_state'] = 'queue/abstract/queue_state.lua', 23 | ['queue.abstract.driver.fifottl'] = 'queue/abstract/driver/fifottl.lua', 24 | ['queue.abstract.driver.utubettl'] = 'queue/abstract/driver/utubettl.lua', 25 | ['queue.abstract.driver.fifo'] = 'queue/abstract/driver/fifo.lua', 26 | ['queue.abstract.driver.utube'] = 'queue/abstract/driver/utube.lua', 27 | ['queue.abstract.driver.limfifottl'] = 'queue/abstract/driver/limfifottl.lua', 28 | ['queue.compat'] = 'queue/compat.lua', 29 | ['queue.util'] = 'queue/util.lua', 30 | ['queue'] = 'queue/init.lua', 31 | ['queue.version'] = 'queue/version.lua' 32 | } 33 | } 34 | 35 | -- vim: syntax=lua 36 | -------------------------------------------------------------------------------- /queue/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | install(FILES ${CMAKE_CURRENT_SOURCE_DIR}/init.lua 2 | DESTINATION ${TARANTOOL_INSTALL_LUADIR}/${PROJECT_NAME}/) 3 | install(FILES ${CMAKE_CURRENT_SOURCE_DIR}/compat.lua 4 | DESTINATION ${TARANTOOL_INSTALL_LUADIR}/${PROJECT_NAME}/) 5 | install(FILES ${CMAKE_CURRENT_SOURCE_DIR}/util.lua 6 | DESTINATION ${TARANTOOL_INSTALL_LUADIR}/${PROJECT_NAME}/) 7 | install(FILES ${CMAKE_CURRENT_SOURCE_DIR}/abstract.lua 8 | DESTINATION ${TARANTOOL_INSTALL_LUADIR}/${PROJECT_NAME}/) 9 | install(FILES ${CMAKE_CURRENT_SOURCE_DIR}/version.lua 10 | DESTINATION ${TARANTOOL_INSTALL_LUADIR}/${PROJECT_NAME}/) 11 | install(FILES ${CMAKE_CURRENT_SOURCE_DIR}/abstract/state.lua 12 | DESTINATION ${TARANTOOL_INSTALL_LUADIR}/${PROJECT_NAME}/abstract) 13 | install(FILES ${CMAKE_CURRENT_SOURCE_DIR}/abstract/queue_session.lua 14 | DESTINATION ${TARANTOOL_INSTALL_LUADIR}/${PROJECT_NAME}/abstract) 15 | install(FILES ${CMAKE_CURRENT_SOURCE_DIR}/abstract/queue_state.lua 16 | DESTINATION ${TARANTOOL_INSTALL_LUADIR}/${PROJECT_NAME}/abstract) 17 | install(FILES ${CMAKE_CURRENT_SOURCE_DIR}/abstract/driver/fifo.lua 18 | DESTINATION ${TARANTOOL_INSTALL_LUADIR}/${PROJECT_NAME}/abstract/driver/) 19 | install(FILES ${CMAKE_CURRENT_SOURCE_DIR}/abstract/driver/utube.lua 20 | DESTINATION ${TARANTOOL_INSTALL_LUADIR}/${PROJECT_NAME}/abstract/driver/) 21 | install(FILES ${CMAKE_CURRENT_SOURCE_DIR}/abstract/driver/fifottl.lua 22 | DESTINATION ${TARANTOOL_INSTALL_LUADIR}/${PROJECT_NAME}/abstract/driver/) 23 | install(FILES ${CMAKE_CURRENT_SOURCE_DIR}/abstract/driver/utubettl.lua 24 | DESTINATION ${TARANTOOL_INSTALL_LUADIR}/${PROJECT_NAME}/abstract/driver/) 25 | install(FILES ${CMAKE_CURRENT_SOURCE_DIR}/abstract/driver/limfifottl.lua 26 | DESTINATION ${TARANTOOL_INSTALL_LUADIR}/${PROJECT_NAME}/abstract/driver/) 27 | -------------------------------------------------------------------------------- /queue/abstract/driver/fifo.lua: -------------------------------------------------------------------------------- 1 | local state = require('queue.abstract.state') 2 | 3 | local num_type = require('queue.compat').num_type 4 | local str_type = require('queue.compat').str_type 5 | 6 | local tube = {} 7 | local method = {} 8 | 9 | -- validate space of queue 10 | local function validate_space(space) 11 | -- check indexes 12 | local indexes = {'task_id', 'status'} 13 | for _, index in pairs(indexes) do 14 | if space.index[index] == nil then 15 | error(string.format('space "%s" does not have "%s" index', 16 | space.name, index)) 17 | end 18 | end 19 | end 20 | 21 | -- create space 22 | function tube.create_space(space_name, opts) 23 | local space_opts = {} 24 | local if_not_exists = opts.if_not_exists or false 25 | space_opts.temporary = opts.temporary or false 26 | space_opts.engine = opts.engine or 'memtx' 27 | space_opts.format = { 28 | {name = 'task_id', type = num_type()}, 29 | {name = 'status', type = str_type()}, 30 | {name = 'data', type = '*'} 31 | } 32 | 33 | local space = box.space[space_name] 34 | if if_not_exists and space then 35 | -- Validate the existing space. 36 | validate_space(box.space[space_name]) 37 | return space 38 | end 39 | 40 | space = box.schema.create_space(space_name, space_opts) 41 | space:create_index('task_id', { 42 | type = 'tree', 43 | parts = {1, num_type()} 44 | }) 45 | space:create_index('status', { 46 | type = 'tree', 47 | parts = {2, str_type(), 1, num_type()} 48 | }) 49 | return space 50 | end 51 | 52 | -- start tube on space 53 | function tube.new(space, on_task_change) 54 | validate_space(space) 55 | 56 | on_task_change = on_task_change or (function() end) 57 | local self = setmetatable({ 58 | space = space, 59 | on_task_change = on_task_change, 60 | }, { __index = method }) 61 | return self 62 | end 63 | 64 | -- method.grant grants provided user to all spaces of driver. 65 | function method.grant(self, user, opts) 66 | box.schema.user.grant(user, 'read,write', 'space', self.space.name, opts) 67 | end 68 | 69 | -- normalize task: cleanup all internal fields 70 | function method.normalize_task(self, task) 71 | return task 72 | end 73 | 74 | -- put task in space 75 | function method.put(self, data, opts) 76 | local max 77 | 78 | -- Taking the maximum of the index is an implicit transactions, so it is 79 | -- always done with 'read-confirmed' mvcc isolation level. 80 | -- It can lead to errors when trying to make parallel 'put' calls with mvcc enabled. 81 | -- It is hapenning because 'max' for several puts in parallel will be the same since 82 | -- read confirmed isolation level makes visible all transactions that finished the commit. 83 | -- To fix it we wrap it with box.begin/commit and set right isolation level. 84 | -- Current fix does not resolve that bug in situations when we already are in transaction 85 | -- since it will open nested transactions. 86 | -- See https://github.com/tarantool/queue/issues/207 87 | -- See https://www.tarantool.io/ru/doc/latest/concepts/atomic/txn_mode_mvcc/ 88 | 89 | if box.cfg.memtx_use_mvcc_engine and (not box.is_in_txn()) then 90 | box.begin({txn_isolation = 'read-committed'}) 91 | max = self.space.index.task_id:max() 92 | box.commit() 93 | else 94 | max = self.space.index.task_id:max() 95 | end 96 | 97 | local id = max and max[1] + 1 or 0 98 | local task = self.space:insert{id, state.READY, data} 99 | self.on_task_change(task, 'put') 100 | return task 101 | end 102 | 103 | -- take task 104 | function method.take(self) 105 | local task 106 | -- Taking the minimum is an implicit transactions, so it is 107 | -- always done with 'read-confirmed' mvcc isolation level. 108 | -- It can lead to errors when trying to make parallel 'take' calls with mvcc enabled. 109 | -- It is hapenning because 'min' for several takes in parallel will be the same since 110 | -- read confirmed isolation level makes visible all transactions that finished the commit. 111 | -- To fix it we wrap it with box.begin/commit and set right isolation level. 112 | -- Current fix does not resolve that bug in situations when we already are in transaction 113 | -- since it will open nested transactions. 114 | -- See https://github.com/tarantool/queue/issues/207 115 | -- See https://www.tarantool.io/ru/doc/latest/concepts/atomic/txn_mode_mvcc/ 116 | if box.cfg.memtx_use_mvcc_engine and (not box.is_in_txn()) then 117 | box.begin({txn_isolation = 'read-committed'}) 118 | task = self.space.index.status:min{state.READY} 119 | box.commit() 120 | else 121 | task = self.space.index.status:min{state.READY} 122 | end 123 | 124 | if task ~= nil and task[2] == state.READY then 125 | task = self.space:update(task[1], { { '=', 2, state.TAKEN } }) 126 | self.on_task_change(task, 'take') 127 | return task 128 | end 129 | end 130 | 131 | -- touch task 132 | function method.touch(self, id, ttr) 133 | error('fifo queue does not support touch') 134 | end 135 | 136 | -- delete task 137 | function method.delete(self, id) 138 | local task = self.space:get(id) 139 | self.space:delete(id) 140 | if task ~= nil then 141 | task = task:transform(2, 1, state.DONE) 142 | self.on_task_change(task, 'delete') 143 | end 144 | return task 145 | end 146 | 147 | -- release task 148 | function method.release(self, id, opts) 149 | local task = self.space:update(id, {{ '=', 2, state.READY }}) 150 | if task ~= nil then 151 | self.on_task_change(task, 'release') 152 | end 153 | return task 154 | end 155 | 156 | -- bury task 157 | function method.bury(self, id) 158 | local task = self.space:update(id, {{ '=', 2, state.BURIED }}) 159 | self.on_task_change(task, 'bury') 160 | return task 161 | end 162 | 163 | -- unbury several tasks 164 | function method.kick(self, count) 165 | for i = 1, count do 166 | local task = self.space.index.status:min{ state.BURIED } 167 | if task == nil then 168 | return i - 1 169 | end 170 | if task[2] ~= state.BURIED then 171 | return i - 1 172 | end 173 | 174 | task = self.space:update(task[1], {{ '=', 2, state.READY }}) 175 | self.on_task_change(task, 'kick') 176 | end 177 | return count 178 | end 179 | 180 | -- peek task 181 | function method.peek(self, id) 182 | return self.space:get{id} 183 | end 184 | 185 | -- get iterator to tasks in a certain state 186 | function method.tasks_by_state(self, task_state) 187 | return self.space.index.status:pairs(task_state) 188 | end 189 | 190 | function method.truncate(self) 191 | self.space:truncate() 192 | end 193 | 194 | -- This driver has no background activity. 195 | -- Implement dummy methods for the API requirement. 196 | function method.start() 197 | return 198 | end 199 | 200 | function method.stop() 201 | return 202 | end 203 | 204 | return tube 205 | -------------------------------------------------------------------------------- /queue/abstract/driver/fifottl.lua: -------------------------------------------------------------------------------- 1 | local log = require('log') 2 | local fiber = require('fiber') 3 | local state = require('queue.abstract.state') 4 | 5 | local util = require('queue.util') 6 | local qc = require('queue.compat') 7 | local num_type = qc.num_type 8 | local str_type = qc.str_type 9 | 10 | local tube = {} 11 | local method = {} 12 | 13 | local i_id = 1 14 | local i_status = 2 15 | local i_next_event = 3 16 | local i_ttl = 4 17 | local i_ttr = 5 18 | local i_pri = 6 19 | local i_created = 7 20 | local i_data = 8 21 | 22 | local function is_expired(task) 23 | local dead_event = task[i_created] + task[i_ttl] 24 | return (dead_event <= fiber.time64()) 25 | end 26 | 27 | -- validate space of queue 28 | local function validate_space(space) 29 | -- check indexes 30 | local indexes = {'task_id', 'status', 'watch'} 31 | for _, index in pairs(indexes) do 32 | if space.index[index] == nil then 33 | error(string.format('space "%s" does not have "%s" index', 34 | space.name, index)) 35 | end 36 | end 37 | end 38 | 39 | -- create space 40 | function tube.create_space(space_name, opts) 41 | opts.ttl = opts.ttl or util.MAX_TIMEOUT 42 | opts.ttr = opts.ttr or opts.ttl 43 | opts.pri = opts.pri or 0 44 | 45 | local space_opts = {} 46 | local if_not_exists = opts.if_not_exists or false 47 | space_opts.temporary = opts.temporary or false 48 | space_opts.engine = opts.engine or 'memtx' 49 | space_opts.format = { 50 | {name = 'task_id', type = num_type()}, 51 | {name = 'status', type = str_type()}, 52 | {name = 'next_event', type = num_type()}, 53 | {name = 'ttl', type = num_type()}, 54 | {name = 'ttr', type = num_type()}, 55 | {name = 'pri', type = num_type()}, 56 | {name = 'created', type = num_type()}, 57 | {name = 'data', type = '*'} 58 | } 59 | 60 | -- 1 2 3 4 5 6 7, 8 61 | -- task_id, status, next_event, ttl, ttr, pri, created, data 62 | local space = box.space[space_name] 63 | if if_not_exists and space then 64 | -- Validate the existing space. 65 | validate_space(box.space[space_name]) 66 | return space 67 | end 68 | 69 | space = box.schema.create_space(space_name, space_opts) 70 | space:create_index('task_id', { 71 | type = 'tree', 72 | parts = {i_id, num_type()} 73 | }) 74 | space:create_index('status', { 75 | type = 'tree', 76 | parts = {i_status, str_type(), i_pri, num_type(), i_id, num_type()} 77 | }) 78 | space:create_index('watch', { 79 | type = 'tree', 80 | parts = {i_status, str_type(), i_next_event, num_type()}, 81 | unique = false 82 | }) 83 | return space 84 | end 85 | 86 | local delayed_state = { state.DELAYED } 87 | local ttl_states = { state.READY, state.BURIED } 88 | local ttr_state = { state.TAKEN } 89 | 90 | local function fifottl_fiber_iteration(self, processed) 91 | local now = util.time() 92 | local task = nil 93 | local estimated = util.MAX_TIMEOUT 94 | 95 | -- delayed tasks 96 | task = self.space.index.watch:min(delayed_state) 97 | if task and task[i_status] == state.DELAYED then 98 | if now >= task[i_next_event] then 99 | task = self.space:update(task[i_id], { 100 | { '=', i_status, state.READY }, 101 | { '=', i_next_event, task[i_created] + task[i_ttl] } 102 | }) 103 | self:on_task_change(task, 'delay') 104 | estimated = 0 105 | processed = processed + 1 106 | else 107 | estimated = tonumber(task[i_next_event] - now) / 1000000 108 | end 109 | end 110 | 111 | -- ttl tasks 112 | for _, state in pairs(ttl_states) do 113 | task = self.space.index.watch:min{ state } 114 | if task ~= nil and task[i_status] == state then 115 | if now >= task[i_next_event] then 116 | task = self:delete(task[i_id]):transform(2, 1, state.DONE) 117 | self:on_task_change(task, 'ttl') 118 | estimated = 0 119 | processed = processed + 1 120 | else 121 | local et = tonumber(task[i_next_event] - now) / 1000000 122 | estimated = et < estimated and et or estimated 123 | end 124 | end 125 | end 126 | 127 | -- ttr tasks 128 | task = self.space.index.watch:min(ttr_state) 129 | if task and task[i_status] == state.TAKEN then 130 | if now >= task[i_next_event] then 131 | task = self.space:update(task[i_id], { 132 | { '=', i_status, state.READY }, 133 | { '=', i_next_event, task[i_created] + task[i_ttl] } 134 | }) 135 | self:on_task_change(task, 'ttr') 136 | estimated = 0 137 | processed = processed + 1 138 | else 139 | local et = tonumber(task[i_next_event] - now) / 1000000 140 | estimated = et < estimated and et or estimated 141 | end 142 | end 143 | 144 | if estimated > 0 or processed > 1000 then 145 | -- free refcounter 146 | estimated = estimated > 0 and estimated or 0 147 | processed = 0 148 | self.cond:wait(estimated) 149 | end 150 | 151 | return processed 152 | end 153 | 154 | -- watch fiber 155 | local function fifottl_fiber(self) 156 | fiber.name('fifottl') 157 | log.info("Started queue fifottl fiber") 158 | local processed = 0 159 | 160 | while true do 161 | if box.info.ro == false then 162 | local stat, err = pcall(fifottl_fiber_iteration, self, processed) 163 | 164 | if not stat and not (err.code == box.error.READONLY) then 165 | log.error("error catched: %s", tostring(err)) 166 | log.error("exiting fiber '%s'", fiber.name()) 167 | return 1 168 | elseif stat then 169 | processed = err 170 | end 171 | else 172 | -- When switching the master to the replica, the fiber will be stopped. 173 | if self.sync_chan:get(0.1) ~= nil then 174 | log.info("Queue fifottl fiber was stopped") 175 | break 176 | end 177 | end 178 | end 179 | end 180 | 181 | -- start tube on space 182 | function tube.new(space, on_task_change, opts) 183 | validate_space(space) 184 | 185 | on_task_change = on_task_change or (function() end) 186 | local self = setmetatable({ 187 | space = space, 188 | on_task_change = function(self, task, stats_data) 189 | -- wakeup fiber 190 | if task ~= nil and self.fiber ~= nil then 191 | self.cond:signal(self.fiber:id()) 192 | end 193 | on_task_change(task, stats_data) 194 | end, 195 | opts = opts, 196 | }, { __index = method }) 197 | 198 | self.cond = qc.waiter() 199 | self.fiber = fiber.create(fifottl_fiber, self) 200 | self.sync_chan = fiber.channel() 201 | 202 | return self 203 | end 204 | 205 | -- method.grant grants provided user to all spaces of driver. 206 | function method.grant(self, user, opts) 207 | box.schema.user.grant(user, 'read,write', 'space', self.space.name, opts) 208 | end 209 | 210 | -- cleanup internal fields in task 211 | function method.normalize_task(self, task) 212 | return task and task:transform(3, 5) 213 | end 214 | 215 | -- put task in space 216 | function method.put(self, data, opts) 217 | local max 218 | 219 | -- Taking the maximum of the index is an implicit transactions, so it is 220 | -- always done with 'read-confirmed' mvcc isolation level. 221 | -- It can lead to errors when trying to make parallel 'put' calls with mvcc enabled. 222 | -- It is hapenning because 'max' for several puts in parallel will be the same since 223 | -- read confirmed isolation level makes visible all transactions that finished the commit. 224 | -- To fix it we wrap it with box.begin/commit and set right isolation level. 225 | -- Current fix does not resolve that bug in situations when we already are in transaction 226 | -- since it will open nested transactions. 227 | -- See https://github.com/tarantool/queue/issues/207 228 | -- See https://www.tarantool.io/ru/doc/latest/concepts/atomic/txn_mode_mvcc/ 229 | 230 | if box.cfg.memtx_use_mvcc_engine and (not box.is_in_txn()) then 231 | box.begin({txn_isolation = 'read-committed'}) 232 | max = self.space.index.task_id:max() 233 | box.commit() 234 | else 235 | max = self.space.index.task_id:max() 236 | end 237 | 238 | local id = max and max[i_id] + 1 or 0 239 | 240 | local status 241 | local ttl = opts.ttl or self.opts.ttl 242 | local ttr = opts.ttr or self.opts.ttr 243 | local pri = opts.pri or self.opts.pri or 0 244 | 245 | local next_event 246 | 247 | if opts.delay ~= nil and opts.delay > 0 then 248 | status = state.DELAYED 249 | ttl = ttl + opts.delay 250 | next_event = util.event_time(opts.delay) 251 | else 252 | status = state.READY 253 | next_event = util.event_time(ttl) 254 | end 255 | 256 | local task = self.space:insert{ 257 | id, 258 | status, 259 | next_event, 260 | util.time(ttl), 261 | util.time(ttr), 262 | pri, 263 | util.time(), 264 | data 265 | } 266 | self:on_task_change(task, 'put') 267 | return task 268 | end 269 | 270 | -- touch task 271 | function method.touch(self, id, delta) 272 | local ops = { 273 | {'+', i_next_event, delta}, 274 | {'+', i_ttl, delta}, 275 | {'+', i_ttr, delta} 276 | } 277 | if delta == util.MAX_TIMEOUT then 278 | ops = { 279 | {'=', i_next_event, delta}, 280 | {'=', i_ttl, delta}, 281 | {'=', i_ttr, delta} 282 | } 283 | end 284 | local task = self.space:update(id, ops) 285 | 286 | self:on_task_change(task, 'touch') 287 | return task 288 | end 289 | 290 | -- take task 291 | function method.take(self) 292 | local task = nil 293 | if box.cfg.memtx_use_mvcc_engine and (not box.is_in_txn()) then 294 | box.begin({txn_isolation = 'read-committed'}) 295 | for _, t in self.space.index.status:pairs({state.READY}) do 296 | if not is_expired(t) then 297 | task = t 298 | break 299 | end 300 | end 301 | box.commit() 302 | else 303 | for _, t in self.space.index.status:pairs({state.READY}) do 304 | if not is_expired(t) then 305 | task = t 306 | break 307 | end 308 | end 309 | end 310 | 311 | if task == nil then 312 | return 313 | end 314 | 315 | local next_event = util.time() + task[i_ttr] 316 | local dead_event = task[i_created] + task[i_ttl] 317 | if next_event > dead_event then 318 | next_event = dead_event 319 | end 320 | 321 | task = self.space:update(task[i_id], { 322 | { '=', i_status, state.TAKEN }, 323 | { '=', i_next_event, next_event } 324 | }) 325 | self:on_task_change(task, 'take') 326 | return task 327 | end 328 | 329 | -- delete task 330 | function method.delete(self, id) 331 | local task = self.space:get(id) 332 | self.space:delete(id) 333 | if task ~= nil then 334 | task = task:transform(2, 1, state.DONE) 335 | self:on_task_change(task, 'delete') 336 | end 337 | return task 338 | end 339 | 340 | -- release task 341 | function method.release(self, id, opts) 342 | local task = self.space:get{id} 343 | if task == nil then 344 | return 345 | end 346 | if opts.delay ~= nil and opts.delay > 0 then 347 | task = self.space:update(id, { 348 | { '=', i_status, state.DELAYED }, 349 | { '=', i_next_event, util.event_time(opts.delay) }, 350 | { '+', i_ttl, util.time(opts.delay) } 351 | }) 352 | else 353 | task = self.space:update(id, { 354 | { '=', i_status, state.READY }, 355 | { '=', i_next_event, task[i_created] + task[i_ttl] } 356 | }) 357 | end 358 | self:on_task_change(task, 'release') 359 | return task 360 | end 361 | 362 | -- bury task 363 | function method.bury(self, id) 364 | -- The `i_next_event` should be updated because if the task has been 365 | -- "buried" after it was "taken" (and the task has "ttr") when the time in 366 | -- `i_next_event` will be interpreted as "ttl" in `fifottl_fiber_iteration` 367 | -- and the task will be deleted. 368 | local task = self.space:get{id} 369 | if task == nil then 370 | return 371 | end 372 | task = self.space:update(id, { 373 | { '=', i_status, state.BURIED }, 374 | { '=', i_next_event, task[i_created] + task[i_ttl] } 375 | }) 376 | self:on_task_change(task, 'bury') 377 | return task 378 | end 379 | 380 | -- unbury several tasks 381 | function method.kick(self, count) 382 | for i = 1, count do 383 | local task = self.space.index.status:min{ state.BURIED } 384 | if task == nil then 385 | return i - 1 386 | end 387 | if task[i_status] ~= state.BURIED then 388 | return i - 1 389 | end 390 | 391 | task = self.space:update(task[i_id], {{ '=', i_status, state.READY }}) 392 | self:on_task_change(task, 'kick') 393 | end 394 | return count 395 | end 396 | 397 | -- peek task 398 | function method.peek(self, id) 399 | return self.space:get{id} 400 | end 401 | 402 | -- get iterator to tasks in a certain state 403 | function method.tasks_by_state(self, task_state) 404 | return self.space.index.status:pairs(task_state) 405 | end 406 | 407 | function method.truncate(self) 408 | self.space:truncate() 409 | end 410 | 411 | function method.start(self) 412 | if self.fiber then 413 | return 414 | end 415 | self.fiber = fiber.create(fifottl_fiber, self) 416 | end 417 | 418 | function method.stop(self) 419 | if not self.fiber then 420 | return 421 | end 422 | self.cond:signal(self.fiber:id()) 423 | self.sync_chan:put(true) 424 | self.fiber = nil 425 | end 426 | 427 | return tube 428 | -------------------------------------------------------------------------------- /queue/abstract/driver/limfifottl.lua: -------------------------------------------------------------------------------- 1 | local fiber = require('fiber') 2 | local fifottl = require('queue.abstract.driver.fifottl') 3 | 4 | local tube = {} 5 | 6 | tube.create_space = function(space_name, opts) 7 | if opts.engine == 'vinyl' then 8 | error('limfifottl queue does not support vinyl engine') 9 | end 10 | return fifottl.create_space(space_name, opts) 11 | end 12 | 13 | -- start tube on space 14 | function tube.new(space, on_task_change, opts) 15 | local state = { 16 | capacity = opts.capacity or 0, 17 | parent = fifottl.new(space, on_task_change, opts) 18 | } 19 | 20 | -- put task in space 21 | local put = function (self, data, opts) 22 | local timeout = opts.timeout or 0 23 | local started = tonumber(fiber.time()) 24 | 25 | while true do 26 | local tube_size = self.space:len() 27 | if tube_size < state.capacity or state.capacity == 0 then 28 | return state.parent.put(self, data, opts) 29 | else 30 | if tonumber(fiber.time()) - started > timeout then 31 | return nil 32 | end 33 | fiber.sleep(.01) 34 | end 35 | end 36 | end 37 | 38 | local len = function (self) 39 | return self.space:len() 40 | end 41 | 42 | return setmetatable({ 43 | put = put, 44 | len = len 45 | }, {__index = state.parent}) 46 | end 47 | 48 | return tube 49 | -------------------------------------------------------------------------------- /queue/abstract/queue_session.lua: -------------------------------------------------------------------------------- 1 | local log = require('log') 2 | local fiber = require('fiber') 3 | local uuid = require('uuid') 4 | 5 | local util = require('queue.util') 6 | local qc = require('queue.compat') 7 | local num_type = qc.num_type 8 | local str_type = qc.str_type 9 | 10 | -- since: 11 | -- https://github.com/tarantool/tarantool/commit/f266559b167b4643c377724eebde9651877d6cc9 12 | local ddl_txn_supported = qc.check_version({2, 2, 1}) 13 | 14 | local queue_session = {} 15 | 16 | -- 17 | -- Replicaset mode switch. 18 | -- 19 | -- Running a queue in master-replica mode requires that 20 | -- `_queue_shared_sessions` space was not temporary. 21 | -- When the queue is running in single mode, 22 | -- the space is converted to temporary mode to increase performance. 23 | -- 24 | local function switch_in_replicaset(replicaset_mode) 25 | if replicaset_mode == nil then 26 | replicaset_mode = false 27 | end 28 | 29 | if not box.space._queue_shared_sessions then 30 | return 31 | end 32 | 33 | if box.space._queue_shared_sessions.temporary 34 | and replicaset_mode == false then 35 | return 36 | end 37 | 38 | if not box.space._queue_shared_sessions.temporary 39 | and replicaset_mode == true then 40 | return 41 | end 42 | 43 | box.schema.create_space('_queue_shared_sessions_mgr', { 44 | temporary = not replicaset_mode, 45 | format = { 46 | { name = 'uuid', type = str_type() }, 47 | { name = 'exp_time', type = num_type() }, 48 | { name = 'active', type = 'boolean' }, 49 | } 50 | }) 51 | 52 | box.space._queue_shared_sessions_mgr:create_index('uuid', { 53 | type = 'tree', 54 | parts = { 1, str_type() }, 55 | unique = true 56 | }) 57 | box.space._queue_shared_sessions_mgr:create_index('active', { 58 | type = 'tree', 59 | parts = { 3, 'boolean', 1, str_type() }, 60 | unique = true 61 | }) 62 | 63 | box.begin() -- Disable implicit yields until the transaction ends. 64 | for _, tuple in box.space._queue_shared_sessions:pairs() do 65 | box.space._queue_shared_sessions_mgr:insert(tuple) 66 | end 67 | 68 | if ddl_txn_supported then 69 | -- We can do it inside a transaction. 70 | box.space._queue_shared_sessions:drop() 71 | box.space._queue_shared_sessions_mgr:rename('_queue_shared_sessions') 72 | end 73 | 74 | local status, err = pcall(box.commit) 75 | if not status then 76 | error(('Error migrate _queue_shared_sessions: %s'):format(tostring(err))) 77 | end 78 | 79 | if not ddl_txn_supported then 80 | -- Do it outside a transaction becase DDL does not support 81 | -- multi-statement transactions. 82 | box.space._queue_shared_sessions:drop() 83 | box.space._queue_shared_sessions_mgr:rename('_queue_shared_sessions') 84 | end 85 | end 86 | 87 | --- Create everything that's needed to work with "shared" sessions. 88 | local function identification_init() 89 | local queue_session_ids = box.space._queue_session_ids 90 | if queue_session_ids == nil then 91 | queue_session_ids = box.schema.create_space('_queue_session_ids', { 92 | temporary = true, 93 | format = { 94 | { name = 'connection_id', type = num_type() }, 95 | { name = 'session_uuid', type = str_type() } 96 | } 97 | }) 98 | 99 | queue_session_ids:create_index('conn_id', { 100 | type = 'tree', 101 | parts = {1, num_type()}, 102 | unique = true 103 | }) 104 | queue_session_ids:create_index('uuid', { 105 | type = 'tree', 106 | parts = {2, str_type()}, 107 | unique = false 108 | }) 109 | end 110 | 111 | local replicaset_mode = queue_session.cfg['in_replicaset'] or false 112 | if box.space._queue_shared_sessions == nil then 113 | box.schema.create_space('_queue_shared_sessions', { 114 | temporary = not replicaset_mode, 115 | format = { 116 | { name = 'uuid', type = str_type() }, 117 | { name = 'exp_time', type = num_type() }, 118 | { name = 'active', type = 'boolean' }, 119 | } 120 | }) 121 | 122 | box.space._queue_shared_sessions:create_index('uuid', { 123 | type = 'tree', 124 | parts = { 1, str_type() }, 125 | unique = true 126 | }) 127 | box.space._queue_shared_sessions:create_index('active', { 128 | type = 'tree', 129 | parts = { 3, 'boolean', 1, str_type() }, 130 | unique = true 131 | }) 132 | else 133 | switch_in_replicaset(queue_session.cfg['in_replicaset']) 134 | 135 | -- At the start of the queue, we make all active sessions inactive, 136 | -- and set an expiration time for them. If the ttr setting is not set, 137 | -- then we delete all sessions. 138 | local ttr = queue_session.cfg['ttr'] or 0 139 | if ttr > 0 then 140 | for _, tuple in box.space._queue_shared_sessions.index.active:pairs{true} do 141 | box.space._queue_shared_sessions:update(tuple[1], { 142 | {'=', 2, util.event_time(ttr)}, 143 | {'=', 3, false}, 144 | }) 145 | end 146 | else 147 | if queue_session._on_session_remove ~= nil then 148 | for _, tuple in box.space._queue_shared_sessions.index.uuid:pairs() do 149 | queue_session._on_session_remove(tuple[1]) 150 | end 151 | end 152 | box.space._queue_shared_sessions:truncate() 153 | end 154 | end 155 | end 156 | 157 | local function cleanup_inactive_sessions() 158 | local cur_time = util.time() 159 | 160 | for _, val in box.space._queue_shared_sessions.index.active:pairs{false} do 161 | local session_uuid = val[1] 162 | local exp_time = val[2] 163 | if cur_time >= exp_time then 164 | if queue_session._on_session_remove ~= nil then 165 | queue_session._on_session_remove(session_uuid) 166 | end 167 | box.space._queue_shared_sessions:delete{session_uuid} 168 | end 169 | end 170 | end 171 | 172 | --- Create an expiration fiber to cleanup expired sessions. 173 | local function create_expiration_fiber() 174 | local exp_fiber = fiber.create(function() 175 | fiber.self():name('queue_expiration_fiber') 176 | while true do 177 | if box.info.ro == false then 178 | local status, err = pcall(cleanup_inactive_sessions) 179 | if status == false then 180 | log.error('An error occurred while cleanup the sessions: %s', 181 | err) 182 | end 183 | end 184 | if queue_session.sync_chan:get(1) ~= nil then 185 | log.info("Queue expiration fiber was stopped") 186 | break 187 | end 188 | end 189 | end) 190 | 191 | return exp_fiber 192 | end 193 | 194 | --- Identifies the connection and return the UUID of the current session. 195 | -- If session_uuid ~= nil: associate the connection with given session. 196 | -- In case of attempt to use an invalid format UUID or expired UUID, 197 | -- an error will be thrown. 198 | local function identify(conn_id, session_uuid) 199 | local queue_session_ids = box.space._queue_session_ids 200 | local session_ids = queue_session_ids:get(conn_id) 201 | local cur_uuid = session_ids and session_ids[2] 202 | 203 | if session_uuid == nil and cur_uuid ~= nil then 204 | -- Just return the UUID of the current session. 205 | return cur_uuid 206 | elseif session_uuid == nil and cur_uuid == nil then 207 | -- Generate new UUID for the session. 208 | cur_uuid = uuid.bin() 209 | queue_session_ids:insert{conn_id, cur_uuid} 210 | box.space._queue_shared_sessions:insert{cur_uuid, 0, true} 211 | elseif session_uuid ~= nil then 212 | -- Validate UUID. 213 | if not pcall(uuid.frombin, session_uuid) then 214 | error('Invalid UUID format.') 215 | end 216 | 217 | -- identify using a previously created session. 218 | -- Check that a session with this uuid exists. 219 | local ids_by_uuid = queue_session_ids.index.uuid:select( 220 | session_uuid, { limit = 1 })[1] 221 | local shared_session = box.space._queue_shared_sessions:get(session_uuid) 222 | if ids_by_uuid == nil and shared_session == nil then 223 | error('The UUID ' .. uuid.frombin(session_uuid):str() .. 224 | ' is unknown.') 225 | end 226 | 227 | if cur_uuid ~= session_uuid then 228 | if cur_uuid ~= nil then 229 | queue_session.disconnect(conn_id) 230 | end 231 | queue_session_ids:insert({conn_id, session_uuid}) 232 | cur_uuid = session_uuid 233 | end 234 | 235 | -- Make session active. 236 | if shared_session then 237 | box.space._queue_shared_sessions:update(shared_session[1], { 238 | {'=', 2, 0}, 239 | {'=', 3, true}, 240 | }) 241 | end 242 | end 243 | 244 | return cur_uuid 245 | end 246 | 247 | local function on_session_remove(callback) 248 | if type(callback) ~= 'function' then 249 | error('The "on_session_remove" argument type must be "function"') 250 | end 251 | queue_session._on_session_remove = callback 252 | end 253 | 254 | --- Remove a connection from the list of active connections and 255 | -- release its tasks if necessary. 256 | local function disconnect(conn_id) 257 | local queue_session_ids = box.space._queue_session_ids 258 | local session_uuid = queue_session.identify(conn_id) 259 | 260 | queue_session_ids:delete{conn_id} 261 | local session_ids = queue_session_ids.index.uuid:select(session_uuid, 262 | { limit = 1 })[1] 263 | 264 | -- If a queue session doesn't have any active connections it should be 265 | -- removed (if ttr is absent) or change the `active` flag to make the 266 | -- session inactive. 267 | if session_ids == nil then 268 | local ttr = queue_session.cfg['ttr'] or 0 269 | if ttr > 0 then 270 | local tuple = box.space._queue_shared_sessions:get{session_uuid} 271 | if tuple == nil then 272 | box.space._queue_shared_sessions:insert{ 273 | session_uuid, util.event_time(ttr), false 274 | } 275 | else 276 | box.space._queue_shared_sessions:update(tuple[1], { 277 | {'=', 2, util.event_time(ttr)}, 278 | {'=', 3, false}, 279 | }) 280 | end 281 | elseif queue_session._on_session_remove ~= nil then 282 | queue_session._on_session_remove(session_uuid) 283 | box.space._queue_shared_sessions.index.uuid:delete{session_uuid} 284 | end 285 | end 286 | end 287 | 288 | local function grant(user) 289 | box.schema.user.grant(user, 'read, write', 'space', '_queue_session_ids', 290 | { if_not_exists = true }) 291 | box.schema.user.grant(user, 'read, write', 'space', '_queue_shared_sessions', 292 | { if_not_exists = true }) 293 | end 294 | 295 | local function start() 296 | identification_init() 297 | queue_session.sync_chan = fiber.channel() 298 | queue_session.expiration_fiber = create_expiration_fiber() 299 | end 300 | 301 | -- When switching the master to the replica, 302 | -- the expiration_fiber must be stopped. 303 | local function stop() 304 | queue_session.sync_chan:put(true) 305 | end 306 | 307 | local function validate_opts(opts) 308 | for key, val in pairs(opts) do 309 | if key == 'ttr' then 310 | if type(val) ~= 'number' or val < 0 then 311 | error('Invalid value of ttr: ' .. tostring(val)) 312 | end 313 | elseif key == 'in_replicaset' then 314 | -- Do nothing. 315 | else 316 | error('Unknown option ' .. tostring(key)) 317 | end 318 | end 319 | end 320 | 321 | --- Configure of queue_session module. 322 | -- If an invalid value or an unknown option 323 | -- is used, an error will be thrown. 324 | local function cfg(self, opts) 325 | opts = opts or {} 326 | -- Check all options before configuring so that 327 | -- the configuration is done transactionally. 328 | validate_opts(opts) 329 | 330 | for key, val in pairs(opts) do 331 | self[key] = val 332 | end 333 | 334 | switch_in_replicaset(self['in_replicaset']) 335 | end 336 | 337 | local function exist_shared(session_uuid) 338 | if box.space._queue_shared_sessions:get{session_uuid} then 339 | return true 340 | end 341 | 342 | return false 343 | end 344 | 345 | queue_session.cfg = setmetatable({}, { __call = cfg }) 346 | 347 | -- methods 348 | local method = { 349 | identify = identify, 350 | disconnect = disconnect, 351 | grant = grant, 352 | on_session_remove = on_session_remove, 353 | start = start, 354 | stop = stop, 355 | exist_shared = exist_shared 356 | } 357 | 358 | return setmetatable(queue_session, { __index = method }) 359 | -------------------------------------------------------------------------------- /queue/abstract/queue_state.lua: -------------------------------------------------------------------------------- 1 | -- Queue states. 2 | 3 | local log = require('log') 4 | local fiber = require('fiber') 5 | 6 | --[[ States switching scheme: 7 | 8 | +-----------+ 9 | | RUNNING | 10 | +-----------+ 11 | ^ | (rw -> ro) 12 | (ro -> rw) | v 13 | +------+ +---------+ +--------+ 14 | | INIT | ---> | STARTUP | | ENDING | 15 | +------+ +---------+ +--------+ 16 | ^ | 17 | (ro -> rw) | v 18 | +-----------+ 19 | | WAITING | 20 | +-----------+ 21 | ]]-- 22 | 23 | local queue_state = { 24 | states = { 25 | INIT = 'i', 26 | STARTUP = 's', 27 | RUNNING = 'r', 28 | ENDING = 'e', 29 | WAITING = 'w', 30 | } 31 | } 32 | 33 | local current = queue_state.states.INIT 34 | 35 | local function get_state_key(value) 36 | for k, v in pairs(queue_state.states) do 37 | if v == value then 38 | return k 39 | end 40 | end 41 | 42 | return nil 43 | end 44 | 45 | local function max_lag() 46 | local max_lag = 0 47 | local n_replica = table.maxn(box.info.replication) 48 | 49 | for i = 1, n_replica do 50 | local replica = box.info.replication[i] 51 | if replica and replica.upstream then 52 | local lag = replica.upstream.lag 53 | if lag > max_lag then 54 | max_lag = lag 55 | end 56 | end 57 | end 58 | 59 | return max_lag 60 | end 61 | 62 | local function create_state_fiber(on_state_change_cb) 63 | log.info('Started queue state fiber') 64 | 65 | fiber.create(function() 66 | fiber.self():name('queue_state_fiber') 67 | while true do 68 | if current == queue_state.states.WAITING then 69 | local rc = pcall(box.ctl.wait_rw) 70 | if rc then 71 | current = queue_state.states.STARTUP 72 | log.info('Queue state changed: STARTUP') 73 | -- Wait for maximum upstream lag * 2. 74 | fiber.sleep(max_lag() * 2) 75 | on_state_change_cb(current) 76 | current = queue_state.states.RUNNING 77 | log.info('Queue state changed: RUNNING') 78 | end 79 | elseif current == queue_state.states.RUNNING then 80 | local rc = pcall(box.ctl.wait_ro) 81 | if rc then 82 | current = queue_state.states.ENDING 83 | on_state_change_cb(current) 84 | log.info('Queue state changed: ENDING') 85 | current = queue_state.states.WAITING 86 | log.info('Queue state changed: WAITING') 87 | end 88 | end 89 | -- Handle server shutdown. 90 | if not pcall(fiber.testcancel) then 91 | break 92 | end 93 | end 94 | end) 95 | 96 | log.info('Finished queue state fiber') 97 | end 98 | 99 | -- Public methods. 100 | local method = {} 101 | 102 | -- Initialise queue states. Must be called on queue starts. 103 | function method.init(tubes) 104 | current = queue_state.states.RUNNING 105 | create_state_fiber(tubes) 106 | end 107 | 108 | -- Show current state in human readable format. 109 | function method.show() 110 | return get_state_key(current) 111 | end 112 | 113 | -- Get current state. 114 | function method.get() 115 | return current 116 | end 117 | 118 | -- Polls queue state with maximum attemps number. 119 | function method.poll(state, max_attempts) 120 | local attempt = 0 121 | while true do 122 | if current == state then 123 | return true 124 | end 125 | attempt = attempt + 1 126 | if attempt == max_attempts then 127 | break 128 | end 129 | fiber.sleep(0.01) 130 | end 131 | 132 | return false 133 | end 134 | 135 | return setmetatable(queue_state, { __index = method }) 136 | -------------------------------------------------------------------------------- /queue/abstract/state.lua: -------------------------------------------------------------------------------- 1 | -- task states 2 | return { 3 | READY = 'r', 4 | TAKEN = 't', 5 | DONE = '-', 6 | BURIED = '!', 7 | DELAYED = '~', 8 | } 9 | -------------------------------------------------------------------------------- /queue/compat.lua: -------------------------------------------------------------------------------- 1 | local fun = require('fun') 2 | local log = require('log') 3 | local json = require('json') 4 | local fiber = require('fiber') 5 | 6 | local iter, op = fun.iter, fun.operator 7 | 8 | local function split(self, sep) 9 | local sep, fields = sep or ":", {} 10 | local pattern = string.format("([^%s]+)", sep) 11 | self:gsub(pattern, function(c) table.insert(fields, c) end) 12 | return fields 13 | end 14 | 15 | local function reducer(res, l, r) 16 | if res ~= nil then 17 | return res 18 | end 19 | if tonumber(l) == tonumber(r) then 20 | return nil 21 | end 22 | return tonumber(l) > tonumber(r) 23 | end 24 | 25 | local function split_version(version_string) 26 | local vtable = split(version_string, '.') 27 | local vtable2 = split(vtable[3], '-') 28 | vtable[3], vtable[4] = vtable2[1], vtable2[2] 29 | return vtable 30 | end 31 | 32 | local function check_version(expected, version) 33 | version = version or _TARANTOOL 34 | if type(version) == 'string' then 35 | version = split_version(version) 36 | end 37 | local res = iter(version):zip(expected):reduce(reducer, nil) 38 | if res or res == nil then res = true end 39 | return res 40 | end 41 | 42 | local function get_actual_numtype(version) 43 | return check_version({1, 7, 2}, version) and 'unsigned' or 'num' 44 | end 45 | 46 | local function get_actual_strtype(version) 47 | return check_version({1, 7, 2}, version) and 'string' or 'str' 48 | end 49 | 50 | local function get_actual_vinylname(version) 51 | return check_version({1, 7}, version) and 'vinyl' or nil 52 | end 53 | 54 | local function get_optname_snapdir(version) 55 | return check_version({1, 7}, version) and 'memtx_dir' or 'snap_dir' 56 | end 57 | 58 | local function get_optname_logger(version) 59 | return check_version({1, 7}, version) and 'log' or 'logger' 60 | end 61 | 62 | local function pack_args(...) 63 | return check_version({1, 7}) and { ... } or ... 64 | end 65 | 66 | local waiter_list = {} 67 | 68 | local function waiter_new() 69 | return setmetatable({ 70 | cond = fiber.cond() 71 | }, { 72 | __index = { 73 | wait = function(self, timeout) 74 | self.cond:wait(timeout) 75 | end, 76 | signal = function(self, wfiber) 77 | self.cond:signal() 78 | end, 79 | free = function(self) 80 | if #waiter_list < 100 then 81 | table.insert(waiter_list, self) 82 | end 83 | end 84 | } 85 | }) 86 | end 87 | 88 | local function waiter_old() 89 | return setmetatable({}, { 90 | __index = { 91 | wait = function(self, timeout) 92 | fiber.sleep(timeout) 93 | end, 94 | signal = function(self, fid) 95 | local wfiber = fiber.find(fid) 96 | if wfiber ~= nil and 97 | wfiber:status() ~= 'dead' and 98 | wfiber:id() ~= fiber.id() then 99 | wfiber:wakeup() 100 | end 101 | end, 102 | free = function(self) 103 | if #waiter_list < 100 then 104 | table.insert(waiter_list, self) 105 | end 106 | end 107 | } 108 | }) 109 | end 110 | 111 | local waiter_actual = check_version({1, 7, 2}) and waiter_new or waiter_old 112 | 113 | local function waiter() 114 | return table.remove(waiter_list) or waiter_actual() 115 | end 116 | 117 | return { 118 | split_version = split_version, 119 | check_version = check_version, 120 | vinyl_name = get_actual_vinylname, 121 | num_type = get_actual_numtype, 122 | str_type = get_actual_strtype, 123 | snapdir_optname = get_optname_snapdir, 124 | logger_optname = get_optname_logger, 125 | pack_args = pack_args, 126 | waiter = waiter 127 | } 128 | -------------------------------------------------------------------------------- /queue/init.lua: -------------------------------------------------------------------------------- 1 | local fiber = require('fiber') 2 | 3 | local abstract = require('queue.abstract') 4 | local queue_state = require('queue.abstract.queue_state') 5 | local qc = require('queue.compat') 6 | local queue = nil 7 | 8 | -- load all core drivers 9 | local core_drivers = { 10 | fifo = require('queue.abstract.driver.fifo'), 11 | fifottl = require('queue.abstract.driver.fifottl'), 12 | utube = require('queue.abstract.driver.utube'), 13 | utubettl = require('queue.abstract.driver.utubettl'), 14 | limfifottl = require('queue.abstract.driver.limfifottl') 15 | } 16 | 17 | -- since: 18 | -- https://github.com/locker/tarantool/commit/8cf5151cb4f05cee3fd0ea831add2b3187a01fe4 19 | local watchers_supported = qc.check_version({2, 10, 0}) 20 | 21 | local function register_driver(driver_name, tube_ctr) 22 | if type(tube_ctr.create_space) ~= 'function' or 23 | type(tube_ctr.new) ~= 'function' then 24 | error('tube control methods must contain functions "create_space"' 25 | .. ' and "new"') 26 | end 27 | if queue.driver[driver_name] then 28 | error(('overriding registered driver "%s"'):format(driver_name)) 29 | end 30 | queue.driver[driver_name] = tube_ctr 31 | end 32 | 33 | local deferred_opts = {} 34 | 35 | -- We cannot call queue.cfg() while tarantool is in read_only mode. 36 | -- This method stores settings for later original queue.cfg() call. 37 | local function deferred_cfg(opts) 38 | opts = opts or {} 39 | 40 | for k, v in pairs(opts) do 41 | deferred_opts[k] = v 42 | end 43 | end 44 | 45 | queue = setmetatable({ 46 | driver = core_drivers, 47 | register_driver = register_driver, 48 | state = queue_state.show, 49 | cfg = deferred_cfg, 50 | _VERSION = require('queue.version'), 51 | }, { __index = function(self, key) 52 | -- In replicaset mode, the user can attempt to call public methods on the replica start. 53 | -- For example, a single script is used for master and replica. 54 | -- Each public method has a check on the state of the queue, so this forwarding is safe. 55 | if deferred_opts['in_replicaset'] == true then 56 | if abstract[key] ~= nil then 57 | return abstract[key] 58 | end 59 | else 60 | print(debug.traceback()) 61 | error('Please configure box.cfg{} in read/write mode first') 62 | end 63 | end 64 | }) 65 | 66 | -- Used to store the original methods 67 | local orig_cfg = nil 68 | local orig_call = nil 69 | 70 | local wrapper_impl, handle_instance_mode 71 | 72 | local function rw_waiter() 73 | fiber.name('queue instance rw waiter') 74 | local wait_cond = fiber.cond() 75 | local w = box.watch('box.status', function(_, new_status) 76 | if new_status.is_ro == false then 77 | wait_cond:signal() 78 | end 79 | end) 80 | wait_cond:wait() 81 | w:unregister() 82 | handle_instance_mode() 83 | end 84 | 85 | local function cfg_wrapper(...) 86 | box.cfg = orig_cfg 87 | return wrapper_impl(...) 88 | end 89 | 90 | local function cfg_call_wrapper(cfg, ...) 91 | local cfg_mt = getmetatable(box.cfg) 92 | cfg_mt.__call = orig_call 93 | return wrapper_impl(...) 94 | end 95 | 96 | local function wrap_box_cfg() 97 | if type(box.cfg) == 'function' then 98 | -- box.cfg before the first box.cfg call 99 | orig_cfg = box.cfg 100 | box.cfg = cfg_wrapper 101 | elseif type(box.cfg) == 'table' then 102 | if box.info.ro_reason == 'config' or not watchers_supported then 103 | -- box.cfg after the first box.cfg call. 104 | -- The another call could switch the mode. 105 | local cfg_mt = getmetatable(box.cfg) 106 | orig_call = cfg_mt.__call 107 | cfg_mt.__call = cfg_call_wrapper 108 | else 109 | -- Wait for the rw state. 110 | fiber.new(rw_waiter) 111 | end 112 | else 113 | error('The box.cfg type is unexpected: ' .. type(box.cfg)) 114 | end 115 | end 116 | 117 | function handle_instance_mode() 118 | if box.info.ro == false then 119 | local abstract = require 'queue.abstract' 120 | for name, val in pairs(abstract) do 121 | rawset(queue, name, val) 122 | end 123 | abstract.driver = queue.driver 124 | -- Now the "register_driver" method from abstract will be used. 125 | queue.register_driver = nil 126 | setmetatable(queue, getmetatable(abstract)) 127 | queue.cfg(deferred_opts) 128 | queue.start() 129 | else 130 | -- Delay a start until the box will be configured 131 | -- with read_only = false 132 | wrap_box_cfg() 133 | end 134 | end 135 | 136 | function wrapper_impl(...) 137 | local result = { pcall(box.cfg,...) } 138 | if result[1] then 139 | table.remove(result, 1) 140 | else 141 | wrap_box_cfg() 142 | error(result[2]) 143 | end 144 | 145 | handle_instance_mode() 146 | return unpack(result) 147 | end 148 | 149 | --- Implementation of the “lazy start” procedure. 150 | -- The queue module is loaded immediately if the instance was 151 | -- configured with read_only = false. Otherwise, a start is 152 | -- delayed until the instance will be configured with read_only = false. 153 | local function queue_init() 154 | if rawget(box, 'space') ~= nil and box.info.ro == false then 155 | -- The box was configured with read_only = false 156 | queue = require('queue.abstract') 157 | queue.driver = core_drivers 158 | queue.start() 159 | else 160 | wrap_box_cfg() 161 | end 162 | end 163 | 164 | queue_init() 165 | 166 | return queue 167 | -------------------------------------------------------------------------------- /queue/util.lua: -------------------------------------------------------------------------------- 1 | local fiber = require('fiber') 2 | 3 | -- MAX_TIMEOUT == 100 years 4 | local MAX_TIMEOUT = 365 * 86400 * 100 5 | -- Set to TIMEOUT_INFINITY 6 | -- instead returns time for next event 7 | local TIMEOUT_INFINITY = 18446744073709551615ULL 8 | 9 | --- Convert seconds to microseconds. 10 | -- If tm == nil then returns current system time 11 | -- (in microseconds since the epoch). 12 | -- If tm < 0 return 0. 13 | local function time(tm) 14 | if tm == nil then 15 | tm = fiber.time64() 16 | elseif tm < 0 then 17 | tm = 0 18 | else 19 | tm = tm * 1000000 20 | end 21 | return 0ULL + tm 22 | end 23 | 24 | --- Calculates the system time (in microseconds) of an event that 25 | -- will occur after a given time period(tm, specified in seconds). 26 | -- If tm <= 0 then returns current system time. 27 | local function event_time(tm) 28 | if tm == nil or tm < 0 then 29 | tm = 0 30 | elseif tm > MAX_TIMEOUT then 31 | return TIMEOUT_INFINITY 32 | end 33 | tm = 0ULL + tm * 1000000 + fiber.time64() 34 | return tm 35 | end 36 | 37 | local util = { 38 | MAX_TIMEOUT = MAX_TIMEOUT, 39 | TIMEOUT_INFINITY = TIMEOUT_INFINITY 40 | } 41 | 42 | -- methods 43 | local method = { 44 | time = time, 45 | event_time = event_time 46 | } 47 | 48 | return setmetatable(util, { __index = method }) 49 | -------------------------------------------------------------------------------- /queue/version.lua: -------------------------------------------------------------------------------- 1 | -- Сontains the module version. 2 | -- Requires manual update in case of release commit. 3 | 4 | return '1.4.4' 5 | -------------------------------------------------------------------------------- /rpm/prebuild.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -exu # Strict shell (w/o -o pipefail) 4 | 5 | curl -LsSf https://www.tarantool.io/release/1.10/installer.sh | sudo bash 6 | -------------------------------------------------------------------------------- /rpm/tarantool-queue.spec: -------------------------------------------------------------------------------- 1 | Name: tarantool-queue 2 | Version: 1.0.1 3 | Release: 1%{?dist} 4 | Summary: Persistent in-memory queues for Tarantool 5 | Group: Applications/Databases 6 | License: BSD 7 | URL: https://github.com/tarantool/queue 8 | Source0: https://github.com/tarantool/%{name}/archive/%{version}/%{name}-%{version}.tar.gz 9 | BuildArch: noarch 10 | BuildRequires: tarantool >= 1.7 11 | BuildRequires: tarantool-devel >= 1.7 12 | BuildRequires: /usr/bin/prove 13 | Requires: tarantool >= 1.7 14 | %description 15 | A collection of persistent queue implementations for Tarantool. 16 | 17 | %prep 18 | %setup -q -n %{name}-%{version} 19 | 20 | %build 21 | %cmake -DCMAKE_BUILD_TYPE=RelWithDebInfo 22 | 23 | # %check 24 | # make check 25 | # 26 | %define luapkgdir %{_datadir}/tarantool/queue/ 27 | %install 28 | %if 0%{?fedora} >= 33 || 0%{?rhel} >= 8 29 | %cmake_install 30 | %else 31 | %make_install 32 | %endif 33 | 34 | %files 35 | %dir %{luapkgdir} 36 | %{luapkgdir}/*.lua 37 | %{luapkgdir}/abstract/*.lua 38 | %{luapkgdir}/abstract/driver/*.lua 39 | %doc README.md 40 | %{!?_licensedir:%global license %doc} 41 | %license LICENSE 42 | 43 | %changelog 44 | * Wed Apr 18 2018 Alexander Turenko 1.0.1 45 | - Update tarantool dependency to 1.7 46 | 47 | * Thu Apr 06 2016 Eugene Blikh 1.0.1-6 48 | - RPM spec uses CMake now (depend on tarantool0devel) 49 | 50 | * Thu Feb 18 2016 Roman Tsisyk 1.0.0-1 51 | - Initial version of the RPM spec 52 | -------------------------------------------------------------------------------- /t/000-init.t: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env tarantool 2 | local yaml = require('yaml') 3 | local fiber = require('fiber') 4 | 5 | local test = require('tap').test() 6 | test:plan(2) 7 | 8 | local queue = require('queue') 9 | 10 | local engine = os.getenv('ENGINE') or 'memtx' 11 | 12 | test:test('access to queue until box.cfg is started', function(test) 13 | test:plan(3) 14 | test:isnil(rawget(box, 'space'), 'box is not started yet') 15 | 16 | local s, e = pcall(function() return queue.tube end) 17 | test:ok(not s, 'exception was generated') 18 | test:ok(string.match(e, 'Please configure box.cfg') ~= nil, 'Exception text') 19 | end) 20 | 21 | local state = require('queue.abstract.state') 22 | 23 | local tnt = require('t.tnt') 24 | tnt.cfg{} 25 | 26 | test:test('access to queue after box.cfg{}', function(test) 27 | test:plan(9) 28 | test:istable(queue.tube, 'queue.tube is table') 29 | test:is(#queue.tube, 0, 'queue.tube is empty') 30 | 31 | local tube = queue.create_tube('test', 'fifo', { engine = engine }) 32 | test:ok(queue.tube.test, 'tube "test" is created') 33 | 34 | test:ok(queue.tube.test:put(123), 'put') 35 | 36 | local task = queue.tube.test:take(.1) 37 | test:ok(task, 'task was taken') 38 | test:is(task[3], 123, 'task.data') 39 | test:ok(queue.tube.test:ack(task[1]), 'task.ack') 40 | 41 | test:ok(queue.tube.test:drop(), 'tube.drop') 42 | test:isnil(queue.tube.test, 'tube is really removed') 43 | end) 44 | 45 | tnt.finish() 46 | os.exit(test:check() and 0 or 1) 47 | -- vim: set ft=lua : 48 | -------------------------------------------------------------------------------- /t/001-tube-init.t: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env tarantool 2 | local yaml = require('yaml') 3 | local fiber = require('fiber') 4 | 5 | local tnt = require('t.tnt') 6 | 7 | local test = require('tap').test() 8 | test:plan(1) 9 | 10 | local engine = os.getenv('ENGINE') or 'memtx' 11 | 12 | local mock_tube = { create_space = function() end, new = function() end } 13 | 14 | test:test('test queue mock addition', function(test) 15 | test:plan(3) 16 | 17 | local queue = require('queue') 18 | queue.register_driver('mock', mock_tube) 19 | test:is(queue.driver.mock, mock_tube) 20 | 21 | local res, err = pcall(queue.register_driver, 'mock', mock_tube) 22 | local check = not res and 23 | string.match(err, 'overriding registered driver') ~= nil 24 | test:ok(check, 'check a driver override failure') 25 | 26 | tnt.cfg{} 27 | 28 | test:is(queue.driver.mock, mock_tube) 29 | end) 30 | 31 | tnt.finish() 32 | os.exit(test:check() and 0 or 1) 33 | -- vim: set ft=lua : 34 | -------------------------------------------------------------------------------- /t/010-fifo.t: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env tarantool 2 | local yaml = require('yaml') 3 | local fiber = require('fiber') 4 | 5 | local test = require('tap').test() 6 | test:plan(15) 7 | 8 | local queue = require('queue') 9 | local state = require('queue.abstract.state') 10 | 11 | local engine = os.getenv('ENGINE') or 'memtx' 12 | 13 | local tnt = require('t.tnt') 14 | tnt.cfg{} 15 | 16 | test:ok(rawget(box, 'space'), 'box started') 17 | test:ok(queue, 'queue is loaded') 18 | 19 | local tube = queue.create_tube('test', 'fifo', { engine = engine }) 20 | local tube2 = queue.create_tube('test_stat', 'fifo', { engine = engine }) 21 | test:ok(tube, 'test tube created') 22 | test:is(tube.name, 'test', 'tube.name') 23 | test:is(tube.type, 'fifo', 'tube.type') 24 | 25 | test:test('Fifo statistics', function(test) 26 | test:plan(13) 27 | tube2:put('stat_0') 28 | tube2:put('stat_1') 29 | tube2:put('stat_2') 30 | tube2:put('stat_3') 31 | tube2:put('stat_4') 32 | tube2:delete(4) 33 | tube2:take(.001) 34 | tube2:release(0) 35 | tube2:take(.001) 36 | tube2:ack(0) 37 | tube2:bury(1) 38 | tube2:bury(2) 39 | tube2:kick(1) 40 | tube2:take(.001) 41 | 42 | local stats = queue.statistics('test_stat') 43 | 44 | -- check tasks statistics 45 | test:is(stats.tasks.taken, 1, 'tasks.taken') 46 | test:is(stats.tasks.buried, 1, 'tasks.buried') 47 | test:is(stats.tasks.ready, 1, 'tasks.ready') 48 | test:is(stats.tasks.done, 2, 'tasks.done') 49 | test:is(stats.tasks.delayed, 0, 'tasks.delayed') 50 | test:is(stats.tasks.total, 3, 'tasks.total') 51 | 52 | -- check function call statistics 53 | test:is(stats.calls.delete, 1, 'calls.delete') 54 | test:is(stats.calls.ack, 1, 'calls.ack') 55 | test:is(stats.calls.take, 3, 'calls.take') 56 | test:is(stats.calls.kick, 1, 'calls.kick') 57 | test:is(stats.calls.bury, 2, 'calls.bury') 58 | test:is(stats.calls.put, 5, 'calls.put') 59 | test:is(stats.calls.release, 1, 'calls.release') 60 | end) 61 | 62 | test:test('Easy put/take/ack', function(test) 63 | test:plan(5) 64 | 65 | test:ok(tube:put{123}, 'task was put') 66 | local task = tube:take() 67 | test:ok(task, 'task was taken') 68 | test:is(task[2], 't', 'task status') 69 | task = tube:ack(task[1]) 70 | test:ok(task, 'task was acked') 71 | test:is(task[2], '-', 'task status') 72 | end) 73 | 74 | 75 | test:test('fifo', function(test) 76 | test:plan(5) 77 | 78 | test:test('order', function(test) 79 | test:plan(16) 80 | test:ok(tube:put('first'), 'put task 1') 81 | test:ok(tube:put(2), 'put task 2') 82 | test:ok(tube:put('third'), 'put task 3') 83 | test:ok(tube:put(4), 'put task 4') 84 | 85 | local task1 = tube:take() 86 | test:ok(task1, 'task 1 was taken') 87 | test:is(task1[2], 't', 'task1 status') 88 | test:is(task1[3], 'first', 'task1.data') 89 | 90 | local task2 = tube:take() 91 | test:ok(task2, 'task 2 was taken') 92 | test:is(task2[2], 't', 'task2 status') 93 | test:is(task2[3], 2, 'task2.data') 94 | 95 | local task3 = tube:take() 96 | test:ok(task3, 'task 3 was taken') 97 | test:is(task3[2], 't', 'task3 status') 98 | test:is(task3[3], 'third', 'task3.data') 99 | 100 | local task4 = tube:take() 101 | test:ok(task4, 'task 4 was taken') 102 | test:is(task4[2], 't', 'task4 status') 103 | test:is(task4[3], 4, 'task4.data') 104 | end) 105 | 106 | test:test('timeouts', function(test) 107 | test:plan(3) 108 | 109 | test:isnil(tube:take(.1), 'task was not taken: timeout') 110 | 111 | fiber.create(function() 112 | test:ok(tube:take(1), 'task was taken') 113 | end) 114 | 115 | fiber.sleep(0.1) 116 | test:ok(tube:put(123), 'task was put') 117 | fiber.sleep(0.1) 118 | end) 119 | 120 | test:test('release', function(test) 121 | test:plan(20) 122 | test:ok(tube:put('first'), 'put task 1') 123 | test:ok(tube:put(2), 'put task 2') 124 | test:ok(tube:put('third'), 'put task 3') 125 | test:ok(tube:put(4), 'put task 4') 126 | 127 | local task1 = tube:take() 128 | test:ok(task1, 'task 1 was taken') 129 | test:is(task1[2], 't', 'task1 status') 130 | test:is(task1[3], 'first', 'task1.data') 131 | 132 | local task2 = tube:take() 133 | test:ok(task2, 'task 2 was taken') 134 | test:is(task2[2], 't', 'task2 status') 135 | test:is(task2[3], 2, 'task2.data') 136 | 137 | 138 | test:ok(tube:release(task1[1]), 'task1 was released') 139 | 140 | local task1a = tube:take() 141 | test:ok(task1a, 'task 1 was taken again') 142 | test:is(task1a[2], 't', 'task1 status') 143 | test:is(task1a[3], 'first', 'task1.data') 144 | 145 | local task3 = tube:take() 146 | test:ok(task3, 'task 3 was taken') 147 | test:is(task3[2], 't', 'task3 status') 148 | test:is(task3[3], 'third', 'task3.data') 149 | 150 | local task4 = tube:take() 151 | test:ok(task4, 'task 4 was taken') 152 | test:is(task4[2], 't', 'task4 status') 153 | test:is(task4[3], 4, 'task4.data') 154 | end) 155 | 156 | test:test('release timeouts', function(test) 157 | test:plan(5) 158 | 159 | test:ok(tube:put('test123'), 'task was put') 160 | local task = tube:take(.1) 161 | test:is(task[3], 'test123', 'task.data') 162 | 163 | fiber.create(function() 164 | local task = tube:take(1) 165 | test:ok(task, 'task was taken') 166 | test:is(task[3], 'test123', 'task.data') 167 | end) 168 | 169 | fiber.sleep(0.1) 170 | test:ok(tube:release(task[1]), 'task was released') 171 | fiber.sleep(0.1) 172 | end) 173 | 174 | 175 | test:test('bury/kick/delete/peek', function(test) 176 | test:plan(18) 177 | test:ok(tube:put('first'), 'put task 1') 178 | test:ok(tube:put(2), 'put task 2') 179 | test:ok(tube:put('third'), 'put task 3') 180 | test:ok(tube:put(4), 'put task 4') 181 | 182 | local task1 = tube:take() 183 | local task2 = tube:take() 184 | local task3 = tube:take() 185 | local task4 = tube:take() 186 | 187 | task1 = tube:bury(task1[1]) 188 | test:is(task1[2], state.BURIED, 'task1 is buried') 189 | test:ok(tube:bury(task2[1]), 'task2 is buried') 190 | test:ok(tube:bury(task3[1]), 'task3 is buried') 191 | test:ok(tube:bury(task4[1]), 'task4 is buried') 192 | 193 | fiber.create(function() 194 | for i = 1, 3 do 195 | tube:take() 196 | end 197 | end) 198 | 199 | fiber.sleep(0.1) 200 | test:is(tube:kick(3), 3, '3 tasks were kicken') 201 | fiber.sleep(0.1) 202 | 203 | test:is(tube:peek(task1[1])[2], state.TAKEN, 'task1 unburied and taken') 204 | test:is(tube:peek(task2[1])[2], state.TAKEN, 'task2 unburied and taken') 205 | test:is(tube:peek(task3[1])[2], state.TAKEN, 'task3 unburied and taken') 206 | test:is(tube:peek(task4[1])[2], state.BURIED, 'task4 is still buried') 207 | 208 | test:is(tube:kick(100500), 1, 'one task was unburied') 209 | test:is(tube:peek(task4[1])[2], state.READY, 'task4 unburied and ready') 210 | test:is(tube:delete(task4[1])[2], state.DONE, 'task4 was removed') 211 | 212 | local s, e = pcall(function() tube:peek(task4[1]) end) 213 | test:ok(not s, "Task not found exception") 214 | test:ok(string.match(e, "Task %d+ not found") ~= nil, 215 | "Task not found message") 216 | 217 | end) 218 | end) 219 | 220 | test:test('creating existing tube', function(test) 221 | test:plan(2) 222 | local s, e = pcall(function() queue.create_tube('test', 'fifo', { engine = engine }) end) 223 | test:ok(not s, 'exception was thrown') 224 | test:ok(e:match("Space 'test' already exists") ~= nil, 'text of exception') 225 | end) 226 | 227 | test:test('tempspace', function(test) 228 | if engine ~= 'vinyl' then 229 | test:plan(2) 230 | tube = queue.create_tube('test1', 'fifo', { temporary = true }) 231 | test:ok(tube, 'tube was created') 232 | local space_r = box.space._space:get{queue.tube.test1.raw.space.id} 233 | test:ok(space_r[6].temporary, 'really tempspace') 234 | else 235 | test:plan(0) 236 | end 237 | end) 238 | 239 | test:test('disconnect test', function(test) 240 | test:plan(6) 241 | 242 | tube:put(1) 243 | tube:put(2) 244 | tube:put(3) 245 | 246 | local task1 = tube:take(.1) 247 | test:ok(task1, 'task1 was taken') 248 | local task2 = tube:take(.1) 249 | test:ok(task2, 'task2 was taken') 250 | local task3 = tube:take(.1) 251 | test:ok(task3, 'task3 was taken') 252 | 253 | queue._on_consumer_disconnect() 254 | 255 | test:is(tube:peek(task1[1])[2], state.READY, 'task1 was marked as READY') 256 | test:is(tube:peek(task2[1])[2], state.READY, 'task2 was marked as READY') 257 | test:is(tube:peek(task3[1])[2], state.READY, 'task3 was marked as READY') 258 | end) 259 | 260 | test:test('if not exists tests', function(test) 261 | if engine ~= 'vinyl' then 262 | test:plan(1) 263 | local tube_dup = queue.create_tube('test1', 'fifo', { if_not_exists = true }) 264 | test:is(tube_dup, tube, '') 265 | else 266 | test:plan(0) 267 | end 268 | end) 269 | 270 | test:test('truncate test', function(test) 271 | test:plan(3) 272 | local len = tube.raw.space:count() 273 | test:ok(len > 0, 'we have something in tube') 274 | test:ok(len, tube:truncate(), 'we delete everything from tube') 275 | test:is(tube.raw.space:count(), 0, 'nothing in tube after it') 276 | end) 277 | 278 | test:test('if_not_exists test', function(test) 279 | test:plan(2) 280 | local tube = queue.create_tube('test_ine', 'fifo', { 281 | if_not_exists = true, engine = engine 282 | }) 283 | local tube_new = queue.create_tube('test_ine', 'fifo', { 284 | if_not_exists = true, engine = engine 285 | }) 286 | test:is(tube, tube_new, "if_not_exists if tube exists") 287 | 288 | queue.tube['test_ine'] = nil 289 | local tube_new = queue.create_tube('test_ine', 'fifo', { 290 | if_not_exists = true, engine = engine 291 | }) 292 | test:isnt(tube, tube_new, "if_not_exists if tube doesn't exists") 293 | end) 294 | 295 | test:test('Get tasks by state test', function(test) 296 | test:plan(2) 297 | local tube = queue.create_tube('test_task_it', 'fifo') 298 | 299 | for i = 1, 10 do 300 | tube:put('test_data' .. tostring(i)) 301 | end 302 | for i = 1, 4 do 303 | tube:take(0.001) 304 | end 305 | 306 | local count_taken = 0 307 | local count_ready = 0 308 | 309 | for _, task in tube.raw:tasks_by_state(state.READY) do 310 | if task[2] == state.READY then 311 | count_ready = count_ready + 1 312 | end 313 | end 314 | 315 | for _, task in tube.raw:tasks_by_state(state.TAKEN) do 316 | if task[2] == state.TAKEN then 317 | count_taken = count_taken + 1 318 | end 319 | end 320 | 321 | test:is(count_ready, 6, 'Check tasks count in a ready state') 322 | test:is(count_taken, 4, 'Check tasks count in a taken state') 323 | end) 324 | 325 | tnt.finish() 326 | os.exit(test:check() and 0 or 1) 327 | -- vim: set ft=lua : 328 | -------------------------------------------------------------------------------- /t/020-fifottl.t: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env tarantool 2 | local fiber = require('fiber') 3 | 4 | local test = require('tap').test() 5 | test:plan(16) 6 | 7 | local queue = require('queue') 8 | local state = require('queue.abstract.state') 9 | local queue_state = require('queue.abstract.queue_state') 10 | 11 | local qc = require('queue.compat') 12 | 13 | local tnt = require('t.tnt') 14 | tnt.cfg{} 15 | 16 | local engine = os.getenv('ENGINE') or 'vinyl' 17 | 18 | test:ok(rawget(box, 'space'), 'box started') 19 | test:ok(queue, 'queue is loaded') 20 | 21 | local tube = queue.create_tube('test', 'fifottl', { engine = engine }) 22 | local tube2 = queue.create_tube('test_stat', 'fifottl', { engine = engine }) 23 | test:ok(tube, 'test tube created') 24 | test:is(tube.name, 'test', 'tube.name') 25 | test:is(tube.type, 'fifottl', 'tube.type') 26 | 27 | test:test('Fifottl statistics', function(test) 28 | test:plan(13) 29 | tube2:put('stat_0') 30 | tube2:put('stat_1') 31 | tube2:put('stat_2') 32 | tube2:put('stat_3') 33 | tube2:put('stat_4') 34 | tube2:put('stat_5', {delay=1000}) 35 | tube2:delete(4) 36 | tube2:take(.001) 37 | tube2:release(0) 38 | tube2:take(.001) 39 | tube2:ack(0) 40 | tube2:bury(1) 41 | tube2:bury(2) 42 | tube2:kick(1) 43 | tube2:take(.001) 44 | 45 | local stats = queue.statistics('test_stat') 46 | 47 | -- check tasks statistics 48 | test:is(stats.tasks.taken, 1, 'tasks.taken') 49 | test:is(stats.tasks.buried, 1, 'tasks.buried') 50 | test:is(stats.tasks.ready, 1, 'tasks.ready') 51 | test:is(stats.tasks.done, 2, 'tasks.done') 52 | test:is(stats.tasks.delayed, 1, 'tasks.delayed') 53 | test:is(stats.tasks.total, 4, 'tasks.total') 54 | 55 | -- check function call statistics 56 | test:is(stats.calls.delete, 1, 'calls.delete') 57 | test:is(stats.calls.ack, 1, 'calls.ack') 58 | test:is(stats.calls.take, 3, 'calls.take') 59 | test:is(stats.calls.kick, 1, 'calls.kick') 60 | test:is(stats.calls.bury, 2, 'calls.bury') 61 | test:is(stats.calls.put, 6, 'calls.put') 62 | test:is(stats.calls.release, 1, 'calls.release') 63 | end) 64 | 65 | 66 | test:test('put/take/peek', function(test) 67 | test:plan(11) 68 | 69 | local task = tube:put('abc') 70 | 71 | test:ok(task, "task was put") 72 | test:is(task[2], state.READY, "task.state") 73 | 74 | local peek = tube:peek(task[1]) 75 | test:is_deeply(task[1], peek[1], "put and peek tasks are the same") 76 | test:is_deeply(task[2], peek[2], "put and peek tasks are the same") 77 | test:is_deeply(task[3], peek[3], "put and peek tasks are the same") 78 | 79 | local taken = tube:take( .1 ) 80 | test:ok(taken, 'task was taken') 81 | 82 | test:is(task[1], taken[1], 'task.id') 83 | test:is(taken[2], state.TAKEN, 'task.status') 84 | 85 | local ack = tube:ack(taken[1]) 86 | test:ok(ack, 'task was acked') 87 | 88 | local s, e = pcall(function() tube:peek(task[1]) end) 89 | test:ok(not s, "peek status") 90 | test:ok(string.match(e, 'Task %d+ not found') ~= nil, 'peek error message') 91 | end) 92 | 93 | test:test('delayed tasks', function(test) 94 | test:plan(12) 95 | 96 | local task = tube:put('cde', { delay = 0.1, ttl = 0.1, ttr = 0.01 }) 97 | test:ok(task, 'delayed task was put') 98 | test:is(task[3], 'cde', 'task.data') 99 | test:is(task[2], state.DELAYED, 'state is DELAYED') 100 | 101 | test:isnil(tube:take(.01), 'delayed task was not taken') 102 | 103 | local taken = tube:take(.15) 104 | test:ok(taken, 'delayed task was taken after timeout') 105 | test:is(taken[3], 'cde', 'task.data') 106 | 107 | local retaken = tube:take(0.05) 108 | test:ok(retaken, "retake task after ttr") 109 | test:is(retaken[3], 'cde', 'task.data') 110 | 111 | fiber.sleep(0.2) 112 | local s, e = pcall(function() tube:peek(retaken[1]) end) 113 | test:ok(not s, 'Task is not in database (TTL)') 114 | test:ok(string.match(e, 'Task %d+ not found') ~= nil, 'peek error message') 115 | 116 | 117 | s, e = pcall(function() tube:ack(retaken[1]) end) 118 | test:ok(not s, 'Task is not ackable (TTL)') 119 | test:ok(string.match(e, 'Task was not taken') ~= nil, 'peek error message') 120 | end) 121 | 122 | test:test('delete/peek', function(test) 123 | test:plan(10) 124 | 125 | local task = tube:put('abc') 126 | test:ok(task, 'task was put') 127 | test:is(task[2], state.READY, 'task is READY') 128 | 129 | local taken = tube:take(.1) 130 | test:ok(taken, 'task was taken') 131 | test:is(taken[3], 'abc', 'task.data') 132 | test:is(taken[2], state.TAKEN, 'task is really taken') 133 | 134 | local removed = tube:delete(task[1]) 135 | test:ok(removed, 'tube:delete') 136 | test:is(removed[3], 'abc', 'removed.data') 137 | test:is(removed[2], state.DONE, 'removed.status') 138 | 139 | local s, e = pcall(function() tube:ack(task[1]) end) 140 | test:ok(not s, "Can't ack removed task") 141 | test:ok(string.match(e, 'Task was not taken') ~= nil, 'peek error message') 142 | end) 143 | 144 | test:test('bury/peek/kick', function(test) 145 | test:plan(17) 146 | 147 | local task = tube:put('abc') 148 | test:ok(task, 'task was put') 149 | test:is(task[2], state.READY, 'task is READY') 150 | 151 | local taken = tube:take(.1) 152 | test:ok(taken, 'task was taken') 153 | test:is(taken[3], 'abc', 'task.data') 154 | test:is(taken[2], state.TAKEN, 'task is really taken') 155 | 156 | local buried = tube:bury(task[1]) 157 | test:ok(buried, 'tube:bury') 158 | test:is(buried[3], 'abc', 'buried.data') 159 | test:is(buried[2], state.BURIED, 'buried.status') 160 | 161 | local s, e = pcall(function() tube:ack(task[1]) end) 162 | test:ok(not s, "Can't ack removed task") 163 | test:ok(string.match(e, 'Task was not taken') ~= nil, 'peek error message') 164 | 165 | local peek = tube:peek(task[1]) 166 | test:is(peek[1], buried[1], 'task was peek') 167 | test:is(peek[2], buried[2], 'task.status') 168 | test:is(peek[3], buried[3], 'task.data') 169 | 170 | fiber.create(function() 171 | local retaken = tube:take(0.1) 172 | test:ok(retaken, 'buried task was retaken') 173 | 174 | test:is(retaken[1], buried[1], 'task.id') 175 | test:is(retaken[2], state.TAKEN, 'task.status') 176 | test:is(retaken[3], buried[3], 'task.data') 177 | end) 178 | 179 | tube:kick(1) 180 | fiber.sleep(0.1) 181 | end) 182 | 183 | test:test('if_not_exists test', function(test) 184 | test:plan(2) 185 | local tube = queue.create_tube('test_ine', 'fifottl', { 186 | if_not_exists = true, engine = engine 187 | }) 188 | local tube_new = queue.create_tube('test_ine', 'fifottl', { 189 | if_not_exists = true, engine = engine 190 | }) 191 | test:is(tube, tube_new, "if_not_exists if tube exists") 192 | 193 | queue.tube['test_ine'] = nil 194 | local tube_new = queue.create_tube('test_ine', 'fifottl', { 195 | if_not_exists = true, engine = engine 196 | }) 197 | test:isnt(tube, tube_new, "if_not_exists if tube doesn't exists") 198 | end) 199 | 200 | test:test('touch test', function(test) 201 | test:plan(3) 202 | tube:put('abc', {ttl=0.2, ttr=0.1}) 203 | local task = tube:take() 204 | tube:touch(task[1], 0.3) 205 | fiber.sleep(0.1) 206 | test:is(task[2], 't') 207 | task = tube:ack(task[1]) 208 | test:is(task[2], '-') 209 | test:isnt(task, nil) 210 | end) 211 | 212 | test:test('read_only test', function(test) 213 | test:plan(7) 214 | tube:put('abc', { delay = 0.1 }) 215 | local ttl_fiber = tube.raw.fiber 216 | box.cfg{ read_only = true } 217 | -- Wait for a change in the state of the queue to waiting no more than 10 seconds. 218 | local rc = queue_state.poll(queue_state.states.WAITING, 10) 219 | test:ok(rc, "queue state changed to waiting") 220 | fiber.sleep(0.11) 221 | test:is(ttl_fiber:status(), 'dead', 222 | "check that background fiber is canceled") 223 | test:isnil(tube.raw.fiber, 224 | "check that background fiber object is cleaned") 225 | if qc.check_version({1, 7}) then 226 | local task = tube:take(0.2) 227 | test:isnil(task, "check that task wasn't moved to ready state") 228 | else 229 | local stat, task = pcall(tube.take, tube, 0.2) 230 | test:is(stat, false, "check that task wasn't taken") 231 | end 232 | box.cfg{ read_only = false } 233 | local rc = queue_state.poll(queue_state.states.RUNNING, 10) 234 | test:ok(rc, "queue state changed to running") 235 | test:is(tube.raw.fiber:status(), 'suspended', 236 | "check that background fiber started") 237 | local task = tube:take() 238 | test:isnt(task, nil, "check that we can take task") 239 | tube:ack(task[1]) 240 | end) 241 | 242 | test:test('ttl after delay test', function(test) 243 | local TTL = 10 244 | local TTR = 20 245 | local DELTA = 5 246 | test:plan(2) 247 | box.cfg{} 248 | local tube = queue.create_tube('test_ttl_release', 'fifottl', { if_not_exists = true }) 249 | tube:put({'test_task'}, { ttl = 10, ttr = 20 }) 250 | tube:take() 251 | tube:release(0, { delay = DELTA }) 252 | local task = box.space.test_ttl_release:get(0) 253 | test:is(task.ttl, (TTL + DELTA) * 1000000, 'check TTL after release') 254 | test:is(task.ttr, TTR * 1000000, 'check TTR after release') 255 | end) 256 | 257 | -- gh-96: infinite loop after dropping a tube with a burried task 258 | test:test('buried task in a dropped queue', function(test) 259 | test:plan(1) 260 | 261 | local TASK_ID = 1 262 | local tube = queue.create_tube('test_drop_with_burried', 'fifottl', 263 | {ttr = 0.1, if_not_exist = true}) 264 | 265 | tube:put({foo = 'bar'}) 266 | local task = tube:take(0) 267 | tube:bury(task[TASK_ID]) 268 | 269 | tube:drop() 270 | fiber.sleep(0.2) 271 | test:ok(true, 'queue does not hang') 272 | end) 273 | 274 | test:test('Get tasks by state test', function(test) 275 | test:plan(2) 276 | local tube = queue.create_tube('test_task_it', 'fifottl') 277 | 278 | for i = 1, 10 do 279 | tube:put('test_data' .. tostring(i)) 280 | end 281 | for i = 1, 4 do 282 | tube:take(0.001) 283 | end 284 | 285 | local count_taken = 0 286 | local count_ready = 0 287 | 288 | for _, task in tube.raw:tasks_by_state(state.READY) do 289 | if task[2] == state.READY then 290 | count_ready = count_ready + 1 291 | end 292 | end 293 | 294 | for _, task in tube.raw:tasks_by_state(state.TAKEN) do 295 | if task[2] == state.TAKEN then 296 | count_taken = count_taken + 1 297 | end 298 | end 299 | 300 | test:is(count_ready, 6, 'Check tasks count in a ready state') 301 | test:is(count_taken, 4, 'Check tasks count in a taken state') 302 | end) 303 | 304 | tnt.finish() 305 | os.exit(test:check() and 0 or 1) 306 | -- vim: set ft=lua : 307 | -------------------------------------------------------------------------------- /t/030-utube.t: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env tarantool 2 | local yaml = require('yaml') 3 | local fiber = require('fiber') 4 | 5 | local test = (require('tap')).test() 6 | test:plan(12) 7 | 8 | local queue = require('queue') 9 | local state = require('queue.abstract.state') 10 | 11 | local tnt = require('t.tnt') 12 | tnt.cfg{} 13 | 14 | local engine = os.getenv('ENGINE') or 'memtx' 15 | 16 | test:ok(rawget(box, 'space'), 'box started') 17 | test:ok(queue, 'queue is loaded') 18 | 19 | local tube = queue.create_tube('test', 'utube', { engine = engine }) 20 | local tube2 = queue.create_tube('test_stat', 'utube', { engine = engine }) 21 | local tube_ready, tube2_ready 22 | if engine ~= 'vinyl' then 23 | tube_ready = queue.create_tube('test_ready', 'utube', 24 | { engine = engine, storage_mode = queue.driver.utube.STORAGE_MODE_READY_BUFFER }) 25 | tube2_ready = queue.create_tube('test_stat_ready', 'utube', 26 | { engine = engine, storage_mode = queue.driver.utube.STORAGE_MODE_READY_BUFFER }) 27 | end 28 | test:ok(tube, 'test tube created') 29 | test:is(tube.name, 'test', 'tube.name') 30 | test:is(tube.type, 'utube', 'tube.type') 31 | 32 | test:test('Utube statistics', function(test) 33 | if engine ~= 'vinyl' then 34 | test:plan(13 * 2) 35 | else 36 | test:plan(13) 37 | end 38 | for _, tube_stat in ipairs({tube2, tube2_ready}) do 39 | if tube_stat == nil then 40 | break 41 | end 42 | 43 | tube_stat:put('stat_0') 44 | tube_stat:put('stat_1') 45 | tube_stat:put('stat_2') 46 | tube_stat:put('stat_3') 47 | tube_stat:put('stat_4') 48 | tube_stat:delete(4) 49 | tube_stat:take(.001) 50 | tube_stat:release(0) 51 | tube_stat:take(.001) 52 | tube_stat:ack(0) 53 | tube_stat:bury(1) 54 | tube_stat:bury(2) 55 | tube_stat:kick(1) 56 | tube_stat:take(.001) 57 | 58 | local stats = queue.statistics(tube_stat.name) 59 | 60 | -- check tasks statistics 61 | test:is(stats.tasks.taken, 1, 'tasks.taken') 62 | test:is(stats.tasks.buried, 1, 'tasks.buried') 63 | test:is(stats.tasks.ready, 1, 'tasks.ready') 64 | test:is(stats.tasks.done, 2, 'tasks.done') 65 | test:is(stats.tasks.delayed, 0, 'tasks.delayed') 66 | test:is(stats.tasks.total, 3, 'tasks.total') 67 | 68 | -- check function call statistics 69 | test:is(stats.calls.delete, 1, 'calls.delete') 70 | test:is(stats.calls.ack, 1, 'calls.ack') 71 | test:is(stats.calls.take, 3, 'calls.take') 72 | test:is(stats.calls.kick, 1, 'calls.kick') 73 | test:is(stats.calls.bury, 2, 'calls.bury') 74 | test:is(stats.calls.put, 5, 'calls.put') 75 | test:is(stats.calls.release, 1, 'calls.release') 76 | end 77 | end) 78 | 79 | 80 | test:test('Easy put/take/ack', function(test) 81 | if engine ~= 'vinyl' then 82 | test:plan(12 * 2) 83 | else 84 | test:plan(12) 85 | end 86 | 87 | for _, test_tube in ipairs({tube, tube_ready}) do 88 | if test_tube == nil then 89 | break 90 | end 91 | 92 | test:ok(test_tube:put(123, {utube = 1}), 'task was put') 93 | test:ok(test_tube:put(345, {utube = 1}), 'task was put') 94 | local task = test_tube:take() 95 | test:ok(task, 'task was taken') 96 | test:is(task[2], state.TAKEN, 'task status') 97 | test:is(task[3], 123, 'task.data') 98 | test:ok(test_tube:take(.1) == nil, 'second task was not taken (the same tube)') 99 | 100 | task = test_tube:ack(task[1]) 101 | test:ok(task, 'task was acked') 102 | test:is(task[2], '-', 'task status') 103 | test:is(task[3], 123, 'task.data') 104 | 105 | task = test_tube:take(.1) 106 | test:ok(task, 'task2 was taken') 107 | test:is(task[3], 345, 'task.data') 108 | test:is(task[2], state.TAKEN, 'task.status') 109 | end 110 | end) 111 | 112 | test:test('ack in utube', function(test) 113 | if engine ~= 'vinyl' then 114 | test:plan(8 * 2) 115 | else 116 | test:plan(8) 117 | end 118 | 119 | for _, test_tube in ipairs({tube, tube_ready}) do 120 | if test_tube == nil then 121 | break 122 | end 123 | 124 | test:ok(test_tube:put(123, {utube = 'abc'}), 'task was put') 125 | test:ok(test_tube:put(345, {utube = 'abc'}), 'task was put') 126 | 127 | local state = 0 128 | fiber.create(function() 129 | fiber.sleep(0.1) 130 | local taken = test_tube:take() 131 | test:ok(taken, 'second task was taken') 132 | test:is(taken[3], 345, 'task.data') 133 | state = state + 1 134 | end) 135 | 136 | local taken = test_tube:take(.1) 137 | state = 1 138 | test:ok(taken, 'task was taken') 139 | test:is(taken[3], 123, 'task.data') 140 | fiber.sleep(0.3) 141 | test:is(state, 1, 'state was not changed') 142 | test_tube:ack(taken[1]) 143 | fiber.sleep(0.2) 144 | test:is(state, 2, 'state was changed') 145 | end 146 | end) 147 | test:test('bury in utube', function(test) 148 | if engine ~= 'vinyl' then 149 | test:plan(8 * 2) 150 | else 151 | test:plan(8) 152 | end 153 | 154 | for _, test_tube in ipairs({tube, tube_ready}) do 155 | if test_tube == nil then 156 | break 157 | end 158 | 159 | test:ok(test_tube:put(567, {utube = 'cde'}), 'task was put') 160 | test:ok(test_tube:put(789, {utube = 'cde'}), 'task was put') 161 | 162 | local state = 0 163 | fiber.create(function() 164 | fiber.sleep(0.1) 165 | local taken = test_tube:take() 166 | test:ok(taken, 'second task was taken') 167 | test:is(taken[3], 789, 'task.data') 168 | state = state + 1 169 | end) 170 | 171 | local taken = test_tube:take(.1) 172 | state = 1 173 | test:ok(taken, 'task was taken') 174 | test:is(taken[3], 567, 'task.data') 175 | fiber.sleep(0.3) 176 | test:is(state, 1, 'state was not changed') 177 | test_tube:bury(taken[1]) 178 | fiber.sleep(0.2) 179 | test:is(state, 2, 'state was changed') 180 | end 181 | end) 182 | test:test('instant bury', function(test) 183 | if engine ~= 'vinyl' then 184 | test:plan(1 * 2) 185 | else 186 | test:plan(1) 187 | end 188 | tube:put(1, {ttr=60}) 189 | local taken = tube:take(.1) 190 | test:is(tube:bury(taken[1])[2], '!', 'task is buried') 191 | 192 | if tube_ready ~= nil then 193 | tube_ready:put(1, {ttr=60}) 194 | local taken = tube_ready:take(.1) 195 | test:is(tube_ready:bury(taken[1])[2], '!', 'task is buried') 196 | end 197 | end) 198 | 199 | test:test('if_not_exists test', function(test) 200 | test:plan(2) 201 | local tube = queue.create_tube('test_ine', 'utube', { 202 | if_not_exists = true, engine = engine 203 | }) 204 | local tube_new = queue.create_tube('test_ine', 'utube', { 205 | if_not_exists = true, engine = engine 206 | }) 207 | test:is(tube, tube_new, "if_not_exists if tube exists") 208 | 209 | queue.tube['test_ine'] = nil 210 | local tube_new = queue.create_tube('test_ine', 'utube', { 211 | if_not_exists = true, engine = engine 212 | }) 213 | test:isnt(tube, tube_new, "if_not_exists if tube doesn't exists") 214 | end) 215 | 216 | test:test('Get tasks by state test', function(test) 217 | test:plan(2) 218 | local tube = queue.create_tube('test_task_it', 'utube') 219 | 220 | for i = 1, 10 do 221 | tube:put('test_data' .. tostring(i), { utube = i }) 222 | end 223 | for i = 1, 4 do 224 | tube:take(0.001) 225 | end 226 | 227 | local count_taken = 0 228 | local count_ready = 0 229 | 230 | for _, task in tube.raw:tasks_by_state(state.READY) do 231 | if task[2] == state.READY then 232 | count_ready = count_ready + 1 233 | end 234 | end 235 | 236 | for _, task in tube.raw:tasks_by_state(state.TAKEN) do 237 | if task[2] == state.TAKEN then 238 | count_taken = count_taken + 1 239 | end 240 | end 241 | 242 | test:is(count_ready, 6, 'Check tasks count in a ready state') 243 | test:is(count_taken, 4, 'Check tasks count in a taken state') 244 | end) 245 | 246 | tnt.finish() 247 | os.exit(test:check() and 0 or 1) 248 | -- vim: set ft=lua : 249 | -------------------------------------------------------------------------------- /t/040-utubettl.t: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env tarantool 2 | local yaml = require('yaml') 3 | local fiber = require('fiber') 4 | 5 | local test = (require('tap')).test() 6 | test:plan(18) 7 | 8 | local queue = require('queue') 9 | local state = require('queue.abstract.state') 10 | local queue_state = require('queue.abstract.queue_state') 11 | 12 | local qc = require('queue.compat') 13 | 14 | local tnt = require('t.tnt') 15 | tnt.cfg{} 16 | 17 | local engine = os.getenv('ENGINE') or 'memtx' 18 | 19 | test:ok(rawget(box, 'space'), 'box started') 20 | test:ok(queue, 'queue is loaded') 21 | 22 | local tube = queue.create_tube('test', 'utubettl', { engine = engine }) 23 | local tube2 = queue.create_tube('test_stat', 'utubettl', { engine = engine }) 24 | local tube_ready, tube2_ready 25 | if engine ~= 'vinyl' then 26 | tube_ready = queue.create_tube('test_ready', 'utubettl', 27 | { engine = engine, storage_mode = queue.driver.utubettl.STORAGE_MODE_READY_BUFFER }) 28 | tube2_ready = queue.create_tube('test_stat_ready', 'utubettl', 29 | { engine = engine, storage_mode = queue.driver.utubettl.STORAGE_MODE_READY_BUFFER }) 30 | end 31 | test:ok(tube, 'test tube created') 32 | test:is(tube.name, 'test', 'tube.name') 33 | test:is(tube.type, 'utubettl', 'tube.type') 34 | 35 | test:test('Utubettl statistics', function(test) 36 | if engine ~= 'vinyl' then 37 | test:plan(13 * 2) 38 | else 39 | test:plan(13) 40 | end 41 | for _, tube_stat in ipairs({tube2, tube2_ready}) do 42 | if tube_stat == nil then 43 | break 44 | end 45 | 46 | tube_stat:put('stat_0') 47 | tube_stat:put('stat_1') 48 | tube_stat:put('stat_2') 49 | tube_stat:put('stat_3') 50 | tube_stat:put('stat_4') 51 | tube_stat:put('stat_5', {delay=1000}) 52 | tube_stat:delete(4) 53 | tube_stat:take(.001) 54 | tube_stat:release(0) 55 | tube_stat:take(.001) 56 | tube_stat:ack(0) 57 | tube_stat:bury(1) 58 | tube_stat:bury(2) 59 | tube_stat:kick(1) 60 | tube_stat:take(.001) 61 | 62 | local stats = queue.statistics(tube_stat.name) 63 | 64 | -- check tasks statistics 65 | test:is(stats.tasks.taken, 1, 'tasks.taken') 66 | test:is(stats.tasks.buried, 1, 'tasks.buried') 67 | test:is(stats.tasks.ready, 1, 'tasks.ready') 68 | test:is(stats.tasks.done, 2, 'tasks.done') 69 | test:is(stats.tasks.delayed, 1, 'tasks.delayed') 70 | test:is(stats.tasks.total, 4, 'tasks.total') 71 | 72 | -- check function call statistics 73 | test:is(stats.calls.delete, 1, 'calls.delete') 74 | test:is(stats.calls.ack, 1, 'calls.ack') 75 | test:is(stats.calls.take, 3, 'calls.take') 76 | test:is(stats.calls.kick, 1, 'calls.kick') 77 | test:is(stats.calls.bury, 2, 'calls.bury') 78 | test:is(stats.calls.put, 6, 'calls.put') 79 | test:is(stats.calls.release, 1, 'calls.release') 80 | end 81 | end) 82 | 83 | 84 | test:test('Easy put/take/ack', function(test) 85 | if engine ~= 'vinyl' then 86 | test:plan(12 * 2) 87 | else 88 | test:plan(12) 89 | end 90 | 91 | for _, test_tube in ipairs({tube, tube_ready}) do 92 | if test_tube == nil then 93 | break 94 | end 95 | 96 | test:ok(test_tube:put(123, {utube = 1}), 'task was put') 97 | test:ok(test_tube:put(345, {utube = 1}), 'task was put') 98 | local task = test_tube:take() 99 | test:ok(task, 'task was taken') 100 | test:is(task[2], state.TAKEN, 'task status') 101 | test:is(task[3], 123, 'task.data') 102 | test:ok(test_tube:take(.1) == nil, 'second task was not taken (the same tube)') 103 | 104 | task = test_tube:ack(task[1]) 105 | test:ok(task, 'task was acked') 106 | test:is(task[2], '-', 'task status') 107 | test:is(task[3], 123, 'task.data') 108 | 109 | task = test_tube:take(.1) 110 | test:ok(task, 'task2 was taken') 111 | test:is(task[3], 345, 'task.data') 112 | test:is(task[2], state.TAKEN, 'task.status') 113 | end 114 | end) 115 | 116 | test:test('ttr put/take', function(test) 117 | if engine ~= 'vinyl' then 118 | test:plan(3 * 2) 119 | else 120 | test:plan(3) 121 | end 122 | 123 | local my_queue = queue.create_tube('trr_test', 'utubettl', { engine = engine }) 124 | test:ok(my_queue:put('ttr1', { ttr = 1 }), 'put ttr task') 125 | test:ok(my_queue:take(0.1) ~= nil, 'take this task') 126 | fiber.sleep(1.1) 127 | local task = my_queue:peek(0) 128 | test:is(task[2], state.READY, 'Ready state returned after one second') 129 | 130 | if engine ~= 'vinyl' then 131 | local my_queue_ready = queue.create_tube('trr_test_v2', 'utubettl', 132 | { engine = engine, storage_mode = queue.driver.utubettl.STORAGE_MODE_READY_BUFFER }) 133 | test:ok(my_queue_ready:put('ttr1', { ttr = 1 }), 'put ttr task') 134 | test:ok(my_queue_ready:take(0.1) ~= nil, 'take this task') 135 | fiber.sleep(1.1) 136 | local task = my_queue_ready:peek(0) 137 | test:is(task[2], state.READY, 'Ready state returned after one second') 138 | end 139 | end) 140 | 141 | test:test('ack in utube', function(test) 142 | if engine ~= 'vinyl' then 143 | test:plan(8 * 2) 144 | else 145 | test:plan(8) 146 | end 147 | 148 | for _, test_tube in ipairs({tube, tube_ready}) do 149 | if test_tube == nil then 150 | break 151 | end 152 | 153 | test:ok(test_tube:put(123, {utube = 'abc'}), 'task was put') 154 | test:ok(test_tube:put(345, {utube = 'abc'}), 'task was put') 155 | 156 | local state = 0 157 | fiber.create(function() 158 | fiber.sleep(0.1) 159 | local taken = test_tube:take() 160 | test:ok(taken, 'second task was taken') 161 | test:is(taken[3], 345, 'task.data') 162 | state = state + 1 163 | end) 164 | 165 | local taken = test_tube:take(.1) 166 | state = 1 167 | test:ok(taken, 'task was taken') 168 | test:is(taken[3], 123, 'task.data') 169 | fiber.sleep(0.3) 170 | test:is(state, 1, 'state was not changed') 171 | test_tube:ack(taken[1]) 172 | fiber.sleep(0.2) 173 | test:is(state, 2, 'state was changed') 174 | end 175 | end) 176 | test:test('bury in utube', function(test) 177 | if engine ~= 'vinyl' then 178 | test:plan(8 * 2) 179 | else 180 | test:plan(8) 181 | end 182 | 183 | for _, test_tube in ipairs({tube, tube_ready}) do 184 | if test_tube == nil then 185 | break 186 | end 187 | 188 | test:ok(test_tube:put(567, {utube = 'cde'}), 'task was put') 189 | test:ok(test_tube:put(789, {utube = 'cde'}), 'task was put') 190 | 191 | local state = 0 192 | fiber.create(function() 193 | fiber.sleep(0.1) 194 | local taken = test_tube:take() 195 | test:ok(taken, 'second task was taken') 196 | test:is(taken[3], 789, 'task.data') 197 | state = state + 1 198 | end) 199 | 200 | local taken = test_tube:take(.1) 201 | state = 1 202 | test:ok(taken, 'task was taken') 203 | test:is(taken[3], 567, 'task.data') 204 | fiber.sleep(0.3) 205 | test:is(state, 1, 'state was not changed') 206 | test_tube:bury(taken[1]) 207 | fiber.sleep(0.2) 208 | test:is(state, 2, 'state was changed') 209 | end 210 | end) 211 | 212 | test:test('instant bury', function(test) 213 | if engine ~= 'vinyl' then 214 | test:plan(1 * 2) 215 | else 216 | test:plan(1) 217 | end 218 | 219 | tube:put(1, {ttr=60}) 220 | local taken = tube:take(.1) 221 | test:is(tube:bury(taken[1])[2], '!', 'task is buried') 222 | 223 | if tube_ready ~= nil then 224 | tube_ready:put(1, {ttr=60}) 225 | taken = tube_ready:take(.1) 226 | test:is(tube_ready:bury(taken[1])[2], '!', 'task is buried') 227 | end 228 | end) 229 | 230 | test:test('priority in utube', function(test) 231 | if engine ~= 'vinyl' then 232 | test:plan(8 * 2) 233 | else 234 | test:plan(8) 235 | end 236 | 237 | for _, test_tube in ipairs({tube, tube_ready}) do 238 | if test_tube == nil then 239 | break 240 | end 241 | 242 | test:ok(test_tube:put(670, {utube = 'dee', pri = 1}), 'task was put') 243 | test:ok(test_tube:put(671, {utube = 'dee', pri = 0}), 'task was put') 244 | 245 | local taken = test_tube:take(.1) 246 | test:ok(taken, 'task was taken ' .. taken[1]) 247 | test:is(taken[3], 671, 'task.data') 248 | 249 | test_tube:release(taken[1]) 250 | 251 | taken = test_tube:take(.1) 252 | test:ok(taken, 'task was taken ' .. taken[1]) 253 | test:is(taken[3], 671, 'task.data') 254 | test_tube:ack(taken[1]) 255 | 256 | taken = test_tube:take(.1) 257 | test:ok(taken, 'task was taken ' .. taken[1]) 258 | test:is(taken[3], 670, 'task.data') 259 | test_tube:ack(taken[1]) 260 | end 261 | end) 262 | 263 | test:test('release in utube', function(test) 264 | if engine ~= 'vinyl' then 265 | test:plan(8 * 2) 266 | else 267 | test:plan(8) 268 | end 269 | 270 | for _, test_tube in ipairs({tube, tube_ready}) do 271 | if test_tube == nil then 272 | break 273 | end 274 | 275 | test:ok(test_tube:put(678, {utube = 'def'}), 'task was put') 276 | test:ok(test_tube:put(890, {utube = 'def'}), 'task was put') 277 | 278 | local state = 0 279 | fiber.create(function() 280 | fiber.sleep(0.1) 281 | local taken = test_tube:take() 282 | test:ok(taken, 'first task was taken again') 283 | test:is(taken[3], 678, 'task.data') 284 | state = state + 1 285 | end) 286 | 287 | local taken = test_tube:take(.1) 288 | state = 1 289 | test:ok(taken, 'task was taken ' .. taken[1]) 290 | test:is(taken[3], 678, 'task.data') 291 | fiber.sleep(0.3) 292 | test:is(state, 1, 'state was not changed') 293 | test_tube:release(taken[1]) 294 | fiber.sleep(0.2) 295 | test:is(state, 2, 'state was changed') 296 | end 297 | end) 298 | 299 | test:test('release[delay] in utube', function(test) 300 | if engine ~= 'vinyl' then 301 | test:plan(8 * 2) 302 | else 303 | test:plan(8) 304 | end 305 | 306 | for _, test_tube in ipairs({tube, tube_ready}) do 307 | if test_tube == nil then 308 | break 309 | end 310 | 311 | test:ok(test_tube:put(789, {utube = 'efg'}), 'task was put') 312 | test:ok(test_tube:put(901, {utube = 'efg'}), 'task was put') 313 | 314 | local state = 0 315 | fiber.create(function() 316 | fiber.sleep(0.1) 317 | local taken = test_tube:take() 318 | test:ok(taken, 'second task was taken') 319 | test:is(taken[3], 901, 'task.data') 320 | state = state + 1 321 | end) 322 | 323 | local taken = test_tube:take(.1) 324 | state = 1 325 | test:ok(taken, 'task was taken ' .. taken[1]) 326 | test:is(taken[3], 789, 'task.data') 327 | fiber.sleep(0.3) 328 | test:is(state, 1, 'state was not changed') 329 | test_tube:release(taken[1], { delay = 10 }) 330 | fiber.sleep(0.2) 331 | test:is(state, 2, 'state was changed') 332 | end 333 | end) 334 | 335 | test:test('if_not_exists test', function(test) 336 | test:plan(2) 337 | local tube = queue.create_tube('test_ine', 'utubettl', { 338 | if_not_exists = true, engine = engine 339 | }) 340 | local tube_new = queue.create_tube('test_ine', 'utubettl', { 341 | if_not_exists = true, engine = engine 342 | }) 343 | test:is(tube, tube_new, "if_not_exists if tube exists") 344 | 345 | queue.tube['test_ine'] = nil 346 | local tube_new = queue.create_tube('test_ine', 'utubettl', { 347 | if_not_exists = true, engine = engine 348 | }) 349 | test:isnt(tube, tube_new, "if_not_exists if tube doesn't exists") 350 | end) 351 | 352 | test:test('read_only test', function(test) 353 | test:plan(7) 354 | tube:put('abc', { delay = 0.1 }) 355 | local ttl_fiber = tube.raw.fiber 356 | box.cfg{ read_only = true } 357 | -- Wait for a change in the state of the queue to waiting no more than 10 seconds. 358 | local rc = queue_state.poll(queue_state.states.WAITING, 10) 359 | test:ok(rc, "queue state changed to waiting") 360 | fiber.sleep(0.11) 361 | test:is(ttl_fiber:status(), 'dead', 362 | "check that background fiber is canceled") 363 | test:isnil(tube.raw.fiber, 364 | "check that background fiber object is cleaned") 365 | if qc.check_version({1, 7}) then 366 | local task = tube:take(0.2) 367 | test:isnil(task, "check that task wasn't moved to ready state") 368 | else 369 | local stat, task = pcall(tube.take, tube, 0.2) 370 | test:is(stat, false, "check that task wasn't taken") 371 | end 372 | box.cfg{ read_only = false } 373 | local rc = queue_state.poll(queue_state.states.RUNNING, 10) 374 | test:ok(rc, "queue state changed to running") 375 | test:is(tube.raw.fiber:status(), 'suspended', 376 | "check that background fiber started") 377 | local task = tube:take() 378 | test:isnt(task, nil, "check that we can take task") 379 | tube:ack(task[1]) 380 | end) 381 | 382 | test:test('ttl after delay test', function(test) 383 | local TTL = 10 384 | local TTR = 20 385 | local DELTA = 5 386 | test:plan(2) 387 | box.cfg{} 388 | local tube = queue.create_tube('test_ttl_release', 'utubettl', { if_not_exists = true }) 389 | tube:put({'test_task'}, { ttl = 10, ttr = 20 }) 390 | tube:take() 391 | tube:release(0, { delay = DELTA }) 392 | local task = box.space.test_ttl_release:get(0) 393 | test:is(task.ttl, (TTL + DELTA) * 1000000, 'check TTL after release') 394 | test:is(task.ttr, TTR * 1000000, 'check TTR after release') 395 | end) 396 | 397 | test:test('Get tasks by state test', function(test) 398 | test:plan(2) 399 | local tube = queue.create_tube('test_task_it', 'utubettl') 400 | 401 | for i = 1, 10 do 402 | tube:put('test_data' .. tostring(i), { utube = i }) 403 | end 404 | for i = 1, 4 do 405 | tube:take(0.001) 406 | end 407 | 408 | local count_taken = 0 409 | local count_ready = 0 410 | 411 | for _, task in tube.raw:tasks_by_state(state.READY) do 412 | if task[2] == state.READY then 413 | count_ready = count_ready + 1 414 | end 415 | end 416 | 417 | for _, task in tube.raw:tasks_by_state(state.TAKEN) do 418 | if task[2] == state.TAKEN then 419 | count_taken = count_taken + 1 420 | end 421 | end 422 | 423 | test:is(count_ready, 6, 'Check tasks count in a ready state') 424 | test:is(count_taken, 4, 'Check tasks count in a taken state') 425 | end) 426 | 427 | tnt.finish() 428 | os.exit(test:check() and 0 or 1) 429 | -- vim: set ft=lua : 430 | -------------------------------------------------------------------------------- /t/050-ttl.t: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env tarantool 2 | local fiber = require('fiber') 3 | 4 | local test = require('tap').test() 5 | test:plan(7) 6 | 7 | local queue = require('queue') 8 | 9 | local engine = os.getenv('ENGINE') or 'memtx' 10 | 11 | local tnt = require('t.tnt') 12 | tnt.cfg{ 13 | wal_mode = (engine == 'memtx' and 'none' or nil) 14 | } 15 | 16 | local ttl = 0.1 17 | 18 | test:ok(queue, 'queue is loaded') 19 | 20 | local function test_take_after_ttl(test, tube, ttl) 21 | local attempts, max_attempts = 0, 2 22 | local before = fiber.time64() 23 | local after = before 24 | 25 | while after - before < ttl * 1000000 and attempts < max_attempts do 26 | attempts = attempts + 1 27 | fiber.sleep(ttl) 28 | after = fiber.time64() 29 | end 30 | 31 | if after - before < ttl * 1000000 then 32 | test:fail('failed to sleep ttl, is system clock changed?') 33 | else 34 | test:isnil(tube:take(.1), 'no task is taken') 35 | end 36 | end 37 | 38 | test:test('one message per queue ffttl', function (test) 39 | test:plan(20) 40 | local tube = queue.create_tube('ompq_ffttl', 'fifottl', { engine = engine }) 41 | for i = 1, 20 do 42 | tube:put('ompq_' .. i, {ttl=ttl}) 43 | 44 | test_take_after_ttl(test, tube, ttl) 45 | end 46 | end) 47 | 48 | test:test('one message per queue utttl', function (test) 49 | test:plan(20) 50 | local tube = queue.create_tube('ompq_utttl', 'utubettl', { engine = engine }) 51 | for i = 1, 20 do 52 | tube:put('ompq_' .. i, {ttl=ttl}) 53 | 54 | test_take_after_ttl(test, tube, ttl) 55 | end 56 | end) 57 | 58 | test:test('one message per queue utttl_ready', function (test) 59 | if engine == 'vinyl' then 60 | return 61 | end 62 | 63 | test:plan(20) 64 | local tube = queue.create_tube('ompq_utttl_ready', 'utubettl', 65 | { engine = engine, storage_mode = queue.driver.utubettl.STORAGE_MODE_READY_BUFFER }) 66 | for i = 1, 20 do 67 | tube:put('ompq_' .. i, {ttl=ttl}) 68 | 69 | test_take_after_ttl(test, tube, ttl) 70 | end 71 | end) 72 | 73 | test:test('many messages, one queue ffttl', function (test) 74 | test:plan(20) 75 | for i = 1, 20 do 76 | local tube = queue.create_tube('mmpq_ffttl_' .. i, 'fifottl', { engine = engine }) 77 | tube:put('mmpq_' .. i, {ttl=ttl}) 78 | 79 | test_take_after_ttl(test, tube, ttl) 80 | end 81 | end) 82 | 83 | test:test('many messages, one queue utttl', function (test) 84 | test:plan(20) 85 | for i = 1, 20 do 86 | local tube = queue.create_tube('mmpq_utttl_' .. i, 'utubettl', { engine = engine }) 87 | tube:put('mmpq_' .. i, {ttl=ttl}) 88 | 89 | test_take_after_ttl(test, tube, ttl) 90 | end 91 | end) 92 | 93 | test:test('many messages, one queue utttl_ready', function (test) 94 | if engine == 'vinyl' then 95 | return 96 | end 97 | 98 | test:plan(20) 99 | for i = 1, 20 do 100 | local tube = queue.create_tube('mmpq_utttl_ready_' .. i, 'utubettl', 101 | { engine = engine, storage_mode = queue.driver.utubettl.STORAGE_MODE_READY_BUFFER }) 102 | tube:put('mmpq_' .. i, {ttl=ttl}) 103 | 104 | test_take_after_ttl(test, tube, ttl) 105 | end 106 | end) 107 | 108 | tnt.finish() 109 | os.exit(test:check() and 0 or 1) 110 | -- vim: set ft=lua : 111 | -------------------------------------------------------------------------------- /t/060-async.t: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env tarantool 2 | local yaml = require('yaml') 3 | local fiber = require('fiber') 4 | 5 | local test = require('tap').test() 6 | test:plan(6) 7 | 8 | local queue = require('queue') 9 | local state = require('queue.abstract.state') 10 | 11 | local tnt = require('t.tnt') 12 | tnt.cfg{} 13 | 14 | local engine = os.getenv('ENGINE') or 'memtx' 15 | 16 | test:ok(rawget(box, 'space'), 'box started') 17 | test:ok(queue, 'queue is loaded') 18 | 19 | local tube = queue.create_tube('test', 'fifo', { engine = engine }) 20 | test:ok(tube, 'test tube created') 21 | test:is(tube.name, 'test', 'tube.name') 22 | test:is(tube.type, 'fifo', 'tube.type') 23 | 24 | test:test('concurent take', function(test) 25 | test:plan(16) 26 | 27 | local channel = fiber.channel(1000) 28 | test:ok(channel, 'channel created') 29 | 30 | local res = {} 31 | for i = 1, 5 do 32 | fiber.create(function(i) 33 | local taken = tube:take(1) 34 | test:ok(taken, 'Task was taken ' .. i) 35 | table.insert(res, { taken }) 36 | channel:put(true) 37 | end, i) 38 | end 39 | 40 | fiber.sleep(.1) 41 | test:ok(tube:put(1), 'task 1 was put') 42 | 43 | for i = 2, 5 do 44 | fiber.create(function(i) 45 | fiber.sleep(.5 / i) 46 | test:ok(tube:put(i), 'task ' .. i .. ' was put') 47 | end, i) 48 | end 49 | 50 | for i = 1, 5 do 51 | test:ok(channel:get(1 / i), 'take was done ' .. i) 52 | end 53 | end) 54 | 55 | 56 | tnt.finish() 57 | os.exit(test:check() and 0 or 1) 58 | -- vim: set ft=lua: 59 | -------------------------------------------------------------------------------- /t/070-compat.t: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env tarantool 2 | 3 | local test = require('tap').test() 4 | test:plan(3) 5 | 6 | local qcompat = require('queue.compat') 7 | 8 | test:ok(qcompat, 'queue compatibility layer exists') 9 | 10 | test:test('*_version', function(test) 11 | test:plan(10) 12 | 13 | local split_version = qcompat.split_version 14 | local check_version = qcompat.check_version 15 | 16 | test:is_deeply(split_version("1.6.8-173"), {"1", "6", "8", "173"}, "check split_version 1") 17 | test:is_deeply(split_version("1.7.1-0"), {"1", "7", "1", "0"}, "check split_version 2") 18 | test:is_deeply(split_version("1.7.1"), {"1", "7", "1"}, "check split_version 3") 19 | 20 | test:is(check_version({1, 7, 1}, "1.8.1-0"), true, "check supported version") 21 | test:is(check_version({1, 7, 1}, "1.7.1-0"), true, "check supported version") 22 | test:is(check_version({1, 7, 1}, "1.7.1-1"), true, "check supported version") 23 | test:is(check_version({1, 7, 2}, "1.8.1"), true, "check supported version") 24 | test:is(check_version({1, 7, 1}, "1.6.9"), false, "check unsupported version") 25 | test:is(check_version({1, 7, 1}, "1.6.9-100"), false, "check unsupported version") 26 | test:is(check_version({1, 7, 1}, "1.6.9-100"), false, "check unsupported version") 27 | end) 28 | 29 | test:test("check compatibility names", function(test) 30 | test:plan(7) 31 | 32 | local vinyl_name = qcompat.vinyl_name 33 | local num_name = qcompat.num_type 34 | local str_name = qcompat.str_type 35 | 36 | test:is(vinyl_name("1.7.1-168"), "vinyl", "check new name (vinyl)") 37 | test:is(num_name("1.7.2-1"), "unsigned", "check new name (unsigned)") 38 | test:is(num_name("1.6.9-168"), "num", "check old name (num)") 39 | test:is(num_name("1.7.1-168"), "num", "check old name (num)") 40 | test:is(str_name("1.7.2-1"), "string", "check new name (string)") 41 | test:is(str_name("1.6.9-168"), "str", "check old name (str)") 42 | test:is(str_name("1.7.1-168"), "str", "check old name (str)") 43 | end) 44 | 45 | os.exit(test:check() and 0 or 1) 46 | -- vim: set ft=lua: 47 | -------------------------------------------------------------------------------- /t/080-otc-cb.t: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env tarantool 2 | local test = require('tap').test() 3 | test:plan(1) 4 | 5 | local tnt = require('t.tnt') 6 | tnt.cfg{} 7 | 8 | local engine = os.getenv('ENGINE') or 'memtx' 9 | 10 | local queue = require('queue') 11 | 12 | local function tube_check_simple(tube) 13 | tube:put{123} 14 | local task = tube:take(0) 15 | tube:ack(task[1]) 16 | end 17 | 18 | test:test('on_task_change callback', function(test) 19 | test:plan(5) 20 | local cnt = 0 21 | 22 | -- simple callbacks, to check that they're being called 23 | local function cb(t1, t2) cnt = cnt + 1 end 24 | local function new_cb(t1, t2) cnt = cnt + 2 end 25 | 26 | -- init with initial callback 27 | local tube = queue.create_tube('test2', 'fifo', { 28 | on_task_change = cb, engine = engine 29 | }) 30 | 31 | tube_check_simple(tube) 32 | test:is(cnt, 3, 'check counter') 33 | 34 | -- set new callback 35 | test:is(tube:on_task_change(new_cb), cb, "check rv of on_task_change") 36 | 37 | tube_check_simple(tube) 38 | test:is(cnt, 9, 'check counter after new cb is applied') 39 | 40 | -- delete callback 41 | test:is(tube:on_task_change(), new_cb, "check rv of on_task_change") 42 | 43 | tube_check_simple(tube) 44 | test:is(cnt, 9, 'check counter after new cb is applied') 45 | end) 46 | 47 | tnt.finish() 48 | os.exit(test:check() and 0 or 1) 49 | -- vim: set ft=lua : 50 | -------------------------------------------------------------------------------- /t/090-grant-check.t: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env tarantool 2 | local test = require('tap').test() 3 | test:plan(9) 4 | 5 | local test_user = 'test' 6 | local test_pass = '1234' 7 | local test_host = 'localhost' 8 | local test_port = '3388' 9 | 10 | local uri = require('uri') 11 | local netbox = require('net.box') 12 | 13 | local tnt = require('t.tnt') 14 | 15 | tnt.cfg{ 16 | listen = ('%s:%s'):format(test_host, test_port) 17 | -- listen = uri.format({ host = test_host, service = test_port }) 18 | } 19 | 20 | local engine = os.getenv('ENGINE') or 'memtx' 21 | 22 | local qc = require('queue.compat') 23 | 24 | local test_drivers_grant_cases = { 25 | { 26 | name = 'fifo', 27 | queue_type = 'fifo', 28 | }, 29 | { 30 | name = 'fifottl', 31 | queue_type = 'fifottl', 32 | }, 33 | { 34 | name = 'utube_default', 35 | queue_type = 'utube', 36 | storage_mode = 'default', 37 | }, 38 | { 39 | name = 'utube_ready_buffer', 40 | queue_type = 'utube', 41 | storage_mode = 'ready_buffer', 42 | }, 43 | { 44 | name = 'utubettl_default', 45 | queue_type = 'utubettl', 46 | storage_mode = 'default', 47 | }, 48 | { 49 | name = 'utubettl_ready_buffer', 50 | queue_type = 'utubettl', 51 | storage_mode = 'ready_buffer', 52 | }, 53 | } 54 | 55 | for _, tc in pairs(test_drivers_grant_cases) do 56 | test:test('test dirvers grant ' .. tc.name, function(test) 57 | local queue = require('queue') 58 | box.schema.user.create(test_user, { password = test_pass }) 59 | 60 | test:plan(2) 61 | 62 | local tube_opts = { engine = engine } 63 | if tc.storage_mode ~= nil and tc.storage_mode ~= 'default' then 64 | tube_opts.storage_mode = tc.storage_mode 65 | tube_opts.engine = 'memtx' 66 | end 67 | local tube = queue.create_tube('test', tc.queue_type, tube_opts) 68 | tube:put('help') 69 | 70 | tube:grant(test_user) 71 | 72 | box.session.su(test_user) 73 | local a = tube:take() 74 | test:is(a[1], 0, 'we aren\'t getting any error') 75 | 76 | local c = tube:ack(a[1]) 77 | test:is(c[1], 0, 'we aren\'t getting any error') 78 | 79 | box.session.su('admin') 80 | 81 | box.schema.user.drop(test_user) 82 | tube:drop() 83 | end) 84 | end 85 | 86 | test:test('check for space grants', function(test) 87 | -- prepare for tests 88 | local queue = require('queue') 89 | box.schema.user.create(test_user, { password = test_pass }) 90 | 91 | test:plan(5) 92 | 93 | local tube = queue.create_tube('test', 'fifo', { engine = engine }) 94 | tube:put('help'); 95 | local task = tube:take(); 96 | test:is(task[1], 0, 'we can get record') 97 | tube:release(task[1]) 98 | 99 | -- checking without grants 100 | box.session.su('test') 101 | local stat, er = pcall(tube.take, tube) 102 | test:is(stat, false, 'we\'re getting error') 103 | box.session.su('admin') 104 | 105 | -- checking with grants 106 | tube:grant('test') 107 | box.session.su('test') 108 | local a = tube:take() 109 | test:is(a[1], 0, 'we aren\'t getting any error') 110 | local b = tube:take(0.1) 111 | test:isnil(b, 'we aren\'t getting any error') 112 | local c = tube:ack(a[1]) 113 | test:is(a[1], 0, 'we aren\'t getting any error') 114 | box.session.su('admin') 115 | 116 | -- checking double grants 117 | tube:grant('test') 118 | 119 | box.schema.user.drop(test_user) 120 | tube:drop() 121 | end) 122 | 123 | test:test('check for call grants', function(test) 124 | -- prepare for tests 125 | rawset(_G, 'queue', require('queue')) 126 | box.schema.user.create(test_user, { password = test_pass }) 127 | 128 | test:plan(12) 129 | 130 | local tube = queue.create_tube('test', 'fifo', { engine = engine }) 131 | tube:put('help'); 132 | local task = tube:take(); 133 | test:is(task[1], 0, 'we can get record') 134 | tube:release(task[1]) 135 | 136 | -- checking without grants 137 | box.session.su('test') 138 | local stat, er = pcall(tube.take, tube) 139 | test:is(stat, false, 'we\'re getting error') 140 | box.session.su('admin') 141 | 142 | -- checking with grants 143 | tube:grant('test') 144 | 145 | box.session.su('test') 146 | local a = tube:take() 147 | test:is(a[1], 0, 'we aren\'t getting any error') 148 | local b = tube:take(0.1) 149 | test:isnil(b, 'we aren\'t getting any error') 150 | local c = tube:release(a[1]) 151 | test:is(a[1], 0, 'we aren\'t getting any error') 152 | box.session.su('admin') 153 | 154 | local nb_connect = netbox.connect 155 | if nb_connect == nil then 156 | nb_connect = netbox.new 157 | end 158 | local con = nb_connect( 159 | ('%s:%s@%s:%s'):format(test_user, test_pass, test_host, test_port) 160 | ) 161 | --[[ 162 | local con = netbox.connect(uri.format({ 163 | login = test_user, password = test_pass, 164 | host = test_host, service = test_port, 165 | }, true)) 166 | ]]-- 167 | 168 | local stat, err = pcall(con.call, con, 'queue.tube.test:take') 169 | test:is(stat, false, 'we\'re getting error') 170 | 171 | -- granting call 172 | tube:grant('test', { call = true }) 173 | 174 | local qc_arg_unpack = function(arg) 175 | if qc.check_version({1, 7}) then 176 | return arg 177 | end 178 | return arg and arg[1] 179 | end 180 | 181 | local a = con:call('queue.tube.test:take') 182 | test:is(qc_arg_unpack(a[1]), 0, 'we aren\'t getting any error') 183 | local b = con:call('queue.tube.test:take', qc.pack_args(0.1)) 184 | test:isnil( 185 | qc_arg_unpack(qc_arg_unpack(b)), 186 | 'we aren\'t getting any error' 187 | ) 188 | local c = con:call('queue.tube.test:ack', qc.pack_args(qc_arg_unpack(a[1]))) 189 | test:is(qc_arg_unpack(a[1]), 0, 'we aren\'t getting any error') 190 | 191 | local d = con:call('queue.tube.test:put', {'help'}) 192 | test:is(qc_arg_unpack(d[1]), 0, 'we aren\'t getting any error') 193 | 194 | local e = con:call('queue.statistics') 195 | test:is(type(qc_arg_unpack(e)), 'table', 'we aren\'t getting any error') 196 | 197 | tube:grant('test', { truncate = true }) 198 | local f = con:call('queue.tube.test:truncate') 199 | test:isnil(qc_arg_unpack(f), 'we aren\'t getting any error') 200 | 201 | -- check grants again 202 | tube:grant('test', { call = true }) 203 | 204 | rawset(_G, 'queue', nil) 205 | tube:drop() 206 | end) 207 | 208 | test:test('check tube existence', function(test) 209 | test:plan(14) 210 | local queue = require('queue') 211 | test:is(#queue.tube(), 0, 'checking for empty tube list') 212 | assert(#queue.tube() == 0) 213 | 214 | local tube1 = queue.create_tube('test1', 'fifo', { engine = engine }) 215 | test:is(#queue.tube(), 1, 'checking for not empty tube list') 216 | 217 | local tube2 = queue.create_tube('test2', 'fifo', { engine = engine }) 218 | test:is(#queue.tube(), 2, 'checking for not empty tube list') 219 | 220 | test:is(queue.tube('test1'), true, 'checking for tube existence') 221 | test:is(queue.tube('test2'), true, 'checking for tube existence') 222 | test:is(queue.tube('test3'), false, 'checking for tube nonexistence') 223 | 224 | tube2:drop() 225 | test:is(#queue.tube(), 1, 'checking for not empty tube list') 226 | 227 | test:is(queue.tube('test1'), true, 'checking for tube existence') 228 | test:is(queue.tube('test2'), false, 'checking for tube nonexistence') 229 | test:is(queue.tube('test3'), false, 'checking for tube nonexistence') 230 | 231 | tube1:drop() 232 | test:is(#queue.tube(), 0, 'checking for empty tube list') 233 | 234 | test:is(queue.tube('test1'), false, 'checking for tube nonexistence') 235 | test:is(queue.tube('test2'), false, 'checking for tube nonexistence') 236 | test:is(queue.tube('test3'), false, 'checking for tube nonexistence') 237 | end) 238 | 239 | tnt.finish() 240 | 241 | os.exit(test:check() and 0 or 1) 242 | -- vim: set ft=lua : 243 | -------------------------------------------------------------------------------- /t/100-limfifottl.t: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env tarantool 2 | local yaml = require('yaml') 3 | local fiber = require('fiber') 4 | 5 | local queue = require('queue') 6 | local state = require('queue.abstract.state') 7 | 8 | local engine = os.getenv('ENGINE') or 'memtx' 9 | 10 | local tnt = require('t.tnt') 11 | tnt.cfg{} 12 | 13 | local test = require('tap').test() 14 | 15 | if engine == 'memtx' then 16 | test:plan(9) 17 | 18 | test:ok(rawget(box, 'space'), 'box started') 19 | test:ok(queue, 'queue is loaded') 20 | 21 | local tube = queue.create_tube('lim3_tube', 'limfifottl', { engine = engine, capacity = 3 }) 22 | local unlim_tube = queue.create_tube('unlim_tube', 'limfifottl', { engine = engine }) 23 | 24 | test:ok(tube, 'test tube created') 25 | test:is(tube.name, 'lim3_tube', 'tube.name') 26 | test:is(tube.type, 'limfifottl', 'tube.type') 27 | 28 | test:test('Put timeout is reached', function(test) 29 | test:plan(4) 30 | 31 | test:ok(tube:put{1}, 'task 1 was put') 32 | test:ok(tube:put{2}, 'task 2 was put') 33 | test:ok(tube:put{3}, 'task 3 was put') 34 | test:is(tube:put({4}, {timeout = 0.1}), nil, 'task 4 wasn\'t put cause timeout') 35 | end) 36 | 37 | test:test('Put after freeing up space', function(test) 38 | test:plan(3) 39 | local put_fiber = fiber.create(function() 40 | test:ok(tube:put({4}, {timeout = 1}), 'task 4 was put') 41 | end) 42 | 43 | local task = tube:take() 44 | test:ok(task, 'task 3 was taken') 45 | test:is(tube:ack(task[1])[2], state.DONE, 'task 3 is done') 46 | 47 | while put_fiber:status() ~= 'dead' do 48 | fiber.sleep(.01) 49 | end 50 | end) 51 | 52 | test:test('Get current queue length', function(test) 53 | test:plan(1) 54 | test:is(tube.raw:len(), 3, 'tube length is 3') 55 | end) 56 | 57 | test:test('Unlimited tube put', function(test) 58 | test:plan(3) 59 | 60 | test:is(unlim_tube:take(0), nil, 'tube is empty') 61 | test:ok(unlim_tube:put{1}, 'task 1 was put') 62 | test:ok(unlim_tube:put{2}, 'task 2 was put') 63 | end) 64 | else 65 | test:plan(1) 66 | local ok = pcall(queue.create_tube, 'unsupported_engine', 'limfifottl', { engine = engine }) 67 | test:ok(not ok, 'vinyl engine is not allowed') 68 | end 69 | 70 | tnt.finish() 71 | os.exit(test:check() and 0 or 1) 72 | -- vim: set ft=lua : 73 | -------------------------------------------------------------------------------- /t/110-disconnect-trigger-check.t: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env tarantool 2 | 3 | local fiber = require('fiber') 4 | local netbox = require('net.box') 5 | local os = require('os') 6 | local queue = require('queue') 7 | local tap = require('tap') 8 | local tnt = require('t.tnt') 9 | 10 | local tube 11 | local test = tap.test('lost a session id after yield') 12 | 13 | -- The test cases are in check_result(). 14 | test:plan(2) 15 | 16 | -- Verify that _queue_taken space is empty. 17 | local function check_result() 18 | if tube == nil then 19 | return 20 | end 21 | 22 | -- tube:drop() is most simple way to check that _queue_taken 23 | -- is empty. It give an error if it is not so. 24 | local ok, res = pcall(tube.drop, tube) 25 | test:is(ok, true, 'drop empty queue') 26 | test:is(res, true, 'tube:drop() result is true') 27 | 28 | tnt.finish() 29 | end 30 | 31 | -- Yield in queue's on_disconnect trigger (which handles a client 32 | -- disconnection) may lead to a situation when _queue_taken 33 | -- temporary space is not cleaned and becomes inconsistent with 34 | -- 'status' field in space. This appears only on 35 | -- tarantool versions affected by gh-4627. 36 | -- 37 | -- See https://github.com/tarantool/queue/issues/103 38 | -- See https://github.com/tarantool/tarantool/issues/4627 39 | local function test_lost_session_id_after_yield() 40 | -- We must check the results of a test after 41 | -- the queue._on_consumer_disconnect trigger 42 | -- has been done. 43 | -- 44 | -- Triggers are run in LIFO order. 45 | box.session.on_disconnect(check_result) 46 | 47 | local listen = 'localhost:1918' 48 | tnt.cfg{listen = listen} 49 | 50 | local driver = 'fifottl' 51 | tube = queue.create_tube('test_tube', driver, {if_not_exists = true}) 52 | 53 | rawset(_G, 'queue', require('queue')) 54 | tube:grant('guest', {call = true}) 55 | 56 | -- We need at least two tasks to trigger box.session.id() 57 | -- call after a yield in the queue._on_consumer_disconnect 58 | -- trigger (in the version of queue before the fix). See 59 | -- more below. 60 | queue.tube.test_tube:put('1') 61 | queue.tube.test_tube:put('2') 62 | local connection = netbox.connect(listen) 63 | connection:call('queue.tube.test_tube:take') 64 | connection:call('queue.tube.test_tube:take') 65 | 66 | -- After disconnection of a client the _on_consumer_disconnect 67 | -- trigger is run. It changes 'status' field for tuples in 68 | -- space in a loop and removes the corresponding 69 | -- tuples from _queue_taken space. The version before the fix 70 | -- operates in this way: 71 | -- 72 | -- | <_on_consumer_disconnect> 73 | -- | for task in tasks of the client: 74 | -- | call 75 | -- | 76 | -- | 77 | -- | delete _queue_taken tuple using box.session.id() 78 | -- | update space using task_id -- !! yield 79 | -- 80 | -- So the deletion from _queue_taken may be unable to delete 81 | -- right tuples for second and following tasks, because 82 | -- box.session.id() may give a garbage. 83 | connection:close() 84 | 85 | -- Wait for check_result() trigger, which will ensure that 86 | -- _queue_taken space is cleaned and will exit successfully 87 | -- in the case (or exit abnormally otherwise). 88 | fiber.sleep(5) 89 | 90 | -- Wrong session id may lead to 'Task was not taken in the 91 | -- session' error in the _on_consumer_disconnect and so the 92 | -- second on_disconnect trigger (check_result) will not be 93 | -- fired. 94 | os.exit(test:check() and 0 or 1) 95 | end 96 | 97 | test_lost_session_id_after_yield() 98 | -- vim: set ft=lua : 99 | -------------------------------------------------------------------------------- /t/120-take-task-after-reconnect.t: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env tarantool 2 | 3 | local fiber = require('fiber') 4 | local netbox = require('net.box') 5 | local os = require('os') 6 | local queue = require('queue') 7 | local tap = require('tap') 8 | local tnt = require('t.tnt') 9 | 10 | 11 | local test = tap.test('take a task after reconnect') 12 | test:plan(1) 13 | 14 | local listen = 'localhost:1918' 15 | tnt.cfg{listen = listen} 16 | 17 | 18 | local function test_take_task_after_disconnect(test) 19 | test:plan(1) 20 | local driver = 'fifottl' 21 | local tube = queue.create_tube('test_tube', driver, 22 | {if_not_exists = true}) 23 | rawset(_G, 'queue', require('queue')) 24 | tube:grant('guest', {call = true}) 25 | local task_id = tube:put('test_data')[1] 26 | -- Now we have one task in a ready state 27 | 28 | local connection = netbox.connect(listen) 29 | local fiber_1 = fiber.create(function() 30 | connection:call('queue.tube.test_tube:take') 31 | connection:call('queue.tube.test_tube:take') 32 | end) 33 | 34 | -- This is not a best practice but we need to use the fiber.sleep() 35 | -- (not fiber.yield()). 36 | -- Expected results from a sleep() calling: 37 | -- 1) Execute first connection:call('queue.tube.test_tube:take') 38 | -- Now one task in a taken state 39 | -- 2) Call the second connection:call('queue.tube.test_tube:take') 40 | -- and to hang the fiber_1 41 | -- 3) Start a fiber on the server side of connection which will execute 42 | -- second queue.tube.test_tube:take call and hang because the queue 43 | -- is empty 44 | fiber.sleep(0.1) 45 | 46 | connection:close() 47 | 48 | fiber.sleep(0.1) 49 | -- The taken task will be released (cause - disconnection). 50 | -- After that the fiber which waiting of a ready task (into take procedure) 51 | -- will try to take this task (before the fix). 52 | 53 | 54 | test:is(tube:peek(task_id)[2] == 'r', true, 'Task in ready state') 55 | end 56 | 57 | 58 | test:test('Don\'t take a task after disconnect', test_take_task_after_disconnect) 59 | 60 | 61 | tnt.finish() 62 | os.exit(test:check() and 0 or 1) 63 | -- vim: set ft=lua : 64 | -------------------------------------------------------------------------------- /t/130-release-all-tasks-on-start.t: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env tarantool 2 | 3 | local os = require('os') 4 | local queue = require('queue') 5 | local tap = require('tap') 6 | local tnt = require('t.tnt') 7 | 8 | local test = tap.test('release all tasks on start') 9 | test:plan(1) 10 | 11 | --- 12 | -- Accept gh-66, we need to release all taken tasks on start. 13 | -- Instead of tarantool reboot, we will additionally call queue.start() 14 | -- to simulate the reload of the module. This is not a clean enough, 15 | -- because fibers launched by the module are not cleaned up. 16 | -- See https://github.com/tarantool/queue/issues/66 17 | -- 18 | -- All tricks in this test are done by professionals, don't try 19 | -- to repeat it yourself!!! 20 | local function check_release_tasks_on_start() 21 | tnt.cfg() 22 | -- The "fifottl" driver was choosen for check gh-121. 23 | -- We cann't use opts == nil as argument for driver "release" 24 | -- method. This is the policy of the module "queue" (check 25 | -- opts in abstract.lua, instead to check inside the driver). 26 | -- See https://github.com/tarantool/queue/issues/121 27 | local driver = 'fifottl' 28 | local tube = queue.create_tube('test_tube', driver) 29 | 30 | tube:put('1') 31 | tube:put('2') 32 | tube:put('3') 33 | 34 | tube:take() 35 | tube:take() 36 | tube:take() 37 | 38 | -- Simulate the module reload. 39 | queue.start() 40 | 41 | local ready_tasks_num = queue.statistics()['test_tube']['tasks']['ready'] 42 | test:is(ready_tasks_num, 3, 'check release tasks on start') 43 | end 44 | 45 | check_release_tasks_on_start() 46 | 47 | tnt.finish() 48 | os.exit(test:check() and 0 or 1) 49 | -- vim: set ft=lua : 50 | -------------------------------------------------------------------------------- /t/140-register-driver-after-cfg.t: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env tarantool 2 | local tap = require('tap') 3 | local tnt = require('t.tnt') 4 | 5 | local test = tap.test('test driver register') 6 | test:plan(3) 7 | 8 | local mock_tube = { 9 | create_space = function() end, 10 | new = function() end 11 | } 12 | 13 | -- As opposed to 001-tube-init.t, queue initialization 14 | -- and driver registration are done after cfg(). 15 | local function check_driver_register() 16 | tnt.cfg() 17 | local queue = require('queue') 18 | queue.register_driver('mock', mock_tube) 19 | test:is(queue.driver.mock, mock_tube, 'driver has been registered') 20 | 21 | local standart_drivers = { 22 | 'fifo', 23 | 'fifottl', 24 | 'limfifottl', 25 | 'utube', 26 | 'utubettl' 27 | } 28 | local check_standart_drivers = true 29 | 30 | for _, v in pairs(standart_drivers) do 31 | if queue.driver[v] == nil then 32 | check_standart_drivers = false 33 | break 34 | end 35 | end 36 | 37 | test:ok(check_standart_drivers, 'standard drivers are defined') 38 | 39 | local res, err = pcall(queue.register_driver, 'mock', mock_tube) 40 | local check = not res and 41 | string.match(err, 'overriding registered driver') ~= nil 42 | test:ok(check, 'check a driver override failure') 43 | end 44 | 45 | check_driver_register() 46 | 47 | tnt.finish() 48 | os.exit(test:check() and 0 or 1) 49 | -- vim: set ft=lua : 50 | -------------------------------------------------------------------------------- /t/150-lazy-start.t: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env tarantool 2 | local tap = require('tap') 3 | local tnt = require('t.tnt') 4 | 5 | local test = tap.test('test driver register') 6 | test:plan(3) 7 | 8 | local function check_lazy_start() 9 | -- Needed for bootstrap 10 | tnt.cfg{} 11 | 12 | tnt.cfg{read_only = true} 13 | local queue = require('queue') 14 | 15 | local err_msg = 'Please configure box.cfg{} in read/write mode first' 16 | local res, err = pcall(function() queue.stats() end) 17 | local check = not res and string.match(err,err_msg) ~= nil 18 | test:ok(check, 'check queue delayed start') 19 | 20 | tnt.cfg({read_only = true}) 21 | res, err = pcall(function() queue.stats() end) 22 | check = not res and string.match(err, err_msg) ~= nil 23 | test:ok(check, 'check box reconfiguration with read_only = true') 24 | 25 | tnt.cfg({read_only = false}) 26 | res = pcall(function() queue.stats() end) 27 | test:ok(res, 'queue has been started') 28 | end 29 | 30 | check_lazy_start() 31 | 32 | tnt.finish() 33 | os.exit(test:check() and 0 or 1) 34 | -- vim: set ft=lua : 35 | -------------------------------------------------------------------------------- /t/160-validate-space.t: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env tarantool 2 | 3 | local tap = require('tap') 4 | local os = require('os') 5 | local tnt = require('t.tnt') 6 | local test = tap.test('test space validation') 7 | 8 | local fifo = require('queue.abstract.driver.fifo') 9 | local fifottl = require('queue.abstract.driver.fifottl') 10 | local utube = require('queue.abstract.driver.utube') 11 | local utubettl = require('queue.abstract.driver.utubettl') 12 | 13 | local engine = os.getenv('ENGINE') or 'memtx' 14 | 15 | test:plan(8) 16 | tnt.cfg{} 17 | 18 | local function test_corrupted_space(test, driver, indexes) 19 | test:plan(table.getn(indexes)) 20 | 21 | -- Can't drop primary key in space while secondary keys exist. 22 | -- So, drop other indexes previously. 23 | local function remove_task_id_index(space, indexes) 24 | for _, index in pairs(indexes) do 25 | if index ~= 'task_id' then 26 | space.index[index]:drop() 27 | end 28 | end 29 | space.index.task_id:drop() 30 | end 31 | 32 | for _, index in pairs(indexes) do 33 | test:test(index .. ' index does not exist', function(test) 34 | test:plan(2) 35 | 36 | local space = driver.create_space('corrupted_space', 37 | {engine = engine}) 38 | 39 | if index == 'task_id' then 40 | remove_task_id_index(space, indexes) 41 | else 42 | space.index[index]:drop() 43 | end 44 | 45 | local res, err = pcall(driver.new, space) 46 | local err_match_msg = string.format('space "corrupted_space"' .. 47 | ' does not have "%s" index', index) 48 | test:ok(not res, 'exception was thrown') 49 | test:ok(err:match(err_match_msg) ~= nil, 'text of exception') 50 | 51 | space:drop() 52 | end) 53 | end 54 | end 55 | 56 | local function test_name_conflict(test, driver) 57 | test:plan(2) 58 | 59 | local conflict_space = box.schema.create_space('conflict_tube') 60 | local res, err = pcall(driver.create_space,'conflict_tube', 61 | {engine = engine, if_not_exists = true}) 62 | 63 | test:ok(not res, 'exception was thrown') 64 | test:ok(err:match('space "conflict_tube" does not' .. 65 | ' have "task_id" index') ~= nil, 'text of exception') 66 | 67 | conflict_space:drop() 68 | end 69 | 70 | test:test('test corrupted space fifo', function(test) 71 | test_corrupted_space(test, fifo, {'task_id', 'status'}) 72 | end) 73 | 74 | test:test('test corrupted space fifottl', function(test) 75 | test_corrupted_space(test, fifottl, {'task_id', 'status', 'watch'}) 76 | end) 77 | 78 | test:test('test corrupted space utube', function(test) 79 | test_corrupted_space(test, utube, {'task_id', 'status', 'utube'}) 80 | end) 81 | 82 | test:test('test corrupted space utubettl', function(test) 83 | test_corrupted_space(test, utubettl, 84 | {'task_id', 'status', 'utube', 'watch', 'utube_pri'}) 85 | end) 86 | 87 | test:test('Space name conflict fifo', function(test) 88 | test_name_conflict(test, fifo) 89 | end) 90 | 91 | test:test('Space name conflict fifo', function(test) 92 | local fifo = require('queue.abstract.driver.fifo') 93 | test_name_conflict(test, fifottl) 94 | end) 95 | 96 | test:test('Space name conflict fifo', function(test) 97 | local fifo = require('queue.abstract.driver.fifo') 98 | test_name_conflict(test, utube) 99 | end) 100 | 101 | test:test('Space name conflict fifo', function(test) 102 | local fifo = require('queue.abstract.driver.fifo') 103 | test_name_conflict(test, utubettl) 104 | end) 105 | 106 | tnt.finish() 107 | os.exit(test:check() and 0 or 1) 108 | -- vim: set ft=lua : 109 | -------------------------------------------------------------------------------- /t/170-register-driver-after-reload.t: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env tarantool 2 | 3 | local os = require('os') 4 | local queue = require('queue') 5 | local tap = require('tap') 6 | local tnt = require('t.tnt') 7 | 8 | local test = tap.test('custom driver registration after reload') 9 | test:plan(1) 10 | 11 | tnt.cfg() 12 | 13 | --- Accept gh-137, we need to check custom driver registration 14 | -- after restart. Instead of tarantool reboot, we will additionally 15 | -- call queue.start() to simulate the reload of the module. This 16 | -- is not a clean enough, because queue module doesn't provide the 17 | -- hot restart. 18 | -- 19 | -- All tricks in this test are done by professionals, don't try 20 | -- to repeat it yourself!!! 21 | local function check_driver_registration_after_reload() 22 | local fifo = require('queue.abstract.driver.fifo') 23 | queue.register_driver('fifo_cust', fifo) 24 | 25 | local tube = queue.create_tube('tube_cust', 'fifo_cust') 26 | tube:put('1') 27 | local task_id = tube:take()[1] 28 | 29 | -- Simulate the module reload. 30 | queue.driver.fifo_cust = nil 31 | queue.start() 32 | 33 | -- Check the task has been released after reload. 34 | queue.register_driver('fifo_cust', fifo) 35 | local task_status = queue.tube.tube_cust:peek(task_id)[2] 36 | test:is(task_status, 'r', 'check driver registration after reload') 37 | end 38 | 39 | check_driver_registration_after_reload() 40 | 41 | tnt.finish() 42 | os.exit(test:check() and 0 or 1) 43 | -- vim: set ft=lua : 44 | -------------------------------------------------------------------------------- /t/180-work-with-uuid.t: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env tarantool 2 | 3 | local fiber = require('fiber') 4 | local netbox = require('net.box') 5 | local tap = require('tap') 6 | local os = require('os') 7 | local tnt = require('t.tnt') 8 | 9 | local test = tap.test('test work with uuid') 10 | test:plan(4) 11 | 12 | local listen = 'localhost:1918' 13 | tnt.cfg{ listen = listen } 14 | 15 | rawset(_G, 'queue', require('queue')) 16 | 17 | test:test('test UUID format validation', function(test) 18 | test:plan(1) 19 | 20 | -- We need to call the `grant()` to have sufficient rights 21 | -- to call `identify.` 22 | local tube = queue.create_tube('test_tube', 'fifo') 23 | tube:grant('guest', { call = true }) 24 | 25 | local conn = netbox.connect(listen) 26 | local invalid_uuid = 'just-for-fun' 27 | local ok, err = pcall(conn.call, conn, 'queue.identify', {invalid_uuid}) 28 | 29 | local check_validation = not ok and err:match('Invalid UUID format.') 30 | test:ok(check_validation, 'UUID format validation has been checked.') 31 | conn:close() 32 | 33 | tube:drop() 34 | end) 35 | 36 | test:test('test work with two consumers with the same uuid', function(test) 37 | test:plan(4) 38 | 39 | -- Preparing of the tube. 40 | local tube = queue.create_tube('test_tube', 'fifo') 41 | tube:grant('guest', { call = true }) 42 | local uuid_con1 43 | local uuid_con2 44 | tube:put('test_data') 45 | 46 | local cond = fiber.cond() 47 | 48 | local fiber_consumer_1 = fiber.new(function() 49 | local conn = netbox.connect(listen) 50 | local task = conn:call('queue.tube.test_tube:take') 51 | local task_id = task[1] 52 | uuid_con1 = conn:call('queue.identify') 53 | 54 | -- Wait until consumer 2 connects to the session. 55 | cond:signal() 56 | cond:wait() 57 | 58 | -- Reconnect to the session. 59 | conn:close() 60 | conn = netbox.connect(listen) 61 | uuid_con1 = conn:call('queue.identify', {uuid_con2}) 62 | test:ok(uuid_con1 == uuid_con2, 63 | 'reconnection of consumer 1 has been completed') 64 | 65 | -- Ack the task and close the connection. 66 | task = conn:call('queue.tube.test_tube:ack', {task_id}) 67 | test:ok(task[1] == task_id and task[2] == '-', 'task has been acked') 68 | conn:close() 69 | 70 | -- Wakeup the consumer 2 fiber and wait for it to finishes. 71 | cond:signal() 72 | cond:wait() 73 | end) 74 | 75 | fiber_consumer_1:set_joinable(true) 76 | 77 | local fiber_consumer_2 = fiber.create(function() 78 | -- Waiting for consumer 1 identification 79 | cond:wait() 80 | 81 | -- Connect to server and connect to the same session as consumer 1. 82 | local conn = netbox.connect(listen) 83 | uuid_con2 = conn:call('queue.identify', {uuid_con1}) 84 | test:ok(uuid_con1 == uuid_con2, 85 | 'consumer 2 identification completed correctly') 86 | 87 | -- Wait until consumer 1 will reconnect and "ack" the task. 88 | cond:signal() 89 | cond:wait() 90 | 91 | -- Close the connection and wakeup the consumer 1 fiber. 92 | conn:close() 93 | cond:signal() 94 | end) 95 | 96 | -- Wait for consumers fibers to finishes. 97 | local ok = fiber_consumer_1:join() 98 | test:ok(ok, 'reconnection test done') 99 | 100 | tube:drop() 101 | end) 102 | 103 | test:test('test reconnect and ack the task', function(test) 104 | test:plan(2) 105 | 106 | -- Preparing of the tube. 107 | queue.cfg({ ttr = 60 }) 108 | local tube = queue.create_tube('test_tube', 'fifo') 109 | tube:put('test_data') 110 | tube:grant('guest', { call = true }) 111 | 112 | local fiber_consumer = fiber.new(function() 113 | -- Connect and take a task. 114 | local conn = netbox.connect(listen) 115 | local task = conn:call('queue.tube.test_tube:take') 116 | local task_id = task[1] 117 | local session_id = conn:call('queue.identify') 118 | conn:close() 119 | 120 | -- Reconnect and ack the task. 121 | conn = netbox.connect(listen) 122 | conn:call('queue.identify', {session_id}) 123 | task = conn:call('queue.tube.test_tube:ack', {task_id}) 124 | test:ok(task[1] == task_id and task[2] == '-', 'task has been acked') 125 | 126 | conn:close() 127 | end) 128 | 129 | fiber_consumer:set_joinable(true) 130 | 131 | local ok = fiber_consumer:join() 132 | test:ok(ok, 'reconnect and ack the task test done') 133 | 134 | tube:drop() 135 | end) 136 | 137 | test:test('test expiration', function(test) 138 | test:plan(5) 139 | 140 | -- Preparing of the tube. 141 | local tube = queue.create_tube('test_tube', 'fifo') 142 | tube:put('test_data') 143 | tube:grant('guest', { call = true }) 144 | 145 | local fiber_consumer = fiber.new(function() 146 | queue.cfg({ ttr = 1 }) 147 | -- Connect and take a task. 148 | local conn = netbox.connect(listen) 149 | local task_id = conn:call('queue.tube.test_tube:take')[1] 150 | local uuid_con = conn:call('queue.identify') 151 | conn:close() 152 | 153 | -- Check that the task is in a "taken" state before ttr expires. 154 | fiber.sleep(0.1) 155 | test:ok(tube:peek(task_id)[2] == 't', 'task in taken state') 156 | fiber.sleep(2) 157 | 158 | -- The task must be released after the ttr expires. 159 | test:ok(tube:peek(task_id)[2] == 'r', 'task in ready state after ttr') 160 | 161 | -- The old queue session must expire. 162 | conn = netbox.connect(listen) 163 | local ok, err = pcall(conn.call, conn, 'queue.identify', 164 | {uuid_con}) 165 | local check_ident = not ok and err:match('UUID .* is unknown.') 166 | test:ok(check_ident, 'the old queue session has expired.') 167 | conn:close() 168 | 169 | -- If ttr = 0, the task should be released immediately. 170 | queue.cfg({ ttr = 0 }) 171 | conn = netbox.connect(listen) 172 | task_id = conn:call('queue.tube.test_tube:take')[1] 173 | conn:close() 174 | fiber.sleep(0.1) 175 | test:ok(tube:peek(task_id)[2] == 'r', 176 | 'task has been immediately released') 177 | end) 178 | 179 | fiber_consumer:set_joinable(true) 180 | 181 | local ok = fiber_consumer:join() 182 | test:ok(ok, 'expiration test done') 183 | 184 | tube:drop() 185 | end) 186 | 187 | tnt.finish() 188 | os.exit(test:check() and 0 or 1) 189 | -- vim: set ft=lua : 190 | -------------------------------------------------------------------------------- /t/190-work-with-ttl-buried-task.t: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env tarantool 2 | 3 | local os = require('os') 4 | 5 | local fiber = require('fiber') 6 | 7 | local queue = require('queue') 8 | local state = require('queue.abstract.state') 9 | 10 | local tap = require('tap') 11 | local tnt = require('t.tnt') 12 | 13 | local test = tap.test('work with "ttl" of buried task.') 14 | test:plan(1) 15 | 16 | -- Fields in the task tuple. 17 | local TASK_ID = 1 18 | local TASK_STATE = 2 19 | 20 | tnt.cfg{} 21 | 22 | test:test('test work with "ttl", when "bury" after "take"', function(test) 23 | -- Before the patch if a task has been "buried" after it was "taken" 24 | -- (and the task has "ttr") when the time in `i_next_event` will be 25 | -- interpreted as "ttl" in `{fifottl,utubettl}_fiber_iteration` and 26 | -- the task will be deleted. 27 | local drivers = {'fifottl', 'utubettl'} 28 | test:plan(3 * table.getn(drivers)) 29 | 30 | local TTR = 0.2 31 | local TTL = 1 32 | 33 | for _, driver in pairs(drivers) do 34 | local tube = queue.create_tube('test_tube', driver, {if_not_exists = true}) 35 | local task = tube:put('task1', {ttl = TTL, ttr = TTR}) 36 | 37 | -- "Take" a task and "bury" it. 38 | task = tube:take(0) 39 | local id = task[TASK_ID] 40 | tube:bury(id) 41 | 42 | -- Check status of the task. 43 | task = tube:peek(id) 44 | test:is(task[TASK_STATE], state.BURIED, 45 | ('task "buried", driver: "%s"'):format(driver)) 46 | 47 | -- Check status of the task after "ttr" has expired. 48 | fiber.sleep(TTR * 2) 49 | task = tube:peek(id) 50 | test:is(task[TASK_STATE], state.BURIED, 51 | ('task is still "buried", driver: "%s"'):format(driver)) 52 | 53 | -- Check status of the task after "ttl" has expired. 54 | fiber.sleep(TTL * 2) 55 | local ok, res = pcall(tube.peek, tube, id) 56 | test:ok(res:match(string.format('Task %d not found', id)), 57 | ('task done, driver: "%s"'):format(driver)) 58 | 59 | tube:drop() 60 | end 61 | end) 62 | 63 | tnt.finish() 64 | os.exit(test:check() and 0 or 1) 65 | 66 | -- vim: set ft=lua : 67 | -------------------------------------------------------------------------------- /t/200-master-replica.t: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env tarantool 2 | 3 | local fio = require('fio') 4 | local log = require('log') 5 | local tnt = require('t.tnt') 6 | local test = require('tap').test('') 7 | local uuid = require('uuid') 8 | local queue = require('queue') 9 | local fiber = require('fiber') 10 | 11 | local session = require('queue.abstract.queue_session') 12 | local queue_state = require('queue.abstract.queue_state') 13 | rawset(_G, 'queue', require('queue')) 14 | 15 | local qc = require('queue.compat') 16 | if not qc.check_version({2, 4, 1}) then 17 | log.info('Tests skipped, tarantool version < 2.4.1') 18 | return 19 | end 20 | 21 | -- Replica connection handler. 22 | local conn = {} 23 | 24 | test:plan(8) 25 | 26 | test:test('Check master-replica setup', function(test) 27 | test:plan(9) 28 | local engine = os.getenv('ENGINE') or 'memtx' 29 | tnt.cluster.cfg{} 30 | 31 | test:ok(rawget(box, 'space'), 'box started') 32 | test:ok(queue, 'queue is loaded') 33 | 34 | test:ok(tnt.cluster.wait_replica(), 'wait for replica to connect') 35 | conn = tnt.cluster.connect_replica() 36 | test:ok(conn.error == nil, 'no errors on connect to replica') 37 | test:ok(conn:ping(), 'ping replica') 38 | test:is(queue.state(), 'RUNNING', 'check master queue state') 39 | conn:eval('rawset(_G, "queue", require("queue"))') 40 | test:is(conn:call('queue.state'), 'INIT', 'check replica queue state') 41 | 42 | -- Setup tube. Set ttr = 0.5 for sessions expire testing. 43 | conn:call('queue.cfg', {{ttr = 0.5, in_replicaset = true}}) 44 | test:isnil(conn:call('queue.create_tube', {'test', 'fifo'}), 45 | 'check api call in INIT state') 46 | queue.cfg{ttr = 0.5, in_replicaset = true} 47 | local tube = queue.create_tube('test', 'fifo', {engine = engine}) 48 | test:ok(tube, 'test tube created') 49 | end) 50 | 51 | test:test('Check queue state switching', function(test) 52 | test:plan(4) 53 | box.cfg{read_only = true} 54 | test:ok(queue_state.poll(queue_state.states.WAITING, 10), 55 | "queue state changed to waiting") 56 | test:is(session.expiration_fiber:status(), 'dead', 57 | "check that session expiration fiber is canceled") 58 | box.cfg{read_only = false} 59 | test:ok(queue_state.poll(queue_state.states.RUNNING, 10), 60 | "queue state changed to running") 61 | test:is(session.expiration_fiber:status(), 'suspended', 62 | "check that session expiration fiber started") 63 | end) 64 | 65 | test:test('Check session resuming', function(test) 66 | test:plan(16) 67 | local client = tnt.cluster.connect_master() 68 | test:ok(client.error == nil, 'no errors on client connect to master') 69 | local session_uuid = client:call('queue.identify') 70 | local uuid_obj = uuid.frombin(session_uuid) 71 | 72 | test:ok(queue.tube.test:put('testdata'), 'put task') 73 | local task_master = client:call('queue.tube.test:take') 74 | test:ok(task_master, 'task was taken') 75 | test:is(task_master[3], 'testdata', 'task.data') 76 | 77 | local qt = box.space._queue_taken_2:select() 78 | test:is(uuid.frombin(qt[1][4]):str(), uuid_obj:str(), 79 | 'task taken by actual uuid') 80 | 81 | -- Switch roles. 82 | box.cfg{read_only = true} 83 | queue_state.poll(queue_state.states.WAITING, 10) 84 | test:is(queue.state(), 'WAITING', 'master state is waiting') 85 | conn:eval('box.cfg{read_only=false}') 86 | conn:eval([[ 87 | queue_state = require('queue.abstract.queue_state') 88 | queue_state.poll(queue_state.states.RUNNING, 10) 89 | ]]) 90 | test:is(conn:call('queue.state'), 'RUNNING', 'replica state is running') 91 | 92 | local cfg = conn:eval('return queue.cfg') 93 | test:is(cfg.ttr, 0.5, 'check cfg applied after lazy start') 94 | 95 | test:ok(conn:call('queue.identify', {session_uuid}), 'identify old session') 96 | local stat = conn:call('queue.statistics') 97 | test:is(stat.test.tasks.taken, 1, 'taken tasks count') 98 | test:is(stat.test.tasks.done, 0, 'done tasks count') 99 | local task_replica = conn:call('queue.tube.test:ack', {task_master[1]}) 100 | test:is(task_replica[3], 'testdata', 'check task data') 101 | local stat = conn:call('queue.statistics') 102 | test:is(stat.test.tasks.taken, 0, 'taken tasks count after ack()') 103 | test:is(stat.test.tasks.done, 1, 'done tasks count after ack()') 104 | 105 | -- Switch roles back. 106 | conn:eval('box.cfg{read_only=true}') 107 | conn:eval([[ 108 | queue_state = require('queue.abstract.queue_state') 109 | queue_state.poll(queue_state.states.WAITING, 10) 110 | ]]) 111 | box.cfg{read_only = false} 112 | queue_state.poll(queue_state.states.RUNNING, 10) 113 | test:is(queue.state(), 'RUNNING', 'master state is running') 114 | test:is(conn:call('queue.state'), 'WAITING', 'replica state is waiting') 115 | client:close() 116 | end) 117 | 118 | test:test('Check session resuming (client disconnected)', function(test) 119 | test:plan(17) 120 | local client = tnt.cluster.connect_master() 121 | test:ok(client.error == nil, 'no errors on client connect to master') 122 | local session_uuid = client:call('queue.identify') 123 | local uuid_obj = uuid.frombin(session_uuid) 124 | 125 | test:ok(queue.tube.test:put('testdata'), 'put task') 126 | local task_master = client:call('queue.tube.test:take') 127 | test:ok(task_master, 'task was taken') 128 | test:is(task_master[3], 'testdata', 'task.data') 129 | client:close() 130 | 131 | local qt = box.space._queue_taken_2:select() 132 | test:is(uuid.frombin(qt[1][4]):str(), uuid_obj:str(), 133 | 'task taken by actual uuid') 134 | 135 | -- Wait for disconnect callback. 136 | local attempts = 0 137 | while true do 138 | local tuple = box.space._queue_shared_sessions:get(session_uuid) 139 | 140 | if tuple then 141 | test:is(uuid.frombin(tuple[1]):str(), uuid_obj:str(), 142 | 'check inactive sessions') 143 | break 144 | end 145 | 146 | attempts = attempts + 1 147 | if attempts == 10 then 148 | test:ok(false, 'check inactive sessions') 149 | return false 150 | end 151 | fiber.sleep(0.01) 152 | end 153 | 154 | -- Switch roles. 155 | box.cfg{read_only = true} 156 | queue_state.poll(queue_state.states.WAITING, 10) 157 | test:is(queue.state(), 'WAITING', 'master state is waiting') 158 | conn:eval('box.cfg{read_only=false}') 159 | conn:eval([[ 160 | queue_state = require('queue.abstract.queue_state') 161 | queue_state.poll(queue_state.states.RUNNING, 10) 162 | ]]) 163 | test:is(conn:call('queue.state'), 'RUNNING', 'replica state is running') 164 | 165 | local cfg = conn:eval('return queue.cfg') 166 | test:is(cfg.ttr, 0.5, 'check cfg applied after lazy start') 167 | 168 | test:ok(conn:call('queue.identify', {session_uuid}), 'identify old session') 169 | local stat = conn:call('queue.statistics') 170 | test:is(stat.test.tasks.taken, 1, 'taken tasks count') 171 | test:is(stat.test.tasks.done, 1, 'done tasks count') 172 | local task_replica = conn:call('queue.tube.test:ack', {task_master[1]}) 173 | test:is(task_replica[3], 'testdata', 'check task data') 174 | local stat = conn:call('queue.statistics') 175 | test:is(stat.test.tasks.taken, 0, 'taken tasks count after ack()') 176 | test:is(stat.test.tasks.done, 2, 'done tasks count after ack()') 177 | 178 | -- Switch roles back. 179 | conn:eval('box.cfg{read_only=true}') 180 | conn:eval([[ 181 | queue_state = require('queue.abstract.queue_state') 182 | queue_state.poll(queue_state.states.WAITING, 10) 183 | ]]) 184 | box.cfg{read_only = false} 185 | queue_state.poll(queue_state.states.RUNNING, 10) 186 | test:is(queue.state(), 'RUNNING', 'master state is running') 187 | test:is(conn:call('queue.state'), 'WAITING', 'replica state is waiting') 188 | end) 189 | 190 | test:test('Check task is cleaned after migrate', function(test) 191 | test:plan(8) 192 | local client = tnt.cluster.connect_master() 193 | local session_uuid = client:call('queue.identify') 194 | local uuid_obj = uuid.frombin(session_uuid) 195 | test:ok(queue.tube.test:put('testdata'), 'put task') 196 | test:ok(client:call('queue.tube.test:take'), 'take task from master') 197 | 198 | -- Switch roles. 199 | box.cfg{read_only = true} 200 | 201 | queue_state.poll(queue_state.states.WAITING, 10) 202 | test:is(queue.state(), 'WAITING', 'master state is waiting') 203 | conn:eval('box.cfg{read_only=false}') 204 | conn:eval([[ 205 | queue_state = require('queue.abstract.queue_state') 206 | queue_state.poll(queue_state.states.RUNNING, 10) 207 | ]]) 208 | test:is(conn:call('queue.state'), 'RUNNING', 'replica state is running') 209 | 210 | -- Check task. 211 | local stat = conn:call('queue.statistics') 212 | test:is(stat.test.tasks.taken, 1, 'taken tasks count before timeout') 213 | fiber.sleep(1.5) 214 | local stat = conn:call('queue.statistics') 215 | test:is(stat.test.tasks.taken, 0, 'taken tasks count after timeout') 216 | 217 | -- Switch roles back. 218 | conn:eval('box.cfg{read_only=true}') 219 | conn:eval([[ 220 | queue_state = require('queue.abstract.queue_state') 221 | queue_state.poll(queue_state.states.WAITING, 10) 222 | ]]) 223 | box.cfg{read_only = false} 224 | queue_state.poll(queue_state.states.RUNNING, 10) 225 | test:is(queue.state(), 'RUNNING', 'master state is running') 226 | test:is(conn:call('queue.state'), 'WAITING', 'replica state is waiting') 227 | client:close() 228 | end) 229 | 230 | test:test('Check task is cleaned after migrate (client disconnected)', function(test) 231 | test:plan(9) 232 | local client = tnt.cluster.connect_master() 233 | local session_uuid = client:call('queue.identify') 234 | local uuid_obj = uuid.frombin(session_uuid) 235 | test:ok(queue.tube.test:put('testdata'), 'put task') 236 | test:ok(client:call('queue.tube.test:take'), 'take task from master') 237 | client:close() 238 | 239 | -- Wait for disconnect callback. 240 | local attempts = 0 241 | while true do 242 | local tuple = box.space._queue_shared_sessions:get(session_uuid) 243 | 244 | if tuple then 245 | test:is(uuid.frombin(tuple[1]):str(), uuid_obj:str(), 246 | 'check inactive sessions') 247 | break 248 | end 249 | 250 | attempts = attempts + 1 251 | if attempts == 10 then 252 | test:ok(false, 'check inactive sessions') 253 | return false 254 | end 255 | fiber.sleep(0.01) 256 | end 257 | 258 | -- Switch roles. 259 | box.cfg{read_only = true} 260 | 261 | queue_state.poll(queue_state.states.WAITING, 10) 262 | test:is(queue.state(), 'WAITING', 'master state is waiting') 263 | conn:eval('box.cfg{read_only=false}') 264 | conn:eval([[ 265 | queue_state = require('queue.abstract.queue_state') 266 | queue_state.poll(queue_state.states.RUNNING, 10) 267 | ]]) 268 | test:is(conn:call('queue.state'), 'RUNNING', 'replica state is running') 269 | 270 | -- Check task. 271 | local stat = conn:call('queue.statistics') 272 | test:is(stat.test.tasks.taken, 1, 'taken tasks count before timeout') 273 | fiber.sleep(1.5) 274 | local stat = conn:call('queue.statistics') 275 | test:is(stat.test.tasks.taken, 0, 'taken tasks count after timeout') 276 | 277 | -- Switch roles back. 278 | conn:eval('box.cfg{read_only=true}') 279 | conn:eval([[ 280 | queue_state = require('queue.abstract.queue_state') 281 | queue_state.poll(queue_state.states.WAITING, 10) 282 | ]]) 283 | box.cfg{read_only = false} 284 | queue_state.poll(queue_state.states.RUNNING, 10) 285 | test:is(queue.state(), 'RUNNING', 'master state is running') 286 | test:is(conn:call('queue.state'), 'WAITING', 'replica state is waiting') 287 | end) 288 | 289 | test:test('Check in_replicaset switching', function(test) 290 | test:plan(17) 291 | test:ok(queue.tube.test:put('testdata0'), 'put task #0') 292 | test:ok(queue.tube.test:put('testdata1'), 'put task #1') 293 | local client = tnt.cluster.connect_master() 294 | test:ok(client:call('queue.tube.test:take'), 'take task') 295 | test:ok(client:call('queue.tube.test:take'), 'take task') 296 | client:close() 297 | -- Wait for disconnect callback. 298 | local attempts = 0 299 | while true do 300 | if #box.space._queue_shared_sessions:select() == 1 then 301 | break 302 | end 303 | 304 | attempts = attempts + 1 305 | if attempts == 10 then 306 | test:ok(false, 'check inactive sessions') 307 | return false 308 | end 309 | fiber.sleep(0.01) 310 | end 311 | test:is(box.space._queue_shared_sessions.temporary, false, 312 | '_queue_shared_sessions is not temporary') 313 | test:is(box.space._queue_taken_2.temporary, false, 314 | '_queue_taken_2 is not temporary') 315 | test:is(#box.space._queue_taken_2:select(), 2, 316 | 'check _queue_taken_2 data') 317 | queue.cfg{in_replicaset = false} 318 | test:is(box.space._queue_shared_sessions.temporary, true, 319 | '_queue_shared_sessions is temporary') 320 | test:is(box.space._queue_taken_2.temporary, true, 321 | '_queue_taken_2 is temporary') 322 | test:is(#box.space._queue_taken_2:select(), 2, 323 | 'check _queue_taken_2 data') 324 | test:is(#box.space._queue_shared_sessions:select(), 1, 325 | 'check _queue_shared_sessions data') 326 | queue.cfg{in_replicaset = true} 327 | test:is(box.space._queue_shared_sessions.temporary, false, 328 | '_queue_shared_sessions is not temporary') 329 | test:is(box.space._queue_taken_2.temporary, false, 330 | '_queue_taken_2 is not temporary') 331 | test:is(#box.space._queue_taken_2:select(), 2, 332 | 'check _queue_taken_2 data') 333 | test:is(#box.space._queue_shared_sessions:select(), 1, 334 | 'check _queue_shared_sessions data') 335 | test:is(queue.statistics().test.tasks.taken, 2, 336 | 'taken tasks count before release_all') 337 | queue.tube.test:release_all() 338 | test:is(queue.statistics().test.tasks.taken, 0, 339 | 'taken tasks count after release_all') 340 | end) 341 | 342 | -- gh-202 343 | test:test('Check that tubes indexes is actual after role change', function(test) 344 | local engine = os.getenv('ENGINE') or 'memtx' 345 | test:plan(10) 346 | box.cfg{read_only = true} 347 | queue_state.poll(queue_state.states.WAITING, 10) 348 | test:is(queue.state(), 'WAITING', 'master state is waiting') 349 | conn:eval('box.cfg{read_only=false}') 350 | conn:eval([[ 351 | queue_state = require('queue.abstract.queue_state') 352 | queue_state.poll(queue_state.states.RUNNING, 10) 353 | ]]) 354 | test:is(conn:call('queue.state'), 'RUNNING', 'replica state is running') 355 | conn:eval([[queue.create_tube('repl_tube', 'fifo', {engine =]] .. engine .. [[})]]) 356 | 357 | -- Switch roles back. 358 | conn:eval('box.cfg{read_only=true}') 359 | conn:eval([[ 360 | queue_state = require('queue.abstract.queue_state') 361 | queue_state.poll(queue_state.states.WAITING, 10) 362 | ]]) 363 | box.cfg{read_only = false} 364 | queue_state.poll(queue_state.states.RUNNING, 10) 365 | test:is(queue.state(), 'RUNNING', 'master state is running') 366 | test:is(conn:call('queue.state'), 'WAITING', 'replica state is waiting') 367 | test:ok(queue.tube.repl_tube, 'repl_tube is accessible') 368 | 369 | box.cfg{read_only = true} 370 | queue_state.poll(queue_state.states.WAITING, 10) 371 | test:is(queue.state(), 'WAITING', 'master state is waiting') 372 | conn:eval('box.cfg{read_only=false}') 373 | conn:eval([[ 374 | queue_state = require('queue.abstract.queue_state') 375 | queue_state.poll(queue_state.states.RUNNING, 10) 376 | ]]) 377 | test:is(conn:call('queue.state'), 'RUNNING', 'replica state is running') 378 | conn:eval('queue.tube.repl_tube:drop()') 379 | 380 | -- Switch roles back. 381 | conn:eval('box.cfg{read_only=true}') 382 | conn:eval([[ 383 | queue_state = require('queue.abstract.queue_state') 384 | queue_state.poll(queue_state.states.WAITING, 10) 385 | ]]) 386 | box.cfg{read_only = false} 387 | queue_state.poll(queue_state.states.RUNNING, 10) 388 | test:is(queue.state(), 'RUNNING', 'master state is running') 389 | test:is(conn:call('queue.state'), 'WAITING', 'replica state is waiting') 390 | test:isnil(queue.tube.repl_tube, "repl_tube is not indexed") 391 | end) 392 | 393 | rawset(_G, 'queue', nil) 394 | conn:eval('rawset(_G, "queue", nil)') 395 | conn:close() 396 | tnt.finish() 397 | os.exit(test:check() and 0 or 1) 398 | -- vim: set ft=lua : 399 | -------------------------------------------------------------------------------- /t/210-cfg-in-replicaset.t: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env tarantool 2 | 3 | local test = require('tap').test('') 4 | local queue = require('queue') 5 | 6 | rawset(_G, 'queue', require('queue')) 7 | 8 | test:plan(2) 9 | 10 | test:test('Check in_replicaset switches', function(test) 11 | test:plan(4) 12 | local engine = os.getenv('ENGINE') or 'memtx' 13 | 14 | box.cfg({}) 15 | 16 | local status, err = pcall(queue.cfg, {in_replicaset = true}) 17 | test:ok(status, 'in_replicaset = true switched') 18 | local status, _ = pcall(queue.cfg, {in_replicaset = false}) 19 | test:ok(status, 'in_replicaset = false switched') 20 | local status, _ = pcall(queue.cfg, {in_replicaset = true}) 21 | test:ok(status, 'in_replicaset = true switched') 22 | local status, _ = pcall(queue.cfg, {in_replicaset = false}) 23 | test:ok(status, 'in_replicaset = false switched') 24 | end) 25 | 26 | test:test('Check error create temporary tube', function(test) 27 | test:plan(2) 28 | local engine = os.getenv('ENGINE') or 'memtx' 29 | 30 | box.cfg({}) 31 | 32 | queue.cfg{ttr = 0.5, in_replicaset = true} 33 | local opts = {temporary = true, engine = engine} 34 | local status, err = pcall(queue.create_tube, 'test', 'fifo', opts) 35 | test:is(status, false, 'test tube should not be created') 36 | local founded = string.find(err, 37 | 'Cannot create temporary tube in replicaset mode') 38 | test:ok(founded, 'unexpected error') 39 | end) 40 | 41 | rawset(_G, 'queue', nil) 42 | os.exit(test:check() and 0 or 1) 43 | -- vim: set ft=lua : 44 | -------------------------------------------------------------------------------- /t/220-mvcc.t: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env tarantool 2 | 3 | local qc = require('queue.compat') 4 | local log = require('log') 5 | if not qc.check_version({2, 6, 1}) then 6 | log.info('Tests skipped, tarantool version < 2.6.1') 7 | return 8 | end 9 | 10 | local fiber = require('fiber') 11 | 12 | local test = require('tap').test() 13 | test:plan(6) 14 | 15 | local queue = require('queue') 16 | 17 | local tnt = require('t.tnt') 18 | tnt.cfg{memtx_use_mvcc_engine = true} 19 | 20 | local engine = 'memtx' 21 | 22 | test:ok(rawget(box, 'space'), 'box started') 23 | test:ok(queue, 'queue is loaded') 24 | 25 | local tube = queue.create_tube('test', 'fifo', { engine = engine, temporary = false}) 26 | test:ok(tube, 'test tube created') 27 | test:is(tube.name, 'test', 'tube.name') 28 | test:is(tube.type, 'fifo', 'tube.type') 29 | 30 | --- That test checks that https://github.com/tarantool/queue/pull/211 is fixed. 31 | -- Previously trying to make parallel 'put' or 'take' calls failed with mvcc enabled. 32 | test:test('concurent put and take with mvcc', function(test) 33 | test:plan(6) 34 | -- channels are used to wait for fibers to finish 35 | -- and check results of the 'take'/'put'. 36 | local channel_put = fiber.channel(2) 37 | test:ok(channel_put, 'channel created') 38 | local channel_take = fiber.channel(2) 39 | test:ok(channel_take, 'channel created') 40 | 41 | for i = 1, 2 do 42 | fiber.create(function(i) 43 | local err = pcall(tube.put, tube, i) 44 | channel_put:put(err) 45 | end, i) 46 | end 47 | 48 | for i = 1, 2 do 49 | local res = channel_put:get(1) 50 | test:ok(res, 'task ' .. i .. ' was put') 51 | end 52 | 53 | for i = 1, 2 do 54 | fiber.create(function(i) 55 | local err = pcall(tube.take, tube) 56 | channel_take:put(err) 57 | end, i) 58 | end 59 | 60 | for i = 1, 2 do 61 | local res = channel_take:get() 62 | test:ok(res, 'task ' .. i .. ' was taken') 63 | end 64 | end) 65 | 66 | tnt.finish() 67 | os.exit(test:check() and 0 or 1) 68 | 69 | -- vim: set ft=lua: 70 | -------------------------------------------------------------------------------- /t/230-orphan-not-stalling-init.t: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env tarantool 2 | 3 | local test = require('tap').test('') 4 | local queue = require('queue') 5 | local tnt = require('t.tnt') 6 | local fio = require('fio') 7 | local fiber = require('fiber') 8 | 9 | rawset(_G, 'queue', require('queue')) 10 | 11 | local qc = require('queue.compat') 12 | if not qc.check_version({2, 10, 0}) then 13 | require('log').info('Tests skipped, tarantool version < 2.10.0 ' .. 14 | 'does not support the lazy init') 15 | return 16 | end 17 | 18 | local snapdir_optname = qc.snapdir_optname 19 | local logger_optname = qc.logger_optname 20 | 21 | test:plan(1) 22 | 23 | test:test('Check orphan mode not stalling queue', function(test) 24 | test:plan(4) 25 | local engine = os.getenv('ENGINE') or 'memtx' 26 | tnt.cluster.cfg{} 27 | 28 | local dir_replica = fio.tempdir() 29 | local cmd_replica = { 30 | arg[-1], 31 | '-e', 32 | [[ 33 | box.cfg { 34 | replication = { 35 | 'replicator:password@127.0.0.1:3399', 36 | 'replicator:password@127.0.0.1:3398', 37 | }, 38 | listen = '127.0.0.1:3396', 39 | wal_dir = ']] .. dir_replica .. '\'' .. 40 | ',' .. snapdir_optname() .. ' = \'' .. dir_replica .. '\'' .. 41 | ',' .. logger_optname() .. ' = \'' .. 42 | fio.pathjoin(dir_replica, 'tarantool.log') .. '\'' .. 43 | '}' 44 | } 45 | 46 | replica = require('popen').new(cmd_replica, { 47 | stdin = 'devnull', 48 | stdout = 'devnull', 49 | stderr = 'devnull', 50 | }) 51 | 52 | local attempts = 0 53 | -- Wait for replica to connect. 54 | while box.info.replication[3] == nil or 55 | box.info.replication[3].downstream.status ~= 'follow' do 56 | 57 | attempts = attempts + 1 58 | if attempts == 30 then 59 | error('wait for replica connection') 60 | end 61 | fiber.sleep(0.1) 62 | end 63 | 64 | local conn = require('net.box').connect('127.0.0.1:3396') 65 | 66 | conn:eval([[ 67 | box.cfg{ 68 | replication = { 69 | 'replicator:password@127.0.0.1:3399', 70 | 'replicator:password@127.0.0.1:3398', 71 | 'replicator:password@127.0.0.1:3396', 72 | }, 73 | listen = '127.0.0.1:3397', 74 | replication_connect_quorum = 4, 75 | } 76 | ]]) 77 | 78 | conn:eval('rawset(_G, "queue", require("queue"))') 79 | 80 | test:is(conn:call('queue.state'), 'INIT', 'check queue state') 81 | test:is(conn:call('box.info').ro, true, 'check read only') 82 | test:is(conn:call('box.info').ro_reason, 'orphan', 'check ro reason') 83 | 84 | conn:eval('box.cfg{replication_connect_quorum = 2}') 85 | 86 | local attempts = 0 87 | while conn:call('queue.state') ~= 'RUNNING' and attempts < 50 do 88 | fiber.sleep(0.1) 89 | attempts = attempts + 1 90 | end 91 | test:is(conn:call('queue.state'), 'RUNNING', 'check queue state after orphan') 92 | end) 93 | 94 | rawset(_G, 'queue', nil) 95 | tnt.finish() 96 | os.exit(test:check() and 0 or 1) 97 | -- vim: set ft=lua : 98 | -------------------------------------------------------------------------------- /t/benchmark/async_python.lua: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env tarantool 2 | 3 | box.cfg{listen = 3301, wal_mode='none'} 4 | box.schema.user.grant('guest', 'read,write,execute', 'universe') 5 | 6 | queue = require('queue') 7 | local fiber = require('fiber') 8 | 9 | queue.create_tube('mail_msg', 'fifottl') 10 | 11 | local producer = fiber.create(function() 12 | while true do 13 | queue.tube.mail_msg:put('1') 14 | fiber.sleep(0.2) 15 | end 16 | end) 17 | -------------------------------------------------------------------------------- /t/benchmark/async_python.py: -------------------------------------------------------------------------------- 1 | import psutil 2 | 3 | from hurry.filesize import size, alternative 4 | 5 | import gevent 6 | import gtarantool 7 | 8 | def worker(tnt, timeout): 9 | while True: 10 | try: 11 | task = tnt.call("queue.tube.mail_msg:take", (timeout, )) 12 | if task: 13 | tnt.call("queue.tube.mail_msg:ack", (task[0][0], )) 14 | except Exception as e: 15 | print "Worker connection is closed", e 16 | return 17 | 18 | def spawn_workers(workers, timeout, connection): 19 | jobs = [] 20 | for _ in range(workers): 21 | jobs.append(gevent.spawn(worker, connection, timeout)) 22 | return 23 | 24 | def find_pid_benchmark(): 25 | for proc in psutil.process_iter(): 26 | if proc.name().find('tarantool') >= 0: 27 | return proc 28 | return None 29 | 30 | def spawn_printer(): 31 | proc = find_pid_benchmark() 32 | if proc == None: 33 | raise Exception("Can't find benchmark process") 34 | tnt = gtarantool.connect("127.0.0.1", 3301) 35 | while True: 36 | try: 37 | stat = tnt.call("box.stat")[0][0] 38 | except: 39 | print "Printer connection is closed" 40 | return 41 | print "%6.2f CPU | %6s MEM [ DELETE %6d rps | INSERT %6d rps | SELECT %6d rps ]" % ( 42 | proc.cpu_percent(), size(proc.memory_info().rss, system=alternative), 43 | stat['DELETE']['rps'], 44 | stat['INSERT']['rps'], 45 | stat['SELECT']['rps'] 46 | ) 47 | gevent.sleep(1) 48 | 49 | def main(): 50 | workers = 200 51 | timeout = 2.5 52 | wcon = gtarantool.connect("127.0.0.1", 3301) 53 | spawn_workers(workers, timeout, wcon) 54 | spawn_printer() 55 | return 0 56 | 57 | if __name__ == "__main__": 58 | exit(main()) 59 | -------------------------------------------------------------------------------- /t/benchmark/busy_utubes.lua: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env tarantool 2 | 3 | local clock = require('clock') 4 | local os = require('os') 5 | local fiber = require('fiber') 6 | local queue = require('queue') 7 | 8 | -- Set the number of consumers. 9 | local consumers_count = 10 10 | -- Set the number of tasks processed by one consumer per iteration. 11 | local batch_size = 150000 12 | 13 | local barrier = fiber.cond() 14 | local wait_count = 0 15 | 16 | box.cfg() 17 | 18 | local test_queue = queue.create_tube('test_queue', 'utube', 19 | {temporary = true, storage_mode = queue.driver.utube.STORAGE_MODE_READY_BUFFER}) 20 | 21 | local function prepare_tasks() 22 | local test_data = 'test data' 23 | 24 | for i = 1, consumers_count do 25 | for _ = 1, batch_size do 26 | test_queue:put(test_data, {utube = tostring(i)}) 27 | end 28 | end 29 | end 30 | 31 | local function prepare_consumers() 32 | local consumers = {} 33 | 34 | for i = 1, consumers_count do 35 | consumers[i] = fiber.create(function() 36 | wait_count = wait_count + 1 37 | -- Wait for all consumers to start. 38 | barrier:wait() 39 | 40 | -- Ack the tasks. 41 | for _ = 1, batch_size do 42 | local task = test_queue:take() 43 | fiber.yield() 44 | test_queue:ack(task[1]) 45 | end 46 | 47 | wait_count = wait_count + 1 48 | end) 49 | end 50 | 51 | return consumers 52 | end 53 | 54 | local function multi_consumer_bench() 55 | --- Wait for all consumer fibers. 56 | local wait_all = function() 57 | while (wait_count ~= consumers_count) do 58 | fiber.yield() 59 | end 60 | wait_count = 0 61 | end 62 | 63 | fiber.set_max_slice(100) 64 | 65 | prepare_tasks() 66 | 67 | -- Wait for all consumers to start. 68 | local consumers = prepare_consumers() 69 | wait_all() 70 | 71 | -- Start timing of task confirmation. 72 | local start_ack_time = clock.proc64() 73 | barrier:broadcast() 74 | -- Wait for all tasks to be acked. 75 | wait_all() 76 | -- Complete the timing of task confirmation. 77 | local complete_time = clock.proc64() 78 | 79 | -- Print the result in milliseconds. 80 | print(string.format("Time it takes to confirm the tasks: %i ms", 81 | tonumber((complete_time - start_ack_time) / 10^6))) 82 | end 83 | 84 | -- Start benchmark. 85 | multi_consumer_bench() 86 | 87 | -- Cleanup. 88 | test_queue:drop() 89 | 90 | os.exit(0) 91 | -------------------------------------------------------------------------------- /t/benchmark/many_utubes.lua: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env tarantool 2 | 3 | local clock = require('clock') 4 | local os = require('os') 5 | local fiber = require('fiber') 6 | local queue = require('queue') 7 | 8 | -- Set the number of consumers. 9 | local consumers_count = 30000 10 | 11 | local barrier = fiber.cond() 12 | local wait_count = 0 13 | 14 | box.cfg() 15 | 16 | local test_queue = queue.create_tube('test_queue', 'utube', 17 | {temporary = true, storage_mode = queue.driver.utube.STORAGE_MODE_READY_BUFFER}) 18 | 19 | local function prepare_tasks() 20 | local test_data = 'test data' 21 | 22 | for i = 1, consumers_count do 23 | test_queue:put(test_data, {utube = tostring(i)}) 24 | end 25 | end 26 | 27 | local function prepare_consumers() 28 | local consumers = {} 29 | 30 | for i = 1, consumers_count do 31 | consumers[i] = fiber.create(function() 32 | wait_count = wait_count + 1 33 | -- Wait for all consumers to start. 34 | barrier:wait() 35 | 36 | -- Ack the task. 37 | local task = test_queue:take() 38 | test_queue:ack(task[1]) 39 | 40 | wait_count = wait_count + 1 41 | end) 42 | end 43 | 44 | return consumers 45 | end 46 | 47 | local function multi_consumer_bench() 48 | --- Wait for all consumer fibers. 49 | local wait_all = function() 50 | while (wait_count ~= consumers_count) do 51 | fiber.yield() 52 | end 53 | wait_count = 0 54 | end 55 | 56 | fiber.set_max_slice(100) 57 | 58 | -- Wait for all consumers to start. 59 | local consumers = prepare_consumers() 60 | wait_all() 61 | 62 | -- Start timing creation of tasks. 63 | local start_put_time = clock.proc64() 64 | prepare_tasks() 65 | -- Start timing of task confirmation. 66 | local start_ack_time = clock.proc64() 67 | barrier:broadcast() 68 | -- Wait for all tasks to be acked. 69 | wait_all() 70 | -- Complete the timing of task confirmation. 71 | local complete_time = clock.proc64() 72 | 73 | -- Print results in milliseconds. 74 | print(string.format("Time it takes to fill the queue: %i ms", 75 | tonumber((start_ack_time - start_put_time) / 10^6))) 76 | print(string.format("Time it takes to confirm the tasks: %i ms", 77 | tonumber((complete_time - start_ack_time) / 10^6))) 78 | end 79 | 80 | -- Start benchmark. 81 | multi_consumer_bench() 82 | 83 | -- Cleanup. 84 | test_queue:drop() 85 | 86 | os.exit(0) 87 | -------------------------------------------------------------------------------- /t/benchmark/multi_consumer_work.lua: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env tarantool 2 | 3 | local clock = require('clock') 4 | local os = require('os') 5 | local fiber = require('fiber') 6 | local queue = require('queue') 7 | 8 | -- Set the number of consumers. 9 | local consumers_count = 100 10 | -- Set the number of tasks processed by one consumer per iteration. 11 | local batch_size = 100 12 | 13 | local barrier = fiber.cond() 14 | local wait_count = 0 15 | 16 | box.cfg() 17 | 18 | local test_queue = queue.create_tube('test_queue', 'fifo', {temporary = true}) 19 | 20 | local function prepare_consumers() 21 | local consumers = {} 22 | local test_data = 'test data' 23 | 24 | for i = 1, consumers_count do 25 | consumers[i] = fiber.create(function() 26 | wait_count = wait_count + 1 27 | -- Wait for all consumers to start. 28 | barrier:wait() 29 | 30 | -- Create test tasks. 31 | for j = 1, batch_size do 32 | test_queue:put(test_data) 33 | end 34 | 35 | wait_count = wait_count + 1 36 | -- Wait for all consumers to create tasks. 37 | barrier:wait() 38 | 39 | -- Ack the tasks. 40 | for j = 1, batch_size do 41 | local task = test_queue:take() 42 | test_queue:ack(task[1]) 43 | end 44 | 45 | wait_count = wait_count + 1 46 | end) 47 | end 48 | 49 | return consumers 50 | end 51 | 52 | local function multi_consumer_bench() 53 | --- Wait for all consumer fibers. 54 | local wait_all = function() 55 | while (wait_count ~= consumers_count) do 56 | fiber.yield() 57 | end 58 | wait_count = 0 59 | end 60 | 61 | -- Wait for all consumers to start. 62 | local consumers = prepare_consumers() 63 | wait_all() 64 | 65 | -- Start timing creation of tasks. 66 | local start_put_time = clock.proc64() 67 | barrier:broadcast() 68 | -- Wait for all consumers to create tasks. 69 | wait_all() 70 | -- Complete timing creation of tasks. 71 | local start_ack_time = clock.proc64() 72 | barrier:broadcast() 73 | -- Wait for all tasks to be acked. 74 | wait_all() 75 | -- Complete the timing of task confirmation. 76 | local complete_time = clock.proc64() 77 | 78 | --Print the result in microseconds 79 | print(string.format("Time it takes to fill the queue: %i", 80 | tonumber((start_ack_time - start_put_time) / 10^3))) 81 | print(string.format("Time it takes to confirm the tasks: %i", 82 | tonumber((complete_time - start_ack_time) / 10^3))) 83 | end 84 | 85 | -- Start benchmark. 86 | multi_consumer_bench() 87 | 88 | -- Cleanup. 89 | test_queue:drop() 90 | 91 | os.exit(0) 92 | -------------------------------------------------------------------------------- /t/tnt/init.lua: -------------------------------------------------------------------------------- 1 | local fio = require('fio') 2 | local log = require('log') 3 | local yaml = require('yaml') 4 | local errno = require('errno') 5 | local fiber = require('fiber') 6 | local netbox = require('net.box') 7 | 8 | local dir = os.getenv('QUEUE_TMP') 9 | local cleanup = false 10 | 11 | local qc = require('queue.compat') 12 | local vinyl_name = qc.vinyl_name 13 | local snapdir_optname = qc.snapdir_optname 14 | local logger_optname = qc.logger_optname 15 | 16 | local bind_master = os.getenv('QUEUE_MASTER_ADDR') 17 | local bind_replica = os.getenv('QUEUE_REPLICA_ADDR') 18 | local dir_replica = nil 19 | local replica = nil 20 | 21 | if bind_master == nil then 22 | bind_master = '127.0.0.1:3398' 23 | end 24 | 25 | if bind_replica == nil then 26 | bind_replica = '127.0.0.1:3399' 27 | end 28 | 29 | if dir == nil then 30 | dir = fio.tempdir() 31 | cleanup = true 32 | end 33 | 34 | local function tnt_prepare(cfg_args) 35 | cfg_args = cfg_args or {} 36 | local files = fio.glob(fio.pathjoin(dir, '*')) 37 | for _, file in pairs(files) do 38 | if fio.basename(file) ~= 'tarantool.log' then 39 | log.info("skip removing %s", file) 40 | fio.unlink(file) 41 | end 42 | end 43 | 44 | cfg_args['wal_dir'] = dir 45 | cfg_args[snapdir_optname()] = dir 46 | cfg_args[logger_optname()] = fio.pathjoin(dir, 'tarantool.log') 47 | if vinyl_name() then 48 | local vinyl_optname = vinyl_name() .. '_dir' 49 | cfg_args[vinyl_optname] = dir 50 | end 51 | 52 | box.cfg(cfg_args) 53 | end 54 | 55 | -- Creates master and replica setup for queue states switching tests. 56 | local function tnt_cluster_prepare(cfg_args) 57 | -- Since version 2.4.1, Tarantool has the popen built-in module 58 | -- that supports execution of external programs. 59 | if not qc.check_version({2, 4, 1}) then 60 | error('this test requires tarantool >= 2.4.1') 61 | return false 62 | end 63 | 64 | -- Prepare master. 65 | cfg_args = cfg_args or {} 66 | local files = fio.glob(fio.pathjoin(dir, '*')) 67 | for _, file in pairs(files) do 68 | if fio.basename(file) ~= 'tarantool.log' then 69 | log.info("skip removing %s", file) 70 | fio.unlink(file) 71 | end 72 | end 73 | 74 | cfg_args['wal_dir'] = dir 75 | cfg_args['read_only'] = false 76 | cfg_args[snapdir_optname()] = dir 77 | cfg_args[logger_optname()] = fio.pathjoin(dir, 'tarantool.log') 78 | cfg_args['listen'] = bind_master 79 | cfg_args['replication'] = {'replicator:password@' .. bind_master} 80 | if vinyl_name() then 81 | local vinyl_optname = vinyl_name() .. '_dir' 82 | cfg_args[vinyl_optname] = dir 83 | end 84 | 85 | box.cfg(cfg_args) 86 | -- Allow guest all operations. 87 | box.schema.user.grant('guest', 'read, write, execute, create, drop', 'universe') 88 | box.schema.user.create('replicator', {password = 'password'}) 89 | box.schema.user.grant('replicator', 'replication') 90 | 91 | -- Prepare replica. 92 | dir_replica = fio.tempdir() 93 | 94 | local vinyl_opt = nil 95 | if vinyl_name() then 96 | vinyl_opt = ', ' .. vinyl_name() .. '_dir = \'' .. dir_replica .. '\'' 97 | else 98 | vinyl_opt = '' 99 | end 100 | 101 | local cmd_replica = { 102 | arg[-1], 103 | '-e', 104 | [[ 105 | box.cfg { 106 | read_only = true, 107 | replication = 'replicator:password@]] .. bind_master .. 108 | '\', listen = \'' .. bind_replica .. 109 | '\', wal_dir = \'' .. dir_replica .. 110 | '\', ' .. snapdir_optname() .. ' = \'' .. dir_replica .. 111 | '\', ' .. logger_optname() .. ' = \'' .. 112 | fio.pathjoin(dir_replica, 'tarantool.log') .. '\'' .. 113 | vinyl_opt .. 114 | '}' 115 | } 116 | 117 | replica = require('popen').new(cmd_replica, { 118 | stdin = 'devnull', 119 | stdout = 'devnull', 120 | stderr = 'devnull', 121 | }) 122 | 123 | local attempts = 0 124 | 125 | -- Wait for replica to connect. 126 | while box.info.replication[2] == nil or box.info.replication[2].downstream.status ~= 'follow' do 127 | attempts = attempts + 1 128 | if attempts == 30 then 129 | error('wait for replica connection') 130 | end 131 | fiber.sleep(0.1) 132 | end 133 | 134 | box.cfg({replication = 135 | {'replicator:password@' .. bind_replica, 136 | 'replicator:password@' .. bind_master 137 | } 138 | }) 139 | 140 | -- Wait for connect to replica. 141 | attempts = 0 142 | 143 | while box.info.replication[2].upstream.status ~= 'follow' do 144 | attempts = attempts + 1 145 | if attempts == 30 then 146 | error('wait for replica failed') 147 | end 148 | fiber.sleep(0.1) 149 | end 150 | end 151 | 152 | local function connect_replica() 153 | if not replica then 154 | return nil 155 | end 156 | 157 | return netbox.connect(bind_replica) 158 | end 159 | 160 | local function connect_master() 161 | return netbox.connect(bind_master) 162 | end 163 | 164 | -- Wait for replica to connect. 165 | local function wait_replica() 166 | local attempts = 0 167 | 168 | while true do 169 | if #box.info.replication == 2 then 170 | return true 171 | end 172 | attempts = attempts + 1 173 | if attempts == 10 then 174 | return false 175 | end 176 | fiber.sleep(0.1) 177 | end 178 | 179 | return false 180 | end 181 | 182 | return { 183 | finish = function(code) 184 | local files = fio.glob(fio.pathjoin(dir, '*')) 185 | for _, file in pairs(files) do 186 | if fio.basename(file) == 'tarantool.log' and not cleanup then 187 | log.info("skip removing %s", file) 188 | else 189 | log.info("remove %s", file) 190 | fio.unlink(file) 191 | end 192 | end 193 | if cleanup then 194 | log.info("rmdir %s", dir) 195 | fio.rmdir(dir) 196 | end 197 | if dir_replica then 198 | local files = fio.glob(fio.pathjoin(dir, '*')) 199 | for _, file in pairs(files) do 200 | log.info("remove %s", file) 201 | fio.unlink(file) 202 | end 203 | end 204 | if replica then 205 | replica:kill() 206 | replica:wait() 207 | end 208 | end, 209 | 210 | dir = function() 211 | return dir 212 | end, 213 | 214 | cleanup = function() 215 | return cleanup 216 | end, 217 | 218 | logfile = function() 219 | return fio.pathjoin(dir, 'tarantool.log') 220 | end, 221 | 222 | log = function() 223 | local fh = fio.open(fio.pathjoin(dir, 'tarantool.log'), 'O_RDONLY') 224 | if fh == nil then 225 | box.error(box.error.PROC_LUA, errno.strerror()) 226 | end 227 | 228 | local data = fh:read(16384) 229 | fh:close() 230 | return data 231 | end, 232 | 233 | cfg = tnt_prepare, 234 | cluster = { 235 | cfg = tnt_cluster_prepare, 236 | wait_replica = wait_replica, 237 | connect_replica = connect_replica, 238 | connect_master = connect_master 239 | } 240 | } 241 | --------------------------------------------------------------------------------