├── .github └── workflows │ ├── fast_testing.yaml │ ├── packaging.yml │ ├── publish.yml │ └── reusable_testing.yml ├── .gitignore ├── .luacheckrc ├── CHANGELOG.md ├── LICENSE ├── Makefile ├── README.md ├── cartridge └── roles │ └── expirationd.lua ├── debian ├── .gitignore ├── changelog ├── compat ├── control ├── copyright ├── docs ├── prebuild.sh ├── rules ├── source │ └── format └── tarantool-expirationd.install ├── doc └── ldoc │ ├── assets │ ├── ldoc.css │ └── ldoc.ltp │ └── config.ld ├── expirationd-scm-1.rockspec ├── expirationd ├── init.lua └── version.lua ├── roles └── expirationd.lua ├── rpm ├── prebuild.sh └── tarantool-expirationd.spec └── test ├── entrypoint ├── srv_base.lua └── srv_role.lua ├── helper.lua ├── helper_server.lua ├── integration ├── cartridge_role_test.lua ├── master_replica_test.lua ├── reload_test.lua ├── simple_app │ ├── config.yaml │ └── instances.yml └── tarantool_role_test.lua └── unit ├── atomic_iteration_test.lua ├── callbacks_and_delays_test.lua ├── cartridge_role_test.lua ├── cfg_test.lua ├── continue_test.lua ├── custom_index_test.lua ├── expiration_process_test.lua ├── expirationd_stats_test.lua ├── iterate_with_test.lua ├── iterator_type_test.lua ├── metrics_test.lua ├── process_while_test.lua ├── ro_test.lua ├── space_index_test.lua ├── start_key_test.lua ├── tarantool_role_test.lua ├── task_stop_test.lua ├── update_and_kill_test.lua └── version_test.lua /.github/workflows/fast_testing.yaml: -------------------------------------------------------------------------------- 1 | name: fast_testing 2 | 3 | on: 4 | push: 5 | pull_request: 6 | workflow_dispatch: 7 | 8 | jobs: 9 | linux: 10 | # We want to run on external PRs, but not on our own internal 11 | # PRs as they'll be run by the push to the branch. 12 | # 13 | # The main trick is described here: 14 | # https://github.com/Dart-Code/Dart-Code/pull/2375 15 | if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name != github.repository 16 | 17 | strategy: 18 | fail-fast: false 19 | matrix: 20 | tarantool: 21 | - '1.10' 22 | - '2.4' 23 | - '2.5' 24 | - '2.6' 25 | - '2.7' 26 | - '2.8' 27 | - '2.10' 28 | cartridge-version: 29 | - '' 30 | - '2.7.4' 31 | metrics-version: 32 | - '' 33 | - '0.10.0' 34 | - '0.11.0' 35 | - '0.13.0' 36 | coveralls: [false] 37 | include: 38 | - tarantool: '2.10' 39 | cartridge-version: '2.7.4' 40 | metrics-version: '0.13.0' 41 | coveralls: true 42 | - tarantool: 'debug-master' 43 | cartridge-version: '' 44 | metrics-version: '0.13.0' 45 | coveralls: false 46 | 47 | env: 48 | TNT_DEBUG_PATH: /home/runner/tnt-debug 49 | 50 | runs-on: ubuntu-20.04 51 | steps: 52 | - name: Install tarantool ${{ matrix.tarantool }} 53 | if: startsWith(matrix.tarantool, 'debug') != true 54 | uses: tarantool/setup-tarantool@v1 55 | with: 56 | tarantool-version: ${{ matrix.tarantool }} 57 | 58 | - name: Create variables for Tarantool ${{ matrix.tarantool }} 59 | if: startsWith(matrix.tarantool, 'debug') 60 | run: | 61 | branch=$(echo ${{ matrix.tarantool }} | cut -d- -f2) 62 | commit_hash=$(git ls-remote https://github.com/tarantool/tarantool.git --branch ${branch} | head -c 8) 63 | echo "TNT_BRANCH=${branch}" >> $GITHUB_ENV 64 | echo "VERSION_POSTFIX=-${commit_hash}" >> $GITHUB_ENV 65 | shell: bash 66 | 67 | - name: Cache tarantool build 68 | if: startsWith(matrix.tarantool, 'debug') 69 | id: cache-tnt-debug 70 | uses: actions/cache@v3 71 | with: 72 | path: ${{ env.TNT_DEBUG_PATH }} 73 | key: cache-tnt-${{ matrix.tarantool }}${{ env.VERSION_POSTFIX }} 74 | 75 | - name: Clone tarantool ${{ matrix.tarantool }} 76 | if: startsWith(matrix.tarantool, 'debug') && steps.cache-tnt-debug.outputs.cache-hit != 'true' 77 | uses: actions/checkout@v3 78 | with: 79 | repository: tarantool/tarantool 80 | ref: ${{ env.TNT_BRANCH }} 81 | path: tarantool 82 | fetch-depth: 0 83 | submodules: true 84 | 85 | - name: Build tarantool ${{ matrix.tarantool }} 86 | if: startsWith(matrix.tarantool, 'debug') && steps.cache-tnt-debug.outputs.cache-hit != 'true' 87 | run: | 88 | sudo apt-get -y install git build-essential cmake make zlib1g-dev \ 89 | libreadline-dev libncurses5-dev libssl-dev \ 90 | libunwind-dev libicu-dev python3 python3-yaml \ 91 | python3-six python3-gevent 92 | cd ${GITHUB_WORKSPACE}/tarantool 93 | mkdir build && cd build 94 | cmake .. -DCMAKE_BUILD_TYPE=Debug -DENABLE_DIST=ON 95 | make 96 | make DESTDIR=${TNT_DEBUG_PATH} install 97 | 98 | - name: Install tarantool ${{ matrix.tarantool }} 99 | if: startsWith(matrix.tarantool, 'debug') 100 | run: | 101 | sudo cp -rvP ${TNT_DEBUG_PATH}/usr/local/* /usr/local/ 102 | 103 | - name: Clone the module 104 | uses: actions/checkout@v3 105 | 106 | - name: Cache rocks 107 | uses: actions/cache@v3 108 | id: cache-rocks 109 | with: 110 | path: .rocks/ 111 | key: "cache-rocks-${{ matrix.tarantool }}${{ env.VERSION_POSTFIX }}-\ 112 | ${{ matrix.cartridge-version }}-\ 113 | ${{ matrix.metrics-version }}" 114 | 115 | - name: Setup tt 116 | run: | 117 | curl -L https://tarantool.io/release/2/installer.sh | sudo bash 118 | sudo apt install -y tt 119 | tt version 120 | 121 | - name: Install requirements 122 | run: make deps 123 | if: steps.cache-rocks.outputs.cache-hit != 'true' 124 | 125 | - name: Install metrics 126 | if: matrix.metrics-version != '' 127 | run: | 128 | tt rocks install metrics ${{ matrix.metrics-version }} 129 | 130 | - name: Install cartridge 131 | if: matrix.cartridge-version != '' 132 | run: | 133 | tt rocks install cartridge ${{ matrix.cartridge-version }} 134 | 135 | - run: echo $PWD/.rocks/bin >> $GITHUB_PATH 136 | 137 | - run: make check 138 | 139 | - run: make coverage 140 | 141 | - name: Send code coverage to coveralls.io 142 | run: make coveralls 143 | if: ${{ matrix.coveralls }} 144 | env: 145 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 146 | -------------------------------------------------------------------------------- /.github/workflows/packaging.yml: -------------------------------------------------------------------------------- 1 | name: packaging 2 | 3 | on: 4 | pull_request: 5 | workflow_dispatch: 6 | push: 7 | branches: 8 | - 'master' 9 | tags: 10 | - '*' 11 | 12 | jobs: 13 | # Run not only on tags, otherwise dependent job will skip. 14 | version-check: 15 | runs-on: ubuntu-20.04 16 | steps: 17 | - name: Check module version 18 | # We need this step to run only on push with tag. 19 | if: ${{ github.event_name == 'push' && startsWith(github.ref, 'refs/tags/') }} 20 | uses: tarantool/actions/check-module-version@master 21 | with: 22 | module-name: 'expirationd' 23 | 24 | package: 25 | runs-on: ubuntu-20.04 26 | needs: version-check 27 | 28 | strategy: 29 | fail-fast: false 30 | matrix: 31 | platform: 32 | - { os: 'debian', dist: 'stretch' } 33 | - { os: 'debian', dist: 'buster' } 34 | - { os: 'debian', dist: 'bullseye' } 35 | - { os: 'el', dist: '7' } 36 | - { os: 'el', dist: '8' } 37 | - { os: 'fedora', dist: '30' } 38 | - { os: 'fedora', dist: '31' } 39 | - { os: 'fedora', dist: '32' } 40 | - { os: 'fedora', dist: '33' } 41 | - { os: 'fedora', dist: '34' } 42 | - { os: 'fedora', dist: '35' } 43 | - { os: 'fedora', dist: '36' } 44 | - { os: 'ubuntu', dist: 'xenial' } 45 | - { os: 'ubuntu', dist: 'bionic' } 46 | - { os: 'ubuntu', dist: 'focal' } 47 | - { os: 'ubuntu', dist: 'groovy' } 48 | - { os: 'ubuntu', dist: 'jammy' } 49 | 50 | env: 51 | OS: ${{ matrix.platform.os }} 52 | DIST: ${{ matrix.platform.dist }} 53 | 54 | steps: 55 | - name: Clone the module 56 | uses: actions/checkout@v3 57 | # `actions/checkout` performs shallow clone of repo. To provide 58 | # proper version of the package to `packpack` we need to have 59 | # complete repository, otherwise it will be `0.0.1`. 60 | with: 61 | fetch-depth: 0 62 | 63 | - name: Clone the packpack tool 64 | uses: actions/checkout@v3 65 | with: 66 | repository: packpack/packpack 67 | path: packpack 68 | 69 | - name: Fetch tags 70 | # Found that Github checkout Actions pulls all the tags, but 71 | # right it deannotates the testing tag, check: 72 | # https://github.com/actions/checkout/issues/290 73 | # But we use 'git describe ..' calls w/o '--tags' flag and it 74 | # prevents us from getting the needed tag for packages version 75 | # setup. To avoid of it, let's fetch it manually, to be sure 76 | # that all tags will exist always. 77 | run: git fetch --tags -f 78 | 79 | - name: Create packages 80 | run: ./packpack/packpack 81 | 82 | - name: Deploy packages 83 | # We need this step to run only on push with tag. 84 | if: ${{ github.event_name == 'push' && startsWith(github.ref, 'refs/tags/') }} 85 | env: 86 | RWS_URL_PART: https://rws.tarantool.org/tarantool-modules 87 | RWS_AUTH: ${{ secrets.RWS_AUTH }} 88 | PRODUCT_NAME: tarantool-expirationd 89 | working-directory: build 90 | run: | 91 | CURL_CMD="curl -LfsS \ 92 | -X PUT ${RWS_URL_PART}/${OS}/${DIST} \ 93 | -u ${RWS_AUTH} \ 94 | -F product=${PRODUCT_NAME}" 95 | 96 | shopt -s nullglob 97 | for f in *.deb *.rpm *.dsc *.tar.xz *.tar.gz; do 98 | CURL_CMD+=" -F $(basename ${f})=@${f}" 99 | done 100 | 101 | echo ${CURL_CMD} 102 | 103 | ${CURL_CMD} 104 | -------------------------------------------------------------------------------- /.github/workflows/publish.yml: -------------------------------------------------------------------------------- 1 | name: publish 2 | 3 | on: 4 | push: 5 | branches: [master] 6 | tags: ['*'] 7 | 8 | jobs: 9 | version-check: 10 | # We need this job to run only on push with tag. 11 | if: ${{ github.event_name == 'push' && startsWith(github.ref, 'refs/tags/') }} 12 | runs-on: ubuntu-20.04 13 | steps: 14 | - name: Check module version 15 | uses: tarantool/actions/check-module-version@master 16 | with: 17 | module-name: 'expirationd' 18 | 19 | publish-rockspec-scm-1: 20 | if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }} 21 | runs-on: ubuntu-20.04 22 | steps: 23 | - uses: actions/checkout@v3 24 | - uses: tarantool/rocks.tarantool.org/github-action@master 25 | with: 26 | auth: ${{ secrets.ROCKS_AUTH }} 27 | files: expirationd-scm-1.rockspec 28 | 29 | publish-rockspec-tag: 30 | if: ${{ github.event_name == 'push' && startsWith(github.ref, 'refs/tags/') }} 31 | needs: version-check 32 | runs-on: ubuntu-20.04 33 | steps: 34 | - uses: actions/checkout@v3 35 | 36 | # Create a rockspec for the release. 37 | - run: printf '%s=%s\n' TAG "${GITHUB_REF##*/}" >> "${GITHUB_ENV}" 38 | - run: sed -E 39 | -e 's/branch = ".+"/tag = "${{ env.TAG }}"/g' 40 | -e 's/version = ".+"/version = "${{ env.TAG }}-1"/g' 41 | expirationd-scm-1.rockspec > expirationd-${{ env.TAG }}-1.rockspec 42 | 43 | - name: Setup tt 44 | run: | 45 | curl -L https://tarantool.io/release/2/installer.sh | sudo bash 46 | sudo apt install -y tt 47 | tt version 48 | 49 | # Create a rock for the release (.all.rock). 50 | # 51 | # `tt rocks pack ` creates 52 | # .all.rock tarball. It speeds up 53 | # `tt rocks install ` and 54 | # frees it from dependency on git. 55 | # 56 | # Don't confuse this command with 57 | # `tt rocks pack `, which creates a 58 | # source tarball (.src.rock). 59 | # 60 | # Important: Don't upload binary rocks to 61 | # rocks.tarantool.org. Lua/C modules should be packed into 62 | # .src.rock instead. See [1] for description of rock types. 63 | # 64 | # [1]: https://github.com/luarocks/luarocks/wiki/Types-of-rocks 65 | - uses: tarantool/setup-tarantool@v1 66 | with: 67 | tarantool-version: '1.10' 68 | - run: tt rocks install expirationd-${{ env.TAG }}-1.rockspec 69 | - run: tt rocks pack expirationd ${{ env.TAG }} 70 | 71 | # Upload .rockspec and .all.rock. 72 | - uses: tarantool/rocks.tarantool.org/github-action@master 73 | with: 74 | auth: ${{ secrets.ROCKS_AUTH }} 75 | files: | 76 | expirationd-${{ env.TAG }}-1.rockspec 77 | expirationd-${{ env.TAG }}-1.all.rock 78 | 79 | publish-ldoc: 80 | if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }} 81 | runs-on: ubuntu-latest 82 | steps: 83 | - name: Install Tarantool 84 | uses: tarantool/setup-tarantool@v1 85 | with: 86 | tarantool-version: 2.8 87 | 88 | - name: Clone the module 89 | uses: actions/checkout@v3 90 | 91 | - name: Cache rocks 92 | uses: actions/cache@v3 93 | id: cache-rocks 94 | with: 95 | path: .rocks/ 96 | key: cache-rocks-${{ matrix.tarantool }}-01 97 | 98 | - name: Setup tt 99 | run: | 100 | curl -L https://tarantool.io/release/2/installer.sh | sudo bash 101 | sudo apt install -y tt 102 | tt version 103 | 104 | - name: Install requirements 105 | run: make deps 106 | if: steps.cache-rocks.outputs.cache-hit != 'true' 107 | 108 | - run: echo $PWD/.rocks/bin >> $GITHUB_PATH 109 | 110 | - name: Build API documentation with LDoc 111 | run: make apidoc 112 | 113 | - name: Publish generated API documentation to GitHub Pages 114 | uses: JamesIves/github-pages-deploy-action@4.1.4 115 | with: 116 | branch: gh-pages 117 | folder: doc/apidoc 118 | -------------------------------------------------------------------------------- /.github/workflows/reusable_testing.yml: -------------------------------------------------------------------------------- 1 | name: reusable_testing 2 | 3 | on: 4 | workflow_call: 5 | inputs: 6 | artifact_name: 7 | description: The name of the Tarantool build artifact 8 | default: ubuntu-focal 9 | required: false 10 | type: string 11 | 12 | jobs: 13 | run_tests: 14 | runs-on: ubuntu-20.04 15 | steps: 16 | - name: Clone the expirationd module 17 | uses: actions/checkout@v4 18 | with: 19 | repository: ${{ github.repository_owner }}/expirationd 20 | 21 | - name: Download the Tarantool build artifact 22 | uses: actions/download-artifact@v4 23 | with: 24 | name: ${{ inputs.artifact_name }} 25 | 26 | - name: Install Tarantool 27 | # Now we're lucky: all dependencies are already installed. Check package 28 | # dependencies when migrating to other OS version. 29 | run: sudo dpkg -i tarantool_*.deb tarantool-common_*.deb tarantool-dev_*.deb 30 | 31 | - name: Setup tt 32 | run: | 33 | curl -L https://tarantool.io/release/2/installer.sh | sudo bash 34 | sudo apt install -y tt 35 | tt version 36 | 37 | - run: make deps 38 | - run: echo $PWD/.rocks/bin >> $GITHUB_PATH 39 | - run: make test 40 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *~ 2 | *.snap 3 | *.xlog 4 | *.log 5 | VERSION 6 | luacov.stats.out 7 | luacov.report.out 8 | tags 9 | .rocks 10 | doc/apidoc/ 11 | .idea 12 | *.swp 13 | -------------------------------------------------------------------------------- /.luacheckrc: -------------------------------------------------------------------------------- 1 | globals = { 2 | "box", 3 | "_TARANTOOL", 4 | } 5 | 6 | ignore = { 7 | -- Unused argument . 8 | "212/self", 9 | -- Redefining a local variable. 10 | "411", 11 | -- Shadowing a local variable. 12 | "421", 13 | -- Shadowing an upvalue. 14 | "431", 15 | -- Shadowing an upvalue argument. 16 | "432", 17 | } 18 | 19 | include_files = { 20 | '.luacheckrc', 21 | '*.rockspec', 22 | '**/*.lua', 23 | } 24 | 25 | exclude_files = { 26 | '.rocks', 27 | } 28 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | 3 | All notable changes to this project will be documented in this file. 4 | 5 | The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) 6 | and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html). 7 | 8 | ## [Unreleased] 9 | 10 | ### Added 11 | 12 | ### Changed 13 | 14 | ### Fixed 15 | 16 | ## 1.6.0 - 2024-03-25 17 | 18 | The release introduces a role for Tarantool 3.0. 19 | 20 | ### Added 21 | 22 | - Tarantool 3.0 role (#160). 23 | 24 | ### Changed 25 | 26 | - Updated the 'space_index_test.lua' to drop and recreate the test space 27 | atomically. This prevents the space access failure in the expirationd 28 | task fiber if the `space:drop` function is transactional (#157). 29 | - Updated version of `luatest` in `make deps` to 1.0.1 to support Tarantool 3.0 30 | role tests (#160). 31 | 32 | ## 1.5.0 - 2023-08-23 33 | 34 | The release adds an ability to use functions from `box.func` with the Tarantool 35 | Cartridge role. 36 | 37 | ### Added 38 | 39 | - An ability to use persistent functions in `box.func` with cartridge. A user 40 | can configure the role with persistent functions as callback for a 41 | task (#153). 42 | 43 | ## 1.4.0 - 2023-03-16 44 | 45 | The release adds `_VERSION` constant for the module. 46 | 47 | ### Added 48 | 49 | - Add versioning support (#136). 50 | 51 | ## 1.3.1 - 2023-01-17 52 | 53 | The release adds a missed ability to configure the expirationd using 54 | Tarantool Cartridge role configuration. 55 | 56 | ### Fixed 57 | 58 | - Incorrect check of the Tarantool version in tests to determine a bug in 59 | the vinyl engine that breaks the tests (#103). 60 | - There is no way to configure the module using Tarantool Cartridge role 61 | configuration (#131). 62 | 63 | ## 1.3.0 - 2022-08-11 64 | 65 | This release adds a Tarantool Cartridge role for expirationd package and 66 | improves the default behavior. 67 | 68 | ### Added 69 | 70 | - Continue a task from a last tuple (#54). 71 | - Process a task on a writable space by default (#42). 72 | - Wait until a space or an index is created (#68, #116). 73 | - Tarantool Cartridge role (#107). 74 | - Shuffle tests (#118). 75 | - GitHub Actions workflow with debug Tarantool build (#102). 76 | - GitHub Actions workflow for deploying module packages to S3 based 77 | repositories (#43). 78 | 79 | ### Changed 80 | 81 | - Decrease tarantool-checks dependency from 3.1 to 2.1 (#124). 82 | - expirationd.start() parameter `space_id` has been renamed to `space` (#112). 83 | 84 | ### Deprecated 85 | 86 | - Obsolete functions: task_stats, kill_task, get_task, get_tasks, run_task, 87 | show_task_list. 88 | 89 | ### Fixed 90 | 91 | - Do not restart a work fiber if an index does not exist (#64). 92 | - Build and installation of rpm/deb packages (#124). 93 | - test_mvcc_vinyl_tx_conflict (#104, #105). 94 | - Flaky 'simple expires test' (#90). 95 | - Changelogs. 96 | 97 | ## 1.2.0 - 2022-06-27 98 | 99 | This release adds a lot of test fixes, documentation and CI improvements. The 100 | main new feature is support of metrics package. Collecting statistics using the 101 | metrics package is enabled by default if the package metrics >= 0.11.0 102 | is installed. 103 | 104 | 4 counters will be created: 105 | 106 | 1. expirationd_checked_count 107 | 2. expirationd_expired_count 108 | 3. expirationd_restarts 109 | 4. expirationd_working_time 110 | 111 | The meaning of counters is same as for expirationd.stats(). 112 | 113 | It can be disabled using the expirationd.cfg call: 114 | 115 | ```Lua 116 | expirationd.cfg({metrics = false}) 117 | ``` 118 | 119 | ### Added 120 | 121 | - Check types of function arguments with checks module (#58). 122 | - Messages about obsolete methods. 123 | - Metrics support (#100). 124 | - Tests use new version of API. 125 | - Tests for expirationd.stats() (#77). 126 | - Gather code coverage and send report to coveralls on GitHub CI (#85). 127 | - Print engine passed to tests (#76). 128 | - Support to generate documentation using make (#79). 129 | - Note about using expirationd with replication (#14). 130 | - New target deps to Makefile that install lua dependencies (#79). 131 | - GitHub CI for publishing API documentation (#79). 132 | - Describe prerequisites and installation steps in README.md. 133 | 134 | ### Changed 135 | 136 | - Update documentation and convert to LDoc format (#60). 137 | - Update comparison table in README.md (#53). 138 | - Bump luatest version to 0.5.6. 139 | 140 | ### Fixed 141 | 142 | - Prevent iteration through a functional index for Tarantool < 2.8.4 (#101). 143 | - Processing tasks with zero length box.cfg.replication (#95). 144 | - Remove check for vinyl engine (#76). 145 | - Flakiness (#76, #90, #80). 146 | - Make iterate_with() conform to declared interface (#84). 147 | - Use default 'vinyl_memory' quota for tests (#104). 148 | - A typo in the rpm-package description. 149 | - Function name in example: 150 | function on_full_scan_complete -> function on_full_scan_error. 151 | - Incorrect description of the force option for the expirationd.start (#92, 152 | #96). 153 | 154 | ## 1.1.1 - 2021-09-13 155 | 156 | This release adds a fix for a bug with freezing on stop a task. 157 | 158 | ### Added 159 | 160 | - Enable Lua source code analysis with luacheck (#57). 161 | 162 | ### Fixed 163 | 164 | - Freezes when stopping a task (#69). 165 | 166 | ## 1.1.0 - 2021-07-06 167 | 168 | This release adds a number of features and fixes a bug. 169 | 170 | ### Added 171 | 172 | - The ability to set iteration and full scan delays for a task (#38). 173 | - Callbacks for a task at various stages of the full scan iteration (#25). 174 | - The ability to specify from where to start the iterator (option start_key) 175 | and specify the type of the iterator itself (option iterator_type). 176 | Start key can be set as a function (dynamic parameter) or just a static 177 | value. The type of the iterator can be specified either with the 178 | `box.index.*` constant, or with the name for example, 'EQ' or 179 | box.index.EQ (#50). 180 | - The ability to create a custom iterator that will be created at the selected 181 | index (option iterate_with). One can also pass a predicate that will stop the 182 | full-scan process, if required (process_while) (#50). 183 | - An option atomic_iteration that allows making only one transaction per batch 184 | option. With task:kill(), the batch with transactions will be finalized, and 185 | only after that, the fiber will complete its work (#50). 186 | 187 | ### Fixed 188 | 189 | - Worker iteration for a tree index. The bug can cause an array of tuples for a 190 | check on expiration to be obtained before suspending during the worker 191 | iteration (in case of using a tree index), and some tuples can be 192 | modified/deleted from another fiber while the worker fiber is sleeping. 193 | 194 | ## 1.0.1 - 2018-01-22 195 | 196 | First release with rockspecs. 197 | 198 | ### Added 199 | 200 | - rockspecs. 201 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (C) 2014-2021 Tarantool AUTHORS: 2 | please see AUTHORS file in tarantool/tarantool repository. 3 | 4 | /* 5 | * Redistribution and use in source and binary forms, with or 6 | * without modification, are permitted provided that the following 7 | * conditions are met: 8 | * 9 | * 1. Redistributions of source code must retain the above 10 | * copyright notice, this list of conditions and the 11 | * following disclaimer. 12 | * 13 | * 2. Redistributions in binary form must reproduce the above 14 | * copyright notice, this list of conditions and the following 15 | * disclaimer in the documentation and/or other materials 16 | * provided with the distribution. 17 | * 18 | * THIS SOFTWARE IS PROVIDED BY AUTHORS ``AS IS'' AND 19 | * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 20 | * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 21 | * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL 22 | * AUTHORS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, 23 | * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 24 | * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 | * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR 26 | * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 27 | * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 28 | * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF 29 | * THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 30 | * SUCH DAMAGE. 31 | */ 32 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | # This way everything works as expected ever for 2 | # `make -C /path/to/project` or 3 | # `make -f /path/to/project/Makefile`. 4 | MAKEFILE_PATH := $(abspath $(lastword $(MAKEFILE_LIST))) 5 | PROJECT_DIR := $(patsubst %/,%,$(dir $(MAKEFILE_PATH))) 6 | LUACOV_REPORT := $(PROJECT_DIR)/luacov.report.out 7 | LUACOV_STATS := $(PROJECT_DIR)/luacov.stats.out 8 | 9 | SHELL := $(shell which bash) # Required for brace expansion used in a clean target. 10 | SEED ?= $(shell /bin/bash -c "echo $$RANDOM") 11 | 12 | all: test 13 | 14 | # The template (ldoc.tpl) is written using tarantool specific 15 | # functions like string.split(), string.endswith(), so we run 16 | # ldoc using tarantool. 17 | apidoc: 18 | ldoc -c $(PROJECT_DIR)/doc/ldoc/config.ld \ 19 | -d $(PROJECT_DIR)/doc/apidoc/ expirationd/ 20 | 21 | check: luacheck 22 | 23 | luacheck: 24 | luacheck --config .luacheckrc --codes . 25 | 26 | .PHONY: test 27 | test: 28 | luatest -v --coverage --shuffle all:${SEED} 29 | 30 | $(LUACOV_STATS): test 31 | 32 | coverage: $(LUACOV_STATS) 33 | sed -i -e 's@'"$$(realpath .)"'/@@' $(LUACOV_STATS) 34 | cd $(PROJECT_DIR) && luacov expirationd/*.lua 35 | grep -A999 '^Summary' $(LUACOV_REPORT) 36 | 37 | coveralls: $(LUACOV_STATS) 38 | echo "Send code coverage data to the coveralls.io service" 39 | luacov-coveralls --include ^expirationd --verbose --repo-token ${GITHUB_TOKEN} 40 | 41 | deps: 42 | tt rocks install luatest 1.0.1 43 | tt rocks install luacheck 0.26.0 44 | tt rocks install luacov 0.13.0-1 45 | tt rocks install ldoc --server=https://tarantool.github.io/LDoc/ 46 | tt rocks install luacov-coveralls 0.2.3-1 --server=http://luarocks.org 47 | tt rocks make 48 | 49 | deps-full: deps 50 | tt rocks install cartridge 2.7.4 51 | tt rocks install metrics 0.13.0 52 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![Run tests](https://github.com/tarantool/expirationd/actions/workflows/fast_testing.yaml/badge.svg)](https://github.com/tarantool/expirationd/actions/workflows/fast_testing.yaml) 2 | 3 | Coverage Status 4 | 5 | 6 | # expirationd - data expiration with custom quirks. 7 | 8 | This package can turn Tarantool into a persistent memcache replacement, 9 | but is powerful enough so that your own expiration strategy can be defined. 10 | 11 | You define two functions: one takes a tuple as an input and returns 12 | true in case it's expired and false otherwise. The other takes the 13 | tuple and performs the expiry itself: either deletes it (memcache), or 14 | does something smarter, like put a smaller representation of the data 15 | being deleted into some other space. 16 | 17 | There are a number of similar modules: 18 | - [moonwalker](https://github.com/tarantool/moonwalker) triggered manually, 19 | useful for batch transactions, a performance about 600/700k rec/sec 20 | - [expirationd](https://github.com/tarantool/expirationd/issues/53) always 21 | expires tuples with using indices and using any condition, without guarantee 22 | for time expiration. 23 | - [indexpirationd](https://github.com/moonlibs/indexpiration) always expires 24 | tuples with indices, has a nice precision (up to ms) for time to expire. 25 | 26 | Table below may help you to choose a proper module for your requirements: 27 | 28 | | Module | Reaction time | Uses indices | Arbitrary condition | Expiration trigger | 29 | |---------------|---------------|--------------|---------------------|------------------------------------| 30 | | indexpiration | High (ms) | Yes | No | synchronous (fiber with condition) | 31 | | expirationd | Medium (sec) | Yes | Yes | synchronous (fiber with condition) | 32 | | moonwalker | NA | No | Yes | asynchronous (using crontab etc) | 33 | 34 | ### Prerequisites 35 | 36 | * Tarantool 1.10+ (`tarantool` package, see [documentation](https://www.tarantool.io/en/download/)). 37 | 38 | ### Installation 39 | 40 | You can: 41 | 42 | * Install the module using [tt](https://github.com/tarantool/tt): 43 | 44 | ``` bash 45 | tt rocks install expirationd 46 | ``` 47 | 48 | * Install the module using LuaRocks: 49 | 50 | ``` bash 51 | luarocks install --local --server=https://rocks.tarantool.org expirationd 52 | ``` 53 | 54 | ### Documentation 55 | 56 | See API documentation in https://tarantool.github.io/expirationd/ 57 | 58 | Note about using expirationd with replication: by default expirationd processes 59 | tasks for all types of spaces only on the writable instance. It does not 60 | process tasks on read-only instance for [non-local persistent spaces](https://www.tarantool.io/en/doc/latest/reference/configuration/#confval-read_only). 61 | It means that expirationd *will not* start task processing on a replica for 62 | regular spaces. One can force running task on replica with option `force` in 63 | `start()` module function. The option force let a user control where to start 64 | task processing and where don't. 65 | 66 | ### Examples 67 | 68 | Simple version: 69 | 70 | ```lua 71 | box.cfg{} 72 | space = box.space.old 73 | job_name = "clean_all" 74 | expirationd = require("expirationd") 75 | 76 | function is_expired(args, tuple) 77 | return true 78 | end 79 | 80 | function delete_tuple(space, args, tuple) 81 | box.space[space]:delete{tuple[1]} 82 | end 83 | 84 | expirationd.start(job_name, space.id, is_expired, { 85 | process_expired_tuple = delete_tuple, 86 | args = nil, 87 | tuples_per_iteration = 50, 88 | full_scan_time = 3600 89 | }) 90 | ``` 91 | 92 | Сustomized version: 93 | 94 | ```lua 95 | expirationd.start(job_name, space.id, is_expired, { 96 | -- name or id of the index in the specified space to iterate over 97 | index = "exp", 98 | -- one transaction per batch 99 | -- default is false 100 | atomic_iteration = true, 101 | -- delete data that was added a year ago 102 | -- default is nil 103 | start_key = function( task ) 104 | return clock.time() - (365*24*60*60) 105 | end, 106 | -- delete it from the oldest to the newest 107 | -- default is ALL 108 | iterator_type = "GE", 109 | -- stop full_scan if delete a lot 110 | -- returns true by default 111 | process_while = function( task ) 112 | if task.args.max_expired_tuples >= task.expired_tuples_count then 113 | task.expired_tuples_count = 0 114 | return false 115 | end 116 | return true 117 | end, 118 | -- this function must return an iterator over the tuples 119 | iterate_with = function( task ) 120 | return task.index:pairs({ task.start_key() }, { iterator = task.iterator_type }) 121 | :take_while( function( tuple ) 122 | return task:process_while() 123 | end ) 124 | end, 125 | args = { 126 | max_expired_tuples = 1000 127 | } 128 | }) 129 | ``` 130 | 131 | ## Testing 132 | 133 | ``` 134 | $ make deps-full 135 | $ make test 136 | ``` 137 | 138 | Regression tests running in continuous integration that uses luatest are 139 | executed in shuffle mode. It means that every time order of tests is 140 | pseudorandom with predefined seed. If tests in CI are failed it is better to 141 | reproduce these failures with the same seed: 142 | 143 | ```sh 144 | $ make SEED=1334 test 145 | luatest -v --coverage --shuffle all:1334 146 | ... 147 | ``` 148 | 149 | ## Cartridge role 150 | 151 | `cartridge.roles.expirationd` is a Tarantool Cartridge role for the expirationd 152 | package with features: 153 | 154 | * It registers expirationd as a Tarantool Cartridge service for easy access to 155 | all [API calls](https://tarantool.github.io/expirationd/#Module_functions): 156 | ```Lua 157 | local task = cartridge.service_get('expirationd').start("task_name", id, is_expired) 158 | task:kill() 159 | ``` 160 | * You could configure the expirationd role with `cfg` entry. 161 | [expirationd.cfg()](https://tarantool.github.io/expirationd/#cfg) has the 162 | same parameters with the same meaning. 163 | 164 | Be careful, values from the clusterwide configuration are applied by default 165 | to all nodes on each 166 | [apply_config()](https://www.tarantool.io/en/doc/latest/book/cartridge/cartridge_dev/). 167 | Changing the configuration manually with 168 | [expirationd.cfg()](https://tarantool.github.io/expirationd/#cfg) 169 | only affects the current node and does not update values in the clusterwide 170 | configuration. The manual change will be overwritten by a next 171 | `apply_config` call. 172 | * You can use persistent functions (i.e. created by `box.schema.func.create`). 173 | When configuring, role tries firstly get function from global namespace 174 | (`_G`) and if function was not found then role tries search in `box.func` for 175 | function with the same name. 176 | 177 | Be careful! At the moment of validating and applying config of expirationd 178 | role all persistent functions must be created before, so to configure 179 | cartridge application correctly you must do it in two steps: at the first 180 | step you have to confgure migrations with creating persistent functions and 181 | run them, at the second one put expirationd config. 182 | * The role stops all expirationd tasks on an instance on the role termination. 183 | * The role can automatically start or kill old tasks from the role 184 | configuration: 185 | 186 | ```yaml 187 | expirationd: 188 | cfg: 189 | metrics: true 190 | task_name1: 191 | space: 579 192 | is_expired: is_expired_func_name_in__G 193 | is_master_only: true 194 | options: 195 | args: 196 | - any 197 | atomic_iteration: false 198 | force: false 199 | force_allow_functional_index: true 200 | full_scan_delay: 1 201 | full_scan_time: 1 202 | index: 0 203 | iterate_with: iterate_with_func_name_in__G 204 | iteration_delay: 1 205 | iterator_type: ALL 206 | on_full_scan_complete: on_full_scan_complete_func_name_in__G 207 | on_full_scan_error: on_full_scan_error_func_name_in__G 208 | on_full_scan_start: on_full_scan_start_func_name_in__G 209 | on_full_scan_success: on_full_scan_success_func_name_in__G 210 | process_expired_tuple: process_expired_tuple_func_name_in__G 211 | process_while: process_while_func_name_in__G 212 | start_key: 213 | - 1 214 | tuples_per_iteration: 100 215 | vinyl_assumed_space_len: 100 216 | vinyl_assumed_space_len_factor: 1 217 | task_name2: 218 | ... 219 | ``` 220 | 221 | [expirationd.start()](https://tarantool.github.io/expirationd/#start) has 222 | the same parameters with the same meaning except for the additional optional 223 | param `is_master_only`. If `true`, the task should run only on a master 224 | instance. By default, the value is `false`. 225 | 226 | You need to be careful with parameters-functions. The string is a key in 227 | the global variable `_G`, the value must be a function. You need to define 228 | the key before initializing the role: 229 | 230 | ```Lua 231 | rawset(_G, "is_expired_func_name_in__G", function(args, tuple) 232 | -- code of the function 233 | end) 234 | ``` 235 | 236 | ## Tarantool 3.0 role 237 | 238 | `roles.expirationd` is a Tarantool 3.0 role for the expirationd 239 | package with the following features: 240 | 241 | * You can configure the expirationd role with `cfg` entry (check example). 242 | Cluster configuration allows to set the same parameters as 243 | in [expirationd.cfg()](https://tarantool.github.io/expirationd/#cfg) 244 | * You can use persistent functions (i.e. created by `box.schema.func.create`) 245 | for expirationd `cfg` entries. 246 | When configuring, role tries first to get a function from global namespace (`_G`) 247 | and if the function was not found then role tries to search in `box.func` 248 | for a function with the same name. 249 | If some functions from config are missing, 250 | expirationd will wait for their creation and start tasks when all of them are found. 251 | You can check logs to see what functions are missing. 252 | * The role stops all expirationd tasks on an instance on the role termination. 253 | * The role can automatically start or kill old tasks from the role 254 | configuration. 255 | 256 | ```yaml 257 | roles: [roles.expirationd] 258 | roles_cfg: 259 | roles.expirationd: 260 | cfg: 261 | metrics: true 262 | task_name1: 263 | space: users 264 | is_expired: is_expired_func_name 265 | is_master_only: true 266 | options: 267 | args: 268 | - any 269 | atomic_iteration: false 270 | force: false 271 | force_allow_functional_index: true 272 | full_scan_delay: 1 273 | full_scan_time: 1 274 | index: 0 275 | iterate_with: iterate_with_func_name_in__G 276 | iteration_delay: 1 277 | iterator_type: ALL 278 | on_full_scan_complete: on_full_scan_complete_func_name_in__G 279 | on_full_scan_error: on_full_scan_error_func_name_in__G 280 | on_full_scan_start: on_full_scan_start_func_name_in__G 281 | on_full_scan_success: on_full_scan_success_func_name_in__G 282 | process_expired_tuple: process_expired_tuple_func_name_in__G 283 | process_while: process_while_func_name_in__G 284 | start_key: 285 | - 1 286 | tuples_per_iteration: 100 287 | vinyl_assumed_space_len: 100 288 | vinyl_assumed_space_len_factor: 1 289 | ``` 290 | 291 | [expirationd.start()](https://tarantool.github.io/expirationd/#start) has 292 | the same parameters with the same meaning except for the additional optional 293 | param `is_master_only`. If `true`, the task should run only on a master 294 | instance. By default, the value is `false`. 295 | 296 | You need to be careful with function parameters. Task will not start until it 297 | finds all functions from config. You can define them in user code: 298 | 299 | ```Lua 300 | box.schema.func.create('is_expired_func_name', { 301 | body = "function(...) return true end", 302 | if_not_exists = true 303 | }) 304 | 305 | -- Or you could define a global variable. 306 | rawset(_G, "process_while_func_name_in__G", function(...) 307 | return true 308 | end) 309 | ``` 310 | 311 | -------------------------------------------------------------------------------- /cartridge/roles/expirationd.lua: -------------------------------------------------------------------------------- 1 | local expirationd = require("expirationd") 2 | local role_name = "expirationd" 3 | local started = require("cartridge.vars").new(role_name) 4 | 5 | local function dumb_fn() end 6 | 7 | local function load_function(func_name) 8 | if func_name == nil or type(func_name) ~= 'string' then 9 | return nil 10 | end 11 | 12 | local func = rawget(_G, func_name) 13 | if func == nil then 14 | if type(box.cfg) == 'function' then 15 | -- After restart `validate_config` sometimes is run before box.cfg 16 | -- call and it leads to error like `Please call box.cfg first` and 17 | -- at this moment we are not able to check real availability of 18 | -- functions in box.func. If we return nil, we fail configration 19 | -- but it is not an error actually, because we cannot do any 20 | -- checks. Thus we decided to return dumb_fn to avoid 21 | -- misconfiguration at this stage and in hope that `apply_config` 22 | -- will do real check. 23 | -- P.S. Of course it is a bad solution, but... 24 | return dumb_fn 25 | end 26 | if not box.schema.func.exists(func_name) then 27 | return nil 28 | end 29 | 30 | return function(...) 31 | return box.func[func_name]:call({...}) 32 | end 33 | end 34 | 35 | if type(func) ~= 'function' then 36 | return nil 37 | end 38 | 39 | return func 40 | end 41 | 42 | local function get_param(param_name, value, types) 43 | local types_map = { 44 | b = {type = "boolean", err = "a boolean"}, 45 | n = {type = "number", err = "a number"}, 46 | s = {type = "string", err = "a string"}, 47 | f = {type = "string", transform = load_function, err = "a function name in _G or in box.func"}, 48 | t = {type = "table", err = "a table"}, 49 | any = {err = "any type"}, 50 | } 51 | 52 | local found = false 53 | for _, t in ipairs(types) do 54 | local type_opts = types_map[t] 55 | if type_opts == nil then 56 | error(role_name .. ": unsupported type option") 57 | end 58 | if not type_opts.type or type(value) == type_opts.type then 59 | if type_opts.transform then 60 | local tmp = type_opts.transform(value) 61 | if tmp then 62 | value = tmp 63 | found = true 64 | break 65 | end 66 | else 67 | found = true 68 | break 69 | end 70 | end 71 | end 72 | 73 | if not found then 74 | local err = role_name .. ": " .. param_name .. " must be " 75 | for i, t in ipairs(types) do 76 | err = err .. types_map[t].err 77 | if i ~= #types then 78 | err = err .. " or " 79 | end 80 | end 81 | return false, err 82 | end 83 | return true, value 84 | end 85 | 86 | local function get_task_options(opts) 87 | local opts_map = { 88 | args = {"any"}, 89 | atomic_iteration = {"b"}, 90 | force = {"b"}, 91 | force_allow_functional_index = {"b"}, 92 | full_scan_delay = {"n"}, 93 | full_scan_time = {"n"}, 94 | index = {"n", "s"}, 95 | iterate_with = {"f"}, 96 | iteration_delay = {"n"}, 97 | iterator_type = {"n", "s"}, 98 | on_full_scan_complete = {"f"}, 99 | on_full_scan_error = {"f"}, 100 | on_full_scan_start = {"f"}, 101 | on_full_scan_success = {"f"}, 102 | process_expired_tuple = {"f"}, 103 | process_while = {"f"}, 104 | start_key = {"f", "t"}, 105 | tuples_per_iteration = {"n"}, 106 | vinyl_assumed_space_len_factor = {"n"}, 107 | vinyl_assumed_space_len = {"n"}, 108 | } 109 | if opts == nil then 110 | return 111 | end 112 | 113 | for opt, val in pairs(opts) do 114 | if type(opt) ~= "string" then 115 | error(role_name .. ": an option must be a string") 116 | end 117 | if opts_map[opt] == nil then 118 | error(role_name .. ": unsupported option '" .. opt .. "'") 119 | end 120 | local ok, res = get_param("options." .. opt, val, opts_map[opt]) 121 | if not ok then 122 | error(res) 123 | end 124 | opts[opt] = res 125 | end 126 | 127 | return opts 128 | end 129 | 130 | local function get_task_config(task_conf) 131 | -- setmetatable resets __newindex write protection on a copy 132 | local conf = setmetatable(table.deepcopy(task_conf), {}) 133 | local params_map = { 134 | space = {required = true, types = {"n", "s"}}, 135 | is_expired = {required = true, types = {"f"}}, 136 | is_master_only = {required = false, types = {"b"}}, 137 | options = {required = false, types = {"t"}}, 138 | } 139 | for k, _ in pairs(conf) do 140 | if type(k) ~= "string" then 141 | error(role_name .. ": param must be a string") 142 | end 143 | if params_map[k] == nil then 144 | error(role_name .. ": unsupported param " .. k) 145 | end 146 | end 147 | 148 | for param, opts in pairs(params_map) do 149 | if opts.required and conf[param] == nil then 150 | error(role_name .. ": " .. param .. " is required") 151 | end 152 | if conf[param] ~= nil then 153 | local ok, res = get_param(param, conf[param], opts.types) 154 | if not ok then 155 | error(res) 156 | end 157 | conf[param] = res 158 | end 159 | end 160 | 161 | conf.options = get_task_options(conf.options) 162 | return conf 163 | end 164 | 165 | local function get_cfg(cfg) 166 | local conf = setmetatable(table.deepcopy(cfg), {}) 167 | local params_map = { 168 | metrics = {"b"}, 169 | } 170 | 171 | for k, _ in pairs(conf) do 172 | if type(k) ~= "string" then 173 | error(role_name .. ": config option must be a string") 174 | end 175 | if params_map[k] == nil then 176 | error(role_name .. ": unsupported config option " .. k) 177 | end 178 | end 179 | 180 | for param, types in pairs(params_map) do 181 | if conf[param] ~= nil then 182 | local ok, res = get_param(param, conf[param], types) 183 | if not ok then 184 | error(res) 185 | end 186 | end 187 | end 188 | 189 | return conf 190 | end 191 | 192 | local function init() 193 | 194 | end 195 | 196 | local function validate_config(conf_new) 197 | local conf = conf_new[role_name] or {} 198 | 199 | for task_name, task_conf in pairs(conf) do 200 | local ok, res = get_param("task name", task_name, {"s"}) 201 | if not ok then 202 | error(res) 203 | end 204 | local ok, res = get_param("task params", task_conf, {"t"}) 205 | if not ok then 206 | error(res) 207 | end 208 | local ok, ret = pcall(get_task_config, task_conf) 209 | if not ok then 210 | if task_name == "cfg" then 211 | get_cfg(task_conf) 212 | else 213 | error(ret) 214 | end 215 | end 216 | end 217 | 218 | return true 219 | end 220 | 221 | local function apply_config(conf_new, opts) 222 | local conf = conf_new[role_name] or {} 223 | 224 | -- finishes tasks from an old configuration 225 | for i=#started,1,-1 do 226 | local task_name = started[i] 227 | local ok, _ = pcall(expirationd.task, task_name) 228 | if ok then 229 | if conf[task_name] then 230 | expirationd.task(task_name):stop() 231 | else 232 | expirationd.task(task_name):kill() 233 | end 234 | end 235 | table.remove(started, i) 236 | end 237 | 238 | if conf["cfg"] ~= nil then 239 | local ok = pcall(get_task_config, conf["cfg"]) 240 | if not ok then 241 | local cfg = get_cfg(conf["cfg"]) 242 | expirationd.cfg(cfg) 243 | conf["cfg"] = nil 244 | end 245 | end 246 | 247 | for task_name, task_conf in pairs(conf) do 248 | task_conf = get_task_config(task_conf) 249 | 250 | local skip = task_conf.is_master_only and not opts.is_master 251 | if not skip then 252 | local task = expirationd.start(task_name, task_conf.space, 253 | task_conf.is_expired, 254 | task_conf.options) 255 | if task == nil then 256 | error(role_name .. ": unable to start task " .. task_name) 257 | end 258 | table.insert(started, task_name) 259 | end 260 | end 261 | end 262 | 263 | local function stop() 264 | for _, task_name in pairs(expirationd.tasks()) do 265 | local task = expirationd.task(task_name) 266 | task:stop() 267 | end 268 | end 269 | 270 | return setmetatable({ 271 | role_name = role_name, 272 | init = init, 273 | validate_config = validate_config, 274 | apply_config = apply_config, 275 | stop = stop, 276 | }, { __index = expirationd }) 277 | -------------------------------------------------------------------------------- /debian/.gitignore: -------------------------------------------------------------------------------- 1 | tarantool-expirationd/ 2 | files 3 | stamp-* 4 | *.substvars 5 | *.log 6 | -------------------------------------------------------------------------------- /debian/changelog: -------------------------------------------------------------------------------- 1 | tarantool-expirationd (1.5.0-1) unstable; urgency=medium 2 | 3 | * Add an ability to use persistent functions in `box.func` with cartridge. 4 | 5 | -- Oleg Jukovec Wed, 23 Aug 2023 12:00:00 +0300 6 | 7 | tarantool-expirationd (1.4.0-1) unstable; urgency=medium 8 | 9 | * Add _VERSION constant 10 | 11 | -- Oleg Jukovec Thu, 16 Mar 2023 12:00:00 +0300 12 | 13 | tarantool-expirationd (1.3.1-1) unstable; urgency=medium 14 | 15 | * Fix check of the Tarantool version in tests to determine a bug in 16 | the vinyl engine 17 | * Add a way to configure the module using Tarantool Cartridge role 18 | configuration 19 | 20 | -- Oleg Jukovec Fri, 17 Jan 2023 12:00:00 +0300 21 | 22 | tarantool-expirationd (1.3.0-1) unstable; urgency=medium 23 | 24 | * Continue a task from a last tuple 25 | * Decrease tarantool-checks dependency from 3.1 to 2.1 26 | * Process a task on a writable space by default 27 | * Wait until a space or an index is created 28 | * Tarantool Cartridge role 29 | * Fix build and installation of rpm/deb packages 30 | * Do not restart a work fiber if an index does not exist 31 | * expirationd.start() parameter space_id has been renamed to space 32 | 33 | -- Oleg Jukovec Thu, 11 Aug 2022 12:00:00 +0300 34 | 35 | tarantool-expirationd (1.2.0-1) unstable; urgency=medium 36 | 37 | * Check types of function arguments with checks module 38 | * Add messages about obsolete methods 39 | * Add metrics support 40 | * Prevent iteration through a functional index for Tarantool < 2.8.4 41 | * Fix processing tasks with zero length box.cfg.replication 42 | * Make iterate_with() conform to declared interface 43 | * Update documentation and convert to LDoc format 44 | * Support to generate documentation using make 45 | * Update comparison table in README.md 46 | * Add note about using expirationd with replication 47 | * Fix function name in example: 48 | function on_full_scan_complete -> function on_full_scan_error 49 | * Describe prerequisites and installation steps in README.md 50 | * Bump luatest version to 0.5.6 51 | * Fix incorrect description of the force option for the expirationd.start 52 | 53 | -- Oleg Jukovec Mon, 27 Jun 2022 12:00:00 +0300 54 | 55 | tarantool-expirationd (1.1.1-1) unstable; urgency=medium 56 | 57 | * Fix freezes when stopping a task 58 | * Enable Lua source code analysis with luacheck 59 | 60 | -- Sergey Bronnikov Mon, 13 Sep 2021 12:00:00 +0300 61 | 62 | tarantool-expirationd (1.1.0-1) unstable; urgency=medium 63 | 64 | * Add the ability to set iteration and full scan delays for a task. 65 | * Add callbacks for a task at various stages of the full scan iteration. 66 | * Add the ability to specify from where to start the iterator 67 | (option start_key) and specify the type of the iterator itself 68 | (option iterator_type) 69 | * Add the ability to create a custom iterator that will be created at the 70 | selected index (option iterate_with) 71 | * Add an option atomic_iteration that allows making only one transaction per 72 | batch option 73 | * Fix worker iteration for a tree index 74 | 75 | -- Sergey Bronnikov Tue, 06 Jul 2021 12:00:00 +0300 76 | 77 | tarantool-expirationd (1.0.1-1) unstable; urgency=medium 78 | 79 | * First release with rockspecs 80 | 81 | -- Roman Tsisyk Sat, 22 Jan 2018 12:00:00 +0300 82 | 83 | tarantool-expirationd (1.0.0-1) unstable; urgency=medium 84 | 85 | * Initial release 86 | 87 | -- Roman Tsisyk Thu, 18 Feb 2016 10:11:03 +0300 88 | -------------------------------------------------------------------------------- /debian/compat: -------------------------------------------------------------------------------- 1 | 9 2 | -------------------------------------------------------------------------------- /debian/control: -------------------------------------------------------------------------------- 1 | Source: tarantool-expirationd 2 | Priority: optional 3 | Section: database 4 | Maintainer: Oleg Jukovec 5 | Build-Depends: debhelper (>= 9), 6 | tarantool (>= 1.7.4.0), 7 | # For /usr/bin/prove 8 | perl (>= 5.10.0) 9 | Standards-Version: 3.9.6 10 | Homepage: https://github.com/tarantool/expirationd 11 | Vcs-Git: git://github.com/tarantool/expirationd.git 12 | Vcs-Browser: https://github.com/tarantool/expirationd 13 | 14 | Package: tarantool-expirationd 15 | Architecture: all 16 | Depends: tarantool (>= 1.7.4.0), tarantool-checks (>= 2.1), ${misc:Depends} 17 | Description: Expiration daemon for Tarantool 18 | This package can turn Tarantool into a persistent memcache replacement, 19 | but is powerful enough so that your own expiration strategy can be defined. 20 | . 21 | You define two functions: one takes a tuple as an input and returns true in 22 | case it's expirted and false otherwise. The other takes the tuple and 23 | performs the expiry itself: either deletes it (memcache), or does something 24 | smarter, like put a smaller representation of the data being deleted into 25 | some other space. 26 | -------------------------------------------------------------------------------- /debian/copyright: -------------------------------------------------------------------------------- 1 | Format: http://www.debian.org/doc/packaging-manuals/copyright-format/1.0/ 2 | Debianized-By: Roman Tsisyk 3 | Upstream-Name: tarantool-expirationd 4 | Upstream-Contact: support@tarantool.org 5 | Source: https://github.com/tarantool/tarantool-expirationd 6 | 7 | Files: * 8 | Copyright: 2014-2021 Tarantool AUTHORS 9 | License: BSD-2-Clause 10 | Redistribution and use in source and binary forms, with or without 11 | modification, are permitted provided that the following conditions 12 | are met: 13 | 1. Redistributions of source code must retain the above copyright 14 | notice, this list of conditions and the following disclaimer. 15 | 2. Redistributions in binary form must reproduce the above copyright 16 | notice, this list of conditions and the following disclaimer in the 17 | documentation and/or other materials provided with the distribution. 18 | . 19 | THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 20 | ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 21 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 22 | ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 23 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 24 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 25 | OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 26 | HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 27 | LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 28 | OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 29 | SUCH DAMAGE. 30 | -------------------------------------------------------------------------------- /debian/docs: -------------------------------------------------------------------------------- 1 | README.md 2 | -------------------------------------------------------------------------------- /debian/prebuild.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -e -o pipefail 4 | 5 | curl -LsSf https://www.tarantool.io/release/1.10/installer.sh | sudo bash 6 | -------------------------------------------------------------------------------- /debian/rules: -------------------------------------------------------------------------------- 1 | #!/usr/bin/make -f 2 | 3 | %: 4 | dh $@ 5 | 6 | override_dh_auto_build: 7 | 8 | 9 | override_dh_auto_test: 10 | DEB_BUILD_OPTIONS=nocheck dh_auto_test 11 | -------------------------------------------------------------------------------- /debian/source/format: -------------------------------------------------------------------------------- 1 | 3.0 (quilt) 2 | -------------------------------------------------------------------------------- /debian/tarantool-expirationd.install: -------------------------------------------------------------------------------- 1 | expirationd usr/share/tarantool/ 2 | cartridge/roles/expirationd.lua usr/share/tarantool/cartridge/roles/ 3 | roles/expirationd.lua usr/share/tarantool/roles/ 4 | -------------------------------------------------------------------------------- /doc/ldoc/assets/ldoc.css: -------------------------------------------------------------------------------- 1 | body { 2 | font-size: 16px; 3 | font-family: "Open Sans", sans-serif; 4 | margin: 0; 5 | } 6 | 7 | a:link { color: #fd4c4d; } 8 | a:visited { color: #fd4c4d; } 9 | a:hover { color: #fd4c4d; } 10 | 11 | h1 { font-size:26px; font-weight: normal; } 12 | h2 { font-size:22px; font-weight: normal; } 13 | h3 { font-size:18px; font-weight: normal; } 14 | h4 { font-size:16px; font-weight: bold; } 15 | 16 | hr { 17 | height: 1px; 18 | background: #c1cce4; 19 | border: 0px; 20 | margin: 15px 0; 21 | } 22 | 23 | #content code, tt { 24 | font-family: monospace; 25 | box-sizing: border-box; 26 | background-color: #f9f2f4; 27 | color: #c7254e; 28 | border-radius: 6px; 29 | font-size: 85%; 30 | padding: 1px 0.4em; 31 | } 32 | 33 | #content h1 > code { 34 | background-color: #333333; 35 | color: white; 36 | padding: 1px 0.4em; 37 | } 38 | 39 | .parameter { 40 | font-family: monospace; 41 | font-weight: bold; 42 | color: #c7254e; 43 | } 44 | 45 | .parameter-info { 46 | color: #555555; 47 | } 48 | 49 | .type { 50 | font-style: italic 51 | } 52 | 53 | .parameter-description { 54 | display: inline-block; 55 | padding-left: 15pt; 56 | margin-bottom: 5px; 57 | } 58 | 59 | .parameter-description > p { 60 | margin-top: 0.3em; 61 | margin-bottom: 0.5em; 62 | } 63 | 64 | p.name { 65 | font-family: monospace; 66 | } 67 | 68 | #main { 69 | position: relative; 70 | } 71 | 72 | #navigation { 73 | position: absolute; 74 | color: white; 75 | background-color: #262626; 76 | border-bottom: 1px solid #d3dbec; 77 | 78 | height: 100%; 79 | width: 18em; 80 | vertical-align: top; 81 | overflow: visible; 82 | } 83 | 84 | #navigation a:link { 85 | color: white; 86 | } 87 | #navigation a:visited { 88 | color: white; 89 | } 90 | #navigation a:hover { 91 | color: white; 92 | } 93 | 94 | #navigation br { 95 | display: none; 96 | } 97 | 98 | #navigation h1 { 99 | border-bottom: 1px solid #404040; 100 | padding: 15px; 101 | margin-top: 0px; 102 | margin-bottom: 0px; 103 | } 104 | 105 | #navigation h2 { 106 | font-size: 18px; 107 | border-bottom: 1px solid #404040; 108 | padding-left: 15px; 109 | padding-right: 15px; 110 | padding-top: 10px; 111 | padding-bottom: 10px; 112 | margin-top: 30px; 113 | margin-bottom: 0px; 114 | } 115 | 116 | #content h1 { 117 | background-color: black; 118 | color: white; 119 | padding: 15px; 120 | margin: 0px; 121 | } 122 | 123 | #content > p { 124 | padding-left: 15px; 125 | } 126 | 127 | #content h2 { 128 | background-color: #f9f2f4; 129 | padding: 15px; 130 | padding-top: 15px; 131 | padding-bottom: 15px; 132 | margin-top: 0px; 133 | } 134 | 135 | #content h2 a { 136 | color: #fd4c4d; 137 | text-decoration: none; 138 | } 139 | 140 | #content h2 a:hover { 141 | text-decoration: underline; 142 | } 143 | 144 | #content h3 { 145 | font-style: italic; 146 | padding-top: 15px; 147 | padding-bottom: 4px; 148 | margin-right: 15px; 149 | margin-left: 15px; 150 | margin-bottom: 5px; 151 | border-bottom: solid 1px #bcd; 152 | } 153 | 154 | #content h4 { 155 | margin-right: 15px; 156 | margin-left: 15px; 157 | border-bottom: solid 1px #bcd; 158 | } 159 | 160 | pre { 161 | background-color: #f9f2f4; 162 | border-radius: 6px; 163 | border: 1px solid #cccccc; 164 | padding: 10px; 165 | overflow: auto; 166 | font-family: monospace; 167 | font-size: 85%; 168 | } 169 | 170 | #content ul pre.example { 171 | margin-left: 0px; 172 | } 173 | 174 | table.index { 175 | /* border: 1px #00007f; */ 176 | } 177 | table.index td { text-align: left; vertical-align: top; } 178 | 179 | #navigation ul 180 | { 181 | font-size:1em; 182 | list-style-type: none; 183 | margin: 1px 1px 10px 1px; 184 | } 185 | 186 | #navigation li { 187 | text-indent: -1em; 188 | display: block; 189 | margin: 3px 0px 0px 22px; 190 | } 191 | 192 | #navigation li li a { 193 | margin: 0px 3px 0px -1em; 194 | } 195 | 196 | #content { 197 | margin-left: 18em; 198 | } 199 | 200 | #content .function dd > p { 201 | margin-top: 0.3em; 202 | margin-bottom: 0.5em; 203 | } 204 | 205 | #content table { 206 | padding-left: 15px; 207 | padding-right: 15px; 208 | background-color: white; 209 | } 210 | 211 | #content p, #content table, #content ol, #content ul, #content dl { 212 | max-width: 900px; 213 | } 214 | 215 | table.module_list, table.function_list { 216 | border-width: 1px; 217 | border-style: solid; 218 | border-color: #cccccc; 219 | border-collapse: collapse; 220 | margin: 15px; 221 | } 222 | table.module_list td, table.function_list td { 223 | border-width: 1px; 224 | padding-left: 10px; 225 | padding-right: 10px; 226 | padding-top: 5px; 227 | padding-bottom: 5px; 228 | border: solid 1px #cccccc; 229 | } 230 | table.module_list td.name, table.function_list td.name { 231 | background-color: white; min-width: 200px; border-right-width: 0px; 232 | } 233 | table.module_list td.summary, table.function_list td.summary { 234 | background-color: white; width: 100%; border-left-width: 0px; 235 | } 236 | 237 | dl.function { 238 | margin-right: 15px; 239 | margin-left: 15px; 240 | border-bottom: solid 1px #cccccc; 241 | border-left: solid 1px #cccccc; 242 | border-right: solid 1px #cccccc; 243 | background-color: white; 244 | } 245 | 246 | dl.function dt { 247 | color: #c7254e; 248 | font-family: monospace; 249 | border-top: solid 1px #cccccc; 250 | padding: 15px 15px 0.5em 15px; 251 | } 252 | 253 | dl.function dd { 254 | margin-left: 15px; 255 | margin-right: 15px; 256 | margin-top: 5px; 257 | margin-bottom: 15px; 258 | } 259 | 260 | #content dl.function dd h3 { 261 | margin-top: 0px; 262 | margin-left: 0px; 263 | padding-left: 0px; 264 | font-size: 16px; 265 | color: #555555; 266 | border-bottom: solid 1px #def; 267 | } 268 | 269 | #content dl.function dd ul, #content dl.function dd ol { 270 | padding: 0px; 271 | padding-left: 15px; 272 | list-style-type: none; 273 | } 274 | 275 | /* Highlight XXX notes. */ 276 | #content .xxx { 277 | box-sizing: border-box; 278 | background-color: yellow; 279 | color: black; 280 | border-radius: 6px; 281 | border: #c7254e solid 1px; 282 | padding: 0.1em 0.2em; 283 | } 284 | 285 | ul.nowrap { 286 | overflow:auto; 287 | white-space:nowrap; 288 | } 289 | 290 | .section-description { 291 | padding-left: 15px; 292 | padding-right: 15px; 293 | } 294 | 295 | /* stop sublists from having initial vertical space */ 296 | ul ul { margin-top: 0px; } 297 | ol ul { margin-top: 0px; } 298 | ol ol { margin-top: 0px; } 299 | ul ol { margin-top: 0px; } 300 | 301 | /* make the target distinct; helps when we're navigating to a function */ 302 | a:target + * { 303 | background-color: #FF9; 304 | } 305 | 306 | 307 | /* styles for prettification of source */ 308 | pre .comment { color: #bbccaa; } 309 | pre .constant { color: #a8660d; } 310 | pre .escape { color: #844631; } 311 | pre .keyword { color: #ffc090; font-weight: bold; } 312 | pre .library { color: #0e7c6b; } 313 | pre .marker { color: #512b1e; background: #fedc56; font-weight: bold; } 314 | pre .string { color: #8080ff; } 315 | pre .number { color: #f8660d; } 316 | pre .operator { color: #2239a8; font-weight: bold; } 317 | pre .preprocessor, pre .prepro { color: #a33243; } 318 | pre .global { color: #c040c0; } 319 | pre .user-keyword { color: #800080; } 320 | pre .prompt { color: #558817; } 321 | pre .url { color: #272fc2; text-decoration: underline; } 322 | 323 | html body #content ul.markdown_list { 324 | list-style-type: disc; 325 | } 326 | 327 | #content table.markdown_table { 328 | padding: 0px; 329 | border-collapse: collapse; 330 | } 331 | .markdown_table td, th { 332 | border: 1px solid #999999; 333 | padding: 7px; 334 | } 335 | .markdown_table th { 336 | background-color: #ccc; 337 | } 338 | 339 | .markdown_table tr:nth-child(odd) td { 340 | background-color: white; 341 | } 342 | 343 | .markdown_table tr:nth-child(even) td { 344 | background-color: #f9f2f4; 345 | } 346 | -------------------------------------------------------------------------------- /doc/ldoc/assets/ldoc.ltp: -------------------------------------------------------------------------------- 1 | 3 | 4 | 5 | 6 | $(ldoc.title) 7 | 8 | # if ldoc.custom_css then -- add custom CSS file if configured. 9 | 10 | # end 11 | 12 | 13 | 14 |
15 | 16 |
17 | 18 |
19 |
20 |
21 | 22 | 23 |
24 | 25 | # local no_spaces = ldoc.no_spaces 26 | # local use_li = ldoc.use_li 27 | # local display_name = ldoc.display_name 28 | # local iter = ldoc.modules.iter 29 | # local function M(txt,item) return ldoc.markup(txt,item,ldoc.plain) end 30 | # local nowrap = ldoc.wrap and '' or 'nowrap' 31 | 32 | # local function parse_source_file_table(filename, start_lineno) 33 | # local content = ldoc.include_file(filename) 34 | # local skip = true 35 | # local res = {} 36 | # for lineno, line in ldoc.ipairs(content:split('\n')) do 37 | # if lineno == start_lineno then 38 | # skip = false 39 | # goto continue 40 | # end 41 | # if not skip and line:endswith('}') then 42 | # return res 43 | # end 44 | # if not skip then 45 | # local key, value = line:match("^ *([%w_]-) *= *([%w']-),?$") 46 | # if key then 47 | # res[key] = value 48 | # end 49 | # end 50 | # ::continue:: 51 | # end 52 | # return res 53 | # end 54 | 55 | # -- Copied from config.ld. 56 | # -- 57 | # -- A section description is not processed by 58 | # -- `custom_display_name_handler`. 59 | # local function convert_markdown_references(text) 60 | # local refs = {} 61 | # for _, line in ldoc.pairs(text:split('\n')) do 62 | # local anchor, url = line:lstrip():match('^%[([0-9])%]: (.*)$') 63 | # if anchor then 64 | # refs[anchor] = url 65 | # end 66 | # end 67 | # 68 | # for anchor, url in ldoc.pairs(refs) do 69 | # text = text:gsub('\n *%[' .. anchor .. '%]: [^\n]*', '') 70 | # text = text:gsub('%[(.-)%]%[' .. anchor .. '%]', 71 | # '%1') 72 | # end 73 | # return text 74 | # end 75 | 76 | # -- Copied from config.ld. 77 | # -- 78 | # -- A section description is not processed by 79 | # -- `custom_display_name_handler`. 80 | # local function convert_markdown_backticks(text) 81 | # return text:gsub('`(.-)`', '%1') 82 | # end 83 | 84 | 85 | 86 | 135 | 136 |
137 | 138 | # if ldoc.body then -- verbatim HTML as contents; 'non-code' entries 139 | $(ldoc.body) 140 | # elseif module then -- module documentation 141 |

$(ldoc.module_typename(module)) $(module.name)

142 |

$(M(module.summary,module))

143 |

$(M(module.description,module))

144 | # if module.tags.include then 145 | $(M(ldoc.include_file(module.tags.include))) 146 | # end 147 | # if module.see then 148 | # local li,il = use_li(module.see) 149 |

See also:

150 |
    151 | # for see in iter(module.see) do 152 | $(li)$(see.label)$(il) 153 | # end -- for 154 |
155 | # end -- if see 156 | # if module.usage then 157 | # local li,il = use_li(module.usage) 158 |

Usage:

159 |
    160 | # for usage in iter(module.usage) do 161 | $(li)
    $(ldoc.escape(usage))
    $(il) 162 | # end -- for 163 |
164 | # end -- if usage 165 | # if module.info then 166 |

Info:

167 |
    168 | # for tag, value in module.info:iter() do 169 |
  • $(tag): $(M(value,module))
    • 170 | # end 171 |
172 | # end -- if module.info 173 | 174 | 175 | # if not ldoc.no_summary then 176 | # -- bang out the tables of item types for this module (e.g Functions, Tables, etc) 177 | # for kind,items in module.kinds() do 178 |

$(kind)

179 | 180 | # for item in items() do 181 | 182 | 183 | 184 | 185 | # end -- for items 186 |
$(display_name(item))$(M(item.summary:split('\n', 1)[1],item))
187 | #end -- for kinds 188 | 189 |
190 |
191 | 192 | #end -- if not no_summary 193 | 194 | # --- currently works for both Functions and Tables. The params field either contains 195 | # --- function parameters or table fields. 196 | # local show_return = not ldoc.no_return_or_parms 197 | # local show_parms = show_return 198 | # for kind, items in module.kinds() do 199 | # local kitem = module.kinds:get_item(kind) 200 | # local has_description = kitem and ldoc.descript(kitem) ~= "" 201 |

$(kind)

202 | $(M(module.kinds:get_section_description(kind),nil)) 203 | # if kitem then 204 | # if has_description then 205 |
206 | # local item_text = ldoc.descript(kitem) 207 | # item_text = convert_markdown_references(item_text) 208 | # item_text = convert_markdown_backticks(item_text) 209 | $(M(item_text,kitem)) 210 |
211 | # end 212 | # if kitem.usage then 213 |

Usage:

214 | 
$(ldoc.prettify(kitem.usage[1]))
215 | # end 216 | # end 217 |
218 | # for item in items() do 219 |
220 | 221 | # if kitem and (kitem.name == 'options' or kitem.name == 'defaults') then 222 | $(item.summary) 223 | # else 224 | $(display_name(item)) 225 | # end 226 | # if ldoc.prettify_files and ldoc.is_file_prettified[item.module.file.filename] then 227 | line $(item.lineno) 228 | # end 229 |
230 |
231 | # if kitem and (kitem.name == 'options' or kitem.name == 'defaults') then 232 | $(M(item.description))

233 | # else 234 |

$(M(ldoc.descript(item),item))

235 | # end 236 | 237 | # if ldoc.custom_tags then 238 | # for custom in iter(ldoc.custom_tags) do 239 | # local tag = item.tags[custom[1]] 240 | # if tag and not custom.hidden then 241 | # local li,il = use_li(tag) 242 |

$(custom.title or custom[1]):

243 |
    244 | # for value in iter(tag) do 245 | $(li)$(custom.format and custom.format(value) or M(value))$(il) 246 | # end -- for 247 | # end -- if tag 248 |
249 | # end -- iter tags 250 | # end 251 | 252 | # if show_parms and item.params and #item.params > 0 then 253 | # local subnames = module.kinds:type_of(item).subnames 254 | # if subnames then 255 |

$(subnames):

256 | # end 257 |
    258 | # for parm in iter(item.params) do 259 | # local source_table = {} 260 | # if kitem and kitem.name == 'defaults' then 261 | # source_table = parse_source_file_table(item.file.filename, 262 | # item.lineno) 263 | # end 264 | # local param,sublist = item:subparam(parm) 265 | # if sublist then 266 | # local name = item:display_name_of(sublist) 267 | # local tp = ldoc.typename(item:type_of_param(sublist)) 268 | # local def = item:default_of_param(sublist) 269 | # local sep = def and ',' or ')' 270 |
  • $(name) 271 | # if tp ~= '' then 272 | 273 | ($(tp)$(sep) 274 | # end 275 | # if def == true then 276 | 277 | optional) 278 | 279 | # elseif def then 280 | default $(def) 281 | # end 282 | # if item:readonly(p) then 283 | readonly 284 | # end 285 | # if tp ~= '' then 286 | 287 | # end 288 | 289 |
    290 | 291 |

    $(M(item.params.map[sublist],item))

    292 |     293 |
      294 | # end 295 | # for p in iter(param) do 296 | # local name = item:display_name_of(p) 297 | # local tp = ldoc.typename(item:type_of_param(p)) 298 | # local def = item:default_of_param(p) 299 | # sep = def and ',' or ')' 300 |
    • $(name) 301 | # if tp ~= '' then 302 | 303 | ($(tp)$(sep) 304 | # end 305 | # if def == true then 306 | 307 | optional) 308 | 309 | # elseif def then 310 | default $(def) 311 | # end 312 | # if item:readonly(p) then 313 | readonly 314 | # end 315 | # if tp ~= '' then 316 | 317 | # end 318 | # if source_table[p] ~= nil then 319 | = $(source_table[p]) 320 | # end 321 | # if M(item.params.map[p], item) ~= '' then 322 |
      323 | # end 324 | 325 |

      $(M(item.params.map[p],item))

      326 |       327 |
      • 328 | # end 329 | # if sublist then 330 |
    331 | # end 332 | # end -- for 333 |
334 | # end -- if params 335 | 336 | # if show_return and item.retgroups then local groups = item.retgroups 337 |

Returns:

338 | # for i,group in ldoc.ipairs(groups) do local li,il = use_li(group) 339 |
    340 | # for r in group:iter() do local type, ctypes = item:return_type(r); local rt = ldoc.typename(type) 341 | $(li) 342 | # if rt ~= '' then 343 | $(rt) 344 | # end 345 | $(M(r.text,item))$(il) 346 | # if ctypes then 347 |
      348 | # for c in ctypes:iter() do 349 |
    • $(c.name) 350 | $(ldoc.typename(c.type)) 351 | $(M(c.comment,item))
      • 352 | # end 353 |
    354 | # end -- if ctypes 355 | # end -- for r 356 |
357 | # if i < #groups then 358 |

Or

359 | # end 360 | # end -- for group 361 | # end -- if returns 362 | 363 | # if show_return and item.raise then 364 |

Raises:

365 | $(M(item.raise,item)) 366 | # end 367 | 368 | # if item.see then 369 | # local li,il = use_li(item.see) 370 |

See also:

371 |
    372 | # for see in iter(item.see) do 373 | $(li)$(see.label)$(il) 374 | # end -- for 375 |
376 | # end -- if see 377 | 378 | # if item.usage then 379 | # local li,il = use_li(item.usage) 380 |

Usage:

381 |
    382 | # for usage in iter(item.usage) do 383 | $(li)
    $(ldoc.prettify(usage))
    $(il) 384 | # end -- for 385 |
386 | # end -- if usage 387 | 388 |
389 | # end -- for items 390 |
391 | # end -- for kinds 392 | 393 | # else -- if module; project-level contents 394 | 395 | # if ldoc.description then 396 |

$(M(ldoc.description,nil))

397 | # end 398 | # if ldoc.full_description then 399 |

$(M(ldoc.full_description,nil))

400 | # end 401 | 402 | # for kind, mods in ldoc.kinds() do 403 |

$(kind)

404 | # kind = kind:lower() 405 | 406 | # for m in mods() do 407 | 408 | 409 | 410 | 411 | # end -- for modules 412 |
$(m.name)$(M(ldoc.strip_header(m.summary),m))
413 | # end -- for kinds 414 | # end -- if module 415 | 416 |
417 |
418 |
419 | 420 | 421 | -------------------------------------------------------------------------------- /doc/ldoc/config.ld: -------------------------------------------------------------------------------- 1 | project = 'expirationd' 2 | description = 'Expiration daemon module for Tarantool' 3 | file = 'expirationd' 4 | title = 'expirationd API reference' 5 | 6 | template = 'doc/ldoc/assets' 7 | style = 'doc/ldoc/assets' 8 | no_lua_ref = true 9 | --no_summary = true 10 | 11 | tparam_alias('table', 'table') 12 | tparam_alias('integer', 'integer') 13 | tparam_alias('boolean', 'boolean') 14 | 15 | alias('array', function(tags, value, modifiers) 16 | if modifiers == nil then 17 | return 'param', value, {type = '{...}'} 18 | end 19 | 20 | -- next() is not in the scope. 21 | local subtype = modifiers.number ~= nil and 'number' or 22 | modifiers.string ~= nil and 'string' or 23 | modifiers.table ~= nil and 'table' or 24 | modifiers.integer ~= nil and 'integer' or 25 | modifiers.boolean ~= nil and 'boolean' 26 | return 'param', value, {type = ('{%s, ...}'):format(subtype)} 27 | end) 28 | alias('anchor', 'table') 29 | 30 | -- Convert markdown reference style links into HTML. 31 | local function convert_markdown_references(text) 32 | local refs = {} 33 | for _, line in pairs(text:split('\n')) do 34 | local anchor, url = line:lstrip():match('^%[([0-9])%]: (.*)$') 35 | if anchor then 36 | refs[anchor] = url 37 | end 38 | end 39 | 40 | for anchor, url in pairs(refs) do 41 | text = text:gsub('\n *%[' .. anchor .. '%]: [^\n]*', '') 42 | text = text:gsub('%[(.-)%]%[' .. anchor .. '%]', 43 | '%1') 44 | end 45 | return text 46 | end 47 | 48 | -- Convert three-backticks code blocks to `
` HTML
 49 | -- elements.
 50 | local function convert_markdown_codeblocks(text)
 51 |     local res = ''
 52 |     for _, block in pairs(text:split('\n\n')) do
 53 |         local codeblock = block:match('^ *```\n(.*)[\n ]*```[\n ]*$')
 54 |         if codeblock then
 55 |             block = '
' .. codeblock:rstrip() .. '
\n'
 56 |         end
 57 |         res = res .. '\n\n' .. block
 58 |     end
 59 |     return res:sub(3)
 60 | end
 61 | 
 62 | -- Convert backticks to `` HTML elements.
 63 | local function convert_markdown_backticks(text)
 64 |     return text:gsub('`(.-)`', '%1')
 65 | end
 66 | 
 67 | -- Convert blocks of lines starting from '- ' to an HTML list.
 68 | local function convert_markdown_lists(text)
 69 |     local res = ''
 70 |     for _, block in pairs(text:split('\n\n')) do
 71 |         local list = '
    \n'
 72 |         local is_list = true
 73 |         for _, line in pairs(block:split('\n')) do
 74 |             local list_item = line:match('^ *%- (.*)$')
 75 |             is_list = is_list and list_item
 76 |             if not is_list then
 77 |                 break
 78 |             end
 79 |             list = list .. '
  • ' .. list_item .. '
    • \n'
 80 |         end
 81 |         list = list .. '
'
 82 |         block = is_list and list or block
 83 |         res = res .. '\n\n' .. block
 84 |     end
 85 |     return res:sub(3)
 86 | end
 87 | 
 88 | -- Convert GFM syntax markdown tables into HTML tables.
 89 | local function convert_markdown_tables(text)
 90 |     local res = ''
 91 |     for _, block in pairs(text:split('\n\n')) do
 92 |         local tbl = '\n'
 93 |         local is_table = true
 94 |         local table_line = 1
 95 |         for _, line in pairs(block:split('\n')) do
 96 |             is_table = is_table and (line == '' or line:match('^ *|'))
 97 |             if not is_table then
 98 |                 break
 99 |             end
100 |             if line ~= '' then
101 |                 tbl = tbl .. '\n'
102 |                 for _, table_item in pairs(line:split('|')) do
103 |                     if not table_item:match('^[ -]*$') then
104 |                         if table_line == 1 then
105 |                             tbl = tbl .. '\n'
106 |                         else
107 |                             tbl = tbl .. '\n'
108 |                         end
109 |                     end
110 |                 end
111 |                 tbl = tbl .. '\n'
112 |                 table_line = table_line + 1
113 |             end
114 |         end
115 |         tbl = tbl .. '
' .. table_item:strip() .. '' .. table_item:strip() .. '
\n'
116 |         block = is_table and tbl or block
117 |         res = res .. '\n\n' .. block
118 |     end
119 |     return res:sub(3)
120 | end
121 | 
122 | local function highlight_xxx_notes(text)
123 |     local from = 'XXX'
124 |     local to = 'XXX'
125 | 
126 |     -- Don't wrap several times.
127 |     text = text:gsub(to, '\x01')
128 |     text = text:gsub(from, to)
129 |     return text:gsub('\x01', to)
130 | end
131 | 
132 | -- Apply a transformation `fun` to text properties of
133 | -- given `item`, which are expected to be multiparagraph
134 | -- free form text.
135 | local function apply_to_descriptions(fun, item)
136 |     if item.summary then
137 |         item.summary = fun(item.summary)
138 |     end
139 | 
140 |     if item.description then
141 |         item.description = fun(item.description)
142 |     end
143 | 
144 |     if item.params and item.params.map then
145 |         for k, v in pairs(item.params.map) do
146 |             item.params.map[k] = fun(v)
147 |         end
148 |     end
149 | 
150 |     if item.retgroups then
151 |         for _, group in ipairs(item.retgroups) do
152 |             for _, group_item in ipairs(group) do
153 |                 if group_item.text then
154 |                     group_item.text = fun(group_item.text)
155 |                 end
156 |             end
157 |         end
158 |     end
159 | end
160 | 
161 | custom_display_name_handler = function(item, default_handler)
162 |     apply_to_descriptions(convert_markdown_references, item)
163 |     apply_to_descriptions(convert_markdown_codeblocks, item)
164 |     apply_to_descriptions(convert_markdown_backticks, item)
165 |     apply_to_descriptions(convert_markdown_lists, item)
166 |     apply_to_descriptions(convert_markdown_tables, item)
167 |     apply_to_descriptions(highlight_xxx_notes, item)
168 |     return default_handler(item)
169 | end
170 | 
171 | -- vim: ft=lua:
172 | 


--------------------------------------------------------------------------------
/expirationd-scm-1.rockspec:
--------------------------------------------------------------------------------
 1 | package = "expirationd"
 2 | version = "scm-1"
 3 | source = {
 4 |     url = "git+https://github.com/tarantool/expirationd.git",
 5 |     branch = "master",
 6 | }
 7 | description = {
 8 |     summary = "Expiration daemon for Tarantool",
 9 |     homepage = "https://github.com/tarantool/expirationd",
10 |     license = "BSD2",
11 |     maintainer = "Oleg Jukovec "
12 | }
13 | dependencies = {
14 |     "lua >= 5.1", -- actually tarantool > 1.6
15 |     "checks >= 2.1",
16 | }
17 | build = {
18 |     type = "builtin",
19 |     modules = {
20 |         ["expirationd"] = "expirationd/init.lua",
21 |         ["expirationd.version"] = "expirationd/version.lua",
22 |         ["cartridge.roles.expirationd"] = "cartridge/roles/expirationd.lua",
23 |         ["roles.expirationd"] = "roles/expirationd.lua"
24 |     }
25 | }
26 | -- vim: syntax=lua
27 | 


--------------------------------------------------------------------------------
/expirationd/version.lua:
--------------------------------------------------------------------------------
1 | -- Сontains the module version.
2 | -- Requires manual update in case of release commit.
3 | 
4 | return '1.6.0'
5 | 


--------------------------------------------------------------------------------
/roles/expirationd.lua:
--------------------------------------------------------------------------------
  1 | local expirationd = require("expirationd")
  2 | local fiber = require("fiber")
  3 | local log = require("log")
  4 | 
  5 | local role_name = "roles.expirationd"
  6 | local started = {}
  7 | 
  8 | 
  9 | local function load_function(func_name)
 10 |     if func_name == nil or type(func_name) ~= 'string' then
 11 |         return nil
 12 |     end
 13 | 
 14 |     local func = rawget(_G, func_name)
 15 |     if func ~= nil then
 16 |         if type(func) ~= 'function' then
 17 |             return nil
 18 |         end
 19 | 
 20 |         return func
 21 |     elseif box.schema.func.exists(func_name) then
 22 |         return function(...)
 23 |             return box.func[func_name]:call({...})
 24 |         end
 25 |     else
 26 |         return nil
 27 |     end
 28 | end
 29 | 
 30 | local types_map = {
 31 |     b = {type = "boolean", err = "a boolean"},
 32 |     n = {type = "number", err = "a number"},
 33 |     s = {type = "string", err = "a string"},
 34 |     f = {type = "string", transform = load_function, err = "a function name in _G or in box.func"},
 35 |     t = {type = "table", err = "a table"},
 36 |     any = {err = "any type"},
 37 | }
 38 | 
 39 | local opts_map = {
 40 |     args = {"any"},
 41 |     atomic_iteration = {"b"},
 42 |     force = {"b"},
 43 |     force_allow_functional_index = {"b"},
 44 |     full_scan_delay = {"n"},
 45 |     full_scan_time = {"n"},
 46 |     index = {"n", "s"},
 47 |     iterate_with = {"f"},
 48 |     iteration_delay = {"n"},
 49 |     iterator_type = {"n", "s"},
 50 |     on_full_scan_complete = {"f"},
 51 |     on_full_scan_error = {"f"},
 52 |     on_full_scan_start = {"f"},
 53 |     on_full_scan_success = {"f"},
 54 |     process_expired_tuple = {"f"},
 55 |     process_while = {"f"},
 56 |     start_key = {"f", "t"},
 57 |     tuples_per_iteration = {"n"},
 58 |     vinyl_assumed_space_len_factor = {"n"},
 59 |     vinyl_assumed_space_len = {"n"},
 60 | }
 61 | 
 62 | local function table_contains(table, element)
 63 |     for _, value in pairs(table) do
 64 |       if value == element then
 65 |         return true
 66 |       end
 67 |     end
 68 |     return false
 69 | end
 70 | 
 71 | 
 72 | local function get_param(param_name, value, types)
 73 |     local found = false
 74 |     for _, t in ipairs(types) do
 75 |         local type_opts = types_map[t]
 76 |         if type_opts == nil then
 77 |             error(role_name .. ": unsupported type option")
 78 |         end
 79 |         if not type_opts.type or type(value) == type_opts.type then
 80 |             if type_opts.transform then
 81 |                 local tmp = type_opts.transform(value)
 82 |                 if tmp then
 83 |                     value = tmp
 84 |                     found = true
 85 |                     break
 86 |                 end
 87 |             else
 88 |                 found = true
 89 |                 break
 90 |             end
 91 |         end
 92 |     end
 93 | 
 94 |     -- Small hack because in tarantool role we wait for functions to be created.
 95 |     -- So, if type of value is function and it is allowed
 96 |     -- and it is not found we do not return an error.
 97 |     if table_contains(types, "f") and not found then
 98 |         for _, t in ipairs(types) do
 99 |             local type_opts = types_map[t]
100 |             if t == "f" and type(value) == type_opts.type then
101 |                 return nil, true, nil
102 |             end
103 |         end
104 |     end
105 | 
106 |     if not found then
107 |         local err = role_name .. ": " .. param_name .. " must be "
108 |         for i, t in ipairs(types) do
109 |             err = err .. types_map[t].err
110 |             if i ~= #types then
111 |                 err = err .. " or "
112 |             end
113 |         end
114 |         return nil, false, err
115 |     end
116 | 
117 |     return value, true, nil
118 | end
119 | 
120 | local function get_task_options(opts)
121 |     if opts == nil then
122 |         return
123 |     end
124 | 
125 |     local missed_functions = {}
126 | 
127 |     for opt, val in pairs(opts) do
128 |         if type(opt) ~= "string" then
129 |             error(role_name .. ": an option must be a string")
130 |         end
131 |         if opts_map[opt] == nil then
132 |             error(role_name .. ": unsupported option '" .. opt .. "'")
133 |         end
134 |         local res, ok, err = get_param("options." .. opt, val, opts_map[opt])
135 |         if not ok then
136 |             error(err)
137 |         end
138 |         if ok and res == nil and opts_map[opt][1] == "f" then
139 |             table.insert(missed_functions, val)
140 |         end
141 |         opts[opt] = res
142 |     end
143 | 
144 |     return opts, missed_functions
145 | end
146 | 
147 | local function get_task_config(task_conf)
148 |     -- setmetatable resets __newindex write protection on a copy.
149 |     local conf = setmetatable(table.deepcopy(task_conf), {})
150 |     local params_map = {
151 |         space = {required = true, types = {"n", "s"}},
152 |         is_expired = {required = true, types = {"f"}},
153 |         is_master_only = {required = false, types = {"b"}},
154 |         options = {required = false, types = {"t"}},
155 |     }
156 |     for k, _ in pairs(conf) do
157 |         if type(k) ~= "string" then
158 |             error(role_name .. ": param must be a string")
159 |         end
160 |         if params_map[k] == nil then
161 |             error(role_name .. ": unsupported param " .. k)
162 |         end
163 |     end
164 |     local missed_functions = {}
165 |     for param, opts in pairs(params_map) do
166 |         if opts.required and conf[param] == nil then
167 |             error(role_name .. ": " .. param .. " is required")
168 |         end
169 |         if conf[param] ~= nil then
170 |             local res, ok, err = get_param(param, conf[param], opts.types)
171 |             if not ok then
172 |                  error(err)
173 |             end
174 |             if ok and res == nil and opts.types[1] == "f" then
175 |                 table.insert(missed_functions, conf[param])
176 |             end
177 |             conf[param] = res
178 |         end
179 |     end
180 | 
181 |     local missed_functions_opts
182 |     conf.options, missed_functions_opts = get_task_options(conf.options)
183 |     if missed_functions_opts ~= nil then
184 |         for _, func in pairs(missed_functions_opts) do
185 |             table.insert(missed_functions, func)
186 |         end
187 |     end
188 |     return conf, missed_functions
189 | end
190 | 
191 | local function get_cfg(cfg)
192 |     local conf = setmetatable(table.deepcopy(cfg), {})
193 |     local params_map = {
194 |         metrics = {"b"},
195 |     }
196 | 
197 |     for k, _ in pairs(conf) do
198 |         if type(k) ~= "string" then
199 |             error(role_name .. ": config option must be a string")
200 |         end
201 |         if params_map[k] == nil then
202 |             error(role_name .. ": unsupported config option " .. k)
203 |         end
204 |     end
205 | 
206 |     for param, types in pairs(params_map) do
207 |         if conf[param] ~= nil then
208 |             local _, ok, err = get_param(param, conf[param], types)
209 |             if not ok then
210 |                 error(err)
211 |             end
212 |         end
213 |     end
214 | 
215 |     return conf
216 | end
217 | 
218 | local function validate_config(conf_new)
219 |     local conf = conf_new or {}
220 | 
221 |     for task_name, task_conf in pairs(conf) do
222 |         local _, ok, err = get_param("task name", task_name, {"s"})
223 |         if not ok then
224 |             error(err)
225 |         end
226 |         local _, ok, err = get_param("task params", task_conf, {"t"})
227 |         if not ok then
228 |             error(err)
229 |         end
230 |         local ok, ret = pcall(get_task_config, task_conf)
231 |         if not ok then
232 |             if task_name == "cfg" then
233 |                 get_cfg(task_conf)
234 |             else
235 |                 error(ret)
236 |             end
237 |         end
238 |     end
239 | 
240 |     return true
241 | end
242 | 
243 | local function load_task(task_conf, task_name)
244 |     local timeout = 1
245 |     local warning_delay = 60
246 |     local start = fiber.clock()
247 |     local task_config, missed_functions = get_task_config(task_conf)
248 | 
249 |     fiber.name(role_name .. ":" .. task_name)
250 | 
251 |     local skip = task_conf.is_master_only and not box.info.ro
252 |     if skip then
253 |         return
254 |     end
255 | 
256 |     while #missed_functions ~= 0 do
257 |         fiber.sleep(timeout)
258 |         if fiber.clock() - start > warning_delay then
259 |             local message = role_name .. ": " .. task_name .. ": waiting for functions: "
260 |             for i, func in pairs(missed_functions) do
261 |                 if i == #missed_functions then
262 |                     message = message .. func .. '.'
263 |                 else
264 |                     message = message .. func .. ', '
265 |                 end
266 |             end
267 | 
268 |             log.warn(message)
269 |             start = fiber.clock()
270 |         end
271 |         task_config, missed_functions = get_task_config(task_conf)
272 |     end
273 | 
274 |     local task = expirationd.start(task_name, task_config.space,
275 |                                    task_config.is_expired,
276 |                                    task_config.options)
277 |     if task == nil then
278 |         error(role_name .. ": unable to start task " .. task_name)
279 |     end
280 |     table.insert(started, task_name)
281 | end
282 | 
283 | local function apply_config(conf)
284 |     -- Finishes tasks from an old configuration
285 |     for i = #started, 1, -1 do
286 |         local task_name = started[i]
287 |         local ok, task = pcall(expirationd.task, task_name)
288 |         -- We don't need to do anything if there is no task
289 |         if ok then
290 |             if conf[task_name] then
291 |                 task:stop()
292 |             else
293 |                 task:kill()
294 |             end
295 |         end
296 |         table.remove(started, i)
297 |     end
298 | 
299 |     if conf["cfg"] ~= nil then
300 |         local ok = pcall(get_task_config, conf["cfg"])
301 |         if not ok then
302 |             local cfg = get_cfg(conf["cfg"])
303 |             expirationd.cfg(cfg)
304 |             conf["cfg"] = nil
305 |         end
306 |     end
307 | 
308 |     for task_name, task_conf in pairs(conf) do
309 |         fiber.new(load_task, task_conf, task_name)
310 |     end
311 | end
312 | 
313 | local function stop()
314 |     for _, task_name in pairs(expirationd.tasks()) do
315 |         local task = expirationd.task(task_name)
316 |         task:stop()
317 |     end
318 | end
319 | 
320 | return {
321 |     validate = validate_config,
322 |     apply = apply_config,
323 |     stop = stop,
324 | }
325 | 


--------------------------------------------------------------------------------
/rpm/prebuild.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 | 
3 | set -exu  # Strict shell (w/o -o pipefail)
4 | 
5 | curl -LsSf https://www.tarantool.io/release/1.10/installer.sh | sudo bash
6 | 


--------------------------------------------------------------------------------
/rpm/tarantool-expirationd.spec:
--------------------------------------------------------------------------------
 1 | Name: tarantool-expirationd
 2 | Version: 1.0.0
 3 | Release: 1%{?dist}
 4 | Summary: Expiration daemon for Tarantool
 5 | Group: Applications/Databases
 6 | License: BSD
 7 | URL: https://github.com/tarantool/tarantool-expirationd
 8 | Source0: https://github.com/tarantool/%{name}/archive/%{version}/%{name}-%{version}.tar.gz
 9 | BuildArch: noarch
10 | BuildRequires: tarantool >= 1.7.4.0
11 | BuildRequires: /usr/bin/prove
12 | Requires: tarantool >= 1.7.4.0
13 | Requires: tarantool-checks >= 2.1
14 | %description
15 | This package can turn Tarantool into a persistent memcache replacement,
16 | but is powerful enough so that your own expiration strategy can be defined.
17 | 
18 | You define two functions: one takes a tuple as an input and returns true in
19 | case it's expired and false otherwise. The other takes the tuple and
20 | performs the expiry itself: either deletes it (memcache), or does something
21 | smarter, like put a smaller representation of the data being deleted into
22 | some other space.
23 | 
24 | %prep
25 | %setup -q -n %{name}-%{version}
26 | 
27 | %install
28 | install -d %{buildroot}%{_datarootdir}/tarantool/
29 | install -d %{buildroot}%{_datarootdir}/tarantool/expirationd/
30 | install -m 0644 expirationd/* %{buildroot}%{_datarootdir}/tarantool/expirationd/
31 | install -d %{buildroot}%{_datarootdir}/tarantool/cartridge/roles/
32 | install -m 0644 cartridge/roles/expirationd.lua %{buildroot}%{_datarootdir}/tarantool/cartridge/roles/expirationd.lua
33 | install -d %{buildroot}%{_datarootdir}/tarantool/roles/
34 | install -m 0644 roles/expirationd.lua %{buildroot}%{_datarootdir}/tarantool/roles/expirationd.lua
35 | 
36 | %files
37 | %{_datarootdir}/tarantool/expirationd/
38 | %{_datarootdir}/tarantool/cartridge
39 | %{_datarootdir}/tarantool/roles
40 | %doc README.md
41 | %{!?_licensedir:%global license %doc}
42 | %license LICENSE
43 | 
44 | %changelog
45 | 
46 | * Wed Aug 23 2023 Oleg Jukovec  1.5.0-1
47 | 
48 | - Add an ability to use persistent functions in `box.func` with cartridge.
49 | 
50 | * Thu Mar 16 2023 Oleg Jukovec  1.4.0-1
51 | 
52 | - Add _VERSION constant.
53 | 
54 | * Fri Jan 17 2023 Oleg Jukovec  1.3.1-1
55 | - Fix check of the Tarantool version in tests to determine a bug in
56 |   the vinyl engine
57 | - Add a way to configure the module using Tarantool Cartridge role
58 |   configuration
59 | 
60 | * Mon Jun 27 2022 Oleg Jukovec  1.2.0-1
61 | - Check types of function arguments with checks module
62 | - Add messages about obsolete methods
63 | - Add metrics support
64 | - Prevent iteration through a functional index for Tarantool < 2.8.4
65 | - Fix processing tasks with zero length box.cfg.replication
66 | - Make iterate_with() conform to declared interface
67 | - Update documentation and convert to LDoc format
68 | - Support to generate documentation using make
69 | - Update comparison table in README.md
70 | - Add note about using expirationd with replication
71 | - Fix a typo in the rpm-package description
72 | - Fix function name in example:
73 |   function on_full_scan_complete -> function on_full_scan_error
74 | - Describe prerequisites and installation steps in README.md
75 | - Bump luatest version to 0.5.6
76 | - Fix incorrect description of the force option for the expirationd.start
77 | 
78 | * Mon Sep 13 2021 Sergey Bronnikov  1.1.1-1
79 | - Fix freezes when stopping a task
80 | - Enable Lua source code analysis with luacheck
81 | 
82 | * Tue Jul 06 2021 Sergey Bronnikov  1.1.0-1
83 | - Add the ability to set iteration and full scan delays for a task.
84 | - Add callbacks for a task at various stages of the full scan iteration.
85 | - Add the ability to specify from where to start the iterator
86 |   (option start_key) and specify the type of the iterator itself
87 |   (option iterator_type)
88 | - Add the ability to create a custom iterator that will be created at the
89 |   selected index (option iterate_with)
90 | - Add an option atomic_iteration that allows making only one transaction per
91 |   batch option
92 | - Fix worker iteration for a tree index
93 | 
94 | * Sat Jan 22 2018 Roman Tsisyk  1.0.1-1
95 | - First release with rockspecs
96 | 
97 | * Thu Jun 18 2015 Roman Tsisyk  1.0.0-1
98 | - Initial version of the RPM spec
99 | 


--------------------------------------------------------------------------------
/test/entrypoint/srv_base.lua:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env tarantool
 2 | 
 3 | require('strict').on()
 4 | _G.is_initialized = function() return false end
 5 | 
 6 | local log = require('log')
 7 | local errors = require('errors')
 8 | local cartridge = require('cartridge')
 9 | 
10 | package.preload['customers-storage'] = function()
11 |     return {
12 |         role_name = 'customers-storage',
13 |         init = function()
14 |             local customers_space = box.schema.space.create('customers', {
15 |                 format = {
16 |                     {name = 'id', type = 'unsigned'},
17 |                 },
18 |                 if_not_exists = true,
19 |                 engine = 'memtx',
20 |             })
21 | 
22 |             customers_space:create_index('id', {
23 |                 parts = { {field = 'id'} },
24 |                 unique = true,
25 |                 type = 'TREE',
26 |                 if_not_exists = true,
27 |             })
28 |         end,
29 |     }
30 | end
31 | 
32 | local ok, err = errors.pcall('CartridgeCfgError', cartridge.cfg, {
33 |     advertise_uri = 'localhost:3301',
34 |     http_port = 8081,
35 |     bucket_count = 3000,
36 |     roles = {
37 |         'customers-storage',
38 |         'cartridge.roles.vshard-router',
39 |         'cartridge.roles.vshard-storage',
40 |     },
41 |     roles_reload_allowed = true
42 | })
43 | 
44 | if not ok then
45 |     log.error('%s', err)
46 |     os.exit(1)
47 | end
48 | 
49 | _G.is_initialized = cartridge.is_healthy
50 | 


--------------------------------------------------------------------------------
/test/entrypoint/srv_role.lua:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env tarantool
 2 | 
 3 | require('strict').on()
 4 | _G.is_initialized = function() return false end
 5 | 
 6 | local log = require('log')
 7 | local errors = require('errors')
 8 | local fiber = require('fiber')
 9 | local cartridge = require('cartridge')
10 | local hotreload = require('cartridge.hotreload')
11 | 
12 | package.preload['customers-storage'] = function()
13 |     return {
14 |         role_name = 'customers-storage',
15 |         init = function()
16 |             local customers_space = box.schema.space.create('customers', {
17 |                 format = {
18 |                     {name = 'id', type = 'unsigned'},
19 |                 },
20 |                 if_not_exists = true,
21 |                 engine = 'memtx',
22 |             })
23 | 
24 |             customers_space:create_index('id', {
25 |                 parts = { {field = 'id'} },
26 |                 unique = true,
27 |                 type = 'TREE',
28 |                 if_not_exists = true,
29 |             })
30 |         end,
31 |     }
32 | end
33 | 
34 | local ok, err = errors.pcall('CartridgeCfgError', cartridge.cfg, {
35 |     advertise_uri = 'localhost:3301',
36 |     http_port = 8081,
37 |     bucket_count = 3000,
38 |     roles = {
39 |         'customers-storage',
40 |         'cartridge.roles.vshard-router',
41 |         'cartridge.roles.vshard-storage',
42 |         'cartridge.roles.expirationd'
43 |     },
44 |     roles_reload_allowed = true,
45 | })
46 | 
47 | if not ok then
48 |     log.error('%s', err)
49 |     os.exit(1)
50 | end
51 | 
52 | _G.is_initialized = cartridge.is_healthy
53 | _G.always_true_test = function() return true end
54 | _G.is_expired_test_continue = function(_, tuple)
55 |     if rawget(_G, "is_expired_test_first_tuple") == nil then
56 |         rawset(_G, "is_expired_test_first_tuple", tuple)
57 |     end
58 | 
59 |     local cnt = rawget(_G, "is_expired_test_wait_cnt") or 0
60 |     cnt = cnt + 1
61 |     rawset(_G, "is_expired_test_wait_cnt", cnt)
62 |     if cnt == 5 then
63 |         fiber.sleep(60)
64 |     end
65 |     return true
66 | end
67 | 
68 | hotreload.whitelist_globals({"always_true_test"})
69 | hotreload.whitelist_globals({"is_expired_test_continue"})
70 | 


--------------------------------------------------------------------------------
/test/helper.lua:
--------------------------------------------------------------------------------
  1 | local t = require("luatest")
  2 | local fio = require("fio")
  3 | 
  4 | local helpers = require("luatest.helpers")
  5 | 
  6 | helpers.project_root = fio.dirname(debug.sourcedir())
  7 | 
  8 | function helpers.create_space(space_name, engine)
  9 |     local space_format = {
 10 |         {
 11 |             name = "id",
 12 |             type = "number"
 13 |         },
 14 |         {
 15 |             name = "first_name",
 16 |             type = "string"
 17 |         },
 18 |         {
 19 |             name = "value",
 20 |             type = "number",
 21 |             is_nullable = true
 22 |         },
 23 |         {
 24 |             name = "count",
 25 |             type = "number",
 26 |             is_nullable = true
 27 |         },
 28 |         {
 29 |             name = "non_unique_id",
 30 |             type = "number",
 31 |             is_nullable = true,
 32 |         },
 33 |         {
 34 |             name = "json_path_field",
 35 |             is_nullable = true,
 36 |         },
 37 |         {
 38 |             name = "multikey_field",
 39 |             is_nullable = true
 40 |         },
 41 |         {
 42 |             name = "functional_field",
 43 |             is_nullable = true
 44 |         },
 45 |     }
 46 | 
 47 |     local space = box.schema.create_space(space_name, {
 48 |         engine = engine
 49 |     })
 50 |     space:format(space_format)
 51 | 
 52 |     return space
 53 | end
 54 | 
 55 | function helpers.create_space_with_tree_index(engine)
 56 |     local space = helpers.create_space("tree", engine)
 57 | 
 58 |     space:create_index("primary", {
 59 |         type = "TREE",
 60 |         parts = {
 61 |             {
 62 |                 field = 1
 63 |             }
 64 |         }
 65 |     })
 66 |     space:create_index("index_for_first_name", {
 67 |         type = "TREE",
 68 |         parts = {
 69 |             {
 70 |                 field = 2
 71 |             }
 72 |         }
 73 |     })
 74 |     space:create_index("multipart_index", {
 75 |         type = "TREE",
 76 |         parts = {
 77 |             {
 78 |                 field = 3,
 79 |                 is_nullable = true
 80 |             },
 81 |             {
 82 |                 field = 4,
 83 |                 is_nullable = true
 84 |             }
 85 |         }
 86 |     })
 87 |     space:create_index("non_unique_index", {
 88 |         type = "TREE",
 89 |         parts = {
 90 |             {
 91 |                 field = 5,
 92 |                 is_nullable = true
 93 |             }
 94 |         },
 95 |         unique = false
 96 |     })
 97 | 
 98 |     if _TARANTOOL >= "2" then
 99 |         space:create_index("json_path_index", {
100 |             type = "TREE",
101 |             parts = {
102 |                 {
103 |                     field = 6,
104 |                     type = "scalar",
105 |                     path = "age",
106 |                     is_nullable = true
107 |                 }
108 |             }
109 |         })
110 |         space:create_index("multikey_index", {
111 |             type = "TREE",
112 |             parts = {
113 |                 {
114 |                     field = 7,
115 |                     type = "str",
116 |                     path = "data[*].name"
117 |                 }
118 |             }
119 |         })
120 |         if engine ~= "vinyl" then
121 |             space:create_index("functional_index", {
122 |                 type = "TREE",
123 |                 parts = {
124 |                     {
125 |                         field = 1,
126 |                         type = "string"
127 |                     }
128 |                 },
129 |                 func = "tree_func"
130 |             })
131 |         end
132 |     end
133 | 
134 |     return space
135 | end
136 | 
137 | function helpers.create_space_with_hash_index(engine)
138 |     local space = helpers.create_space("hash", engine)
139 |     space:create_index("primary", {
140 |         type = "HASH",
141 |         parts = {
142 |             {
143 |                 field = 1
144 |             }
145 |         }
146 |     })
147 |     space:create_index("index_for_first_name", {
148 |         type = "HASH",
149 |         parts = {
150 |             {
151 |                 field = 2
152 |             }
153 |         }
154 |     })
155 |     space:create_index("multipart_index", {
156 |         type = "HASH",
157 |         parts = {
158 |             {
159 |                 field = 1
160 |             },
161 |             {
162 |                 field = 2
163 |             }
164 |          }
165 |      })
166 | 
167 |     return space
168 | end
169 | 
170 | function helpers.create_space_with_bitset_index(engine)
171 |     local space = helpers.create_space("bitset", engine)
172 |     space:create_index("primary", {
173 |         type = "TREE",
174 |         parts = {
175 |             {
176 |                 field = 1
177 |             }
178 |         }
179 |     })
180 |     space:create_index("index_for_first_name", {
181 |         type = "BITSET",
182 |         parts = {
183 |             {
184 |                 field = 2,
185 |                 type = "string"
186 |             }
187 |         },
188 |         unique = false
189 |     })
190 | 
191 |     return space
192 | end
193 | 
194 | t.after_suite(function()
195 |     fio.rmtree(t.datadir)
196 | end)
197 | 
198 | t.before_suite(function()
199 |     t.datadir = fio.tempdir()
200 |     box.cfg{
201 |         wal_dir    = t.datadir,
202 |         memtx_dir  = t.datadir,
203 |         vinyl_dir  = t.datadir,
204 |     }
205 | 
206 |     local tree_code = [[function(tuple)
207 |         if tuple[8] then
208 |             return {string.sub(tuple[8],2,2)}
209 |         end
210 |         return {tuple[2]}
211 |     end]]
212 |     if _TARANTOOL >= "2" then
213 |         box.schema.func.create("tree_func", {
214 |             body = tree_code,
215 |             is_deterministic = true,
216 |             is_sandboxed = true
217 |         })
218 |     end
219 | end)
220 | 
221 | function helpers.is_expired_true()
222 |     return true
223 | end
224 | 
225 | function helpers.is_metrics_supported()
226 |     local is_package, metrics = pcall(require, "metrics")
227 |     if not is_package then
228 |         return false
229 |     end
230 |     -- metrics >= 0.11.0 is required
231 |     local counter = require('metrics.collectors.counter')
232 |     return metrics.unregister_callback and counter.remove
233 | end
234 | 
235 | function helpers.iterate_with_func(task)
236 |     return task.index:pairs(task.start_key(), { iterator = task.iterator_type })
237 |        :take_while(
238 |             function()
239 |                 return task:process_while()
240 |             end
241 |         )
242 | end
243 | 
244 | helpers.iteration_result = {}
245 | function helpers.is_expired_debug(_, tuple)
246 |     table.insert(helpers.iteration_result, tuple)
247 |     return true
248 | end
249 | 
250 | function helpers.tarantool_version()
251 |     local major_minor_patch = _G._TARANTOOL:split('-', 1)[1]
252 |     local major_minor_patch_parts = major_minor_patch:split('.', 2)
253 | 
254 |     local major = tonumber(major_minor_patch_parts[1])
255 |     local minor = tonumber(major_minor_patch_parts[2])
256 |     local patch = tonumber(major_minor_patch_parts[3])
257 | 
258 |     return major, minor, patch
259 | end
260 | 
261 | function helpers.vinyl_is_supported()
262 |     local major, minor, patch = helpers.tarantool_version()
263 | 
264 |     -- The issue: https://github.com/tarantool/tarantool/issues/6448
265 |     --
266 |     -- The problem was introduced in 1.10.2 and fixed in 1.10.12, 2.8.3 and
267 |     -- after a 2.10 release.
268 |     return (major == 1 and minor <= 9) or
269 |         (major == 1 and minor == 10 and patch <= 1) or
270 |         (major == 1 and minor == 10 and patch >= 12) or
271 |         (major == 1 and minor >= 11) or
272 |         (major == 2 and minor == 8 and patch >= 3) or
273 |         (major == 2 and minor >= 10) or
274 |         (major >= 3)
275 | end
276 | 
277 | function helpers.memtx_func_index_is_supported()
278 |     local major, minor, patch = helpers.tarantool_version()
279 | 
280 |     -- The issue: https://github.com/tarantool/tarantool/issues/6786
281 |     --
282 |     -- Functional indexes for memtx storage engine are introduced in 2.2.1 with
283 |     -- a bug. The 1.10 series does not support them at all. The problem was
284 |     -- fixed in 2.8.4 and after a 2.10 release.
285 |     return (major == 2 and minor == 8 and patch >= 4) or
286 |         (major == 2 and minor >= 10) or
287 |         (major >= 3)
288 | end
289 | 
290 | function helpers.single_yield_transactional_ddl_is_supported()
291 |     local major, minor, patch = helpers.tarantool_version()
292 | 
293 |     -- The issue: https://github.com/tarantool/tarantool/issues/4083
294 |     --
295 |     -- A limited transactional DDL support has been introduced in 2.2.1, it
296 |     -- allows to wrap a single-yield DDL statement set into a transaction if
297 |     -- the yielding statement is the first in the transaction.
298 |     return (major == 2 and minor == 2 and patch >= 1) or
299 |         (major == 2 and minor >= 3) or
300 |         (major >= 3)
301 | end
302 | 
303 | function helpers.tarantool_role_is_supported()
304 |     local major, _, _ = helpers.tarantool_version()
305 |     return major >= 3
306 | end
307 | 
308 | function helpers.error_function()
309 |     error("error function call")
310 | end
311 | 
312 | function helpers.get_error_function(error_msg)
313 |     return function()
314 |         error(error_msg)
315 |     end
316 | end
317 | 
318 | function helpers.create_persistent_function(name, body)
319 |     box.schema.func.create(name, {
320 |         body = body or "function(...) return true end",
321 |         if_not_exists = true
322 |     })
323 | end
324 | 
325 | local root = fio.dirname(fio.dirname(fio.abspath(package.search('test.helper'))))
326 | 
327 | helpers.lua_path = root .. '/?.lua;' ..
328 |     root .. '/?/init.lua;' ..
329 |     root .. '/.rocks/share/tarantool/?.lua;' ..
330 |     root .. '/.rocks/share/tarantool/?/init.lua'
331 | 
332 | return helpers
333 | 


--------------------------------------------------------------------------------
/test/helper_server.lua:
--------------------------------------------------------------------------------
  1 | -- https://github.com/tarantool/tarantool/blob/5040fba9cf1da942371721e36e81c7372699600c/test/luatest_helpers/server.lua
  2 | local fun = require('fun')
  3 | local yaml = require('yaml')
  4 | local urilib = require('uri')
  5 | local fio = require('fio')
  6 | local luatest = require('luatest')
  7 | 
  8 | -- Join paths in an intuitive way.
  9 | --
 10 | -- If a component is nil, it is skipped.
 11 | --
 12 | -- If a component is an absolute path, it skips all the previous
 13 | -- components.
 14 | --
 15 | -- The wrapper is written for two components for simplicity.
 16 | local function pathjoin(a, b)
 17 |     -- No first path -- skip it.
 18 |     if a == nil then
 19 |         return b
 20 |     end
 21 |     -- No second path -- skip it.
 22 |     if b == nil then
 23 |         return a
 24 |     end
 25 |     -- The absolute path is checked explicitly due to gh-8816.
 26 |     if b:startswith('/') then
 27 |         return b
 28 |     end
 29 |     return fio.pathjoin(a, b)
 30 | end
 31 | 
 32 | -- Determine advertise URI for given instance from a cluster
 33 | -- configuration.
 34 | local function find_advertise_uri(config, instance_name, dir)
 35 |     if config == nil or next(config) == nil then
 36 |         return nil
 37 |     end
 38 | 
 39 |     -- Determine listen and advertise options that are in effect
 40 |     -- for the given instance.
 41 |     local advertise
 42 |     local listen
 43 | 
 44 |     for _, group in pairs(config.groups or {}) do
 45 |         for _, replicaset in pairs(group.replicasets or {}) do
 46 |             local instance = (replicaset.instances or {})[instance_name]
 47 |             if instance == nil then
 48 |                 break
 49 |             end
 50 |             if instance.iproto ~= nil then
 51 |                 if instance.iproto.advertise ~= nil then
 52 |                     advertise = advertise or instance.iproto.advertise.client
 53 |                 end
 54 |                 listen = listen or instance.iproto.listen
 55 |             end
 56 |             if replicaset.iproto ~= nil then
 57 |                 if replicaset.iproto.advertise ~= nil then
 58 |                     advertise = advertise or replicaset.iproto.advertise.client
 59 |                 end
 60 |                 listen = listen or replicaset.iproto.listen
 61 |             end
 62 |             if group.iproto ~= nil then
 63 |                 if group.iproto.advertise ~= nil then
 64 |                     advertise = advertise or group.iproto.advertise.client
 65 |                 end
 66 |                 listen = listen or group.iproto.listen
 67 |             end
 68 |         end
 69 |     end
 70 | 
 71 |     if config.iproto ~= nil then
 72 |         if config.iproto.advertise ~= nil then
 73 |             advertise = advertise or config.iproto.advertise.client
 74 |         end
 75 |         listen = listen or config.iproto.listen
 76 |     end
 77 | 
 78 |     local uris
 79 |     if advertise ~= nil then
 80 |         uris = {{uri = advertise}}
 81 |     else
 82 |         uris = listen
 83 |     end
 84 | 
 85 |     for _, uri in ipairs(uris or {}) do
 86 |         uri = table.copy(uri)
 87 |         uri.uri = uri.uri:gsub('{{ *instance_name *}}', instance_name)
 88 |         uri.uri = uri.uri:gsub('unix/:%./', ('unix/:%s/'):format(dir))
 89 |         local u = urilib.parse(uri)
 90 |         if u.ipv4 ~= '0.0.0.0' and u.ipv6 ~= '::' and u.service ~= '0' then
 91 |             return uri
 92 |         end
 93 |     end
 94 |     error('No suitable URI to connect is found')
 95 | end
 96 | 
 97 | local Server = luatest.Server:inherit({})
 98 | 
 99 | -- Adds the following options:
100 | --
101 | -- * config_file (string)
102 | --
103 | --   An argument of the `--config <...>` CLI option.
104 | --
105 | --   Used to deduce advertise URI to connect net.box to the
106 | --   instance.
107 | --
108 | --   The special value '' means running without `--config <...>`
109 | --   CLI option (but still pass `--name `).
110 | -- * remote_config (table)
111 | --
112 | --   If `config_file` is not passed, this config value is used to
113 | --   deduce the advertise URI to connect net.box to the instance.
114 | Server.constructor_checks = fun.chain(Server.constructor_checks, {
115 |     config_file = 'string',
116 |     remote_config = '?table',
117 | }):tomap()
118 | 
119 | function Server:initialize()
120 |     if self.config_file ~= nil then
121 |         self.command = arg[-1]
122 | 
123 |         self.args = fun.chain(self.args or {}, {
124 |             '--name', self.alias
125 |         }):totable()
126 | 
127 |         if self.config_file ~= '' then
128 |             table.insert(self.args, '--config')
129 |             table.insert(self.args, self.config_file)
130 | 
131 |             -- Take into account self.chdir to calculate a config
132 |             -- file path.
133 |             local config_file_path = pathjoin(self.chdir, self.config_file)
134 | 
135 |             -- Read the provided config file.
136 |             local fh, err = fio.open(config_file_path, {'O_RDONLY'})
137 |             if fh == nil then
138 |                 error(('Unable to open file %q: %s'):format(config_file_path,
139 |                     err))
140 |             end
141 |             self.config = yaml.decode(fh:read())
142 |             fh:close()
143 |         end
144 | 
145 |         if self.net_box_uri == nil then
146 |             local config = self.config or self.remote_config
147 | 
148 |             -- NB: listen and advertise URIs are relative to
149 |             -- process.work_dir, which, in turn, is relative to
150 |             -- self.chdir.
151 |             local work_dir
152 |             if config.process ~= nil and config.process.work_dir ~= nil then
153 |                 work_dir = config.process.work_dir
154 |             end
155 |             local dir = pathjoin(self.chdir, work_dir)
156 |             self.net_box_uri = find_advertise_uri(config, self.alias, dir)
157 |         end
158 |     end
159 |     getmetatable(getmetatable(self)).initialize(self)
160 | end
161 | 
162 | function Server:connect_net_box()
163 |     getmetatable(getmetatable(self)).connect_net_box(self)
164 | 
165 |     if self.config_file == nil then
166 |         return
167 |     end
168 | 
169 |     if not self.net_box then
170 |         return
171 |     end
172 | 
173 |     -- Replace the ready condition.
174 |     local saved_eval = self.net_box.eval
175 |     self.net_box.eval = function(self, expr, args, opts)
176 |         if expr == 'return _G.ready' then
177 |             expr = "return require('config'):info().status == 'ready' or " ..
178 |                           "require('config'):info().status == 'check_warnings'"
179 |         end
180 |         return saved_eval(self, expr, args, opts)
181 |     end
182 | end
183 | 
184 | -- Enable the startup waiting if the advertise URI of the instance
185 | -- is determined.
186 | function Server:start(opts)
187 |     opts = opts or {}
188 |     if self.config_file and opts.wait_until_ready == nil then
189 |         opts.wait_until_ready = self.net_box_uri ~= nil
190 |     end
191 |     getmetatable(getmetatable(self)).start(self, opts)
192 | end
193 | 
194 | return Server
195 | 


--------------------------------------------------------------------------------
/test/integration/cartridge_role_test.lua:
--------------------------------------------------------------------------------
  1 | local fio = require('fio')
  2 | local t = require('luatest')
  3 | local helpers = require('test.helper')
  4 | local g = t.group('cartridge_expirationd_intergration_role')
  5 | local is_cartridge_helpers, cartridge_helpers = pcall(require, 'cartridge.test-helpers')
  6 | 
  7 | g.before_all(function(cg)
  8 |     if is_cartridge_helpers then
  9 |         local entrypoint_path = fio.pathjoin(helpers.project_root,
 10 |                                              'test',
 11 |                                              'entrypoint',
 12 |                                              'srv_role.lua')
 13 |         cg.cluster = cartridge_helpers.Cluster:new({
 14 |             datadir = fio.tempdir(),
 15 |             server_command = entrypoint_path,
 16 |             use_vshard = true,
 17 |             replicasets = {
 18 |                 {
 19 |                     uuid = helpers.uuid('a'),
 20 |                     alias = 'router',
 21 |                     roles = { 'vshard-router' },
 22 |                     servers = {
 23 |                         { instance_uuid = helpers.uuid('a', 1), alias = 'router' },
 24 |                     },
 25 |                 },
 26 |                 {
 27 |                     uuid = helpers.uuid('b'),
 28 |                     alias = 's-1',
 29 |                     roles = { 'vshard-storage', 'customers-storage', 'expirationd' },
 30 |                     servers = {
 31 |                         { instance_uuid = helpers.uuid('b', 1), alias = 's1-master' },
 32 |                         { instance_uuid = helpers.uuid('b', 2), alias = 's1-slave' },
 33 |                     }
 34 |                 }
 35 |             },
 36 |         })
 37 |         cg.cluster:start()
 38 |     end
 39 | end)
 40 | 
 41 | g.after_all(function(cg)
 42 |     if is_cartridge_helpers then
 43 |         cg.cluster:stop()
 44 |         fio.rmtree(cg.cluster.datadir)
 45 |     end
 46 | end)
 47 | 
 48 | g.before_each(function(cg)
 49 |     t.skip_if(not is_cartridge_helpers, "cartridge is not installed")
 50 |     cg.cluster.main_server:upload_config({})
 51 | end)
 52 | 
 53 | g.after_each(function(cg)
 54 |     cg.cluster:server('s1-master').net_box:eval([[
 55 |         box.space.customers:truncate()
 56 |     ]])
 57 | end)
 58 | 
 59 | function g.test_expirationd_service_calls(cg)
 60 |     local result, err = cg.cluster:server('s1-master').net_box:eval([[
 61 |         local expirationd = require('expirationd')
 62 |         local cartridge = require('cartridge')
 63 |         local service = cartridge.service_get("expirationd")
 64 | 
 65 |         for k, v in pairs(expirationd) do
 66 |             if service[k] == nil then
 67 |                 return false
 68 |             end
 69 |         end
 70 |         return true
 71 |     ]])
 72 |     t.assert_equals({result, err}, {true, nil})
 73 | end
 74 | 
 75 | -- init/stop/validate_config/apply_config well tested in test/unit/role_test.lua
 76 | -- here we just ensure that it works as expected
 77 | function g.test_start_task_from_config(cg)
 78 |     t.assert_equals(3, cg.cluster:server('s1-master').net_box:eval([[
 79 |         box.space.customers:insert({1})
 80 |         box.space.customers:insert({2})
 81 |         box.space.customers:insert({3})
 82 |         return #box.space.customers:select({}, {limit = 10})
 83 |     ]]))
 84 |     cg.cluster.main_server:upload_config({
 85 |         expirationd = {
 86 |             test_task = {
 87 |                 space = "customers",
 88 |                 is_expired = "always_true_test",
 89 |                 is_master_only = true,
 90 |             }
 91 |         },
 92 |     })
 93 |     t.assert_equals(cg.cluster:server('s1-master').net_box:eval([[
 94 |         local cartridge = require("cartridge")
 95 |         return #cartridge.service_get("expirationd").tasks()
 96 |     ]]), 1)
 97 |     helpers.retrying({}, function()
 98 |         t.assert_equals(cg.cluster:server('s1-master').net_box:eval([[
 99 |             return #box.space.customers:select({}, {limit = 10})
100 |         ]]), 0)
101 |     end)
102 | 
103 |     -- is_master == false
104 |     t.assert_equals(cg.cluster:server("s1-slave").net_box:eval([[
105 |         local cartridge = require("cartridge")
106 |         return #cartridge.service_get("expirationd").tasks()
107 |     ]]), 0)
108 |     helpers.retrying({}, function()
109 |         t.assert_equals(cg.cluster:server('s1-slave').net_box:eval([[
110 |             return #box.space.customers:select({}, {limit = 10})
111 |         ]]), 0)
112 |     end)
113 | end
114 | 
115 | -- init/stop/validate_config/apply_config well tested in test/unit/role_test.lua
116 | -- here we just ensure that it works as expected
117 | function g.test_start_task_from_config_with_functions_from_box_func(cg)
118 |     t.skip_if(_TARANTOOL < '2', 'Restricted support in Tarantool 1.10')
119 |     t.assert_equals(3, cg.cluster:server('s1-master').net_box:eval([[
120 |         box.schema.func.create('forever_true_test', {
121 |             body = "function(...) return true end",
122 |             if_not_exists = true
123 |         })
124 |         box.space.customers:insert({1})
125 |         box.space.customers:insert({2})
126 |         box.space.customers:insert({3})
127 |         return #box.space.customers:select({}, {limit = 10})
128 |     ]]))
129 |     cg.cluster.main_server:upload_config({
130 |         expirationd = {
131 |             test_task = {
132 |                 space = "customers",
133 |                 is_expired = "forever_true_test",
134 |                 is_master_only = true,
135 |             }
136 |         },
137 |     })
138 |     t.assert_equals(cg.cluster:server('s1-master').net_box:eval([[
139 |         local cartridge = require("cartridge")
140 |         return #cartridge.service_get("expirationd").tasks()
141 |     ]]), 1)
142 |     helpers.retrying({}, function()
143 |         t.assert_equals(cg.cluster:server('s1-master').net_box:eval([[
144 |             return #box.space.customers:select({}, {limit = 10})
145 |         ]]), 0)
146 |     end)
147 | 
148 |     -- is_master == false
149 |     t.assert_equals(cg.cluster:server("s1-slave").net_box:eval([[
150 |         local cartridge = require("cartridge")
151 |         return #cartridge.service_get("expirationd").tasks()
152 |     ]]), 0)
153 |     helpers.retrying({}, function()
154 |         t.assert_equals(cg.cluster:server('s1-slave').net_box:eval([[
155 |             return #box.space.customers:select({}, {limit = 10})
156 |         ]]), 0)
157 |     end)
158 | end
159 | 
160 | function g.test_continue_after_hotreload(cg)
161 |     t.assert_equals(10, cg.cluster:server('s1-master').net_box:eval([[
162 |         for i = 1,10 do
163 |             box.space.customers:insert({i})
164 |         end
165 |         return #box.space.customers:select({}, {limit = 20})
166 |     ]]))
167 |     cg.cluster.main_server:upload_config({
168 |         expirationd = {
169 |             test_task = {
170 |                 space = "customers",
171 |                 is_expired = "is_expired_test_continue",
172 |                 is_master_only = true,
173 |                 options = {
174 |                     process_expired_tuple = "always_true_test",
175 |                 },
176 |             }
177 |         },
178 |     })
179 | 
180 |     t.assert_equals(cg.cluster:server('s1-master').net_box:eval([[
181 |         local cartridge = require("cartridge")
182 |         return #cartridge.service_get("expirationd").tasks()
183 |     ]]), 1)
184 |     helpers.retrying({}, function()
185 |         t.assert_equals(cg.cluster:server('s1-master').net_box:eval([[
186 |             return _G.is_expired_test_first_tuple
187 |         ]]), {1})
188 |     end)
189 | 
190 |     cg.cluster:server('s1-master').net_box:eval([[
191 |         return require('cartridge.roles').reload()
192 |     ]])
193 | 
194 |     t.assert_equals(cg.cluster:server('s1-master').net_box:eval([[
195 |         local cartridge = require("cartridge")
196 |         return #cartridge.service_get("expirationd").tasks()
197 |     ]]), 1)
198 |     t.assert_equals(cg.cluster:server('s1-master').net_box:eval([[
199 |         return #box.space.customers:select({}, {limit = 20})
200 |     ]]), 10)
201 |     helpers.retrying({}, function()
202 |         t.assert_equals(cg.cluster:server('s1-master').net_box:eval([[
203 |             return _G.is_expired_test_first_tuple
204 |         ]]), {5})
205 |     end)
206 | end
207 | 


--------------------------------------------------------------------------------
/test/integration/master_replica_test.lua:
--------------------------------------------------------------------------------
  1 | local fio = require('fio')
  2 | local t = require('luatest')
  3 | local helpers = require('test.helper')
  4 | local g = t.group('expirationd_master_replica')
  5 | local is_cartridge_helpers, cartridge_helpers = pcall(require, 'cartridge.test-helpers')
  6 | 
  7 | g.before_all(function(cg)
  8 |     if is_cartridge_helpers then
  9 |         local entrypoint_path = fio.pathjoin(helpers.project_root,
 10 |                                              'test',
 11 |                                              'entrypoint',
 12 |                                              'srv_base.lua')
 13 |         cg.cluster = cartridge_helpers.Cluster:new({
 14 |             datadir = fio.tempdir(),
 15 |             server_command = entrypoint_path,
 16 |             use_vshard = true,
 17 |             replicasets = {
 18 |                 {
 19 |                     uuid = helpers.uuid('a'),
 20 |                     alias = 'router',
 21 |                     roles = { 'vshard-router' },
 22 |                     servers = {
 23 |                         { instance_uuid = helpers.uuid('a', 1), alias = 'router' },
 24 |                     },
 25 |                 },
 26 |                 {
 27 |                     uuid = helpers.uuid('b'),
 28 |                     alias = 's-1',
 29 |                     roles = { 'vshard-storage', 'customers-storage' },
 30 |                     servers = {
 31 |                         { instance_uuid = helpers.uuid('b', 1), alias = 's1-master' },
 32 |                         { instance_uuid = helpers.uuid('b', 2), alias = 's1-replica' },
 33 |                     }
 34 |                 }
 35 |             },
 36 |         })
 37 |         cg.cluster:start()
 38 |     end
 39 | end)
 40 | 
 41 | g.after_all(function(cg)
 42 |     if is_cartridge_helpers then
 43 |         cg.cluster:stop()
 44 |         fio.rmtree(cg.cluster.datadir)
 45 |     end
 46 | end)
 47 | 
 48 | g.before_each(function()
 49 |     t.skip_if(not is_cartridge_helpers, "cartridge is not installed")
 50 | end)
 51 | 
 52 | g.after_each(function(cg)
 53 |     cg.cluster:server('s1-master').net_box:eval([[
 54 |         box.space.customers:truncate()
 55 |     ]])
 56 | end)
 57 | 
 58 | local tuples_cnt = 3
 59 | local function insert_tuples(cg)
 60 |     cg.cluster:server('s1-master').net_box:eval([[
 61 |         box.space.customers:insert({1})
 62 |         box.space.customers:insert({2})
 63 |         box.space.customers:insert({3})
 64 |     ]])
 65 | end
 66 | 
 67 | local expirationd_eval = string.format([[
 68 |     local deleted = 0
 69 |     local task = require('expirationd').start("clean_all", 'customers',
 70 |                                               function()
 71 |                                                   deleted = deleted + 1
 72 |                                                   return true
 73 |                                               end,
 74 |                                               {full_scan_delay = 0})
 75 |     local retry = 100
 76 |     for _ = 1, 100 do
 77 |         if deleted == %d then
 78 |             break
 79 |         end
 80 |         require("fiber").yield()
 81 |     end
 82 |     task:kill()
 83 |     return #box.space.customers:select({}, {limit = 10})
 84 | ]], tuples_cnt)
 85 | 
 86 | function g.test_expirationd_on_master_processing(cg)
 87 |     insert_tuples(cg)
 88 |     local ret = cg.cluster:server('s1-master').net_box:eval([[
 89 |         return #box.space.customers:select({}, {limit = 10})
 90 |     ]])
 91 |     t.assert_equals(ret, tuples_cnt)
 92 | 
 93 |     ret = cg.cluster:server('s1-master').net_box:eval(expirationd_eval)
 94 |     t.assert_equals(ret, 0)
 95 | end
 96 | 
 97 | function g.test_expirationd_on_replica_no_processing(cg)
 98 |     insert_tuples(cg)
 99 |     local ret = cg.cluster:server('s1-master').net_box:eval([[
100 |         return #box.space.customers:select({}, {limit = 10})
101 |     ]])
102 |     t.assert_equals(ret, tuples_cnt)
103 | 
104 |     -- wait tuples on the replica
105 |     helpers.retrying({}, function()
106 |         local ret = cg.cluster:server('s1-replica').net_box:eval([[
107 |             return #box.space.customers:select({}, {limit = 10})
108 |         ]])
109 |         t.assert_equals(ret, tuples_cnt)
110 |     end)
111 | 
112 |     ret = cg.cluster:server('s1-replica').net_box:eval(expirationd_eval)
113 |     t.assert_equals(ret, tuples_cnt)
114 | end
115 | 


--------------------------------------------------------------------------------
/test/integration/reload_test.lua:
--------------------------------------------------------------------------------
  1 | local fio = require('fio')
  2 | local t = require('luatest')
  3 | local helpers = require('test.helper')
  4 | local g = t.group('expirationd_reload')
  5 | local is_cartridge_helpers, cartridge_helpers = pcall(require, 'cartridge.test-helpers')
  6 | 
  7 | g.before_all(function(cg)
  8 |     if is_cartridge_helpers then
  9 |         local entrypoint_path = fio.pathjoin(helpers.project_root,
 10 |                                              'test',
 11 |                                              'entrypoint',
 12 |                                              'srv_base.lua')
 13 |         cg.cluster = cartridge_helpers.Cluster:new({
 14 |             datadir = fio.tempdir(),
 15 |             server_command = entrypoint_path,
 16 |             use_vshard = true,
 17 |             replicasets = {
 18 |                 {
 19 |                     uuid = helpers.uuid('a'),
 20 |                     alias = 'router',
 21 |                     roles = { 'vshard-router' },
 22 |                     servers = {
 23 |                         { instance_uuid = helpers.uuid('a', 1), alias = 'router' },
 24 |                     },
 25 |                 },
 26 |                 {
 27 |                     uuid = helpers.uuid('b'),
 28 |                     alias = 's-1',
 29 |                     roles = { 'vshard-storage', 'customers-storage' },
 30 |                     servers = {
 31 |                         { instance_uuid = helpers.uuid('b', 1), alias = 's1-master' },
 32 |                     }
 33 |                 }
 34 |             },
 35 |         })
 36 |         cg.cluster:start()
 37 |     end
 38 | end)
 39 | 
 40 | g.after_all(function(cg)
 41 |     if is_cartridge_helpers then
 42 |         cg.cluster:stop()
 43 |         fio.rmtree(cg.cluster.datadir)
 44 |     end
 45 | end)
 46 | 
 47 | g.before_each(function()
 48 |     t.skip_if(not is_cartridge_helpers, "cartridge is not installed")
 49 | end)
 50 | 
 51 | g.after_each(function()
 52 |     g.cluster:server('s1-master').net_box:eval([[
 53 |         box.space.customers:truncate()
 54 |     ]])
 55 | end)
 56 | 
 57 | local function reload_roles(srv)
 58 |     local ok, err = srv.net_box:eval([[
 59 |         return require('cartridge.roles').reload()
 60 |     ]])
 61 | 
 62 |     t.assert_equals({ok, err}, {true, nil})
 63 | end
 64 | 
 65 | local walk_task_name = "walk_all"
 66 | local task_sleep_on_10_eval = string.format([[
 67 |     local expirationd = require('expirationd')
 68 |     local fiber = require("fiber")
 69 |     local helpers = require("test.helper")
 70 | 
 71 |     for i = 1,100 do
 72 |         box.space.customers:insert({i})
 73 |     end
 74 | 
 75 |     local tuples_cnt = 0
 76 |     local is_expired_sleep = function()
 77 |         tuples_cnt = tuples_cnt + 1
 78 |         if tuples_cnt == 10 then
 79 |             fiber.sleep(60)
 80 |         end
 81 |         return true
 82 |     end
 83 |     task = expirationd.start("%s", box.space.customers.id, is_expired_sleep,
 84 |                              {process_expired_tuple = function() return true end,
 85 |                               force = true})
 86 | 
 87 |     helpers.retrying({}, function()
 88 |         if tuples_cnt < 10 then
 89 |             error("the task do not reach a target tuple")
 90 |         end
 91 |     end)
 92 | ]], walk_task_name)
 93 | 
 94 | local task_first_tuple_eval = string.format([[
 95 |     local expirationd = require('expirationd')
 96 |     local helpers = require("test.helper")
 97 | 
 98 |     local tuple = nil
 99 |     local is_expired_tuple = function(arg, t)
100 |         if tuple == nil then
101 |             tuple = t
102 |         end
103 |         return true
104 |     end
105 |     task = expirationd.start("%s", box.space.customers.id, is_expired_tuple,
106 |                              {force = true})
107 | 
108 |     helpers.retrying({}, function()
109 |         if tuple == nil then
110 |             error("the task is not started")
111 |         end
112 |     end)
113 | 
114 |     task:kill()
115 | 
116 |     return tuple or {}
117 | ]], walk_task_name)
118 | 
119 | function g.test_task_continue_after_reload(cg)
120 |     local ok = cg.cluster:server('s1-master').net_box:eval(task_sleep_on_10_eval .. [[
121 |         return true
122 |     ]])
123 |     t.assert_equals(ok, true)
124 | 
125 |     reload_roles(cg.cluster:server('s1-master'))
126 | 
127 |     local tuple = cg.cluster:server('s1-master').net_box:eval(task_first_tuple_eval)
128 |     t.assert_equals(tuple, {10})
129 | end
130 | 
131 | function g.test_task_continue_after_stop_and_reload(cg)
132 |     local ok = cg.cluster:server('s1-master').net_box:eval(task_sleep_on_10_eval .. [[
133 |         task:stop()
134 |         return true
135 |     ]])
136 |     t.assert_equals(ok, true)
137 | 
138 |     reload_roles(cg.cluster:server('s1-master'))
139 | 
140 |     local tuple = cg.cluster:server('s1-master').net_box:eval(task_first_tuple_eval)
141 |     t.assert_equals(tuple, {10})
142 | end
143 | 
144 | function g.test_task_not_continue_after_kill_and_reload(cg)
145 |     local ok = cg.cluster:server('s1-master').net_box:eval(task_sleep_on_10_eval .. [[
146 |         task:kill()
147 |         return true
148 |     ]])
149 |     t.assert_equals(ok, true)
150 | 
151 |     reload_roles(cg.cluster:server('s1-master'))
152 | 
153 |     local tuple = cg.cluster:server('s1-master').net_box:eval(task_first_tuple_eval)
154 |     t.assert_equals(tuple, {1})
155 | end
156 | 
157 | function g.test_cfg_metrics_disable_after_reload(cg)
158 |     t.skip_if(not helpers.is_metrics_supported(),
159 |               "metrics >= 0.11.0 is not installed")
160 | 
161 |     cg.cluster:server('router').net_box:eval([[
162 |         local expirationd = require('expirationd')
163 |         expirationd.cfg({metrics = false})
164 |     ]])
165 | 
166 |     reload_roles(cg.cluster:server('router'))
167 | 
168 |     local ret = cg.cluster:server('router').net_box:eval([[
169 |         return require('expirationd').cfg.metrics
170 |     ]])
171 |     t.assert_equals(ret, false)
172 | end
173 | 
174 | function g.test_cfg_metrics_enable_after_reload(cg)
175 |     t.skip_if(not helpers.is_metrics_supported(),
176 |               "metrics >= 0.11.0 is not installed")
177 | 
178 |     cg.cluster:server('router').net_box:eval([[
179 |         local expirationd = require('expirationd')
180 |         expirationd.cfg({metrics = true})
181 |     ]])
182 | 
183 |     reload_roles(cg.cluster:server('router'))
184 | 
185 |     local ret = cg.cluster:server('router').net_box:eval([[
186 |         return require('expirationd').cfg.metrics
187 |     ]])
188 |     t.assert_equals(ret, true)
189 | end
190 | 
191 | local function has_expirationd(metrics)
192 |     local expected = "expirationd_"
193 |     for _, v in ipairs(metrics) do
194 |         if string.sub(v.metric_name, 1, string.len(expected)) == expected then
195 |             return true
196 |         end
197 |     end
198 |     return false
199 | end
200 | 
201 | function g.test_cfg_metrics_clean_after_reload(cg)
202 |     t.skip_if(not helpers.is_metrics_supported(),
203 |               "metrics >= 0.11.0 is not installed")
204 | 
205 |     local metrics = cg.cluster:server('s1-master').net_box:eval([[
206 |         local metrics = require('metrics')
207 |         local expirationd = require('expirationd')
208 | 
209 |         expirationd.cfg({metrics = true})
210 |         expirationd.start("stats_basic", 'customers',
211 |                           function()
212 |                               return true
213 |                           end)
214 |         metrics.invoke_callbacks()
215 |         return metrics.collect()
216 |     ]])
217 |     t.assert(has_expirationd(metrics))
218 | 
219 |     reload_roles(cg.cluster:server('s1-master'))
220 | 
221 |     local metrics = cg.cluster:server('s1-master').net_box:eval([[
222 |         local metrics = require('metrics')
223 |         local expirationd = require('expirationd')
224 | 
225 |         metrics.invoke_callbacks()
226 |         return metrics.collect()
227 |     ]])
228 |     t.assert_not(has_expirationd(metrics))
229 | end
230 | 


--------------------------------------------------------------------------------
/test/integration/simple_app/config.yaml:
--------------------------------------------------------------------------------
 1 | credentials:
 2 |   users:
 3 |     guest:
 4 |       roles: [super]
 5 | 
 6 | groups:
 7 |   group-001:
 8 |     replicasets:
 9 |       replicaset-001:
10 |         roles: [roles.expirationd]
11 |         roles_cfg:
12 |           roles.expirationd:
13 |             task_name1:
14 |               space: users
15 |               is_expired: forever_true_test
16 |         instances:
17 |           master:
18 |             iproto:
19 |               listen:
20 |                 - uri: '127.0.0.1:3313'
21 |             database:
22 |               mode: rw
23 | 


--------------------------------------------------------------------------------
/test/integration/simple_app/instances.yml:
--------------------------------------------------------------------------------
1 | master:
2 | 


--------------------------------------------------------------------------------
/test/integration/tarantool_role_test.lua:
--------------------------------------------------------------------------------
  1 | local t = require('luatest')
  2 | local fio = require('fio')
  3 | 
  4 | local helpers = require('test.helper')
  5 | local Server = require('test.helper_server')
  6 | 
  7 | local g = t.group('tarantool_role_integration_test')
  8 | 
  9 | g.before_all(function (cg)
 10 |     t.skip_if(not helpers.tarantool_role_is_supported(),
 11 |              'Tarantool role is supported only for Tarantool starting from v3.0.0')
 12 | 
 13 |     local workdir = fio.tempdir()
 14 |     cg.router = Server:new({
 15 |         config_file = fio.abspath(fio.pathjoin('test', 'integration', 'simple_app', 'config.yaml')),
 16 |         env = {LUA_PATH = helpers.lua_path},
 17 |         chdir = workdir,
 18 |         alias = 'master',
 19 |         workdir = workdir,
 20 |     })
 21 | end)
 22 | 
 23 | g.before_each(function(cg)
 24 |     fio.mktree(cg.router.workdir)
 25 | 
 26 |     -- We start instance before each test because
 27 |     -- we need to force reload of expirationd role and also instance environment
 28 |     -- from previous tests can influence test result.
 29 |     -- (e.g function creation, when testing that role doesn't start w/o it)
 30 |     -- Restarting instance is the easiest way to achieve it.
 31 |     -- It takes around 1s to start an instance, which considering small amount
 32 |     -- of integration tests is not a problem.
 33 |     cg.router:start{wait_until_ready = true}
 34 | 
 35 |     cg.router:exec(function()
 36 |         box.watch('box.status', function(_, status)
 37 |             if status.is_ro == false then
 38 |                 box.schema.create_space('users', {if_not_exists = true})
 39 | 
 40 |                 box.space.users:format({
 41 |                     {name = 'id', type = 'unsigned'},
 42 |                     {name = 'first name', type = 'string'},
 43 |                     {name = 'second name', type = 'string', is_nullable = true},
 44 |                     {name = 'age', type = 'number', is_nullable = false},
 45 |                 })
 46 | 
 47 |                 box.space.users:create_index('primary', {
 48 |                     parts = {
 49 |                         {field = 1, type = 'unsigned'},
 50 |                     },
 51 |                 })
 52 | 
 53 |                 box.space.users:insert{1, 'Samantha', 'Carter', 30}
 54 |                 box.space.users:insert{2, 'Fay', 'Rivers', 41}
 55 |                 box.space.users:insert{3, 'Zachariah', 'Peters', 13}
 56 |                 box.space.users:insert{4, 'Milo', 'Walters', 74}
 57 |             end
 58 |         end)
 59 |     end)
 60 | end)
 61 | 
 62 | g.after_each(function(cg)
 63 |     cg.router:stop()
 64 |     fio.rmtree(cg.router.workdir)
 65 | end)
 66 | 
 67 | g.test_simple_role_config = function(cg)
 68 |     cg.router:exec(function()
 69 |         box.schema.func.create('forever_true_test', {
 70 |             body = "function(...) return true end",
 71 |             if_not_exists = true
 72 |         })
 73 |     end)
 74 | 
 75 |     helpers.retrying({}, function()
 76 |         t.assert_equals(cg.router:exec(function()
 77 |             return #box.space.users:select({}, {limit = 10})
 78 |         end), 0)
 79 |     end)
 80 | end
 81 | 
 82 | g.test_waiting_for_functions = function(cg)
 83 |     -- Function for is_expired config entry was not created, so expirationd
 84 |     -- is waiting for it's creation.
 85 |     helpers.retrying({}, function()
 86 |         t.assert_equals(cg.router:exec(function()
 87 |             return box.space.users:select({}, {limit = 10})
 88 |         end), {
 89 |             {1, 'Samantha', 'Carter', 30},
 90 |             {2, 'Fay', 'Rivers', 41},
 91 |             {3, 'Zachariah', 'Peters', 13},
 92 |             {4, 'Milo', 'Walters', 74},
 93 |         })
 94 |     end)
 95 | 
 96 |     -- Create is_expired function and check that expirationd task started.
 97 |     cg.router:exec(function()
 98 |         box.schema.func.create('forever_true_test', {
 99 |             body = "function(...) return true end",
100 |             if_not_exists = true
101 |         })
102 |     end)
103 | 
104 |     helpers.retrying({}, function()
105 |         t.assert_equals(cg.router:exec(function()
106 |             return #box.space.users:select({}, {limit = 10})
107 |         end), 0)
108 |     end)
109 | 
110 | end
111 | 
112 | g.test_function_created_in_g = function(cg)
113 |     cg.router:exec(function()
114 |         rawset(_G, 'forever_true_test', function()
115 |             return true
116 |         end)
117 |     end)
118 | 
119 |     helpers.retrying({}, function()
120 |         t.assert_equals(cg.router:exec(function()
121 |             return #box.space.users:select({}, {limit = 10})
122 |         end), 0)
123 |     end)
124 | end
125 | 


--------------------------------------------------------------------------------
/test/unit/atomic_iteration_test.lua:
--------------------------------------------------------------------------------
  1 | local fiber = require("fiber")
  2 | local expirationd = require("expirationd")
  3 | local t = require("luatest")
  4 | 
  5 | local helpers = require("test.helper")
  6 | 
  7 | local g = t.group('atomic_iteration', {
  8 |     {index_type = 'TREE', engine = 'vinyl'},
  9 |     {index_type = 'TREE', engine = 'memtx'},
 10 |     {index_type = 'HASH', engine = 'memtx'},
 11 | })
 12 | 
 13 | g.before_each({index_type = 'TREE'}, function(cg)
 14 |     g.space = helpers.create_space_with_tree_index(cg.params.engine)
 15 | end)
 16 | 
 17 | g.before_each({index_type = 'HASH'}, function(cg)
 18 |     g.space = helpers.create_space_with_hash_index(cg.params.engine)
 19 | end)
 20 | 
 21 | g.after_each(function(g)
 22 |     g.space:drop()
 23 | end)
 24 | 
 25 | function g.test_passing(cg)
 26 |     local task = expirationd.start("clean_all", cg.space.id, helpers.is_expired_true)
 27 |     t.assert_equals(task.atomic_iteration, false)
 28 |     task:kill()
 29 | 
 30 |     task = expirationd.start("clean_all", cg.space.id, helpers.is_expired_true,
 31 |             {atomic_iteration = true})
 32 |     t.assert_equals(task.atomic_iteration, true)
 33 |     task:kill()
 34 | 
 35 |     -- errors
 36 |     t.assert_error_msg_content_equals("bad argument options.atomic_iteration to nil (?boolean expected, got number)",
 37 |             expirationd.start, "clean_all", cg.space.id, helpers.is_expired_true,
 38 |             {atomic_iteration = 1})
 39 | end
 40 | 
 41 | function g.test_memtx(cg)
 42 |     t.skip_if(cg.params.engine ~= 'memtx', 'Unsupported engine')
 43 | 
 44 |     helpers.iteration_result = {}
 45 | 
 46 |     local space = cg.space
 47 |     local transactions = {}
 48 |     local function f(iterator)
 49 |         local transaction = {}
 50 |         -- old / new_tuple is not passed for vinyl
 51 |         for _, old_tuple in iterator() do
 52 |             table.insert(transaction, old_tuple:totable())
 53 |         end
 54 |         table.insert(transactions, transaction)
 55 |     end
 56 | 
 57 |     local true_box_begin = box.begin
 58 | 
 59 |     -- mock box.begin
 60 |     box.begin = function ()
 61 |         true_box_begin()
 62 |         box.on_commit(f)
 63 |     end
 64 | 
 65 |     -- tuples expired in one atomic_iteration
 66 |     space:insert({1, "3"})
 67 |     space:insert({2, "2"})
 68 |     space:insert({3, "1"})
 69 | 
 70 | 
 71 |     local task = expirationd.start("clean_all", space.id, helpers.is_expired_debug,
 72 |             {atomic_iteration = true})
 73 |     -- wait for tuples expired
 74 |     helpers.retrying({}, function()
 75 |         if space.index[0].type == "HASH" then
 76 |             t.assert_equals(helpers.iteration_result, {{3, "1"}, {2, "2"}, {1, "3"}})
 77 |         else
 78 |             t.assert_equals(helpers.iteration_result, {{1, "3"}, {2, "2"}, {3, "1"}})
 79 |         end
 80 |     end)
 81 |     task:kill()
 82 |     helpers.iteration_result = {}
 83 | 
 84 |     -- check out three row transaction
 85 |     if space.index[0].type == "HASH" then
 86 |         t.assert_equals(transactions, {
 87 |             { {3, "1"}, {2, "2"}, {1, "3"} }
 88 |         })
 89 |     else
 90 |         t.assert_equals(transactions, {
 91 |             { {1, "3"}, {2, "2"}, {3, "1"} }
 92 |         })
 93 |     end
 94 |     transactions = {}
 95 | 
 96 |     -- tuples expired in two atomic_iteration
 97 |     space:insert({1, "3"})
 98 |     space:insert({2, "2"})
 99 |     space:insert({3, "1"})
100 | 
101 |     task = expirationd.start("clean_all", space.id, helpers.is_expired_debug,
102 |             {atomic_iteration = true, tuples_per_iteration = 2})
103 |     -- wait for tuples expired
104 |     -- 2 seconds because suspend will be yield in task
105 |     helpers.retrying({}, function()
106 |         if space.index[0].type == "HASH" then
107 |             t.assert_equals(helpers.iteration_result, {{3, "1"}, {2, "2"}, {1, "3"}})
108 |         else
109 |             t.assert_equals(helpers.iteration_result, {{1, "3"}, {2, "2"}, {3, "1"}})
110 |         end
111 |     end)
112 |     task:kill()
113 |     helpers.iteration_result = {}
114 | 
115 |     if space.index[0].type == "HASH" then
116 |         t.assert_equals(transactions, {
117 |             { {3, "1"}, {2, "2"} }, -- check two row transaction
118 |             { {1, "3"} }            -- check single row transactions
119 |         })
120 |     else
121 |         t.assert_equals(transactions, {
122 |             { {1, "3"}, {2, "2"} }, -- check two row transaction
123 |             { {3, "1"} }            -- check single row transactions
124 |         })
125 |     end
126 | 
127 |     transactions = {}
128 | 
129 |     -- unmock
130 |     box.begin = true_box_begin
131 | end
132 | 
133 | -- it's not check tarantool or vinyl as engine
134 | -- just check expirationd task continue work after conflicts
135 | function g.test_mvcc_vinyl_tx_conflict(cg)
136 |     t.skip_if(cg.params.engine ~= 'vinyl', 'Unsupported engine')
137 |     local tuples_cnt = 10
138 | 
139 |     for i = 1,tuples_cnt do
140 |         cg.space:insert({i, tostring(i), nil, nil, 0})
141 |     end
142 | 
143 |     local updaters = {}
144 |     for i = 1,tuples_cnt do
145 |         local updater = fiber.new(function()
146 |             fiber.name(string.format("updater of %d", i), { truncate = true })
147 |             cg.space:update({i}, { {"+", 5, 1} })
148 |         end)
149 |         updater:set_joinable(true)
150 |         table.insert(updaters, updater)
151 |     end
152 | 
153 |     local is_expired = function(args, tuple)
154 |         -- The idea is to switch explicity to an updater fiber in the middle of
155 |         -- an expirationd's transaction:
156 |         -- Delete from expirationd + update from an updater == conflict at the
157 |         -- expirationd's transaction.
158 |         fiber.yield()
159 |         return helpers.is_expired_debug(args, tuple)
160 |     end
161 | 
162 |     helpers.iteration_result = {}
163 |     local task = expirationd.start("clean_all", cg.space.id, is_expired,
164 |                                    {atomic_iteration = true})
165 |     -- ensure that expirationd task does not delete a tuple yet
166 |     t.assert_equals(helpers.iteration_result, {})
167 | 
168 |     for _, updater in pairs(updaters) do
169 |         updater:join()
170 |     end
171 | 
172 |     helpers.retrying({}, function()
173 |         t.assert_equals(cg.space:select(), {})
174 |     end)
175 |     t.assert_gt(box.stat.vinyl().tx.conflict, 0)
176 |     t.assert_gt(#helpers.iteration_result, tuples_cnt)
177 |     t.assert_equals(box.stat.vinyl().tx.conflict, box.stat.vinyl().tx.rollback)
178 |     task:kill()
179 | end
180 | 
181 | -- Create a task that use atomic_iteration and check that task is gone after
182 | -- kill.
183 | function g.test_kill_task(cg)
184 |     local task_name = 'clean_all'
185 | 
186 |     for i = 1, 100 do
187 |         cg.space:insert({i, tostring(i)})
188 |     end
189 | 
190 |     local task = expirationd.start(task_name, cg.space.id, helpers.is_expired_debug, {
191 |         atomic_iteration = true,
192 |         tuples_per_iteration = 10,
193 |     })
194 |     task:kill()
195 | 
196 |     -- There are two methods to know about task state:
197 |     -- expirationd.task(task_name) and expirationd.stats() that returns
198 |     -- statistics for each task. expirationd.task(task_name) raise error if
199 |     -- task is not found. So we use stats() here and check that there are no
200 |     -- stats for task with our task name.
201 |     local stats = expirationd.stats()
202 |     t.assert_equals(stats[task_name], nil)
203 | end
204 | 


--------------------------------------------------------------------------------
/test/unit/callbacks_and_delays_test.lua:
--------------------------------------------------------------------------------
  1 | local expirationd = require('expirationd')
  2 | local fiber = require('fiber')
  3 | local t = require('luatest')
  4 | 
  5 | local helpers = require('test.helper')
  6 | 
  7 | local g = t.group('callbacks_and_delays', {
  8 |     {index_type = 'TREE', engine = 'vinyl'},
  9 |     {index_type = 'TREE', engine = 'memtx'},
 10 |     {index_type = 'HASH', engine = 'memtx'},
 11 | })
 12 | 
 13 | g.before_each({index_type = 'TREE'}, function(cg)
 14 |     t.skip_if(cg.params.engine == 'vinyl' and not helpers.vinyl_is_supported(),
 15 |         'Blocked by https://github.com/tarantool/tarantool/issues/6448')
 16 |     cg.space = helpers.create_space_with_tree_index(cg.params.engine)
 17 | end)
 18 | 
 19 | g.before_each({index_type = 'HASH'}, function(cg)
 20 |     cg.space = helpers.create_space_with_hash_index(cg.params.engine)
 21 | end)
 22 | 
 23 | g.before_each(function(cg)
 24 |     cg.task_name = 'test'
 25 | 
 26 |     local space = cg.space
 27 |     local total = 10
 28 |     for i = 1, total do
 29 |         space:insert({i, tostring(i)})
 30 |     end
 31 |     t.assert_equals(space:count(), total)
 32 | end)
 33 | 
 34 | g.after_each(function(cg)
 35 |     if cg.task ~= nil then
 36 |         cg.task:kill()
 37 |     end
 38 |     cg.space:drop()
 39 | end)
 40 | 
 41 | function g.test_delays_and_scan_callbacks(cg)
 42 |     local space = cg.space
 43 |     local task_name = cg.task_name
 44 | 
 45 |     -- To check all delays (iteration and full scan), two full scan
 46 |     -- iterations will be performed.
 47 |     local first_fullscan_done = false
 48 |     local cond = fiber.cond()
 49 |     local start_time = 0
 50 |     local complete_time = 0
 51 | 
 52 |     local check_full_scan_delay = true
 53 |     local check_iteration_delay = true
 54 |     local iteration_delay = 1
 55 |     local full_scan_delay = 2
 56 |     local full_scan_success_counter = 0
 57 | 
 58 |     local check_full_scan_delay_cb = function()
 59 |         start_time = fiber.time()
 60 |         if first_fullscan_done and check_full_scan_delay then
 61 |             -- Check the full scan delay with an accuracy of 0.1 seconds.
 62 |             -- Difference between start time of the second full scan
 63 |             -- and complete_time of the first full scan.
 64 |             check_full_scan_delay = math.abs(start_time - complete_time
 65 |                 - full_scan_delay) < 0.1
 66 |         end
 67 |     end
 68 | 
 69 |     local call_counter = function()
 70 |         -- Must be called twice.
 71 |         full_scan_success_counter = full_scan_success_counter + 1
 72 |     end
 73 | 
 74 |     local check_iteration_delay_cb = function()
 75 |         complete_time = fiber.time()
 76 |         if first_fullscan_done then
 77 |             cond:signal()
 78 |         else
 79 |             first_fullscan_done = true
 80 |             -- Check the accuracy of iteration delay.
 81 |             -- Difference between start time and complete_time of the first full scan.
 82 |             if check_iteration_delay then
 83 |                 check_iteration_delay = math.abs(complete_time - start_time -
 84 |                     iteration_delay) < 2
 85 |             end
 86 |         end
 87 |     end
 88 | 
 89 |     cg.task = expirationd.start(task_name, space.id,
 90 |         helpers.is_expired_true,
 91 |         {
 92 |             iteration_delay = iteration_delay,
 93 |             full_scan_delay = full_scan_delay,
 94 |             tuples_per_iteration = 5,
 95 |             on_full_scan_start = check_full_scan_delay_cb,
 96 |             on_full_scan_success = call_counter,
 97 |             on_full_scan_complete = check_iteration_delay_cb,
 98 |             vinyl_assumed_space_len = 5, -- iteration_delay will be 1 sec
 99 |         }
100 |     )
101 | 
102 |     cond:wait()
103 |     cg.task:kill()
104 |     cg.task = nil
105 | 
106 |     t.assert(check_full_scan_delay)
107 |     t.assert(check_iteration_delay)
108 |     t.assert_equals(full_scan_success_counter, 2)
109 |     t.assert_equals(space:count(), 0)
110 | end
111 | 
112 | function g.test_error_callback(cg)
113 |     local space = cg.space
114 |     local task_name = cg.task_name
115 | 
116 |     local cond = fiber.cond()
117 | 
118 |     local error_cb_called = false
119 |     local complete_cb_called = false
120 |     local error_message = 'The error is occurred'
121 | 
122 |     local check_error_cb_called = function(_, err)
123 |         if err:find(error_message) then
124 |             error_cb_called = true
125 |         end
126 |     end
127 | 
128 |     local check_complete_cb_called = function()
129 |         complete_cb_called = true
130 |         cond:signal()
131 |     end
132 | 
133 |     cg.task = expirationd.start(task_name, space.id,
134 |         helpers.get_error_function(error_message),
135 |         {
136 |             -- The callbacks can be called multiple times because guardian_loop
137 |             -- will restart the task.
138 |             on_full_scan_error = check_error_cb_called,
139 |             on_full_scan_complete = check_complete_cb_called
140 |         }
141 |     )
142 | 
143 |     cond:wait()
144 | 
145 |     -- The 'error' callback has been invoked.
146 |     t.assert(error_cb_called)
147 |     -- The 'complete' callback has been invoked.
148 |     t.assert(complete_cb_called)
149 | end
150 | 


--------------------------------------------------------------------------------
/test/unit/cfg_test.lua:
--------------------------------------------------------------------------------
 1 | local expirationd = require("expirationd")
 2 | local t = require("luatest")
 3 | local helpers = require("test.helper")
 4 | local g = t.group('expirationd_cfg')
 5 | 
 6 | local metrics_required_msg = "metrics >= 0.11.0 is not installed"
 7 | local metrics_not_required_msg = "metrics >= 0.11.0 is installed"
 8 | 
 9 | g.before_all(function()
10 |     g.default_cfg = { metrics = expirationd.cfg.metrics }
11 | end)
12 | 
13 | g.after_each(function()
14 |     expirationd.cfg(g.default_cfg)
15 | end)
16 | 
17 | function g.test_cfg_default_if_installed()
18 |     t.skip_if(not helpers.is_metrics_supported(), metrics_required_msg)
19 |     t.assert_equals(expirationd.cfg.metrics, true)
20 | end
21 | 
22 | function g.test_cfg_default_if_uninstalled()
23 |     t.skip_if(helpers.is_metrics_supported(), metrics_not_required_msg)
24 |     t.assert_equals(expirationd.cfg.metrics, false)
25 | end
26 | 
27 | function g.test_cfg_newindex()
28 |     t.assert_error_msg_content_equals("Use expirationd.cfg{} instead",
29 |                                       function()
30 |                                           expirationd.cfg.any_key = false
31 |                                       end)
32 | end
33 | 
34 | function g.test_cfg_metrics_set_unset()
35 |     t.skip_if(not helpers.is_metrics_supported(), metrics_required_msg)
36 | 
37 |     expirationd.cfg({metrics = true})
38 |     t.assert_equals(expirationd.cfg.metrics, true)
39 |     expirationd.cfg({metrics = false})
40 |     t.assert_equals(expirationd.cfg.metrics, false)
41 | end
42 | 
43 | function g.test_cfg_metrics_multiple_set_unset()
44 |     t.skip_if(not helpers.is_metrics_supported(), metrics_required_msg)
45 | 
46 |     expirationd.cfg({metrics = true})
47 |     expirationd.cfg({metrics = true})
48 |     t.assert_equals(expirationd.cfg.metrics, true)
49 |     expirationd.cfg({metrics = false})
50 |     expirationd.cfg({metrics = false})
51 |     t.assert_equals(expirationd.cfg.metrics, false)
52 | end
53 | 
54 | function g.test_cfg_metrics_set_unsupported()
55 |     t.skip_if(helpers.is_metrics_supported(), metrics_not_required_msg)
56 | 
57 |     t.assert_error_msg_content_equals("metrics >= 0.11.0 is required",
58 |                                       function()
59 |                                           expirationd.cfg({metrics = true})
60 |                                       end)
61 |     t.assert_equals(expirationd.cfg.metrics, false)
62 | end
63 | 


--------------------------------------------------------------------------------
/test/unit/continue_test.lua:
--------------------------------------------------------------------------------
  1 | local expirationd = require("expirationd")
  2 | local fiber = require("fiber")
  3 | local t = require("luatest")
  4 | local helpers = require("test.helper")
  5 | local g = t.group('expirationd_continue')
  6 | 
  7 | local task_name = "walk_all"
  8 | g.before_each(function()
  9 |     g.space = helpers.create_space_with_tree_index('memtx')
 10 |     for _, task in ipairs(expirationd.tasks()) do
 11 |          if task == task_name then
 12 |              expirationd.task(task_name):kill()
 13 |          end
 14 |     end
 15 | end)
 16 | 
 17 | g.after_each(function()
 18 |     g.space:drop()
 19 |     if box.space.tmp ~= nil then
 20 |         box.space.tmp:drop()
 21 |     end
 22 | end)
 23 | 
 24 | local tuples_wait_event = {{1, "1"}, {2, "2"}, {3, "3"}, {4, "4"}, {5, "5"}}
 25 | local tuples_all = {{1, "1"}, {2, "2"}, {3, "3"}, {4, "4"}, {5, "5"},
 26 |                     {6, "6"}, {7, "7"}, {8, "8"}, {9, "9"}, {10, "10"}}
 27 | local tuples_repeat = {{1, "1"}, {2, "2"}, {3, "3"}, {4, "4"}, {5, "5"},
 28 |                        {1, "1"}, {2, "2"}, {3, "3"}, {4, "4"}, {5, "5"},
 29 |                        {6, "6"}, {7, "7"}, {8, "8"}, {9, "9"}, {10, "10"}}
 30 | 
 31 | local function insert_tuples(space)
 32 |     for i = 1,10 do
 33 |         space:insert({i, tostring(i)})
 34 |     end
 35 | end
 36 | 
 37 | local function start_walk_task(space, sleep)
 38 |     local cnt = 0
 39 |     local is_expired = function(args, tuple)
 40 |         cnt = cnt + 1
 41 |         if cnt == 6 then
 42 |             if sleep then
 43 |                 fiber.sleep(60)
 44 |             else
 45 |                 error("test error in iteration")
 46 |             end
 47 |         end
 48 |         return helpers.is_expired_debug(args, tuple)
 49 |     end
 50 |     local task = expirationd.start(task_name, space.id, is_expired,
 51 |                                    {process_expired_tuple = function() return true end})
 52 |     return task
 53 | end
 54 | 
 55 | function g.test_task_continue_after_error()
 56 |     insert_tuples(g.space)
 57 | 
 58 |     helpers.iteration_result = {}
 59 |     local task = start_walk_task(g.space, false)
 60 |     helpers.retrying({}, function()
 61 |         t.assert_equals(helpers.iteration_result, tuples_wait_event)
 62 |     end)
 63 | 
 64 |     helpers.retrying({}, function()
 65 |         t.assert_equals(helpers.iteration_result, tuples_all)
 66 |     end)
 67 | 
 68 |     task:kill()
 69 | end
 70 | 
 71 | function g.test_task_continue_after_stop_start()
 72 |     insert_tuples(g.space)
 73 | 
 74 |     helpers.iteration_result = {}
 75 |     local task = start_walk_task(g.space, true)
 76 |     helpers.retrying({}, function()
 77 |         t.assert_equals(helpers.iteration_result, tuples_wait_event)
 78 |     end)
 79 | 
 80 |     task:stop()
 81 |     task:start()
 82 | 
 83 |     helpers.retrying({}, function()
 84 |         t.assert_equals(helpers.iteration_result, tuples_all)
 85 |     end)
 86 | 
 87 |     task:kill()
 88 | end
 89 | 
 90 | function g.test_task_continue_after_stop_recreate()
 91 |     insert_tuples(g.space)
 92 | 
 93 |     helpers.iteration_result = {}
 94 |     local task = start_walk_task(g.space, true)
 95 |     helpers.retrying({}, function()
 96 |         t.assert_equals(helpers.iteration_result, tuples_wait_event)
 97 |     end)
 98 | 
 99 |     task:stop()
100 |     local task = expirationd.start(task_name, g.space.id, helpers.is_expired_debug,
101 |                                    {process_expired_tuple = function() return true end})
102 | 
103 | 
104 |     helpers.retrying({}, function()
105 |         t.assert_equals(helpers.iteration_result, tuples_all)
106 |     end)
107 | 
108 |     task:kill()
109 | end
110 | 
111 | function g.test_task_not_continue_after_kill_start()
112 |     insert_tuples(g.space)
113 | 
114 |     helpers.iteration_result = {}
115 |     local task = start_walk_task(g.space, true)
116 |     helpers.retrying({}, function()
117 |         t.assert_equals(helpers.iteration_result, tuples_wait_event)
118 |     end)
119 | 
120 |     task:kill()
121 | 
122 |     local task = expirationd.start(task_name, g.space.id, helpers.is_expired_debug,
123 |                                    {process_expired_tuple = function() return true end})
124 |     helpers.retrying({}, function()
125 |         t.assert_equals(helpers.iteration_result, tuples_repeat)
126 |     end)
127 | 
128 |     task:kill()
129 | end
130 | 
131 | function g.test_task_not_continue_after_restart()
132 |     insert_tuples(g.space)
133 | 
134 |     helpers.iteration_result = {}
135 |     local task = start_walk_task(g.space, true)
136 |     helpers.retrying({}, function()
137 |         t.assert_equals(helpers.iteration_result, tuples_wait_event)
138 |     end)
139 | 
140 |     task:restart()
141 | 
142 |     helpers.retrying({}, function()
143 |         t.assert_equals(helpers.iteration_result, tuples_repeat)
144 |     end)
145 | 
146 |     task:kill()
147 | end
148 | 
149 | function g.test_task_not_continue_after_index_changed()
150 |     local space = box.schema.create_space("tmp")
151 |     local index = space:create_index("primary", {type = "TREE", parts = {{field = 1}}})
152 |     insert_tuples(space)
153 | 
154 |     helpers.iteration_result = {}
155 |     local task = start_walk_task(space, false)
156 |     helpers.retrying({}, function()
157 |         t.assert_equals(helpers.iteration_result, tuples_wait_event)
158 |     end)
159 | 
160 |     index:alter({parts = {{field = 2}}})
161 | 
162 |     helpers.retrying({}, function()
163 |         t.assert_equals(helpers.iteration_result,
164 |                         {{1, "1"}, {2, "2"}, {3, "3"}, {4, "4"}, {5, "5"},
165 |                          {1, "1"}, {10, "10"}, {2, "2"}, {3, "3"}, {4, "4"},
166 |                          {5, "5"}, {6, "6"}, {7, "7"}, {8, "8"}, {9, "9"}})
167 |     end)
168 | 
169 |     task:kill()
170 |     space:drop()
171 | end
172 | 
173 | function g.test_task_not_continue_after_stop_recreate_other_space()
174 |     insert_tuples(g.space)
175 |     local space = box.schema.create_space("tmp")
176 |     space:create_index("primary", {type = "TREE", parts = {{field = 1}}})
177 |     insert_tuples(space)
178 | 
179 |     helpers.iteration_result = {}
180 |     local task = start_walk_task(g.space, true)
181 |     helpers.retrying({}, function()
182 |         t.assert_equals(helpers.iteration_result, tuples_wait_event)
183 |     end)
184 | 
185 |     task:stop()
186 |     local task = expirationd.start(task_name, space.id, helpers.is_expired_debug,
187 |                                    {process_expired_tuple = function() return true end})
188 | 
189 |     helpers.retrying({}, function()
190 |         t.assert_equals(helpers.iteration_result, tuples_repeat)
191 |     end)
192 | 
193 |     task:kill()
194 |     space:drop()
195 | end
196 | 
197 | function g.test_task_not_continue_after_stop_recreate_other_index()
198 |     insert_tuples(g.space)
199 | 
200 |     helpers.iteration_result = {}
201 |     local task = start_walk_task(g.space, true)
202 |     helpers.retrying({}, function()
203 |         t.assert_equals(helpers.iteration_result, tuples_wait_event)
204 |     end)
205 | 
206 |     task:stop()
207 |     local task = expirationd.start(task_name, g.space.id, helpers.is_expired_debug,
208 |                                    {process_expired_tuple = function() return true end,
209 |                                     index = "index_for_first_name" })
210 | 
211 | 
212 |     helpers.retrying({}, function()
213 |         t.assert_equals(helpers.iteration_result,
214 |                         {{1, "1"}, {2, "2"}, {3, "3"}, {4, "4"}, {5, "5"},
215 |                          {1, "1"}, {10, "10"}, {2, "2"}, {3, "3"}, {4, "4"},
216 |                          {5, "5"}, {6, "6"}, {7, "7"}, {8, "8"}, {9, "9"}})
217 |     end)
218 | 
219 |     task:kill()
220 | end
221 | 
222 | function g.test_task_not_continue_after_stop_recreate_other_iterator()
223 |     insert_tuples(g.space)
224 | 
225 |     helpers.iteration_result = {}
226 |     local task = start_walk_task(g.space, true)
227 |     helpers.retrying({}, function()
228 |         t.assert_equals(helpers.iteration_result, tuples_wait_event)
229 |     end)
230 | 
231 |     task:stop()
232 |     local task = expirationd.start(task_name, g.space.id, helpers.is_expired_debug,
233 |                                    {process_expired_tuple = function() return true end,
234 |                                     iterator_type = box.index.GE})
235 | 
236 | 
237 |     helpers.retrying({}, function()
238 |         t.assert_equals(helpers.iteration_result, tuples_repeat)
239 |     end)
240 | 
241 |     task:kill()
242 | end
243 | 


--------------------------------------------------------------------------------
/test/unit/custom_index_test.lua:
--------------------------------------------------------------------------------
  1 | local expirationd = require("expirationd")
  2 | local t = require("luatest")
  3 | 
  4 | local helpers = require("test.helper")
  5 | 
  6 | local g = t.group('custom_index', {
  7 |     {index_type = 'TREE', engine = 'vinyl'},
  8 |     {index_type = 'TREE', engine = 'memtx'},
  9 |     {index_type = 'HASH', engine = 'memtx'},
 10 | })
 11 | 
 12 | g.before_each({index_type = 'TREE'}, function(cg)
 13 |     t.skip_if(cg.params.engine == 'vinyl' and not helpers.vinyl_is_supported(),
 14 |         'Blocked by https://github.com/tarantool/tarantool/issues/6448 on ' ..
 15 | 		'this Tarantool version')
 16 |     g.space = helpers.create_space_with_tree_index(cg.params.engine)
 17 | end)
 18 | 
 19 | g.before_each({index_type = 'HASH'}, function(cg)
 20 |     g.space = helpers.create_space_with_hash_index(cg.params.engine)
 21 | end)
 22 | 
 23 | g.before_each({index_type = 'BITSET'}, function(cg)
 24 |     g.space = helpers.create_space_with_bitset_index(cg.params.engine)
 25 | end)
 26 | 
 27 | g.after_each(function(g)
 28 |     g.space:drop()
 29 | end)
 30 | 
 31 | function g.test_passing(cg)
 32 |     t.skip_if(cg.params.index_type == 'BITSET', 'Unsupported index type')
 33 | 
 34 |     local task = expirationd.start("clean_all", cg.space.id, helpers.is_expired_true)
 35 |     -- if we don't specify index, program should use primary index
 36 |     t.assert_equals(task.index, cg.space.index[0].id)
 37 |     task:kill()
 38 | 
 39 |     -- index by name
 40 |     task = expirationd.start("clean_all", cg.space.id, helpers.is_expired_true,
 41 |             {index = "index_for_first_name"})
 42 |     t.assert_equals(task.index, cg.space.index[1].name)
 43 |     task:kill()
 44 | 
 45 |     -- index by id
 46 |     task = expirationd.start("clean_all", cg.space.id, helpers.is_expired_true,
 47 |             {index = 1})
 48 |     t.assert_equals(task.index, cg.space.index[1].id)
 49 |     task:kill()
 50 | end
 51 | 
 52 | function g.test_tree_index_errors(cg)
 53 |     t.skip_if(cg.params.index_type ~= 'TREE', 'Unsupported index type')
 54 | 
 55 |     t.assert_error_msg_contains("bad argument options.index to nil (?number|string expected, got table)",
 56 |             expirationd.start, "clean_all", cg.space.id, helpers.is_expired_true,
 57 |             {index = { 10 }})
 58 | end
 59 | 
 60 | function g.test_tree_index(cg)
 61 |     t.skip_if(cg.params.index_type ~= 'TREE', 'Unsupported index type')
 62 | 
 63 |     helpers.iteration_result = {}
 64 | 
 65 |     local space = cg.space
 66 |     space:insert({1, "3"})
 67 |     space:insert({2, "2"})
 68 |     space:insert({3, "1"})
 69 | 
 70 |     -- check default primary index
 71 |     local task = expirationd.start("clean_all", space.id, helpers.is_expired_debug)
 72 |     -- wait for tuples expired
 73 |     helpers.retrying({}, function()
 74 |         t.assert_equals(helpers.iteration_result, {
 75 |             {1, "3"},
 76 |             {2, "2"},
 77 |             {3, "1"}
 78 |         })
 79 |     end)
 80 |     task:kill()
 81 |     helpers.iteration_result = {}
 82 | 
 83 |     space:insert({1, "3"})
 84 |     space:insert({2, "2"})
 85 |     space:insert({3, "1"})
 86 | 
 87 |     -- check custom index
 88 |     task = expirationd.start("clean_all", space.id, helpers.is_expired_debug,
 89 |             {index = "index_for_first_name"})
 90 |     -- wait for tuples expired
 91 |     helpers.retrying({}, function()
 92 |         t.assert_equals(helpers.iteration_result, {
 93 |             {3, "1"},
 94 |             {2, "2"},
 95 |             {1, "3"}
 96 |         })
 97 |     end)
 98 |     task:kill()
 99 | end
100 | 
101 | function g.test_tree_index_multipart(cg)
102 |     t.skip_if(cg.params.index_type ~= 'TREE', 'Unsupported index type')
103 | 
104 |     helpers.iteration_result = {}
105 | 
106 |     local space = cg.space
107 |     space:insert({1, "1", 2, 1})
108 |     space:insert({2, "2", 2, 2})
109 |     space:insert({3, "3", 1, 3})
110 | 
111 |     local task = expirationd.start("clean_all", space.id, helpers.is_expired_debug,
112 |             {index = "multipart_index"})
113 |     -- wait for tuples expired
114 |     helpers.retrying({}, function()
115 |         t.assert_equals(helpers.iteration_result, {
116 |             {3, "3", 1, 3},
117 |             {1, "1", 2, 1},
118 |             {2, "2", 2, 2}
119 |         })
120 |     end)
121 |     task:kill()
122 | end
123 | 
124 | function g.test_tree_index_non_unique(cg)
125 |     t.skip_if(cg.params.index_type ~= 'TREE', 'Unsupported index type')
126 | 
127 |     helpers.iteration_result = {}
128 | 
129 |     cg.space:insert({1, "3", nil, nil, 1})
130 |     cg.space:insert({2, "2", nil, nil, 2})
131 |     cg.space:insert({3, "1", nil, nil, 1})
132 | 
133 |     local task = expirationd.start("clean_all", cg.space.id, helpers.is_expired_debug,
134 |             {index = "non_unique_index"})
135 |     -- wait for tuples expired
136 |     helpers.retrying({}, function()
137 |         t.assert_equals(helpers.iteration_result, {
138 |                     {1, "3", nil, nil, 1},
139 |                     {3, "1", nil, nil, 1},
140 |                     {2, "2", nil, nil, 2}
141 |                 })
142 |     end)
143 |     task:kill()
144 | end
145 | 
146 | function g.test_tree_index_json_path(cg)
147 |     t.skip_if(_TARANTOOL < "2", 'Unsupported Tarantool version')
148 |     t.skip_if(cg.params.index_type ~= 'TREE', 'Unsupported index type')
149 | 
150 |     helpers.iteration_result = {}
151 | 
152 |     local space = cg.space
153 |     space:insert({1, "1", nil, nil, nil, { age  = 3 }})
154 |     space:insert({2, "2", nil, nil, nil, { age  = 1 }})
155 |     space:insert({3, "3", nil, nil, nil, { age  = 2 }})
156 |     space:insert({4, "4", nil, nil, nil, { days = 3 }})
157 |     space:insert({5, "5", nil, nil, nil, { days = 1 }})
158 |     space:insert({6, "6", nil, nil, nil, { days = 2 }})
159 | 
160 | 
161 |     local task = expirationd.start("clean_all", space.id, helpers.is_expired_debug,
162 |             {index = "json_path_index"})
163 |     -- wait for tuples expired
164 |     helpers.retrying({}, function()
165 |         t.assert_equals(helpers.iteration_result, {
166 |             {4, "4", nil, nil, nil, { days = 3 }},
167 |             {5, "5", nil, nil, nil, { days = 1 }},
168 |             {6, "6", nil, nil, nil, { days = 2 }},
169 |             {2, "2", nil, nil, nil, { age = 1 }},
170 |             {3, "3", nil, nil, nil, { age = 2 }},
171 |             {1, "1", nil, nil, nil, { age = 3 }}
172 |         })
173 |     end)
174 |     task:kill()
175 | end
176 | 
177 | function g.test_tree_index_multikey(cg)
178 |     t.skip_if(_TARANTOOL < "2", "Unsupported Tarantool version")
179 |     t.skip_if(cg.params.index_type ~= 'TREE', 'Unsupported index type')
180 | 
181 |     helpers.iteration_result = {}
182 | 
183 |     cg.space:insert({1, "1", nil, nil, nil, nil, {data = {{name = "A"},
184 |                                                        {name = "B"}},
185 |                                                extra_field = 1}})
186 | 
187 |     local task = expirationd.start("clean_all", cg.space.id, helpers.is_expired_debug,
188 |             {index = "multikey_index"})
189 |     -- wait for tuples expired
190 |     helpers.retrying({}, function()
191 |         -- met only once, since we delete and cannot walk a second time on name = "B"
192 |         t.assert_equals(helpers.iteration_result, {
193 |             {1, "1", nil, nil, nil, nil, {data = {{name = "A"},
194 |                                                   {name = "B"}},
195 |                                           extra_field = 1}}
196 |         })
197 |     end)
198 |     task:kill()
199 | end
200 | 
201 | function g.test_memtx_tree_functional_index_broken_error(cg)
202 |     t.skip_if(cg.params.engine == 'vinyl', 'Unsupported engine')
203 |     t.skip_if(cg.params.index_type ~= 'TREE', 'Unsupported index type')
204 |     t.skip_if(cg.space.index["functional_index"] == nil,
205 |         "Functional indexes are not supported by Tarantool")
206 |     t.skip_if(helpers.memtx_func_index_is_supported(),
207 |         "No errors expected from https://github.com/tarantool/tarantool/issues/6786 " ..
208 | 		"on this Tarantool version")
209 | 
210 |     local expected = "Functional indices are not supported for Tarantool < 2.8.4," ..
211 |                      " see options.force_allow_functional_index"
212 | 
213 |     cg.space:insert({1, "1", nil, nil, nil, nil, nil, "12"})
214 |     cg.space:insert({2, "2", nil, nil, nil, nil, nil, "21"})
215 | 
216 |     t.assert_error_msg_contains(expected, expirationd.start, "clean_all", cg.space.id,
217 |             helpers.is_expired_debug, {index = "functional_index"})
218 |     t.assert_items_equals(cg.space:select({}, {limit = 10}), {
219 |             {2, "2", nil, nil, nil, nil, nil, "21"},
220 |             {1, "1", nil, nil, nil, nil, nil, "12"}
221 |     })
222 | 
223 |     cg.space:truncate()
224 | end
225 | 
226 | function g.test_memtx_tree_functional_index_force_broken(cg)
227 |     t.skip_if(cg.params.engine == 'vinyl', 'Unsupported engine')
228 |     t.skip_if(cg.params.index_type ~= 'TREE', 'Unsupported index type')
229 |     t.skip_if(cg.space.index["functional_index"] == nil,
230 |         "Functional indexes are not supported by Tarantool")
231 |     t.skip_if(helpers.memtx_func_index_is_supported(),
232 |         "No errors expected from https://github.com/tarantool/tarantool/issues/6786 " ..
233 | 		"on this Tarantool version")
234 | 
235 |     helpers.iteration_result = {}
236 | 
237 |     cg.space:insert({1, "1", nil, nil, nil, nil, nil, "12"})
238 |     cg.space:insert({2, "2", nil, nil, nil, nil, nil, "21"})
239 | 
240 | 	-- The problem occurs when we iterate through a functional index and delete
241 | 	-- a current tuple. A possible solution is somehow to process tuples chunk
242 | 	-- by chunk using select calls instead of iterating with index:pairs().
243 |     local select_with = function()
244 |         local index = cg.space.index["functional_index"]
245 |         return pairs(index:select({}, {iterator = "ALL", limit = 100}))
246 |     end
247 | 
248 |     local task = expirationd.start("clean_all", cg.space.id, helpers.is_expired_debug,
249 |             {index = "functional_index", force_allow_functional_index = true,
250 |              iterate_with = select_with})
251 | 
252 |     -- wait for tuples expired
253 |     helpers.retrying({}, function()
254 |         -- sort by second character to eighth field
255 |         t.assert_equals(helpers.iteration_result, {
256 |             {2, "2", nil, nil, nil, nil, nil, "21"},
257 |             {1, "1", nil, nil, nil, nil, nil, "12"}
258 |         })
259 |     end)
260 |     task:kill()
261 | end
262 | 
263 | -- Vinyl is not supported yet:
264 | -- https://github.com/tarantool/tarantool/issues/4492
265 | function g.test_memtx_tree_functional_index(cg)
266 |     t.skip_if(cg.params.engine == 'vinyl', 'Unsupported engine')
267 |     t.skip_if(cg.params.index_type ~= 'TREE', 'Unsupported index type')
268 |     t.skip_if(cg.space.index["functional_index"] == nil,
269 |         "Functional indexes are not supported by Tarantool")
270 |     t.skip_if(not helpers.memtx_func_index_is_supported(),
271 |         "Blocked by https://github.com/tarantool/tarantool/issues/6786 on " ..
272 | 		"this Tarantool version")
273 | 
274 |     helpers.iteration_result = {}
275 | 
276 |     cg.space:insert({1, "1", nil, nil, nil, nil, nil, "12"})
277 |     cg.space:insert({2, "2", nil, nil, nil, nil, nil, "21"})
278 | 
279 |     local task = expirationd.start("clean_all", cg.space.id, helpers.is_expired_debug,
280 |             {index = "functional_index"})
281 |     -- wait for tuples expired
282 |     helpers.retrying({}, function()
283 |         -- sort by second character to eighth field
284 |         t.assert_equals(helpers.iteration_result, {
285 |             {2, "2", nil, nil, nil, nil, nil, "21"},
286 |             {1, "1", nil, nil, nil, nil, nil, "12"}
287 |         })
288 |     end)
289 |     task:kill()
290 | end
291 | 
292 | function g.test_hash_index(cg)
293 |     t.skip_if(cg.params.index_type ~= 'HASH', 'Unsupported index type')
294 | 
295 |     helpers.iteration_result = {}
296 |     cg.space:insert({1, "a"})
297 |     cg.space:insert({2, "b"})
298 |     cg.space:insert({3, "c"})
299 | 
300 |     -- check default primary index
301 |     local task = expirationd.start("clean_all", cg.space.id, helpers.is_expired_debug)
302 |     -- wait for tuples expired
303 |     helpers.retrying({}, function()
304 |         t.assert_equals(helpers.iteration_result, {
305 |             {3, "c"},
306 |             {2, "b"},
307 |             {1, "a"}
308 |         })
309 |     end)
310 |     task:kill()
311 | 
312 |     helpers.iteration_result = {}
313 |     cg.space:insert({1, "a"})
314 |     cg.space:insert({2, "b"})
315 |     cg.space:insert({3, "c"})
316 | 
317 |     task = expirationd.start("clean_all", cg.space.id, helpers.is_expired_debug,
318 |             {index = "index_for_first_name"})
319 |     -- wait for tuples expired
320 |     helpers.retrying({}, function()
321 |         t.assert_equals(helpers.iteration_result, {
322 |             {1, "a"},
323 |             {3, "c"},
324 |             {2, "b"}
325 |         })
326 |     end)
327 |     helpers.iteration_result = {}
328 |     task:kill()
329 | end
330 | 
331 | function g.test_hash_index_multipart(cg)
332 |     t.skip_if(cg.params.index_type ~= 'HASH', 'Unsupported index type')
333 | 
334 |     helpers.iteration_result = {}
335 | 
336 |     cg.space:insert({1, "1"})
337 |     cg.space:insert({2, "2"})
338 |     cg.space:insert({3, "3"})
339 | 
340 |     local task = expirationd.start("clean_all", cg.space.id, helpers.is_expired_debug,
341 |             {index = "multipart_index"})
342 |     -- wait for tuples expired
343 |     helpers.retrying({}, function()
344 |         t.assert_equals(helpers.iteration_result, {
345 |             {2, "2"},
346 |             {1, "1"},
347 |             {3, "3"}
348 |         })
349 |     end)
350 |     task:kill()
351 | end
352 | 


--------------------------------------------------------------------------------
/test/unit/expiration_process_test.lua:
--------------------------------------------------------------------------------
  1 | local expirationd = require('expirationd')
  2 | local fiber = require('fiber')
  3 | local t = require('luatest')
  4 | 
  5 | local helpers = require('test.helper')
  6 | 
  7 | local g = t.group('expiration_process', {
  8 |     {index_type = 'TREE', engine = 'vinyl'},
  9 |     {index_type = 'TREE', engine = 'memtx'},
 10 |     {index_type = 'HASH', engine = 'memtx'},
 11 | })
 12 | 
 13 | g.before_each({index_type = 'TREE'}, function(cg)
 14 |     t.skip_if(cg.params.engine == 'vinyl' and not helpers.vinyl_is_supported(),
 15 |         'Blocked by https://github.com/tarantool/tarantool/issues/6448')
 16 |     g.space = helpers.create_space_with_tree_index(cg.params.engine)
 17 | end)
 18 | 
 19 | g.before_each({index_type = 'HASH'}, function(cg)
 20 |     g.space = helpers.create_space_with_hash_index(cg.params.engine)
 21 | end)
 22 | 
 23 | g.before_each(function(cg)
 24 |     local space_archive = helpers.create_space('archived_tree', cg.params.engine)
 25 |     space_archive:create_index('primary')
 26 |     g.space_archive = space_archive
 27 | 
 28 |     cg.task_name = 'test'
 29 | end)
 30 | 
 31 | g.after_each(function(cg)
 32 |     if cg.task ~= nil then
 33 |         cg.task:kill()
 34 |     end
 35 |     cg.space:drop()
 36 |     cg.space_archive:drop()
 37 | end)
 38 | 
 39 | -- Check tuple's expiration by timestamp.
 40 | local function check_tuple_expire_by_timestamp(args, tuple)
 41 |     local tuple_expire_time = tuple[args.field_no]
 42 | 
 43 |     local current_time = fiber.time()
 44 |     return current_time >= tuple_expire_time
 45 | end
 46 | 
 47 | -- Put expired tuple in archive.
 48 | local function put_tuple_to_archive(space_id, args, tuple)
 49 |     -- Delete expired tuple.
 50 |     box.space[space_id]:delete({tuple.id})
 51 |     local id, first_name = tuple.id, tuple.first_name
 52 |     if args.archive_space_id ~= nil and id ~= nil and first_name ~= nil then
 53 |         box.space[args.archive_space_id]:insert({id, first_name, fiber.time()})
 54 |     end
 55 | end
 56 | 
 57 | -- Checking that we can use custom is_tuple_expired, process_expired_tuple,
 58 | -- these basic functions are included in expiration_process.
 59 | -- We also test the timestamp expiration check.
 60 | function g.test_archive_by_timestamp(cg)
 61 |     local space = cg.space
 62 |     local space_archive = cg.space_archive
 63 |     local task_name = cg.task_name
 64 | 
 65 |     local total = 10
 66 |     local todelete = 5
 67 |     local time = fiber.time()
 68 |     local deleted = {}
 69 |     local nondeleted = {}
 70 |     for i = 1, total do
 71 |         local tuple
 72 |         if i <= todelete then
 73 |             -- This tuples should be deleted by the expirationd.
 74 |             tuple = {i, tostring(i), time}
 75 |             table.insert(deleted, tuple)
 76 |         else
 77 |             -- This tuples should still exist.
 78 |             tuple = {i, tostring(i), time + 60}
 79 |             table.insert(nondeleted, tuple)
 80 |         end
 81 |         space:insert(tuple)
 82 |     end
 83 | 
 84 |     cg.task = expirationd.start(task_name, space.id,
 85 |         check_tuple_expire_by_timestamp,
 86 |         {
 87 |             process_expired_tuple = put_tuple_to_archive,
 88 |             args = {
 89 |                 field_no = 3,
 90 |                 archive_space_id = space_archive.id
 91 |             },
 92 |         })
 93 |     local task = cg.task
 94 |     local start_time = fiber.time()
 95 | 
 96 |     -- We sure that the task will be executed.
 97 |     helpers.retrying({}, function()
 98 |         t.assert_equals(space_archive:count(), #deleted)
 99 |     end)
100 | 
101 |     -- Check the validity of the task parameters.
102 |     t.assert_equals(task.name, 'test')
103 |     t.assert_equals(task.name, task_name)
104 |     t.assert_equals(task.start_time, start_time)
105 |     t.assert_equals(task.restarts, 1)
106 | 
107 |     -- Check tuple processing.
108 |     t.assert_equals(space_archive:count(), #deleted)
109 |     t.assert_equals(task.expired_tuples_count, #deleted)
110 |     t.assert_items_include(space:select(nil, {limit = 1000}), nondeleted)
111 | end
112 | 
113 | function g.test_broken_is_tuple_expired(cg)
114 |     local space = cg.space
115 |     local space_archive = cg.space_archive
116 |     local task_name = cg.task_name
117 | 
118 |     local full_scan_counter = 0
119 |     cg.task = expirationd.start(
120 |         task_name,
121 |         space.id,
122 |         helpers.error_function,
123 |         {
124 |             process_expired_tuple = put_tuple_to_archive,
125 |             args = {
126 |                 field_no = 3,
127 |                 archive_space_id = space_archive.id,
128 |             },
129 |             on_full_scan_complete = function()
130 |                 full_scan_counter = full_scan_counter + 1
131 |             end
132 |         }
133 |     )
134 |     local task = cg.task
135 | 
136 |     t.assert_equals(task.restarts, 1)
137 | 
138 |     -- Check that task is alive and running.
139 |     helpers.retrying({}, function()
140 |         t.assert_ge(full_scan_counter, 3)
141 |     end)
142 | end
143 | 
144 | function g.test_broken_process_expired_tuple(cg)
145 |     local space = cg.space
146 |     local space_archive = cg.space_archive
147 |     local task_name = cg.task_name
148 | 
149 |     local full_scan_counter = 0
150 |     cg.task = expirationd.start(
151 |         task_name,
152 |         space.id,
153 |         check_tuple_expire_by_timestamp,
154 |         {
155 |             process_expired_tuple = helpers.error_function,
156 |             args = {
157 |                 field_no = 3,
158 |                 archive_space_id = space_archive.id,
159 |             },
160 |             on_full_scan_complete = function()
161 |                 full_scan_counter = full_scan_counter + 1
162 |             end
163 |         }
164 |     )
165 |     local task = cg.task
166 | 
167 |     t.assert_equals(task.restarts, 1)
168 | 
169 |     -- Check that task is alive and running.
170 |     helpers.retrying({}, function()
171 |         t.assert_ge(full_scan_counter, 3)
172 |     end)
173 | end
174 | 
175 | function g.test_check_tuples_not_expired_by_timestamp(cg)
176 |     local space = cg.space
177 |     local space_archive = cg.space_archive
178 |     local task_name = cg.task_name
179 | 
180 |     local total = 5
181 |     local time = fiber.time()
182 |     for i = 1, total do
183 |         space:insert({i, tostring(i), time + 2})
184 |     end
185 | 
186 |     local full_scan_counter = 0
187 |     cg.task = expirationd.start(task_name, space.id, check_tuple_expire_by_timestamp,
188 |         {
189 |             process_expired_tuple = put_tuple_to_archive,
190 |             args = {
191 |                 field_no = 3,
192 |                 archive_space_id = space_archive.id
193 |             },
194 |             on_full_scan_complete = function()
195 |                 full_scan_counter = full_scan_counter + 1
196 |             end
197 |         })
198 |     local task = cg.task
199 | 
200 |     -- Tuples are not expired after run.
201 |     -- Сheck that after the expiration starts,
202 |     -- no tuples will be archived since the timestamp has an advantage of 2 seconds.
203 |     helpers.retrying({}, function()
204 |         t.assert(full_scan_counter > 0)
205 |         t.assert_equals(task.expired_tuples_count, 0)
206 |         t.assert_equals(space_archive:count(), 0)
207 |     end)
208 | 
209 |     -- Wait and check: all tuples must be expired.
210 |     helpers.retrying({}, function()
211 |         t.assert_equals(task.expired_tuples_count, total)
212 |         t.assert_equals(space_archive:count(), total)
213 |     end)
214 | end
215 | 
216 | function g.test_default_tuple_drop_function(cg)
217 |     local space = cg.space
218 |     local task_name = cg.task_name
219 |     local space_archive = cg.space_archive
220 | 
221 |     local total = 10
222 |     local time = fiber.time()
223 |     for i = 1, total do
224 |         space:insert({i, tostring(i), time})
225 |     end
226 |     -- Tuples are in space.
227 |     t.assert_equals(space:count(), total)
228 | 
229 |     cg.task = expirationd.start(task_name, space.id, check_tuple_expire_by_timestamp,
230 |         {
231 |             args = {
232 |                 field_no = 3,
233 |             },
234 |         })
235 |     local task = cg.task
236 | 
237 |     -- All tuples are expired with default function.
238 |     helpers.retrying({}, function()
239 |         t.assert_equals(task.expired_tuples_count, total)
240 |         t.assert_equals(space_archive:count(), 0)
241 |         t.assert_equals(space:count(), 0)
242 |     end)
243 | end
244 | 
245 | function g.test_tuples_per_iteration(cg)
246 |     local space = cg.space
247 |     local space_archive = cg.space_archive
248 |     local task_name = cg.task_name
249 | 
250 |     local total = 10
251 |     local time = fiber.time()
252 |     for i = 1, total do
253 |         space:insert({i, tostring(i), time})
254 |     end
255 |     t.assert_equals(space:count(), total)
256 | 
257 |     cg.task = expirationd.start(task_name, space.id, check_tuple_expire_by_timestamp,
258 |         {
259 |             process_expired_tuple = put_tuple_to_archive,
260 |             args = {
261 |                 field_no = 3,
262 |                 archive_space_id = space_archive.id
263 |             },
264 |             iteration_delay = 1,
265 |             vinyl_assumed_space_len = 5, -- Iteration_delay will be 1 sec.
266 |             tuples_per_iteration = 5,
267 |         })
268 |     local task = cg.task
269 | 
270 |     -- Test first expire part.
271 |     local worker_fiber = task.worker_fiber
272 |     helpers.retrying({}, function()
273 |         t.assert_equals(task.expired_tuples_count, total / 2)
274 |         t.assert_equals(worker_fiber:status(), 'suspended')
275 |     end)
276 | 
277 |     -- Test second expire part.
278 |     helpers.retrying({}, function()
279 |         t.assert_equals(task.expired_tuples_count, total)
280 |     end)
281 | end
282 | 


--------------------------------------------------------------------------------
/test/unit/expirationd_stats_test.lua:
--------------------------------------------------------------------------------
  1 | local expirationd = require("expirationd")
  2 | local fiber = require("fiber")
  3 | local t = require("luatest")
  4 | 
  5 | local helpers = require("test.helper")
  6 | 
  7 | local g = t.group('expirationd_stats', {
  8 |     {index_type = 'TREE', engine = 'vinyl'},
  9 |     {index_type = 'TREE', engine = 'memtx'},
 10 |     {index_type = 'HASH', engine = 'memtx'},
 11 | })
 12 | 
 13 | g.before_each({index_type = 'TREE'}, function(cg)
 14 |     t.skip_if(cg.params.engine == 'vinyl' and not helpers.vinyl_is_supported(),
 15 |         'Blocked by https://github.com/tarantool/tarantool/issues/6448 on ' ..
 16 | 		'this Tarantool version')
 17 |     g.space = helpers.create_space_with_tree_index(cg.params.engine)
 18 | end)
 19 | 
 20 | g.before_each({index_type = 'HASH'}, function(cg)
 21 |     g.space = helpers.create_space_with_hash_index(cg.params.engine)
 22 | end)
 23 | 
 24 | g.after_each(function(g)
 25 |     g.space:drop()
 26 | end)
 27 | 
 28 | function g.test_stats_basic(cg)
 29 |     local task = expirationd.start("stats_basic", cg.space.id, helpers.is_expired_true)
 30 |     local stats = expirationd.stats("stats_basic")
 31 |     t.assert_equals(stats, {
 32 |         checked_count = 0,
 33 |         expired_count = 0,
 34 |         restarts = 1,
 35 |         working_time = 0,
 36 |     })
 37 |     task:kill()
 38 | end
 39 | 
 40 | function g.test_stats_expired_count(cg)
 41 |     helpers.iteration_result = {}
 42 |     cg.space:insert({1, "a"})
 43 |     cg.space:insert({2, "b"})
 44 |     cg.space:insert({3, "c"})
 45 | 
 46 |     local iteration_result
 47 |     if cg.params.index_type == 'TREE' then
 48 |         iteration_result = {
 49 |             {1, "a"},
 50 |             {2, "b"},
 51 |             {3, "c"},
 52 |         }
 53 |     elseif cg.params.index_type == 'HASH' then
 54 |         iteration_result = {
 55 |             {3, "c"},
 56 |             {2, "b"},
 57 |             {1, "a"},
 58 |         }
 59 |     else
 60 |         error('Expected result is undefined.')
 61 |     end
 62 | 
 63 |     expirationd.start("stats_expired_count", cg.space.id, helpers.is_expired_debug)
 64 |     helpers.retrying({}, function()
 65 |         t.assert_covers(helpers.iteration_result, iteration_result)
 66 |     end)
 67 | 
 68 |     helpers.iteration_result = {}
 69 |     cg.space:insert({1, "a"})
 70 |     cg.space:insert({2, "b"})
 71 |     cg.space:insert({3, "c"})
 72 | 
 73 |     local task = expirationd.start("stats_expired_count", cg.space.id, helpers.is_expired_debug)
 74 |     helpers.retrying({}, function()
 75 |         t.assert_equals(helpers.iteration_result, iteration_result)
 76 |     end)
 77 |     local stats = expirationd.stats("stats_expired_count")
 78 |     t.assert_items_equals(stats, {
 79 |         checked_count = 3,
 80 |         expired_count = 3,
 81 |         restarts = 1,
 82 |         working_time = 0,
 83 |     })
 84 |     task:kill()
 85 | end
 86 | 
 87 | function g.test_stats_restarts(cg)
 88 |     local task = expirationd.start("stats_restarts", cg.space.id, helpers.is_expired_true)
 89 |     task:restart()
 90 |     task:restart()
 91 |     local stats = expirationd.stats("stats_restarts")
 92 |     t.assert_equals(stats, {
 93 |         checked_count = 0,
 94 |         expired_count = 0,
 95 |         restarts = 3,
 96 |         working_time = 0,
 97 |     })
 98 |     task:kill()
 99 | end
100 | 
101 | function g.test_stats_working_time(cg)
102 |     local task = expirationd.start("stats_working_time", cg.space.id, helpers.is_expired_true)
103 |     local running_time = 1
104 |     local threshold = 0.3
105 | 
106 |     local start_time = fiber.clock()
107 |     fiber.sleep(running_time)
108 |     local duration = fiber.clock() - start_time
109 | 
110 |     local stats = expirationd.stats("stats_working_time")
111 |     t.assert_almost_equals(stats.working_time, duration, threshold)
112 |     stats.working_time = nil
113 |     t.assert_equals(stats, {
114 |         checked_count = 0,
115 |         expired_count = 0,
116 |         restarts = 1,
117 |     })
118 |     task:kill()
119 | end
120 | 


--------------------------------------------------------------------------------
/test/unit/iterate_with_test.lua:
--------------------------------------------------------------------------------
 1 | local expirationd = require("expirationd")
 2 | local t = require("luatest")
 3 | 
 4 | local helpers = require("test.helper")
 5 | 
 6 | local g = t.group('iterate_with', {
 7 |     {index_type = 'TREE', engine = 'vinyl'},
 8 |     {index_type = 'TREE', engine = 'memtx'},
 9 |     {index_type = 'HASH', engine = 'memtx'},
10 | })
11 | 
12 | g.before_each({index_type = 'TREE'}, function(cg)
13 |     g.space = helpers.create_space_with_tree_index(cg.params.engine)
14 | end)
15 | 
16 | g.before_each({index_type = 'HASH'}, function(cg)
17 |     g.space = helpers.create_space_with_hash_index(cg.params.engine)
18 | end)
19 | 
20 | g.after_each(function(g)
21 |     g.space:drop()
22 | end)
23 | 
24 | function g.test_passing(cg)
25 |     local task = expirationd.start("clean_all", cg.space.id, helpers.is_expired_true,
26 |             { iterate_with = helpers.iterate_with_func })
27 |     -- default process_while always return false, iterations never stopped by this function
28 |     t.assert_equals(task.iterate_with, helpers.iterate_with_func)
29 |     task:kill()
30 | 
31 |     -- errors
32 |     t.assert_error_msg_contains("bad argument options.iterate_with to nil (?function expected, got string)",
33 |             expirationd.start, "clean_all", cg.space.id, helpers.is_expired_true,
34 |             { iterate_with = "" })
35 | end
36 | 


--------------------------------------------------------------------------------
/test/unit/metrics_test.lua:
--------------------------------------------------------------------------------
  1 | local expirationd = require("expirationd")
  2 | local t = require("luatest")
  3 | local helpers = require("test.helper")
  4 | local g = t.group('expirationd_metrics')
  5 | 
  6 | g.before_all(function()
  7 |     g.default_cfg = { metrics = expirationd.cfg.metrics }
  8 | end)
  9 | 
 10 | g.before_each(function()
 11 |     t.skip_if(not helpers.is_metrics_supported(),
 12 |               "metrics >= 0.11.0 is not installed")
 13 |     g.space = helpers.create_space_with_tree_index('memtx')
 14 |     -- kill live tasks (it can still live after failed tests)
 15 |     for _, t in ipairs(expirationd.tasks()) do
 16 |         expirationd.kill(t)
 17 |     end
 18 |     -- disable and clean metrics by default
 19 |     expirationd.cfg({metrics = false})
 20 |     require('metrics').clear()
 21 | end)
 22 | 
 23 | local task = nil
 24 | g.after_each(function(g)
 25 |     expirationd.cfg(g.default_cfg)
 26 |     g.space:drop()
 27 |     if task ~= nil then
 28 |         task:kill()
 29 |         task = nil
 30 |     end
 31 | end)
 32 | 
 33 | local function get_metrics()
 34 |     local metrics = require('metrics')
 35 |     metrics.invoke_callbacks()
 36 |     return metrics.collect()
 37 | end
 38 | 
 39 | local function assert_metrics_equals(t, value, expected)
 40 |     local copy = table.deepcopy(value)
 41 |     for _, v in ipairs(copy) do
 42 |          v['timestamp'] = nil
 43 |     end
 44 |     t.assert_items_equals(copy, expected)
 45 | end
 46 | 
 47 | local function assert_metrics_restarts_equals(t, value, expected)
 48 |     local copy = {}
 49 |     for _, v in ipairs(value) do
 50 |         if v['metric_name'] == "expirationd_restarts" then
 51 |             table.insert(copy, v)
 52 |         end
 53 |     end
 54 |     assert_metrics_equals(t, copy, expected)
 55 | end
 56 | 
 57 | function g.test_metrics_disabled(cg)
 58 |     expirationd.cfg({metrics = false})
 59 |     task = expirationd.start("stats_basic", cg.space.id, helpers.is_expired_true)
 60 | 
 61 |     local metrics = get_metrics()
 62 | 
 63 |     assert_metrics_equals(t, metrics, {})
 64 |     task:kill()
 65 |     task = nil
 66 | end
 67 | 
 68 | local metrics_basic = {
 69 |     {
 70 |         label_pairs = {name = "stats_basic"},
 71 |         metric_name = "expirationd_expired_count",
 72 |         value = 0,
 73 |     },
 74 |     {
 75 |         label_pairs = {name = "stats_basic"},
 76 |         metric_name = "expirationd_checked_count",
 77 |         value = 0,
 78 |     },
 79 |     {
 80 |         label_pairs = {name = "stats_basic"},
 81 |         metric_name = "expirationd_working_time",
 82 |         value = 0,
 83 |     },
 84 |     {
 85 |         label_pairs = {name = "stats_basic"},
 86 |         metric_name = "expirationd_restarts",
 87 |         value = 1,
 88 |     },
 89 | }
 90 | 
 91 | function g.test_metrics_basic(cg)
 92 |     expirationd.cfg({metrics = true})
 93 |     task = expirationd.start("stats_basic", cg.space.id, helpers.is_expired_true)
 94 | 
 95 |     local metrics = get_metrics()
 96 |     assert_metrics_equals(t, metrics, metrics_basic)
 97 |     task:kill()
 98 |     task = nil
 99 | end
100 | 
101 | function g.test_metrics_no_values_after_kill(cg)
102 |     expirationd.cfg({metrics = true})
103 |     task = expirationd.start("stats_basic", cg.space.id, helpers.is_expired_true)
104 | 
105 |     local metrics = get_metrics()
106 |     assert_metrics_equals(t, metrics, metrics_basic)
107 | 
108 |     task:kill()
109 |     task = nil
110 | 
111 |     local metrics = get_metrics()
112 |     assert_metrics_equals(t, metrics, {})
113 | end
114 | 
115 | function g.test_metrics_multiple_tasks_and_kill(cg)
116 |     expirationd.cfg({metrics = true})
117 |     local task1 = expirationd.start("stats_basic1", cg.space.id, helpers.is_expired_true)
118 |     local task2 = expirationd.start("stats_basic2", cg.space.id, helpers.is_expired_true)
119 |     task2:restart()
120 |     local task3 = expirationd.start("stats_basic3", cg.space.id, helpers.is_expired_true)
121 |     task3:restart()
122 |     task3:restart()
123 | 
124 |     local before_kill_metrics = get_metrics()
125 | 
126 |     task1:kill()
127 |     local after1_kill_metrics = get_metrics()
128 | 
129 |     task3:kill()
130 |     local after13_kill_metrics = get_metrics()
131 | 
132 |     task2:kill()
133 |     local after123_kill_metrics = get_metrics()
134 | 
135 |     assert_metrics_restarts_equals(t, before_kill_metrics, {
136 |         {
137 |             label_pairs = {name = "stats_basic1"},
138 |             metric_name = "expirationd_restarts",
139 |             value = 1,
140 |         },
141 |         {
142 |             label_pairs = {name = "stats_basic2"},
143 |             metric_name = "expirationd_restarts",
144 |             value = 2,
145 |         },
146 |         {
147 |             label_pairs = {name = "stats_basic3"},
148 |             metric_name = "expirationd_restarts",
149 |             value = 3,
150 |         },
151 |     })
152 |     assert_metrics_restarts_equals(t, after1_kill_metrics, {
153 |         {
154 |             label_pairs = {name = "stats_basic2"},
155 |             metric_name = "expirationd_restarts",
156 |             value = 2,
157 |         },
158 |         {
159 |             label_pairs = {name = "stats_basic3"},
160 |             metric_name = "expirationd_restarts",
161 |             value = 3,
162 |         },
163 |     })
164 |     assert_metrics_restarts_equals(t, after13_kill_metrics, {
165 |         {
166 |             label_pairs = {name = "stats_basic2"},
167 |             metric_name = "expirationd_restarts",
168 |             value = 2,
169 |         },
170 |     })
171 |     assert_metrics_restarts_equals(t, after123_kill_metrics, {})
172 | end
173 | 
174 | function g.test_metrics_no_values_after_disable(cg)
175 |     expirationd.cfg({metrics = true})
176 |     task = expirationd.start("stats_basic", cg.space.id, helpers.is_expired_true)
177 | 
178 |     local metrics = get_metrics()
179 |     assert_metrics_equals(t, metrics, metrics_basic)
180 | 
181 |     expirationd.cfg({metrics = false})
182 |     local metrics = get_metrics()
183 | 
184 |     assert_metrics_equals(t, metrics, {})
185 |     task:kill()
186 |     task = nil
187 | end
188 | 
189 | function g.test_metrics_new_values_after_restart(cg)
190 |     expirationd.cfg({metrics = true})
191 |     task = expirationd.start("stats_basic", cg.space.id, helpers.is_expired_true)
192 |     task:restart()
193 | 
194 |     local metrics = get_metrics()
195 |     assert_metrics_equals(t, metrics, {
196 |         {
197 |             label_pairs = {name = "stats_basic"},
198 |             metric_name = "expirationd_expired_count",
199 |             value = 0,
200 |         },
201 |         {
202 |             label_pairs = {name = "stats_basic"},
203 |             metric_name = "expirationd_checked_count",
204 |             value = 0,
205 |         },
206 |         {
207 |             label_pairs = {name = "stats_basic"},
208 |             metric_name = "expirationd_working_time",
209 |             value = 0,
210 |         },
211 |         {
212 |             label_pairs = {name = "stats_basic"},
213 |             metric_name = "expirationd_restarts",
214 |             value = 2,
215 |         }
216 |     })
217 | 
218 |     task:kill()
219 |     local metrics = get_metrics()
220 |     assert_metrics_equals(t, metrics, {})
221 | 
222 |     task = expirationd.start("stats_basic", cg.space.id, helpers.is_expired_true)
223 |     local metrics = get_metrics()
224 |     assert_metrics_equals(t, metrics, metrics_basic)
225 | 
226 |     task:kill()
227 |     task = nil
228 | end
229 | 
230 | function g.test_metrics_expired_count(cg)
231 |     local iteration_result = {
232 |         {1, "a"},
233 |         {2, "b"},
234 |         {3, "c"},
235 |     }
236 | 
237 |     helpers.iteration_result = {}
238 |     cg.space:insert({1, "a"})
239 |     cg.space:insert({2, "b"})
240 |     cg.space:insert({3, "c"})
241 | 
242 |     expirationd.cfg({metrics = true})
243 |     task = expirationd.start("stats_expired_count", cg.space.id, helpers.is_expired_debug)
244 |     helpers.retrying({}, function()
245 |         t.assert_equals(helpers.iteration_result, iteration_result)
246 |     end)
247 | 
248 |     local metrics = get_metrics()
249 |     assert_metrics_equals(t, metrics, {
250 |         {
251 |             label_pairs = {name = "stats_expired_count"},
252 |             metric_name = "expirationd_expired_count",
253 |             value = 3,
254 |         },
255 |         {
256 |             label_pairs = {name = "stats_expired_count"},
257 |             metric_name = "expirationd_checked_count",
258 |             value = 3,
259 |         },
260 |         {
261 |             label_pairs = {name = "stats_expired_count"},
262 |             metric_name = "expirationd_working_time",
263 |             value = 0,
264 |         },
265 |         {
266 |             label_pairs = {name = "stats_expired_count"},
267 |             metric_name = "expirationd_restarts",
268 |             value = 1,
269 |         },
270 |     })
271 | 
272 |     task:kill()
273 |     task = nil
274 | end
275 | 


--------------------------------------------------------------------------------
/test/unit/process_while_test.lua:
--------------------------------------------------------------------------------
 1 | local expirationd = require("expirationd")
 2 | local t = require("luatest")
 3 | 
 4 | local helpers = require("test.helper")
 5 | 
 6 | local g = t.group('process_while', {
 7 |     {index_type = 'TREE', engine = 'vinyl'},
 8 |     {index_type = 'TREE', engine = 'memtx'},
 9 |     {index_type = 'HASH', engine = 'memtx'},
10 | })
11 | 
12 | g.before_each({index_type = 'TREE'}, function(cg)
13 |     g.space = helpers.create_space_with_tree_index(cg.params.engine)
14 | end)
15 | 
16 | g.before_each({index_type = 'HASH'}, function(cg)
17 |     g.space = helpers.create_space_with_hash_index(cg.params.engine)
18 | end)
19 | 
20 | g.after_each(function(g)
21 |     g.space:drop()
22 | end)
23 | 
24 | function g.test_passing(cg)
25 |     local task = expirationd.start("clean_all", cg.space.id, helpers.is_expired_true)
26 |     -- default process_while always return false, iterations never stopped by this function
27 |     t.assert_equals(task.process_while(), true)
28 |     task:kill()
29 | 
30 |     local function process_while()
31 |         return false
32 |     end
33 | 
34 |     task = expirationd.start("clean_all", cg.space.id, helpers.is_expired_true,
35 |             {process_while = process_while})
36 |     t.assert_equals(task.process_while(), false)
37 |     task:kill()
38 | 
39 |     -- errors
40 |     t.assert_error_msg_contains("bad argument options.process_while to nil (?function expected, got string)",
41 |             expirationd.start, "clean_all", cg.space.id, helpers.is_expired_true,
42 |             { process_while = "" })
43 | end
44 | 
45 | local function process_while(task)
46 |     if task.checked_tuples_count >= 1 then return false end
47 |     return true
48 | end
49 | 
50 | function g.test_tree_index(cg)
51 |     t.skip_if(cg.params.index_type ~= 'TREE', 'Unsupported index type')
52 | 
53 |     local space = cg.space
54 |     helpers.iteration_result = {}
55 |     space:insert({1, "3"})
56 |     space:insert({2, "2"})
57 |     space:insert({3, "1"})
58 |     local task = expirationd.start("clean_all", space.id, helpers.is_expired_debug,
59 |             {process_while = process_while})
60 |     -- wait for tuples expired
61 |     helpers.retrying({}, function()
62 |         t.assert_equals(helpers.iteration_result, {{1, "3"}})
63 |     end)
64 |     task:kill()
65 | end
66 | 
67 | function g.test_hash_index(cg)
68 |     t.skip_if(cg.params.index_type ~= 'HASH', 'Unsupported index type')
69 | 
70 |     helpers.iteration_result = {}
71 |     cg.space:insert({1, "3"})
72 |     cg.space:insert({2, "2"})
73 |     cg.space:insert({3, "1"})
74 | 
75 |     local task = expirationd.start("clean_all", cg.space.id, helpers.is_expired_debug,
76 |             {process_while = process_while})
77 |     -- wait for tuples expired
78 |     helpers.retrying({}, function()
79 |         t.assert_equals(helpers.iteration_result, {{3, "1"}})
80 |     end)
81 |     task:kill()
82 | end
83 | 


--------------------------------------------------------------------------------
/test/unit/ro_test.lua:
--------------------------------------------------------------------------------
  1 | local expirationd = require("expirationd")
  2 | local fiber = require("fiber")
  3 | local t = require("luatest")
  4 | local helpers = require("test.helper")
  5 | local g = t.group('expirationd_ro')
  6 | 
  7 | g.before_all(function()
  8 |     -- we need to restore a state after fail
  9 |     g.default_box_cfg = {read_only = box.cfg.read_only}
 10 | end)
 11 | 
 12 | g.before_each(function()
 13 |     box.cfg({read_only = false})
 14 |     g.space = helpers.create_space_with_tree_index('memtx')
 15 |     box.cfg(g.default_box_cfg)
 16 | end)
 17 | 
 18 | local local_space = nil
 19 | g.after_each(function()
 20 |     box.cfg({read_only = false})
 21 |     g.space:drop()
 22 |     if local_space ~= nil then
 23 |         local_space:drop()
 24 |         local_space = nil
 25 |     end
 26 |     box.cfg(g.default_box_cfg)
 27 | end)
 28 | 
 29 | local function create_id_index(space)
 30 |     space:create_index("primary", {
 31 |         type = "TREE",
 32 |         parts = {
 33 |             {
 34 |                 field = 1
 35 |             }
 36 |         }
 37 |     })
 38 | end
 39 | 
 40 | local tuples = {{1, "1"}, {2, "2"}, {3, "3"}}
 41 | local function insert_tuples(space)
 42 |     for i = 1,3 do
 43 |         space:insert({i, tostring(i)})
 44 |     end
 45 | end
 46 | 
 47 | function g.test_ro_temporary()
 48 |     box.cfg({read_only = false})
 49 |     local_space = box.schema.create_space("temporary", {temporary = true})
 50 |     create_id_index(local_space)
 51 |     insert_tuples(local_space)
 52 | 
 53 |     box.cfg({read_only = true})
 54 | 
 55 |     helpers.iteration_result = {}
 56 |     local task = expirationd.start("clean_all", local_space.id, helpers.is_expired_debug,
 57 |             {full_scan_delay = 0})
 58 | 
 59 |     helpers.retrying({}, function()
 60 |         t.assert_equals(helpers.iteration_result, tuples)
 61 |     end)
 62 | 
 63 |     task:kill()
 64 |     box.cfg({read_only = false})
 65 | end
 66 | 
 67 | function g.test_ro_local()
 68 |     box.cfg({read_only = false})
 69 |     local_space = box.schema.create_space("is_local", {is_local = true})
 70 |     create_id_index(local_space)
 71 |     insert_tuples(local_space)
 72 | 
 73 |     box.cfg({read_only = true})
 74 | 
 75 |     helpers.iteration_result = {}
 76 |     local task = expirationd.start("clean_all", local_space.id, helpers.is_expired_debug,
 77 |             {full_scan_delay = 0})
 78 | 
 79 |     helpers.retrying({}, function()
 80 |         t.assert_equals(helpers.iteration_result, tuples)
 81 |     end)
 82 | 
 83 |     task:kill()
 84 |     box.cfg({read_only = false})
 85 | end
 86 | 
 87 | function g.test_switch_ro_to_rw()
 88 |     box.cfg({read_only = false})
 89 |     insert_tuples(g.space)
 90 | 
 91 |     box.cfg({read_only = true})
 92 | 
 93 |     helpers.iteration_result = {}
 94 |     local task = expirationd.start("clean_all", g.space.id, helpers.is_expired_debug,
 95 |             {full_scan_delay = 0})
 96 | 
 97 |     fiber.yield()
 98 |     helpers.retrying({}, function()
 99 |         t.assert_equals(helpers.iteration_result, {})
100 |     end)
101 | 
102 |     box.cfg({read_only = false})
103 | 
104 |     helpers.retrying({}, function()
105 |         t.assert_equals(helpers.iteration_result, tuples)
106 |     end)
107 | 
108 |     task:kill()
109 | end
110 | 
111 | function g.test_switch_rw_to_ro()
112 |     box.cfg({read_only = false})
113 |     insert_tuples(g.space)
114 | 
115 |     local is_expired_yield = function()
116 |         fiber.yield()
117 |         return true
118 |     end
119 |     helpers.iteration_result = {}
120 |     local task = expirationd.start("clean_all", g.space.id, is_expired_yield,
121 |             {full_scan_delay = 0})
122 | 
123 |     box.cfg({read_only = true})
124 | 
125 |     for _ = 1, 10 do
126 |         fiber.yield()
127 |     end
128 | 
129 |     helpers.retrying({}, function()
130 |         t.assert_equals(g.space:select({}, {limit = 10}), tuples)
131 |     end)
132 | 
133 |     task:kill()
134 | end
135 | 


--------------------------------------------------------------------------------
/test/unit/start_key_test.lua:
--------------------------------------------------------------------------------
  1 | local expirationd = require("expirationd")
  2 | local t = require("luatest")
  3 | 
  4 | local helpers = require("test.helper")
  5 | 
  6 | local g = t.group('start_key', {
  7 |     {index_type = 'TREE', engine = 'vinyl'},
  8 |     {index_type = 'TREE', engine = 'memtx'},
  9 |     {index_type = 'HASH', engine = 'memtx'},
 10 | })
 11 | 
 12 | g.before_each({index_type = 'TREE'}, function(cg)
 13 |     t.skip_if(cg.params.engine == 'vinyl' and not helpers.vinyl_is_supported(),
 14 |         'Blocked by https://github.com/tarantool/tarantool/issues/6448 on ' ..
 15 | 		'this Tarantool version')
 16 |     g.space = helpers.create_space_with_tree_index(cg.params.engine)
 17 | end)
 18 | 
 19 | g.before_each({index_type = 'HASH'}, function(cg)
 20 |     g.space = helpers.create_space_with_hash_index(cg.params.engine)
 21 | end)
 22 | 
 23 | g.after_each(function(g)
 24 |     g.space:drop()
 25 | end)
 26 | 
 27 | function g.test_passing(cg)
 28 |     t.skip_if(cg.params.index_type ~= 'TREE', 'Unsupported index type')
 29 | 
 30 |     local task = expirationd.start("clean_all", cg.space.id, helpers.is_expired_true)
 31 |     -- default start element is nil, iterate all elements
 32 |     t.assert_equals(task.start_key(), nil)
 33 |     task:kill()
 34 | 
 35 |     task = expirationd.start("clean_all", cg.space.id, helpers.is_expired_true,
 36 |             {start_key = box.NULL})
 37 |     -- default start element is nil, iterate all elements
 38 |     t.assert_equals(task.start_key(), box.NULL)
 39 |     task:kill()
 40 | 
 41 |     task = expirationd.start("clean_all", cg.space.id, helpers.is_expired_true,
 42 |             {start_key = 1})
 43 |     t.assert_equals(task.start_key(), 1)
 44 |     task:kill()
 45 | 
 46 |     task = expirationd.start("clean_all", cg.space.id, helpers.is_expired_true,
 47 |             { index = "multipart_index", start_key = {1, 1}})
 48 |     t.assert_equals(task.start_key(), {1, 1})
 49 |     task:kill()
 50 | 
 51 |     -- errors
 52 |     t.assert_error_msg_content_equals(
 53 |             "Supplied key type of part 0 does not match index part type: expected number",
 54 |             expirationd.start, "clean_all", cg.space.id, helpers.is_expired_true,
 55 |             { start_key = "" })
 56 |     t.assert_error_msg_content_equals(
 57 |             "Supplied key type of part 0 does not match index part type: expected number",
 58 |             expirationd.start, "clean_all", cg.space.id, helpers.is_expired_true,
 59 |             { index = "multipart_index", start_key = "" })
 60 |     t.assert_error_msg_content_equals(
 61 |             "Supplied key type of part 0 does not match index part type: expected number",
 62 |             expirationd.start, "clean_all", cg.space.id, helpers.is_expired_true,
 63 |             { index = "multipart_index", start_key = {"", ""} })
 64 |     t.assert_error_msg_content_equals(
 65 |             "Supplied key type of part 1 does not match index part type: expected number",
 66 |             expirationd.start, "clean_all", cg.space.id, helpers.is_expired_true,
 67 |             { index = "multipart_index", start_key = {1, ""} })
 68 | end
 69 | 
 70 | function g.test_tree_index(cg)
 71 |     t.skip_if(cg.params.index_type ~= 'TREE', 'Unsupported index type')
 72 | 
 73 |     local space = cg.space
 74 |     -- without start key
 75 |     helpers.iteration_result = {}
 76 |     space:insert({1, "3"})
 77 |     space:insert({2, "2"})
 78 |     space:insert({3, "1"})
 79 | 
 80 |     local task = expirationd.start("clean_all", space.id, helpers.is_expired_debug)
 81 |     -- wait for tuples expired
 82 |     helpers.retrying({}, function()
 83 |         t.assert_equals(helpers.iteration_result, {
 84 |             {1, "3"},
 85 |             {2, "2"},
 86 |             {3, "1"}
 87 |         })
 88 |     end)
 89 |     task:kill()
 90 | 
 91 |     -- box.NULL
 92 |     helpers.iteration_result = {}
 93 |     space:insert({1, "3"})
 94 |     space:insert({2, "2"})
 95 |     space:insert({3, "1"})
 96 | 
 97 |     task = expirationd.start("clean_all", space.id, helpers.is_expired_debug,
 98 |                     {start_key = box.NULL})
 99 |     -- wait for tuples expired
100 |     helpers.retrying({}, function()
101 |         t.assert_equals(helpers.iteration_result, {
102 |             {1, "3"},
103 |             {2, "2"},
104 |             {3, "1"}
105 |         })
106 |     end)
107 |     task:kill()
108 | 
109 |     -- with start key
110 |     helpers.iteration_result = {}
111 |     space:insert({1, "3"})
112 |     space:insert({2, "2"})
113 |     space:insert({3, "1"})
114 | 
115 |     task = expirationd.start("clean_all", space.id, helpers.is_expired_debug,
116 |             {start_key = 2})
117 |     -- wait for tuples expired
118 |     helpers.retrying({}, function()
119 |         t.assert_equals(helpers.iteration_result, {
120 |             {2, "2"},
121 |             {3, "1"}
122 |         })
123 |     end)
124 |     task:kill()
125 | end
126 | 
127 | function g.test_hash_index(cg)
128 |     t.skip_if(cg.params.index_type ~= 'HASH', 'Unsupported index type')
129 | 
130 |     -- without start key
131 |     helpers.iteration_result = {}
132 |     cg.space:insert({1, "3"})
133 |     cg.space:insert({2, "2"})
134 |     cg.space:insert({3, "1"})
135 | 
136 |     local task = expirationd.start("clean_all", cg.space.id, helpers.is_expired_debug)
137 |     -- wait for tuples expired
138 |     helpers.retrying({}, function()
139 |         t.assert_equals(helpers.iteration_result, {
140 |             {3, "1"},
141 |             {2, "2"},
142 |             {1, "3"}
143 |         })
144 |     end)
145 |     task:kill()
146 | 
147 |     -- box.NULL
148 |     helpers.iteration_result = {}
149 |     cg.space:insert({1, "3"})
150 |     cg.space:insert({2, "2"})
151 |     cg.space:insert({3, "1"})
152 | 
153 |     task = expirationd.start("clean_all", cg.space.id, helpers.is_expired_debug,
154 |             {start_key = box.NULL})
155 |     -- wait for tuples expired
156 |     helpers.retrying({}, function()
157 |         t.assert_equals(helpers.iteration_result, {
158 |             {3, "1"},
159 |             {2, "2"},
160 |             {1, "3"}
161 |         })
162 |     end)
163 |     task:kill()
164 | 
165 |     -- with start key
166 |     helpers.iteration_result = {}
167 |     cg.space:insert({1, "3"})
168 |     cg.space:insert({2, "2"})
169 |     cg.space:insert({3, "1"})
170 | 
171 |     task = expirationd.start("clean_all", cg.space.id, helpers.is_expired_debug,
172 |             {start_key = 2})
173 |     -- wait for tuples expired
174 |     helpers.retrying({}, function()
175 |         t.assert_equals(helpers.iteration_result, {
176 |             {3, "1"},
177 |             {2, "2"},
178 |             {1, "3"}
179 |         })
180 |     end)
181 |     task:kill()
182 | end
183 | 


--------------------------------------------------------------------------------
/test/unit/task_stop_test.lua:
--------------------------------------------------------------------------------
 1 | local expirationd = require("expirationd")
 2 | local fiber = require("fiber")
 3 | local t = require("luatest")
 4 | 
 5 | local helpers = require("test.helper")
 6 | 
 7 | local g = t.group('task_stop', t.helpers.matrix({
 8 |     engine = {
 9 |         'memtx',
10 |         'vinyl',
11 |     },
12 | }))
13 | 
14 | g.before_each(function(cg)
15 |     g.space = helpers.create_space_with_tree_index(cg.params.engine)
16 | end)
17 | 
18 | g.after_each(function(g)
19 |     g.space:drop()
20 | end)
21 | 
22 | function g.test_cancel_on_pcall(cg)
23 |     local function on_full_scan_complete()
24 |         pcall(fiber.sleep, 1)
25 |     end
26 |     local one_hour = 3600
27 |     local task = expirationd.start("clean_all", cg.space.id, helpers.is_expired_true, {
28 |         full_scan_delay = one_hour,
29 |         on_full_scan_complete = on_full_scan_complete
30 |     })
31 |     helpers.retrying({}, function()
32 |         t.assert(task.worker_fiber)
33 |     end)
34 |     -- We need to execute in a separate fiber,
35 |     -- since pcall does not check testcancel and stop may freeze up.
36 |     local f = fiber.create(task.stop, task)
37 |     helpers.retrying({timeout = 5}, function()
38 |         t.assert_equals(f:status(), "dead")
39 |     end)
40 |     task:kill()
41 | end
42 | 


--------------------------------------------------------------------------------
/test/unit/update_and_kill_test.lua:
--------------------------------------------------------------------------------
  1 | local fiber = require("fiber")
  2 | local expirationd = require("expirationd")
  3 | local t = require("luatest")
  4 | 
  5 | local helpers = require("test.helper")
  6 | 
  7 | local g = t.group('update_and_kill', {
  8 |     {index_type = 'TREE', engine = 'vinyl'},
  9 |     {index_type = 'TREE', engine = 'memtx'},
 10 |     {index_type = 'HASH', engine = 'memtx'},
 11 | })
 12 | 
 13 | g.before_each({index_type = 'TREE'}, function(cg)
 14 |     t.skip_if(cg.params.engine == 'vinyl' and not helpers.vinyl_is_supported(),
 15 |         'Blocked by https://github.com/tarantool/tarantool/issues/6448')
 16 |     cg.first_space = helpers.create_space_with_tree_index(cg.params.engine)
 17 |     local second_space = helpers.create_space('second_space', cg.params.engine)
 18 |     second_space:create_index('primary')
 19 |     cg.second_space = second_space
 20 | end)
 21 | 
 22 | g.before_each({index_type = 'HASH'}, function(cg)
 23 |     cg.first_space = helpers.create_space_with_hash_index(cg.params.engine)
 24 |     local second_space = helpers.create_space('second_space', cg.params.engine)
 25 |     second_space:create_index('primary', {type = 'HASH'})
 26 |     cg.second_space = second_space
 27 | end)
 28 | 
 29 | g.after_each(function(cg)
 30 |     for _, task_name in pairs(expirationd.tasks()) do
 31 |         expirationd.kill(task_name)
 32 |     end
 33 |     cg.first_space:drop()
 34 |     cg.second_space:drop()
 35 | end)
 36 | 
 37 | g.after_test('test_expirationd_update', function()
 38 |     -- Back old link in require. It's necessary to avoid problem of double update call.
 39 |     -- The problem that we can't use old link properly after double update.
 40 |     -- Old link wouldn't see new tasks which were started by new link. That could happen without this line.
 41 |     -- Expirationd module changes only one previous link that stores in package.loaded.
 42 |     package.loaded["expirationd"] = expirationd
 43 | end)
 44 | 
 45 | function g.test_expirationd_update(cg)
 46 |     local first_space = cg.first_space
 47 |     local second_space = cg.second_space
 48 | 
 49 |     local first_expd_link = require("expirationd")
 50 | 
 51 |     -- Start tasks by first expirationd link.
 52 |     local first_expd_tasks_cnt = 4
 53 |     local first_expd_task_name_prefix = "first_"
 54 |     for i = 1, first_expd_tasks_cnt do
 55 |         first_expd_link.start(first_expd_task_name_prefix .. i, first_space.id, helpers.is_expired_true)
 56 |     end
 57 | 
 58 |     -- Check updating in progress message.
 59 |     local chan = fiber.channel(1)
 60 |     fiber.create(function()
 61 |         first_expd_link.update()
 62 |         chan:put(1)
 63 |     end)
 64 |     local _, err = pcall(function() first_expd_link.start() end)
 65 |     t.assert_str_contains(err, "Wait until update is done")
 66 |     chan:get()
 67 | 
 68 |     -- Check that links are not equals.
 69 |     local second_expd_link = require("expirationd")
 70 |     t.assert_not_equals(
 71 |         tostring(first_expd_link):match("0x.*"),
 72 |         tostring(second_expd_link):match("0x.*"))
 73 | 
 74 |     -- Start tasks by second expirationd link.
 75 |     local second_expd_tasks_cnt = 4
 76 |     local second_expd_task_name_prefix = "second_"
 77 |     for i = 1, second_expd_tasks_cnt do
 78 |         second_expd_link.start(second_expd_task_name_prefix .. i, second_space.id, helpers.is_expired_true)
 79 |     end
 80 | 
 81 |     -- Check that we have all tasks be shared between both tasks.
 82 |     t.assert_equals(first_expd_link.tasks(), second_expd_link.tasks())
 83 | 
 84 |     -- And tasks work correctly.
 85 |     for _, space in pairs({first_space, second_space}) do
 86 |         local total = 10
 87 |         for i = 1, total do
 88 |             space:insert({i, tostring(i)})
 89 |         end
 90 | 
 91 |         t.assert_equals(space:count(), total)
 92 |         helpers.retrying({}, function()
 93 |             t.assert_equals(space:count(), 0)
 94 |         end)
 95 |     end
 96 | end
 97 | 
 98 | function g.test_zombie_task_kill(cg)
 99 |     local space = cg.first_space
100 |     local task_name = 'test'
101 | 
102 |     local one_hour = 3600
103 |     local task = expirationd.start(task_name, space.id, helpers.is_expired_true,
104 |         {
105 |             full_scan_delay = one_hour,
106 |         }
107 |     )
108 | 
109 |     local first_task_fiber
110 |     helpers.retrying({}, function()
111 |         first_task_fiber = task.worker_fiber
112 |         t.assert_equals(first_task_fiber:status(), "suspended")
113 |     end)
114 |     local total = 10
115 |     for i = 1, total do
116 |         space:insert({ i, tostring(i) })
117 |     end
118 |     t.assert_equals(space:count(), total)
119 | 
120 |     -- Run again and check - it must kill first task.
121 |     task = expirationd.start(task_name, space.id, helpers.is_expired_true)
122 | 
123 |     t.assert_equals(task.restarts, 1)
124 |     -- Check is first fiber killed.
125 |     t.assert_equals(first_task_fiber:status(), "dead")
126 | 
127 |     helpers.retrying({}, function()
128 |         t.assert_equals(space:count(), 0)
129 |     end)
130 | end
131 | 


--------------------------------------------------------------------------------
/test/unit/version_test.lua:
--------------------------------------------------------------------------------
1 | local t = require("luatest")
2 | local g = t.group('expirationd_versioning')
3 | local expirationd = require('expirationd')
4 | 
5 | g.test_version = function()
6 |     t.assert_type(expirationd._VERSION, 'string')
7 |     t.assert_not_equals(string.find(expirationd._VERSION, "^%d+%.%d+%.%d+$"), nil)
8 | end
9 | 


--------------------------------------------------------------------------------