├── .github
└── workflows
│ ├── nebula-ci.yml
│ ├── nebula-publish.yml
│ └── nebula-snapshot.yml
├── .gitignore
├── CHANGELOG.md
├── CONTRIBUTING.md
├── LICENSE
├── OSSMETADATA
├── README.md
├── build.gradle
├── dyno-queues-core
├── .gitignore
├── build.gradle
└── src
│ ├── main
│ └── java
│ │ └── com
│ │ └── netflix
│ │ └── dyno
│ │ └── queues
│ │ ├── DynoQueue.java
│ │ ├── Message.java
│ │ └── ShardSupplier.java
│ └── test
│ └── java
│ └── com
│ └── netflix
│ └── dyno
│ └── queues
│ └── TestMessage.java
├── dyno-queues-redis
├── .gitignore
├── build.gradle
└── src
│ ├── main
│ ├── java
│ │ └── com
│ │ │ └── netflix
│ │ │ └── dyno
│ │ │ └── queues
│ │ │ ├── demo
│ │ │ └── DynoQueueDemo.java
│ │ │ ├── redis
│ │ │ ├── QueueMonitor.java
│ │ │ ├── QueueUtils.java
│ │ │ ├── RedisDynoQueue.java
│ │ │ ├── RedisQueues.java
│ │ │ ├── conn
│ │ │ │ ├── DynoClientProxy.java
│ │ │ │ ├── DynoJedisPipe.java
│ │ │ │ ├── JedisProxy.java
│ │ │ │ ├── Pipe.java
│ │ │ │ ├── RedisConnection.java
│ │ │ │ └── RedisPipe.java
│ │ │ ├── sharding
│ │ │ │ ├── RoundRobinStrategy.java
│ │ │ │ └── ShardingStrategy.java
│ │ │ └── v2
│ │ │ │ ├── MultiRedisQueue.java
│ │ │ │ ├── QueueBuilder.java
│ │ │ │ └── RedisPipelineQueue.java
│ │ │ └── shard
│ │ │ ├── ConsistentAWSDynoShardSupplier.java
│ │ │ ├── ConsistentDynoShardSupplier.java
│ │ │ ├── DynoShardSupplier.java
│ │ │ └── SingleShardSupplier.java
│ └── resources
│ │ └── demo.properties
│ └── test
│ └── java
│ └── com
│ └── netflix
│ └── dyno
│ └── queues
│ ├── jedis
│ └── JedisMock.java
│ └── redis
│ ├── BaseQueueTests.java
│ ├── CustomShardingStrategyTest.java
│ ├── DefaultShardingStrategyTest.java
│ ├── DynoShardSupplierTest.java
│ ├── RedisDynoQueueTest.java
│ ├── benchmark
│ ├── BenchmarkTestsDynoJedis.java
│ ├── BenchmarkTestsJedis.java
│ ├── BenchmarkTestsNoPipelines.java
│ └── QueueBenchmark.java
│ └── v2
│ ├── DynoJedisTests.java
│ ├── JedisTests.java
│ ├── MultiQueueTests.java
│ └── RedisDynoQueueTest.java
├── gradle
└── wrapper
│ ├── gradle-wrapper.jar
│ └── gradle-wrapper.properties
├── gradlew
├── gradlew.bat
└── settings.gradle
/.github/workflows/nebula-ci.yml:
--------------------------------------------------------------------------------
1 | name: "CI"
2 | on:
3 | push:
4 | branches:
5 | - '*'
6 | tags-ignore:
7 | - '*'
8 | pull_request:
9 |
10 | jobs:
11 | build:
12 | runs-on: ubuntu-latest
13 | services:
14 | redis:
15 | image: redis
16 | options: >-
17 | --health-cmd "redis-cli ping"
18 | --health-interval 10s
19 | --health-timeout 5s
20 | --health-retries 5
21 | ports:
22 | - 6379:6379
23 | strategy:
24 | matrix:
25 | # test against JDK 8
26 | java: [ 8 ]
27 | name: CI with Java ${{ matrix.java }}
28 | steps:
29 | - uses: actions/checkout@v4
30 | - run: |
31 | git config --global user.name "Netflix OSS Maintainers"
32 | git config --global user.email "netflixoss@netflix.com"
33 | - name: Setup jdk
34 | uses: actions/setup-java@v1
35 | with:
36 | java-version: ${{ matrix.java }}
37 | - uses: actions/cache@v4
38 | id: gradle-cache
39 | with:
40 | path: ~/.gradle/caches
41 | key: ${{ runner.os }}-gradle-${{ hashFiles('**/gradle/dependency-locks/*.lockfile') }}
42 | restore-keys: |
43 | - ${{ runner.os }}-gradle-
44 | - uses: actions/cache@v4
45 | id: gradle-wrapper-cache
46 | with:
47 | path: ~/.gradle/wrapper
48 | key: ${{ runner.os }}-gradlewrapper-${{ hashFiles('gradle/wrapper/*') }}
49 | restore-keys: |
50 | - ${{ runner.os }}-gradlewrapper-
51 | - name: Build with Gradle
52 | run: ./gradlew --info --stacktrace build
53 | env:
54 | CI_NAME: github_actions
55 | CI_BUILD_NUMBER: ${{ github.sha }}
56 | CI_BUILD_URL: 'https://github.com/${{ github.repository }}'
57 | CI_BRANCH: ${{ github.ref }}
58 | COVERALLS_REPO_TOKEN: ${{ secrets.GITHUB_TOKEN }}
59 |
--------------------------------------------------------------------------------
/.github/workflows/nebula-publish.yml:
--------------------------------------------------------------------------------
1 | name: "Publish candidate/release to NetflixOSS and Maven Central"
2 | on:
3 | push:
4 | tags:
5 | - v*.*.*
6 | - v*.*.*-rc.*
7 | release:
8 | types:
9 | - published
10 |
11 | jobs:
12 | build:
13 | runs-on: ubuntu-latest
14 | services:
15 | redis:
16 | image: redis
17 | options: >-
18 | --health-cmd "redis-cli ping"
19 | --health-interval 10s
20 | --health-timeout 5s
21 | --health-retries 5
22 | ports:
23 | - 6379:6379
24 | steps:
25 | - uses: actions/checkout@v4
26 | - run: |
27 | git config --global user.name "Netflix OSS Maintainers"
28 | git config --global user.email "netflixoss@netflix.com"
29 | - name: Setup jdk 8
30 | uses: actions/setup-java@v1
31 | with:
32 | java-version: 1.8
33 | - uses: actions/cache@v4
34 | id: gradle-cache
35 | with:
36 | path: ~/.gradle/caches
37 | key: ${{ runner.os }}-gradle-${{ hashFiles('**/gradle/dependency-locks/*.lockfile') }}
38 | restore-keys: |
39 | - ${{ runner.os }}-gradle-
40 | - uses: actions/cache@v4
41 | id: gradle-wrapper-cache
42 | with:
43 | path: ~/.gradle/wrapper
44 | key: ${{ runner.os }}-gradlewrapper-${{ hashFiles('gradle/wrapper/*') }}
45 | restore-keys: |
46 | - ${{ runner.os }}-gradlewrapper-
47 | - name: Publish candidate
48 | if: contains(github.ref, '-rc.')
49 | run: ./gradlew --info --stacktrace -Prelease.useLastTag=true candidate
50 | env:
51 | NETFLIX_OSS_SIGNING_KEY: ${{ secrets.ORG_SIGNING_KEY }}
52 | NETFLIX_OSS_SIGNING_PASSWORD: ${{ secrets.ORG_SIGNING_PASSWORD }}
53 | NETFLIX_OSS_REPO_USERNAME: ${{ secrets.ORG_NETFLIXOSS_USERNAME }}
54 | NETFLIX_OSS_REPO_PASSWORD: ${{ secrets.ORG_NETFLIXOSS_PASSWORD }}
55 | - name: Publish release
56 | if: (!contains(github.ref, '-rc.'))
57 | run: ./gradlew --info -Prelease.useLastTag=true final
58 | env:
59 | NETFLIX_OSS_SONATYPE_USERNAME: ${{ secrets.ORG_SONATYPE_USERNAME }}
60 | NETFLIX_OSS_SONATYPE_PASSWORD: ${{ secrets.ORG_SONATYPE_PASSWORD }}
61 | NETFLIX_OSS_SIGNING_KEY: ${{ secrets.ORG_SIGNING_KEY }}
62 | NETFLIX_OSS_SIGNING_PASSWORD: ${{ secrets.ORG_SIGNING_PASSWORD }}
63 | NETFLIX_OSS_REPO_USERNAME: ${{ secrets.ORG_NETFLIXOSS_USERNAME }}
64 | NETFLIX_OSS_REPO_PASSWORD: ${{ secrets.ORG_NETFLIXOSS_PASSWORD }}
65 |
--------------------------------------------------------------------------------
/.github/workflows/nebula-snapshot.yml:
--------------------------------------------------------------------------------
1 | name: "Publish snapshot to NetflixOSS and Maven Central"
2 |
3 | on:
4 | push:
5 | branches:
6 | - master
7 |
8 | jobs:
9 | build:
10 | runs-on: ubuntu-latest
11 | services:
12 | redis:
13 | image: redis
14 | options: >-
15 | --health-cmd "redis-cli ping"
16 | --health-interval 10s
17 | --health-timeout 5s
18 | --health-retries 5
19 | ports:
20 | - 6379:6379
21 | steps:
22 | - uses: actions/checkout@v4
23 | with:
24 | fetch-depth: 0
25 | - run: |
26 | git config --global user.name "Netflix OSS Maintainers"
27 | git config --global user.email "netflixoss@netflix.com"
28 | - name: Set up JDK
29 | uses: actions/setup-java@v1
30 | with:
31 | java-version: 8
32 | - uses: actions/cache@v4
33 | id: gradle-cache
34 | with:
35 | path: |
36 | ~/.gradle/caches
37 | key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle') }}
38 | - uses: actions/cache@v4
39 | id: gradle-wrapper-cache
40 | with:
41 | path: |
42 | ~/.gradle/wrapper
43 | key: ${{ runner.os }}-gradlewrapper-${{ hashFiles('gradle/wrapper/*') }}
44 | - name: Build
45 | run: ./gradlew build snapshot
46 | env:
47 | NETFLIX_OSS_SIGNING_KEY: ${{ secrets.ORG_SIGNING_KEY }}
48 | NETFLIX_OSS_SIGNING_PASSWORD: ${{ secrets.ORG_SIGNING_PASSWORD }}
49 | NETFLIX_OSS_REPO_USERNAME: ${{ secrets.ORG_NETFLIXOSS_USERNAME }}
50 | NETFLIX_OSS_REPO_PASSWORD: ${{ secrets.ORG_NETFLIXOSS_PASSWORD }}
51 |
--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
1 | dyno-queues-core/.settings
2 | dyno-queues-core/build
3 | build
4 | redis-3.0.7/
5 | redis-3.0.7.tar.gz
6 |
7 | *.iml
8 | .idea
9 | .gradle
10 | .classpath
11 | .project
12 |
13 |
14 | dyno-queues-core/out/
15 | dyno-queues-redis/out/
16 |
17 | # publishing secrets
18 | secrets/signing-key
19 |
--------------------------------------------------------------------------------
/CHANGELOG.md:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Netflix/dyno-queues/6104363527d0c6801101b8516306212e68d3517d/CHANGELOG.md
--------------------------------------------------------------------------------
/CONTRIBUTING.md:
--------------------------------------------------------------------------------
1 | # Contributing to Dyno-queues
2 |
3 | We are following the Gitflow workflow. The active development branch is [dev](https://github.com/Netflix/dyno-queues/tree/dev), the stable branch is [master](https://github.com/Netflix/dyno-queues/tree/master).
4 |
5 | Contributions will be accepted to the [dev](https://github.com/Netflix/dyno-queues/tree/dev) only.
6 |
7 | ## How to provide a patch for a new feature
8 |
9 | 1. If it is a major feature, please create an [Issue]( https://github.com/Netflix/dyno-queues/issues ) and discuss with the project leaders.
10 |
11 | 2. If in step 1 you get an acknowledge from the project leaders, use the
12 | following procedure to submit a patch:
13 |
14 | a. Fork Dynomite on github ( http://help.github.com/fork-a-repo/ )
15 |
16 | b. Create a topic branch (git checkout -b my_branch)
17 |
18 | c. Push to your branch (git push origin my_branch)
19 |
20 | d. Initiate a pull request on github ( http://help.github.com/send-pull-requests/ )
21 |
22 | e. Done :)
23 |
24 | For minor fixes just open a pull request to the [dev]( https://github.com/Netflix/dyno-queues/tree/dev ) branch on Github.
25 |
--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
1 | Apache License
2 | Version 2.0, January 2004
3 | http://www.apache.org/licenses/
4 |
5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
6 |
7 | 1. Definitions.
8 |
9 | "License" shall mean the terms and conditions for use, reproduction,
10 | and distribution as defined by Sections 1 through 9 of this document.
11 |
12 | "Licensor" shall mean the copyright owner or entity authorized by
13 | the copyright owner that is granting the License.
14 |
15 | "Legal Entity" shall mean the union of the acting entity and all
16 | other entities that control, are controlled by, or are under common
17 | control with that entity. For the purposes of this definition,
18 | "control" means (i) the power, direct or indirect, to cause the
19 | direction or management of such entity, whether by contract or
20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the
21 | outstanding shares, or (iii) beneficial ownership of such entity.
22 |
23 | "You" (or "Your") shall mean an individual or Legal Entity
24 | exercising permissions granted by this License.
25 |
26 | "Source" form shall mean the preferred form for making modifications,
27 | including but not limited to software source code, documentation
28 | source, and configuration files.
29 |
30 | "Object" form shall mean any form resulting from mechanical
31 | transformation or translation of a Source form, including but
32 | not limited to compiled object code, generated documentation,
33 | and conversions to other media types.
34 |
35 | "Work" shall mean the work of authorship, whether in Source or
36 | Object form, made available under the License, as indicated by a
37 | copyright notice that is included in or attached to the work
38 | (an example is provided in the Appendix below).
39 |
40 | "Derivative Works" shall mean any work, whether in Source or Object
41 | form, that is based on (or derived from) the Work and for which the
42 | editorial revisions, annotations, elaborations, or other modifications
43 | represent, as a whole, an original work of authorship. For the purposes
44 | of this License, Derivative Works shall not include works that remain
45 | separable from, or merely link (or bind by name) to the interfaces of,
46 | the Work and Derivative Works thereof.
47 |
48 | "Contribution" shall mean any work of authorship, including
49 | the original version of the Work and any modifications or additions
50 | to that Work or Derivative Works thereof, that is intentionally
51 | submitted to Licensor for inclusion in the Work by the copyright owner
52 | or by an individual or Legal Entity authorized to submit on behalf of
53 | the copyright owner. For the purposes of this definition, "submitted"
54 | means any form of electronic, verbal, or written communication sent
55 | to the Licensor or its representatives, including but not limited to
56 | communication on electronic mailing lists, source code control systems,
57 | and issue tracking systems that are managed by, or on behalf of, the
58 | Licensor for the purpose of discussing and improving the Work, but
59 | excluding communication that is conspicuously marked or otherwise
60 | designated in writing by the copyright owner as "Not a Contribution."
61 |
62 | "Contributor" shall mean Licensor and any individual or Legal Entity
63 | on behalf of whom a Contribution has been received by Licensor and
64 | subsequently incorporated within the Work.
65 |
66 | 2. Grant of Copyright License. Subject to the terms and conditions of
67 | this License, each Contributor hereby grants to You a perpetual,
68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable
69 | copyright license to reproduce, prepare Derivative Works of,
70 | publicly display, publicly perform, sublicense, and distribute the
71 | Work and such Derivative Works in Source or Object form.
72 |
73 | 3. Grant of Patent License. Subject to the terms and conditions of
74 | this License, each Contributor hereby grants to You a perpetual,
75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable
76 | (except as stated in this section) patent license to make, have made,
77 | use, offer to sell, sell, import, and otherwise transfer the Work,
78 | where such license applies only to those patent claims licensable
79 | by such Contributor that are necessarily infringed by their
80 | Contribution(s) alone or by combination of their Contribution(s)
81 | with the Work to which such Contribution(s) was submitted. If You
82 | institute patent litigation against any entity (including a
83 | cross-claim or counterclaim in a lawsuit) alleging that the Work
84 | or a Contribution incorporated within the Work constitutes direct
85 | or contributory patent infringement, then any patent licenses
86 | granted to You under this License for that Work shall terminate
87 | as of the date such litigation is filed.
88 |
89 | 4. Redistribution. You may reproduce and distribute copies of the
90 | Work or Derivative Works thereof in any medium, with or without
91 | modifications, and in Source or Object form, provided that You
92 | meet the following conditions:
93 |
94 | (a) You must give any other recipients of the Work or
95 | Derivative Works a copy of this License; and
96 |
97 | (b) You must cause any modified files to carry prominent notices
98 | stating that You changed the files; and
99 |
100 | (c) You must retain, in the Source form of any Derivative Works
101 | that You distribute, all copyright, patent, trademark, and
102 | attribution notices from the Source form of the Work,
103 | excluding those notices that do not pertain to any part of
104 | the Derivative Works; and
105 |
106 | (d) If the Work includes a "NOTICE" text file as part of its
107 | distribution, then any Derivative Works that You distribute must
108 | include a readable copy of the attribution notices contained
109 | within such NOTICE file, excluding those notices that do not
110 | pertain to any part of the Derivative Works, in at least one
111 | of the following places: within a NOTICE text file distributed
112 | as part of the Derivative Works; within the Source form or
113 | documentation, if provided along with the Derivative Works; or,
114 | within a display generated by the Derivative Works, if and
115 | wherever such third-party notices normally appear. The contents
116 | of the NOTICE file are for informational purposes only and
117 | do not modify the License. You may add Your own attribution
118 | notices within Derivative Works that You distribute, alongside
119 | or as an addendum to the NOTICE text from the Work, provided
120 | that such additional attribution notices cannot be construed
121 | as modifying the License.
122 |
123 | You may add Your own copyright statement to Your modifications and
124 | may provide additional or different license terms and conditions
125 | for use, reproduction, or distribution of Your modifications, or
126 | for any such Derivative Works as a whole, provided Your use,
127 | reproduction, and distribution of the Work otherwise complies with
128 | the conditions stated in this License.
129 |
130 | 5. Submission of Contributions. Unless You explicitly state otherwise,
131 | any Contribution intentionally submitted for inclusion in the Work
132 | by You to the Licensor shall be under the terms and conditions of
133 | this License, without any additional terms or conditions.
134 | Notwithstanding the above, nothing herein shall supersede or modify
135 | the terms of any separate license agreement you may have executed
136 | with Licensor regarding such Contributions.
137 |
138 | 6. Trademarks. This License does not grant permission to use the trade
139 | names, trademarks, service marks, or product names of the Licensor,
140 | except as required for reasonable and customary use in describing the
141 | origin of the Work and reproducing the content of the NOTICE file.
142 |
143 | 7. Disclaimer of Warranty. Unless required by applicable law or
144 | agreed to in writing, Licensor provides the Work (and each
145 | Contributor provides its Contributions) on an "AS IS" BASIS,
146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
147 | implied, including, without limitation, any warranties or conditions
148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
149 | PARTICULAR PURPOSE. You are solely responsible for determining the
150 | appropriateness of using or redistributing the Work and assume any
151 | risks associated with Your exercise of permissions under this License.
152 |
153 | 8. Limitation of Liability. In no event and under no legal theory,
154 | whether in tort (including negligence), contract, or otherwise,
155 | unless required by applicable law (such as deliberate and grossly
156 | negligent acts) or agreed to in writing, shall any Contributor be
157 | liable to You for damages, including any direct, indirect, special,
158 | incidental, or consequential damages of any character arising as a
159 | result of this License or out of the use or inability to use the
160 | Work (including but not limited to damages for loss of goodwill,
161 | work stoppage, computer failure or malfunction, or any and all
162 | other commercial damages or losses), even if such Contributor
163 | has been advised of the possibility of such damages.
164 |
165 | 9. Accepting Warranty or Additional Liability. While redistributing
166 | the Work or Derivative Works thereof, You may choose to offer,
167 | and charge a fee for, acceptance of support, warranty, indemnity,
168 | or other liability obligations and/or rights consistent with this
169 | License. However, in accepting such obligations, You may act only
170 | on Your own behalf and on Your sole responsibility, not on behalf
171 | of any other Contributor, and only if You agree to indemnify,
172 | defend, and hold each Contributor harmless for any liability
173 | incurred by, or claims asserted against, such Contributor by reason
174 | of your accepting any such warranty or additional liability.
175 |
176 | END OF TERMS AND CONDITIONS
177 |
178 | APPENDIX: How to apply the Apache License to your work.
179 |
180 | To apply the Apache License to your work, attach the following
181 | boilerplate notice, with the fields enclosed by brackets "{}"
182 | replaced with your own identifying information. (Don't include
183 | the brackets!) The text should be enclosed in the appropriate
184 | comment syntax for the file format. We also recommend that a
185 | file or class name and description of purpose be included on the
186 | same "printed page" as the copyright notice for easier
187 | identification within third-party archives.
188 |
189 | Copyright {yyyy} {name of copyright owner}
190 |
191 | Licensed under the Apache License, Version 2.0 (the "License");
192 | you may not use this file except in compliance with the License.
193 | You may obtain a copy of the License at
194 |
195 | http://www.apache.org/licenses/LICENSE-2.0
196 |
197 | Unless required by applicable law or agreed to in writing, software
198 | distributed under the License is distributed on an "AS IS" BASIS,
199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
200 | See the License for the specific language governing permissions and
201 | limitations under the License.
--------------------------------------------------------------------------------
/OSSMETADATA:
--------------------------------------------------------------------------------
1 | osslifecycle=active
2 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | ## DISCLAIMER: THIS PROJECT IS NO LONGER ACTIVELY MAINTAINED
2 |
3 |
4 |
5 | # Dyno Queues
6 | [](https://travis-ci.com/Netflix/dyno-queues)
7 | [](https://gitter.im/Netflix/dynomite?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)
8 |
9 | Dyno Queues is a recipe that provides task queues utilizing [Dynomite](https://github.com/Netflix/dynomite).
10 |
11 |
12 | ## Dyno Queues Features
13 |
14 | + Time based queues. Each queue element has a timestamp associated and is only polled out after that time.
15 | + Priority queues
16 | + No strict FIFO semantics. However, within a shard, the elements are delivered in FIFO (depending upon the priority)
17 |
--------------------------------------------------------------------------------
/build.gradle:
--------------------------------------------------------------------------------
1 | buildscript {
2 | repositories {
3 | mavenCentral()
4 | maven {
5 | url = 'https://plugins.gradle.org/m2'
6 | }
7 | }
8 |
9 | dependencies {
10 | classpath 'com.netflix.nebula:gradle-aggregate-javadocs-plugin:3.0.1'
11 | classpath 'com.netflix.nebula:gradle-extra-configurations-plugin:4.0.1'
12 | }
13 | }
14 |
15 | plugins {
16 | id 'com.netflix.nebula.netflixoss' version '11.5.0'
17 | }
18 |
19 | // Establish version and status
20 | ext.githubProjectName = rootProject.name // Change if github project name is not the same as the root project's name
21 |
22 | apply plugin: 'project-report'
23 |
24 | subprojects {
25 | apply plugin: 'nebula.netflixoss'
26 | apply plugin: 'java-library'
27 | apply plugin: 'project-report'
28 |
29 | sourceCompatibility = 1.8
30 | targetCompatibility = 1.8
31 |
32 | repositories {
33 | mavenCentral()
34 |
35 | // oss-candidate for -rc.* verions:
36 | maven {
37 | url "https://dl.bintray.com/netflixoss/oss-candidate"
38 | }
39 | }
40 |
41 | group = "com.netflix.${githubProjectName}"
42 | }
43 |
44 |
--------------------------------------------------------------------------------
/dyno-queues-core/.gitignore:
--------------------------------------------------------------------------------
1 | /bin/
2 |
--------------------------------------------------------------------------------
/dyno-queues-core/build.gradle:
--------------------------------------------------------------------------------
1 |
2 | dependencies {
3 | api 'com.netflix.dyno:dyno-core:1.7.2-rc2'
4 | testImplementation "junit:junit:4.11"
5 | }
6 |
--------------------------------------------------------------------------------
/dyno-queues-core/src/main/java/com/netflix/dyno/queues/DynoQueue.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright 2016 Netflix, Inc.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 | /**
17 | *
18 | */
19 | package com.netflix.dyno.queues;
20 |
21 | import java.io.Closeable;
22 | import java.util.List;
23 | import java.util.Map;
24 | import java.util.Set;
25 | import java.util.concurrent.TimeUnit;
26 |
27 | /**
28 | * @author Viren
29 | * Abstraction of a dyno queue.
30 | */
31 | public interface DynoQueue extends Closeable {
32 |
33 | /**
34 | *
35 | * @return Returns the name of the queue
36 | */
37 | public String getName();
38 |
39 | /**
40 | *
41 | * @return Time in milliseconds before the messages that are popped and not acknowledge are pushed back into the queue.
42 | * @see #ack(String)
43 | */
44 | public int getUnackTime();
45 |
46 | /**
47 | *
48 | * @param messages messages to be pushed onto the queue
49 | * @return Returns the list of message ids
50 | */
51 | public List push(List messages);
52 |
53 | /**
54 | *
55 | * @param messageCount number of messages to be popped out of the queue.
56 | * @param wait Amount of time to wait if there are no messages in queue
57 | * @param unit Time unit for the wait period
58 | * @return messages. Can be less than the messageCount if there are fewer messages available than the message count.
59 | * If the popped messages are not acknowledge in a timely manner, they are pushed back into the queue.
60 | * @see #peek(int)
61 | * @see #ack(String)
62 | * @see #getUnackTime()
63 | *
64 | */
65 | public List pop(int messageCount, int wait, TimeUnit unit);
66 |
67 | /**
68 | * Pops "messageId" from the local shard if it exists.
69 | * Note that if "messageId" is present in a different shard, we will be unable to pop it.
70 | *
71 | * @param messageId ID of message to pop
72 | * @return Returns a "Message" object if pop was successful. 'null' otherwise.
73 | */
74 | public Message popWithMsgId(String messageId);
75 |
76 | /**
77 | * Provides a peek into the queue without taking messages out.
78 | *
79 | * Note: This peeks only into the 'local' shard.
80 | *
81 | * @param messageCount number of messages to be peeked.
82 | * @return List of peeked messages.
83 | * @see #pop(int, int, TimeUnit)
84 | */
85 | public List peek(int messageCount);
86 |
87 | /**
88 | * Provides an acknowledgement for the message. Once ack'ed the message is removed from the queue forever.
89 | * @param messageId ID of the message to be acknowledged
90 | * @return true if the message was found pending acknowledgement and is now ack'ed. false if the message id is invalid or message is no longer present in the queue.
91 | */
92 | public boolean ack(String messageId);
93 |
94 |
95 | /**
96 | * Bulk version for {@link #ack(String)}
97 | * @param messages Messages to be acknowledged. Each message MUST be populated with id and shard information.
98 | */
99 | public void ack(List messages);
100 |
101 | /**
102 | * Sets the unack timeout on the message (changes the default timeout to the new value). Useful when extended lease is required for a message by consumer before sending ack.
103 | * @param messageId ID of the message to be acknowledged
104 | * @param timeout time in milliseconds for which the message will remain in un-ack state. If no ack is received after the timeout period has expired, the message is put back into the queue
105 | * @return true if the message id was found and updated with new timeout. false otherwise.
106 | */
107 | public boolean setUnackTimeout(String messageId, long timeout);
108 |
109 |
110 | /**
111 | * Updates the timeout for the message.
112 | * @param messageId ID of the message to be acknowledged
113 | * @param timeout time in milliseconds for which the message will remain invisible and not popped out of the queue.
114 | * @return true if the message id was found and updated with new timeout. false otherwise.
115 | */
116 | public boolean setTimeout(String messageId, long timeout);
117 |
118 | /**
119 | *
120 | * @param messageId Remove the message from the queue
121 | * @return true if the message id was found and removed. False otherwise.
122 | */
123 | public boolean remove(String messageId);
124 | public boolean atomicRemove(String messageId);
125 |
126 | /**
127 | * Enqueues 'message' if it doesn't exist in any of the shards or unack sets.
128 | *
129 | * @param message Message to enqueue if it doesn't exist.
130 | * @return true if message was enqueued. False if messageId already exists.
131 | */
132 | public boolean ensure(Message message);
133 |
134 | /**
135 | * Checks the message bodies (i.e. the data in the hash map), and returns true on the first match with
136 | * 'predicate'.
137 | *
138 | * Matching is done based on 'lua pattern' matching.
139 | * http://lua-users.org/wiki/PatternsTutorial
140 | *
141 | * Disclaimer: This is a potentially expensive call, since we will iterate over the entire hash map in the
142 | * worst case. Use mindfully.
143 | *
144 | * @param predicate The predicate to check against.
145 | * @return 'true' if any of the messages contain 'predicate'; 'false' otherwise.
146 | */
147 | public boolean containsPredicate(String predicate);
148 |
149 | /**
150 | * Checks the message bodies (i.e. the data in the hash map), and returns true on the first match with
151 | * 'predicate'.
152 | *
153 | * Matching is done based on 'lua pattern' matching.
154 | * http://lua-users.org/wiki/PatternsTutorial
155 | *
156 | * Disclaimer: This is a potentially expensive call, since we will iterate over the entire hash map in the
157 | * worst case. Use mindfully.
158 | *
159 | * @param predicate The predicate to check against.
160 | * @param localShardOnly If this is true, it will only check if the message exists in the local shard as opposed to
161 | * all shards. Note that this will only work if the Dynomite cluster ring size is 1 (i.e. one
162 | * instance per AZ).
163 | * @return 'true' if any of the messages contain 'predicate'; 'false' otherwise.
164 | */
165 | public boolean containsPredicate(String predicate, boolean localShardOnly);
166 |
167 | /**
168 | * Checks the message bodies (i.e. the data in the hash map), and returns the ID of the first message to match with
169 | * 'predicate'.
170 | *
171 | * Matching is done based on 'lua pattern' matching.
172 | * http://lua-users.org/wiki/PatternsTutorial
173 | *
174 | * Disclaimer: This is a potentially expensive call, since we will iterate over the entire hash map in the
175 | * worst case. Use mindfully.
176 | *
177 | * @param predicate The predicate to check against.
178 | * @return Message ID as string if any of the messages contain 'predicate'; 'null' otherwise.
179 | */
180 | public String getMsgWithPredicate(String predicate);
181 |
182 | /**
183 | * Checks the message bodies (i.e. the data in the hash map), and returns the ID of the first message to match with
184 | * 'predicate'.
185 | *
186 | * Matching is done based on 'lua pattern' matching.
187 | * http://lua-users.org/wiki/PatternsTutorial
188 | *
189 | * Disclaimer: This is a potentially expensive call, since we will iterate over the entire hash map in the
190 | * worst case. Use mindfully.
191 | *
192 | * @param predicate The predicate to check against.
193 | * @param localShardOnly If this is true, it will only check if the message exists in the local shard as opposed to
194 | * all shards. Note that this will only work if the Dynomite cluster ring size is 1 (i.e. one
195 | * instance per AZ).
196 | * @return Message ID as string if any of the messages contain 'predicate'; 'null' otherwise.
197 | */
198 | public String getMsgWithPredicate(String predicate, boolean localShardOnly);
199 |
200 | /**
201 | * Pops the message with the highest priority that matches 'predicate'.
202 | *
203 | * Note: Can be slow for large queues.
204 | *
205 | * @param predicate The predicate to check against.
206 | * @param localShardOnly If this is true, it will only check if the message exists in the local shard as opposed to
207 | * all shards. Note that this will only work if the Dynomite cluster ring size is 1 (i.e. one
208 | * instance per AZ).
209 | * @return
210 | */
211 | public Message popMsgWithPredicate(String predicate, boolean localShardOnly);
212 |
213 | /**
214 | *
215 | * @param messageId message to be retrieved.
216 | * @return Retrieves the message stored in the queue by the messageId. Null if not found.
217 | */
218 | public Message get(String messageId);
219 |
220 | /**
221 | *
222 | * Attempts to return all the messages found in the hashmap. It's a best-effort return of all payloads, i.e. it may
223 | * not 100% match with what's in the queue metadata at any given time and is read with a non-quorum connection.
224 | *
225 | * @return Returns a list of all messages found in the message hashmap.
226 | */
227 | public List getAllMessages();
228 |
229 | /**
230 | *
231 | * Same as get(), but uses the non quorum connection.
232 | * @param messageId message to be retrieved.
233 | * @return Retrieves the message stored in the queue by the messageId. Null if not found.
234 | */
235 | public Message localGet(String messageId);
236 |
237 | public List bulkPop(int messageCount, int wait, TimeUnit unit);
238 | public List unsafeBulkPop(int messageCount, int wait, TimeUnit unit);
239 |
240 | /**
241 | *
242 | * @return Size of the queue.
243 | * @see #shardSizes()
244 | */
245 | public long size();
246 |
247 | /**
248 | *
249 | * @return Map of shard name to the # of messages in the shard.
250 | * @see #size()
251 | */
252 | public Map> shardSizes();
253 |
254 | /**
255 | * Truncates the entire queue. Use with caution!
256 | */
257 | public void clear();
258 |
259 | /**
260 | * Process un-acknowledged messages. The messages which are polled by the client but not ack'ed are moved back to queue
261 | */
262 | public void processUnacks();
263 | public void atomicProcessUnacks();
264 |
265 | /**
266 | *
267 | * Attempts to return the items present in the local queue shard but not in the hashmap, if any.
268 | * (Ideally, we would not require this function, however, in some configurations, especially with multi-region write
269 | * traffic sharing the same queue, we may find ourselves with stale items in the queue shards)
270 | *
271 | * @return List of stale messages IDs.
272 | */
273 | public List findStaleMessages();
274 |
275 | /*
276 | * <=== Begin unsafe* functions. ===>
277 | *
278 | * The unsafe functions listed below are not advisable to use.
279 | * The reason they are listed as unsafe is that they operate over all shards of a queue which means that
280 | * due to the eventually consistent nature of Dynomite, the calling application may see duplicate item(s) that
281 | * may have already been popped in a different rack, by another instance of the same application.
282 | *
283 | * Why are these functions made available then?
284 | * There are some users of dyno-queues who have use-cases that are completely okay with dealing with duplicate
285 | * items.
286 | */
287 |
288 | /**
289 | * Provides a peek into all shards of the queue without taking messages out.
290 | * Note: This function does not guarantee ordering of items based on shards like unsafePopAllShards().
291 | *
292 | * @param messageCount The number of messages to peek.
293 | * @return A list of up to 'count' messages.
294 | */
295 | public List unsafePeekAllShards(final int messageCount);
296 |
297 |
298 | /**
299 | * Allows popping from all shards of the queue.
300 | *
301 | * Note: The local shard will always be looked into first and other shards will be filled behind it (if 'messageCount' is
302 | * greater than the number of elements in the local shard). This way we ensure the chances of duplicates are less.
303 | *
304 | * @param messageCount number of messages to be popped out of the queue.
305 | * @param wait Amount of time to wait for each shard if there are no messages in shard.
306 | * @param unit Time unit for the wait period
307 | * @return messages. Can be less than the messageCount if there are fewer messages available than the message count.
308 | * If the popped messages are not acknowledge in a timely manner, they are pushed back into
309 | * the queue.
310 | * @see #peek(int)
311 | * @see #ack(String)
312 | * @see #getUnackTime()
313 | *
314 | */
315 | public List unsafePopAllShards(int messageCount, int wait, TimeUnit unit);
316 |
317 |
318 | /**
319 | * Same as popWithMsgId(), but allows popping from any shard.
320 | *
321 | * @param messageId ID of message to pop
322 | * @return Returns a "Message" object if pop was successful. 'null' otherwise.
323 | */
324 | public Message unsafePopWithMsgIdAllShards(String messageId);
325 |
326 | }
327 |
--------------------------------------------------------------------------------
/dyno-queues-core/src/main/java/com/netflix/dyno/queues/Message.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright 2016 Netflix, Inc.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 | /**
17 | *
18 | */
19 | package com.netflix.dyno.queues;
20 |
21 | import java.util.concurrent.TimeUnit;
22 |
23 | /**
24 | * @author Viren
25 | *
26 | */
27 | public class Message {
28 |
29 | private String id;
30 |
31 | private String payload;
32 |
33 | private long timeout;
34 |
35 | private int priority;
36 |
37 | private String shard;
38 |
39 | public Message() {
40 |
41 | }
42 |
43 | public Message(String id, String payload) {
44 | this.id = id;
45 | this.payload = payload;
46 | }
47 |
48 | /**
49 | * @return the id
50 | */
51 | public String getId() {
52 | return id;
53 | }
54 |
55 | /**
56 | * @param id
57 | * the id to set
58 | */
59 | public void setId(String id) {
60 | this.id = id;
61 | }
62 |
63 | /**
64 | * @return the payload
65 | */
66 | public String getPayload() {
67 | return payload;
68 | }
69 |
70 | /**
71 | * @param payload the payload to set
72 | *
73 | */
74 | public void setPayload(String payload) {
75 | this.payload = payload;
76 | }
77 |
78 | /**
79 | *
80 | * @param timeout Timeout in milliseconds - The message is only given to the consumer after the specified milliseconds have elapsed.
81 | */
82 | public void setTimeout(long timeout) {
83 | this.timeout = timeout;
84 | }
85 |
86 | /**
87 | * Helper method for the {@link #setTimeout(long)}
88 | * @param time timeout time
89 | * @param unit unit for the time
90 | * @see #setTimeout(long)
91 | */
92 | public void setTimeout(long time, TimeUnit unit) {
93 | this.timeout = TimeUnit.MILLISECONDS.convert(time, unit);
94 | }
95 |
96 | /**
97 | *
98 | * @return Returns the timeout for the message
99 | */
100 | public long getTimeout() {
101 | return timeout;
102 | }
103 |
104 | /**
105 | * Sets the message priority. Higher priority message is retrieved ahead of lower priority ones
106 | * @param priority priority for the message.
107 | */
108 | public void setPriority(int priority) {
109 | if (priority < 0 || priority > 99) {
110 | throw new IllegalArgumentException("priority MUST be between 0 and 99 (inclusive)");
111 | }
112 | this.priority = priority;
113 | }
114 |
115 | public int getPriority() {
116 | return priority;
117 | }
118 |
119 | /**
120 | * @return the shard
121 | */
122 | public String getShard() {
123 | return shard;
124 | }
125 |
126 | /**
127 | * @param shard the shard to set
128 | *
129 | */
130 | public void setShard(String shard) {
131 | this.shard = shard;
132 | }
133 |
134 | @Override
135 | public int hashCode() {
136 | final int prime = 31;
137 | int result = 1;
138 | result = prime * result + ((id == null) ? 0 : id.hashCode());
139 | return result;
140 | }
141 |
142 | @Override
143 | public boolean equals(Object obj) {
144 | if (this == obj)
145 | return true;
146 | if (obj == null)
147 | return false;
148 | if (getClass() != obj.getClass())
149 | return false;
150 | Message other = (Message) obj;
151 | if (id == null) {
152 | if (other.id != null)
153 | return false;
154 | } else if (!id.equals(other.id))
155 | return false;
156 | return true;
157 | }
158 |
159 | @Override
160 | public String toString() {
161 | return "Message [id=" + id + ", payload=" + payload + ", timeout=" + timeout + ", priority=" + priority + "]";
162 | }
163 |
164 |
165 | }
166 |
--------------------------------------------------------------------------------
/dyno-queues-core/src/main/java/com/netflix/dyno/queues/ShardSupplier.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright 2016 Netflix, Inc.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 | /**
17 | *
18 | */
19 | package com.netflix.dyno.queues;
20 |
21 | import com.netflix.dyno.connectionpool.Host;
22 |
23 | import java.util.Set;
24 |
25 |
26 | /**
27 | * @author Viren
28 | *
29 | */
30 | public interface ShardSupplier {
31 |
32 | /**
33 | *
34 | * @return Provides the set of all the available queue shards. The elements are evenly distributed amongst these shards
35 | */
36 | public Set getQueueShards();
37 |
38 | /**
39 | *
40 | * @return Name of the current shard. Used when popping elements out of the queue
41 | */
42 | public String getCurrentShard();
43 |
44 | /**
45 | *
46 | * @param host
47 | * @return shard for this host based on the rack
48 | */
49 | public String getShardForHost(Host host);
50 | }
51 |
--------------------------------------------------------------------------------
/dyno-queues-core/src/test/java/com/netflix/dyno/queues/TestMessage.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright 2016 Netflix, Inc.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 | /**
17 | *
18 | */
19 | package com.netflix.dyno.queues;
20 |
21 | import static org.junit.Assert.*;
22 |
23 | import java.util.concurrent.TimeUnit;
24 |
25 | import org.junit.Test;
26 |
27 | /**
28 | * @author Viren
29 | *
30 | */
31 | public class TestMessage {
32 |
33 | @Test
34 | public void test() {
35 | Message msg = new Message();
36 | msg.setPayload("payload");
37 | msg.setTimeout(10, TimeUnit.SECONDS);
38 | assertEquals(msg.toString(), 10 * 1000, msg.getTimeout());
39 | msg.setTimeout(10);
40 | assertEquals(msg.toString(), 10, msg.getTimeout());
41 | }
42 |
43 | @Test(expected = IllegalArgumentException.class)
44 | public void testPrioirty() {
45 | Message msg = new Message();
46 | msg.setPriority(-1);
47 | }
48 |
49 | @Test(expected = IllegalArgumentException.class)
50 | public void testPrioirty2() {
51 | Message msg = new Message();
52 | msg.setPriority(100);
53 | }
54 | }
55 |
--------------------------------------------------------------------------------
/dyno-queues-redis/.gitignore:
--------------------------------------------------------------------------------
1 | /bin/
2 |
--------------------------------------------------------------------------------
/dyno-queues-redis/build.gradle:
--------------------------------------------------------------------------------
1 | dependencies {
2 |
3 | api project(':dyno-queues-core')
4 |
5 | api "com.google.inject:guice:3.0"
6 |
7 | api 'com.netflix.dyno:dyno-core:1.7.2-rc2'
8 | api 'com.netflix.dyno:dyno-jedis:1.7.2-rc2'
9 | api 'com.netflix.dyno:dyno-demo:1.7.2-rc2'
10 |
11 | api 'com.netflix.archaius:archaius-core:0.7.5'
12 | api 'com.netflix.servo:servo-core:0.12.17'
13 | api 'com.netflix.eureka:eureka-client:1.8.1'
14 | api 'com.fasterxml.jackson.core:jackson-databind:2.4.4'
15 |
16 | testImplementation 'org.rarefiedredis.redis:redis-java:0.0.17'
17 | testImplementation "junit:junit:4.11"
18 | }
19 |
20 | tasks.withType(Test) {
21 | maxParallelForks = 1
22 | }
23 |
--------------------------------------------------------------------------------
/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/demo/DynoQueueDemo.java:
--------------------------------------------------------------------------------
1 | package com.netflix.dyno.queues.demo;
2 |
3 | import com.netflix.dyno.demo.redis.DynoJedisDemo;
4 | import com.netflix.dyno.jedis.DynoJedisClient;
5 | import com.netflix.dyno.queues.DynoQueue;
6 | import com.netflix.dyno.queues.Message;
7 | import com.netflix.dyno.queues.redis.RedisQueues;
8 | import com.netflix.dyno.queues.redis.v2.QueueBuilder;
9 | import com.netflix.dyno.queues.shard.ConsistentAWSDynoShardSupplier;
10 | import org.slf4j.Logger;
11 | import org.slf4j.LoggerFactory;
12 |
13 | import java.io.IOException;
14 | import java.util.ArrayList;
15 | import java.util.Arrays;
16 | import java.util.List;
17 | import java.util.Properties;
18 | import java.util.concurrent.TimeUnit;
19 |
20 | public class DynoQueueDemo extends DynoJedisDemo {
21 |
22 | private static final Logger logger = LoggerFactory.getLogger(DynoQueue.class);
23 |
24 | public DynoQueueDemo(String clusterName, String localRack) {
25 | super(clusterName, localRack);
26 | }
27 |
28 | public DynoQueueDemo(String primaryCluster, String shadowCluster, String localRack) {
29 | super(primaryCluster, shadowCluster, localRack);
30 | }
31 |
32 | /**
33 | * Provide the cluster name to connect to as an argument to the function.
34 | * throws java.lang.RuntimeException: java.net.ConnectException: Connection timed out (Connection timed out)
35 | * if the cluster is not reachable.
36 | *
37 | * @param args: cluster-name version
38 | *
39 | * cluster-name: Name of cluster to run demo against
40 | * version: Possible values = 1 or 2; (for V1 or V2)
41 | */
42 | public static void main(String[] args) throws IOException {
43 | final String clusterName = args[0];
44 |
45 | if (args.length < 2) {
46 | throw new IllegalArgumentException("Need to pass in cluster-name and version of dyno-queues to run as arguments");
47 | }
48 |
49 | int version = Integer.parseInt(args[1]);
50 | final DynoQueueDemo demo = new DynoQueueDemo(clusterName, "us-east-1e");
51 | Properties props = new Properties();
52 | props.load(DynoQueueDemo.class.getResourceAsStream("/demo.properties"));
53 | for (String name : props.stringPropertyNames()) {
54 | System.setProperty(name, props.getProperty(name));
55 | }
56 |
57 | try {
58 | demo.initWithRemoteClusterFromEurekaUrl(args[0], 8102, false);
59 |
60 | if (version == 1) {
61 | demo.runSimpleV1Demo(demo.client);
62 | } else if (version == 2) {
63 | demo.runSimpleV2QueueDemo(demo.client);
64 | }
65 | Thread.sleep(10000);
66 |
67 | } catch (Exception ex) {
68 | ex.printStackTrace();
69 | } finally {
70 | demo.stop();
71 | logger.info("Done");
72 | }
73 | }
74 |
75 |
76 | private void runSimpleV1Demo(DynoJedisClient dyno) throws IOException {
77 | String region = System.getProperty("LOCAL_DATACENTER");
78 | String localRack = System.getProperty("LOCAL_RACK");
79 |
80 | String prefix = "dynoQueue_";
81 |
82 | ConsistentAWSDynoShardSupplier ss = new ConsistentAWSDynoShardSupplier(dyno.getConnPool().getConfiguration().getHostSupplier(), region, localRack);
83 |
84 | RedisQueues queues = new RedisQueues(dyno, dyno, prefix, ss, 50_000, 50_000);
85 |
86 | List payloads = new ArrayList<>();
87 | payloads.add(new Message("id1", "searchable payload123"));
88 | payloads.add(new Message("id2", "payload 2"));
89 | payloads.add(new Message("id3", "payload 3"));
90 | payloads.add(new Message("id4", "payload 4"));
91 | payloads.add(new Message("id5", "payload 5"));
92 | payloads.add(new Message("id6", "payload 6"));
93 | payloads.add(new Message("id7", "payload 7"));
94 | payloads.add(new Message("id8", "payload 8"));
95 | payloads.add(new Message("id9", "payload 9"));
96 | payloads.add(new Message("id10", "payload 10"));
97 | payloads.add(new Message("id11", "payload 11"));
98 | payloads.add(new Message("id12", "payload 12"));
99 | payloads.add(new Message("id13", "payload 13"));
100 | payloads.add(new Message("id14", "payload 14"));
101 | payloads.add(new Message("id15", "payload 15"));
102 |
103 | DynoQueue V1Queue = queues.get("simpleQueue");
104 |
105 | // Clear the queue in case the server already has the above key.
106 | V1Queue.clear();
107 |
108 | // Test push() API
109 | List pushed_msgs = V1Queue.push(payloads);
110 |
111 | // Test ensure() API
112 | Message msg1 = payloads.get(0);
113 | logger.info("Does Message with ID '" + msg1.getId() + "' already exist? -> " + !V1Queue.ensure(msg1));
114 |
115 | // Test containsPredicate() API
116 | logger.info("Does the predicate 'searchable' exist in the queue? -> " + V1Queue.containsPredicate("searchable"));
117 |
118 | // Test getMsgWithPredicate() API
119 | logger.info("Get MSG ID that contains 'searchable' in the queue -> " + V1Queue.getMsgWithPredicate("searchable pay*"));
120 |
121 | // Test getMsgWithPredicate(predicate, localShardOnly=true) API
122 | // NOTE: This only works on single ring sized Dynomite clusters.
123 | logger.info("Get MSG ID that contains 'searchable' in the queue -> " + V1Queue.getMsgWithPredicate("searchable pay*", true));
124 | logger.info("Get MSG ID that contains '3' in the queue -> " + V1Queue.getMsgWithPredicate("3", true));
125 |
126 | Message poppedWithPredicate = V1Queue.popMsgWithPredicate("searchable pay*", false);
127 | V1Queue.ack(poppedWithPredicate.getId());
128 |
129 | List specific_pops = new ArrayList<>();
130 | // We'd only be able to pop from the local shard with popWithMsgId(), so try to pop the first payload ID we see in the local shard.
131 | // Until then pop all messages not in the local shard with unsafePopWithMsgIdAllShards().
132 | for (int i = 1; i < payloads.size(); ++i) {
133 | Message popWithMsgId = V1Queue.popWithMsgId(payloads.get(i).getId());
134 | if (popWithMsgId != null) {
135 | specific_pops.add(popWithMsgId);
136 | break;
137 | } else {
138 | // If we were unable to pop using popWithMsgId(), that means the message ID does not exist in the local shard.
139 | // Ensure that we can pop with unsafePopWithMsgIdAllShards().
140 | Message unsafeSpecificPop = V1Queue.unsafePopWithMsgIdAllShards(payloads.get(i).getId());
141 | assert(unsafeSpecificPop != null);
142 | boolean ack = V1Queue.ack(unsafeSpecificPop.getId());
143 | assert(ack);
144 | }
145 | }
146 |
147 | // Test ack()
148 | boolean ack_successful = V1Queue.ack(specific_pops.get(0).getId());
149 | assert(ack_successful);
150 |
151 | // Test remove()
152 | // Note: This checks for "id9" specifically as it implicitly expects every 3rd element we push to be in our
153 | // local shard.
154 | boolean removed = V1Queue.remove("id9");
155 | assert(removed);
156 |
157 | // Test pop(). Even though we try to pop 3 messages, there will only be one remaining message in our local shard.
158 | List popped_msgs = V1Queue.pop(1, 1000, TimeUnit.MILLISECONDS);
159 | V1Queue.ack(popped_msgs.get(0).getId());
160 |
161 | // Test unsafePeekAllShards()
162 | List peek_all_msgs = V1Queue.unsafePeekAllShards(5);
163 | for (Message msg : peek_all_msgs) {
164 | logger.info("Message peeked (ID : payload) -> " + msg.getId() + " : " + msg.getPayload());
165 | }
166 |
167 | // Test unsafePopAllShards()
168 | List pop_all_msgs = V1Queue.unsafePopAllShards(7, 1000, TimeUnit.MILLISECONDS);
169 | for (Message msg : pop_all_msgs) {
170 | logger.info("Message popped (ID : payload) -> " + msg.getId() + " : " + msg.getPayload());
171 | boolean ack = V1Queue.ack(msg.getId());
172 | assert(ack);
173 | }
174 |
175 | V1Queue.clear();
176 | V1Queue.close();
177 | }
178 |
179 | private void runSimpleV2QueueDemo(DynoJedisClient dyno) throws IOException {
180 | String prefix = "dynoQueue_";
181 |
182 | DynoQueue queue = new QueueBuilder()
183 | .setQueueName("test")
184 | .setRedisKeyPrefix(prefix)
185 | .useDynomite(dyno, dyno)
186 | .setUnackTime(50_000)
187 | .build();
188 |
189 | Message msg = new Message("id1", "message payload");
190 | queue.push(Arrays.asList(msg));
191 |
192 | int count = 10;
193 | List polled = queue.pop(count, 1, TimeUnit.SECONDS);
194 | logger.info(polled.toString());
195 |
196 | queue.ack("id1");
197 | queue.close();
198 | }
199 | }
200 |
--------------------------------------------------------------------------------
/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/QueueMonitor.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright 2016 Netflix, Inc.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 | package com.netflix.dyno.queues.redis;
17 |
18 | import java.io.Closeable;
19 | import java.io.IOException;
20 | import java.util.concurrent.Executors;
21 | import java.util.concurrent.ScheduledExecutorService;
22 | import java.util.concurrent.TimeUnit;
23 |
24 | import com.netflix.servo.DefaultMonitorRegistry;
25 | import com.netflix.servo.MonitorRegistry;
26 | import com.netflix.servo.monitor.BasicCounter;
27 | import com.netflix.servo.monitor.BasicStopwatch;
28 | import com.netflix.servo.monitor.BasicTimer;
29 | import com.netflix.servo.monitor.MonitorConfig;
30 | import com.netflix.servo.monitor.StatsMonitor;
31 | import com.netflix.servo.monitor.Stopwatch;
32 | import com.netflix.servo.stats.StatsConfig;
33 |
34 | /**
35 | * @author Viren
36 | * Monitoring for the queue, publishes the metrics using servo
37 | * https://github.com/Netflix/servo
38 | */
39 | public class QueueMonitor implements Closeable {
40 |
41 | public BasicTimer peek;
42 |
43 | public BasicTimer ack;
44 |
45 | public BasicTimer size;
46 |
47 | public BasicTimer processUnack;
48 |
49 | public BasicTimer remove;
50 |
51 | public BasicTimer get;
52 |
53 | public StatsMonitor queueDepth;
54 |
55 | public StatsMonitor batchSize;
56 |
57 | public StatsMonitor pop;
58 |
59 | public StatsMonitor push;
60 |
61 | public BasicCounter misses;
62 |
63 | public StatsMonitor prefetch;
64 |
65 | private String queueName;
66 |
67 | private String shardName;
68 |
69 | private ScheduledExecutorService executor;
70 |
71 | private static final String className = QueueMonitor.class.getSimpleName();
72 |
73 | public QueueMonitor(String queueName, String shardName) {
74 |
75 | String totalTagName = "total";
76 | executor = Executors.newScheduledThreadPool(1);
77 |
78 | this.queueName = queueName;
79 | this.shardName = shardName;
80 |
81 | peek = new BasicTimer(create("peek"), TimeUnit.MILLISECONDS);
82 | ack = new BasicTimer(create("ack"), TimeUnit.MILLISECONDS);
83 | size = new BasicTimer(create("size"), TimeUnit.MILLISECONDS);
84 | processUnack = new BasicTimer(create("processUnack"), TimeUnit.MILLISECONDS);
85 | remove = new BasicTimer(create("remove"), TimeUnit.MILLISECONDS);
86 | get = new BasicTimer(create("get"), TimeUnit.MILLISECONDS);
87 | misses = new BasicCounter(create("queue_miss"));
88 |
89 |
90 | StatsConfig statsConfig = new StatsConfig.Builder().withPublishCount(true).withPublishMax(true).withPublishMean(true).withPublishMin(true).withPublishTotal(true).build();
91 |
92 | queueDepth = new StatsMonitor(create("queueDepth"), statsConfig, executor, totalTagName, true);
93 | batchSize = new StatsMonitor(create("batchSize"), statsConfig, executor, totalTagName, true);
94 | pop = new StatsMonitor(create("pop"), statsConfig, executor, totalTagName, true);
95 | push = new StatsMonitor(create("push"), statsConfig, executor, totalTagName, true);
96 | prefetch = new StatsMonitor(create("prefetch"), statsConfig, executor, totalTagName, true);
97 |
98 | MonitorRegistry registry = DefaultMonitorRegistry.getInstance();
99 |
100 | registry.register(pop);
101 | registry.register(push);
102 | registry.register(peek);
103 | registry.register(ack);
104 | registry.register(size);
105 | registry.register(processUnack);
106 | registry.register(remove);
107 | registry.register(get);
108 | registry.register(queueDepth);
109 | registry.register(misses);
110 | registry.register(batchSize);
111 | registry.register(prefetch);
112 | }
113 |
114 | private MonitorConfig create(String name) {
115 | return MonitorConfig.builder(name).withTag("class", className).withTag("shard", shardName).withTag("queueName", queueName).build();
116 | }
117 |
118 | public Stopwatch start(StatsMonitor sm, int batchCount) {
119 | int count = (batchCount == 0) ? 1 : batchCount;
120 | Stopwatch sw = new BasicStopwatch() {
121 |
122 | @Override
123 | public void stop() {
124 | super.stop();
125 | long duration = getDuration(TimeUnit.MILLISECONDS) / count;
126 | sm.record(duration);
127 | batchSize.record(count);
128 | }
129 |
130 | };
131 | sw.start();
132 | return sw;
133 | }
134 |
135 | @Override
136 | public void close() throws IOException {
137 | executor.shutdown();
138 | }
139 | }
140 |
--------------------------------------------------------------------------------
/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/QueueUtils.java:
--------------------------------------------------------------------------------
1 | package com.netflix.dyno.queues.redis;
2 |
3 | import com.fasterxml.jackson.annotation.JsonInclude;
4 | import com.fasterxml.jackson.databind.DeserializationFeature;
5 | import com.fasterxml.jackson.databind.ObjectMapper;
6 | import com.fasterxml.jackson.databind.SerializationFeature;
7 | import com.netflix.dyno.connectionpool.exception.DynoException;
8 |
9 | import java.util.concurrent.Callable;
10 | import java.util.concurrent.ExecutionException;
11 |
12 | /**
13 | * Helper class to consolidate functions which might be reused across different DynoQueue implementations.
14 | */
15 | public class QueueUtils {
16 |
17 | private static final int retryCount = 2;
18 |
19 | /**
20 | * Execute function with retries if required
21 | *
22 | * @param opName
23 | * @param keyName
24 | * @param r
25 | * @param
26 | * @return
27 | */
28 | public static R execute(String opName, String keyName, Callable r) {
29 | return executeWithRetry(opName, keyName, r, 0);
30 | }
31 |
32 | private static R executeWithRetry(String opName, String keyName, Callable r, int retryNum) {
33 |
34 | try {
35 |
36 | return r.call();
37 |
38 | } catch (ExecutionException e) {
39 |
40 | if (e.getCause() instanceof DynoException) {
41 | if (retryNum < retryCount) {
42 | return executeWithRetry(opName, keyName, r, ++retryNum);
43 | }
44 | }
45 | throw new RuntimeException(e.getCause());
46 | } catch (Exception e) {
47 | throw new RuntimeException(
48 | "Operation: ( " + opName + " ) failed on key: [" + keyName + " ].", e);
49 | }
50 | }
51 |
52 | /**
53 | * Construct standard objectmapper to use within the DynoQueue instances to read/write Message objects
54 | *
55 | * @return
56 | */
57 | public static ObjectMapper constructObjectMapper() {
58 | ObjectMapper om = new ObjectMapper();
59 | om.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
60 | om.configure(DeserializationFeature.FAIL_ON_IGNORED_PROPERTIES, false);
61 | om.configure(DeserializationFeature.FAIL_ON_NULL_FOR_PRIMITIVES, false);
62 | om.setSerializationInclusion(JsonInclude.Include.NON_NULL);
63 | om.setSerializationInclusion(JsonInclude.Include.NON_EMPTY);
64 | om.disable(SerializationFeature.INDENT_OUTPUT);
65 | return om;
66 | }
67 | }
68 |
--------------------------------------------------------------------------------
/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisQueues.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright 2016 Netflix, Inc.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 | package com.netflix.dyno.queues.redis;
17 |
18 | import com.netflix.dyno.jedis.DynoJedisClient;
19 | import com.netflix.dyno.queues.DynoQueue;
20 | import com.netflix.dyno.queues.ShardSupplier;
21 | import com.netflix.dyno.queues.redis.sharding.RoundRobinStrategy;
22 | import com.netflix.dyno.queues.redis.sharding.ShardingStrategy;
23 | import redis.clients.jedis.commands.JedisCommands;
24 |
25 | import java.io.Closeable;
26 | import java.io.IOException;
27 | import java.time.Clock;
28 | import java.util.Collection;
29 | import java.util.Set;
30 | import java.util.concurrent.ConcurrentHashMap;
31 |
32 | /**
33 | * @author Viren
34 | *
35 | * Please note that you should take care for disposing resource related to RedisQueue instances - that means you
36 | * should call close() on RedisQueue instance.
37 | */
38 | public class RedisQueues implements Closeable {
39 |
40 | private final Clock clock;
41 |
42 | private final JedisCommands quorumConn;
43 |
44 | private final JedisCommands nonQuorumConn;
45 |
46 | private final Set allShards;
47 |
48 | private final String shardName;
49 |
50 | private final String redisKeyPrefix;
51 |
52 | private final int unackTime;
53 |
54 | private final int unackHandlerIntervalInMS;
55 |
56 | private final ConcurrentHashMap queues;
57 |
58 | private final ShardingStrategy shardingStrategy;
59 |
60 | private final boolean singleRingTopology;
61 |
62 | /**
63 | * @param quorumConn Dyno connection with dc_quorum enabled
64 | * @param nonQuorumConn Dyno connection to local Redis
65 | * @param redisKeyPrefix prefix applied to the Redis keys
66 | * @param shardSupplier Provider for the shards for the queues created
67 | * @param unackTime Time in millisecond within which a message needs to be acknowledged by the client, after which the message is re-queued.
68 | * @param unackHandlerIntervalInMS Time in millisecond at which the un-acknowledgement processor runs
69 | */
70 | public RedisQueues(JedisCommands quorumConn, JedisCommands nonQuorumConn, String redisKeyPrefix, ShardSupplier shardSupplier, int unackTime, int unackHandlerIntervalInMS) {
71 | this(Clock.systemDefaultZone(), quorumConn, nonQuorumConn, redisKeyPrefix, shardSupplier, unackTime, unackHandlerIntervalInMS, new RoundRobinStrategy());
72 | }
73 |
74 | /**
75 | * @param quorumConn Dyno connection with dc_quorum enabled
76 | * @param nonQuorumConn Dyno connection to local Redis
77 | * @param redisKeyPrefix prefix applied to the Redis keys
78 | * @param shardSupplier Provider for the shards for the queues created
79 | * @param unackTime Time in millisecond within which a message needs to be acknowledged by the client, after which the message is re-queued.
80 | * @param unackHandlerIntervalInMS Time in millisecond at which the un-acknowledgement processor runs
81 | * @param shardingStrategy sharding strategy responsible for calculating message's destination shard
82 | */
83 | public RedisQueues(JedisCommands quorumConn, JedisCommands nonQuorumConn, String redisKeyPrefix, ShardSupplier shardSupplier, int unackTime, int unackHandlerIntervalInMS, ShardingStrategy shardingStrategy) {
84 | this(Clock.systemDefaultZone(), quorumConn, nonQuorumConn, redisKeyPrefix, shardSupplier, unackTime, unackHandlerIntervalInMS, shardingStrategy);
85 | }
86 |
87 |
88 | /**
89 | * @param clock Time provider
90 | * @param quorumConn Dyno connection with dc_quorum enabled
91 | * @param nonQuorumConn Dyno connection to local Redis
92 | * @param redisKeyPrefix prefix applied to the Redis keys
93 | * @param shardSupplier Provider for the shards for the queues created
94 | * @param unackTime Time in millisecond within which a message needs to be acknowledged by the client, after which the message is re-queued.
95 | * @param unackHandlerIntervalInMS Time in millisecond at which the un-acknowledgement processor runs
96 | * @param shardingStrategy sharding strategy responsible for calculating message's destination shard
97 | */
98 | public RedisQueues(Clock clock, JedisCommands quorumConn, JedisCommands nonQuorumConn, String redisKeyPrefix, ShardSupplier shardSupplier, int unackTime, int unackHandlerIntervalInMS, ShardingStrategy shardingStrategy) {
99 | this.clock = clock;
100 | this.quorumConn = quorumConn;
101 | this.nonQuorumConn = nonQuorumConn;
102 | this.redisKeyPrefix = redisKeyPrefix;
103 | this.allShards = shardSupplier.getQueueShards();
104 | this.shardName = shardSupplier.getCurrentShard();
105 | this.unackTime = unackTime;
106 | this.unackHandlerIntervalInMS = unackHandlerIntervalInMS;
107 | this.queues = new ConcurrentHashMap<>();
108 | this.shardingStrategy = shardingStrategy;
109 |
110 | if (quorumConn instanceof DynoJedisClient) {
111 | this.singleRingTopology = ((DynoJedisClient) quorumConn).getConnPool().getPools().size() == 3;
112 | } else {
113 | this.singleRingTopology = false;
114 | }
115 | }
116 |
117 | /**
118 | *
119 | * @param queueName Name of the queue
120 | * @return Returns the DynoQueue hosting the given queue by name
121 | * @see DynoQueue
122 | * @see RedisDynoQueue
123 | */
124 | public DynoQueue get(String queueName) {
125 |
126 | String key = queueName.intern();
127 |
128 | return queues.computeIfAbsent(key, (keyToCompute) -> new RedisDynoQueue(clock, redisKeyPrefix, queueName, allShards, shardName, unackHandlerIntervalInMS, shardingStrategy, singleRingTopology)
129 | .withUnackTime(unackTime)
130 | .withNonQuorumConn(nonQuorumConn)
131 | .withQuorumConn(quorumConn));
132 | }
133 |
134 | /**
135 | *
136 | * @return Collection of all the registered queues
137 | */
138 | public Collection queues() {
139 | return this.queues.values();
140 | }
141 |
142 | @Override
143 | public void close() throws IOException {
144 | queues.values().forEach(queue -> {
145 | try {
146 | queue.close();
147 | } catch (final IOException e) {
148 | throw new RuntimeException(e.getCause());
149 | }
150 | });
151 | }
152 | }
153 |
--------------------------------------------------------------------------------
/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/conn/DynoClientProxy.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright 2018 Netflix, Inc.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 | /**
17 | *
18 | */
19 | package com.netflix.dyno.queues.redis.conn;
20 |
21 | import java.util.Set;
22 |
23 | import com.netflix.dyno.jedis.DynoJedisClient;
24 |
25 | import redis.clients.jedis.Tuple;
26 |
27 | /**
28 | * @author Viren
29 | *
30 | * Dynomite connection
31 | */
32 | public class DynoClientProxy implements RedisConnection {
33 |
34 | private DynoJedisClient jedis;
35 |
36 |
37 | public DynoClientProxy(DynoJedisClient jedis) {
38 | this.jedis = jedis;
39 | }
40 |
41 | @Override
42 | public RedisConnection getResource() {
43 | return this;
44 | }
45 |
46 | @Override
47 | public void close() {
48 | //nothing!
49 | }
50 |
51 | @Override
52 | public Pipe pipelined() {
53 | return new DynoJedisPipe(jedis.pipelined());
54 | }
55 |
56 | @Override
57 | public String hget(String key, String member) {
58 | return jedis.hget(key, member);
59 | }
60 |
61 | @Override
62 | public Long zrem(String key, String member) {
63 | return jedis.zrem(key, member);
64 | }
65 |
66 | @Override
67 | public Long hdel(String key, String member) {
68 | return jedis.hdel(key, member);
69 |
70 | }
71 |
72 | @Override
73 | public Double zscore(String key, String member) {
74 | return jedis.zscore(key, member);
75 | }
76 |
77 | @Override
78 | public void zadd(String key, double score, String member) {
79 | jedis.zadd(key, score, member);
80 | }
81 |
82 | @Override
83 | public void hset(String key, String member, String json) {
84 | jedis.hset(key, member, json);
85 | }
86 |
87 | @Override
88 | public long zcard(String key) {
89 | return jedis.zcard(key);
90 | }
91 |
92 | @Override
93 | public void del(String key) {
94 | jedis.del(key);
95 | }
96 |
97 | @Override
98 | public Set zrangeByScore(String key, int min, double max, int offset, int count) {
99 | return jedis.zrangeByScore(key, min, max, offset, count);
100 | }
101 |
102 | @Override
103 | public Set zrangeByScoreWithScores(String key, int min, double max, int offset, int count) {
104 | return jedis.zrangeByScoreWithScores(key, min, max, offset, count);
105 | }
106 |
107 | }
108 |
--------------------------------------------------------------------------------
/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/conn/DynoJedisPipe.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright 2018 Netflix, Inc.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 | /**
17 | *
18 | */
19 | package com.netflix.dyno.queues.redis.conn;
20 |
21 |
22 | import com.netflix.dyno.jedis.DynoJedisPipeline;
23 | import redis.clients.jedis.Response;
24 | import redis.clients.jedis.params.ZAddParams;
25 |
26 | /**
27 | * @author Viren
28 | * Pipeline abstraction for Dynomite Pipeline.
29 | */
30 | public class DynoJedisPipe implements Pipe {
31 |
32 | private DynoJedisPipeline pipe;
33 |
34 | private boolean modified;
35 |
36 | public DynoJedisPipe(DynoJedisPipeline pipe) {
37 | this.pipe = pipe;
38 | this.modified = false;
39 | }
40 |
41 | @Override
42 | public void hset(String key, String field, String value) {
43 | pipe.hset(key, field, value);
44 | this.modified = true;
45 |
46 | }
47 |
48 | @Override
49 | public Response zadd(String key, double score, String member) {
50 | this.modified = true;
51 | return pipe.zadd(key, score, member);
52 | }
53 |
54 | @Override
55 | public Response zadd(String key, double score, String member, ZAddParams zParams) {
56 | this.modified = true;
57 | return pipe.zadd(key, score, member, zParams);
58 | }
59 |
60 | @Override
61 | public Response zrem(String key, String member) {
62 | this.modified = true;
63 | return pipe.zrem(key, member);
64 | }
65 |
66 | @Override
67 | public Response hget(String key, String member) {
68 | this.modified = true;
69 | return pipe.hget(key, member);
70 | }
71 |
72 | @Override
73 | public Response hdel(String key, String member) {
74 | this.modified = true;
75 | return pipe.hdel(key, member);
76 | }
77 |
78 | @Override
79 | public void sync() {
80 | if (modified) {
81 | pipe.sync();
82 | modified = false;
83 | }
84 | }
85 |
86 | @Override
87 | public void close() throws Exception {
88 | pipe.close();
89 | }
90 |
91 |
92 | }
93 |
--------------------------------------------------------------------------------
/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/conn/JedisProxy.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright 2018 Netflix, Inc.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 | /**
17 | *
18 | */
19 | package com.netflix.dyno.queues.redis.conn;
20 |
21 | import java.util.Set;
22 |
23 | import redis.clients.jedis.Jedis;
24 | import redis.clients.jedis.JedisPool;
25 | import redis.clients.jedis.Tuple;
26 |
27 | /**
28 | * @author Viren
29 | *
30 | * This class provides the abstraction of a Jedis Connection Pool. Used when using Redis directly without Dynomite.
31 | */
32 | public class JedisProxy implements RedisConnection {
33 |
34 | private JedisPool pool;
35 |
36 | private Jedis jedis;
37 |
38 | public JedisProxy(JedisPool pool) {
39 | this.pool = pool;
40 | }
41 |
42 | public JedisProxy(Jedis jedis) {
43 | this.jedis = jedis;
44 | }
45 |
46 | @Override
47 | public RedisConnection getResource() {
48 | Jedis jedis = pool.getResource();
49 | return new JedisProxy(jedis);
50 | }
51 |
52 | @Override
53 | public void close() {
54 | jedis.close();
55 | }
56 |
57 | @Override
58 | public Pipe pipelined() {
59 | return new RedisPipe(jedis.pipelined());
60 | }
61 |
62 | @Override
63 | public String hget(String key, String member) {
64 | return jedis.hget(key, member);
65 | }
66 |
67 | @Override
68 | public Long zrem(String key, String member) {
69 | return jedis.zrem(key, member);
70 | }
71 |
72 | @Override
73 | public Long hdel(String key, String member) {
74 | return jedis.hdel(key, member);
75 |
76 | }
77 |
78 | @Override
79 | public Double zscore(String key, String member) {
80 | return jedis.zscore(key, member);
81 | }
82 |
83 | @Override
84 | public void zadd(String key, double unackScore, String member) {
85 | jedis.zadd(key, unackScore, member);
86 | }
87 |
88 | @Override
89 | public void hset(String key, String member, String json) {
90 | jedis.hset(key, member, json);
91 | }
92 |
93 | @Override
94 | public long zcard(String key) {
95 | return jedis.zcard(key);
96 | }
97 |
98 | @Override
99 | public void del(String key) {
100 | jedis.del(key);
101 | }
102 |
103 | @Override
104 | public Set zrangeByScore(String key, int min, double max, int offset, int count) {
105 | return jedis.zrangeByScore(key, min, max, offset, count);
106 | }
107 |
108 | @Override
109 | public Set zrangeByScoreWithScores(String key, int min, double max, int offset, int count) {
110 | return jedis.zrangeByScoreWithScores(key, min, max, offset, count);
111 | }
112 |
113 | }
114 |
--------------------------------------------------------------------------------
/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/conn/Pipe.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright 2018 Netflix, Inc.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 | package com.netflix.dyno.queues.redis.conn;
17 |
18 | import com.netflix.dyno.jedis.DynoJedisPipeline;
19 | import redis.clients.jedis.Pipeline;
20 | import redis.clients.jedis.Response;
21 | import redis.clients.jedis.params.ZAddParams;
22 |
23 | /**
24 | *
25 | * @author Viren
26 | *
27 | * Abstraction of Redis Pipeline.
28 | * The abstraction is required as there is no common interface between DynoJedisPipeline and Jedis' Pipeline classes.
29 | *
30 | * @see DynoJedisPipeline
31 | * @see Pipeline
32 | * The commands here reflects the RedisCommand structure.
33 | *
34 | */
35 | public interface Pipe {
36 |
37 | /**
38 | *
39 | * @param key The Key
40 | * @param field Field
41 | * @param value Value of the Field
42 | */
43 | public void hset(String key, String field, String value);
44 |
45 | /**
46 | *
47 | * @param key The Key
48 | * @param score Score for the member
49 | * @param member Member to be added within the key
50 | * @return
51 | */
52 | public Response zadd(String key, double score, String member);
53 |
54 | /**
55 | *
56 | * @param key The Key
57 | * @param score Score for the member
58 | * @param member Member to be added within the key
59 | * @param zParams Parameters
60 | * @return
61 | */
62 | public Response zadd(String key, double score, String member, ZAddParams zParams);
63 |
64 | /**
65 | *
66 | * @param key The Key
67 | * @param member Member
68 | * @return
69 | */
70 | public Response zrem(String key, String member);
71 |
72 | /**
73 | *
74 | * @param key The Key
75 | * @param member Member
76 | * @return
77 | */
78 | public Response hget(String key, String member);
79 |
80 | /**
81 | *
82 | * @param key
83 | * @param member
84 | * @return
85 | */
86 | public Response hdel(String key, String member);
87 |
88 | /**
89 | *
90 | */
91 | public void sync();
92 |
93 | /**
94 | *
95 | * @throws Exception
96 | */
97 | public void close() throws Exception;
98 | }
--------------------------------------------------------------------------------
/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/conn/RedisConnection.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright 2018 Netflix, Inc.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 | package com.netflix.dyno.queues.redis.conn;
17 |
18 | import java.util.Set;
19 |
20 | import com.netflix.dyno.jedis.DynoJedisClient;
21 |
22 | import redis.clients.jedis.Jedis;
23 | import redis.clients.jedis.Tuple;
24 |
25 | /**
26 | * Abstraction of Redis connection.
27 | *
28 | * @author viren
29 | *
30 | * The methods are 1-1 proxies from Jedis. See Jedis documentation for the details.
31 | *
32 | * @see Jedis
33 | * @see DynoJedisClient
34 | */
35 | public interface RedisConnection {
36 |
37 | /**
38 | *
39 | * @return Returns the underlying connection resource. For connection pool, returns the actual connection
40 | */
41 | public RedisConnection getResource();
42 |
43 | public String hget(String messkeyageStoreKey, String member);
44 |
45 | public Long zrem(String key, String member);
46 |
47 | public Long hdel(String key, String member);
48 |
49 | public Double zscore(String key, String member);
50 |
51 | public void zadd(String key, double score, String member);
52 |
53 | public void hset(String key, String id, String json);
54 |
55 | public long zcard(String key);
56 |
57 | public void del(String key);
58 |
59 | public Set zrangeByScore(String key, int min, double max, int offset, int count);
60 |
61 | public Set zrangeByScoreWithScores(String key, int min, double max, int offset, int count);
62 |
63 | public void close();
64 |
65 | public Pipe pipelined();
66 |
67 | }
--------------------------------------------------------------------------------
/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/conn/RedisPipe.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright 2018 Netflix, Inc.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 | /**
17 | *
18 | */
19 | package com.netflix.dyno.queues.redis.conn;
20 |
21 | import redis.clients.jedis.Pipeline;
22 | import redis.clients.jedis.Response;
23 | import redis.clients.jedis.params.ZAddParams;
24 |
25 | /**
26 | * @author Viren
27 | *
28 | * Pipeline abstraction for direct redis connection - when not using Dynomite.
29 | */
30 | public class RedisPipe implements Pipe {
31 |
32 | private Pipeline pipe;
33 |
34 | public RedisPipe(Pipeline pipe) {
35 | this.pipe = pipe;
36 | }
37 |
38 | @Override
39 | public void hset(String key, String field, String value) {
40 | pipe.hset(key, field, value);
41 |
42 | }
43 |
44 | @Override
45 | public Response zadd(String key, double score, String member) {
46 | return pipe.zadd(key, score, member);
47 | }
48 |
49 | @Override
50 | public Response zadd(String key, double score, String member, ZAddParams zParams) {
51 | return pipe.zadd(key, score, member, zParams);
52 | }
53 |
54 | @Override
55 | public Response zrem(String key, String member) {
56 | return pipe.zrem(key, member);
57 | }
58 |
59 | @Override
60 | public Response hget(String key, String member) {
61 | return pipe.hget(key, member);
62 | }
63 |
64 | @Override
65 | public Response hdel(String key, String member) {
66 | return pipe.hdel(key, member);
67 | }
68 |
69 | @Override
70 | public void sync() {
71 | pipe.sync();
72 | }
73 |
74 | @Override
75 | public void close() throws Exception {
76 | pipe.close();
77 | }
78 |
79 |
80 | }
81 |
--------------------------------------------------------------------------------
/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/sharding/RoundRobinStrategy.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright 2018 Netflix, Inc.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 | package com.netflix.dyno.queues.redis.sharding;
17 |
18 | import com.netflix.dyno.queues.Message;
19 |
20 | import java.util.List;
21 | import java.util.concurrent.atomic.AtomicInteger;
22 |
23 | public class RoundRobinStrategy implements ShardingStrategy {
24 |
25 | private final AtomicInteger nextShardIndex = new AtomicInteger(0);
26 |
27 | /**
28 | * Get shard based on round robin strategy.
29 | * @param allShards
30 | * @param message is ignored in round robin strategy
31 | * @return
32 | */
33 | @Override
34 | public String getNextShard(List allShards, Message message) {
35 | int index = nextShardIndex.incrementAndGet();
36 | if (index >= allShards.size()) {
37 | nextShardIndex.set(0);
38 | index = 0;
39 | }
40 | return allShards.get(index);
41 | }
42 | }
43 |
--------------------------------------------------------------------------------
/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/sharding/ShardingStrategy.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright 2018 Netflix, Inc.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 | package com.netflix.dyno.queues.redis.sharding;
17 |
18 | import com.netflix.dyno.queues.Message;
19 |
20 | import java.util.List;
21 |
22 | /**
23 | * Expose common interface that allow to apply custom sharding strategy.
24 | */
25 | public interface ShardingStrategy {
26 | String getNextShard(List allShards, Message message);
27 | }
28 |
--------------------------------------------------------------------------------
/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/MultiRedisQueue.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright 2017 Netflix, Inc.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 | package com.netflix.dyno.queues.redis.v2;
17 |
18 | import com.netflix.dyno.queues.DynoQueue;
19 | import com.netflix.dyno.queues.Message;
20 |
21 | import java.io.IOException;
22 | import java.lang.UnsupportedOperationException;
23 | import java.util.HashMap;
24 | import java.util.LinkedList;
25 | import java.util.List;
26 | import java.util.Map;
27 | import java.util.Map.Entry;
28 | import java.util.concurrent.TimeUnit;
29 | import java.util.concurrent.atomic.AtomicInteger;
30 | import java.util.stream.Collectors;
31 |
32 | /**
33 | * @author Viren
34 | * MultiRedisQueue exposes a single queue using multiple redis queues. Each RedisQueue is a shard.
35 | * When pushing elements to the queue, does a round robin to push the message to one of the shards.
36 | * When polling, the message is polled from the current shard (shardName) the instance is associated with.
37 | */
38 | public class MultiRedisQueue implements DynoQueue {
39 |
40 | private List shards;
41 |
42 | private String name;
43 |
44 | private Map queues = new HashMap<>();
45 |
46 | private RedisPipelineQueue me;
47 |
48 | public MultiRedisQueue(String queueName, String shardName, Map queues) {
49 | this.name = queueName;
50 | this.queues = queues;
51 | this.me = queues.get(shardName);
52 | if (me == null) {
53 | throw new IllegalArgumentException("List of shards supplied (" + queues.keySet() + ") does not contain current shard name: " + shardName);
54 | }
55 | this.shards = queues.keySet().stream().collect(Collectors.toList());
56 | }
57 |
58 | @Override
59 | public String getName() {
60 | return name;
61 | }
62 |
63 | @Override
64 | public int getUnackTime() {
65 | return me.getUnackTime();
66 | }
67 |
68 | @Override
69 | public List push(List messages) {
70 | int size = queues.size();
71 | int partitionSize = messages.size() / size;
72 | List ids = new LinkedList<>();
73 |
74 | for (int i = 0; i < size - 1; i++) {
75 | RedisPipelineQueue queue = queues.get(getNextShard());
76 | int start = i * partitionSize;
77 | int end = start + partitionSize;
78 | ids.addAll(queue.push(messages.subList(start, end)));
79 | }
80 | RedisPipelineQueue queue = queues.get(getNextShard());
81 | int start = (size - 1) * partitionSize;
82 |
83 | ids.addAll(queue.push(messages.subList(start, messages.size())));
84 | return ids;
85 | }
86 |
87 | @Override
88 | public List pop(int messageCount, int wait, TimeUnit unit) {
89 | return me.pop(messageCount, wait, unit);
90 | }
91 |
92 | @Override
93 | public Message popWithMsgId(String messageId) {
94 | throw new UnsupportedOperationException();
95 | }
96 |
97 | @Override
98 | public Message unsafePopWithMsgIdAllShards(String messageId) {
99 | throw new UnsupportedOperationException();
100 | }
101 |
102 | @Override
103 | public List peek(int messageCount) {
104 | return me.peek(messageCount);
105 | }
106 |
107 | @Override
108 | public boolean ack(String messageId) {
109 | for (DynoQueue q : queues.values()) {
110 | if (q.ack(messageId)) {
111 | return true;
112 | }
113 | }
114 | return false;
115 | }
116 |
117 | @Override
118 | public void ack(List messages) {
119 | Map> byShard = messages.stream().collect(Collectors.groupingBy(Message::getShard));
120 | for (Entry> e : byShard.entrySet()) {
121 | queues.get(e.getKey()).ack(e.getValue());
122 | }
123 | }
124 |
125 | @Override
126 | public boolean setUnackTimeout(String messageId, long timeout) {
127 | for (DynoQueue q : queues.values()) {
128 | if (q.setUnackTimeout(messageId, timeout)) {
129 | return true;
130 | }
131 | }
132 | return false;
133 | }
134 |
135 | @Override
136 | public boolean setTimeout(String messageId, long timeout) {
137 | for (DynoQueue q : queues.values()) {
138 | if (q.setTimeout(messageId, timeout)) {
139 | return true;
140 | }
141 | }
142 | return false;
143 | }
144 |
145 | @Override
146 | public boolean remove(String messageId) {
147 | for (DynoQueue q : queues.values()) {
148 | if (q.remove(messageId)) {
149 | return true;
150 | }
151 | }
152 | return false;
153 | }
154 |
155 | @Override
156 | public boolean ensure(Message message) {
157 | throw new UnsupportedOperationException();
158 | }
159 |
160 |
161 | @Override
162 | public boolean containsPredicate(String predicate) {
163 | return containsPredicate(predicate, false);
164 | }
165 |
166 | @Override
167 | public String getMsgWithPredicate(String predicate) {
168 | return getMsgWithPredicate(predicate, false);
169 | }
170 |
171 | @Override
172 | public boolean containsPredicate(String predicate, boolean localShardOnly) {
173 | throw new UnsupportedOperationException();
174 | }
175 |
176 | @Override
177 | public String getMsgWithPredicate(String predicate, boolean localShardOnly) {
178 | throw new UnsupportedOperationException();
179 | }
180 |
181 | @Override
182 | public Message popMsgWithPredicate(String predicate, boolean localShardOnly) {
183 | throw new UnsupportedOperationException();
184 | }
185 |
186 | @Override
187 | public List bulkPop(int messageCount, int wait, TimeUnit unit) {
188 | throw new UnsupportedOperationException();
189 | }
190 |
191 | @Override
192 | public List unsafeBulkPop(int messageCount, int wait, TimeUnit unit) {
193 | throw new UnsupportedOperationException();
194 | }
195 |
196 | @Override
197 | public Message get(String messageId) {
198 | for (DynoQueue q : queues.values()) {
199 | Message msg = q.get(messageId);
200 | if (msg != null) {
201 | return msg;
202 | }
203 | }
204 | return null;
205 | }
206 |
207 | @Override
208 | public Message localGet(String messageId) {
209 | throw new UnsupportedOperationException();
210 | }
211 |
212 | @Override
213 | public long size() {
214 | long size = 0;
215 | for (DynoQueue q : queues.values()) {
216 | size += q.size();
217 | }
218 | return size;
219 | }
220 |
221 | @Override
222 | public Map> shardSizes() {
223 | Map> sizes = new HashMap<>();
224 | for (Entry e : queues.entrySet()) {
225 | sizes.put(e.getKey(), e.getValue().shardSizes().get(e.getKey()));
226 | }
227 | return sizes;
228 | }
229 |
230 | @Override
231 | public void clear() {
232 | for (DynoQueue q : queues.values()) {
233 | q.clear();
234 | }
235 |
236 | }
237 |
238 | @Override
239 | public void close() throws IOException {
240 | for (RedisPipelineQueue queue : queues.values()) {
241 | queue.close();
242 | }
243 | }
244 |
245 | @Override
246 | public List getAllMessages() {
247 | throw new UnsupportedOperationException();
248 | }
249 |
250 | @Override
251 | public void processUnacks() {
252 | for (RedisPipelineQueue queue : queues.values()) {
253 | queue.processUnacks();
254 | }
255 | }
256 |
257 | @Override
258 | public void atomicProcessUnacks() {
259 | throw new UnsupportedOperationException();
260 | }
261 |
262 | @Override
263 | public List findStaleMessages() { throw new UnsupportedOperationException(); }
264 |
265 | @Override
266 | public boolean atomicRemove(String messageId) {
267 | throw new UnsupportedOperationException();
268 | }
269 |
270 | private AtomicInteger nextShardIndex = new AtomicInteger(0);
271 |
272 | private String getNextShard() {
273 | int indx = nextShardIndex.incrementAndGet();
274 | if (indx >= shards.size()) {
275 | nextShardIndex.set(0);
276 | indx = 0;
277 | }
278 | String s = shards.get(indx);
279 | return s;
280 | }
281 |
282 | @Override
283 | public List unsafePeekAllShards(final int messageCount) {
284 | throw new UnsupportedOperationException();
285 | }
286 |
287 | @Override
288 | public List unsafePopAllShards(int messageCount, int wait, TimeUnit unit) {
289 | throw new UnsupportedOperationException();
290 | }
291 |
292 | }
293 |
--------------------------------------------------------------------------------
/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/QueueBuilder.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright 2018 Netflix, Inc.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 | /**
17 | *
18 | */
19 | package com.netflix.dyno.queues.redis.v2;
20 |
21 | import com.google.common.collect.Collections2;
22 | import com.google.common.collect.Lists;
23 | import com.netflix.appinfo.AmazonInfo;
24 | import com.netflix.appinfo.AmazonInfo.MetaDataKey;
25 | import com.netflix.appinfo.InstanceInfo;
26 | import com.netflix.appinfo.InstanceInfo.InstanceStatus;
27 | import com.netflix.discovery.EurekaClient;
28 | import com.netflix.discovery.shared.Application;
29 | import com.netflix.dyno.connectionpool.Host;
30 | import com.netflix.dyno.connectionpool.HostBuilder;
31 | import com.netflix.dyno.connectionpool.HostSupplier;
32 | import com.netflix.dyno.connectionpool.impl.utils.ConfigUtils;
33 | import com.netflix.dyno.jedis.DynoJedisClient;
34 | import com.netflix.dyno.queues.DynoQueue;
35 | import com.netflix.dyno.queues.ShardSupplier;
36 | import com.netflix.dyno.queues.redis.conn.DynoClientProxy;
37 | import com.netflix.dyno.queues.redis.conn.JedisProxy;
38 | import com.netflix.dyno.queues.redis.conn.RedisConnection;
39 | import com.netflix.dyno.queues.shard.DynoShardSupplier;
40 | import redis.clients.jedis.JedisPool;
41 | import redis.clients.jedis.JedisPoolConfig;
42 |
43 | import java.time.Clock;
44 | import java.util.ArrayList;
45 | import java.util.Collection;
46 | import java.util.HashMap;
47 | import java.util.List;
48 | import java.util.Map;
49 |
50 | /**
51 | * @author Viren
52 | * Builder for the queues.
53 | *
54 | */
55 | public class QueueBuilder {
56 |
57 | private Clock clock;
58 |
59 | private String queueName;
60 |
61 | private String redisKeyPrefix;
62 |
63 | private int unackTime;
64 |
65 | private String currentShard;
66 |
67 | private ShardSupplier shardSupplier;
68 |
69 | private HostSupplier hs;
70 |
71 | private EurekaClient eurekaClient;
72 |
73 | private String applicationName;
74 |
75 | private Collection hosts;
76 |
77 | private JedisPoolConfig redisPoolConfig;
78 |
79 | private DynoJedisClient dynoQuorumClient;
80 |
81 | private DynoJedisClient dynoNonQuorumClient;
82 |
83 | /**
84 | * @param clock the Clock instance to set
85 | * @return instance of QueueBuilder
86 | */
87 | public QueueBuilder setClock(Clock clock) {
88 | this.clock = clock;
89 | return this;
90 | }
91 |
92 | public QueueBuilder setApplicationName(String appName) {
93 | this.applicationName = appName;
94 | return this;
95 | }
96 |
97 | public QueueBuilder setEurekaClient(EurekaClient eurekaClient) {
98 | this.eurekaClient = eurekaClient;
99 | return this;
100 | }
101 |
102 | /**
103 | * @param queueName the queueName to set
104 | * @return instance of QueueBuilder
105 | */
106 | public QueueBuilder setQueueName(String queueName) {
107 | this.queueName = queueName;
108 | return this;
109 | }
110 |
111 | /**
112 | * @param redisKeyPrefix Prefix used for all the keys in Redis
113 | * @return instance of QueueBuilder
114 | */
115 | public QueueBuilder setRedisKeyPrefix(String redisKeyPrefix) {
116 | this.redisKeyPrefix = redisKeyPrefix;
117 | return this;
118 | }
119 |
120 | /**
121 | * @param redisPoolConfig
122 | * @return instance of QueueBuilder
123 | */
124 | public QueueBuilder useNonDynomiteRedis(JedisPoolConfig redisPoolConfig, List redisHosts) {
125 | this.redisPoolConfig = redisPoolConfig;
126 | this.hosts = redisHosts;
127 | return this;
128 | }
129 |
130 | /**
131 | *
132 | * @param dynoQuorumClient
133 | * @param dynoNonQuorumClient
134 | * @return
135 | */
136 | public QueueBuilder useDynomite(DynoJedisClient dynoQuorumClient, DynoJedisClient dynoNonQuorumClient) {
137 | this.dynoQuorumClient = dynoQuorumClient;
138 | this.dynoNonQuorumClient = dynoNonQuorumClient;
139 | this.hs = dynoQuorumClient.getConnPool().getConfiguration().getHostSupplier();
140 | return this;
141 | }
142 |
143 | /**
144 | * @param unackTime Time in millisecond, after which the uncked messages will be re-queued for the delivery
145 | * @return instance of QueueBuilder
146 | */
147 | public QueueBuilder setUnackTime(int unackTime) {
148 | this.unackTime = unackTime;
149 | return this;
150 | }
151 |
152 | /**
153 | * @param currentShard Name of the current shard
154 | * @return instance of QueueBuilder
155 | */
156 | public QueueBuilder setCurrentShard(String currentShard) {
157 | this.currentShard = currentShard;
158 | return this;
159 | }
160 |
161 | /**
162 | * @param shardSupplier
163 | * @return
164 | */
165 | public QueueBuilder setShardSupplier(ShardSupplier shardSupplier) {
166 | this.shardSupplier = shardSupplier;
167 | return this;
168 | }
169 |
170 | /**
171 | *
172 | * @return Build an instance of the queue with supplied parameters.
173 | * @see MultiRedisQueue
174 | * @see RedisPipelineQueue
175 | */
176 | public DynoQueue build() {
177 |
178 | boolean useDynomiteCluster = dynoQuorumClient != null;
179 | if (useDynomiteCluster) {
180 | if(hs == null) {
181 | hs = dynoQuorumClient.getConnPool().getConfiguration().getHostSupplier();
182 | }
183 | this.hosts = hs.getHosts();
184 | }
185 |
186 | if (shardSupplier == null) {
187 | String region = ConfigUtils.getDataCenter();
188 | String az = ConfigUtils.getLocalZone();
189 | shardSupplier = new DynoShardSupplier(hs, region, az);
190 | }
191 | if(currentShard == null) {
192 | currentShard = shardSupplier.getCurrentShard();
193 | }
194 |
195 | if (clock == null) {
196 | clock = Clock.systemDefaultZone();
197 | }
198 |
199 | Map shardMap = new HashMap<>();
200 | for (Host host : hosts) {
201 | String shard = shardSupplier.getShardForHost(host);
202 | shardMap.put(shard, host);
203 | }
204 |
205 |
206 | Map queues = new HashMap<>();
207 |
208 | for (String queueShard : shardMap.keySet()) {
209 |
210 | Host host = shardMap.get(queueShard);
211 | String hostAddress = host.getIpAddress();
212 | if (hostAddress == null || "".equals(hostAddress)) {
213 | hostAddress = host.getHostName();
214 | }
215 | RedisConnection redisConn = null;
216 | RedisConnection redisConnRead = null;
217 |
218 | if (useDynomiteCluster) {
219 | redisConn = new DynoClientProxy(dynoQuorumClient);
220 | if(dynoNonQuorumClient == null) {
221 | dynoNonQuorumClient = dynoQuorumClient;
222 | }
223 | redisConnRead = new DynoClientProxy(dynoNonQuorumClient);
224 | } else {
225 | JedisPool pool = new JedisPool(redisPoolConfig, hostAddress, host.getPort(), 0);
226 | redisConn = new JedisProxy(pool);
227 | redisConnRead = new JedisProxy(pool);
228 | }
229 |
230 | RedisPipelineQueue q = new RedisPipelineQueue(clock, redisKeyPrefix, queueName, queueShard, unackTime, unackTime, redisConn);
231 | q.setNonQuorumPool(redisConnRead);
232 |
233 | queues.put(queueShard, q);
234 | }
235 |
236 | if (queues.size() == 1) {
237 | //This is a queue with a single shard
238 | return queues.values().iterator().next();
239 | }
240 |
241 | MultiRedisQueue queue = new MultiRedisQueue(queueName, currentShard, queues);
242 | return queue;
243 | }
244 |
245 |
246 | private HostSupplier getHostSupplierFromEureka(String applicationName) {
247 | return () -> {
248 | Application app = eurekaClient.getApplication(applicationName);
249 | List hosts = new ArrayList<>();
250 |
251 | if (app == null) {
252 | return hosts;
253 | }
254 |
255 | List ins = app.getInstances();
256 |
257 | if (ins == null || ins.isEmpty()) {
258 | return hosts;
259 | }
260 |
261 | hosts = Lists.newArrayList(Collections2.transform(ins,
262 |
263 | info -> {
264 |
265 | Host.Status status = info.getStatus() == InstanceStatus.UP ? Host.Status.Up : Host.Status.Down;
266 | String rack = null;
267 | if (info.getDataCenterInfo() instanceof AmazonInfo) {
268 | AmazonInfo amazonInfo = (AmazonInfo) info.getDataCenterInfo();
269 | rack = amazonInfo.get(MetaDataKey.availabilityZone);
270 | }
271 | //Host host = new Host(info.getHostName(), info.getIPAddr(), rack, status);
272 | Host host = new HostBuilder()
273 | .setHostname(info.getHostName())
274 | .setIpAddress(info.getIPAddr())
275 | .setRack(rack).setStatus(status)
276 | .createHost();
277 | return host;
278 | }));
279 | return hosts;
280 | };
281 | }
282 | }
283 |
--------------------------------------------------------------------------------
/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/shard/ConsistentAWSDynoShardSupplier.java:
--------------------------------------------------------------------------------
1 | package com.netflix.dyno.queues.shard;
2 |
3 | import com.netflix.dyno.connectionpool.HostSupplier;
4 |
5 | import java.util.HashMap;
6 | import java.util.Map;
7 |
8 | public class ConsistentAWSDynoShardSupplier extends ConsistentDynoShardSupplier {
9 |
10 | /**
11 | * Dynomite based shard supplier. Keeps the number of shards in parity with the hosts and regions
12 | *
13 | * Note: This ensures that all racks use the same shard names. This fixes issues with the now deprecated DynoShardSupplier
14 | * that would write to the wrong shard if there are cross-region writers/readers.
15 | *
16 | * @param hs Host supplier
17 | * @param region current region
18 | * @param localRack local rack identifier
19 | */
20 | public ConsistentAWSDynoShardSupplier(HostSupplier hs, String region, String localRack) {
21 | super(hs, region, localRack);
22 | Map rackToHashMapEntries = new HashMap() {{
23 | this.put("us-east-1c", "c");
24 | this.put("us-east-1d", "d");
25 | this.put("us-east-1e", "e");
26 |
27 | this.put("eu-west-1a", "c");
28 | this.put("eu-west-1b", "d");
29 | this.put("eu-west-1c", "e");
30 |
31 | this.put("us-west-2a", "c");
32 | this.put("us-west-2b", "d");
33 | this.put("us-west-2c", "e");
34 | }};
35 | setRackToShardMap(rackToHashMapEntries);
36 | }
37 | }
--------------------------------------------------------------------------------
/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/shard/ConsistentDynoShardSupplier.java:
--------------------------------------------------------------------------------
1 | package com.netflix.dyno.queues.shard;
2 |
3 | import com.netflix.dyno.connectionpool.Host;
4 | import com.netflix.dyno.connectionpool.HostSupplier;
5 | import com.netflix.dyno.queues.ShardSupplier;
6 |
7 | import java.util.*;
8 |
9 | abstract class ConsistentDynoShardSupplier implements ShardSupplier {
10 |
11 | protected HostSupplier hs;
12 |
13 | protected String region;
14 |
15 | protected String localRack;
16 |
17 | protected Map rackToShardMap;
18 |
19 | /**
20 | * Dynomite based shard supplier. Keeps the number of shards in parity with the hosts and regions
21 | * @param hs Host supplier
22 | * @param region current region
23 | * @param localRack local rack identifier
24 | */
25 | public ConsistentDynoShardSupplier(HostSupplier hs, String region, String localRack) {
26 | this.hs = hs;
27 | this.region = region;
28 | this.localRack = localRack;
29 | }
30 |
31 | public void setRackToShardMap(Map rackToShardMapEntries) {
32 | rackToShardMap = new HashMap<>(rackToShardMapEntries);
33 | }
34 |
35 | @Override
36 | public String getCurrentShard() {
37 | return rackToShardMap.get(localRack);
38 | }
39 |
40 | @Override
41 | public Set getQueueShards() {
42 | Set queueShards = new HashSet<>();
43 | List hosts = hs.getHosts();
44 | for (Host host : hosts) {
45 | queueShards.add(rackToShardMap.get(host.getRack()));
46 | }
47 | return queueShards;
48 | }
49 |
50 | @Override
51 | public String getShardForHost(Host host) {
52 | return rackToShardMap.get(host.getRack());
53 | }
54 | }
--------------------------------------------------------------------------------
/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/shard/DynoShardSupplier.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright 2016 Netflix, Inc.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 | /**
17 | *
18 | */
19 | package com.netflix.dyno.queues.shard;
20 |
21 | import com.netflix.dyno.connectionpool.Host;
22 | import com.netflix.dyno.connectionpool.HostSupplier;
23 | import com.netflix.dyno.queues.ShardSupplier;
24 |
25 | import java.util.Set;
26 | import java.util.function.Function;
27 | import java.util.stream.Collectors;
28 |
29 | /**
30 | * @author Viren
31 | *
32 | * NOTE: This class is deprecated and should not be used. It still remains for backwards compatibility for legacy applications
33 | * New applications must use 'ConsistentAWSDynoShardSupplier' or extend 'ConsistentDynoShardSupplier' for non-AWS environments.
34 | *
35 | */
36 | @Deprecated
37 | public class DynoShardSupplier implements ShardSupplier {
38 |
39 | private HostSupplier hs;
40 |
41 | private String region;
42 |
43 | private String localRack;
44 |
45 | private Function rackToShardMap = rack -> rack.substring(rack.length()-1);
46 |
47 | /**
48 | * Dynomite based shard supplier. Keeps the number of shards in parity with the hosts and regions
49 | * @param hs Host supplier
50 | * @param region current region
51 | * @param localRack local rack identifier
52 | */
53 | public DynoShardSupplier(HostSupplier hs, String region, String localRack) {
54 | this.hs = hs;
55 | this.region = region;
56 | this.localRack = localRack;
57 | }
58 |
59 | @Override
60 | public String getCurrentShard() {
61 | return rackToShardMap.apply(localRack);
62 | }
63 |
64 | @Override
65 | public Set getQueueShards() {
66 | return hs.getHosts().stream().map(host -> host.getRack()).map(rackToShardMap).collect(Collectors.toSet());
67 | }
68 |
69 | @Override
70 | public String getShardForHost(Host host) {
71 | return rackToShardMap.apply(host.getRack());
72 | }
73 | }
74 |
--------------------------------------------------------------------------------
/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/shard/SingleShardSupplier.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright 2016 Netflix, Inc.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 | /**
17 | *
18 | */
19 | package com.netflix.dyno.queues.shard;
20 |
21 | import com.google.common.collect.Sets;
22 | import com.netflix.dyno.connectionpool.Host;
23 | import com.netflix.dyno.queues.ShardSupplier;
24 |
25 | import java.util.Set;
26 |
27 | /**
28 | * @author Viren
29 | *
30 | */
31 | public class SingleShardSupplier implements ShardSupplier {
32 |
33 | private String shardName;
34 |
35 | public SingleShardSupplier(String shardName) {
36 | this.shardName = shardName;
37 | }
38 |
39 | @Override
40 | public String getCurrentShard() {
41 | return shardName;
42 | }
43 |
44 | @Override
45 | public String getShardForHost(Host host) {
46 | return shardName;
47 | }
48 |
49 | @Override
50 | public Set getQueueShards() {
51 | return Sets.newHashSet(shardName);
52 | }
53 | }
54 |
--------------------------------------------------------------------------------
/dyno-queues-redis/src/main/resources/demo.properties:
--------------------------------------------------------------------------------
1 | ####
2 | ## Properties to initialize the demo app
3 | #
4 | #LOCAL_DATACENTER=us-east-1
5 | #LOCAL_RACK=us-east-1c
6 | #NETFLIX_STACK=dyno_demo
7 | #EC2_AVAILABILITY_ZONE=us-east-1c
8 | dyno.demo.lbStrategy=TokenAware
9 | dyno.demo.retryPolicy=RetryNTimes:2
10 | dyno.demo.port=8102
11 | dyno.demo.discovery.prod=discoveryreadonly.%s.dynprod.netflix.net:7001/v2/apps
12 | dyno.demo.discovery.test=discoveryreadonly.%s.dyntest.netflix.net:7001/v2/apps
13 |
--------------------------------------------------------------------------------
/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/BaseQueueTests.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright 2016 Netflix, Inc.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 | package com.netflix.dyno.queues.redis;
17 |
18 | import com.google.common.util.concurrent.Uninterruptibles;
19 | import com.netflix.dyno.queues.DynoQueue;
20 | import com.netflix.dyno.queues.Message;
21 | import org.junit.Before;
22 | import org.junit.Test;
23 |
24 | import java.util.Arrays;
25 | import java.util.LinkedList;
26 | import java.util.List;
27 | import java.util.Map;
28 | import java.util.Random;
29 | import java.util.Set;
30 | import java.util.UUID;
31 | import java.util.concurrent.CopyOnWriteArrayList;
32 | import java.util.concurrent.CountDownLatch;
33 | import java.util.concurrent.ExecutionException;
34 | import java.util.concurrent.Executors;
35 | import java.util.concurrent.ScheduledExecutorService;
36 | import java.util.concurrent.TimeUnit;
37 | import java.util.concurrent.atomic.AtomicInteger;
38 | import java.util.stream.Collectors;
39 |
40 | import static org.junit.Assert.assertEquals;
41 | import static org.junit.Assert.assertFalse;
42 | import static org.junit.Assert.assertNotNull;
43 | import static org.junit.Assert.assertNull;
44 | import static org.junit.Assert.assertTrue;
45 |
46 | public abstract class BaseQueueTests {
47 |
48 |
49 | private String queueName;
50 |
51 | protected static final String redisKeyPrefix = "testdynoqueues";
52 |
53 | protected DynoQueue rdq;
54 |
55 | protected String messageKeyPrefix;
56 |
57 |
58 | public abstract DynoQueue getQueue(String redisKeyPrefix, String queueName);
59 |
60 | public BaseQueueTests(String queueName) {
61 | this.queueName = queueName;
62 | this.messageKeyPrefix = redisKeyPrefix + ".MESSAGE.";
63 |
64 | this.rdq = getQueue(redisKeyPrefix, queueName);
65 | this.rdq.clear();
66 |
67 | }
68 |
69 |
70 | @Test
71 | public void testGetName() {
72 | assertEquals(queueName, rdq.getName());
73 | }
74 |
75 | @Test
76 | public void testGetUnackTime() {
77 | assertEquals(1_000, rdq.getUnackTime());
78 | }
79 |
80 | @Test
81 | public void testTimeoutUpdate() {
82 |
83 | rdq.clear();
84 |
85 | String id = UUID.randomUUID().toString();
86 | Message msg = new Message(id, "Hello World-" + id);
87 | msg.setTimeout(100, TimeUnit.MILLISECONDS);
88 | rdq.push(Arrays.asList(msg));
89 |
90 | List popped = rdq.pop(1, 10, TimeUnit.MILLISECONDS);
91 | assertNotNull(popped);
92 | assertEquals(0, popped.size());
93 |
94 | Uninterruptibles.sleepUninterruptibly(500, TimeUnit.MILLISECONDS);
95 |
96 | popped = rdq.pop(1, 1, TimeUnit.SECONDS);
97 | assertNotNull(popped);
98 | assertEquals(1, popped.size());
99 |
100 | boolean updated = rdq.setUnackTimeout(id, 500);
101 | assertTrue(updated);
102 | popped = rdq.pop(1, 1, TimeUnit.SECONDS);
103 | assertNotNull(popped);
104 | assertEquals(0, popped.size());
105 |
106 | Uninterruptibles.sleepUninterruptibly(1000, TimeUnit.MILLISECONDS);
107 | rdq.processUnacks();
108 | popped = rdq.pop(1, 1, TimeUnit.SECONDS);
109 | assertNotNull(popped);
110 | assertEquals(1, popped.size());
111 |
112 | updated = rdq.setUnackTimeout(id, 10_000); //10 seconds!
113 | assertTrue(updated);
114 | rdq.processUnacks();
115 | popped = rdq.pop(1, 1, TimeUnit.SECONDS);
116 | assertNotNull(popped);
117 | assertEquals(0, popped.size());
118 |
119 | updated = rdq.setUnackTimeout(id, 0);
120 | assertTrue(updated);
121 | rdq.processUnacks();
122 | popped = rdq.pop(1, 1, TimeUnit.SECONDS);
123 | assertNotNull(popped);
124 | assertEquals(1, popped.size());
125 |
126 | rdq.ack(id);
127 | Map> size = rdq.shardSizes();
128 | Map values = size.get("a");
129 | long total = values.values().stream().mapToLong(v -> v).sum();
130 | assertEquals(0, total);
131 |
132 | popped = rdq.pop(1, 1, TimeUnit.SECONDS);
133 | assertNotNull(popped);
134 | assertEquals(0, popped.size());
135 | }
136 |
137 | @Test
138 | public void testConcurrency() throws InterruptedException, ExecutionException {
139 |
140 | rdq.clear();
141 |
142 | final int count = 100;
143 | final AtomicInteger published = new AtomicInteger(0);
144 |
145 | ScheduledExecutorService ses = Executors.newScheduledThreadPool(6);
146 | CountDownLatch publishLatch = new CountDownLatch(1);
147 | Runnable publisher = new Runnable() {
148 |
149 | @Override
150 | public void run() {
151 | List messages = new LinkedList<>();
152 | for (int i = 0; i < 10; i++) {
153 | Message msg = new Message(UUID.randomUUID().toString(), "Hello World-" + i);
154 | msg.setPriority(new Random().nextInt(98));
155 | messages.add(msg);
156 | }
157 | if (published.get() >= count) {
158 | publishLatch.countDown();
159 | return;
160 | }
161 |
162 | published.addAndGet(messages.size());
163 | rdq.push(messages);
164 | }
165 | };
166 |
167 | for (int p = 0; p < 3; p++) {
168 | ses.scheduleWithFixedDelay(publisher, 1, 1, TimeUnit.MILLISECONDS);
169 | }
170 | publishLatch.await();
171 | CountDownLatch latch = new CountDownLatch(count);
172 | List allMsgs = new CopyOnWriteArrayList<>();
173 | AtomicInteger consumed = new AtomicInteger(0);
174 | AtomicInteger counter = new AtomicInteger(0);
175 | Runnable consumer = () -> {
176 | if (consumed.get() >= count) {
177 | return;
178 | }
179 | List popped = rdq.pop(100, 1, TimeUnit.MILLISECONDS);
180 | allMsgs.addAll(popped);
181 | consumed.addAndGet(popped.size());
182 | popped.stream().forEach(p -> latch.countDown());
183 | counter.incrementAndGet();
184 | };
185 | for (int c = 0; c < 2; c++) {
186 | ses.scheduleWithFixedDelay(consumer, 1, 10, TimeUnit.MILLISECONDS);
187 | }
188 | Uninterruptibles.awaitUninterruptibly(latch);
189 | System.out.println("Consumed: " + consumed.get() + ", all: " + allMsgs.size() + " counter: " + counter.get());
190 | Set uniqueMessages = allMsgs.stream().collect(Collectors.toSet());
191 |
192 | assertEquals(count, allMsgs.size());
193 | assertEquals(count, uniqueMessages.size());
194 | List more = rdq.pop(1, 1, TimeUnit.SECONDS);
195 | // If we published more than we consumed since we could've published more than we consumed in which case this
196 | // will not be empty
197 | if(published.get() == consumed.get())
198 | assertEquals(0, more.size());
199 | else
200 | assertEquals(1, more.size());
201 |
202 | ses.shutdownNow();
203 | }
204 |
205 | @Test
206 | public void testSetTimeout() {
207 |
208 | rdq.clear();
209 |
210 | Message msg = new Message("x001yx", "Hello World");
211 | msg.setPriority(3);
212 | msg.setTimeout(10_000);
213 | rdq.push(Arrays.asList(msg));
214 |
215 | List popped = rdq.pop(1, 1, TimeUnit.SECONDS);
216 | assertTrue(popped.isEmpty());
217 |
218 | boolean updated = rdq.setTimeout(msg.getId(), 0);
219 | assertTrue(updated);
220 | popped = rdq.pop(2, 1, TimeUnit.SECONDS);
221 | assertEquals(1, popped.size());
222 | assertEquals(0, popped.get(0).getTimeout());
223 | }
224 |
225 | @Test
226 | public void testAll() {
227 |
228 | rdq.clear();
229 | assertEquals(0, rdq.size());
230 |
231 | int count = 10;
232 | List messages = new LinkedList<>();
233 | for (int i = 0; i < count; i++) {
234 | Message msg = new Message("" + i, "Hello World-" + i);
235 | msg.setPriority(count - i);
236 | messages.add(msg);
237 | }
238 | rdq.push(messages);
239 |
240 | messages = rdq.peek(count);
241 |
242 | assertNotNull(messages);
243 | assertEquals(count, messages.size());
244 | long size = rdq.size();
245 | assertEquals(count, size);
246 |
247 | // We did a peek - let's ensure the messages are still around!
248 | List messages2 = rdq.peek(count);
249 | assertNotNull(messages2);
250 | assertEquals(messages, messages2);
251 |
252 | List poped = rdq.pop(count, 1, TimeUnit.SECONDS);
253 | assertNotNull(poped);
254 | assertEquals(count, poped.size());
255 | assertEquals(messages, poped);
256 |
257 | Uninterruptibles.sleepUninterruptibly(2, TimeUnit.SECONDS);
258 | rdq.processUnacks();
259 |
260 | for (Message msg : messages) {
261 | Message found = rdq.get(msg.getId());
262 | assertNotNull(found);
263 | assertEquals(msg.getId(), found.getId());
264 | assertEquals(msg.getTimeout(), found.getTimeout());
265 | }
266 | assertNull(rdq.get("some fake id"));
267 |
268 | List messages3 = rdq.pop(count, 1, TimeUnit.SECONDS);
269 | if (messages3.size() < count) {
270 | List messages4 = rdq.pop(count, 1, TimeUnit.SECONDS);
271 | messages3.addAll(messages4);
272 | }
273 |
274 | assertNotNull(messages3);
275 | assertEquals(10, messages3.size());
276 | assertEquals(messages.stream().map(msg -> msg.getId()).sorted().collect(Collectors.toList()), messages3.stream().map(msg -> msg.getId()).sorted().collect(Collectors.toList()));
277 | assertEquals(10, messages3.stream().map(msg -> msg.getId()).collect(Collectors.toSet()).size());
278 | messages3.stream().forEach(System.out::println);
279 |
280 | for (Message msg : messages3) {
281 | assertTrue(rdq.ack(msg.getId()));
282 | assertFalse(rdq.ack(msg.getId()));
283 | }
284 | Uninterruptibles.sleepUninterruptibly(2, TimeUnit.SECONDS);
285 | messages3 = rdq.pop(count, 1, TimeUnit.SECONDS);
286 | assertNotNull(messages3);
287 | assertEquals(0, messages3.size());
288 | }
289 |
290 | @Before
291 | public void clear() {
292 | rdq.clear();
293 | }
294 |
295 | @Test
296 | public void testClearQueues() {
297 | rdq.clear();
298 | int count = 10;
299 | List messages = new LinkedList<>();
300 | for (int i = 0; i < count; i++) {
301 | Message msg = new Message("x" + i, "Hello World-" + i);
302 | msg.setPriority(count - i);
303 | messages.add(msg);
304 | }
305 |
306 | rdq.push(messages);
307 | assertEquals(count, rdq.size());
308 | rdq.clear();
309 | assertEquals(0, rdq.size());
310 |
311 | }
312 |
313 | }
314 |
--------------------------------------------------------------------------------
/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/CustomShardingStrategyTest.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright 2016 Netflix, Inc.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 | package com.netflix.dyno.queues.redis;
17 |
18 | import com.netflix.dyno.connectionpool.Host;
19 | import com.netflix.dyno.connectionpool.HostBuilder;
20 | import com.netflix.dyno.connectionpool.HostSupplier;
21 | import com.netflix.dyno.queues.Message;
22 | import com.netflix.dyno.queues.ShardSupplier;
23 | import com.netflix.dyno.queues.jedis.JedisMock;
24 | import com.netflix.dyno.queues.redis.sharding.ShardingStrategy;
25 | import org.junit.Before;
26 | import org.junit.BeforeClass;
27 | import org.junit.Test;
28 |
29 | import java.util.Iterator;
30 | import java.util.LinkedList;
31 | import java.util.List;
32 | import java.util.Set;
33 | import java.util.concurrent.TimeUnit;
34 | import java.util.stream.Collectors;
35 |
36 | import static org.junit.Assert.assertEquals;
37 |
38 | public class CustomShardingStrategyTest {
39 |
40 | public static class HashBasedStrategy implements ShardingStrategy {
41 | @Override
42 | public String getNextShard(List allShards, Message message) {
43 | int hashCodeAbs = Math.abs(message.getId().hashCode());
44 | int calculatedShard = (hashCodeAbs % allShards.size());
45 | return allShards.get(calculatedShard);
46 | }
47 | }
48 |
49 | private static JedisMock dynoClient;
50 |
51 | private static final String queueName = "test_queue";
52 |
53 | private static final String redisKeyPrefix = "testdynoqueues";
54 |
55 | private static RedisDynoQueue shard1DynoQueue;
56 | private static RedisDynoQueue shard2DynoQueue;
57 | private static RedisDynoQueue shard3DynoQueue;
58 |
59 | private static RedisQueues shard1Queue;
60 | private static RedisQueues shard2Queue;
61 | private static RedisQueues shard3Queue;
62 |
63 | private static String messageKey;
64 |
65 | @BeforeClass
66 | public static void setUpBeforeClass() throws Exception {
67 |
68 | HostSupplier hs = new HostSupplier() {
69 | @Override
70 | public List getHosts() {
71 | List hosts = new LinkedList<>();
72 | hosts.add(
73 | new HostBuilder()
74 | .setHostname("localhost")
75 | .setPort(8102)
76 | .setRack("rack1")
77 | .setStatus(Host.Status.Up)
78 | .createHost()
79 | );
80 | hosts.add(
81 | new HostBuilder()
82 | .setHostname("localhost")
83 | .setPort(8102)
84 | .setRack("rack2")
85 | .setStatus(Host.Status.Up)
86 | .createHost()
87 | );
88 | hosts.add(
89 | new HostBuilder()
90 | .setHostname("localhost")
91 | .setPort(8102)
92 | .setRack("rack3")
93 | .setStatus(Host.Status.Up)
94 | .createHost()
95 | );
96 | return hosts;
97 | }
98 | };
99 |
100 | dynoClient = new JedisMock();
101 |
102 | Set allShards = hs.getHosts().stream().map(host -> host.getRack().substring(host.getRack().length() - 2)).collect(Collectors.toSet());
103 |
104 | Iterator iterator = allShards.iterator();
105 | String shard1Name = iterator.next();
106 | String shard2Name = iterator.next();
107 | String shard3Name = iterator.next();
108 |
109 | ShardSupplier shard1Supplier = new ShardSupplier() {
110 |
111 | @Override
112 | public Set getQueueShards() {
113 | return allShards;
114 | }
115 |
116 | @Override
117 | public String getCurrentShard() {
118 | return shard1Name;
119 | }
120 |
121 | @Override
122 | public String getShardForHost(Host host) {
123 | return null;
124 | }
125 | };
126 |
127 | ShardSupplier shard2Supplier = new ShardSupplier() {
128 |
129 | @Override
130 | public Set getQueueShards() {
131 | return allShards;
132 | }
133 |
134 | @Override
135 | public String getCurrentShard() {
136 | return shard2Name;
137 | }
138 |
139 | @Override
140 | public String getShardForHost(Host host) {
141 | return null;
142 | }
143 | };
144 |
145 |
146 | ShardSupplier shard3Supplier = new ShardSupplier() {
147 |
148 | @Override
149 | public Set getQueueShards() {
150 | return allShards;
151 | }
152 |
153 | @Override
154 | public String getCurrentShard() {
155 | return shard3Name;
156 | }
157 |
158 | @Override
159 | public String getShardForHost(Host host) {
160 | return null;
161 | }
162 | };
163 |
164 | messageKey = redisKeyPrefix + ".MESSAGE." + queueName;
165 |
166 | HashBasedStrategy hashBasedStrategy = new HashBasedStrategy();
167 |
168 | shard1Queue = new RedisQueues(dynoClient, dynoClient, redisKeyPrefix, shard1Supplier, 1_000, 1_000_000, hashBasedStrategy);
169 | shard2Queue = new RedisQueues(dynoClient, dynoClient, redisKeyPrefix, shard2Supplier, 1_000, 1_000_000, hashBasedStrategy);
170 | shard3Queue = new RedisQueues(dynoClient, dynoClient, redisKeyPrefix, shard3Supplier, 1_000, 1_000_000, hashBasedStrategy);
171 |
172 | shard1DynoQueue = (RedisDynoQueue) shard1Queue.get(queueName);
173 | shard2DynoQueue = (RedisDynoQueue) shard2Queue.get(queueName);
174 | shard3DynoQueue = (RedisDynoQueue) shard3Queue.get(queueName);
175 | }
176 |
177 | @Before
178 | public void clearAll() {
179 | shard1DynoQueue.clear();
180 | shard2DynoQueue.clear();
181 | shard3DynoQueue.clear();
182 | }
183 |
184 | @Test
185 | public void testAll() {
186 |
187 | List messages = new LinkedList<>();
188 |
189 | Message msg = new Message("1", "Hello World");
190 | msg.setPriority(1);
191 | messages.add(msg);
192 |
193 | /**
194 | * Because my custom sharding strategy that depends on message id, and calculated hash (just Java's hashCode),
195 | * message will always ends on the same shard, so message never duplicates, in test case, I expect that
196 | * message will be received only once.
197 | */
198 | shard1DynoQueue.push(messages);
199 | shard1DynoQueue.push(messages);
200 | shard1DynoQueue.push(messages);
201 |
202 | List popedFromShard1 = shard1DynoQueue.pop(1, 1, TimeUnit.SECONDS);
203 | List popedFromShard2 = shard2DynoQueue.pop(1, 1, TimeUnit.SECONDS);
204 | List popedFromShard3 = shard3DynoQueue.pop(1, 1, TimeUnit.SECONDS);
205 |
206 | assertEquals(0, popedFromShard1.size());
207 | assertEquals(1, popedFromShard2.size());
208 | assertEquals(0, popedFromShard3.size());
209 |
210 | assertEquals(msg, popedFromShard2.get(0));
211 | }
212 | }
213 |
--------------------------------------------------------------------------------
/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/DefaultShardingStrategyTest.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright 2016 Netflix, Inc.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 | package com.netflix.dyno.queues.redis;
17 |
18 | import com.netflix.dyno.connectionpool.Host;
19 | import com.netflix.dyno.connectionpool.HostBuilder;
20 | import com.netflix.dyno.connectionpool.HostSupplier;
21 | import com.netflix.dyno.queues.Message;
22 | import com.netflix.dyno.queues.ShardSupplier;
23 | import com.netflix.dyno.queues.jedis.JedisMock;
24 | import org.junit.Before;
25 | import org.junit.BeforeClass;
26 | import org.junit.Test;
27 |
28 | import java.util.Iterator;
29 | import java.util.LinkedList;
30 | import java.util.List;
31 | import java.util.Set;
32 | import java.util.concurrent.TimeUnit;
33 | import java.util.stream.Collectors;
34 |
35 | import static org.junit.Assert.assertEquals;
36 |
37 | public class DefaultShardingStrategyTest {
38 |
39 | private static JedisMock dynoClient;
40 |
41 | private static final String queueName = "test_queue";
42 |
43 | private static final String redisKeyPrefix = "testdynoqueues";
44 |
45 | private static RedisDynoQueue shard1DynoQueue;
46 | private static RedisDynoQueue shard2DynoQueue;
47 | private static RedisDynoQueue shard3DynoQueue;
48 |
49 | private static RedisQueues shard1Queue;
50 | private static RedisQueues shard2Queue;
51 | private static RedisQueues shard3Queue;
52 |
53 | private static String messageKey;
54 |
55 | @BeforeClass
56 | public static void setUpBeforeClass() throws Exception {
57 |
58 | HostSupplier hs = new HostSupplier() {
59 | @Override
60 | public List getHosts() {
61 | List hosts = new LinkedList<>();
62 | hosts.add(
63 | new HostBuilder()
64 | .setHostname("localhost")
65 | .setPort(8102)
66 | .setRack("us-east-1d")
67 | .setStatus(Host.Status.Up)
68 | .createHost()
69 | );
70 | hosts.add(
71 | new HostBuilder()
72 | .setHostname("localhost")
73 | .setPort(8102)
74 | .setRack("us-east-2d")
75 | .setStatus(Host.Status.Up)
76 | .createHost()
77 | );
78 | hosts.add(
79 | new HostBuilder()
80 | .setHostname("localhost")
81 | .setPort(8102)
82 | .setRack("us-east-3d")
83 | .setStatus(Host.Status.Up)
84 | .createHost()
85 | );
86 | return hosts;
87 | }
88 | };
89 |
90 | dynoClient = new JedisMock();
91 |
92 | Set allShards = hs.getHosts().stream().map(host -> host.getRack().substring(host.getRack().length() - 2)).collect(Collectors.toSet());
93 | Iterator iterator = allShards.iterator();
94 | String shard1Name = iterator.next();
95 | String shard2Name = iterator.next();
96 | String shard3Name = iterator.next();
97 |
98 | ShardSupplier shard1Supplier = new ShardSupplier() {
99 |
100 | @Override
101 | public Set getQueueShards() {
102 | return allShards;
103 | }
104 |
105 | @Override
106 | public String getCurrentShard() {
107 | return shard1Name;
108 | }
109 |
110 | @Override
111 | public String getShardForHost(Host host) {
112 | return null;
113 | }
114 | };
115 |
116 | ShardSupplier shard2Supplier = new ShardSupplier() {
117 |
118 | @Override
119 | public Set getQueueShards() {
120 | return allShards;
121 | }
122 |
123 | @Override
124 | public String getCurrentShard() {
125 | return shard2Name;
126 | }
127 |
128 | @Override
129 | public String getShardForHost(Host host) {
130 | return null;
131 | }
132 | };
133 |
134 |
135 | ShardSupplier shard3Supplier = new ShardSupplier() {
136 |
137 | @Override
138 | public Set getQueueShards() {
139 | return allShards;
140 | }
141 |
142 | @Override
143 | public String getCurrentShard() {
144 | return shard3Name;
145 | }
146 |
147 | @Override
148 | public String getShardForHost(Host host) {
149 | return null;
150 | }
151 | };
152 |
153 | messageKey = redisKeyPrefix + ".MESSAGE." + queueName;
154 |
155 | shard1Queue = new RedisQueues(dynoClient, dynoClient, redisKeyPrefix, shard1Supplier, 1_000, 1_000_000);
156 | shard2Queue = new RedisQueues(dynoClient, dynoClient, redisKeyPrefix, shard2Supplier, 1_000, 1_000_000);
157 | shard3Queue = new RedisQueues(dynoClient, dynoClient, redisKeyPrefix, shard3Supplier, 1_000, 1_000_000);
158 |
159 |
160 | shard1DynoQueue = (RedisDynoQueue) shard1Queue.get(queueName);
161 | shard2DynoQueue = (RedisDynoQueue) shard2Queue.get(queueName);
162 | shard3DynoQueue = (RedisDynoQueue) shard3Queue.get(queueName);
163 | }
164 |
165 | @Before
166 | public void clearAll() {
167 | shard1DynoQueue.clear();
168 | shard2DynoQueue.clear();
169 | shard3DynoQueue.clear();
170 | }
171 |
172 | @Test
173 | public void testAll() {
174 |
175 | List messages = new LinkedList<>();
176 |
177 | Message msg = new Message("1", "Hello World");
178 | msg.setPriority(1);
179 | messages.add(msg);
180 |
181 | /**
182 | * Because of sharding strategy works in round-robin manner, single client, for shard1, should
183 | * push message(even the same) to three different shards.
184 | */
185 | shard1DynoQueue.push(messages);
186 | shard1DynoQueue.push(messages);
187 | shard1DynoQueue.push(messages);
188 |
189 | List popedFromShard1 = shard1DynoQueue.pop(1, 1, TimeUnit.SECONDS);
190 |
191 | List popedFromShard2 = shard2DynoQueue.pop(1, 1, TimeUnit.SECONDS);
192 |
193 | List popedFromShard3 = shard3DynoQueue.pop(1, 1, TimeUnit.SECONDS);
194 |
195 |
196 | assertEquals(1, popedFromShard1.size());
197 | assertEquals(1, popedFromShard2.size());
198 | assertEquals(1, popedFromShard3.size());
199 |
200 | assertEquals(msg, popedFromShard1.get(0));
201 | assertEquals(msg, popedFromShard2.get(0));
202 | assertEquals(msg, popedFromShard3.get(0));
203 |
204 | }
205 |
206 | }
207 |
--------------------------------------------------------------------------------
/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/DynoShardSupplierTest.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright 2016 Netflix, Inc.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 | /**
17 | *
18 | */
19 | package com.netflix.dyno.queues.redis;
20 |
21 | import static org.junit.Assert.assertEquals;
22 | import static org.junit.Assert.assertNotNull;
23 |
24 | import java.util.Arrays;
25 | import java.util.Collection;
26 | import java.util.LinkedList;
27 | import java.util.List;
28 | import java.util.Set;
29 | import java.util.stream.Collectors;
30 |
31 | import com.netflix.dyno.connectionpool.HostBuilder;
32 | import org.junit.Test;
33 |
34 | import com.netflix.dyno.connectionpool.Host;
35 | import com.netflix.dyno.connectionpool.Host.Status;
36 | import com.netflix.dyno.queues.shard.DynoShardSupplier;
37 | import com.netflix.dyno.connectionpool.HostSupplier;
38 |
39 | /**
40 | * @author Viren
41 | *
42 | */
43 | public class DynoShardSupplierTest {
44 |
45 | @Test
46 | public void test(){
47 | HostSupplier hs = new HostSupplier() {
48 | @Override
49 | public List getHosts() {
50 | List hosts = new LinkedList<>();
51 | hosts.add(
52 | new HostBuilder()
53 | .setHostname("host1")
54 | .setPort(8102)
55 | .setRack("us-east-1a")
56 | .setStatus(Host.Status.Up)
57 | .createHost()
58 | );
59 | hosts.add(
60 | new HostBuilder()
61 | .setHostname("host1")
62 | .setPort(8102)
63 | .setRack("us-east-1b")
64 | .setStatus(Host.Status.Up)
65 | .createHost()
66 | );
67 | hosts.add(
68 | new HostBuilder()
69 | .setHostname("host1")
70 | .setPort(8102)
71 | .setRack("us-east-1d")
72 | .setStatus(Host.Status.Up)
73 | .createHost()
74 | );
75 |
76 | return hosts;
77 | }
78 | };
79 | DynoShardSupplier supplier = new DynoShardSupplier(hs, "us-east-1", "a");
80 | String localShard = supplier.getCurrentShard();
81 | Set allShards = supplier.getQueueShards();
82 |
83 | assertNotNull(localShard);
84 | assertEquals("a", localShard);
85 | assertNotNull(allShards);
86 | assertEquals(Arrays.asList("a","b","d").stream().collect(Collectors.toSet()), allShards);
87 | }
88 | }
89 |
--------------------------------------------------------------------------------
/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/RedisDynoQueueTest.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright 2016 Netflix, Inc.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 | package com.netflix.dyno.queues.redis;
17 |
18 | import com.google.common.util.concurrent.Uninterruptibles;
19 | import com.netflix.dyno.connectionpool.Host;
20 | import com.netflix.dyno.connectionpool.HostBuilder;
21 | import com.netflix.dyno.connectionpool.HostSupplier;
22 | import com.netflix.dyno.queues.DynoQueue;
23 | import com.netflix.dyno.queues.Message;
24 | import com.netflix.dyno.queues.ShardSupplier;
25 | import com.netflix.dyno.queues.jedis.JedisMock;
26 | import org.junit.Before;
27 | import org.junit.BeforeClass;
28 | import org.junit.Test;
29 |
30 | import java.util.Arrays;
31 | import java.util.LinkedList;
32 | import java.util.List;
33 | import java.util.Map;
34 | import java.util.Random;
35 | import java.util.Set;
36 | import java.util.UUID;
37 | import java.util.concurrent.CopyOnWriteArrayList;
38 | import java.util.concurrent.CountDownLatch;
39 | import java.util.concurrent.ExecutionException;
40 | import java.util.concurrent.Executors;
41 | import java.util.concurrent.ScheduledExecutorService;
42 | import java.util.concurrent.TimeUnit;
43 | import java.util.concurrent.atomic.AtomicInteger;
44 | import java.util.stream.Collectors;
45 |
46 | import static org.junit.Assert.assertEquals;
47 | import static org.junit.Assert.assertFalse;
48 | import static org.junit.Assert.assertNotNull;
49 | import static org.junit.Assert.assertNull;
50 | import static org.junit.Assert.assertTrue;
51 |
52 | public class RedisDynoQueueTest {
53 |
54 | private static JedisMock dynoClient;
55 |
56 | private static final String queueName = "test_queue";
57 |
58 | private static final String redisKeyPrefix = "testdynoqueues";
59 |
60 | private static RedisDynoQueue rdq;
61 |
62 | private static RedisQueues rq;
63 |
64 | private static String messageKey;
65 |
66 | @BeforeClass
67 | public static void setUpBeforeClass() throws Exception {
68 |
69 | HostSupplier hs = new HostSupplier() {
70 | @Override
71 | public List getHosts() {
72 | List hosts = new LinkedList<>();
73 | hosts.add(
74 | new HostBuilder()
75 | .setHostname("ec2-11-22-33-444.compute-0.amazonaws.com")
76 | .setPort(8102)
77 | .setRack("us-east-1d")
78 | .setStatus(Host.Status.Up)
79 | .createHost()
80 | );
81 | return hosts;
82 | }
83 | };
84 |
85 | dynoClient = new JedisMock();
86 |
87 | Set allShards = hs.getHosts().stream().map(host -> host.getRack().substring(host.getRack().length() - 2)).collect(Collectors.toSet());
88 | String shardName = allShards.iterator().next();
89 | ShardSupplier ss = new ShardSupplier() {
90 |
91 | @Override
92 | public Set getQueueShards() {
93 | return allShards;
94 | }
95 |
96 | @Override
97 | public String getCurrentShard() {
98 | return shardName;
99 | }
100 |
101 | @Override
102 | public String getShardForHost(Host host) {
103 | return null;
104 | }
105 | };
106 | messageKey = redisKeyPrefix + ".MESSAGE." + queueName;
107 |
108 | rq = new RedisQueues(dynoClient, dynoClient, redisKeyPrefix, ss, 1_000, 1_000_000);
109 | DynoQueue rdq1 = rq.get(queueName);
110 | assertNotNull(rdq1);
111 |
112 | rdq = (RedisDynoQueue) rq.get(queueName);
113 | assertNotNull(rdq);
114 |
115 | assertEquals(rdq1, rdq); // should be the same instance.
116 |
117 | }
118 |
119 | @Test
120 | public void testGetName() {
121 | assertEquals(queueName, rdq.getName());
122 | }
123 |
124 | @Test
125 | public void testGetUnackTime() {
126 | assertEquals(1_000, rdq.getUnackTime());
127 | }
128 |
129 | @Test
130 | public void testTimeoutUpdate() {
131 |
132 | rdq.clear();
133 |
134 | String id = UUID.randomUUID().toString();
135 | Message msg = new Message(id, "Hello World-" + id);
136 | msg.setTimeout(100, TimeUnit.MILLISECONDS);
137 | rdq.push(Arrays.asList(msg));
138 |
139 | List popped = rdq.pop(1, 10, TimeUnit.MILLISECONDS);
140 | assertNotNull(popped);
141 | assertEquals(0, popped.size());
142 |
143 | Uninterruptibles.sleepUninterruptibly(500, TimeUnit.MILLISECONDS);
144 |
145 | popped = rdq.pop(1, 1, TimeUnit.SECONDS);
146 | assertNotNull(popped);
147 | assertEquals(1, popped.size());
148 |
149 | boolean updated = rdq.setUnackTimeout(id, 500);
150 | assertTrue(updated);
151 | popped = rdq.pop(1, 1, TimeUnit.SECONDS);
152 | assertNotNull(popped);
153 | assertEquals(0, popped.size());
154 |
155 | Uninterruptibles.sleepUninterruptibly(1000, TimeUnit.MILLISECONDS);
156 | rdq.processUnacks();
157 | popped = rdq.pop(1, 1, TimeUnit.SECONDS);
158 | assertNotNull(popped);
159 | assertEquals(1, popped.size());
160 |
161 | updated = rdq.setUnackTimeout(id, 10_000); //10 seconds!
162 | assertTrue(updated);
163 | rdq.processUnacks();
164 | popped = rdq.pop(1, 1, TimeUnit.SECONDS);
165 | assertNotNull(popped);
166 | assertEquals(0, popped.size());
167 |
168 | updated = rdq.setUnackTimeout(id, 0);
169 | assertTrue(updated);
170 | rdq.processUnacks();
171 | popped = rdq.pop(1, 1, TimeUnit.SECONDS);
172 | assertNotNull(popped);
173 | assertEquals(1, popped.size());
174 |
175 | rdq.ack(id);
176 | Map> size = rdq.shardSizes();
177 | Map values = size.get("1d");
178 | long total = values.values().stream().mapToLong(v -> v).sum();
179 | assertEquals(0, total);
180 |
181 | popped = rdq.pop(1, 1, TimeUnit.SECONDS);
182 | assertNotNull(popped);
183 | assertEquals(0, popped.size());
184 | }
185 |
186 | @Test
187 | public void testConcurrency() throws InterruptedException, ExecutionException {
188 |
189 | rdq.clear();
190 |
191 | final int count = 10_000;
192 | final AtomicInteger published = new AtomicInteger(0);
193 |
194 | ScheduledExecutorService ses = Executors.newScheduledThreadPool(6);
195 | CountDownLatch publishLatch = new CountDownLatch(1);
196 | Runnable publisher = new Runnable() {
197 |
198 | @Override
199 | public void run() {
200 | List messages = new LinkedList<>();
201 | for (int i = 0; i < 10; i++) {
202 | Message msg = new Message(UUID.randomUUID().toString(), "Hello World-" + i);
203 | msg.setPriority(new Random().nextInt(98));
204 | messages.add(msg);
205 | }
206 | if (published.get() >= count) {
207 | publishLatch.countDown();
208 | return;
209 | }
210 |
211 | published.addAndGet(messages.size());
212 | rdq.push(messages);
213 |
214 | }
215 | };
216 |
217 | for (int p = 0; p < 3; p++) {
218 | ses.scheduleWithFixedDelay(publisher, 1, 1, TimeUnit.MILLISECONDS);
219 | }
220 | publishLatch.await();
221 | CountDownLatch latch = new CountDownLatch(count);
222 | List allMsgs = new CopyOnWriteArrayList<>();
223 | AtomicInteger consumed = new AtomicInteger(0);
224 | AtomicInteger counter = new AtomicInteger(0);
225 | Runnable consumer = new Runnable() {
226 |
227 | @Override
228 | public void run() {
229 | if (consumed.get() >= count) {
230 | return;
231 | }
232 | List popped = rdq.pop(100, 1, TimeUnit.MILLISECONDS);
233 | allMsgs.addAll(popped);
234 | consumed.addAndGet(popped.size());
235 | popped.stream().forEach(p -> latch.countDown());
236 | counter.incrementAndGet();
237 | }
238 | };
239 |
240 | for (int c = 0; c < 2; c++) {
241 | ses.scheduleWithFixedDelay(consumer, 1, 10, TimeUnit.MILLISECONDS);
242 | }
243 | Uninterruptibles.awaitUninterruptibly(latch);
244 | System.out.println("Consumed: " + consumed.get() + ", all: " + allMsgs.size() + " counter: " + counter.get());
245 | Set uniqueMessages = allMsgs.stream().collect(Collectors.toSet());
246 |
247 | assertEquals(count, allMsgs.size());
248 | assertEquals(count, uniqueMessages.size());
249 | long start = System.currentTimeMillis();
250 | List more = rdq.pop(1, 1, TimeUnit.SECONDS);
251 | long elapsedTime = System.currentTimeMillis() - start;
252 | assertTrue(elapsedTime >= 1000);
253 | assertEquals(0, more.size());
254 | assertEquals(0, rdq.numIdsToPrefetch.get());
255 |
256 | ses.shutdownNow();
257 | }
258 |
259 | @Test
260 | public void testSetTimeout() {
261 |
262 | rdq.clear();
263 |
264 | Message msg = new Message("x001", "Hello World");
265 | msg.setPriority(3);
266 | msg.setTimeout(20_000);
267 | rdq.push(Arrays.asList(msg));
268 |
269 | List popped = rdq.pop(1, 1, TimeUnit.SECONDS);
270 | assertTrue(popped.isEmpty());
271 |
272 | boolean updated = rdq.setTimeout(msg.getId(), 1);
273 | assertTrue(updated);
274 | popped = rdq.pop(1, 1, TimeUnit.SECONDS);
275 | assertEquals(1, popped.size());
276 | assertEquals(1, popped.get(0).getTimeout());
277 | updated = rdq.setTimeout(msg.getId(), 1);
278 | assertTrue(!updated);
279 | }
280 |
281 | @Test
282 | public void testAll() {
283 |
284 | rdq.clear();
285 |
286 | int count = 10;
287 | List messages = new LinkedList<>();
288 | for (int i = 0; i < count; i++) {
289 | Message msg = new Message("" + i, "Hello World-" + i);
290 | msg.setPriority(count - i);
291 | messages.add(msg);
292 | }
293 | rdq.push(messages);
294 |
295 | messages = rdq.peek(count);
296 |
297 | assertNotNull(messages);
298 | assertEquals(count, messages.size());
299 | long size = rdq.size();
300 | assertEquals(count, size);
301 |
302 | // We did a peek - let's ensure the messages are still around!
303 | List messages2 = rdq.peek(count);
304 | assertNotNull(messages2);
305 | assertEquals(messages, messages2);
306 |
307 | List poped = rdq.pop(count, 1, TimeUnit.SECONDS);
308 | assertNotNull(poped);
309 | assertEquals(count, poped.size());
310 | assertEquals(messages, poped);
311 |
312 | Uninterruptibles.sleepUninterruptibly(2, TimeUnit.SECONDS);
313 | ((RedisDynoQueue) rdq).processUnacks();
314 |
315 | for (Message msg : messages) {
316 | Message found = rdq.get(msg.getId());
317 | assertNotNull(found);
318 | assertEquals(msg.getId(), found.getId());
319 | assertEquals(msg.getTimeout(), found.getTimeout());
320 | }
321 | assertNull(rdq.get("some fake id"));
322 |
323 | List messages3 = rdq.pop(count, 1, TimeUnit.SECONDS);
324 | if (messages3.size() < count) {
325 | List messages4 = rdq.pop(count, 1, TimeUnit.SECONDS);
326 | messages3.addAll(messages4);
327 | }
328 |
329 | assertNotNull(messages3);
330 | assertEquals(10, messages3.size());
331 | assertEquals(messages, messages3);
332 | assertEquals(10, messages3.stream().map(msg -> msg.getId()).collect(Collectors.toSet()).size());
333 | messages3.stream().forEach(System.out::println);
334 | assertTrue(dynoClient.hlen(messageKey) == 10);
335 |
336 | for (Message msg : messages3) {
337 | assertTrue(rdq.ack(msg.getId()));
338 | assertFalse(rdq.ack(msg.getId()));
339 | }
340 | Uninterruptibles.sleepUninterruptibly(2, TimeUnit.SECONDS);
341 | messages3 = rdq.pop(count, 1, TimeUnit.SECONDS);
342 | assertNotNull(messages3);
343 | assertEquals(0, messages3.size());
344 |
345 | int max = 10;
346 | for (Message msg : messages) {
347 | assertEquals(max, msg.getPriority());
348 | rdq.remove(msg.getId());
349 | max--;
350 | }
351 |
352 | size = rdq.size();
353 | assertEquals(0, size);
354 |
355 | assertTrue(dynoClient.hlen(messageKey) == 0);
356 |
357 | }
358 |
359 | @Before
360 | public void clear() {
361 | rdq.clear();
362 | assertTrue(dynoClient.hlen(messageKey) == 0);
363 | }
364 |
365 | @Test
366 | public void testClearQueues() {
367 | rdq.clear();
368 | int count = 10;
369 | List messages = new LinkedList<>();
370 | for (int i = 0; i < count; i++) {
371 | Message msg = new Message("x" + i, "Hello World-" + i);
372 | msg.setPriority(count - i);
373 | messages.add(msg);
374 | }
375 |
376 | rdq.push(messages);
377 | assertEquals(count, rdq.size());
378 | rdq.clear();
379 | assertEquals(0, rdq.size());
380 |
381 | }
382 |
383 | }
384 |
--------------------------------------------------------------------------------
/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/benchmark/BenchmarkTestsDynoJedis.java:
--------------------------------------------------------------------------------
1 | /**
2 | *
3 | */
4 | package com.netflix.dyno.queues.redis.benchmark;
5 |
6 | import com.netflix.dyno.connectionpool.Host;
7 | import com.netflix.dyno.connectionpool.HostBuilder;
8 | import com.netflix.dyno.connectionpool.HostSupplier;
9 | import com.netflix.dyno.connectionpool.TokenMapSupplier;
10 | import com.netflix.dyno.connectionpool.impl.ConnectionPoolConfigurationImpl;
11 | import com.netflix.dyno.connectionpool.impl.lb.HostToken;
12 | import com.netflix.dyno.jedis.DynoJedisClient;
13 | import com.netflix.dyno.queues.redis.v2.QueueBuilder;
14 |
15 | import java.util.*;
16 |
17 | /**
18 | * @author Viren
19 | */
20 | public class BenchmarkTestsDynoJedis extends QueueBenchmark {
21 |
22 | public BenchmarkTestsDynoJedis() {
23 |
24 | List hosts = new ArrayList<>(1);
25 | hosts.add(
26 | new HostBuilder()
27 | .setHostname("localhost")
28 | .setIpAddress("127.0.0.1")
29 | .setPort(6379)
30 | .setRack("us-east-1c")
31 | .setDatacenter("us-east-1")
32 | .setStatus(Host.Status.Up)
33 | .createHost()
34 | );
35 |
36 |
37 | QueueBuilder qb = new QueueBuilder();
38 |
39 | DynoJedisClient.Builder builder = new DynoJedisClient.Builder();
40 | HostSupplier hs = new HostSupplier() {
41 | @Override
42 | public List getHosts() {
43 | return hosts;
44 | }
45 | };
46 |
47 | ConnectionPoolConfigurationImpl cp = new ConnectionPoolConfigurationImpl("test").withTokenSupplier(new TokenMapSupplier() {
48 |
49 | HostToken token = new HostToken(1L, hosts.get(0));
50 |
51 | @Override
52 | public List getTokens(Set activeHosts) {
53 | return Arrays.asList(token);
54 | }
55 |
56 | @Override
57 | public HostToken getTokenForHost(Host host, Set activeHosts) {
58 | return token;
59 | }
60 |
61 |
62 | }).setLocalRack("us-east-1c").setLocalDataCenter("us-east-1");
63 | cp.setSocketTimeout(0);
64 | cp.setConnectTimeout(0);
65 | cp.setMaxConnsPerHost(10);
66 | cp.withHashtag("{}");
67 |
68 | DynoJedisClient client = builder.withApplicationName("test")
69 | .withDynomiteClusterName("test")
70 | .withCPConfig(cp)
71 | .withHostSupplier(hs)
72 | .build();
73 |
74 |
75 | queue = qb
76 | .setCurrentShard("a")
77 | .setQueueName("testq")
78 | .setRedisKeyPrefix("keyprefix")
79 | .setUnackTime(60_000)
80 | .useDynomite(client, client)
81 | .build();
82 | }
83 |
84 |
85 | public static void main(String[] args) throws Exception {
86 | try {
87 |
88 | System.out.println("Start");
89 | BenchmarkTestsDynoJedis tests = new BenchmarkTestsDynoJedis();
90 | tests.run();
91 | } catch (Exception e) {
92 | e.printStackTrace();
93 | } finally {
94 | System.exit(0);
95 | }
96 | }
97 |
98 | }
99 |
--------------------------------------------------------------------------------
/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/benchmark/BenchmarkTestsJedis.java:
--------------------------------------------------------------------------------
1 | /**
2 | *
3 | */
4 | package com.netflix.dyno.queues.redis.benchmark;
5 |
6 | import com.netflix.dyno.connectionpool.Host;
7 | import com.netflix.dyno.connectionpool.HostBuilder;
8 |
9 | import com.netflix.dyno.queues.redis.v2.QueueBuilder;
10 | import redis.clients.jedis.JedisPoolConfig;
11 |
12 | import java.util.LinkedList;
13 | import java.util.List;
14 |
15 | /**
16 | * @author Viren
17 | *
18 | */
19 | public class BenchmarkTestsJedis extends QueueBenchmark {
20 |
21 | public BenchmarkTestsJedis() {
22 | List hosts = new LinkedList<>();
23 | hosts.add(
24 | new HostBuilder()
25 | .setHostname("localhost")
26 | .setPort(6379)
27 | .setRack("us-east-1a")
28 | .createHost()
29 | );
30 |
31 | QueueBuilder qb = new QueueBuilder();
32 |
33 | JedisPoolConfig config = new JedisPoolConfig();
34 | config.setTestOnBorrow(true);
35 | config.setTestOnCreate(true);
36 | config.setMaxTotal(10);
37 | config.setMaxIdle(5);
38 | config.setMaxWaitMillis(60_000);
39 |
40 |
41 | queue = qb
42 | .setCurrentShard("a")
43 | .setQueueName("testq")
44 | .setRedisKeyPrefix("keyprefix")
45 | .setUnackTime(60_000_000)
46 | .useNonDynomiteRedis(config, hosts)
47 | .build();
48 |
49 | System.out.println("Instance: " + queue.getClass().getName());
50 | }
51 |
52 | public static void main(String[] args) throws Exception {
53 | try {
54 |
55 | BenchmarkTestsJedis tests = new BenchmarkTestsJedis();
56 | tests.run();
57 |
58 | } catch (Exception e) {
59 | e.printStackTrace();
60 | } finally {
61 | System.exit(0);
62 | }
63 | }
64 | }
65 |
--------------------------------------------------------------------------------
/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/benchmark/BenchmarkTestsNoPipelines.java:
--------------------------------------------------------------------------------
1 | /**
2 | *
3 | */
4 | package com.netflix.dyno.queues.redis.benchmark;
5 |
6 | import com.netflix.dyno.connectionpool.Host;
7 | import com.netflix.dyno.connectionpool.HostBuilder;
8 | import com.netflix.dyno.connectionpool.HostSupplier;
9 | import com.netflix.dyno.connectionpool.TokenMapSupplier;
10 | import com.netflix.dyno.connectionpool.impl.ConnectionPoolConfigurationImpl;
11 | import com.netflix.dyno.connectionpool.impl.lb.HostToken;
12 | import com.netflix.dyno.jedis.DynoJedisClient;
13 | import com.netflix.dyno.queues.ShardSupplier;
14 | import com.netflix.dyno.queues.redis.RedisQueues;
15 |
16 | import java.util.ArrayList;
17 | import java.util.Arrays;
18 | import java.util.List;
19 | import java.util.Set;
20 | import java.util.stream.Collectors;
21 |
22 | /**
23 | * @author Viren
24 | */
25 | public class BenchmarkTestsNoPipelines extends QueueBenchmark {
26 |
27 |
28 | public BenchmarkTestsNoPipelines() {
29 |
30 | String redisKeyPrefix = "perftestnopipe";
31 | String queueName = "nopipequeue";
32 |
33 | List hosts = new ArrayList<>(1);
34 | hosts.add(
35 | new HostBuilder()
36 | .setHostname("localhost")
37 | .setIpAddress("127.0.0.1")
38 | .setPort(6379)
39 | .setRack("us-east-1c")
40 | .setDatacenter("us-east-1")
41 | .setStatus(Host.Status.Up)
42 | .createHost()
43 | );
44 |
45 | DynoJedisClient.Builder builder = new DynoJedisClient.Builder();
46 | HostSupplier hs = new HostSupplier() {
47 | @Override
48 | public List getHosts() {
49 | return hosts;
50 | }
51 | };
52 |
53 | ConnectionPoolConfigurationImpl cp = new ConnectionPoolConfigurationImpl("test").withTokenSupplier(new TokenMapSupplier() {
54 |
55 | HostToken token = new HostToken(1L, hosts.get(0));
56 |
57 | @Override
58 | public List getTokens(Set activeHosts) {
59 | return Arrays.asList(token);
60 | }
61 |
62 | @Override
63 | public HostToken getTokenForHost(Host host, Set activeHosts) {
64 | return token;
65 | }
66 |
67 |
68 | }).setLocalRack("us-east-1c").setLocalDataCenter("us-east-1");
69 | cp.setSocketTimeout(0);
70 | cp.setConnectTimeout(0);
71 | cp.setMaxConnsPerHost(10);
72 |
73 |
74 | DynoJedisClient client = builder.withApplicationName("test")
75 | .withDynomiteClusterName("test")
76 | .withCPConfig(cp)
77 | .withHostSupplier(hs)
78 | .build();
79 |
80 | Set allShards = hs.getHosts().stream().map(host -> host.getRack().substring(host.getRack().length() - 2)).collect(Collectors.toSet());
81 | String shardName = allShards.iterator().next();
82 | ShardSupplier ss = new ShardSupplier() {
83 |
84 | @Override
85 | public Set getQueueShards() {
86 | return allShards;
87 | }
88 |
89 | @Override
90 | public String getCurrentShard() {
91 | return shardName;
92 | }
93 |
94 | @Override
95 | public String getShardForHost(Host host) {
96 | return null;
97 | }
98 | };
99 |
100 | RedisQueues rq = new RedisQueues(client, client, redisKeyPrefix, ss, 60_000, 1_000_000);
101 | queue = rq.get(queueName);
102 | }
103 |
104 |
105 | public static void main(String[] args) throws Exception {
106 | try {
107 |
108 | BenchmarkTestsNoPipelines tests = new BenchmarkTestsNoPipelines();
109 | tests.run();
110 |
111 | } catch (Exception e) {
112 | e.printStackTrace();
113 | } finally {
114 | System.exit(0);
115 | }
116 | }
117 |
118 | }
119 |
--------------------------------------------------------------------------------
/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/benchmark/QueueBenchmark.java:
--------------------------------------------------------------------------------
1 | package com.netflix.dyno.queues.redis.benchmark;
2 |
3 | import com.netflix.dyno.queues.DynoQueue;
4 | import com.netflix.dyno.queues.Message;
5 |
6 | import java.util.*;
7 | import java.util.concurrent.ExecutorService;
8 | import java.util.concurrent.Executors;
9 | import java.util.concurrent.Future;
10 | import java.util.concurrent.TimeUnit;
11 | import java.util.stream.Collectors;
12 |
13 | public abstract class QueueBenchmark {
14 |
15 | protected DynoQueue queue;
16 |
17 | public void publish() {
18 |
19 | long s = System.currentTimeMillis();
20 | int loopCount = 100;
21 | int batchSize = 3000;
22 | for (int i = 0; i < loopCount; i++) {
23 | List messages = new ArrayList<>(batchSize);
24 | for (int k = 0; k < batchSize; k++) {
25 | String id = UUID.randomUUID().toString();
26 | Message message = new Message(id, getPayload());
27 | messages.add(message);
28 | }
29 | queue.push(messages);
30 | }
31 | long e = System.currentTimeMillis();
32 | long diff = e - s;
33 | long throughput = 1000 * ((loopCount * batchSize) / diff);
34 | System.out.println("Publish time: " + diff + ", throughput: " + throughput + " msg/sec");
35 | }
36 |
37 | public void consume() {
38 | try {
39 | Set ids = new HashSet<>();
40 | long s = System.currentTimeMillis();
41 | int loopCount = 100;
42 | int batchSize = 3500;
43 | int count = 0;
44 | for (int i = 0; i < loopCount; i++) {
45 | List popped = queue.pop(batchSize, 1, TimeUnit.MILLISECONDS);
46 | queue.ack(popped);
47 | Set poppedIds = popped.stream().map(Message::getId).collect(Collectors.toSet());
48 | if (popped.size() != poppedIds.size()) {
49 | //We consumed dups
50 | throw new RuntimeException("Count does not match. expected: " + popped.size() + ", but actual was : " + poppedIds.size() + ", i: " + i);
51 | }
52 | ids.addAll(poppedIds);
53 | count += popped.size();
54 | }
55 | long e = System.currentTimeMillis();
56 | long diff = e - s;
57 | long throughput = 1000 * ((count) / diff);
58 | if (count != ids.size()) {
59 | //We consumed dups
60 | throw new RuntimeException("There were duplicate messages consumed... expected messages to be consumed " + count + ", but actual was : " + ids.size());
61 | }
62 | System.out.println("Consume time: " + diff + ", read throughput: " + throughput + " msg/sec, messages read: " + count);
63 | } catch (Exception e) {
64 | e.printStackTrace();
65 | }
66 | }
67 |
68 | private String getPayload() {
69 | StringBuilder sb = new StringBuilder();
70 | for (int i = 0; i < 1; i++) {
71 | sb.append(UUID.randomUUID().toString());
72 | sb.append(",");
73 | }
74 | return sb.toString();
75 | }
76 |
77 | public void run() throws Exception {
78 |
79 | ExecutorService es = Executors.newFixedThreadPool(2);
80 | List> futures = new LinkedList<>();
81 | for (int i = 0; i < 2; i++) {
82 | Future future = es.submit(() -> {
83 | publish();
84 | consume();
85 | return null;
86 | });
87 | futures.add(future);
88 | }
89 | for (Future future : futures) {
90 | future.get();
91 | }
92 | }
93 | }
94 |
--------------------------------------------------------------------------------
/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/v2/DynoJedisTests.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright 2016 Netflix, Inc.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 | package com.netflix.dyno.queues.redis.v2;
17 |
18 | import com.netflix.dyno.connectionpool.Host;
19 | import com.netflix.dyno.connectionpool.HostBuilder;
20 | import com.netflix.dyno.connectionpool.HostSupplier;
21 | import com.netflix.dyno.connectionpool.TokenMapSupplier;
22 | import com.netflix.dyno.connectionpool.impl.ConnectionPoolConfigurationImpl;
23 | import com.netflix.dyno.connectionpool.impl.lb.HostToken;
24 | import com.netflix.dyno.jedis.DynoJedisClient;
25 | import com.netflix.dyno.queues.DynoQueue;
26 | import com.netflix.dyno.queues.redis.BaseQueueTests;
27 | import redis.clients.jedis.Jedis;
28 |
29 | import java.util.ArrayList;
30 | import java.util.Arrays;
31 | import java.util.List;
32 | import java.util.Set;
33 |
34 | public class DynoJedisTests extends BaseQueueTests {
35 |
36 | private static Jedis dynoClient;
37 |
38 | private static RedisPipelineQueue rdq;
39 |
40 | private static String messageKeyPrefix;
41 |
42 | private static int maxHashBuckets = 32;
43 |
44 | public DynoJedisTests() {
45 | super("dyno_queue_tests");
46 | }
47 |
48 | @Override
49 | public DynoQueue getQueue(String redisKeyPrefix, String queueName) {
50 |
51 | List hosts = new ArrayList<>(1);
52 | hosts.add(
53 | new HostBuilder()
54 | .setHostname("localhost")
55 | .setIpAddress("127.0.0.1")
56 | .setPort(6379)
57 | .setRack("us-east-1a")
58 | .setDatacenter("us-east-1")
59 | .setStatus(Host.Status.Up)
60 | .createHost()
61 | );
62 |
63 |
64 | QueueBuilder qb = new QueueBuilder();
65 |
66 | DynoJedisClient.Builder builder = new DynoJedisClient.Builder();
67 | HostSupplier hs = new HostSupplier() {
68 | @Override
69 | public List getHosts() {
70 | return hosts;
71 | }
72 | };
73 |
74 | ConnectionPoolConfigurationImpl cp = new ConnectionPoolConfigurationImpl("test").withTokenSupplier(new TokenMapSupplier() {
75 |
76 | HostToken token = new HostToken(1L, hosts.get(0));
77 |
78 | @Override
79 | public List getTokens(Set activeHosts) {
80 | return Arrays.asList(token);
81 | }
82 |
83 | @Override
84 | public HostToken getTokenForHost(Host host, Set activeHosts) {
85 | return token;
86 | }
87 |
88 |
89 | }).setLocalRack("us-east-1a").setLocalDataCenter("us-east-1");
90 | cp.setSocketTimeout(0);
91 | cp.setConnectTimeout(0);
92 | cp.setMaxConnsPerHost(10);
93 | cp.withHashtag("{}");
94 |
95 | DynoJedisClient client = builder.withApplicationName("test")
96 | .withDynomiteClusterName("test")
97 | .withCPConfig(cp)
98 | .withHostSupplier(hs)
99 | .build();
100 |
101 | return qb
102 | .setCurrentShard("a")
103 | .setQueueName(queueName)
104 | .setRedisKeyPrefix(redisKeyPrefix)
105 | .setUnackTime(1_000)
106 | .useDynomite(client, client)
107 | .build();
108 | }
109 |
110 |
111 | }
112 |
--------------------------------------------------------------------------------
/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/v2/JedisTests.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright 2016 Netflix, Inc.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 | package com.netflix.dyno.queues.redis.v2;
17 |
18 | import com.netflix.dyno.connectionpool.Host;
19 | import com.netflix.dyno.connectionpool.HostBuilder;
20 | import com.netflix.dyno.queues.DynoQueue;
21 | import com.netflix.dyno.queues.redis.BaseQueueTests;
22 | import redis.clients.jedis.Jedis;
23 | import redis.clients.jedis.JedisPool;
24 | import redis.clients.jedis.JedisPoolConfig;
25 |
26 | import java.util.LinkedList;
27 | import java.util.List;
28 |
29 | /**
30 | *
31 | */
32 | public class JedisTests extends BaseQueueTests {
33 |
34 | private static Jedis dynoClient;
35 |
36 |
37 | private static RedisPipelineQueue rdq;
38 |
39 | private static String messageKeyPrefix;
40 |
41 | private static int maxHashBuckets = 32;
42 |
43 | public JedisTests() {
44 | super("jedis_queue_tests");
45 | }
46 |
47 | @Override
48 | public DynoQueue getQueue(String redisKeyPrefix, String queueName) {
49 | JedisPoolConfig config = new JedisPoolConfig();
50 | config.setTestOnBorrow(true);
51 | config.setTestOnCreate(true);
52 | config.setMaxTotal(10);
53 | config.setMaxIdle(5);
54 | config.setMaxWaitMillis(60_000);
55 | JedisPool pool = new JedisPool(config, "localhost", 6379);
56 | dynoClient = new Jedis("localhost", 6379, 0, 0);
57 | dynoClient.flushAll();
58 |
59 | List hosts = new LinkedList<>();
60 | hosts.add(
61 | new HostBuilder()
62 | .setHostname("localhost")
63 | .setPort(6379)
64 | .setRack("us-east-1a")
65 | .createHost()
66 | );
67 |
68 | QueueBuilder qb = new QueueBuilder();
69 | DynoQueue queue = qb
70 | .setCurrentShard("a")
71 | .setQueueName(queueName)
72 | .setRedisKeyPrefix(redisKeyPrefix)
73 | .setUnackTime(1_000)
74 | .useNonDynomiteRedis(config, hosts)
75 | .build();
76 |
77 | queue.clear();
78 |
79 | return queue;
80 |
81 | }
82 |
83 |
84 | }
85 |
--------------------------------------------------------------------------------
/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/v2/MultiQueueTests.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright 2016 Netflix, Inc.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 | package com.netflix.dyno.queues.redis.v2;
17 |
18 | import com.netflix.dyno.connectionpool.Host;
19 | import com.netflix.dyno.connectionpool.HostBuilder;
20 | import com.netflix.dyno.queues.DynoQueue;
21 | import com.netflix.dyno.queues.Message;
22 | import org.junit.Test;
23 | import redis.clients.jedis.Jedis;
24 | import redis.clients.jedis.JedisPool;
25 | import redis.clients.jedis.JedisPoolConfig;
26 |
27 | import java.util.LinkedList;
28 | import java.util.List;
29 | import java.util.Map;
30 | import java.util.concurrent.TimeUnit;
31 |
32 | import static org.junit.Assert.assertEquals;
33 | import static org.junit.Assert.assertNotNull;
34 | import static org.junit.Assert.assertTrue;
35 |
36 | /**
37 | *
38 | */
39 | public class MultiQueueTests {
40 |
41 | private static Jedis dynoClient;
42 |
43 |
44 | private static RedisPipelineQueue rdq;
45 |
46 | private static String messageKeyPrefix;
47 |
48 | private static int maxHashBuckets = 32;
49 |
50 | public DynoQueue getQueue(String redisKeyPrefix, String queueName) {
51 | JedisPoolConfig config = new JedisPoolConfig();
52 | config.setTestOnBorrow(true);
53 | config.setTestOnCreate(true);
54 | config.setMaxTotal(10);
55 | config.setMaxIdle(5);
56 | config.setMaxWaitMillis(60_000);
57 | JedisPool pool = new JedisPool(config, "localhost", 6379);
58 | dynoClient = new Jedis("localhost", 6379, 0, 0);
59 | dynoClient.flushAll();
60 |
61 | List hosts = new LinkedList<>();
62 | hosts.add(
63 | new HostBuilder()
64 | .setHostname("localhost")
65 | .setPort(6379)
66 | .setRack("us-east-1a")
67 | .createHost()
68 | );
69 | hosts.add(
70 | new HostBuilder()
71 | .setHostname("localhost")
72 | .setPort(6379)
73 | .setRack("us-east-2b")
74 | .createHost()
75 | );
76 |
77 | QueueBuilder qb = new QueueBuilder();
78 | DynoQueue queue = qb
79 | .setCurrentShard("a")
80 | .setQueueName(queueName)
81 | .setRedisKeyPrefix(redisKeyPrefix)
82 | .setUnackTime(50_000)
83 | .useNonDynomiteRedis(config, hosts)
84 | .build();
85 |
86 | queue.clear(); //clear the queue
87 |
88 | return queue;
89 |
90 | }
91 |
92 | @Test
93 | public void testAll() {
94 | DynoQueue queue = getQueue("test", "multi_queue");
95 | assertEquals(MultiRedisQueue.class, queue.getClass());
96 |
97 | long start = System.currentTimeMillis();
98 | List popped = queue.pop(1, 1, TimeUnit.SECONDS);
99 | assertTrue(popped.isEmpty()); //we have not pushed anything!!!!
100 | long elapsedTime = System.currentTimeMillis() - start;
101 | System.out.println("elapsed Time " + elapsedTime);
102 | assertTrue(elapsedTime > 1000);
103 |
104 | List messages = new LinkedList<>();
105 | for (int i = 0; i < 10; i++) {
106 | Message msg = new Message();
107 | msg.setId("" + i);
108 | msg.setPayload("" + i);
109 | messages.add(msg);
110 | }
111 | queue.push(messages);
112 |
113 | assertEquals(10, queue.size());
114 | Map> shards = queue.shardSizes();
115 | assertEquals(2, shards.keySet().size()); //a and b
116 |
117 | Map shardA = shards.get("a");
118 | Map shardB = shards.get("b");
119 |
120 | assertNotNull(shardA);
121 | assertNotNull(shardB);
122 |
123 | Long sizeA = shardA.get("size");
124 | Long sizeB = shardB.get("size");
125 |
126 | assertNotNull(sizeA);
127 | assertNotNull(sizeB);
128 |
129 | assertEquals(5L, sizeA.longValue());
130 | assertEquals(5L, sizeB.longValue());
131 |
132 | start = System.currentTimeMillis();
133 | popped = queue.pop(2, 1, TimeUnit.SECONDS);
134 | elapsedTime = System.currentTimeMillis() - start;
135 | assertEquals(2, popped.size());
136 | System.out.println("elapsed Time " + elapsedTime);
137 | assertTrue(elapsedTime < 1000);
138 |
139 |
140 | start = System.currentTimeMillis();
141 | popped = queue.pop(5, 5, TimeUnit.SECONDS);
142 | elapsedTime = System.currentTimeMillis() - start;
143 | assertEquals(3, popped.size()); //3 remaining in the current shard
144 | System.out.println("elapsed Time " + elapsedTime);
145 | assertTrue(elapsedTime > 5000); //we would have waited for at least 5 second for the last 2 elements!
146 |
147 | }
148 |
149 |
150 | }
151 |
--------------------------------------------------------------------------------
/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/v2/RedisDynoQueueTest.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright 2016 Netflix, Inc.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 | package com.netflix.dyno.queues.redis.v2;
17 |
18 | import static org.junit.Assert.assertEquals;
19 | import static org.junit.Assert.assertFalse;
20 | import static org.junit.Assert.assertNotNull;
21 | import static org.junit.Assert.assertNull;
22 | import static org.junit.Assert.assertTrue;
23 |
24 | import java.util.Arrays;
25 | import java.util.LinkedList;
26 | import java.util.List;
27 | import java.util.Map;
28 | import java.util.Random;
29 | import java.util.Set;
30 | import java.util.UUID;
31 | import java.util.concurrent.CopyOnWriteArrayList;
32 | import java.util.concurrent.CountDownLatch;
33 | import java.util.concurrent.ExecutionException;
34 | import java.util.concurrent.Executors;
35 | import java.util.concurrent.ScheduledExecutorService;
36 | import java.util.concurrent.TimeUnit;
37 | import java.util.concurrent.atomic.AtomicInteger;
38 | import java.util.stream.Collectors;
39 |
40 | import com.netflix.dyno.queues.redis.v2.RedisPipelineQueue;
41 | import org.junit.Before;
42 | import org.junit.BeforeClass;
43 | import org.junit.Test;
44 |
45 | import com.google.common.util.concurrent.Uninterruptibles;
46 | import com.netflix.dyno.queues.Message;
47 | import com.netflix.dyno.queues.redis.conn.JedisProxy;
48 |
49 | import redis.clients.jedis.Jedis;
50 | import redis.clients.jedis.JedisPool;
51 | import redis.clients.jedis.JedisPoolConfig;
52 |
53 | public class RedisDynoQueueTest {
54 |
55 | private static Jedis dynoClient;
56 |
57 | private static final String queueName = "test_queue";
58 |
59 | private static final String redisKeyPrefix = "testdynoqueues";
60 |
61 | private static RedisPipelineQueue rdq;
62 |
63 | private static String messageKeyPrefix;
64 |
65 | private static int maxHashBuckets = 32;
66 |
67 | @BeforeClass
68 | public static void setUpBeforeClass() throws Exception {
69 |
70 |
71 | JedisPoolConfig config = new JedisPoolConfig();
72 | config.setTestOnBorrow(true);
73 | config.setTestOnCreate(true);
74 | config.setMaxTotal(10);
75 | config.setMaxIdle(5);
76 | config.setMaxWaitMillis(60_000);
77 | JedisPool pool = new JedisPool(config, "localhost", 6379);
78 | dynoClient = new Jedis("localhost", 6379, 0, 0);
79 | dynoClient.flushAll();
80 |
81 | rdq = new RedisPipelineQueue(redisKeyPrefix, queueName, "x", 1_000, 1_000, new JedisProxy(pool));
82 | messageKeyPrefix = redisKeyPrefix + ".MSG." + "{" + queueName + ".x}";
83 | }
84 |
85 | @Test
86 | public void testGetName() {
87 | assertEquals(queueName, rdq.getName());
88 | }
89 |
90 | @Test
91 | public void testGetUnackTime() {
92 | assertEquals(1_000, rdq.getUnackTime());
93 | }
94 |
95 | @Test
96 | public void testTimeoutUpdate() {
97 |
98 | rdq.clear();
99 |
100 | String id = UUID.randomUUID().toString();
101 | Message msg = new Message(id, "Hello World-" + id);
102 | msg.setTimeout(100, TimeUnit.MILLISECONDS);
103 | rdq.push(Arrays.asList(msg));
104 |
105 | List popped = rdq.pop(1, 10, TimeUnit.MILLISECONDS);
106 | assertNotNull(popped);
107 | assertEquals(0, popped.size());
108 |
109 | Uninterruptibles.sleepUninterruptibly(500, TimeUnit.MILLISECONDS);
110 |
111 | popped = rdq.pop(1, 1, TimeUnit.SECONDS);
112 | assertNotNull(popped);
113 | assertEquals(1, popped.size());
114 |
115 | boolean updated = rdq.setUnackTimeout(id, 500);
116 | assertTrue(updated);
117 | popped = rdq.pop(1, 1, TimeUnit.SECONDS);
118 | assertNotNull(popped);
119 | assertEquals(0, popped.size());
120 |
121 | Uninterruptibles.sleepUninterruptibly(1000, TimeUnit.MILLISECONDS);
122 | rdq.processUnacks();
123 | popped = rdq.pop(1, 1, TimeUnit.SECONDS);
124 | assertNotNull(popped);
125 | assertEquals(1, popped.size());
126 |
127 | updated = rdq.setUnackTimeout(id, 10_000); //10 seconds!
128 | assertTrue(updated);
129 | rdq.processUnacks();
130 | popped = rdq.pop(1, 1, TimeUnit.SECONDS);
131 | assertNotNull(popped);
132 | assertEquals(0, popped.size());
133 |
134 | updated = rdq.setUnackTimeout(id, 0);
135 | assertTrue(updated);
136 | rdq.processUnacks();
137 | popped = rdq.pop(1, 1, TimeUnit.SECONDS);
138 | assertNotNull(popped);
139 | assertEquals(1, popped.size());
140 |
141 | rdq.ack(id);
142 | Map> size = rdq.shardSizes();
143 | Map values = size.get("x");
144 | long total = values.values().stream().mapToLong(v -> v).sum();
145 | assertEquals(0, total);
146 |
147 | popped = rdq.pop(1, 1, TimeUnit.SECONDS);
148 | assertNotNull(popped);
149 | assertEquals(0, popped.size());
150 | }
151 |
152 | @Test
153 | public void testConcurrency() throws InterruptedException, ExecutionException {
154 |
155 | rdq.clear();
156 |
157 | final int count = 100;
158 | final AtomicInteger published = new AtomicInteger(0);
159 |
160 | ScheduledExecutorService ses = Executors.newScheduledThreadPool(6);
161 | CountDownLatch publishLatch = new CountDownLatch(1);
162 | Runnable publisher = new Runnable() {
163 |
164 | @Override
165 | public void run() {
166 | List messages = new LinkedList<>();
167 | for (int i = 0; i < 10; i++) {
168 | Message msg = new Message(UUID.randomUUID().toString(), "Hello World-" + i);
169 | msg.setPriority(new Random().nextInt(98));
170 | messages.add(msg);
171 | }
172 | if (published.get() >= count) {
173 | publishLatch.countDown();
174 | return;
175 | }
176 |
177 | published.addAndGet(messages.size());
178 | rdq.push(messages);
179 | }
180 | };
181 |
182 | for (int p = 0; p < 3; p++) {
183 | ses.scheduleWithFixedDelay(publisher, 1, 1, TimeUnit.MILLISECONDS);
184 | }
185 | publishLatch.await();
186 | CountDownLatch latch = new CountDownLatch(count);
187 | List allMsgs = new CopyOnWriteArrayList<>();
188 | AtomicInteger consumed = new AtomicInteger(0);
189 | AtomicInteger counter = new AtomicInteger(0);
190 | Runnable consumer = new Runnable() {
191 |
192 | @Override
193 | public void run() {
194 | if (consumed.get() >= count) {
195 | return;
196 | }
197 | List popped = rdq.pop(100, 1, TimeUnit.MILLISECONDS);
198 | allMsgs.addAll(popped);
199 | consumed.addAndGet(popped.size());
200 | popped.stream().forEach(p -> latch.countDown());
201 | counter.incrementAndGet();
202 | }
203 | };
204 | for (int c = 0; c < 2; c++) {
205 | ses.scheduleWithFixedDelay(consumer, 1, 10, TimeUnit.MILLISECONDS);
206 | }
207 | Uninterruptibles.awaitUninterruptibly(latch);
208 | System.out.println("Consumed: " + consumed.get() + ", all: " + allMsgs.size() + " counter: " + counter.get());
209 | Set uniqueMessages = allMsgs.stream().collect(Collectors.toSet());
210 |
211 | assertEquals(count, allMsgs.size());
212 | assertEquals(count, uniqueMessages.size());
213 | List more = rdq.pop(1, 1, TimeUnit.SECONDS);
214 | assertEquals(0, more.size());
215 |
216 | ses.shutdownNow();
217 | }
218 |
219 | @Test
220 | public void testSetTimeout() {
221 |
222 | rdq.clear();
223 |
224 | Message msg = new Message("x001yx", "Hello World");
225 | msg.setPriority(3);
226 | msg.setTimeout(10_000);
227 | rdq.push(Arrays.asList(msg));
228 |
229 | List popped = rdq.pop(1, 1, TimeUnit.SECONDS);
230 | assertTrue(popped.isEmpty());
231 |
232 | boolean updated = rdq.setTimeout(msg.getId(), 0);
233 | assertTrue(updated);
234 | popped = rdq.pop(2, 1, TimeUnit.SECONDS);
235 | assertEquals(1, popped.size());
236 | assertEquals(0, popped.get(0).getTimeout());
237 | }
238 |
239 | @Test
240 | public void testAll() {
241 |
242 | rdq.clear();
243 | assertEquals(0, rdq.size());
244 |
245 | int count = 10;
246 | List messages = new LinkedList<>();
247 | for (int i = 0; i < count; i++) {
248 | Message msg = new Message("" + i, "Hello World-" + i);
249 | msg.setPriority(count - i);
250 | messages.add(msg);
251 | }
252 | rdq.push(messages);
253 |
254 | messages = rdq.peek(count);
255 |
256 | assertNotNull(messages);
257 | assertEquals(count, messages.size());
258 | long size = rdq.size();
259 | assertEquals(count, size);
260 |
261 | // We did a peek - let's ensure the messages are still around!
262 | List messages2 = rdq.peek(count);
263 | assertNotNull(messages2);
264 | assertEquals(messages, messages2);
265 |
266 | List poped = rdq.pop(count, 1, TimeUnit.SECONDS);
267 | assertNotNull(poped);
268 | assertEquals(count, poped.size());
269 | assertEquals(messages, poped);
270 |
271 | Uninterruptibles.sleepUninterruptibly(2, TimeUnit.SECONDS);
272 | rdq.processUnacks();
273 |
274 | for (Message msg : messages) {
275 | Message found = rdq.get(msg.getId());
276 | assertNotNull(found);
277 | assertEquals(msg.getId(), found.getId());
278 | assertEquals(msg.getTimeout(), found.getTimeout());
279 | }
280 | assertNull(rdq.get("some fake id"));
281 |
282 | List messages3 = rdq.pop(count, 1, TimeUnit.SECONDS);
283 | if (messages3.size() < count) {
284 | List messages4 = rdq.pop(count, 1, TimeUnit.SECONDS);
285 | messages3.addAll(messages4);
286 | }
287 |
288 | assertNotNull(messages3);
289 | assertEquals(10, messages3.size());
290 | assertEquals(messages.stream().map(msg -> msg.getId()).sorted().collect(Collectors.toList()), messages3.stream().map(msg -> msg.getId()).sorted().collect(Collectors.toList()));
291 | assertEquals(10, messages3.stream().map(msg -> msg.getId()).collect(Collectors.toSet()).size());
292 | messages3.stream().forEach(System.out::println);
293 | int bucketCounts = 0;
294 | for (int i = 0; i < maxHashBuckets; i++) {
295 | bucketCounts += dynoClient.hlen(messageKeyPrefix + "." + i);
296 | }
297 | assertEquals(10, bucketCounts);
298 |
299 | for (Message msg : messages3) {
300 | assertTrue(rdq.ack(msg.getId()));
301 | assertFalse(rdq.ack(msg.getId()));
302 | }
303 | Uninterruptibles.sleepUninterruptibly(2, TimeUnit.SECONDS);
304 | messages3 = rdq.pop(count, 1, TimeUnit.SECONDS);
305 | assertNotNull(messages3);
306 | assertEquals(0, messages3.size());
307 | }
308 |
309 | @Before
310 | public void clear() {
311 | rdq.clear();
312 | int bucketCounts = 0;
313 | for (int i = 0; i < maxHashBuckets; i++) {
314 | bucketCounts += dynoClient.hlen(messageKeyPrefix + "." + i);
315 | }
316 | assertEquals(0, bucketCounts);
317 | }
318 |
319 | @Test
320 | public void testClearQueues() {
321 | rdq.clear();
322 | int count = 10;
323 | List messages = new LinkedList<>();
324 | for (int i = 0; i < count; i++) {
325 | Message msg = new Message("x" + i, "Hello World-" + i);
326 | msg.setPriority(count - i);
327 | messages.add(msg);
328 | }
329 |
330 | rdq.push(messages);
331 | assertEquals(count, rdq.size());
332 | rdq.clear();
333 | assertEquals(0, rdq.size());
334 |
335 | }
336 |
337 | }
338 |
--------------------------------------------------------------------------------
/gradle/wrapper/gradle-wrapper.jar:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Netflix/dyno-queues/6104363527d0c6801101b8516306212e68d3517d/gradle/wrapper/gradle-wrapper.jar
--------------------------------------------------------------------------------
/gradle/wrapper/gradle-wrapper.properties:
--------------------------------------------------------------------------------
1 | #Wed Apr 24 09:46:53 PDT 2019
2 | distributionBase=GRADLE_USER_HOME
3 | distributionPath=wrapper/dists
4 | zipStoreBase=GRADLE_USER_HOME
5 | zipStorePath=wrapper/dists
6 | distributionUrl=https\://services.gradle.org/distributions/gradle-8.11-all.zip
7 |
--------------------------------------------------------------------------------
/gradlew:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env sh
2 |
3 | ##############################################################################
4 | ##
5 | ## Gradle start up script for UN*X
6 | ##
7 | ##############################################################################
8 |
9 | # Attempt to set APP_HOME
10 | # Resolve links: $0 may be a link
11 | PRG="$0"
12 | # Need this for relative symlinks.
13 | while [ -h "$PRG" ] ; do
14 | ls=`ls -ld "$PRG"`
15 | link=`expr "$ls" : '.*-> \(.*\)$'`
16 | if expr "$link" : '/.*' > /dev/null; then
17 | PRG="$link"
18 | else
19 | PRG=`dirname "$PRG"`"/$link"
20 | fi
21 | done
22 | SAVED="`pwd`"
23 | cd "`dirname \"$PRG\"`/" >/dev/null
24 | APP_HOME="`pwd -P`"
25 | cd "$SAVED" >/dev/null
26 |
27 | APP_NAME="Gradle"
28 | APP_BASE_NAME=`basename "$0"`
29 |
30 | # Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
31 | DEFAULT_JVM_OPTS=""
32 |
33 | # Use the maximum available, or set MAX_FD != -1 to use that value.
34 | MAX_FD="maximum"
35 |
36 | warn () {
37 | echo "$*"
38 | }
39 |
40 | die () {
41 | echo
42 | echo "$*"
43 | echo
44 | exit 1
45 | }
46 |
47 | # OS specific support (must be 'true' or 'false').
48 | cygwin=false
49 | msys=false
50 | darwin=false
51 | nonstop=false
52 | case "`uname`" in
53 | CYGWIN* )
54 | cygwin=true
55 | ;;
56 | Darwin* )
57 | darwin=true
58 | ;;
59 | MINGW* )
60 | msys=true
61 | ;;
62 | NONSTOP* )
63 | nonstop=true
64 | ;;
65 | esac
66 |
67 | CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar
68 |
69 | # Determine the Java command to use to start the JVM.
70 | if [ -n "$JAVA_HOME" ] ; then
71 | if [ -x "$JAVA_HOME/jre/sh/java" ] ; then
72 | # IBM's JDK on AIX uses strange locations for the executables
73 | JAVACMD="$JAVA_HOME/jre/sh/java"
74 | else
75 | JAVACMD="$JAVA_HOME/bin/java"
76 | fi
77 | if [ ! -x "$JAVACMD" ] ; then
78 | die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME
79 |
80 | Please set the JAVA_HOME variable in your environment to match the
81 | location of your Java installation."
82 | fi
83 | else
84 | JAVACMD="java"
85 | which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
86 |
87 | Please set the JAVA_HOME variable in your environment to match the
88 | location of your Java installation."
89 | fi
90 |
91 | # Increase the maximum file descriptors if we can.
92 | if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then
93 | MAX_FD_LIMIT=`ulimit -H -n`
94 | if [ $? -eq 0 ] ; then
95 | if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then
96 | MAX_FD="$MAX_FD_LIMIT"
97 | fi
98 | ulimit -n $MAX_FD
99 | if [ $? -ne 0 ] ; then
100 | warn "Could not set maximum file descriptor limit: $MAX_FD"
101 | fi
102 | else
103 | warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT"
104 | fi
105 | fi
106 |
107 | # For Darwin, add options to specify how the application appears in the dock
108 | if $darwin; then
109 | GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\""
110 | fi
111 |
112 | # For Cygwin, switch paths to Windows format before running java
113 | if $cygwin ; then
114 | APP_HOME=`cygpath --path --mixed "$APP_HOME"`
115 | CLASSPATH=`cygpath --path --mixed "$CLASSPATH"`
116 | JAVACMD=`cygpath --unix "$JAVACMD"`
117 |
118 | # We build the pattern for arguments to be converted via cygpath
119 | ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null`
120 | SEP=""
121 | for dir in $ROOTDIRSRAW ; do
122 | ROOTDIRS="$ROOTDIRS$SEP$dir"
123 | SEP="|"
124 | done
125 | OURCYGPATTERN="(^($ROOTDIRS))"
126 | # Add a user-defined pattern to the cygpath arguments
127 | if [ "$GRADLE_CYGPATTERN" != "" ] ; then
128 | OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)"
129 | fi
130 | # Now convert the arguments - kludge to limit ourselves to /bin/sh
131 | i=0
132 | for arg in "$@" ; do
133 | CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -`
134 | CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option
135 |
136 | if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition
137 | eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"`
138 | else
139 | eval `echo args$i`="\"$arg\""
140 | fi
141 | i=$((i+1))
142 | done
143 | case $i in
144 | (0) set -- ;;
145 | (1) set -- "$args0" ;;
146 | (2) set -- "$args0" "$args1" ;;
147 | (3) set -- "$args0" "$args1" "$args2" ;;
148 | (4) set -- "$args0" "$args1" "$args2" "$args3" ;;
149 | (5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;;
150 | (6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;;
151 | (7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;;
152 | (8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;;
153 | (9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;;
154 | esac
155 | fi
156 |
157 | # Escape application args
158 | save () {
159 | for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done
160 | echo " "
161 | }
162 | APP_ARGS=$(save "$@")
163 |
164 | # Collect all arguments for the java command, following the shell quoting and substitution rules
165 | eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS"
166 |
167 | # by default we should be in the correct project dir, but when run from Finder on Mac, the cwd is wrong
168 | if [ "$(uname)" = "Darwin" ] && [ "$HOME" = "$PWD" ]; then
169 | cd "$(dirname "$0")"
170 | fi
171 |
172 | exec "$JAVACMD" "$@"
173 |
--------------------------------------------------------------------------------
/gradlew.bat:
--------------------------------------------------------------------------------
1 | @if "%DEBUG%" == "" @echo off
2 | @rem ##########################################################################
3 | @rem
4 | @rem Gradle startup script for Windows
5 | @rem
6 | @rem ##########################################################################
7 |
8 | @rem Set local scope for the variables with windows NT shell
9 | if "%OS%"=="Windows_NT" setlocal
10 |
11 | set DIRNAME=%~dp0
12 | if "%DIRNAME%" == "" set DIRNAME=.
13 | set APP_BASE_NAME=%~n0
14 | set APP_HOME=%DIRNAME%
15 |
16 | @rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
17 | set DEFAULT_JVM_OPTS=
18 |
19 | @rem Find java.exe
20 | if defined JAVA_HOME goto findJavaFromJavaHome
21 |
22 | set JAVA_EXE=java.exe
23 | %JAVA_EXE% -version >NUL 2>&1
24 | if "%ERRORLEVEL%" == "0" goto init
25 |
26 | echo.
27 | echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
28 | echo.
29 | echo Please set the JAVA_HOME variable in your environment to match the
30 | echo location of your Java installation.
31 |
32 | goto fail
33 |
34 | :findJavaFromJavaHome
35 | set JAVA_HOME=%JAVA_HOME:"=%
36 | set JAVA_EXE=%JAVA_HOME%/bin/java.exe
37 |
38 | if exist "%JAVA_EXE%" goto init
39 |
40 | echo.
41 | echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME%
42 | echo.
43 | echo Please set the JAVA_HOME variable in your environment to match the
44 | echo location of your Java installation.
45 |
46 | goto fail
47 |
48 | :init
49 | @rem Get command-line arguments, handling Windows variants
50 |
51 | if not "%OS%" == "Windows_NT" goto win9xME_args
52 |
53 | :win9xME_args
54 | @rem Slurp the command line arguments.
55 | set CMD_LINE_ARGS=
56 | set _SKIP=2
57 |
58 | :win9xME_args_slurp
59 | if "x%~1" == "x" goto execute
60 |
61 | set CMD_LINE_ARGS=%*
62 |
63 | :execute
64 | @rem Setup the command line
65 |
66 | set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar
67 |
68 | @rem Execute Gradle
69 | "%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS%
70 |
71 | :end
72 | @rem End local scope for the variables with windows NT shell
73 | if "%ERRORLEVEL%"=="0" goto mainEnd
74 |
75 | :fail
76 | rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of
77 | rem the _cmd.exe /c_ return code!
78 | if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1
79 | exit /b 1
80 |
81 | :mainEnd
82 | if "%OS%"=="Windows_NT" endlocal
83 |
84 | :omega
85 |
--------------------------------------------------------------------------------
/settings.gradle:
--------------------------------------------------------------------------------
1 | rootProject.name='dyno-queues'
2 | include 'dyno-queues-core', 'dyno-queues-redis'
3 |
--------------------------------------------------------------------------------