\n"
28 | else
29 | thread.backtrace.each do |line|
30 | file.write "#{line} \n"
31 | end
32 | end
33 | end
34 | end
35 | end
36 |
37 | begin
38 | server.start
39 | server.join
40 | rescue => ex
41 | Lowkiq.last_words.call ex
42 | raise ex
43 | end
44 |
--------------------------------------------------------------------------------
/spec/schedulers/lag_spec.rb:
--------------------------------------------------------------------------------
1 | RSpec.describe Lowkiq::Schedulers::Lag do
2 | let(:shard_handler) { double("shard_handler") }
3 | let(:metrics) { double("metrics") }
4 | let(:wait) { double("wait") }
5 |
6 | subject { described_class.new wait, metrics }
7 |
8 | it 'process' do
9 | expect(metrics).to receive(:call).and_return([OpenStruct.new(lag: 1.0)])
10 | expect(shard_handler).to receive(:queue_name).and_return("name")
11 | expect(shard_handler).to receive(:shard_index).and_return(0)
12 | expect(shard_handler).to receive(:process).and_return(true)
13 | job = subject.build_job [shard_handler]
14 | job.call
15 | end
16 |
17 | it 'wait' do
18 | expect(metrics).to receive(:call).and_return([OpenStruct.new(lag: 0.0)])
19 | expect(wait).to receive(:call).exactly(1)
20 | expect(shard_handler).to receive(:queue_name).and_return("name")
21 | expect(shard_handler).to receive(:shard_index).and_return(0)
22 |
23 | job = subject.build_job [shard_handler]
24 | job.call
25 | end
26 | end
27 |
--------------------------------------------------------------------------------
/examples/benchmark/lowkiq.rb:
--------------------------------------------------------------------------------
1 | # require 'bundler/setup'
2 | # Bundler.require(:default)
3 |
4 | $jobs_count = 100_000
5 |
6 | Lowkiq.build_scheduler = ->() { Lowkiq.build_seq_scheduler }
7 | Lowkiq.redis = ->() { Redis.new url: ENV.fetch('REDIS_URL'), driver: :hiredis }
8 |
9 | module Worker
10 | extend Lowkiq::Worker
11 |
12 | # self.shards_count = 50
13 |
14 | def self.perform(payloads_by_id)
15 | end
16 | end
17 |
18 | Lowkiq.server_redis_pool.with do |redis|
19 | redis.flushdb
20 | end
21 |
22 | jobs = (0...$jobs_count).map do |i|
23 | { id: i, payload: i.to_s }
24 | end
25 | jobs.each_slice(10_000) do |batch|
26 | Worker.perform_async batch
27 | end
28 |
29 | puts "jobs are enqueued"
30 |
31 | start = Time.now.to_f
32 |
33 | Monitoring = Thread.new do
34 | metrics = Lowkiq::Queue::QueueMetrics.new Lowkiq.client_redis_pool
35 |
36 | loop do
37 | len = metrics.call([Worker.queue_name]).first.length
38 | puts len
39 |
40 | if len == 0
41 | total = Time.now.to_f - start
42 |
43 | puts "total time: #{total}"
44 |
45 | exit 0
46 | end
47 |
48 | sleep 1
49 | end
50 | end
51 |
--------------------------------------------------------------------------------
/frontend/src/dumb/dashboard/total.js:
--------------------------------------------------------------------------------
1 | import React from 'react';
2 | import formatNumber from '../util/format-number';
3 | import formatDuration from '../util/format-duration';
4 |
5 | function Card({title, type, children}) {
6 | return (
7 |
8 |
9 |
{title}
10 |
11 | {children}
12 |
13 |
14 |
21 | {formatDuration(lag)}
22 | {formatNumber(busy)}
23 | {formatNumber(enqueued)}
24 | {formatNumber(fresh)}
25 | {formatNumber(retries)}
26 | {formatNumber(dead)}
27 |
28 | );
29 | }
30 |
--------------------------------------------------------------------------------
/lib/lowkiq/worker.rb:
--------------------------------------------------------------------------------
1 | module Lowkiq
2 | module Worker
3 | attr_accessor :shards_count,
4 | :batch_size,
5 | :max_retry_count,
6 | :queue_name
7 |
8 | def self.extended(mod)
9 | super
10 | mod.shards_count = 5
11 | mod.batch_size = 1
12 | mod.max_retry_count = 25
13 | mod.queue_name = mod.name
14 | end
15 |
16 | # i.e. 15, 16, 31, 96, 271, ... seconds + a random amount of time
17 | def retry_in(retry_count)
18 | (retry_count ** 4) + 15 + (rand(30) * (retry_count + 1))
19 | end
20 |
21 | def retries_exhausted(batch)
22 | # no-op
23 | end
24 |
25 | def perform(payload)
26 | fail "not implemented"
27 | end
28 |
29 | def client_queue
30 | Queue::Queue.new Lowkiq.client_redis_pool, self.queue_name, self.shards_count
31 | end
32 |
33 | def client_queries
34 | Queue::Queries.new Lowkiq.client_redis_pool, self.queue_name
35 | end
36 |
37 | def client_actions
38 | Queue::Actions.new client_queue, client_queries
39 | end
40 |
41 | def perform_async(batch)
42 | Lowkiq.client_wrapper.call(self, batch) {client_queue.push batch}
43 | end
44 |
45 | end
46 | end
47 |
--------------------------------------------------------------------------------
/spec/schedulers/seq_spec.rb:
--------------------------------------------------------------------------------
1 | RSpec.describe Lowkiq::Schedulers::Seq do
2 | let(:shard_handler_0) { double("shard_handler_0") }
3 | let(:shard_handler_1) { double("shard_handler_1") }
4 | let(:shard_handler_2) { double("shard_handler_2") }
5 |
6 | let(:shard_handlers) { [shard_handler_0, shard_handler_1, shard_handler_2] }
7 |
8 | let(:wait) { double("wait") }
9 | let(:threads_count) { 1 }
10 |
11 | let(:subject) { described_class.new wait }
12 | let(:job) { subject.build_job shard_handlers }
13 |
14 | it 'process' do
15 | 2.times do
16 | expect(shard_handler_0).to receive(:process).and_return(false)
17 | job.call
18 |
19 | expect(shard_handler_1).to receive(:process).and_return(true)
20 | job.call
21 |
22 | expect(shard_handler_2).to receive(:process).and_return(true)
23 | job.call
24 | end
25 | end
26 |
27 | it 'wait' do
28 | expect(shard_handler_0).to receive(:process).and_return(false)
29 | job.call
30 |
31 | expect(shard_handler_1).to receive(:process).and_return(false)
32 | job.call
33 |
34 | expect(shard_handler_2).to receive(:process).and_return(false)
35 | job.call
36 |
37 | expect(wait).to receive(:call)
38 | job.call
39 | end
40 | end
41 |
--------------------------------------------------------------------------------
/frontend/src/dumb/dashboard/redis-info.js:
--------------------------------------------------------------------------------
1 | import React from 'react';
2 |
3 | function Card({title, val}) {
4 | return (
5 |
6 |
7 |
{title}
8 |
9 | {val}
10 |
11 |
12 |
23 |
Redis info
24 |
25 |
26 |
27 |
28 |
30 |
32 |
34 |
36 |
38 |
39 |
23 |
24 |
25 | Lowkiq dashboard
26 |
27 |
28 |
29 |
30 |
31 |
32 |
33 |
15 |
16 |
18 |
19 |
23 |
27 |
28 |
29 |
33 |
37 |
38 |
39 | {items.map(item => {
40 | return
;
41 | })}
42 |
12 |
13 |
15 |
16 |
20 |
24 |
28 |
29 |
30 |
34 |
35 |
39 |
40 |
44 |
45 |
46 | {items.map(item => {
47 | return
;
48 | })}
49 |
88 |
89 |
90 | );
91 | }
92 | }
93 |
--------------------------------------------------------------------------------
/frontend/webpack.config.web-api.js:
--------------------------------------------------------------------------------
1 | const path = require('path');
2 | const webpack = require('webpack');
3 |
4 | module.exports = {
5 | mode: 'development',
6 | entry: './main-web-api/demo.js',
7 | devServer: {
8 | hot: true,
9 | contentBase: './main-web-api',
10 | port: 8081,
11 | host: "0.0.0.0",
12 | historyApiFallback: true,
13 | proxy: {
14 | // Docker for Mac only
15 | '/api': 'http://host.docker.internal:8080'
16 | }
17 | },
18 | output: {
19 | filename: 'demo.js',
20 | path: path.resolve(__dirname, 'main-web-api'),
21 | publicPath: '/'
22 | },
23 | devtool: 'eval',
24 | resolve: {
25 | alias: {
26 | 'chart.js': 'chart.js/dist/Chart.js'
27 | }
28 | },
29 | plugins: [
30 | new webpack.HotModuleReplacementPlugin()
31 | ],
32 | module: {
33 | rules: [
34 | {
35 | test: /\.js$/,
36 | include: [
37 | path.join(__dirname, 'src'),
38 | path.join(__dirname, 'main-web-api')
39 | ],
40 | use: {
41 | loader: 'babel-loader',
42 | options: {
43 | presets: ['@babel/preset-env', '@babel/preset-react'],
44 | plugins: [
45 | 'react-hot-loader/babel',
46 | ]
47 | }
48 | }
49 | }, {
50 | test: /\.(scss)$/,
51 | use: [{
52 | loader: 'style-loader'
53 | }, {
54 | loader: 'css-loader'
55 | }, {
56 | loader: 'postcss-loader',
57 | options: {
58 | plugins: () => [
59 | require('autoprefixer')
60 | ]
61 | }
62 | }, {
63 | loader: 'sass-loader'
64 | }]
65 | }, {
66 | test: /\.module\.css$/,
67 | include: path.resolve(__dirname, 'src', 'dumb'),
68 | use: [{
69 | loader: 'style-loader'
70 | }, {
71 | loader: 'css-loader',
72 | options: {
73 | modules: true,
74 | localIdentName: '[local]--[hash:base64:5]'
75 | }
76 | }, {
77 | loader: 'postcss-loader',
78 | options: {
79 | plugins: () => [
80 | require('autoprefixer')
81 | ]
82 | }
83 | }]
84 | }
85 | ]
86 | }
87 | };
88 |
--------------------------------------------------------------------------------
/lib/lowkiq/queue/queue_metrics.rb:
--------------------------------------------------------------------------------
1 | module Lowkiq
2 | module Queue
3 | class QueueMetrics
4 | def initialize(redis_pool)
5 | @redis_pool = redis_pool
6 | @timestamp = Utils::Timestamp.method(:now)
7 | end
8 |
9 | def call(queues)
10 | result = @redis_pool.with do |redis|
11 | #redis.pipelined do
12 | redis.multi do
13 | queues.each { |queue| pipeline redis, queue }
14 | end
15 | end
16 |
17 | result.each_slice(pipeline_count).map do |res|
18 | coerce(res)
19 | end
20 | end
21 |
22 | private
23 |
24 | def pipeline(redis, name)
25 | keys = Keys.new name
26 |
27 | # fresh
28 | redis.zcount keys.all_ids_scored_by_retry_count_zset, -1, -1
29 |
30 | # retries
31 | redis.zcount keys.all_ids_scored_by_retry_count_zset, 0, '+inf'
32 |
33 | # morgue_length
34 | redis.zcard keys.morgue_all_ids_scored_by_updated_at_zset
35 |
36 | # lag [id, score]
37 | redis.zrange keys.all_ids_scored_by_perform_in_zset,
38 | 0, 0, with_scores: true
39 | # processed
40 | redis.get keys.processed_key
41 |
42 | # failed
43 | redis.get keys.failed_key
44 |
45 | # busy []
46 | redis.hvals keys.processing_length_by_shard_hash
47 | end
48 |
49 | def pipeline_count
50 | 7
51 | end
52 |
53 | def coerce(result)
54 | length = result[0] + result[1]
55 | OpenStruct.new length: length,
56 | fresh: result[0],
57 | retries: result[1],
58 | morgue_length: result[2],
59 | lag: coerce_lag(result[3]),
60 | processed: result[4].to_i,
61 | failed: result[5].to_i,
62 | busy: coerce_busy(result[6])
63 | end
64 |
65 | def coerce_lag(res)
66 | _id, score = res.first
67 | return 0.0 if score.nil?
68 | lag = @timestamp.call - score
69 | return 0.0 if lag < 0.0
70 | lag
71 | end
72 |
73 | def coerce_busy(res)
74 | res.map(&:to_i).reduce(0, &:+)
75 | end
76 | end
77 | end
78 | end
79 |
--------------------------------------------------------------------------------
/examples/dummy/lib/app.rb:
--------------------------------------------------------------------------------
1 | require 'logger'
2 | require "zlib"
3 | require "base64"
4 |
5 | # Lowkiq.build_splitter = ->() { Lowkiq.build_by_node_splitter 2, 0 }
6 |
7 | Lowkiq.format_error = -> (error) { error.full_message(highlight: false) }
8 | Lowkiq.dump_error = Proc.new do |msg|
9 | compressed = Zlib::Deflate.deflate(msg.to_s)
10 | Base64.encode64(compressed)
11 | end
12 | Lowkiq.load_error = Proc.new do |input|
13 | decoded = Base64.decode64(input)
14 | Zlib::Inflate.inflate(decoded)
15 | rescue
16 | input
17 | end
18 |
19 | $logger = Logger.new(STDOUT)
20 |
21 | Lowkiq.server_middlewares << -> (worker, batch, &block) do
22 | $logger.info "Started job for #{worker} #{batch}"
23 | block.call
24 | $logger.info "Finished job for #{worker} #{batch}"
25 | end
26 |
27 | Lowkiq.server_middlewares << -> (worker, batch, &block) do
28 | begin
29 | block.call
30 | rescue => e
31 | $logger.error "#{e.message} #{worker} #{batch}"
32 | raise e
33 | end
34 | end
35 |
36 | Lowkiq.last_words = ->(ex) { puts ex }
37 |
38 | module ATestWorker
39 | extend Lowkiq::Worker
40 |
41 | self.max_retry_count = 2
42 | def self.perform_async(jobs)
43 | jobs.each do |job|
44 | job.merge! id: job[:payload][:id]
45 | end
46 | super
47 | end
48 |
49 | def self.perform(batch)
50 | sleep Random.rand
51 |
52 | if Random.rand(5) == 0
53 | fail "error"
54 | end
55 | end
56 | end
57 |
58 | module ATest2Worker
59 | extend Lowkiq::Worker
60 |
61 | self.max_retry_count = 2
62 | def self.perform_async(jobs)
63 | jobs.each do |job|
64 | job.merge! id: job[:payload][:id]
65 | end
66 | super
67 | end
68 |
69 | def self.perform(batch)
70 | sleep Random.rand
71 |
72 | if Random.rand(5) == 0
73 | fail "error"
74 | end
75 | end
76 | end
77 |
78 | Lowkiq.workers = [ ATestWorker, ATest2Worker ]
79 |
80 | ATestWorker.perform_async 1000.times.map { |id| { payload: {id: id},
81 | perform_in: Time.now.to_f + Random.rand(10)} }
82 | ATest2Worker.perform_async 1000.times.map { |id| { payload: {id: id},
83 | perform_in: Time.now.to_f + Random.rand(10)} }
84 |
85 | require 'rack'
86 |
87 | Thread.new do
88 | Rack::Handler::WEBrick.run Lowkiq::Web, Port: 8080, Host: '0.0.0.0'
89 | end
90 |
--------------------------------------------------------------------------------
/frontend/src/dumb/details/job.js:
--------------------------------------------------------------------------------
1 | import React from 'react';
2 | import classNames from 'classnames';
3 | import style from './job.module.css';
4 |
5 | function isPresent(val) {
6 | return val !== null && val !== undefined && val !== '';
7 | }
8 |
9 | function Meta({label, value}) {
10 | return (
11 |
12 | | {label} |
13 | {value} |
14 |
15 | );
16 | }
17 |
18 | function Payload({payload, score}) {
19 | return [
20 | (
21 |
22 | ), (
23 |
24 | | payload |
25 |
26 |
27 | {JSON.stringify(payload, null, 2)}
28 |
29 | |
30 |
31 | )
32 | ];
33 | }
34 |
35 | function Actions({onQueueUp, onDelete}) {
36 | return (
37 |
38 | |
39 | actions
40 | |
41 |
42 |
46 |
50 | |
51 |
52 | );
53 | }
54 |
55 | function time(timestamp) {
56 | const time = new Date(timestamp * 1000).toLocaleString();
57 | return `${timestamp} [${time}]`;
58 | }
59 |
60 | function fmtRetryCount(val) {
61 | if (val === -1) return `${val} [fresh]`;
62 | if (val >= 0) return `${val} [retry]`;
63 | return val;
64 | }
65 |
66 | export default function Job(props) {
67 | const {
68 | id, perform_in, retry_count, updated_at, error, payloads,
69 | actions
70 | } = props;
71 | const tableClasses = classNames(
72 | 'table',
73 | 'table-bordered',
74 | style.table
75 | );
76 | return (
77 |
78 |
79 |
80 |
81 |
82 |
83 | {isPresent(id) && }
84 | {isPresent(perform_in) && }
85 | {isPresent(retry_count) && }
86 | {isPresent(updated_at) && }
87 | {isPresent(error) && {error}} />}
88 | {payloads.map(([payload, score]) => {
89 | return ;
90 | })}
91 | {actions && }
92 |
93 |
94 | );
95 | }
96 |
--------------------------------------------------------------------------------
/frontend/src/web-api/details/enqueued.js:
--------------------------------------------------------------------------------
1 | import React from 'react';
2 | import _get from 'lodash/get';
3 |
4 | import Enqueued from '../../dumb/details/enqueued';
5 | import { RedirectToDashboard } from '../../dumb/util/redirect';
6 |
7 | export default function enqueuedFactory(client) {
8 | return class EnqueuedManager extends React.Component {
9 | constructor(props) {
10 | super(props);
11 |
12 | const selectedFilter = _get(
13 | props,
14 | 'location.state.selectedFilter',
15 | {id: {min: '-inf', max: '+inf', rev: false}}
16 | );
17 |
18 | this.state = {
19 | items: [],
20 | selectedFilter: selectedFilter
21 | };
22 | this.worker = props.match.params.name;
23 | this.handleFilter = this.handleFilter.bind(this);
24 | this.handlePerformAllJobsNow = this.handlePerformAllJobsNow.bind(this);
25 | this.handleKillAllFailedJobs = this.handleKillAllFailedJobs.bind(this);
26 | this.handleDeleteAllFailedJobs = this.handleDeleteAllFailedJobs.bind(this);
27 | }
28 |
29 | fetch(filter) {
30 | return client
31 | .filter(this.worker, filter)
32 | .then( items => this.setState({items, selectedFilter: filter}));
33 | }
34 |
35 | componentDidMount() {
36 | this.fetch(this.state.selectedFilter);
37 | }
38 |
39 | handleFilter(column, interval) {
40 | this.fetch({
41 | [column]: interval
42 | });
43 | }
44 |
45 | // действия происходят в бэкенде в отдельном треде
46 | handlePerformAllJobsNow() {
47 | if (! confirm("Are you sure?") ) return;
48 | client
49 | .perform_all_jobs_now(this.worker)
50 | .then(() => this.setState({toDashboard: true}));
51 | }
52 |
53 | // действия происходят в бэкенде в отдельном треде
54 | handleKillAllFailedJobs() {
55 | if (! confirm("Are you sure?") ) return;
56 | client
57 | .kill_all_failed_jobs(this.worker)
58 | .then(() => this.setState({toDashboard: true}));
59 | }
60 |
61 | // действия происходят в бэкенде в отдельном треде
62 | handleDeleteAllFailedJobs() {
63 | if (! confirm("Are you sure?") ) return;
64 | client
65 | .delete_all_failed_jobs(this.worker)
66 | .then(() => this.setState({toDashboard: true}));
67 | }
68 |
69 | render() {
70 | if (this.state.toDashboard) return ;
71 |
72 | return (
73 |
79 | );
80 | }
81 | };
82 | }
83 |
--------------------------------------------------------------------------------
/frontend/src/web-api/client.js:
--------------------------------------------------------------------------------
1 | import ky from 'ky';
2 |
3 | function prepareMinMax(column, [min, max]) {
4 | let c_min, c_max;
5 |
6 | if (column === 'id') {
7 | if (min === '-inf') {
8 | c_min = '-';
9 | } else {
10 | c_min = '[' + min;
11 | }
12 | } else {
13 | c_min = min.toString();
14 | }
15 |
16 | if (column === 'id') {
17 | if (max === '+inf') {
18 | c_max = '+';
19 | } else {
20 | c_max = '[' + max;
21 | }
22 | } else {
23 | c_max = max.toString();
24 | }
25 |
26 | return [c_min, c_max];
27 | }
28 |
29 | export default class Routes {
30 | constructor(baseUrl, opts = {}) {
31 | this.api = ky.extend({prefixUrl: baseUrl, ...opts});
32 | }
33 |
34 | dashboard() {
35 | return this.api.get('dashboard')
36 | .then(resp => resp.json());
37 | }
38 |
39 | processing_data(worker) {
40 | return this.api
41 | .get(`${worker}/processing_data`)
42 | .then(resp => resp.json());
43 | }
44 |
45 | filter(worker, filter) {
46 | const [column, {rev, min, max}] = Object.entries(filter)[0];
47 | const [c_min, c_max] = prepareMinMax(column, [min, max]);
48 | const path = `${worker}/${rev ? 'rev_' : ''}range_by_${column}`;
49 |
50 | return this.api
51 | .get(path, {searchParams: {min: c_min, max: c_max}})
52 | .then(resp => resp.json());
53 | }
54 |
55 | morgue_filter(worker, filter) {
56 | const [column, {rev, min, max}] = Object.entries(filter)[0];
57 | const [c_min, c_max] = prepareMinMax(column, [min, max]);
58 | const path = `${worker}/morgue_${rev ? 'rev_' : ''}range_by_${column}`;
59 |
60 | return this.api
61 | .get(path, {searchParams: {min: c_min, max: c_max}})
62 | .then(resp => resp.json());
63 | }
64 |
65 | morgue_queue_up(worker, id) {
66 | return this.api
67 | .post(`${worker}/morgue_queue_up?ids[]=${id}`)
68 | .then(() => null);
69 | }
70 |
71 | morgue_delete(worker, id) {
72 | return this.api
73 | .post(`${worker}/morgue_delete?ids[]=${id}`)
74 | .then(() => null);
75 | }
76 |
77 | morgue_queue_up_all_jobs(worker) {
78 | return this.api
79 | .post(`${worker}/morgue_queue_up_all_jobs`)
80 | .then(() => null);
81 | }
82 |
83 | morgue_delete_all_jobs(worker) {
84 | return this.api
85 | .post(`${worker}/morgue_delete_all_jobs`)
86 | .then(() => null);
87 | }
88 |
89 | perform_all_jobs_now(worker) {
90 | return this.api
91 | .post(`${worker}/perform_all_jobs_now`)
92 | .then(() => null);
93 | }
94 |
95 | kill_all_failed_jobs(worker) {
96 | return this.api
97 | .post(`${worker}/kill_all_failed_jobs`)
98 | .then(() => null);
99 | }
100 |
101 | delete_all_failed_jobs(worker) {
102 | return this.api
103 | .post(`${worker}/delete_all_failed_jobs`)
104 | .then(() => null);
105 | }
106 | }
107 |
--------------------------------------------------------------------------------
/frontend/src/web-api/details/dead.js:
--------------------------------------------------------------------------------
1 | import React from 'react';
2 |
3 | import Dead from '../../dumb/details/dead';
4 | import { RedirectToDashboard } from '../../dumb/util/redirect';
5 |
6 | export default function deadFactory(client) {
7 | return class DashboardManager extends React.Component {
8 | constructor(props) {
9 | super(props);
10 | this.state = {
11 | items: [],
12 | selectedFilter: {id: {min: '-inf', max: '+inf', rev: false}}
13 | };
14 | this.worker = props.match.params.name;
15 |
16 | this.handleFilter = this.handleFilter.bind(this);
17 | this.handleQueueUp = this.handleQueueUp.bind(this);
18 | this.handleDelete = this.handleDelete.bind(this);
19 | this.handleQueueUpAllJobs = this.handleQueueUpAllJobs.bind(this);
20 | this.handleDeleteAllJobs = this.handleDeleteAllJobs.bind(this);
21 | }
22 |
23 | // действия происходят в бэкенде в отдельном треде
24 | handleQueueUp(id) {
25 | client
26 | .morgue_queue_up(this.worker, id)
27 | .then(() => this.setState(state => {
28 | const items = state.items.filter( item => item.id !== id );
29 | return {items};
30 | }));
31 | }
32 |
33 | // действия происходят в бэкенде в отдельном треде
34 | handleDelete(id) {
35 | client
36 | .morgue_delete(this.worker, id)
37 | .then(() => this.setState(state => {
38 | const items = state.items.filter( item => item.id !== id );
39 | return {items};
40 | }));
41 | }
42 |
43 | // действия происходят в бэкенде в отдельном треде
44 | handleQueueUpAllJobs() {
45 | if (! confirm("Are you sure?") ) return;
46 | client
47 | .morgue_queue_up_all_jobs(this.worker)
48 | .then(() => this.setState({toDashboard: true}));
49 | }
50 |
51 | // действия происходят в бэкенде в отдельном треде
52 | handleDeleteAllJobs() {
53 | if (! confirm("Are you sure?") ) return;
54 | client
55 | .morgue_delete_all_jobs(this.worker)
56 | .then(() => this.setState({toDashboard: true}));
57 | }
58 |
59 | fetch(filter) {
60 | return client
61 | .morgue_filter(this.worker, filter)
62 | .then( items => items.map((item) => {
63 | item.actions = {
64 | onQueueUp: () => this.handleQueueUp(item.id),
65 | onDelete: () => this.handleDelete(item.id)
66 | };
67 | return item;
68 | }))
69 | .then( items => this.setState({items, selectedFilter: filter}));
70 | }
71 |
72 | componentDidMount() {
73 | this.fetch(this.state.selectedFilter);
74 | }
75 |
76 | handleFilter(column, interval) {
77 | this.fetch({
78 | [column]: interval
79 | });
80 | }
81 |
82 | render() {
83 | if (this.state.toDashboard) return ;
84 |
85 | return (
86 |
91 | );
92 | }
93 | };
94 | }
95 |
--------------------------------------------------------------------------------
/frontend/src/dumb/details/filter.js:
--------------------------------------------------------------------------------
1 | import React from 'react';
2 |
3 | function Interval({min, max, rev}) {
4 | if (!rev && min === '-inf' && max === '+inf') {
5 | return "(-∞, +∞)";
6 | } else if (rev && max === '+inf' && min === '-inf') {
7 | return "(+∞, -∞)";
8 | } else if (!rev && max === '+inf') {
9 | return `[val, +∞)`;
10 | } else if (rev && min === '-inf') {
11 | return `[val, -∞)`;
12 | } else {
13 | throw "invalid interval";
14 | }
15 | }
16 |
17 | function Button({interval, selected, disabled, onClick}) {
18 | let btnType = selected
19 | && interval.rev === selected.rev
20 | && interval.min === selected.min
21 | && interval.max === selected.max
22 | ? 'btn-secondary' : 'btn-outline-secondary';
23 | const handleClick = () => onClick(interval);
24 |
25 | return (
26 |
29 | );
30 | }
31 |
32 | function getVal(selected) {
33 | if (!selected) return "";
34 |
35 | const {min, max, rev} = selected;
36 | if (min !== '-inf' && max === '+inf' && rev === false)
37 | return min;
38 | if (min === '-inf' && max !== '+inf' && rev === true)
39 | return max;
40 | return "";
41 | }
42 |
43 | export default class Filter extends React.Component {
44 | constructor(props) {
45 | super(props);
46 |
47 | this.state = {
48 | value: getVal(props.selected)
49 | };
50 | this.handleChange = this.handleChange.bind(this);
51 | }
52 |
53 | handleChange(e) {
54 | this.setState({value: e.target.value});
55 | }
56 |
57 | render() {
58 | const {label, selected, type, onClick} = this.props;
59 | const {value} = this.state;
60 |
61 | return (
62 |
63 |
64 |
65 |
66 |
67 |
68 |
71 |
72 |
73 |
76 |
77 |
78 |
83 |
84 |
85 |
89 |
93 |
94 |
95 |
96 |
39 | |
40 |
41 |
42 | |
43 |
44 |
45 | |
46 |
47 |
48 | |
49 |
50 |
51 | {formatNumber(busy)}
52 |
53 | |
54 |
55 |
58 | {formatNumber(enqueued)}
59 |
60 | |
61 |
62 |
66 | {formatNumber(fresh)}
67 |
68 | |
69 |
70 |
74 | {formatNumber(retries)}
75 |
76 | |
77 |
78 |
79 | {formatNumber(dead)}
80 |
81 | |
82 |
83 | );
84 | }
85 |
86 | export default function Table({queues}) {
87 | return (
88 |
89 |
90 |
91 |
92 | | Worker |
93 | Processed |
94 | Failed |
95 | Lag |
96 | Busy |
97 | Enqueued |
98 | Fresh |
99 | Retries |
100 | Dead |
101 |
102 |
103 |
104 | {
105 | queues.map( queue => {
106 | return
107 | {routes => |
}
108 | ;
109 | })
110 | }
111 |
112 |
113 |
133 | g. Saint-Petersburg, pr. Moskovskiy, d.94, lit. A, pom. 12- H
134 |
135 | OGRN 1147847386906
136 |
137 | TIN/ 7810385714
138 |
139 | RRC/ 781001001
140 |
141 | Name and email address of the representative:
142 | Sergey Staskov
143 | Sergey.Staskov@bia-tech.ru
144 |
--------------------------------------------------------------------------------
/README.ru.md:
--------------------------------------------------------------------------------
1 | [](https://badge.fury.io/rb/lowkiq)
2 |
3 | # Lowkiq
4 |
5 | Упорядоченная обработка фоновых задач.
6 |
7 | 
8 |
9 | * [Rationale](#rationale)
10 | * [Description](#description)
11 | * [Sidekiq](#sidekiq)
12 | * [Очередь](#%D0%BE%D1%87%D0%B5%D1%80%D0%B5%D0%B4%D1%8C)
13 | + [Алгоритм расчета retry_count и perform_in](#%D0%B0%D0%BB%D0%B3%D0%BE%D1%80%D0%B8%D1%82%D0%BC-%D1%80%D0%B0%D1%81%D1%87%D0%B5%D1%82%D0%B0-retry_count-%D0%B8-perform_in)
14 | + [Правило слияния задач](#%D0%BF%D1%80%D0%B0%D0%B2%D0%B8%D0%BB%D0%BE-%D1%81%D0%BB%D0%B8%D1%8F%D0%BD%D0%B8%D1%8F-%D0%B7%D0%B0%D0%B4%D0%B0%D1%87)
15 | * [Install](#install)
16 | * [Api](#api)
17 | * [Ring app](#ring-app)
18 | * [Настройка](#%D0%BD%D0%B0%D1%81%D1%82%D1%80%D0%BE%D0%B9%D0%BA%D0%B0)
19 | * [Запуск](#%D0%B7%D0%B0%D0%BF%D1%83%D1%81%D0%BA)
20 | * [Остановка](#%D0%BE%D1%81%D1%82%D0%B0%D0%BD%D0%BE%D0%B2%D0%BA%D0%B0)
21 | * [Debug](#debug)
22 | * [Development](#development)
23 | * [Исключения](#%D0%B8%D1%81%D0%BA%D0%BB%D1%8E%D1%87%D0%B5%D0%BD%D0%B8%D1%8F)
24 | * [Rails integration](#rails-integration)
25 | * [Splitter](#splitter)
26 | * [Scheduler](#scheduler)
27 | * [Рекомендации по настройке](#%D1%80%D0%B5%D0%BA%D0%BE%D0%BC%D0%B5%D0%BD%D0%B4%D0%B0%D1%86%D0%B8%D0%B8-%D0%BF%D0%BE-%D0%BD%D0%B0%D1%81%D1%82%D1%80%D0%BE%D0%B9%D0%BA%D0%B5)
28 | + [`SomeWorker.shards_count`](#someworkershards_count)
29 | + [`SomeWorker.max_retry_count`](#someworkermax_retry_count)
30 | * [Изменение количества шардов воркера](#%D0%B8%D0%B7%D0%BC%D0%B5%D0%BD%D0%B5%D0%BD%D0%B8%D0%B5-%D0%BA%D0%BE%D0%BB%D0%B8%D1%87%D0%B5%D1%81%D1%82%D0%B2%D0%B0-%D1%88%D0%B0%D1%80%D0%B4%D0%BE%D0%B2-%D0%B2%D0%BE%D1%80%D0%BA%D0%B5%D1%80%D0%B0)
31 |
32 | ## Rationale
33 |
34 | При использовании Sidekiq мы столкнулись с проблемами при обработке сообщений от сторонней системы.
35 |
36 | Скажем, сообщение представляет собой данные заказа в определенный момент времени.
37 | При изменении атрибутов или статуса отправляется новое сообщение сторонней системой.
38 | Заказы обновляются часто и в очереди рядом находятся сообщения, касающиеся одного и того же заказа.
39 |
40 | Sidekiq не гарантирует строгого порядка сообщений, т.к. очередь обрабатывается в несколько потоков.
41 | Например, пришло 2 сообщения: M1 и M2.
42 | Sidekiq обработчики начинают обрабатывать их параллельно,
43 | при этом M2 может обработаться раньше M1.
44 |
45 | Параллельная обработка данных одного заказа приводит к:
46 |
47 | + dead locks
48 | + затиранию новых данных старыми
49 |
50 | Lowkiq призван устранить эти проблемы, исключая параллельность обработки сообщений в рамках одной сущности.
51 |
52 | ## Description
53 |
54 | Очереди надежны. Lowkiq сохраняет данные об обрабатываемой задаче и при запуске переносит
55 | незавершенные задачи обратно в очередь.
56 |
57 | Задачи в очереди отсортированы по заданному времени исполнения, т.е. это не FIFO очереди.
58 |
59 | Каждая задача имеет идентификатор. Очереди гарантируют, что не может быть ситуации,
60 | когда несколько потоков обрабатывают задачи с одинаковыми идентификаторами.
61 |
62 | Каждая очередь разбивается на постоянный набор шардов.
63 | На основе идентификатора задачи выбирается шард, в который попадет задача.
64 | Таким образом задачи с одним идентификатором всегда попадают в один и тот же шард.
65 | Задачи шарда всегда обрабатываются одним и тем же потоком.
66 | Это гарантирует порядок обработки задач с одинаковым идентификатором и исключает возможность блокировок.
67 |
68 | Кроме идентификатора задача имеет полезную нагрузку или данные задачи (payload).
69 | Для задач с одинаковым идентификаторм происходит слияние полезных нагрузок.
70 | Таким образом одновременно в обработку попадают все накопленные полезные нагрузки задачи.
71 | Это полезно, если нужно обработать только последнее сообщение и отбросить все предыдущие.
72 |
73 | Каждой очереди соответствует воркер, содержащий логику обработки задачи.
74 |
75 | Для обработки всех задач всех очередей используется фиксированное количество тредов.
76 | Добавление или удаление очередей или их шардов не приводит к изменению числа тредов.
77 |
78 | ## Sidekiq
79 |
80 | Если для ваших задач подходит Sidekiq - используйте его.
81 |
82 | Если вы используете плагины вроде
83 | [sidekiq-grouping](https://github.com/gzigzigzeo/sidekiq-grouping),
84 | [sidekiq-unique-jobs](https://github.com/mhenrixon/sidekiq-unique-jobs),
85 | [sidekiq-merger](https://github.com/dtaniwaki/sidekiq-merger)
86 | или реализуете собственный механизм блокировок, то стоит рассмотреть Lowkiq.
87 |
88 | Например, sidekiq-grouping предварительно накапливает пачку задач, ставит ее в очередь и начинает накапливать следующую.
89 | При таком подходе случается ситуация, когда в очереди находятся 2 пачки с данными одного заказа.
90 | Эти пачки начинают обрабатываться одновременно разными тредами, что приводит к изначальной проблеме.
91 |
92 | Lowkiq изначально проектировался так, чтобы не использовать любые блокировки.
93 |
94 | Кроме того, в Lowkiq очереди изначально надежны. Только Sidekiq Pro или плагины добавляют такую функциональность.
95 |
96 | Этот [бенчмарк](examples/benchmark) показывает накладные расходы на взаимодействие с redis.
97 | Для 5 threads, 100'000 blank jobs получились результаты:
98 |
99 | + lowkiq: 214 sec или 2,14 мс на задачу
100 | + sidekiq: 29 sec или 0,29 мс на задачу
101 |
102 | Эта разница связана с принципиально различным устройством очередей.
103 | Sidekiq использует один список для всех воркеров и извлекает задачу целиком за O(1).
104 | Lowkiq использует несколько типов данных, включая сортированные множества для хранения идентификаторов задач.
105 | Таким образом только получение идентификатора задачи занимает O(log(N)).
106 |
107 | ## Очередь
108 |
109 | Каждая задача в очереди имеет аттрибуты:
110 |
111 | + `id` - идентификатор задачи (строка)
112 | + `payloads` - сортированное множество payload'ов (объекты) по их score (вещественное число)
113 | + `perform_in` - запланированное время начала иполнения задачи (unix timestamp, вещественное число)
114 | + `retry_count` - количество совершённых повторов задачи (вещественное число)
115 |
116 | `id` может быть, например, идентификатором реплицируемой сущности
117 | `payloads` - множество,
118 | получаемое в результате группировки полезной нагрузки задачи по `id` и отсортированное по ее `score`.
119 | `payload` может быть ruby объектом, т.к. сериализуется с помощью `Marshal.dump`.
120 | `score` может быть датой (unix timestamp) создания `payload`
121 | или ее монотонно увеличивающимся номером версии.
122 | По умолчанию - текущий unix timestamp.
123 | По умолчанию `perform_in` - текущий unix timestamp.
124 | `retry_count` для новой необработанной задачи равен `-1`, для упавшей один раз - `0`,
125 | т.е. считаются не совершённые, а запланированные повторы.
126 |
127 | Выполнение задачи может закончиться неудачей.
128 | В этом случае ее `retry_count` инкрементируется и по заданной формуле вычисляется новый `perform_in`,
129 | и она ставится обратно в очередь.
130 |
131 | В случае, когда `retry_count` становится `>=` `max_retry_count`
132 | элемент payloads с наименьшим(старейшим) score перемещается в морг,
133 | а оставшиеся элементы помещаются обратно в очередь, при этом
134 | `retry_count` и `perform_in` сбрасываются в `-1` и `now()` соответственно.
135 |
136 | ### Алгоритм расчета retry_count и perform_in
137 |
138 | 0. задача выполнилась и упала
139 | 1. `retry_count++`
140 | 2. `perform_in = now + retry_in(try_count)`
141 | 3. `if retry_count >= max_retry_count` задача перемещается в морг
142 |
143 | | тип | `retry_count` | `perform_in` |
144 | | --- | --- | --- |
145 | | новая не выполнялась | -1 | задан или `now()` |
146 | | новая упала | 0 | `now() + retry_in(0)` |
147 | | повтор упал | 1 | `now() + retry_in(1)` |
148 |
149 | Если `max_retry_count = 1`, то попытки прекращаются.
150 |
151 | ### Правило слияния задач
152 |
153 | Когда применяется:
154 |
155 | + если в очереди была задача и добавляется еще одна с тем же id
156 | + если при обработке возникла ошибка, а в очередь успели добавили задачу с тем же id
157 | + если задачу из морга поставили в очередь, а в очереди уже есть задача с тем же id
158 |
159 | Алгоритм:
160 |
161 | + payloads объединяются, при этом выбирается минимальный score,
162 | т.е. для одинаковых payload выигрывает самая старая
163 | + если объединяется новая и задача из очереди,
164 | то `perform_in` и `retry_count` берутся из задачи из очереди
165 | + если объединяется упавшая задача и задача из очереди,
166 | то `perform_in` и `retry_count` берутся из упавшей
167 | + если объединяется задача из морга и задача из очереди,
168 | то `perform_in = now()`, `retry_count = -1`
169 |
170 | Пример:
171 |
172 | ```
173 | # v1 - первая версия, v2 - вторая
174 | # #{"v1": 1} - сортированное множество одного элемента, payload - "v1", score - 1
175 |
176 | # задача в очереди
177 | { id: "1", payloads: #{"v1": 1, "v2": 2}, retry_count: 0, perform_in: 1536323288 }
178 | # добавляемая задача
179 | { id: "1", payloads: #{"v2": 3, "v3": 4}, retry_count: -1, perform_in: 1536323290 }
180 |
181 | # результат
182 | { id: "1", payloads: #{"v1": 1, "v2": 3, "v3": 4}, retry_count: 0, perform_in: 1536323288 }
183 | ```
184 |
185 | Морг - часть очереди. Задачи в морге не обрабатываются.
186 | Задача в морге имеет следующие атрибуты:
187 |
188 | + id - идентификатор задачи
189 | + payloads
190 |
191 | Задачи в морге можно отсортировать по дате изменения или id.
192 |
193 | Задачу из морга можно переместить в очередь. При этом для нее `retry_count = 0`, `perform_in = now()`.
194 |
195 | ## Install
196 |
197 | ```
198 | # Gemfile
199 |
200 | gem 'lowkiq'
201 | ```
202 |
203 | Redis версии >= 3.2.
204 |
205 | ## Api
206 |
207 | ```ruby
208 | module ATestWorker
209 | extend Lowkiq::Worker
210 |
211 | self.shards_count = 24
212 | self.batch_size = 10
213 | self.max_retry_count = 5
214 |
215 | def self.retry_in(count)
216 | 10 * (count + 1) # (i.e. 10, 20, 30, 40, 50)
217 | end
218 |
219 | def self.perform(payloads_by_id)
220 | # payloads_by_id - хеш
221 | payloads_by_id.each do |id, payloads|
222 | # id - идентификатор задачи
223 | # payloads отсортированы по score, от старых к новым (от минимальных к максимальным)
224 | payloads.each do |payload|
225 | do_some_work(id, payload)
226 | end
227 | end
228 | end
229 | end
230 | ```
231 |
232 | Значения по умолчанию:
233 |
234 | ```ruby
235 | self.shards_count = 5
236 | self.batch_size = 1
237 | self.max_retry_count = 25
238 | self.queue_name = self.name
239 |
240 | # i.e. 15, 16, 31, 96, 271, ... seconds + a random amount of time
241 | def retry_in(retry_count)
242 | (retry_count ** 4) + 15 + (rand(30) * (retry_count + 1))
243 | end
244 | ```
245 |
246 | ```ruby
247 | ATestWorker.perform_async [
248 | { id: 0 },
249 | { id: 1, payload: { attr: 'v1' } },
250 | { id: 2, payload: { attr: 'v1' }, score: Time.now.to_f, perform_in: Time.now.to_f },
251 | ]
252 | # payload по умолчанию равен ""
253 | # score и perform_in по умолчанию равны Time.now.to_f
254 | ```
255 |
256 | Вы можете переопределить `perform_async` и вычислять `id`, `score` и `perform_in` в воркере:
257 |
258 | ```ruby
259 | module ATestWorker
260 | extend Lowkiq::Worker
261 |
262 | def self.perform_async(jobs)
263 | jobs.each do |job|
264 | job.merge! id: job[:payload][:id]
265 | end
266 | super
267 | end
268 |
269 | def self.perform(payloads_by_id)
270 | #...
271 | end
272 | end
273 |
274 | ATestWorker.perform_async 1000.times.map { |id| { payload: {id: id} } }
275 | ```
276 |
277 | ## Ring app
278 |
279 | `Lowkiq::Web` - ring app.
280 |
281 | + `/` - dashboard
282 | + `/api/v1/stats` - длина очереди, длина морга, лаг для каждого воркера и суммарно
283 |
284 | ## Настройка
285 |
286 | Опции и значения по умолчанию:
287 |
288 | + `Lowkiq.poll_interval = 1` - задержка в секундах между опросами очереди на предмет новых задач.
289 | Используется только если на предыдущей итерации очередь оказалась пуста или случилась ошибка.
290 | + `Lowkiq.threads_per_node = 5` - кол-во тредов для каждой ноды.
291 | + `Lowkiq.redis = ->() { Redis.new url: ENV.fetch('REDIS_URL') }` - настройка redis.
292 | + `Lowkiq.client_pool_size = 5` - размер пула редиса для постановки задач в очередь.
293 | + `Lowkiq.pool_timeout = 5` - таймаут клиентского и серверного пула редиса
294 | + `Lowkiq.server_middlewares = []` - список middleware, оборачивающих воркер.
295 | + `Lowkiq.on_server_init = ->() {}` - выполнения кода при инициализации сервера.
296 | + `Lowkiq.build_scheduler = ->() { Lowkiq.build_lag_scheduler }` - планировщик.
297 | + `Lowkiq.build_splitter = ->() { Lowkiq.build_default_splitter }` - сплиттер.
298 | + `Lowkiq.last_words = ->(ex) {}` - обработчик исключений, потомков `StandardError`, вызвавших остановку процесса.
299 |
300 | ```ruby
301 | $logger = Logger.new(STDOUT)
302 |
303 | Lowkiq.server_middlewares << -> (worker, batch, &block) do
304 | $logger.info "Started job for #{worker} #{batch}"
305 | block.call
306 | $logger.info "Finished job for #{worker} #{batch}"
307 | end
308 |
309 | Lowkiq.server_middlewares << -> (worker, batch, &block) do
310 | begin
311 | block.call
312 | rescue => e
313 | $logger.error "#{e.message} #{worker} #{batch}"
314 | raise e
315 | end
316 | end
317 | ```
318 |
319 | ## Запуск
320 |
321 | `lowkiq -r ./path_to_app`
322 |
323 | `path_to_app.rb` должен загрузить приложение. [Пример](examples/dummy/lib/app.rb).
324 |
325 | Ленивая загрузка модулей воркеров недопустима.
326 | Используйте для предварительной загрузки модулей
327 | `require` или [`require_dependency`](https://api.rubyonrails.org/classes/ActiveSupport/Dependencies/Loadable.html#method-i-require_dependency)
328 | для Ruby on Rails.
329 |
330 | ## Остановка
331 |
332 | Послать процессу TERM или INT (Ctrl-C).
333 | Процесс будет ждать завершения всех задач.
334 |
335 | Обратите внимание, если очередь пуста, процесс спит `poll_interval` секунд.
336 | Таким образом завершится не позднее чем через `poll_interval` секунд.
337 |
338 | ## Debug
339 |
340 | Получить trace всех тредов приложения:
341 |
342 | ```
343 | kill -TTIN
344 | cat /tmp/lowkiq_ttin.txt
345 | ```
346 |
347 | ## Development
348 |
349 | ```
350 | docker-compose run --rm --service-port app bash
351 | bundle
352 | rspec
353 | cd examples/dummy ; bundle exec ../../exe/lowkiq -r ./lib/app.rb
354 | ```
355 |
356 | ## Исключения
357 |
358 | `StandardError` выброшенные воркером обрабатываются с помощью middleware.
359 | Такие исключения не приводят к остановке процесса.
360 |
361 | Все прочие исключения приводят к остановке процесса.
362 | При этом Lowkiq дожидается выполнения задач другими тредами.
363 |
364 | `StandardError` выброшенные вне воркера передаются в `Lowkiq.last_words`.
365 | Например это происходит при потере соединения к Redis или при ошибке в коде Lowkiq.
366 |
367 | ## Rails integration
368 |
369 | ```ruby
370 | # config/routes.rb
371 |
372 | Rails.application.routes.draw do
373 | # ...
374 | mount Lowkiq::Web => '/lowkiq'
375 | # ...
376 | end
377 | ```
378 |
379 | ```ruby
380 | # config/initializers/lowkiq.rb
381 |
382 | # загружаем все lowkiq воркеры
383 | Dir["#{Rails.root}/app/lowkiq_workers/**/*.rb"].each { |file| require_dependency file }
384 |
385 | # конфигурация:
386 | # Lowkiq.redis = -> { Redis.new url: ENV.fetch('LOWKIQ_REDIS_URL') }
387 | # Lowkiq.threads_per_node = ENV.fetch('LOWKIQ_THREADS_PER_NODE').to_i
388 | # Lowkiq.client_pool_size = ENV.fetch('LOWKIQ_CLIENT_POOL_SIZE').to_i
389 | # ...
390 |
391 | Lowkiq.server_middlewares << -> (worker, batch, &block) do
392 | logger = Rails.logger
393 | tag = "#{worker}-#{Thread.current.object_id}"
394 |
395 | logger.tagged(tag) do
396 | time_start = Time.now
397 | logger.info "#{time_start} Started job, batch: #{batch}"
398 | begin
399 | block.call
400 | rescue => e
401 | logger.error e.message
402 | raise e
403 | ensure
404 | time_end = Time.now
405 | logger.info "#{time_end} Finished job, duration: #{time_end - time_start} sec"
406 | end
407 | end
408 | end
409 |
410 | # Sentry integration
411 | Lowkiq.server_middlewares << -> (worker, batch, &block) do
412 | opts = {
413 | extra: {
414 | lowkiq: {
415 | worker: worker.name,
416 | batch: batch,
417 | }
418 | }
419 | }
420 |
421 | Raven.capture opts do
422 | block.call
423 | end
424 | end
425 |
426 | # NewRelic integration
427 | if defined? NewRelic
428 | class NewRelicLowkiqMiddleware
429 | include NewRelic::Agent::Instrumentation::ControllerInstrumentation
430 |
431 | def call(worker, batch, &block)
432 | opts = {
433 | category: 'OtherTransaction/LowkiqJob',
434 | class_name: worker.name,
435 | name: :perform,
436 | }
437 |
438 | perform_action_with_newrelic_trace opts do
439 | block.call
440 | end
441 | end
442 | end
443 |
444 | Lowkiq.server_middlewares << NewRelicLowkiqMiddleware.new
445 | end
446 |
447 | # Rails reloader, в том числе отвечает за высвобождение ActiveRecord коннектов
448 | Lowkiq.server_middlewares << -> (worker, batch, &block) do
449 | Rails.application.reloader.wrap do
450 | block.call
451 | end
452 | end
453 |
454 | Lowkiq.on_server_init = ->() do
455 | [[ActiveRecord::Base, ActiveRecord::Base.configurations[Rails.env]]].each do |(klass, init_config)|
456 | klass.connection_pool.disconnect!
457 | config = init_config.merge 'pool' => Lowkiq.threads_per_node
458 | klass.establish_connection(config)
459 | end
460 | end
461 | ```
462 |
463 | Запуск: `bundle exec lowkiq -r ./config/environment.rb`
464 |
465 | ## Splitter
466 |
467 | У каждого воркера есть несколько шардов:
468 |
469 | ```
470 | # worker: shard ids
471 | worker A: 0, 1, 2
472 | worker B: 0, 1, 2, 3
473 | worker C: 0
474 | worker D: 0, 1
475 | ```
476 |
477 | Lowkiq использует фиксированное кол-во тредов для обработки задач, следовательно нужно распределить шарды
478 | между тредами. Этим занимается Splitter.
479 |
480 | Чтобы определить набор шардов, которые будет обрабатывать тред, поместим их в один список:
481 |
482 | ```
483 | A0, A1, A2, B0, B1, B2, B3, C0, D0, D1
484 | ```
485 |
486 | Рассмотрим Default splitter, который равномерно распределяет шарды по тредам единственной ноды.
487 |
488 | Если `threads_per_node` установлено в 3, то распределение будет таким:
489 |
490 | ```
491 | # thread id: shards
492 | t0: A0, B0, B3, D1
493 | t1: A1, B1, C0
494 | t2: A2, B2, D0
495 | ```
496 |
497 | Помимо Default есть ByNode splitter. Он позволяет распределить нагрузку по нескольким процессам (нодам).
498 |
499 | ```
500 | Lowkiq.build_splitter = -> () do
501 | Lowkiq.build_by_node_splitter(
502 | ENV.fetch('LOWKIQ_NUMBER_OF_NODES').to_i,
503 | ENV.fetch('LOWKIQ_NODE_NUMBER').to_i
504 | )
505 | end
506 | ```
507 |
508 | Таким образом, вместо одного процесса нужно запустить несколько и указать переменные окружения:
509 |
510 | ```
511 | # process 0
512 | LOWKIQ_NUMBER_OF_NODES=2 LOWKIQ_NODE_NUMBER=0 bundle exec lowkiq -r ./lib/app.rb
513 |
514 | # process 1
515 | LOWKIQ_NUMBER_OF_NODES=2 LOWKIQ_NODE_NUMBER=1 bundle exec lowkiq -r ./lib/app.rb
516 | ```
517 |
518 | Отмечу, что общее количество тредов будет равно произведению `ENV.fetch('LOWKIQ_NUMBER_OF_NODES')` и `Lowkiq.threads_per_node`.
519 |
520 | Вы можете написать свой сплиттер, если ваше приложение требует особого распределения шардов между тредами или нодами.
521 |
522 | ## Scheduler
523 |
524 | Каждый тред обрабатывает набор шардов. За выбор шарда для обработки отвечает планировщик.
525 | Каждый поток имеет свой собственный экземпляр планировщика.
526 |
527 | Lowkiq имеет 2 планировщика на выбор.
528 | Первый, `Seq` - последовательно перебирает шарды.
529 | Второй, `Lag` - выбирает шард с самой старой задачей, т.е. стремится минимизировать лаг.
530 | Используется по умолчанию.
531 |
532 | Планировщик задается через настройки:
533 |
534 | ```
535 | Lowkiq.build_scheduler = ->() { Lowkiq.build_seq_scheduler }
536 | # или
537 | Lowkiq.build_scheduler = ->() { Lowkiq.build_lag_scheduler }
538 | ```
539 |
540 | ## Рекомендации по настройке
541 |
542 | ### `SomeWorker.shards_count`
543 |
544 | Сумма `shards_count` всех воркеров не должна быть меньше `Lowkiq.threads_per_node`
545 | иначе треды будут простаивать.
546 |
547 | Сумма `shards_count` всех воркеров может быть равна `Lowkiq.threads_per_node`.
548 | В этом случае тред обрабатывает единственный шард. Это имеет смысл только при равномерной нагрузке на очереди.
549 |
550 | Сумма `shards_count` всех воркеров может быть больше `Lowkiq.threads_per_node`.
551 | В этом случае `shards_count` можно рассматривать в качестве приоритета.
552 | Чем он выше, тем чаще задачи этой очереди будут обрабатываться.
553 |
554 | Нет смысла устанавливать `shards_count` одного воркера больше чем `Lowkiq.threads_per_node`,
555 | т.к. каждый тред будет обрабатывать более одного шарда этой очереди, что увеличит накладные расходы.
556 |
557 | ### `SomeWorker.max_retry_count`
558 |
559 | Исходя из `retry_in` и `max_retry_count`,
560 | можно вычислить примерное время, которая задача проведет в очереди.
561 | Под задачей тут понимается payload задачи.
562 | После достижения `max_retry_count` в морг переносится только payload с минимальным score.
563 |
564 | Для `retry_in`, заданного по умолчанию получается следующая таблица:
565 |
566 | ```ruby
567 | def retry_in(retry_count)
568 | (retry_count ** 4) + 15 + (rand(30) * (retry_count + 1))
569 | end
570 | ```
571 |
572 | | `max_retry_count` | кол-во дней жизни задачи |
573 | | --- | --- |
574 | | 14 | 1 |
575 | | 16 | 2 |
576 | | 18 | 3 |
577 | | 19 | 5 |
578 | | 20 | 6 |
579 | | 21 | 8 |
580 | | 22 | 10 |
581 | | 23 | 13 |
582 | | 24 | 16 |
583 | | 25 | 20 |
584 |
585 | `(0...25).map{ |c| retry_in c }.sum / 60 / 60 / 24`
586 |
587 |
588 | ## Изменение количества шардов воркера
589 |
590 | Старайтесь сразу расчитать количество шардов и не именять их количество в будущем.
591 |
592 | Если вы можете отключить добавление новых заданий,
593 | то дождитесь опустошения очередей и выкатите новую версию кода с измененным количеством шардов.
594 |
595 | Если такой возможности нет, воспользуйтесь следующим сценарием.
596 |
597 | Например, есть воркер:
598 |
599 | ```ruby
600 | module ATestWorker
601 | extend Lowkiq::Worker
602 |
603 | self.shards_count = 5
604 |
605 | def self.perform(payloads_by_id)
606 | some_code
607 | end
608 | end
609 | ```
610 |
611 | Теперь нужно указать новое кол-во шардов и задать новое имя очереди:
612 |
613 | ```ruby
614 | module ATestWorker
615 | extend Lowkiq::Worker
616 |
617 | self.shards_count = 10
618 | self.queue_name = "#{self.name}_V2"
619 |
620 | def self.perform(payloads_by_id)
621 | some_code
622 | end
623 | end
624 | ```
625 |
626 | И добавить воркер, перекладывающий задачи из старой очереди в новую:
627 |
628 | ```ruby
629 | module ATestMigrationWorker
630 | extend Lowkiq::Worker
631 |
632 | self.shards_count = 5
633 | self.queue_name = "ATestWorker"
634 |
635 | def self.perform(payloads_by_id)
636 | jobs = payloads_by_id.each_with_object([]) do |(id, payloads), acc|
637 | payloads.each do |payload|
638 | acc << { id: id, payload: payload }
639 | end
640 | end
641 |
642 | ATestWorker.perform_async jobs
643 | end
644 | end
645 | ```
646 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | [](https://badge.fury.io/rb/lowkiq)
2 |
3 | # Lowkiq
4 |
5 | Ordered background jobs processing
6 |
7 | 
8 |
9 | * [Rationale](#rationale)
10 | * [Description](#description)
11 | * [Sidekiq comparison](#sidekiq-comparison)
12 | * [Queue](#queue)
13 | + [Calculation algorithm for `retry_count` and `perform_in`](#calculation-algorithm-for-retry_count-and-perform_in)
14 | + [Job merging rules](#job-merging-rules)
15 | * [Install](#install)
16 | * [Api](#api)
17 | * [Ring app](#ring-app)
18 | * [Configuration](#configuration)
19 | * [Performance](#performance)
20 | * [Execution](#execution)
21 | * [Shutdown](#shutdown)
22 | * [Debug](#debug)
23 | * [Development](#development)
24 | * [Exceptions](#exceptions)
25 | * [Rails integration](#rails-integration)
26 | * [Splitter](#splitter)
27 | * [Scheduler](#scheduler)
28 | * [Recommendations on configuration](#recommendations-on-configuration)
29 | + [`SomeWorker.shards_count`](#someworkershards_count)
30 | + [`SomeWorker.max_retry_count`](#someworkermax_retry_count)
31 | * [Changing of worker's shards amount](#changing-of-workers-shards-amount)
32 | * [Extended error info](#extended-error-info)
33 |
34 | ## Rationale
35 |
36 | We've faced some problems using Sidekiq while processing messages from a side system.
37 | For instance, the message is the data of an order at a particular time.
38 | The side system will send new data of an order on every change.
39 | Orders are frequently updated and a queue contains some closely located messages of the same order.
40 |
41 | Sidekiq doesn't guarantee a strict message order, because a queue is processed by multiple threads.
42 | For example, we've received 2 messages: M1 and M2.
43 | Sidekiq handlers begin to process them parallel,
44 | so M2 can be processed before M1.
45 |
46 | Parallel processing of such kind of messages can result in:
47 |
48 | + deadlocks
49 | + overwriting new data with an old one
50 |
51 | Lowkiq has been created to eliminate such problems by avoiding parallel task processing within one entity.
52 |
53 | ## Description
54 |
55 | Lowkiq's queues are reliable i.e.,
56 | Lowkiq saves information about a job being processed
57 | and returns uncompleted jobs to the queue on startup.
58 |
59 | Jobs in queues are ordered by preassigned execution time, so they are not FIFO queues.
60 |
61 | Every job has its identifier. Lowkiq guarantees that jobs with equal IDs are processed by the same thread.
62 |
63 | Every queue is divided into a permanent set of shards.
64 | A job is placed into a particular shard based on an id of the job.
65 | So jobs with the same id are always placed into the same shard.
66 | All jobs of the shard are always processed with the same thread.
67 | This guarantees the sequential processing of jobs with the same ids and excludes the possibility of locks.
68 |
69 | Besides the id, every job has a payload.
70 | Payloads are accumulated for jobs with the same id.
71 | So all accumulated payloads will be processed together.
72 | It's useful when you need to process only the last message and drop all previous ones.
73 |
74 | A worker corresponds to a queue and contains a job processing logic.
75 |
76 | The fixed number of threads is used to process all jobs of all queues.
77 | Adding or removing queues or their shards won't affect the number of threads.
78 |
79 | ## Sidekiq comparison
80 |
81 | If Sidekiq is good for your tasks you should use it.
82 | But if you use plugins like
83 | [sidekiq-grouping](https://github.com/gzigzigzeo/sidekiq-grouping),
84 | [sidekiq-unique-jobs](https://github.com/mhenrixon/sidekiq-unique-jobs),
85 | [sidekiq-merger](https://github.com/dtaniwaki/sidekiq-merger)
86 | or implement your own lock system, you should look at Lowkiq.
87 |
88 | For example, sidekiq-grouping accumulates a batch of jobs then enqueues it and accumulates the next batch.
89 | With this approach, a queue can contain two batches with data of the same order.
90 | These batches are parallel processed with different threads, so we come back to the initial problem.
91 |
92 | Lowkiq was designed to avoid any type of locking.
93 |
94 | Furthermore, Lowkiq's queues are reliable. Only Sidekiq Pro or plugins can add such functionality.
95 |
96 | This [benchmark](examples/benchmark) shows overhead on Redis usage.
97 | These are the results for 5 threads, 100,000 blank jobs:
98 |
99 | + lowkiq: 155 sec or 1.55 ms per job
100 | + lowkiq +hiredis: 80 sec or 0.80 ms per job
101 | + sidekiq: 15 sec or 0.15 ms per job
102 |
103 | This difference is related to different queues structure.
104 | Sidekiq uses one list for all workers and fetches the job entirely for O(1).
105 | Lowkiq uses several data structures, including sorted sets for keeping ids of jobs.
106 | So fetching only an id of a job takes O(log(N)).
107 |
108 | ## Queue
109 |
110 | Please, look at [the presentation](https://docs.google.com/presentation/d/e/2PACX-1vRdwA2Ck22r26KV1DbY__XcYpj2FdlnR-2G05w1YULErnJLB_JL1itYbBC6_JbLSPOHwJ0nwvnIHH2A/pub?start=false&loop=false&delayms=3000).
111 |
112 | Every job has the following attributes:
113 |
114 | + `id` is a job identifier with string type.
115 | + `payloads` is a sorted set of payloads ordered by its score. A payload is an object. A score is a real number.
116 | + `perform_in` is planned execution time. It's a Unix timestamp with a real number type.
117 | + `retry_count` is amount of retries. It's a real number.
118 |
119 | For example, `id` can be an identifier of a replicated entity.
120 | `payloads` is a sorted set ordered by a score of payload and resulted by grouping a payload of the job by its `id`.
121 | `payload` can be a ruby object because it is serialized by `Marshal.dump`.
122 | `score` can be `payload`'s creation date (Unix timestamp) or it's an incremental version number.
123 | By default, `score` and `perform_in` are current Unix timestamp.
124 | `retry_count` for new unprocessed job equals to `-1`,
125 | for one-time failed is `0`, so the planned retries are counted, not the performed ones.
126 |
127 | Job execution can be unsuccessful. In this case, its `retry_count` is incremented, the new `perform_in` is calculated with determined formula, and it moves back to a queue.
128 |
129 | In case of `retry_count` is getting `>=` `max_retry_count` an element of `payloads` with less (oldest) score is moved to a morgue,
130 | rest elements are moved back to the queue, wherein `retry_count` and `perform_in` are reset to `-1` and `now()` respectively.
131 |
132 | ### Calculation algorithm for `retry_count` and `perform_in`
133 |
134 | 0. a job's been executed and failed
135 | 1. `retry_count++`
136 | 2. `perform_in = now + retry_in (try_count)`
137 | 3. `if retry_count >= max_retry_count` the job will be moved to a morgue.
138 |
139 | | type | `retry_count` | `perform_in` |
140 | | --- | --- | --- |
141 | | new haven't been executed | -1 | set or `now()` |
142 | | new failed | 0 | `now() + retry_in(0)` |
143 | | retry failed | 1 | `now() + retry_in(1)` |
144 |
145 | If `max_retry_count = 1`, retries stop.
146 |
147 | ### Job merging rules
148 |
149 | They are applied when:
150 |
151 | + a job has been in a queue and a new one with the same id is added
152 | + a job is failed, but a new one with the same id has been added
153 | + a job from a morgue is moved back to a queue, but the queue has had a job with the same id
154 |
155 | Algorithm:
156 |
157 | + payloads are merged, the minimal score is chosen for equal payloads
158 | + if a new job and queued job is merged, `perform_in` and `retry_count` is taken from the job from the queue
159 | + if a failed job and queued job is merged, `perform_in` and `retry_count` is taken from the failed one
160 | + if morgue job and queued job is merged, `perform_in = now()`, `retry_count = -1`
161 |
162 | Example:
163 |
164 | ```
165 | # v1 is the first version and v2 is the second
166 | # #{"v1": 1} is a sorted set of a single element, the payload is "v1", the score is 1
167 |
168 | # a job is in a queue
169 | { id: "1", payloads: #{"v1": 1, "v2": 2}, retry_count: 0, perform_in: 1536323288 }
170 | # a job which is being added
171 | { id: "1", payloads: #{"v2": 3, "v3": 4}, retry_count: -1, perform_in: 1536323290 }
172 |
173 | # a resulted job in the queue
174 | { id: "1", payloads: #{"v1": 1, "v2": 3, "v3": 4}, retry_count: 0, perform_in: 1536323288 }
175 | ```
176 |
177 | A morgue is a part of a queue. Jobs in a morgue are not processed.
178 | A job in a morgue has the following attributes:
179 |
180 | + id is the job identifier
181 | + payloads
182 |
183 | A job from morgue can be moved back to the queue, `retry_count` = 0 and `perform_in = now()` would be set.
184 |
185 | ## Install
186 |
187 | ```
188 | # Gemfile
189 |
190 | gem 'lowkiq'
191 | ```
192 |
193 | Redis >= 3.2
194 |
195 | ## Api
196 |
197 | ```ruby
198 | module ATestWorker
199 | extend Lowkiq::Worker
200 |
201 | self.shards_count = 24
202 | self.batch_size = 10
203 | self.max_retry_count = 5
204 |
205 | def self.retry_in(count)
206 | 10 * (count + 1) # (i.e. 10, 20, 30, 40, 50)
207 | end
208 |
209 | def self.retries_exhausted(batch)
210 | batch.each do |job|
211 | Rails.logger.info "retries exhausted for #{name} with error #{job[:error]}"
212 | end
213 | end
214 |
215 | def self.perform(payloads_by_id)
216 | # payloads_by_id is a hash map
217 | payloads_by_id.each do |id, payloads|
218 | # payloads are sorted by score, from old to new (min to max)
219 | payloads.each do |payload|
220 | do_some_work(id, payload)
221 | end
222 | end
223 | end
224 | end
225 | ```
226 |
227 | And then you have to add it to Lowkiq in your initializer file due to problems with autoloading:
228 |
229 | ```ruby
230 | Lowkiq.workers = [ ATestWorker ]
231 | ```
232 |
233 | Default values:
234 |
235 | ```ruby
236 | self.shards_count = 5
237 | self.batch_size = 1
238 | self.max_retry_count = 25
239 | self.queue_name = self.name
240 |
241 | # i.e. 15, 16, 31, 96, 271, ... seconds + a random amount of time
242 | def retry_in(retry_count)
243 | (retry_count ** 4) + 15 + (rand(30) * (retry_count + 1))
244 | end
245 | ```
246 |
247 | ```ruby
248 | ATestWorker.perform_async [
249 | { id: 0 },
250 | { id: 1, payload: { attr: 'v1' } },
251 | { id: 2, payload: { attr: 'v1' }, score: Time.now.to_f, perform_in: Time.now.to_f },
252 | ]
253 | # payload by default equals to ""
254 | # score and perform_in by default equals to Time.now.to_f
255 | ```
256 |
257 | It is possible to redefine `perform_async` and calculate `id`, `score` и `perform_in` in a worker code:
258 |
259 | ```ruby
260 | module ATestWorker
261 | extend Lowkiq::Worker
262 |
263 | def self.perform_async(jobs)
264 | jobs.each do |job|
265 | job.merge! id: job[:payload][:id]
266 | end
267 | super
268 | end
269 |
270 | def self.perform(payloads_by_id)
271 | #...
272 | end
273 | end
274 |
275 | ATestWorker.perform_async 1000.times.map { |id| { payload: {id: id} } }
276 | ```
277 |
278 | ## Ring app
279 |
280 | `Lowkiq::Web` - a ring app.
281 |
282 | + `/` - a dashboard
283 | + `/api/v1/stats` - queue length, morgue length, lag for each worker and total result
284 |
285 | ## Configuration
286 |
287 | Options and their default values are:
288 |
289 | + `Lowkiq.workers = []`- list of workers to use. Since 1.1.0.
290 | + `Lowkiq.poll_interval = 1` - delay in seconds between queue polling for new jobs.
291 | Used only if a queue was empty in a previous cycle or an error occurred.
292 | + `Lowkiq.threads_per_node = 5` - threads per node.
293 | + `Lowkiq.redis = ->() { Redis.new url: ENV.fetch('REDIS_URL') }` - redis connection options
294 | + `Lowkiq.client_pool_size = 5` - redis pool size for queueing jobs
295 | + `Lowkiq.pool_timeout = 5` - client and server redis pool timeout
296 | + `Lowkiq.server_middlewares = []` - a middleware list, used when job is processed
297 | + `Lowkiq.client_middlewares = []` - a middleware list, used when job is enqueued
298 | + `Lowkiq.on_server_init = ->() {}` - a lambda is being executed when server inits
299 | + `Lowkiq.build_scheduler = ->() { Lowkiq.build_lag_scheduler }` is a scheduler
300 | + `Lowkiq.build_splitter = ->() { Lowkiq.build_default_splitter }` is a splitter
301 | + `Lowkiq.last_words = ->(ex) {}` is an exception handler of descendants of `StandardError` caused the process stop
302 | + `Lowkiq.dump_payload = Marshal.method :dump`
303 | + `Lowkiq.load_payload = Marshal.method :load`
304 |
305 | + `Lowkiq.format_error = -> (error) { error.message }` can be used to add error backtrace. Please see [Extended error info](#extended-error-info)
306 | + `Lowkiq.dump_error = -> (msg) { msg }` can be used to implement a custom compression logic for errors. Recommended when using `Lowkiq.format_error`.
307 | + `Lowkiq.load_error = -> (msg) { msg }` can be used to implement a custom decompression logic for errors.
308 |
309 | ```ruby
310 | $logger = Logger.new(STDOUT)
311 |
312 | Lowkiq.server_middlewares << -> (worker, batch, &block) do
313 | $logger.info "Started job for #{worker} #{batch}"
314 | block.call
315 | $logger.info "Finished job for #{worker} #{batch}"
316 | end
317 |
318 | Lowkiq.server_middlewares << -> (worker, batch, &block) do
319 | begin
320 | block.call
321 | rescue => e
322 | $logger.error "#{e.message} #{worker} #{batch}"
323 | raise e
324 | end
325 | end
326 |
327 | Lowkiq.client_middlewares << -> (worker, batch, &block) do
328 | $logger.info "Enqueueing job for #{worker} #{batch}"
329 | block.call
330 | $logger.info "Enqueued job for #{worker} #{batch}"
331 | end
332 |
333 | ```
334 |
335 | ## Performance
336 |
337 | Use [hiredis](https://github.com/redis/hiredis-rb) for better performance.
338 |
339 | ```ruby
340 | # Gemfile
341 |
342 | gem "hiredis"
343 | ```
344 |
345 | ```ruby
346 | # config
347 |
348 | Lowkiq.redis = ->() { Redis.new url: ENV.fetch('REDIS_URL'), driver: :hiredis }
349 | ```
350 |
351 | ## Execution
352 |
353 | `lowkiq -r ./path_to_app`
354 |
355 | `path_to_app.rb` must load app. [Example](examples/dummy/lib/app.rb).
356 |
357 | The lazy loading of worker modules is unacceptable.
358 | For preliminarily loading modules use
359 | `require`
360 | or [`require_dependency`](https://api.rubyonrails.org/classes/ActiveSupport/Dependencies/Loadable.html#method-i-require_dependency)
361 | for Ruby on Rails.
362 |
363 | ## Shutdown
364 |
365 | Send TERM or INT signal to the process (Ctrl-C).
366 | The process will wait for executed jobs to finish.
367 |
368 | Note that if a queue is empty, the process sleeps `poll_interval` seconds,
369 | therefore, the process will not stop until the `poll_interval` seconds have passed.
370 |
371 | ## Debug
372 |
373 | To get trace of all threads of an app:
374 |
375 | ```
376 | kill -TTIN
377 | cat /tmp/lowkiq_ttin.txt
378 | ```
379 |
380 | ## Development
381 |
382 | ```
383 | docker-compose run --rm --service-port app bash
384 | bundle
385 | rspec
386 | cd examples/dummy ; bundle exec ../../exe/lowkiq -r ./lib/app.rb
387 |
388 | # open localhost:8080
389 | ```
390 |
391 | ```
392 | docker-compose run --rm --service-port frontend bash
393 | npm run dumb
394 | # open localhost:8081
395 |
396 | # npm run build
397 | # npm run web-api
398 | ```
399 |
400 | ## Exceptions
401 |
402 | `StandardError` thrown by a worker are handled with middleware. Such exceptions don't lead to process stops.
403 |
404 | All other exceptions cause the process to stop.
405 | Lowkiq will wait for job execution by other threads.
406 |
407 | `StandardError` thrown outside of worker are passed to `Lowkiq.last_words`.
408 | For example, it can happen when Redis connection is lost or when Lowkiq's code has a bug.
409 |
410 | ## Rails integration
411 |
412 | ```ruby
413 | # config/routes.rb
414 |
415 | Rails.application.routes.draw do
416 | # ...
417 | mount Lowkiq::Web => '/lowkiq'
418 | # ...
419 | end
420 | ```
421 |
422 | ```ruby
423 | # config/initializers/lowkiq.rb
424 |
425 | # configuration:
426 | # Lowkiq.redis = -> { Redis.new url: ENV.fetch('LOWKIQ_REDIS_URL') }
427 | # Lowkiq.threads_per_node = ENV.fetch('LOWKIQ_THREADS_PER_NODE').to_i
428 | # Lowkiq.client_pool_size = ENV.fetch('LOWKIQ_CLIENT_POOL_SIZE').to_i
429 | # ...
430 |
431 | # since 1.1.0
432 | Lowkiq.workers = [
433 | ATestWorker,
434 | OtherCoolWorker
435 | ]
436 |
437 | Lowkiq.server_middlewares << -> (worker, batch, &block) do
438 | logger = Rails.logger
439 | tag = "#{worker}-#{Thread.current.object_id}"
440 |
441 | logger.tagged(tag) do
442 | time_start = Time.now
443 | logger.info "#{time_start} Started job, batch: #{batch}"
444 | begin
445 | block.call
446 | rescue => e
447 | logger.error e.message
448 | raise e
449 | ensure
450 | time_end = Time.now
451 | logger.info "#{time_end} Finished job, duration: #{time_end - time_start} sec"
452 | end
453 | end
454 | end
455 |
456 | # Sentry integration
457 | Lowkiq.server_middlewares << -> (worker, batch, &block) do
458 | opts = {
459 | extra: {
460 | lowkiq: {
461 | worker: worker.name,
462 | batch: batch,
463 | }
464 | }
465 | }
466 |
467 | Raven.capture opts do
468 | block.call
469 | end
470 | end
471 |
472 | # NewRelic integration
473 | if defined? NewRelic
474 | class NewRelicLowkiqMiddleware
475 | include NewRelic::Agent::Instrumentation::ControllerInstrumentation
476 |
477 | def call(worker, batch, &block)
478 | opts = {
479 | category: 'OtherTransaction/LowkiqJob',
480 | class_name: worker.name,
481 | name: :perform,
482 | }
483 |
484 | perform_action_with_newrelic_trace opts do
485 | block.call
486 | end
487 | end
488 | end
489 |
490 | Lowkiq.server_middlewares << NewRelicLowkiqMiddleware.new
491 | end
492 |
493 | # Rails reloader, responsible for cleaning of ActiveRecord connections
494 | Lowkiq.server_middlewares << -> (worker, batch, &block) do
495 | Rails.application.reloader.wrap do
496 | block.call
497 | end
498 | end
499 |
500 | Lowkiq.on_server_init = ->() do
501 | [[ActiveRecord::Base, ActiveRecord::Base.configurations[Rails.env]]].each do |(klass, init_config)|
502 | klass.connection_pool.disconnect!
503 | config = init_config.merge 'pool' => Lowkiq.threads_per_node
504 | klass.establish_connection(config)
505 | end
506 | end
507 | ```
508 |
509 | Note: In Rails 7, the worker files wouldn't be loaded by default in the initializers since they are managed by the `main` autoloader. To solve this, we can wrap setting the workers around the `to_prepare` configuration.
510 |
511 | ```ruby
512 | Rails.application.config.to_prepare do
513 | Lowkiq.workers = [
514 | ATestWorker,
515 | OtherCoolWorker
516 | ]
517 | end
518 | ```
519 |
520 | Execution: `bundle exec lowkiq -r ./config/environment.rb`
521 |
522 |
523 | ## Splitter
524 |
525 | Each worker has several shards:
526 |
527 | ```
528 | # worker: shard ids
529 | worker A: 0, 1, 2
530 | worker B: 0, 1, 2, 3
531 | worker C: 0
532 | worker D: 0, 1
533 | ```
534 |
535 | Lowkiq uses a fixed number of threads for job processing, therefore it is necessary to distribute shards between threads.
536 | Splitter does it.
537 |
538 | To define a set of shards, which is being processed by a thread, let's move them to one list:
539 |
540 | ```
541 | A0, A1, A2, B0, B1, B2, B3, C0, D0, D1
542 | ```
543 |
544 | Default splitter evenly distributes shards by threads of a single node.
545 |
546 | If `threads_per_node` is set to 3, the distribution will be:
547 |
548 | ```
549 | # thread id: shards
550 | t0: A0, B0, B3, D1
551 | t1: A1, B1, C0
552 | t2: A2, B2, D0
553 | ```
554 |
555 | Besides Default Lowkiq has the ByNode splitter. It allows dividing the load by several processes (nodes).
556 |
557 | ```
558 | Lowkiq.build_splitter = -> () do
559 | Lowkiq.build_by_node_splitter(
560 | ENV.fetch('LOWKIQ_NUMBER_OF_NODES').to_i,
561 | ENV.fetch('LOWKIQ_NODE_NUMBER').to_i
562 | )
563 | end
564 | ```
565 |
566 | So, instead of a single process, you need to execute multiple ones and to set environment variables up:
567 |
568 | ```
569 | # process 0
570 | LOWKIQ_NUMBER_OF_NODES=2 LOWKIQ_NODE_NUMBER=0 bundle exec lowkiq -r ./lib/app.rb
571 |
572 | # process 1
573 | LOWKIQ_NUMBER_OF_NODES=2 LOWKIQ_NODE_NUMBER=1 bundle exec lowkiq -r ./lib/app.rb
574 | ```
575 |
576 | Summary amount of threads are equal product of `ENV.fetch('LOWKIQ_NUMBER_OF_NODES')` and `Lowkiq.threads_per_node`.
577 |
578 | You can also write your own splitter if your app needs an extra distribution of shards between threads or nodes.
579 |
580 | ## Scheduler
581 |
582 | Every thread processes a set of shards. The scheduler selects shard for processing.
583 | Every thread has its own instance of the scheduler.
584 |
585 | Lowkiq has 2 schedulers for your choice.
586 | `Seq` sequentially looks over shards.
587 | `Lag` chooses shard with the oldest job minimizing the lag. It's used by default.
588 |
589 | The scheduler can be set up through settings:
590 |
591 | ```
592 | Lowkiq.build_scheduler = ->() { Lowkiq.build_seq_scheduler }
593 | # or
594 | Lowkiq.build_scheduler = ->() { Lowkiq.build_lag_scheduler }
595 | ```
596 |
597 | ## Recommendations on configuration
598 |
599 | ### `SomeWorker.shards_count`
600 |
601 | Sum of `shards_count` of all workers shouldn't be less than `Lowkiq.threads_per_node`
602 | otherwise, threads will stay idle.
603 |
604 | Sum of `shards_count` of all workers can be equal to `Lowkiq.threads_per_node`.
605 | In this case, a thread processes a single shard. This makes sense only with a uniform queue load.
606 |
607 | Sum of `shards_count` of all workers can be more than `Lowkiq.threads_per_node`.
608 | In this case, `shards_count` can be counted as a priority.
609 | The larger it is, the more often the tasks of this queue will be processed.
610 |
611 | There is no reason to set `shards_count` of one worker more than `Lowkiq.threads_per_node`,
612 | because every thread will handle more than one shard from this queue, so it increases the overhead.
613 |
614 | ### `SomeWorker.max_retry_count`
615 |
616 | From `retry_in` and `max_retry_count`, you can calculate the approximate time that a payload of a job will be in a queue.
617 | After `max_retry_count` is reached a payload with a minimal score will be moved to a morgue.
618 |
619 | For default `retry_in` we receive the following table.
620 |
621 | ```ruby
622 | def retry_in(retry_count)
623 | (retry_count ** 4) + 15 + (rand(30) * (retry_count + 1))
624 | end
625 | ```
626 |
627 | | `max_retry_count` | amount of days of job's life |
628 | | --- | --- |
629 | | 14 | 1 |
630 | | 16 | 2 |
631 | | 18 | 3 |
632 | | 19 | 5 |
633 | | 20 | 6 |
634 | | 21 | 8 |
635 | | 22 | 10 |
636 | | 23 | 13 |
637 | | 24 | 16 |
638 | | 25 | 20 |
639 |
640 | `(0...25).map{ |c| retry_in c }.sum / 60 / 60 / 24`
641 |
642 |
643 | ## Changing of worker's shards amount
644 |
645 | Try to count the number of shards right away and don't change it in the future.
646 |
647 | If you can disable adding of new jobs, wait for queues to get empty, and deploy the new version of code with a changed amount of shards.
648 |
649 | If you can't do it, follow the next steps:
650 |
651 | A worker example:
652 |
653 | ```ruby
654 | module ATestWorker
655 | extend Lowkiq::Worker
656 |
657 | self.shards_count = 5
658 |
659 | def self.perform(payloads_by_id)
660 | some_code
661 | end
662 | end
663 | ```
664 |
665 | Set the number of shards and the new queue name:
666 |
667 | ```ruby
668 | module ATestWorker
669 | extend Lowkiq::Worker
670 |
671 | self.shards_count = 10
672 | self.queue_name = "#{self.name}_V2"
673 |
674 | def self.perform(payloads_by_id)
675 | some_code
676 | end
677 | end
678 | ```
679 |
680 | Add a worker moving jobs from the old queue to the new one:
681 |
682 | ```ruby
683 | module ATestMigrationWorker
684 | extend Lowkiq::Worker
685 |
686 | self.shards_count = 5
687 | self.queue_name = "ATestWorker"
688 |
689 | def self.perform(payloads_by_id)
690 | jobs = payloads_by_id.each_with_object([]) do |(id, payloads), acc|
691 | payloads.each do |payload|
692 | acc << { id: id, payload: payload }
693 | end
694 | end
695 |
696 | ATestWorker.perform_async jobs
697 | end
698 | end
699 | ```
700 |
701 | ## Extended error info
702 | For failed jobs, lowkiq only stores `error.message` by default. This can be configured by using `Lowkiq.format_error` setting.
703 | `Lowkiq.dump` and `Lowkiq.load_error` can be used to compress and decompress the error messages respectively.
704 | Example:
705 | ```ruby
706 | Lowkiq.format_error = -> (error) { error.full_message(highlight: false) }
707 |
708 | Lowkiq.dump_error = Proc.new do |msg|
709 | compressed = Zlib::Deflate.deflate(msg.to_s)
710 | Base64.encode64(compressed)
711 | end
712 |
713 | Lowkiq.load_error = Proc.new do |input|
714 | decoded = Base64.decode64(input)
715 | Zlib::Inflate.inflate(decoded)
716 | rescue
717 | input
718 | end
719 | ```
720 |
--------------------------------------------------------------------------------