diff --git a/.rubocop.yml b/.rubocop.yml index 3cdfba4..65e4886 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -20,3 +20,6 @@ Rails/EnvironmentVariableAccess: RSpec/IndexedLet: Enabled: false + +Capybara/RSpec/PredicateMatcher: + Enabled: false diff --git a/.rubocop_todo.yml b/.rubocop_todo.yml index 2448f2c..96b4d15 100644 --- a/.rubocop_todo.yml +++ b/.rubocop_todo.yml @@ -1,16 +1,17 @@ # This configuration was generated by # `rubocop --auto-gen-config --auto-gen-only-exclude --no-exclude-limit --no-auto-gen-timestamp` -# using RuboCop version 1.71.1. +# using RuboCop version 1.75.1. # The point is for the user to remove these configuration records # one by one as the offenses are removed from the code base. # Note that changes in the inspected code, or installation of new # versions of RuboCop, may require this file to be generated again. -# Offense count: 13 +# Offense count: 14 Betterment/ActiveJobPerformable: Exclude: - 'lib/delayed/performable_mailer.rb' - 'lib/delayed/performable_method.rb' + - 'lib/delayed/worker.rb' - 'spec/autoloaded/clazz.rb' - 'spec/autoloaded/instance_clazz.rb' - 'spec/autoloaded/instance_struct.rb' @@ -29,6 +30,12 @@ Betterment/DirectDelayedEnqueue: - 'spec/performable_method_spec.rb' - 'spec/worker_spec.rb' +# Offense count: 19 +Betterment/EnvironmentConfiguration: + Exclude: + - 'Rakefile' + - 'lib/delayed/tasks.rb' + # Offense count: 1 # This cop supports safe autocorrection (--autocorrect). # Configuration parameters: Severity, Include. diff --git a/Gemfile.lock b/Gemfile.lock index 1b667aa..68f10cd 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,69 +1,66 @@ PATH remote: . specs: - delayed (0.7.1) + delayed (0.7.2) activerecord (>= 5.2) concurrent-ruby GEM remote: https://rubygems.org/ specs: - actionmailer (7.1.5.1) - actionpack (= 7.1.5.1) - actionview (= 7.1.5.1) - activejob (= 7.1.5.1) - activesupport (= 7.1.5.1) - mail (~> 2.5, >= 2.5.4) - net-imap - net-pop - net-smtp + actionmailer (8.0.2) + actionpack (= 8.0.2) + actionview (= 8.0.2) + activejob (= 8.0.2) + activesupport (= 8.0.2) + mail (>= 2.8.0) rails-dom-testing (~> 2.2) - actionpack (7.1.5.1) - actionview (= 7.1.5.1) - activesupport (= 7.1.5.1) + actionpack (8.0.2) + actionview (= 8.0.2) + activesupport (= 8.0.2) nokogiri (>= 1.8.5) - racc rack (>= 2.2.4) rack-session (>= 1.0.1) rack-test (>= 0.6.3) rails-dom-testing (~> 2.2) rails-html-sanitizer (~> 1.6) - actionview (7.1.5.1) - activesupport (= 7.1.5.1) + useragent (~> 0.16) + actionview (8.0.2) + activesupport (= 8.0.2) builder (~> 3.1) erubi (~> 1.11) rails-dom-testing (~> 2.2) rails-html-sanitizer (~> 1.6) - activejob (7.1.5.1) - activesupport (= 7.1.5.1) + activejob (8.0.2) + activesupport (= 8.0.2) globalid (>= 0.3.6) - activemodel (7.1.5.1) - activesupport (= 7.1.5.1) - activerecord (7.1.5.1) - activemodel (= 7.1.5.1) - activesupport (= 7.1.5.1) + activemodel (8.0.2) + activesupport (= 8.0.2) + activerecord (8.0.2) + activemodel (= 8.0.2) + activesupport (= 8.0.2) timeout (>= 0.4.0) - activesupport (7.1.5.1) + activesupport (8.0.2) base64 benchmark (>= 0.3) bigdecimal - concurrent-ruby (~> 1.0, >= 1.0.2) + concurrent-ruby (~> 1.0, >= 1.3.1) connection_pool (>= 2.2.5) drb i18n (>= 1.6, < 2) logger (>= 1.4.2) minitest (>= 5.1) - mutex_m securerandom (>= 0.3) - tzinfo (~> 2.0) + tzinfo (~> 2.0, >= 2.0.5) + uri (>= 0.13.1) appraisal (2.5.0) bundler rake thor (>= 0.14.0) - ast (2.4.2) + ast (2.4.3) base64 (0.2.0) benchmark (0.4.0) - betterlint (1.18.0) + betterlint (1.19.0) rubocop (~> 1.71) rubocop-graphql (~> 1.5) rubocop-performance (~> 1.23) @@ -71,22 +68,23 @@ GEM rubocop-rake (~> 0.6) rubocop-rspec (~> 2.29) bigdecimal (3.1.9) - builder (3.2.4) - concurrent-ruby (1.2.3) - connection_pool (2.4.1) + builder (3.3.0) + concurrent-ruby (1.3.5) + connection_pool (2.5.0) crass (1.0.6) - date (3.3.4) - diff-lcs (1.5.1) + date (3.4.1) + diff-lcs (1.6.1) drb (2.2.1) - erubi (1.12.0) + erubi (1.13.1) globalid (1.2.1) activesupport (>= 6.1) - i18n (1.14.6) + i18n (1.14.7) concurrent-ruby (~> 1.0) - json (2.7.2) - language_server-protocol (3.17.0.3) - logger (1.6.5) - loofah (2.22.0) + json (2.10.2) + language_server-protocol (3.17.0.4) + lint_roller (1.1.0) + logger (1.7.0) + loofah (2.24.0) crass (~> 1.0.2) nokogiri (>= 1.12.0) mail (2.8.1) @@ -95,40 +93,45 @@ GEM net-pop net-smtp mini_mime (1.1.5) - minitest (5.22.3) - mutex_m (0.2.0) + minitest (5.25.5) mysql2 (0.5.6) - net-imap (0.4.18) + net-imap (0.5.6) date net-protocol net-pop (0.1.2) net-protocol net-protocol (0.2.2) timeout - net-smtp (0.5.0) + net-smtp (0.5.1) net-protocol - nokogiri (1.16.8-aarch64-linux) + nokogiri (1.18.6-aarch64-linux-gnu) + racc (~> 1.4) + nokogiri (1.18.6-aarch64-linux-musl) racc (~> 1.4) - nokogiri (1.16.8-arm-linux) + nokogiri (1.18.6-arm-linux-gnu) racc (~> 1.4) - nokogiri (1.16.8-arm64-darwin) + nokogiri (1.18.6-arm-linux-musl) racc (~> 1.4) - nokogiri (1.16.8-x86-linux) + nokogiri (1.18.6-arm64-darwin) racc (~> 1.4) - nokogiri (1.16.8-x86_64-darwin) + nokogiri (1.18.6-x86_64-darwin) racc (~> 1.4) - nokogiri (1.16.8-x86_64-linux) + nokogiri (1.18.6-x86_64-linux-gnu) racc (~> 1.4) - parallel (1.24.0) - parser (3.3.7.0) + nokogiri (1.18.6-x86_64-linux-musl) + racc (~> 1.4) + parallel (1.26.3) + parser (3.3.7.3) ast (~> 2.4.1) racc - pg (1.5.6) + pg (1.5.9) + prism (1.4.0) racc (1.8.1) - rack (3.0.11) - rack-session (2.0.0) + rack (3.1.12) + rack-session (2.1.0) + base64 (>= 0.1.0) rack (>= 3.0.0) - rack-test (2.1.0) + rack-test (2.2.0) rack (>= 1.3) rails-dom-testing (2.2.0) activesupport (>= 5.0.0) @@ -144,73 +147,89 @@ GEM rspec-core (~> 3.13.0) rspec-expectations (~> 3.13.0) rspec-mocks (~> 3.13.0) - rspec-core (3.13.0) + rspec-core (3.13.3) rspec-support (~> 3.13.0) - rspec-expectations (3.13.0) + rspec-expectations (3.13.3) diff-lcs (>= 1.2.0, < 2.0) rspec-support (~> 3.13.0) - rspec-mocks (3.13.0) + rspec-mocks (3.13.2) diff-lcs (>= 1.2.0, < 2.0) rspec-support (~> 3.13.0) - rspec-support (3.13.1) - rubocop (1.71.1) + rspec-support (3.13.2) + rubocop (1.75.1) json (~> 2.3) - language_server-protocol (>= 3.17.0) + language_server-protocol (~> 3.17.0.2) + lint_roller (~> 1.1.0) parallel (~> 1.10) parser (>= 3.3.0.2) rainbow (>= 2.2.2, < 4.0) regexp_parser (>= 2.9.3, < 3.0) - rubocop-ast (>= 1.38.0, < 2.0) + rubocop-ast (>= 1.43.0, < 2.0) ruby-progressbar (~> 1.7) unicode-display_width (>= 2.4.0, < 4.0) - rubocop-ast (1.38.0) - parser (>= 3.3.1.0) - rubocop-capybara (2.20.0) - rubocop (~> 1.41) - rubocop-factory_bot (2.25.1) - rubocop (~> 1.41) - rubocop-graphql (1.5.4) - rubocop (>= 1.50, < 2) - rubocop-performance (1.23.1) - rubocop (>= 1.48.1, < 2.0) - rubocop-ast (>= 1.31.1, < 2.0) - rubocop-rails (2.29.1) + rubocop-ast (1.43.0) + parser (>= 3.3.7.2) + prism (~> 1.4) + rubocop-capybara (2.22.1) + lint_roller (~> 1.1) + rubocop (~> 1.72, >= 1.72.1) + rubocop-factory_bot (2.27.1) + lint_roller (~> 1.1) + rubocop (~> 1.72, >= 1.72.1) + rubocop-graphql (1.5.5) + lint_roller (~> 1.1) + rubocop (>= 1.72.1, < 2) + rubocop-performance (1.24.0) + lint_roller (~> 1.1) + rubocop (>= 1.72.1, < 2.0) + rubocop-ast (>= 1.38.0, < 2.0) + rubocop-rails (2.30.3) activesupport (>= 4.2.0) + lint_roller (~> 1.1) rack (>= 1.1) - rubocop (>= 1.52.0, < 2.0) - rubocop-ast (>= 1.31.1, < 2.0) - rubocop-rake (0.6.0) - rubocop (~> 1.0) + rubocop (>= 1.72.1, < 2.0) + rubocop-ast (>= 1.38.0, < 2.0) + rubocop-rake (0.7.1) + lint_roller (~> 1.1) + rubocop (>= 1.72.1) rubocop-rspec (2.31.0) rubocop (~> 1.40) rubocop-capybara (~> 2.17) rubocop-factory_bot (~> 2.22) rubocop-rspec_rails (~> 2.28) - rubocop-rspec_rails (2.28.2) - rubocop (~> 1.40) + rubocop-rspec_rails (2.29.1) + rubocop (~> 1.61) ruby-progressbar (1.13.0) securerandom (0.4.1) - sqlite3 (2.5.0-aarch64-linux-gnu) - sqlite3 (2.5.0-arm-linux-gnu) - sqlite3 (2.5.0-arm64-darwin) - sqlite3 (2.5.0-x86-linux-gnu) - sqlite3 (2.5.0-x86_64-darwin) - sqlite3 (2.5.0-x86_64-linux-gnu) - thor (1.3.1) - timecop (0.9.8) + sqlite3 (2.6.0-aarch64-linux-gnu) + sqlite3 (2.6.0-aarch64-linux-musl) + sqlite3 (2.6.0-arm-linux-gnu) + sqlite3 (2.6.0-arm-linux-musl) + sqlite3 (2.6.0-arm64-darwin) + sqlite3 (2.6.0-x86_64-darwin) + sqlite3 (2.6.0-x86_64-linux-gnu) + sqlite3 (2.6.0-x86_64-linux-musl) + thor (1.3.2) + timecop (0.9.10) timeout (0.4.3) tzinfo (2.0.6) concurrent-ruby (~> 1.0) - unicode-display_width (2.5.0) - zeitwerk (2.6.13) + unicode-display_width (3.1.4) + unicode-emoji (~> 4.0, >= 4.0.4) + unicode-emoji (4.0.4) + uri (1.0.3) + useragent (0.16.11) + zeitwerk (2.7.2) PLATFORMS - aarch64-linux - arm-linux + aarch64-linux-gnu + aarch64-linux-musl + arm-linux-gnu + arm-linux-musl arm64-darwin - x86-linux x86_64-darwin - x86_64-linux + x86_64-linux-gnu + x86_64-linux-musl DEPENDENCIES actionmailer @@ -228,4 +247,4 @@ DEPENDENCIES zeitwerk BUNDLED WITH - 2.6.3 + 2.6.6 diff --git a/delayed.gemspec b/delayed.gemspec index e5b3346..baeefbc 100644 --- a/delayed.gemspec +++ b/delayed.gemspec @@ -18,7 +18,7 @@ Gem::Specification.new do |spec| spec.require_paths = ['lib'] spec.summary = 'a multi-threaded, SQL-driven ActiveJob backend used at Betterment to process millions of background jobs per day' - spec.version = '0.7.1' + spec.version = '0.7.2' spec.metadata = { 'changelog_uri' => 'https://github.com/betterment/delayed/blob/main/CHANGELOG.md', 'bug_tracker_uri' => 'https://github.com/betterment/delayed/issues', diff --git a/lib/delayed.rb b/lib/delayed.rb index 2fb9782..0f50aad 100644 --- a/lib/delayed.rb +++ b/lib/delayed.rb @@ -42,8 +42,8 @@ module Delayed mattr_accessor(:default_log_level) { 'info'.freeze } mattr_accessor(:plugins) do [ - Delayed::Plugins::Instrumentation, Delayed::Plugins::Connection, + Delayed::Plugins::Instrumentation, ] end diff --git a/lib/delayed/worker.rb b/lib/delayed/worker.rb index c8cff37..ae26da2 100644 --- a/lib/delayed/worker.rb +++ b/lib/delayed/worker.rb @@ -101,7 +101,12 @@ def work_off(num = 100) pool = Concurrent::FixedThreadPool.new(jobs.length) jobs.each do |job| pool.post do - success.increment if run_job(job) + self.class.lifecycle.run_callbacks(:thread, self, job) do + success.increment if perform(job) + end + rescue Exception => e # rubocop:disable Lint/RescueException + job_say job, "Job thread crashed with #{e.class.name}: #{e.message}", 'error' + job.error = e end end @@ -117,12 +122,8 @@ def work_off(num = 100) [success.value, total - success.value] end - def run_thread_callbacks(job, &block) - self.class.lifecycle.run_callbacks(:thread, self, job, &block) - end - - def run(job) - run_thread_callbacks(job) do + def perform(job) + self.class.lifecycle.run_callbacks(:perform, self, job) do metadata = { status: 'RUNNING', name: job.name, @@ -209,10 +210,6 @@ def handle_failed_job(job, error) reschedule(job) end - def run_job(job) - self.class.lifecycle.run_callbacks(:perform, self, job) { run(job) } - end - # The backend adapter may return either a list or a single job # In some backends, this can be controlled with the `max_claims` config # Either way, we map this to an array of job instances diff --git a/spec/delayed/job_spec.rb b/spec/delayed/job_spec.rb index 7260970..8f8b059 100644 --- a/spec/delayed/job_spec.rb +++ b/spec/delayed/job_spec.rb @@ -533,7 +533,7 @@ def create_job(opts = {}) it 'fails after Worker.max_run_time' do Delayed::Worker.max_run_time = 1.second job = described_class.create payload_object: LongRunningJob.new - worker.run(job) + worker.perform(job) expect(job.error).not_to be_nil expect(job.reload.last_error).to match(/expired/) expect(job.reload.last_error).to match(/Delayed::Worker\.max_run_time is only 1 second/) @@ -558,7 +558,7 @@ def create_job(opts = {}) it 'records last_error when destroy_failed_jobs = false, max_attempts = 1' do Delayed::Worker.max_attempts = 1 - worker.run(@job) + worker.perform(@job) @job.reload expect(@job.error).not_to be_nil expect(@job.last_error).to match(/did not work/) @@ -580,7 +580,7 @@ def create_job(opts = {}) it 're-schedules jobs with handler provided time if present' do job = described_class.enqueue(CustomRescheduleJob.new(99.minutes)) - worker.run(job) + worker.perform(job) job.reload expect((described_class.db_time_now + 99.minutes - job.run_at).abs).to be < 1 @@ -590,7 +590,7 @@ def create_job(opts = {}) error_with_nil_message = StandardError.new expect(error_with_nil_message).to receive(:message).twice.and_return(nil) expect(@job).to receive(:invoke_job).and_raise error_with_nil_message - expect { worker.run(@job) }.not_to raise_error + expect { worker.perform(@job) }.not_to raise_error end end