Fix lease renewal and preserve claimed fences

CHANGELOG.md

@@ -1,5 +1,14 @@
 # Changelog
 
+## 0.17.4 - 2026-10-12
+
+- Renew activations between turns in a drain pass when renewal is due,
+  including when the renewal thread has not run.
+- Keep the activation fence and expiry while an outstanding message claim
+  remains. The next worker pass recovers an expired turn in mailbox order,
+  including with `max_attempts: 1`. Reclaiming increments `attempt_count`;
+  lease loss alone does not create a dead letter.
+
 ## 0.17.3 - 2026-10-09
 
 - Add four problem guides in `docs/guides/`: preventing race conditions in

Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    solid_objects (0.17.3)
+    solid_objects (0.17.4)
       actioncable (>= 7.1)
       actionpack (>= 7.1)
       actionview (>= 7.1)
@@ -392,7 +392,7 @@ CHECKSUMS
   rubocop-rails-omakase (1.1.0) sha256=2af73ac8ee5852de2919abbd2618af9c15c19b512c4cfc1f9a5d3b6ef009109d
   ruby-progressbar (1.13.0) sha256=80fc9c47a9b640d6834e0dc7b3c94c9df37f08cb072b7761e4a71e22cff29b33
   securerandom (0.4.1) sha256=cc5193d414a4341b6e225f0cb4446aceca8e50d5e1888743fac16987638ea0b1
-  solid_objects (0.17.3)
+  solid_objects (0.17.4)
   sqlite3 (2.9.5-aarch64-linux-gnu) sha256=78075b6337d3d182c6d2b4691049ed45cd220826160c9ea18946bf6a1de200dc
   sqlite3 (2.9.5-aarch64-linux-musl) sha256=18c801185deb4adc01ddb281e8f672a39e3d1729979ca91e39439cd3eac0402d
   sqlite3 (2.9.5-arm-linux-gnu) sha256=1bdfca0c7d63998c60b0f4a8e3c8df2d33800ccc4abd2d612eddbbbc92a4c48b

docs/roadmap.md

@@ -23,7 +23,8 @@
   instead of 12. Concurrent creation causes no deadlock on SQLite, PostgreSQL,
   or MySQL. MySQL needs a shared read after a duplicate key, because it uses
   repeatable read. The tests count statements, and do not measure latency.
-- Activation leases, renewal, unique activation tokens, generations, and
+- Activation leases, renewal between turns as well as in the renewal thread,
+  unique activation tokens, generations, and
   fenced commits
 - Bounded activation passes, idle cache, hot-actor yield, and process records
 - At-least-once retries, terminal domain rejection, strict poison ordering,

lib/solid_objects/activation.rb

@@ -139,6 +139,7 @@ def drain_messages(message_id: nil, deadline: nil)
           break
         end
 
+        renew_lease if processed_count.positive? && lease_renewal_due?
         message = claim_next_message
         break unless message
 

lib/solid_objects/lease.rb

@@ -123,6 +123,7 @@ def release
         instance = Instance.lock.find_by(id: instance_id)
         next false unless instance
         next false unless owned_generation?(instance)
+        next false if ClaimedMessage.where(instance_id:).exists?
 
         instance.update!(
           activation_owner_id: nil,

lib/solid_objects/version.rb

@@ -1,5 +1,5 @@
 # rbs_inline: enabled
 
 module SolidObjects
-  VERSION = "0.17.3"
+  VERSION = "0.17.4"
 end

test/integration/activation_lease_recovery_test.rb

@@ -0,0 +1,155 @@
+# rbs_inline: enabled
+
+require "database_test_helper"
+
+class ActivationLeaseRecoveryTest < ActiveSupport::TestCase
+  class Counter < SolidObjects::Actor
+    actor_type "activation-lease-recovery"
+    attribute :count, default: 0
+
+    class << self
+      attr_accessor :before_commit
+    end
+
+    def increment
+      self.count += 1
+      self.class.before_commit&.call
+      count
+    end
+  end
+
+  setup do
+    SolidObjects.configuration.lease_duration = 0.2
+    SolidObjects.configuration.lease_renewal_interval = 0.05
+    SolidObjects.configuration.max_attempts = 1
+    @clock = { now: SolidObjects.database_adapter.database_now }
+    clock = @clock
+    SolidObjects.database_adapter.define_singleton_method(:database_now) { clock.fetch(:now) }
+    Counter.before_commit = nil
+    @events = []
+    @subscription = ActiveSupport::Notifications.subscribe(/solid_objects\.(message|activation|recovery)\./) do |event|
+      @events << event.name
+    end
+  end
+
+  teardown do
+    ActiveSupport::Notifications.unsubscribe(@subscription)
+    SolidObjects.database_adapter.singleton_class.remove_method(:database_now)
+    Counter.before_commit = nil
+  end
+
+  test "keeps renewing across short worker passes for five lease durations" do
+    worker = SolidObjects::Worker.new
+    Counter.before_commit = -> { @clock[:now] += 0.03 }
+    messages = []
+    first_expiration = nil
+
+    34.times do
+      messages << Counter.ref("continuous").async.increment
+      assert_equal 1, worker.run_once
+      first_expiration ||= instance.activation_expires_at
+    end
+
+    assert_operator instance.activation_expires_at, :>, first_expiration + 0.8
+    assert_equal 1, instance.activation_generation
+    messages.each_with_index do |reference, index|
+      message = SolidObjects::Message.find(reference.id)
+      assert_predicate message, :completed?
+      assert_equal index + 1, message.result
+      assert_equal 1, message.attempt_count
+    end
+    assert_equal 34, @events.count("solid_objects.message.completed")
+  ensure
+    worker&.stop
+  end
+
+  test "renews between short turns while draining without a renewal thread" do
+    messages = 34.times.map { Counter.ref("draining").async.increment }
+    registry = SolidObjects::ProcessRegistry.new
+    process = registry.register
+    activation = SolidObjects::ActivationManager.new(owner_id: process.id).claim_next
+    first_expiration = instance.activation_expires_at
+    Counter.before_commit = -> { @clock[:now] += 0.03 }
+
+    assert_equal 34, activation.drain
+    assert_operator instance.activation_expires_at, :>, first_expiration + 0.8
+    assert_equal 1, instance.activation_generation
+    assert messages.all? { |reference| SolidObjects::Message.find(reference.id).completed? }
+  ensure
+    activation&.deactivate
+    registry&.stop
+  end
+
+  test "keeps the expired lease fence while a message claim remains" do
+    worker = SolidObjects::Worker.new
+    first = Counter.ref("expired").async.increment
+    second = Counter.ref("expired").async.increment
+    Counter.before_commit = -> { @clock[:now] += 0.22 }
+
+    assert_equal 0, worker.run_once
+    assert_empty instance.state
+    assert SolidObjects::ClaimedMessage.exists?(message_id: first.id)
+    assert_not_nil instance.activation_expires_at
+    assert_operator instance.activation_expires_at, :<=, @clock.fetch(:now)
+    Counter.before_commit = nil
+
+    assert_equal 2, worker.run_once
+    assert_equal 1, SolidObjects::Message.find(first.id).result
+    assert_equal 2, SolidObjects::Message.find(first.id).attempt_count
+    assert_equal 2, SolidObjects::Message.find(second.id).result
+    assert_equal 1, SolidObjects::Message.find(second.id).attempt_count
+    assert_empty SolidObjects::DeadLetter.all
+    assert_equal 2, @events.count("solid_objects.message.completed")
+  ensure
+    worker&.stop
+  end
+
+  test "recovers on the same worker and preserves mailbox order with one allowed attempt" do
+    worker = SolidObjects::Worker.new
+    first = Counter.ref("recoverable").async.increment
+    second = Counter.ref("recoverable").async.increment
+    Counter.before_commit = -> { @clock[:now] += 0.22 }
+
+    assert_equal 0, worker.run_once
+    assert_empty instance.state
+    Counter.before_commit = nil
+
+    assert_equal 2, worker.run_once
+    assert_equal 1, SolidObjects::Message.find(first.id).result
+    assert_equal 2, SolidObjects::Message.find(second.id).result
+    assert_equal 2, SolidObjects::Message.find(first.id).attempt_count
+    assert_empty SolidObjects::DeadLetter.all
+  ensure
+    worker&.stop
+  end
+
+  test "a stale worker cannot release a second worker's activation or repeat its commits" do
+    first_worker = SolidObjects::Worker.new
+    second_worker = SolidObjects::Worker.new
+    first = Counter.ref("takeover").async.increment
+    second = Counter.ref("takeover").async.increment
+    Counter.before_commit = -> do
+      Counter.before_commit = nil
+      @clock[:now] += 0.22
+      assert_equal 2, Thread.new { second_worker.run_once }.value
+    end
+
+    assert_equal 0, first_worker.run_once
+    assert_equal 1, SolidObjects::Message.find(first.id).result
+    assert_equal 2, SolidObjects::Message.find(second.id).result
+    assert_equal 2, SolidObjects::Message.find(first.id).attempt_count
+    assert_equal 2, @events.count("solid_objects.message.completed")
+    third = Counter.ref("takeover").async.increment
+    assert_equal 1, second_worker.run_once
+    assert_equal 3, SolidObjects::Message.find(third.id).result
+  ensure
+    first_worker&.stop
+    second_worker&.stop
+  end
+
+  private
+
+  def instance
+    SolidObjects::Instance.find_by!(actor_type: Counter.actor_type)
+  end
+end