Randomize delay before retrying a request

ayon_api/server_api.py

@@ -11,6 +11,7 @@
 import re
 import io
 import json
+import random
 import ssl
 import time
 import logging
@@ -120,6 +121,24 @@
 )
 
 
+def _get_retry_delay(delay: float) -> float:
+    """Randomize delay before retry of a request.
+
+    Clients that failed at the same moment, e.g. many machines starting at
+        once against an overloaded server, would otherwise retry at the same
+        moment again. Returned delay is between 50% and 150% of passed
+        delay, so the delay is the same on average.
+
+    Args:
+        delay (float): Delay in seconds.
+
+    Returns:
+        float: Randomized delay in seconds.
+
+    """
+    return delay * random.uniform(0.5, 1.5)
+
+
 def _get_requests_ca_bundle() -> str | None:
     """CA bundle used by 'requests' for ssl verification.
 
@@ -1773,7 +1792,7 @@ def _do_rest_request(
                         response.status_code
                     )
                     if retry_idx != 0:
-                        time.sleep(2)
+                        time.sleep(_get_retry_delay(2))
                     continue
                 break
 
@@ -1821,7 +1840,7 @@ def _do_rest_request(
                 )
 
             if retry_idx != 0:
-                time.sleep(0.1)
+                time.sleep(_get_retry_delay(0.1))
 
         if new_response is not None:
             return new_response

tests/test_request_retries.py

@@ -33,3 +33,35 @@ def request_func(url, **kwargs):
     )
     assert response.status_code == 200
     assert response.data == {"ok": True}
+
+
+def test_retry_delay_is_randomized(monkeypatch):
+    """Clients failing at the same moment should not retry at once."""
+    sleeps = []
+    monkeypatch.setattr("time.sleep", sleeps.append)
+    uniform_args = []
+
+    def uniform(low, high):
+        uniform_args.append((low, high))
+        return high
+
+    monkeypatch.setattr("random.uniform", uniform)
+    con = ServerAPI("http://localhost:0", create_session=False, max_retries=3)
+    responses = [
+        FakeResponse(503),
+        requests.exceptions.ConnectionError("down"),
+        FakeResponse(200, json_data={"ok": True}),
+    ]
+
+    def request_func(url, **kwargs):
+        item = responses.pop(0)
+        if isinstance(item, Exception):
+            raise item
+        return item
+
+    response = con._do_rest_request(
+        request_func, "http://localhost:0/api/x", handle_invalid_token=False
+    )
+    assert response.status_code == 200
+    assert uniform_args == [(0.5, 1.5), (0.5, 1.5)]
+    assert sleeps == [pytest.approx(3.0), pytest.approx(0.15)]