Websocket: Fix connection failing without system CA certificates or IPv6 support

ayon_api/events.py

@@ -919,14 +919,24 @@ def _set_connected(self, connected: bool) -> None:
         else:
             self.emit_event(Event(self.TOPIC_CONNECTION_CLOSED))
 
-    def _on_connection_lost(self, stop_event: threading.Event) -> None:
+    def _on_connection_lost(
+        self,
+        stop_event: threading.Event,
+        exc: Exception | None = None,
+    ) -> None:
         state = self._loop_state
         received_message = state.received_message
         state.received_message = False
         self._close_ws_connection()
         if stop_event.is_set():
             return
 
+        # Log only first failure, reconnection attempts would spam the log
+        if state.connected is not False:
+            self._log.warning(
+                "Lost connection to AYON server websocket: %s",
+                exc if exc is not None else "Closed by server",
+            )
         self._set_connected(False)
 
         # Server closes unauthorized clients after few seconds without
@@ -972,10 +982,10 @@ def _loop(self, stop_event: threading.Event) -> None:
 
                 try:
                     ws_con = self._create_ws_connection()
-                except Exception:
+                except Exception as exc:
                     # Server is not available (connection refused, timeout,
                     #   502 from proxy during restart, ssl error...)
-                    self._on_connection_lost(stop_event)
+                    self._on_connection_lost(stop_event, exc)
                     continue
 
                 with self._thread_lock:
@@ -995,15 +1005,15 @@ def _loop(self, stop_event: threading.Event) -> None:
                 }
                 try:
                     ws_con.send(json.dumps(subscribe_payload))
-                except Exception:
-                    self._on_connection_lost(stop_event)
+                except Exception as exc:
+                    self._on_connection_lost(stop_event, exc)
                 continue
 
             try:
                 op_code, message = ws_con.recv_data()
-            except Exception:
+            except Exception as exc:
                 # Timeout (no heartbeat), closed connection, socket error...
-                self._on_connection_lost(stop_event)
+                self._on_connection_lost(stop_event, exc)
                 continue
 
             if op_code == ABNF.OPCODE_CLOSE:

ayon_api/server_api.py

@@ -15,6 +15,7 @@
 import time
 import logging
 import platform
+import socket
 import uuid
 from contextlib import contextmanager
 import typing
@@ -115,6 +116,64 @@
 )
 
 
+def _get_requests_ca_bundle() -> str | None:
+    """CA bundle used by 'requests' for ssl verification.
+
+    Returns:
+        str | None: Path to CA bundle file or directory.
+
+    """
+    # Same order as 'requests' uses when 'trust_env' is enabled
+    for env_key in ("REQUESTS_CA_BUNDLE", "CURL_CA_BUNDLE"):
+        path = os.environ.get(env_key)
+        if path:
+            return path
+    try:
+        from requests.certs import where
+
+        return where()
+    except Exception:
+        return None
+
+
+def _disable_websocket_ipv6() -> None:
+    """Skip IPv6 addresses in 'websocket' when python does not support IPv6.
+
+    Some DCC pythons are built without IPv6 support (e.g. Flame's on macOS).
+        'websocket' tries resolved addresses in order and fails on the first
+        IPv6 address with 'connect(): bad family' instead of trying the next
+        one. As 'localhost' resolves to '::1' first, websocket never
+        connected while REST requests (which use only IPv4) did work.
+
+    """
+    if socket.has_ipv6:
+        return
+
+    try:
+        from websocket import _http
+    except ImportError:
+        return
+
+    # Private function of 'websocket', skip if not available
+    get_addrinfo_list = getattr(_http, "_get_addrinfo_list", None)
+    if get_addrinfo_list is None or getattr(
+        get_addrinfo_list, "_ayon_ipv6_disabled", False
+    ):
+        return
+
+    def _get_ipv4_addrinfo_list(*args, **kwargs):
+        addrinfo_list, *other = get_addrinfo_list(*args, **kwargs)
+        ipv4_addrinfo_list = [
+            addrinfo
+            for addrinfo in addrinfo_list
+            if addrinfo[0] != socket.AF_INET6
+        ]
+        return (ipv4_addrinfo_list or addrinfo_list, *other)
+
+    _get_ipv4_addrinfo_list._ayon_ipv6_disabled = True
+    _http._get_addrinfo_list = _get_ipv4_addrinfo_list
+
+
 class GraphQlResponse:
     """GraphQl response."""
 
@@ -927,15 +986,27 @@ def create_websocket(
         prepared_sslopt = copy.deepcopy(sslopt) if sslopt else {}
         if self._ssl_verify is False:
             prepared_sslopt.setdefault("cert_reqs", ssl.CERT_NONE)
-        elif isinstance(self._ssl_verify, str):
-            prepared_sslopt.setdefault("ca_certs", self._ssl_verify)
+        else:
+            # Verify with the same CA bundle as REST requests do. Without
+            #   it 'websocket' uses system certificates of the python ssl
+            #   module, which might be missing (e.g. python on macOS), so
+            #   REST would work but websocket would always fail.
+            ca_bundle = self._ssl_verify
+            if not isinstance(ca_bundle, str):
+                ca_bundle = _get_requests_ca_bundle()
+            if ca_bundle:
+                key = "ca_certs"
+                if os.path.isdir(ca_bundle):
+                    key = "ca_cert_path"
+                prepared_sslopt.setdefault(key, ca_bundle)
 
         if self._cert:
             prepared_sslopt.setdefault("certfile", self._cert)
 
         if ws_url.startswith("wss://") and prepared_sslopt:
             ws_kwargs["sslopt"] = prepared_sslopt
 
+        _disable_websocket_ipv6()
         return websocket.create_connection(ws_url, **ws_kwargs)
 
     def _update_session_headers(self) -> None: