diff --git a/ayon_api/events.py b/ayon_api/events.py index 1dda11969..b5bc6d11a 100644 --- a/ayon_api/events.py +++ b/ayon_api/events.py @@ -919,7 +919,11 @@ def _set_connected(self, connected: bool) -> None: else: self.emit_event(Event(self.TOPIC_CONNECTION_CLOSED)) - def _on_connection_lost(self, stop_event: threading.Event) -> None: + def _on_connection_lost( + self, + stop_event: threading.Event, + exc: Exception | None = None, + ) -> None: state = self._loop_state received_message = state.received_message state.received_message = False @@ -927,6 +931,12 @@ def _on_connection_lost(self, stop_event: threading.Event) -> None: if stop_event.is_set(): return + # Log only first failure, reconnection attempts would spam the log + if state.connected is not False: + self._log.warning( + "Lost connection to AYON server websocket: %s", + exc if exc is not None else "Closed by server", + ) self._set_connected(False) # Server closes unauthorized clients after few seconds without @@ -972,10 +982,10 @@ def _loop(self, stop_event: threading.Event) -> None: try: ws_con = self._create_ws_connection() - except Exception: + except Exception as exc: # Server is not available (connection refused, timeout, # 502 from proxy during restart, ssl error...) - self._on_connection_lost(stop_event) + self._on_connection_lost(stop_event, exc) continue with self._thread_lock: @@ -995,15 +1005,15 @@ def _loop(self, stop_event: threading.Event) -> None: } try: ws_con.send(json.dumps(subscribe_payload)) - except Exception: - self._on_connection_lost(stop_event) + except Exception as exc: + self._on_connection_lost(stop_event, exc) continue try: op_code, message = ws_con.recv_data() - except Exception: + except Exception as exc: # Timeout (no heartbeat), closed connection, socket error... - self._on_connection_lost(stop_event) + self._on_connection_lost(stop_event, exc) continue if op_code == ABNF.OPCODE_CLOSE: diff --git a/ayon_api/server_api.py b/ayon_api/server_api.py index 7cb03c4d4..74701c090 100644 --- a/ayon_api/server_api.py +++ b/ayon_api/server_api.py @@ -15,6 +15,7 @@ import time import logging import platform +import socket import uuid from contextlib import contextmanager import typing @@ -115,6 +116,64 @@ ) +def _get_requests_ca_bundle() -> str | None: + """CA bundle used by 'requests' for ssl verification. + + Returns: + str | None: Path to CA bundle file or directory. + + """ + # Same order as 'requests' uses when 'trust_env' is enabled + for env_key in ("REQUESTS_CA_BUNDLE", "CURL_CA_BUNDLE"): + path = os.environ.get(env_key) + if path: + return path + try: + from requests.certs import where + + return where() + except Exception: + return None + + +def _disable_websocket_ipv6() -> None: + """Skip IPv6 addresses in 'websocket' when python does not support IPv6. + + Some DCC pythons are built without IPv6 support (e.g. Flame's on macOS). + 'websocket' tries resolved addresses in order and fails on the first + IPv6 address with 'connect(): bad family' instead of trying the next + one. As 'localhost' resolves to '::1' first, websocket never + connected while REST requests (which use only IPv4) did work. + + """ + if socket.has_ipv6: + return + + try: + from websocket import _http + except ImportError: + return + + # Private function of 'websocket', skip if not available + get_addrinfo_list = getattr(_http, "_get_addrinfo_list", None) + if get_addrinfo_list is None or getattr( + get_addrinfo_list, "_ayon_ipv6_disabled", False + ): + return + + def _get_ipv4_addrinfo_list(*args, **kwargs): + addrinfo_list, *other = get_addrinfo_list(*args, **kwargs) + ipv4_addrinfo_list = [ + addrinfo + for addrinfo in addrinfo_list + if addrinfo[0] != socket.AF_INET6 + ] + return (ipv4_addrinfo_list or addrinfo_list, *other) + + _get_ipv4_addrinfo_list._ayon_ipv6_disabled = True + _http._get_addrinfo_list = _get_ipv4_addrinfo_list + + class GraphQlResponse: """GraphQl response.""" @@ -927,8 +986,19 @@ def create_websocket( prepared_sslopt = copy.deepcopy(sslopt) if sslopt else {} if self._ssl_verify is False: prepared_sslopt.setdefault("cert_reqs", ssl.CERT_NONE) - elif isinstance(self._ssl_verify, str): - prepared_sslopt.setdefault("ca_certs", self._ssl_verify) + else: + # Verify with the same CA bundle as REST requests do. Without + # it 'websocket' uses system certificates of the python ssl + # module, which might be missing (e.g. python on macOS), so + # REST would work but websocket would always fail. + ca_bundle = self._ssl_verify + if not isinstance(ca_bundle, str): + ca_bundle = _get_requests_ca_bundle() + if ca_bundle: + key = "ca_certs" + if os.path.isdir(ca_bundle): + key = "ca_cert_path" + prepared_sslopt.setdefault(key, ca_bundle) if self._cert: prepared_sslopt.setdefault("certfile", self._cert) @@ -936,6 +1006,7 @@ def create_websocket( if ws_url.startswith("wss://") and prepared_sslopt: ws_kwargs["sslopt"] = prepared_sslopt + _disable_websocket_ipv6() return websocket.create_connection(ws_url, **ws_kwargs) def _update_session_headers(self) -> None: