From dcd2a8d0d0c4d84f28f7c2deb1607c7e75fb961e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20S=C3=A1nchez?= Date: Thu, 3 Sep 2026 15:17:40 -0600 Subject: [PATCH 1/2] fix[installer](nginx): replace 0s agent gRPC timeouts with a 24h inactivity backstop --- installer/templates/front-end.go | 29 ++++++++++++++--------------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/installer/templates/front-end.go b/installer/templates/front-end.go index febd7d689..b584265d8 100644 --- a/installer/templates/front-end.go +++ b/installer/templates/front-end.go @@ -70,32 +70,31 @@ server { proxy_request_buffering off; } - # The agent's persistent gRPC streams to agent-manager. nginx's - # grpc_*_timeout of 0 means "no inactivity timeout" (removing the - # directive would silently fall back to a 60s default), so an idle - # AgentStream / PingService / CollectorService is never torn down by the - # proxy. Liveness is carried by the gRPC keepalive (30s ping / 10s - # timeout on both ends) plus grpc_socket_keepalive, which still reaps a - # genuinely dead peer. + # The agent's persistent gRPC streams to agent-manager. nginx has no + # "no timeout" for upstream read/send (0 means "fire immediately", which + # breaks every request), so use a large inactivity backstop of 24h: + # gRPC keepalive (30s ping / 10s timeout on both ends) plus + # grpc_socket_keepalive keep live streams active by resetting the + # inactivity timer, and still reap a genuinely dead peer. location /agent.AgentService/ { grpc_pass grpcs://$utmstack_agent_manager_grpc; - grpc_read_timeout 0; - grpc_send_timeout 0; + grpc_read_timeout 86400; + grpc_send_timeout 86400; client_body_timeout 1h; grpc_socket_keepalive on; } location /agent.PanelService/ { grpc_pass grpcs://$utmstack_agent_manager_grpc; - grpc_read_timeout 0; - grpc_send_timeout 0; + grpc_read_timeout 86400; + grpc_send_timeout 86400; grpc_socket_keepalive on; } location /agent.CollectorService/ { grpc_pass grpcs://$utmstack_agent_manager_grpc; - grpc_read_timeout 0; - grpc_send_timeout 0; + grpc_read_timeout 86400; + grpc_send_timeout 86400; grpc_socket_keepalive on; } @@ -108,8 +107,8 @@ server { location /agent.PingService/ { grpc_pass grpcs://$utmstack_agent_manager_grpc; - grpc_read_timeout 0; - grpc_send_timeout 0; + grpc_read_timeout 86400; + grpc_send_timeout 86400; grpc_socket_keepalive on; } From bf196e0e915ebe9b0363284b29561b37688cac58 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20S=C3=A1nchez?= Date: Thu, 3 Sep 2026 16:13:34 -0600 Subject: [PATCH 2/2] fix[installer](nginx): stop client_body_timeout from evicting idle agent gRPC streams at 60s --- installer/templates/front-end.go | 20 +++++++++++------- installer/templates/proxy.go | 36 +++++++++++++++++++++++++------- 2 files changed, 41 insertions(+), 15 deletions(-) diff --git a/installer/templates/front-end.go b/installer/templates/front-end.go index b584265d8..06e44aaba 100644 --- a/installer/templates/front-end.go +++ b/installer/templates/front-end.go @@ -72,15 +72,15 @@ server { # The agent's persistent gRPC streams to agent-manager. nginx has no # "no timeout" for upstream read/send (0 means "fire immediately", which - # breaks every request), so use a large inactivity backstop of 24h: - # gRPC keepalive (30s ping / 10s timeout on both ends) plus - # grpc_socket_keepalive keep live streams active by resetting the - # inactivity timer, and still reap a genuinely dead peer. + # breaks every request), so use a large inactivity backstop of 24h. + # gRPC keepalive PINGs are HTTP/2 control frames - nginx only ACKs them + # and they do NOT reset these timers; only DATA does. Every timer below + # must therefore be large for streams legitimately silent between calls. location /agent.AgentService/ { grpc_pass grpcs://$utmstack_agent_manager_grpc; grpc_read_timeout 86400; grpc_send_timeout 86400; - client_body_timeout 1h; + client_body_timeout 86400; grpc_socket_keepalive on; } @@ -88,6 +88,7 @@ server { grpc_pass grpcs://$utmstack_agent_manager_grpc; grpc_read_timeout 86400; grpc_send_timeout 86400; + client_body_timeout 86400; grpc_socket_keepalive on; } @@ -95,20 +96,25 @@ server { grpc_pass grpcs://$utmstack_agent_manager_grpc; grpc_read_timeout 86400; grpc_send_timeout 86400; + client_body_timeout 86400; grpc_socket_keepalive on; } # log-input's ingest, whose service lives in the SDK's "plugins" package. + # log-input's ingest: also a persistent collector stream, so the same + # 24h inactivity backstop (a quiet host can go >60s without a log line). location /plugins.Integration/ { grpc_pass grpcs://$utmstack_log_input_grpc; - grpc_read_timeout 900; - grpc_send_timeout 900; + grpc_read_timeout 86400; + grpc_send_timeout 86400; + client_body_timeout 86400; } location /agent.PingService/ { grpc_pass grpcs://$utmstack_agent_manager_grpc; grpc_read_timeout 86400; grpc_send_timeout 86400; + client_body_timeout 86400; grpc_socket_keepalive on; } diff --git a/installer/templates/proxy.go b/installer/templates/proxy.go index 8c16d8352..a7570dadb 100644 --- a/installer/templates/proxy.go +++ b/installer/templates/proxy.go @@ -27,8 +27,15 @@ server { location /agent. { grpc_pass grpc://127.0.0.1:10001; grpc_set_header x-shared-key $shared_key; - grpc_read_timeout 900; - grpc_send_timeout 900; + # Long-lived agent streams: nginx's gRPC keepalive PINGs are HTTP/2 + # control frames and do NOT reset these timers; only DATA does. + # An AgentStream is pure DATA between commands, so the 60s-default + # client_body_timeout (request body inactivity) is what evicted idle + # agents at exactly 60s. Use a 24h inactivity backstop; liveness is + # the app's job (gRPC keepalive + TCP keepalive). + grpc_read_timeout 86400; + grpc_send_timeout 86400; + client_body_timeout 86400; } # log-input's ingest. Separate only because its proto package differs; the @@ -36,8 +43,11 @@ server { location /plugins. { grpc_pass grpc://127.0.0.1:10001; grpc_set_header x-shared-key $shared_key; - grpc_read_timeout 900; - grpc_send_timeout 900; + # Same long-lived-stream treatment: a quiet collector can easily go + # more than 60s without a single log line on the stream. + grpc_read_timeout 86400; + grpc_send_timeout 86400; + client_body_timeout 86400; } location / { @@ -105,8 +115,15 @@ server { location /agent. { grpc_pass grpc://127.0.0.1:10001; grpc_set_header x-shared-key $shared_key; - grpc_read_timeout 900; - grpc_send_timeout 900; + # Long-lived agent streams: nginx's gRPC keepalive PINGs are HTTP/2 + # control frames and do NOT reset these timers; only DATA does. + # An AgentStream is pure DATA between commands, so the 60s-default + # client_body_timeout (request body inactivity) is what evicted idle + # agents at exactly 60s. Use a 24h inactivity backstop; liveness is + # the app's job (gRPC keepalive + TCP keepalive). + grpc_read_timeout 86400; + grpc_send_timeout 86400; + client_body_timeout 86400; } # log-input's ingest. Separate only because its proto package differs; the @@ -114,8 +131,11 @@ server { location /plugins. { grpc_pass grpc://127.0.0.1:10001; grpc_set_header x-shared-key $shared_key; - grpc_read_timeout 900; - grpc_send_timeout 900; + # Same long-lived-stream treatment: a quiet collector can easily go + # more than 60s without a single log line on the stream. + grpc_read_timeout 86400; + grpc_send_timeout 86400; + client_body_timeout 86400; } location / {