Gateway state sync mechanism

mkdocs/docs/concepts/gateways.md

@@ -174,7 +174,6 @@ $ dstack gateway list
 
     - Changing the number of replicas or redeploying replicas is not supported.
     - HTTPS is only supported for AWS gateways with the `acm` [certificate type](#certificate). For other gateways, use an external load balancer for TLS termination.
-    - An unavailable gateway replica prevents any new services or service replicas from being added.
     - All replicas are bound to the same backend and region.
     - At most 3 replicas are allowed per gateway.
 

src/dstack/_internal/core/models/runs.py

@@ -94,6 +94,7 @@ class RunTerminationReason(str, Enum):
     STOPPED_BY_USER = "stopped_by_user"
     ABORTED_BY_USER = "aborted_by_user"
     SERVER_ERROR = "server_error"
+    GATEWAY_ERROR = "gateway_error"
 
     def to_job_termination_reason(self) -> "JobTerminationReason":
         """
@@ -107,6 +108,7 @@ def to_job_termination_reason(self) -> "JobTerminationReason":
             self.STOPPED_BY_USER: JobTerminationReason.TERMINATED_BY_USER,
             self.ABORTED_BY_USER: JobTerminationReason.ABORTED_BY_USER,
             self.SERVER_ERROR: JobTerminationReason.TERMINATED_BY_SERVER,
+            self.GATEWAY_ERROR: JobTerminationReason.TERMINATED_BY_SERVER,
         }
         return mapping[self]
 
@@ -118,6 +120,7 @@ def to_status(self) -> "RunStatus":
             self.STOPPED_BY_USER: RunStatus.TERMINATED,
             self.ABORTED_BY_USER: RunStatus.TERMINATED,
             self.SERVER_ERROR: RunStatus.FAILED,
+            self.GATEWAY_ERROR: RunStatus.FAILED,
         }
         return mapping[self]
 
@@ -126,6 +129,8 @@ def to_error(self) -> Optional[str]:
             return "retry limit exceeded"
         elif self == RunTerminationReason.SERVER_ERROR:
             return "server error"
+        elif self == RunTerminationReason.GATEWAY_ERROR:
+            return "gateway error"
         else:
             return None