Gateway: status route with per-route results, test push, reconnect and unpair

gateway/src/t3/host.ts

@@ -35,6 +35,13 @@ export interface HostConfig {
   environmentId: string;
 }
 
+/** What happened the last time the gateway tried one route. */
+export interface RouteAttempt {
+  ok: boolean;
+  error: string | null;
+  at: string;
+}
+
 export interface ThreadChange {
   host: T3Host;
   previous: ThreadShell | undefined;
@@ -71,6 +78,15 @@ export class T3Host {
   readonly projects = new Map<string, Project>();
   readonly threads = new Map<string, ThreadShell>();
   serverConfig: ServerConfig | null = null;
+  serverVersion: string | null = null;
+  /** Results of the latest connect, by route. Routes after the one that answered were not tried. */
+  readonly routeAttempts = new Map<string, RouteAttempt>();
+  /** When the current connection came up. Null when not online. */
+  onlineSince: string | null = null;
+  /** When the last connection ended. Null if it never connected or is online now. */
+  lastOnlineAt: string | null = null;
+  /** When the next connect is scheduled, while waiting to retry. */
+  nextRetryAt: string | null = null;
 
   private socket: RpcSocket | null = null;
   private sequence: number | null = null;
@@ -78,6 +94,8 @@ export class T3Host {
   private stopped = false;
   private attempt = 0;
   private retryTimer: ReturnType<typeof setTimeout> | null = null;
+  /** Set by reconnect() so a deliberate close connects again at once, with no backoff. */
+  private reconnecting = false;
   /** The route this connection uses. Chosen afresh, preferred first, on every connect. */
   private activeUrl: string;
 
@@ -114,6 +132,22 @@ export class T3Host {
     this.socket?.close();
   }
 
+  /**
+   * Connects again now, starting from the preferred route. For when a better route may be back,
+   * such as the LAN after getting home, or to skip a backoff wait. Does nothing mid-connect.
+   */
+  reconnect() {
+    if (this.stopped || this.status === "connecting") return;
+    this.attempt = 0;
+    if (this.socket) {
+      this.reconnecting = true;
+      this.socket.close();
+      return;
+    }
+    if (this.retryTimer) clearTimeout(this.retryTimer);
+    void this.connect();
+  }
+
   /** Sends one orchestration command and resolves with its sequence. */
   async dispatch(command: Command): Promise<{ sequence: number }> {
     return this.requireSocket().call("orchestration.dispatchCommand", command);
@@ -162,6 +196,11 @@ export class T3Host {
   }
 
   private setStatus(status: HostStatus, error: string | null = null) {
+    if (status === "online") this.onlineSince = new Date().toISOString();
+    else if (this.status === "online") {
+      this.lastOnlineAt = new Date().toISOString();
+      this.onlineSince = null;
+    }
     this.status = status;
     this.lastError = error;
     this.options.onStatus?.(this);
@@ -169,6 +208,7 @@ export class T3Host {
 
   private async connect() {
     if (this.stopped) return;
+    this.nextRetryAt = null;
     this.setStatus("connecting", this.lastError);
     try {
       const descriptor = await this.pickRoute();
@@ -192,6 +232,11 @@ export class T3Host {
       this.setStatus("online");
       await socket.closed;
       this.socket = null;
+      if (this.reconnecting) {
+        this.reconnecting = false;
+        void this.connect();
+        return;
+      }
       this.scheduleRetry("offline", "connection closed");
     } catch (error) {
       this.socket?.close();
@@ -206,6 +251,7 @@ export class T3Host {
   /** Uses the first route whose well-known descriptor answers, and returns that descriptor. */
   private async pickRoute(): Promise<EnvironmentDescriptor> {
     const failures: string[] = [];
+    this.routeAttempts.clear();
     for (const url of this.config.urls) {
       try {
         const response = await fetch(new URL("/.well-known/t3/environment", url), {
@@ -215,9 +261,13 @@ export class T3Host {
         const descriptor = (await response.json()) as EnvironmentDescriptor;
         if (url !== this.activeUrl) this.options.log?.(`${this.label}: using ${url}`);
         this.activeUrl = url;
+        this.serverVersion = descriptor.serverVersion ?? null;
+        this.routeAttempts.set(url, { ok: true, error: null, at: new Date().toISOString() });
         return descriptor;
       } catch (error) {
-        failures.push(`${new URL(url).host} ${error instanceof Error ? error.message : String(error)}`);
+        const message = routeError(error);
+        this.routeAttempts.set(url, { ok: false, error: message, at: new Date().toISOString() });
+        failures.push(`${new URL(url).host} ${message}`);
       }
     }
     throw new HostError(`no route answered (${failures.join(", ")})`, 503);
@@ -246,6 +296,7 @@ export class T3Host {
       status === "auth_invalid" || status === "incompatible"
         ? AUTH_RETRY_MS
         : Math.min(BACKOFF_MAX_MS, BACKOFF_MIN_MS * 2 ** this.attempt++) * (0.5 + Math.random() / 2);
+    this.nextRetryAt = new Date(Date.now() + delay).toISOString();
     this.retryTimer = setTimeout(() => void this.connect(), delay);
   }
 
@@ -301,3 +352,12 @@ export class T3Host {
     this.options.onThreadChange?.({ host: this, previous, next, initial });
   }
 }
+
+/** A short reason a route failed, worded for a watch screen. */
+function routeError(error: unknown): string {
+  if (error instanceof DOMException && error.name === "TimeoutError") return `no answer in ${ROUTE_TIMEOUT_MS / 1000} s`;
+  const message = error instanceof Error ? error.message : String(error);
+  if (/ECONNREFUSED|Unable to connect|refused/i.test(message)) return "refused";
+  if (/ENOTFOUND|getaddrinfo|DNS/i.test(message)) return "name not found";
+  return message;
+}