Gateway: fallback routes per host, for the laptop's VPN address

README.md

@@ -37,6 +37,10 @@ Operators use the `wristcode` CLI inside the container, on junior.gib:
 t3 auth pairing create --scope orchestration:read --scope orchestration:operate --label "WristCode gateway"
 podman exec -it wristcode-gateway wristcode host add https://t3mac.gbrown.org <credential> --label "Mac mini"
 
+# A host reachable more than one way, such as a laptop with a VPN route, lists fallbacks.
+# The gateway tries them in order when the first does not answer.
+podman exec -it wristcode-gateway wristcode host add https://t3laptop.gbrown.org <credential> --label "Laptop" --alt-url https://t3laptopvpn.gbrown.org
+
 # Pair a watch: enter the printed code on the watch within 5 minutes.
 podman exec -it wristcode-gateway wristcode pair
 

gateway/src/cli.ts

@@ -4,14 +4,17 @@
 
 import { loadConfig } from "./config.ts";
 import { openDb } from "./db.ts";
+import { hostUrls } from "./hub.ts";
 import { ORCHESTRATION_PROTOCOL_VERSION, type EnvironmentDescriptor } from "./t3/types.ts";
 
 const USAGE = `wristcode <command>
 
-  host add <url> <pairing-credential> [--label <name>]
+  host add <url> <pairing-credential> [--label <name>] [--alt-url <url>]...
       Connect a T3 Code server. Make the credential on that host with:
       t3 auth pairing create --scope orchestration:read --scope orchestration:operate --label "WristCode gateway"
       Run it again with a fresh credential to renew the gateway's token.
+      --alt-url adds a fallback route to the same server, such as a VPN address, tried in order
+      when <url> does not answer. Repeat it for more.
   host list
   host remove <id or label>
   pair
@@ -39,7 +42,7 @@ async function run() {
       return hostAdd(rest);
     case "host list":
       for (const host of db.hosts.list()) {
-        console.log(`${host.label}\t${host.url}\t${host.id}\ttoken expires ${host.token_expires_at ?? "unknown"}`);
+        console.log(`${host.label}\t${hostUrls(host).join(" ")}\t${host.id}\ttoken expires ${host.token_expires_at ?? "unknown"}`);
       }
       return;
     case "host remove":
@@ -69,9 +72,16 @@ async function run() {
 
 /** Exchanges a T3 pairing credential for a bearer session and saves the host. */
 async function hostAdd(args: string[]) {
-  const labelFlag = args.indexOf("--label");
+  const positional: string[] = [];
-  const label = labelFlag === -1 ? undefined : args[labelFlag + 1];
+  const altUrls: string[] = [];
-  const [rawUrl, credential] = args.filter((_, index) => labelFlag === -1 || (index !== labelFlag && index !== labelFlag + 1));
+  let label: string | undefined;
+  for (let index = 0; index < args.length; index++) {
+    const arg = args[index]!;
+    if (arg === "--label") label = args[++index];
+    else if (arg === "--alt-url") altUrls.push(new URL(args[++index] ?? "").origin);
+    else positional.push(arg);
+  }
+  const [rawUrl, credential] = positional;
   if (!rawUrl || !credential) throw new Error(USAGE);
   const url = new URL(rawUrl).origin;
 
@@ -104,6 +114,7 @@ async function hostAdd(args: string[]) {
     id: descriptor.environmentId,
     label: label ?? descriptor.label,
     url,
+    alt_urls: altUrls.length > 0 ? JSON.stringify(altUrls) : null,
     environment_id: descriptor.environmentId,
     token: token.access_token,
     token_expires_at: token.expires_in ? new Date(Date.now() + token.expires_in * 1000).toISOString() : null,

gateway/src/db.ts

@@ -8,6 +8,8 @@ export interface HostRow {
   id: string;
   label: string;
   url: string;
+  /** Fallback routes to the same server, as a JSON array, tried in order after `url`. */
+  alt_urls: string | null;
   environment_id: string;
   token: string;
   token_expires_at: string | null;
@@ -61,17 +63,19 @@ export function openDb(path: string) {
   db.run("pragma journal_mode = wal");
   db.run("pragma busy_timeout = 5000");
   db.run(SCHEMA);
+  const hostColumns = db.query<{ name: string }, []>("pragma table_info(hosts)").all().map((column) => column.name);
+  if (!hostColumns.includes("alt_urls")) db.run("alter table hosts add column alt_urls text");
 
   return {
     hosts: {
       list: () => db.query<HostRow, []>("select * from hosts order by created_at").all(),
       upsert(host: Omit<HostRow, "created_at">) {
         db.query(
-          `insert into hosts (id, label, url, environment_id, token, token_expires_at, created_at)
+          `insert into hosts (id, label, url, alt_urls, environment_id, token, token_expires_at, created_at)
-           values ($id, $label, $url, $environment_id, $token, $token_expires_at, $created_at)
+           values ($id, $label, $url, $alt_urls, $environment_id, $token, $token_expires_at, $created_at)
            on conflict(environment_id) do update set
-             label = excluded.label, url = excluded.url, token = excluded.token,
+             label = excluded.label, url = excluded.url, alt_urls = excluded.alt_urls,
-             token_expires_at = excluded.token_expires_at`,
+             token = excluded.token, token_expires_at = excluded.token_expires_at`,
         ).run({ ...host, created_at: now() });
       },
       remove: (idOrLabel: string) =>

gateway/src/hub.ts

@@ -19,13 +19,19 @@ export class Hub {
     for (const row of rows) {
       seen.add(row.id);
       const current = this.hosts.get(row.id);
-      if (current && current.row.url === row.url && current.row.token === row.token && current.row.label === row.label) {
+      if (
+        current &&
+        current.row.url === row.url &&
+        current.row.alt_urls === row.alt_urls &&
+        current.row.token === row.token &&
+        current.row.label === row.label
+      ) {
         current.row = row;
         continue;
       }
       current?.host.stop();
       const host = new T3Host(
-        { id: row.id, label: row.label, url: row.url, token: row.token, environmentId: row.environment_id },
+        { id: row.id, label: row.label, urls: hostUrls(row), token: row.token, environmentId: row.environment_id },
         this.options,
       );
       this.hosts.set(row.id, { host, row });
@@ -55,3 +61,9 @@ export class Hub {
     this.hosts.clear();
   }
 }
+
+/** A host's routes, preferred first. */
+export function hostUrls(row: Pick<HostRow, "url" | "alt_urls">): string[] {
+  const alternates: unknown = row.alt_urls ? JSON.parse(row.alt_urls) : [];
+  return [row.url, ...(Array.isArray(alternates) ? alternates.filter((url) => typeof url === "string") : [])];
+}

gateway/src/t3/host.ts

@@ -26,8 +26,11 @@ export type HostStatus =
 export interface HostConfig {
   id: string;
   label: string;
-  /** The server's HTTP origin, such as `https://t3mac.gbrown.org`. */
+  /**
-  url: string;
+   * The server's HTTP origins, preferred first, such as a LAN route then a VPN route. Every one
+   * must reach the same T3 server, since they share the environment ID and the token.
+   */
+  urls: string[];
   token: string;
   environmentId: string;
 }
@@ -49,6 +52,8 @@ export interface HostOptions {
 const BACKOFF_MIN_MS = 1_000;
 const BACKOFF_MAX_MS = 5 * 60_000;
 const AUTH_RETRY_MS = 10 * 60_000;
+/** How long one route gets to answer before the next is tried. */
+const ROUTE_TIMEOUT_MS = 5_000;
 
 export class HostError extends Error {
   constructor(
@@ -73,11 +78,17 @@ export class T3Host {
   private stopped = false;
   private attempt = 0;
   private retryTimer: ReturnType<typeof setTimeout> | null = null;
+  /** The route this connection uses. Chosen afresh, preferred first, on every connect. */
+  private activeUrl: string;
 
   constructor(
     readonly config: HostConfig,
     private readonly options: HostOptions = {},
-  ) {}
+  ) {
+    const [first] = config.urls;
+    if (!first) throw new Error(`${config.label} has no URL`);
+    this.activeUrl = first;
+  }
 
   get id() {
     return this.config.id;
@@ -87,6 +98,11 @@ export class T3Host {
     return this.config.label;
   }
 
+  /** The route in use, or the one that will be tried first. */
+  get url() {
+    return this.activeUrl;
+  }
+
   start() {
     this.stopped = false;
     void this.connect();
@@ -130,7 +146,7 @@ export class T3Host {
   }
 
   private async http<T>(path: string, init: RequestInit = {}): Promise<T> {
-    const response = await fetch(new URL(path, this.config.url), {
+    const response = await fetch(new URL(path, this.activeUrl), {
       ...init,
       headers: {
         authorization: `Bearer ${this.config.token}`,
@@ -155,7 +171,7 @@ export class T3Host {
     if (this.stopped) return;
     this.setStatus("connecting", this.lastError);
     try {
-      const descriptor = await this.http<EnvironmentDescriptor>("/.well-known/t3/environment");
+      const descriptor = await this.pickRoute();
       const protocol = descriptor.orchestrationProtocolVersion ?? 1;
       if (protocol !== ORCHESTRATION_PROTOCOL_VERSION) {
         throw new HostError(`speaks orchestration protocol ${protocol}, the gateway speaks ${ORCHESTRATION_PROTOCOL_VERSION}`, 426);
@@ -164,7 +180,7 @@ export class T3Host {
         throw new HostError(`is environment ${descriptor.environmentId}, expected ${this.config.environmentId}`, 409);
       }
       const { ticket } = await this.http<{ ticket: string }>("/api/auth/websocket-ticket/rpc", { method: "POST" });
-      const wsUrl = new URL("/ws", this.config.url);
+      const wsUrl = new URL("/ws", this.activeUrl);
       wsUrl.protocol = wsUrl.protocol === "https:" ? "wss:" : "ws:";
       wsUrl.searchParams.set("wsTicket", ticket);
       wsUrl.searchParams.set("orchestrationProtocol", String(ORCHESTRATION_PROTOCOL_VERSION));
@@ -187,6 +203,26 @@ export class T3Host {
     }
   }
 
+  /** Uses the first route whose well-known descriptor answers, and returns that descriptor. */
+  private async pickRoute(): Promise<EnvironmentDescriptor> {
+    const failures: string[] = [];
+    for (const url of this.config.urls) {
+      try {
+        const response = await fetch(new URL("/.well-known/t3/environment", url), {
+          signal: AbortSignal.timeout(ROUTE_TIMEOUT_MS),
+        });
+        if (!response.ok) throw new Error(`returned ${response.status}`);
+        const descriptor = (await response.json()) as EnvironmentDescriptor;
+        if (url !== this.activeUrl) this.options.log?.(`${this.label}: using ${url}`);
+        this.activeUrl = url;
+        return descriptor;
+      } catch (error) {
+        failures.push(`${new URL(url).host} ${error instanceof Error ? error.message : String(error)}`);
+      }
+    }
+    throw new HostError(`no route answered (${failures.join(", ")})`, 503);
+  }
+
   private subscribe(socket: RpcSocket) {
     const input = this.hasSnapshot && this.sequence !== null ? { afterSequence: this.sequence } : {};
     const { done } = socket.stream("orchestration.subscribeShell", input, (value) =>

gateway/test/host.test.ts

@@ -0,0 +1,58 @@
+// A host with several routes connects through the first one that answers.
+import { afterEach, describe, expect, it } from "bun:test";
+import type { Server } from "bun";
+
+import { T3Host } from "../src/t3/host.ts";
+
+const ENVIRONMENT_ID = "env-laptop";
+let server: Server<undefined> | undefined;
+afterEach(() => server?.stop(true));
+
+/** Just enough of a T3 server: the descriptor, a ticket, config, and an empty shell snapshot. */
+function fakeT3() {
+  server = Bun.serve({
+    port: 0,
+    fetch(req, srv) {
+      const { pathname } = new URL(req.url);
+      if (pathname === "/.well-known/t3/environment") {
+        return Response.json({ environmentId: ENVIRONMENT_ID, label: "framework-gib", serverVersion: "0.1.5", orchestrationProtocolVersion: 2 });
+      }
+      if (pathname === "/api/auth/websocket-ticket/rpc") return Response.json({ ticket: "t" });
+      if (pathname === "/ws" && srv.upgrade(req)) return undefined;
+      return new Response("not found", { status: 404 });
+    },
+    websocket: {
+      message(ws, data) {
+        const frame = JSON.parse(String(data)) as { _tag: string; id: string; tag: string };
+        if (frame._tag !== "Request") return;
+        if (frame.tag === "server.getConfig") {
+          ws.send(JSON.stringify({ _tag: "Exit", requestId: frame.id, exit: { _tag: "Success", value: {} } }));
+        } else if (frame.tag === "orchestration.subscribeShell") {
+          const snapshot = { snapshotSequence: 1, projects: [], threads: [] };
+          ws.send(JSON.stringify({ _tag: "Chunk", requestId: frame.id, values: [{ kind: "snapshot", snapshot }] }));
+        }
+      },
+    },
+  });
+  return `http://localhost:${server.port}`;
+}
+
+describe("T3Host routes", () => {
+  it("falls back to the next route when the preferred one does not answer", async () => {
+    const working = fakeT3();
+    // Port 9 (discard) refuses connections, standing in for a LAN route that is down.
+    const host = new T3Host({ id: ENVIRONMENT_ID, label: "Laptop", urls: ["http://127.0.0.1:9", working], token: "x", environmentId: ENVIRONMENT_ID });
+    const online = new Promise<void>((resolve) => {
+      const check = setInterval(() => {
+        if (host.status === "online") {
+          clearInterval(check);
+          resolve();
+        }
+      }, 5);
+    });
+    host.start();
+    await online;
+    expect(host.url).toBe(working);
+    host.stop();
+  });
+});

gateway/test/notify.test.ts

@@ -95,6 +95,7 @@ describe("Notifier", () => {
       id: "host-1",
       label: "Mac mini",
       url: "https://t3mac.gbrown.org",
+      alt_urls: null,
       environment_id: "host-1",
       token: "t",
       token_expires_at: new Date(Date.now() + 3 * 24 * 60 * 60_000).toISOString(),

scripts/renew-hosts.sh

@@ -13,12 +13,15 @@ GATEWAY_SSH="${GATEWAY_SSH:-junior.gib}"
 GATEWAY_RUNTIME="${GATEWAY_RUNTIME:-podman}"
 GATEWAY_CONTAINER="${GATEWAY_CONTAINER:-wristcode-gateway}"
 
-# label | SSH host, or "local" for the machine running this script | public T3 URL
+# label | SSH host, or "local" for the machine running this script | public T3 URLs
+# A host can list several URLs, space separated, preferred first. The gateway falls back to the
+# next one when a route does not answer, such as the laptop's VPN route when it is away from home.
 HOSTS=(
   "Desktop|local|https://t3desktop.gbrown.org"
   "Mac mini|mac.gib|https://t3mac.gbrown.org"
   "server-gib|server.gib|https://t3sr.gbrown.org"
   "LittleMan|junior.gib|https://t3jr.gbrown.org"
+  "Laptop|laptop.gib|https://t3laptop.gbrown.org https://t3laptopvpn.gbrown.org"
 )
 
 # Prints a pairing credential. Runs on the T3 host. The CLI is not on PATH in non-login shells,
@@ -36,7 +39,9 @@ done
 '
 
 renew() {
-  local label="$1" ssh_host="$2" url="$3" credential
+  local label="$1" ssh_host="$2" url alternates alternate credential alt_args=""
+  read -r url alternates <<<"$3"
+  for alternate in $alternates; do alt_args+=" --alt-url '$alternate'"; done
   if [ "$ssh_host" = "local" ]; then
     credential="$(bash -c "$MINT")"
   else
@@ -48,7 +53,7 @@ renew() {
   fi
   # The credential travels on stdin, not on a command line other processes could read.
   ssh -o BatchMode=yes "$GATEWAY_SSH" \
-    "read -r credential && $GATEWAY_RUNTIME exec $GATEWAY_CONTAINER wristcode host add '$url' \"\$credential\" --label '$label'" \
+    "read -r credential && $GATEWAY_RUNTIME exec $GATEWAY_CONTAINER wristcode host add '$url' \"\$credential\" --label '$label'$alt_args" \
     <<<"$credential"
 }