1 line
8.4 KiB
Plaintext
1 line
8.4 KiB
Plaintext
{"version":3,"sources":["../src/failover.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2026 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\n\n// Region failover for the Twirp API clients.\n//\n// On a retryable failure (any transport error or HTTP 5xx) the client discovers\n// alternative LiveKit Cloud regions via /settings/regions and replays the\n// request against the next region, with exponential backoff. 4xx responses are\n// returned immediately.\n\n// Total attempts (the original request + fallback regions) and the base retry\n// backoff are fixed, not user-configurable, so retries can't be tuned to values\n// that could overwhelm the server.\nexport const FAILOVER_MAX_ATTEMPTS = 3;\nexport const FAILOVER_BACKOFF_BASE_MS = 200;\n// Below this per-request timeout, a retry is unlikely to help and many clients\n// would retry in lockstep across regions, so a short request gets a single\n// attempt (thundering-herd guard).\nexport const MIN_FAILOVER_TIMEOUT_SECONDS = 5;\n\n/**\n * Total request attempts for a host; 1 means no failover. Failover only engages\n * when enabled, the host is a LiveKit Cloud domain, and the request timeout is\n * long enough to retry. `force` bypasses the cloud-host check (test-only).\n */\nexport function failoverAttempts(\n enabled: boolean,\n hostname: string,\n force = false,\n timeoutSeconds = 0,\n): number {\n if (!enabled || !(force || isCloud(hostname))) {\n return 1;\n }\n if (timeoutSeconds > 0 && timeoutSeconds < MIN_FAILOVER_TIMEOUT_SECONDS) {\n return 1;\n }\n return FAILOVER_MAX_ATTEMPTS;\n}\n\n// Failover only engages for LiveKit Cloud project domains.\nfunction isCloud(hostname: string): boolean {\n return hostname.endsWith('.livekit.cloud');\n}\n\n/** Normalizes a region URL to an http(s) scheme (ws -> http, wss -> https). */\nfunction toHttp(url: string): string {\n return url.startsWith('ws') ? `http${url.slice(2)}` : url;\n}\n\n/** A stable key identifying a host (including port) for dedup across attempts. */\nexport function hostKey(url: URL): string {\n return url.host.toLowerCase();\n}\n\n/** Returns the first region origin whose host has not yet been attempted. */\nexport function pickNext(regionOrigins: string[], attempted: Set<string>): string | undefined {\n for (const origin of regionOrigins) {\n try {\n if (!attempted.has(hostKey(new URL(origin)))) {\n return origin;\n }\n } catch {\n // skip malformed URLs\n }\n }\n return undefined;\n}\n\nexport function sleep(ms: number): Promise<void> {\n return ms > 0 ? new Promise((resolve) => setTimeout(resolve, ms)) : Promise.resolve();\n}\n\ntype CacheEntry = {\n origins: string[];\n fetchedAt: number;\n ttl: number; // ms\n};\n\n// Shared across all clients in the process so the region list is fetched once.\nconst regionCache = new Map<string, CacheEntry>();\n// Coalesces concurrent discovery fetches per origin: while one /settings/regions\n// request is in flight, other callers for the same origin await its result\n// rather than issuing their own (avoids a thundering herd when many requests\n// fail over at once).\nconst inflight = new Map<string, Promise<string[]>>();\n\n/**\n * Returns alternative region origins for `origin`, fetching /settings/regions\n * if the cache is stale. Best-effort: on a fetch failure it serves a stale\n * cached list when available, otherwise an empty list. Forwards `headers` so a\n * valid token — and any test directives — reach the discovery endpoint.\n */\nexport async function regionOrigins(origin: URL, headers: unknown): Promise<string[]> {\n const key = hostKey(origin);\n const cached = regionCache.get(key);\n if (cached && Date.now() - cached.fetchedAt < cached.ttl) {\n return cached.origins;\n }\n\n const existing = inflight.get(key);\n if (existing) {\n return existing;\n }\n const request = (async () => {\n try {\n const { origins, ttl } = await fetchRegions(origin, headers);\n // A zero TTL (e.g. Cache-Control: max-age=0) means \"do not cache\".\n if (ttl > 0) {\n regionCache.set(key, { origins, fetchedAt: Date.now(), ttl });\n }\n return origins;\n } catch {\n return cached?.origins ?? [];\n } finally {\n inflight.delete(key);\n }\n })();\n\n inflight.set(key, request);\n return request;\n}\n\nasync function fetchRegions(\n origin: URL,\n headers: unknown,\n): Promise<{ origins: string[]; ttl: number }> {\n // Forward the caller's headers (auth + any custom), minus body-specific ones.\n const fetchHeaders: Record<string, string> = {};\n for (const [k, v] of Object.entries((headers as Record<string, string>) ?? {})) {\n if (k.toLowerCase() === 'content-type' || k.toLowerCase() === 'content-length') continue;\n fetchHeaders[k] = v;\n }\n\n const response = await fetch(new URL('/settings/regions', origin.origin), {\n method: 'GET',\n headers: fetchHeaders,\n // Short timeout so a slow/unreachable discovery endpoint doesn't stall the\n // failover path.\n signal: AbortSignal.timeout(2000),\n });\n if (!response.ok) {\n throw new Error(`region discovery failed: ${response.status}`);\n }\n const ttl = parseMaxAge(response.headers.get('cache-control'));\n const body = (await response.json()) as { regions?: Array<{ url?: string }> };\n const origins = (body.regions ?? [])\n .filter((r) => !!r.url)\n .map((r) => new URL(toHttp(r.url!)).origin);\n return { origins, ttl };\n}\n\n/**\n * Returns the `max-age` from a Cache-Control header in milliseconds, or 0 when\n * absent, non-positive, or unparseable (meaning \"do not cache\"). Only `max-age`\n * is honored; other directives (including `s-maxage`, which targets shared\n * proxies) are ignored.\n */\nexport function parseMaxAge(cacheControl: string | null): number {\n if (!cacheControl) return 0;\n for (const directive of cacheControl.split(',')) {\n const trimmed = directive.trim().toLowerCase();\n if (trimmed.startsWith('max-age=')) {\n const secs = parseInt(trimmed.slice('max-age='.length), 10);\n return Number.isFinite(secs) && secs > 0 ? secs * 1000 : 0;\n }\n }\n return 0;\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAcO,MAAM,wBAAwB;AAC9B,MAAM,2BAA2B;AAIjC,MAAM,+BAA+B;AAOrC,SAAS,iBACd,SACA,UACA,QAAQ,OACR,iBAAiB,GACT;AACR,MAAI,CAAC,WAAW,EAAE,SAAS,QAAQ,QAAQ,IAAI;AAC7C,WAAO;AAAA,EACT;AACA,MAAI,iBAAiB,KAAK,iBAAiB,8BAA8B;AACvE,WAAO;AAAA,EACT;AACA,SAAO;AACT;AAGA,SAAS,QAAQ,UAA2B;AAC1C,SAAO,SAAS,SAAS,gBAAgB;AAC3C;AAGA,SAAS,OAAO,KAAqB;AACnC,SAAO,IAAI,WAAW,IAAI,IAAI,OAAO,IAAI,MAAM,CAAC,CAAC,KAAK;AACxD;AAGO,SAAS,QAAQ,KAAkB;AACxC,SAAO,IAAI,KAAK,YAAY;AAC9B;AAGO,SAAS,SAASA,gBAAyB,WAA4C;AAC5F,aAAW,UAAUA,gBAAe;AAClC,QAAI;AACF,UAAI,CAAC,UAAU,IAAI,QAAQ,IAAI,IAAI,MAAM,CAAC,CAAC,GAAG;AAC5C,eAAO;AAAA,MACT;AAAA,IACF,QAAQ;AAAA,IAER;AAAA,EACF;AACA,SAAO;AACT;AAEO,SAAS,MAAM,IAA2B;AAC/C,SAAO,KAAK,IAAI,IAAI,QAAQ,CAAC,YAAY,WAAW,SAAS,EAAE,CAAC,IAAI,QAAQ,QAAQ;AACtF;AASA,MAAM,cAAc,oBAAI,IAAwB;AAKhD,MAAM,WAAW,oBAAI,IAA+B;AAQpD,eAAsB,cAAc,QAAa,SAAqC;AACpF,QAAM,MAAM,QAAQ,MAAM;AAC1B,QAAM,SAAS,YAAY,IAAI,GAAG;AAClC,MAAI,UAAU,KAAK,IAAI,IAAI,OAAO,YAAY,OAAO,KAAK;AACxD,WAAO,OAAO;AAAA,EAChB;AAEA,QAAM,WAAW,SAAS,IAAI,GAAG;AACjC,MAAI,UAAU;AACZ,WAAO;AAAA,EACT;AACA,QAAM,WAAW,YAAY;AAC3B,QAAI;AACF,YAAM,EAAE,SAAS,IAAI,IAAI,MAAM,aAAa,QAAQ,OAAO;AAE3D,UAAI,MAAM,GAAG;AACX,oBAAY,IAAI,KAAK,EAAE,SAAS,WAAW,KAAK,IAAI,GAAG,IAAI,CAAC;AAAA,MAC9D;AACA,aAAO;AAAA,IACT,QAAQ;AACN,cAAO,iCAAQ,YAAW,CAAC;AAAA,IAC7B,UAAE;AACA,eAAS,OAAO,GAAG;AAAA,IACrB;AAAA,EACF,GAAG;AAEH,WAAS,IAAI,KAAK,OAAO;AACzB,SAAO;AACT;AAEA,eAAe,aACb,QACA,SAC6C;AAE7C,QAAM,eAAuC,CAAC;AAC9C,aAAW,CAAC,GAAG,CAAC,KAAK,OAAO,QAAS,WAAsC,CAAC,CAAC,GAAG;AAC9E,QAAI,EAAE,YAAY,MAAM,kBAAkB,EAAE,YAAY,MAAM,iBAAkB;AAChF,iBAAa,CAAC,IAAI;AAAA,EACpB;AAEA,QAAM,WAAW,MAAM,MAAM,IAAI,IAAI,qBAAqB,OAAO,MAAM,GAAG;AAAA,IACxE,QAAQ;AAAA,IACR,SAAS;AAAA;AAAA;AAAA,IAGT,QAAQ,YAAY,QAAQ,GAAI;AAAA,EAClC,CAAC;AACD,MAAI,CAAC,SAAS,IAAI;AAChB,UAAM,IAAI,MAAM,4BAA4B,SAAS,MAAM,EAAE;AAAA,EAC/D;AACA,QAAM,MAAM,YAAY,SAAS,QAAQ,IAAI,eAAe,CAAC;AAC7D,QAAM,OAAQ,MAAM,SAAS,KAAK;AAClC,QAAM,WAAW,KAAK,WAAW,CAAC,GAC/B,OAAO,CAAC,MAAM,CAAC,CAAC,EAAE,GAAG,EACrB,IAAI,CAAC,MAAM,IAAI,IAAI,OAAO,EAAE,GAAI,CAAC,EAAE,MAAM;AAC5C,SAAO,EAAE,SAAS,IAAI;AACxB;AAQO,SAAS,YAAY,cAAqC;AAC/D,MAAI,CAAC,aAAc,QAAO;AAC1B,aAAW,aAAa,aAAa,MAAM,GAAG,GAAG;AAC/C,UAAM,UAAU,UAAU,KAAK,EAAE,YAAY;AAC7C,QAAI,QAAQ,WAAW,UAAU,GAAG;AAClC,YAAM,OAAO,SAAS,QAAQ,MAAM,WAAW,MAAM,GAAG,EAAE;AAC1D,aAAO,OAAO,SAAS,IAAI,KAAK,OAAO,IAAI,OAAO,MAAO;AAAA,IAC3D;AAAA,EACF;AACA,SAAO;AACT;","names":["regionOrigins"]} |