feat: add first token latency window stats

tkmind-proxy.mjs

@@ -124,6 +124,37 @@ function createRuntimeRouter({
     const n = Number(value ?? 0);
     return Number.isFinite(n) ? n : 0;
   };
+  const parseFirstTokenSample = (value) => {
+    const parts = String(value ?? '').split(':');
+    return readNumber(parts[1]);
+  };
+  const percentile = (values, pct) => {
+    const sorted = values
+      .map((value) => readNumber(value))
+      .filter((value) => value > 0)
+      .sort((a, b) => a - b);
+    if (sorted.length === 0) return 0;
+    const idx = Math.min(sorted.length - 1, Math.max(0, Math.ceil((pct / 100) * sorted.length) - 1));
+    return sorted[idx];
+  };
+  const firstTokenWindowStats = async (client, workerId, windowMs) => {
+    if (!client) return { count: 0, p50Ms: 0, p95Ms: 0 };
+    const now = Date.now();
+    const samples = await client
+      .sendCommand([
+        'ZRANGEBYSCORE',
+        key('worker', workerId, 'first_token_samples'),
+        String(now - windowMs),
+        String(now),
+      ])
+      .catch(() => []);
+    const values = samples.map(parseFirstTokenSample).filter((value) => value > 0);
+    return {
+      count: values.length,
+      p50Ms: percentile(values, 50),
+      p95Ms: percentile(values, 95),
+    };
+  };
   const scoreWorker = async (client, target) => {
     const workerId = workerIdForTarget(target);
     const values = await client.mGet([
@@ -225,9 +256,11 @@ function createRuntimeRouter({
       const workerId = workerIdForTarget(target);
       const sample = Math.max(0, Math.round(Number(latencyMs) || 0));
       const ewmaKey = key('worker', workerId, 'ewma_first_token_ms');
+      const samplesKey = key('worker', workerId, 'first_token_samples');
       const prev = readNumber(await client.get(ewmaKey).catch(() => null));
       const next = prev > 0 ? Math.round(prev * 0.8 + sample * 0.2) : sample;
-      const now = String(Date.now());
+      const nowMs = Date.now();
+      const now = String(nowMs);
       await client
         .multi()
         .set(ewmaKey, String(next))
@@ -237,6 +270,18 @@ function createRuntimeRouter({
         .set(key('worker', workerId, 'heartbeat'), now, { EX: 30 })
         .exec()
         .catch(() => null);
+      await client
+        .sendCommand([
+          'ZADD',
+          samplesKey,
+          String(nowMs),
+          `${now}:${sample}:${Math.random().toString(36).slice(2)}`,
+        ])
+        .catch(() => null);
+      await client
+        .sendCommand(['ZREMRANGEBYSCORE', samplesKey, '0', String(nowMs - 60 * 60 * 1000)])
+        .catch(() => null);
+      await client.sendCommand(['EXPIRE', samplesKey, String(2 * 60 * 60)]).catch(() => null);
     },
     async getStatus() {
       const client = await getClient();
@@ -270,6 +315,8 @@ function createRuntimeRouter({
               ])
               .catch(() => [])
           : [];
+        const firstToken5m = await firstTokenWindowStats(client, workerId, 5 * 60 * 1000);
+        const firstToken1h = await firstTokenWindowStats(client, workerId, 60 * 60 * 1000);
         workers.push({
           id: workerId,
           target,
@@ -294,6 +341,8 @@ function createRuntimeRouter({
           lastFirstTokenMs: readNumber(values?.[18]),
           lastFirstTokenAt: values?.[19] ? Number(values[19]) : null,
           firstTokenCount: readNumber(values?.[20]),
+          firstToken5m,
+          firstToken1h,
           score: workerScoreFromValues(values),
         });
       }