memind/services/orchestrator/executor-worker-runtime.mjs

function workerError(code, message, options = {}) {
  const error = new Error(message);
  error.code = code;
  error.retryable = options.retryable === true;
  return error;
}

function safeJson(response) {
  return response.json().catch(() => null);
}

export function createExecutorWorkerClient({
  baseUrl,
  serviceToken = '',
  workerToken = '',
  fetchImpl = globalThis.fetch,
} = {}) {
  const target = String(baseUrl ?? '').trim().replace(/\/$/, '');
  if (!target) throw new Error('Executor worker client requires baseUrl');
  if (typeof fetchImpl !== 'function') throw new Error('Executor worker client requires fetch');

  async function request(path, { method = 'GET', body = null } = {}) {
    const response = await fetchImpl(`${target}${path}`, {
      method,
      headers: {
        accept: 'application/json',
        ...(body == null ? {} : { 'content-type': 'application/json' }),
        ...(serviceToken ? { authorization: `Bearer ${serviceToken}` } : {}),
        ...(workerToken ? { 'x-orchestrator-worker-token': workerToken } : {}),
      },
      ...(body == null ? {} : { body: JSON.stringify(body) }),
    });
    if (response.status === 204) return null;
    const payload = await safeJson(response);
    if (!response.ok) {
      const error = workerError(
        payload?.error?.code ?? 'EXECUTOR_WORKER_HTTP_ERROR',
        payload?.error?.message ?? `Executor worker request failed (${response.status})`,
        { retryable: response.status >= 500 || response.status === 429 },
      );
      error.status = response.status;
      throw error;
    }
    return payload;
  }

  return {
    claim(input) {
      return request('/v1/workers/claim', { method: 'POST', body: input });
    },
    start(jobId, input) {
      return request(`/v1/workers/jobs/${encodeURIComponent(jobId)}/start`, {
        method: 'POST',
        body: input,
      });
    },
    heartbeat(jobId, input) {
      return request(`/v1/workers/jobs/${encodeURIComponent(jobId)}/heartbeat`, {
        method: 'POST',
        body: input,
      });
    },
    progress(jobId, input) {
      return request(`/v1/workers/jobs/${encodeURIComponent(jobId)}/progress`, {
        method: 'POST',
        body: input,
      });
    },
    complete(jobId, input) {
      return request(`/v1/workers/jobs/${encodeURIComponent(jobId)}/complete`, {
        method: 'POST',
        body: input,
      });
    },
    fail(jobId, input) {
      return request(`/v1/workers/jobs/${encodeURIComponent(jobId)}/fail`, {
        method: 'POST',
        body: input,
      });
    },
    recoverExpired(input = {}) {
      return request('/v1/workers/recover-expired', { method: 'POST', body: input });
    },
    stats() {
      return request('/v1/workers/stats');
    },
  };
}

export function createWorkerAdapterRegistry(initialAdapters = []) {
  const adapters = new Map();
  return {
    register(adapter) {
      const id = String(adapter?.id ?? '').trim().toLowerCase();
      if (!id || typeof adapter?.submit !== 'function') {
        throw new Error('Worker adapter requires id and submit');
      }
      if (adapters.has(id)) throw new Error(`Worker adapter already registered: ${id}`);
      adapters.set(id, adapter);
      return adapter;
    },
    get(id) {
      return adapters.get(String(id ?? '').trim().toLowerCase()) ?? null;
    },
    list() {
      return [...adapters.values()].map((adapter) => ({
        id: adapter.id,
        kind: adapter.kind ?? 'worker',
        capabilities: [...(adapter.capabilities ?? [])],
      }));
    },
    ids() {
      return [...adapters.keys()];
    },
    async health() {
      const result = {};
      for (const [id, adapter] of adapters) {
        try {
          result[id] = typeof adapter.health === 'function'
            ? await adapter.health()
            : { ok: true };
        } catch (error) {
          result[id] = {
            ok: false,
            code: String(error?.code ?? 'EXECUTOR_ADAPTER_HEALTH_FAILED').slice(0, 128),
          };
        }
      }
      return result;
    },
    _initialize() {
      for (const adapter of initialAdapters) this.register(adapter);
      return this;
    },
  }._initialize();
}

export function createExecutorWorkerRuntime({
  client,
  registry = createWorkerAdapterRegistry(),
  workerId,
  heartbeatMs = 10_000,
  leaseDurationMs = 30_000,
  logger = console,
} = {}) {
  const normalizedWorkerId = String(workerId ?? '').trim();
  if (!normalizedWorkerId) throw new Error('Executor worker runtime requires workerId');
  if (!client?.claim) throw new Error('Executor worker runtime requires client');
  let stopping = false;
  const active = new Map();
  let lastAdapterHealth = {};

  async function executeAdapter(adapter, job, leaseToken, controller) {
    return adapter.submit(job.request, {
      jobId: job.jobId,
      signal: controller.signal,
      emit(type, data) {
        return client.progress(job.jobId, { leaseToken, type, data });
      },
    });
  }

  async function processClaim(claim) {
    const job = claim.job;
    const leaseToken = claim.leaseToken;
    const controller = new AbortController();
    active.set(job.jobId, { controller, leaseToken });
    const timeoutMs = Math.max(500, Number(job.timeoutMs ?? job.request?.controls?.timeoutMs) || 900_000);
    const timeout = setTimeout(() => controller.abort('executor_timeout'), timeoutMs);
    let heartbeatFailure = null;
    const heartbeat = setInterval(() => {
      void client.heartbeat(job.jobId, { leaseToken, leaseDurationMs }).catch((error) => {
        heartbeatFailure = error;
        controller.abort('heartbeat_failed');
      });
    }, Math.max(1_000, Math.min(heartbeatMs, Math.floor(leaseDurationMs / 2))));
    heartbeat.unref?.();
    try {
      await client.start(job.jobId, { leaseToken });
      if (!job.request) {
        throw workerError(
          'EXECUTOR_JOB_REQUEST_MISSING',
          'Claimed executor job does not contain an execution request',
        );
      }
      const primary = registry.get(job.executor);
      if (!primary) {
        throw workerError(
          'EXECUTOR_ADAPTER_NOT_INSTALLED',
          `Worker does not have adapter ${job.executor}`,
        );
      }
      let result;
      try {
        result = await executeAdapter(primary, job, leaseToken, controller);
      } catch (primaryError) {
        const fallbackId = String(job.fallbackExecutor ?? '').trim();
        const fallback = job.fallbackAvailable ? registry.get(fallbackId) : null;
        if (!fallback || controller.signal.aborted) throw primaryError;
        await client.progress(job.jobId, {
          leaseToken,
          type: 'executor_job_fallback_started',
          data: {
            from: job.executor,
            to: fallbackId,
            reason: String(primaryError?.code ?? 'EXECUTOR_PRIMARY_FAILED').slice(0, 128),
          },
        });
        result = await executeAdapter(fallback, {
          ...job,
          executor: fallbackId,
          request: { ...job.request, executor: fallbackId },
        }, leaseToken, controller);
      }
      if (heartbeatFailure) throw heartbeatFailure;
      await client.complete(job.jobId, { leaseToken, result });
      return { processed: true, jobId: job.jobId, status: 'succeeded' };
    } catch (error) {
      const failure = controller.signal.aborted && !heartbeatFailure
        ? {
            code: 'EXECUTOR_TIMEOUT_OR_CANCELLED',
            message: 'Executor was cancelled or exceeded its timeout',
            retryable: false,
          }
        : {
            code: String(error?.code ?? 'EXECUTOR_WORKER_FAILED').slice(0, 128),
            message: String(error?.message ?? error ?? 'Executor worker failed').slice(0, 2048),
            retryable: error?.retryable === true,
          };
      try {
        await client.fail(job.jobId, {
          leaseToken,
          error: failure,
          retryDelayMs: failure.retryable ? 2_000 : 0,
        });
      } catch (reportError) {
        logger.error?.(
          `[executor-worker] failed to report ${job.jobId}: ${reportError.message}`,
        );
        throw reportError;
      }
      return { processed: true, jobId: job.jobId, status: 'failed', error: failure };
    } finally {
      clearTimeout(timeout);
      clearInterval(heartbeat);
      active.delete(job.jobId);
    }
  }

  return {
    async runOnce() {
      if (stopping) return { processed: false, reason: 'stopping' };
      lastAdapterHealth = await registry.health();
      const healthyExecutors = registry.ids().filter(
        (executor) => lastAdapterHealth[executor]?.ok === true,
      );
      const claim = await client.claim({
        workerId: normalizedWorkerId,
        executors: healthyExecutors.length ? healthyExecutors : registry.ids(),
        leaseDurationMs,
        draining: healthyExecutors.length === 0,
        adapterHealth: lastAdapterHealth,
      });
      if (!claim) {
        return {
          processed: false,
          reason: healthyExecutors.length ? 'queue_empty' : 'adapters_unhealthy',
        };
      }
      return processClaim(claim);
    },
    async recoverExpired(options = {}) {
      return client.recoverExpired(options);
    },
    status() {
      return {
        version: 'executor-worker-runtime-v1',
        workerId: normalizedWorkerId,
        stopping,
        activeJobs: [...active.keys()],
        adapters: registry.list(),
        adapterHealth: lastAdapterHealth,
        heartbeatMs,
        leaseDurationMs,
      };
    },
    async stop() {
      stopping = true;
      await client.claim({
        workerId: normalizedWorkerId,
        executors: registry.ids(),
        leaseDurationMs,
        draining: true,
        adapterHealth: lastAdapterHealth,
      }).catch(() => null);
      for (const { controller } of active.values()) controller.abort('worker_stopping');
    },
  };
}

export const executorWorkerRuntimeInternals = {
  safeJson,
  workerError,
};