Add ez-assistant and kerberos service folders
This commit is contained in:
kelin
@@ -0,0 +1,8 @@
|
||||
{
|
||||
"id": "diagnostics-otel",
|
||||
"configSchema": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"properties": {}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
import type { MoltbotPluginApi } from "clawdbot/plugin-sdk";
|
||||
import { emptyPluginConfigSchema } from "clawdbot/plugin-sdk";
|
||||
|
||||
import { createDiagnosticsOtelService } from "./src/service.js";
|
||||
|
||||
const plugin = {
|
||||
id: "diagnostics-otel",
|
||||
name: "Diagnostics OpenTelemetry",
|
||||
description: "Export diagnostics events to OpenTelemetry",
|
||||
configSchema: emptyPluginConfigSchema(),
|
||||
register(api: MoltbotPluginApi) {
|
||||
api.registerService(createDiagnosticsOtelService());
|
||||
},
|
||||
};
|
||||
|
||||
export default plugin;
|
||||
@@ -0,0 +1,24 @@
|
||||
{
|
||||
"name": "@moltbot/diagnostics-otel",
|
||||
"version": "2026.1.26",
|
||||
"type": "module",
|
||||
"description": "Moltbot diagnostics OpenTelemetry exporter",
|
||||
"moltbot": {
|
||||
"extensions": [
|
||||
"./index.ts"
|
||||
]
|
||||
},
|
||||
"dependencies": {
|
||||
"@opentelemetry/api": "^1.9.0",
|
||||
"@opentelemetry/api-logs": "^0.211.0",
|
||||
"@opentelemetry/exporter-logs-otlp-http": "^0.211.0",
|
||||
"@opentelemetry/exporter-metrics-otlp-http": "^0.211.0",
|
||||
"@opentelemetry/exporter-trace-otlp-http": "^0.211.0",
|
||||
"@opentelemetry/resources": "^2.5.0",
|
||||
"@opentelemetry/sdk-logs": "^0.211.0",
|
||||
"@opentelemetry/sdk-metrics": "^2.5.0",
|
||||
"@opentelemetry/sdk-node": "^0.211.0",
|
||||
"@opentelemetry/sdk-trace-base": "^2.5.0",
|
||||
"@opentelemetry/semantic-conventions": "^1.39.0"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,220 @@
|
||||
import { beforeEach, describe, expect, test, vi } from "vitest";
|
||||
|
||||
const registerLogTransportMock = vi.hoisted(() => vi.fn());
|
||||
|
||||
const telemetryState = vi.hoisted(() => {
|
||||
const counters = new Map<string, { add: ReturnType<typeof vi.fn> }>();
|
||||
const histograms = new Map<string, { record: ReturnType<typeof vi.fn> }>();
|
||||
const tracer = {
|
||||
startSpan: vi.fn((_name: string, _opts?: unknown) => ({
|
||||
end: vi.fn(),
|
||||
setStatus: vi.fn(),
|
||||
})),
|
||||
};
|
||||
const meter = {
|
||||
createCounter: vi.fn((name: string) => {
|
||||
const counter = { add: vi.fn() };
|
||||
counters.set(name, counter);
|
||||
return counter;
|
||||
}),
|
||||
createHistogram: vi.fn((name: string) => {
|
||||
const histogram = { record: vi.fn() };
|
||||
histograms.set(name, histogram);
|
||||
return histogram;
|
||||
}),
|
||||
};
|
||||
return { counters, histograms, tracer, meter };
|
||||
});
|
||||
|
||||
const sdkStart = vi.hoisted(() => vi.fn().mockResolvedValue(undefined));
|
||||
const sdkShutdown = vi.hoisted(() => vi.fn().mockResolvedValue(undefined));
|
||||
const logEmit = vi.hoisted(() => vi.fn());
|
||||
const logShutdown = vi.hoisted(() => vi.fn().mockResolvedValue(undefined));
|
||||
|
||||
vi.mock("@opentelemetry/api", () => ({
|
||||
metrics: {
|
||||
getMeter: () => telemetryState.meter,
|
||||
},
|
||||
trace: {
|
||||
getTracer: () => telemetryState.tracer,
|
||||
},
|
||||
SpanStatusCode: {
|
||||
ERROR: 2,
|
||||
},
|
||||
}));
|
||||
|
||||
vi.mock("@opentelemetry/sdk-node", () => ({
|
||||
NodeSDK: class {
|
||||
start = sdkStart;
|
||||
shutdown = sdkShutdown;
|
||||
},
|
||||
}));
|
||||
|
||||
vi.mock("@opentelemetry/exporter-metrics-otlp-http", () => ({
|
||||
OTLPMetricExporter: class {},
|
||||
}));
|
||||
|
||||
vi.mock("@opentelemetry/exporter-trace-otlp-http", () => ({
|
||||
OTLPTraceExporter: class {},
|
||||
}));
|
||||
|
||||
vi.mock("@opentelemetry/exporter-logs-otlp-http", () => ({
|
||||
OTLPLogExporter: class {},
|
||||
}));
|
||||
|
||||
vi.mock("@opentelemetry/sdk-logs", () => ({
|
||||
BatchLogRecordProcessor: class {},
|
||||
LoggerProvider: class {
|
||||
addLogRecordProcessor = vi.fn();
|
||||
getLogger = vi.fn(() => ({
|
||||
emit: logEmit,
|
||||
}));
|
||||
shutdown = logShutdown;
|
||||
},
|
||||
}));
|
||||
|
||||
vi.mock("@opentelemetry/sdk-metrics", () => ({
|
||||
PeriodicExportingMetricReader: class {},
|
||||
}));
|
||||
|
||||
vi.mock("@opentelemetry/sdk-trace-base", () => ({
|
||||
ParentBasedSampler: class {},
|
||||
TraceIdRatioBasedSampler: class {},
|
||||
}));
|
||||
|
||||
vi.mock("@opentelemetry/resources", () => ({
|
||||
Resource: class {
|
||||
// eslint-disable-next-line @typescript-eslint/no-useless-constructor
|
||||
constructor(_value?: unknown) {}
|
||||
},
|
||||
}));
|
||||
|
||||
vi.mock("@opentelemetry/semantic-conventions", () => ({
|
||||
SemanticResourceAttributes: {
|
||||
SERVICE_NAME: "service.name",
|
||||
},
|
||||
}));
|
||||
|
||||
vi.mock("clawdbot/plugin-sdk", async () => {
|
||||
const actual = await vi.importActual<typeof import("clawdbot/plugin-sdk")>("clawdbot/plugin-sdk");
|
||||
return {
|
||||
...actual,
|
||||
registerLogTransport: registerLogTransportMock,
|
||||
};
|
||||
});
|
||||
|
||||
import { createDiagnosticsOtelService } from "./service.js";
|
||||
import { emitDiagnosticEvent } from "clawdbot/plugin-sdk";
|
||||
|
||||
describe("diagnostics-otel service", () => {
|
||||
beforeEach(() => {
|
||||
telemetryState.counters.clear();
|
||||
telemetryState.histograms.clear();
|
||||
telemetryState.tracer.startSpan.mockClear();
|
||||
telemetryState.meter.createCounter.mockClear();
|
||||
telemetryState.meter.createHistogram.mockClear();
|
||||
sdkStart.mockClear();
|
||||
sdkShutdown.mockClear();
|
||||
logEmit.mockClear();
|
||||
logShutdown.mockClear();
|
||||
registerLogTransportMock.mockReset();
|
||||
});
|
||||
|
||||
test("records message-flow metrics and spans", async () => {
|
||||
const registeredTransports: Array<(logObj: Record<string, unknown>) => void> = [];
|
||||
const stopTransport = vi.fn();
|
||||
registerLogTransportMock.mockImplementation((transport) => {
|
||||
registeredTransports.push(transport);
|
||||
return stopTransport;
|
||||
});
|
||||
|
||||
const service = createDiagnosticsOtelService();
|
||||
await service.start({
|
||||
config: {
|
||||
diagnostics: {
|
||||
enabled: true,
|
||||
otel: {
|
||||
enabled: true,
|
||||
endpoint: "http://otel-collector:4318",
|
||||
protocol: "http/protobuf",
|
||||
traces: true,
|
||||
metrics: true,
|
||||
logs: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
logger: {
|
||||
info: vi.fn(),
|
||||
warn: vi.fn(),
|
||||
error: vi.fn(),
|
||||
debug: vi.fn(),
|
||||
},
|
||||
});
|
||||
|
||||
emitDiagnosticEvent({
|
||||
type: "webhook.received",
|
||||
channel: "telegram",
|
||||
updateType: "telegram-post",
|
||||
});
|
||||
emitDiagnosticEvent({
|
||||
type: "webhook.processed",
|
||||
channel: "telegram",
|
||||
updateType: "telegram-post",
|
||||
durationMs: 120,
|
||||
});
|
||||
emitDiagnosticEvent({
|
||||
type: "message.queued",
|
||||
channel: "telegram",
|
||||
source: "telegram",
|
||||
queueDepth: 2,
|
||||
});
|
||||
emitDiagnosticEvent({
|
||||
type: "message.processed",
|
||||
channel: "telegram",
|
||||
outcome: "completed",
|
||||
durationMs: 55,
|
||||
});
|
||||
emitDiagnosticEvent({
|
||||
type: "queue.lane.dequeue",
|
||||
lane: "main",
|
||||
queueSize: 3,
|
||||
waitMs: 10,
|
||||
});
|
||||
emitDiagnosticEvent({
|
||||
type: "session.stuck",
|
||||
state: "processing",
|
||||
ageMs: 125_000,
|
||||
});
|
||||
emitDiagnosticEvent({
|
||||
type: "run.attempt",
|
||||
runId: "run-1",
|
||||
attempt: 2,
|
||||
});
|
||||
|
||||
expect(telemetryState.counters.get("moltbot.webhook.received")?.add).toHaveBeenCalled();
|
||||
expect(telemetryState.histograms.get("moltbot.webhook.duration_ms")?.record).toHaveBeenCalled();
|
||||
expect(telemetryState.counters.get("moltbot.message.queued")?.add).toHaveBeenCalled();
|
||||
expect(telemetryState.counters.get("moltbot.message.processed")?.add).toHaveBeenCalled();
|
||||
expect(telemetryState.histograms.get("moltbot.message.duration_ms")?.record).toHaveBeenCalled();
|
||||
expect(telemetryState.histograms.get("moltbot.queue.wait_ms")?.record).toHaveBeenCalled();
|
||||
expect(telemetryState.counters.get("moltbot.session.stuck")?.add).toHaveBeenCalled();
|
||||
expect(telemetryState.histograms.get("moltbot.session.stuck_age_ms")?.record).toHaveBeenCalled();
|
||||
expect(telemetryState.counters.get("moltbot.run.attempt")?.add).toHaveBeenCalled();
|
||||
|
||||
const spanNames = telemetryState.tracer.startSpan.mock.calls.map((call) => call[0]);
|
||||
expect(spanNames).toContain("moltbot.webhook.processed");
|
||||
expect(spanNames).toContain("moltbot.message.processed");
|
||||
expect(spanNames).toContain("moltbot.session.stuck");
|
||||
|
||||
expect(registerLogTransportMock).toHaveBeenCalledTimes(1);
|
||||
expect(registeredTransports).toHaveLength(1);
|
||||
registeredTransports[0]?.({
|
||||
0: "{\"subsystem\":\"diagnostic\"}",
|
||||
1: "hello",
|
||||
_meta: { logLevelName: "INFO", date: new Date() },
|
||||
});
|
||||
expect(logEmit).toHaveBeenCalled();
|
||||
|
||||
await service.stop?.();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,566 @@
|
||||
import { metrics, trace, SpanStatusCode } from "@opentelemetry/api";
|
||||
import type { SeverityNumber } from "@opentelemetry/api-logs";
|
||||
import { OTLPLogExporter } from "@opentelemetry/exporter-logs-otlp-http";
|
||||
import { OTLPMetricExporter } from "@opentelemetry/exporter-metrics-otlp-http";
|
||||
import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-http";
|
||||
import { Resource } from "@opentelemetry/resources";
|
||||
import { BatchLogRecordProcessor, LoggerProvider } from "@opentelemetry/sdk-logs";
|
||||
import { PeriodicExportingMetricReader } from "@opentelemetry/sdk-metrics";
|
||||
import { NodeSDK } from "@opentelemetry/sdk-node";
|
||||
import { ParentBasedSampler, TraceIdRatioBasedSampler } from "@opentelemetry/sdk-trace-base";
|
||||
import { SemanticResourceAttributes } from "@opentelemetry/semantic-conventions";
|
||||
|
||||
import type { MoltbotPluginService, DiagnosticEventPayload } from "clawdbot/plugin-sdk";
|
||||
import { onDiagnosticEvent, registerLogTransport } from "clawdbot/plugin-sdk";
|
||||
|
||||
const DEFAULT_SERVICE_NAME = "moltbot";
|
||||
|
||||
function normalizeEndpoint(endpoint?: string): string | undefined {
|
||||
const trimmed = endpoint?.trim();
|
||||
return trimmed ? trimmed.replace(/\/+$/, "") : undefined;
|
||||
}
|
||||
|
||||
function resolveOtelUrl(endpoint: string | undefined, path: string): string | undefined {
|
||||
if (!endpoint) return undefined;
|
||||
if (endpoint.includes("/v1/")) return endpoint;
|
||||
return `${endpoint}/${path}`;
|
||||
}
|
||||
|
||||
function resolveSampleRate(value: number | undefined): number | undefined {
|
||||
if (typeof value !== "number" || !Number.isFinite(value)) return undefined;
|
||||
if (value < 0 || value > 1) return undefined;
|
||||
return value;
|
||||
}
|
||||
|
||||
export function createDiagnosticsOtelService(): MoltbotPluginService {
|
||||
let sdk: NodeSDK | null = null;
|
||||
let logProvider: LoggerProvider | null = null;
|
||||
let stopLogTransport: (() => void) | null = null;
|
||||
let unsubscribe: (() => void) | null = null;
|
||||
|
||||
return {
|
||||
id: "diagnostics-otel",
|
||||
async start(ctx) {
|
||||
const cfg = ctx.config.diagnostics;
|
||||
const otel = cfg?.otel;
|
||||
if (!cfg?.enabled || !otel?.enabled) return;
|
||||
|
||||
const protocol = otel.protocol ?? process.env.OTEL_EXPORTER_OTLP_PROTOCOL ?? "http/protobuf";
|
||||
if (protocol !== "http/protobuf") {
|
||||
ctx.logger.warn(`diagnostics-otel: unsupported protocol ${protocol}`);
|
||||
return;
|
||||
}
|
||||
|
||||
const endpoint = normalizeEndpoint(otel.endpoint ?? process.env.OTEL_EXPORTER_OTLP_ENDPOINT);
|
||||
const headers = otel.headers ?? undefined;
|
||||
const serviceName =
|
||||
otel.serviceName?.trim() || process.env.OTEL_SERVICE_NAME || DEFAULT_SERVICE_NAME;
|
||||
const sampleRate = resolveSampleRate(otel.sampleRate);
|
||||
|
||||
const tracesEnabled = otel.traces !== false;
|
||||
const metricsEnabled = otel.metrics !== false;
|
||||
const logsEnabled = otel.logs === true;
|
||||
if (!tracesEnabled && !metricsEnabled && !logsEnabled) return;
|
||||
|
||||
const resource = new Resource({
|
||||
[SemanticResourceAttributes.SERVICE_NAME]: serviceName,
|
||||
});
|
||||
|
||||
const traceUrl = resolveOtelUrl(endpoint, "v1/traces");
|
||||
const metricUrl = resolveOtelUrl(endpoint, "v1/metrics");
|
||||
const logUrl = resolveOtelUrl(endpoint, "v1/logs");
|
||||
const traceExporter = tracesEnabled
|
||||
? new OTLPTraceExporter({
|
||||
...(traceUrl ? { url: traceUrl } : {}),
|
||||
...(headers ? { headers } : {}),
|
||||
})
|
||||
: undefined;
|
||||
|
||||
const metricExporter = metricsEnabled
|
||||
? new OTLPMetricExporter({
|
||||
...(metricUrl ? { url: metricUrl } : {}),
|
||||
...(headers ? { headers } : {}),
|
||||
})
|
||||
: undefined;
|
||||
|
||||
const metricReader = metricExporter
|
||||
? new PeriodicExportingMetricReader({
|
||||
exporter: metricExporter,
|
||||
...(typeof otel.flushIntervalMs === "number"
|
||||
? { exportIntervalMillis: Math.max(1000, otel.flushIntervalMs) }
|
||||
: {}),
|
||||
})
|
||||
: undefined;
|
||||
|
||||
if (tracesEnabled || metricsEnabled) {
|
||||
sdk = new NodeSDK({
|
||||
resource,
|
||||
...(traceExporter ? { traceExporter } : {}),
|
||||
...(metricReader ? { metricReader } : {}),
|
||||
...(sampleRate !== undefined
|
||||
? {
|
||||
sampler: new ParentBasedSampler({
|
||||
root: new TraceIdRatioBasedSampler(sampleRate),
|
||||
}),
|
||||
}
|
||||
: {}),
|
||||
});
|
||||
|
||||
await sdk.start();
|
||||
}
|
||||
|
||||
const logSeverityMap: Record<string, SeverityNumber> = {
|
||||
TRACE: 1 as SeverityNumber,
|
||||
DEBUG: 5 as SeverityNumber,
|
||||
INFO: 9 as SeverityNumber,
|
||||
WARN: 13 as SeverityNumber,
|
||||
ERROR: 17 as SeverityNumber,
|
||||
FATAL: 21 as SeverityNumber,
|
||||
};
|
||||
|
||||
const meter = metrics.getMeter("moltbot");
|
||||
const tracer = trace.getTracer("moltbot");
|
||||
|
||||
const tokensCounter = meter.createCounter("moltbot.tokens", {
|
||||
unit: "1",
|
||||
description: "Token usage by type",
|
||||
});
|
||||
const costCounter = meter.createCounter("moltbot.cost.usd", {
|
||||
unit: "1",
|
||||
description: "Estimated model cost (USD)",
|
||||
});
|
||||
const durationHistogram = meter.createHistogram("moltbot.run.duration_ms", {
|
||||
unit: "ms",
|
||||
description: "Agent run duration",
|
||||
});
|
||||
const contextHistogram = meter.createHistogram("moltbot.context.tokens", {
|
||||
unit: "1",
|
||||
description: "Context window size and usage",
|
||||
});
|
||||
const webhookReceivedCounter = meter.createCounter("moltbot.webhook.received", {
|
||||
unit: "1",
|
||||
description: "Webhook requests received",
|
||||
});
|
||||
const webhookErrorCounter = meter.createCounter("moltbot.webhook.error", {
|
||||
unit: "1",
|
||||
description: "Webhook processing errors",
|
||||
});
|
||||
const webhookDurationHistogram = meter.createHistogram("moltbot.webhook.duration_ms", {
|
||||
unit: "ms",
|
||||
description: "Webhook processing duration",
|
||||
});
|
||||
const messageQueuedCounter = meter.createCounter("moltbot.message.queued", {
|
||||
unit: "1",
|
||||
description: "Messages queued for processing",
|
||||
});
|
||||
const messageProcessedCounter = meter.createCounter("moltbot.message.processed", {
|
||||
unit: "1",
|
||||
description: "Messages processed by outcome",
|
||||
});
|
||||
const messageDurationHistogram = meter.createHistogram("moltbot.message.duration_ms", {
|
||||
unit: "ms",
|
||||
description: "Message processing duration",
|
||||
});
|
||||
const queueDepthHistogram = meter.createHistogram("moltbot.queue.depth", {
|
||||
unit: "1",
|
||||
description: "Queue depth on enqueue/dequeue",
|
||||
});
|
||||
const queueWaitHistogram = meter.createHistogram("moltbot.queue.wait_ms", {
|
||||
unit: "ms",
|
||||
description: "Queue wait time before execution",
|
||||
});
|
||||
const laneEnqueueCounter = meter.createCounter("moltbot.queue.lane.enqueue", {
|
||||
unit: "1",
|
||||
description: "Command queue lane enqueue events",
|
||||
});
|
||||
const laneDequeueCounter = meter.createCounter("moltbot.queue.lane.dequeue", {
|
||||
unit: "1",
|
||||
description: "Command queue lane dequeue events",
|
||||
});
|
||||
const sessionStateCounter = meter.createCounter("moltbot.session.state", {
|
||||
unit: "1",
|
||||
description: "Session state transitions",
|
||||
});
|
||||
const sessionStuckCounter = meter.createCounter("moltbot.session.stuck", {
|
||||
unit: "1",
|
||||
description: "Sessions stuck in processing",
|
||||
});
|
||||
const sessionStuckAgeHistogram = meter.createHistogram("moltbot.session.stuck_age_ms", {
|
||||
unit: "ms",
|
||||
description: "Age of stuck sessions",
|
||||
});
|
||||
const runAttemptCounter = meter.createCounter("moltbot.run.attempt", {
|
||||
unit: "1",
|
||||
description: "Run attempts",
|
||||
});
|
||||
|
||||
if (logsEnabled) {
|
||||
const logExporter = new OTLPLogExporter({
|
||||
...(logUrl ? { url: logUrl } : {}),
|
||||
...(headers ? { headers } : {}),
|
||||
});
|
||||
logProvider = new LoggerProvider({ resource });
|
||||
logProvider.addLogRecordProcessor(
|
||||
new BatchLogRecordProcessor(logExporter, {
|
||||
...(typeof otel.flushIntervalMs === "number"
|
||||
? { scheduledDelayMillis: Math.max(1000, otel.flushIntervalMs) }
|
||||
: {}),
|
||||
}),
|
||||
);
|
||||
const otelLogger = logProvider.getLogger("moltbot");
|
||||
|
||||
stopLogTransport = registerLogTransport((logObj) => {
|
||||
const safeStringify = (value: unknown) => {
|
||||
try {
|
||||
return JSON.stringify(value);
|
||||
} catch {
|
||||
return String(value);
|
||||
}
|
||||
};
|
||||
const meta = (logObj as Record<string, unknown>)._meta as
|
||||
| {
|
||||
logLevelName?: string;
|
||||
date?: Date;
|
||||
name?: string;
|
||||
parentNames?: string[];
|
||||
path?: {
|
||||
filePath?: string;
|
||||
fileLine?: string;
|
||||
fileColumn?: string;
|
||||
filePathWithLine?: string;
|
||||
method?: string;
|
||||
};
|
||||
}
|
||||
| undefined;
|
||||
const logLevelName = meta?.logLevelName ?? "INFO";
|
||||
const severityNumber = logSeverityMap[logLevelName] ?? (9 as SeverityNumber);
|
||||
|
||||
const numericArgs = Object.entries(logObj)
|
||||
.filter(([key]) => /^\d+$/.test(key))
|
||||
.sort((a, b) => Number(a[0]) - Number(b[0]))
|
||||
.map(([, value]) => value);
|
||||
|
||||
let bindings: Record<string, unknown> | undefined;
|
||||
if (typeof numericArgs[0] === "string" && numericArgs[0].trim().startsWith("{")) {
|
||||
try {
|
||||
const parsed = JSON.parse(numericArgs[0]);
|
||||
if (parsed && typeof parsed === "object" && !Array.isArray(parsed)) {
|
||||
bindings = parsed as Record<string, unknown>;
|
||||
numericArgs.shift();
|
||||
}
|
||||
} catch {
|
||||
// ignore malformed json bindings
|
||||
}
|
||||
}
|
||||
|
||||
let message = "";
|
||||
if (numericArgs.length > 0 && typeof numericArgs[numericArgs.length - 1] === "string") {
|
||||
message = String(numericArgs.pop());
|
||||
} else if (numericArgs.length === 1) {
|
||||
message = safeStringify(numericArgs[0]);
|
||||
numericArgs.length = 0;
|
||||
}
|
||||
if (!message) {
|
||||
message = "log";
|
||||
}
|
||||
|
||||
const attributes: Record<string, string | number | boolean> = {
|
||||
"moltbot.log.level": logLevelName,
|
||||
};
|
||||
if (meta?.name) attributes["moltbot.logger"] = meta.name;
|
||||
if (meta?.parentNames?.length) {
|
||||
attributes["moltbot.logger.parents"] = meta.parentNames.join(".");
|
||||
}
|
||||
if (bindings) {
|
||||
for (const [key, value] of Object.entries(bindings)) {
|
||||
if (typeof value === "string" || typeof value === "number" || typeof value === "boolean") {
|
||||
attributes[`moltbot.${key}`] = value;
|
||||
} else if (value != null) {
|
||||
attributes[`moltbot.${key}`] = safeStringify(value);
|
||||
}
|
||||
}
|
||||
}
|
||||
if (numericArgs.length > 0) {
|
||||
attributes["moltbot.log.args"] = safeStringify(numericArgs);
|
||||
}
|
||||
if (meta?.path?.filePath) attributes["code.filepath"] = meta.path.filePath;
|
||||
if (meta?.path?.fileLine) attributes["code.lineno"] = Number(meta.path.fileLine);
|
||||
if (meta?.path?.method) attributes["code.function"] = meta.path.method;
|
||||
if (meta?.path?.filePathWithLine) {
|
||||
attributes["moltbot.code.location"] = meta.path.filePathWithLine;
|
||||
}
|
||||
|
||||
otelLogger.emit({
|
||||
body: message,
|
||||
severityText: logLevelName,
|
||||
severityNumber,
|
||||
attributes,
|
||||
timestamp: meta?.date ?? new Date(),
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
const spanWithDuration = (
|
||||
name: string,
|
||||
attributes: Record<string, string | number>,
|
||||
durationMs?: number,
|
||||
) => {
|
||||
const startTime =
|
||||
typeof durationMs === "number" ? Date.now() - Math.max(0, durationMs) : undefined;
|
||||
const span = tracer.startSpan(name, {
|
||||
attributes,
|
||||
...(startTime ? { startTime } : {}),
|
||||
});
|
||||
return span;
|
||||
};
|
||||
|
||||
const recordModelUsage = (evt: Extract<DiagnosticEventPayload, { type: "model.usage" }>) => {
|
||||
const attrs = {
|
||||
"moltbot.channel": evt.channel ?? "unknown",
|
||||
"moltbot.provider": evt.provider ?? "unknown",
|
||||
"moltbot.model": evt.model ?? "unknown",
|
||||
};
|
||||
|
||||
const usage = evt.usage;
|
||||
if (usage.input) tokensCounter.add(usage.input, { ...attrs, "moltbot.token": "input" });
|
||||
if (usage.output) tokensCounter.add(usage.output, { ...attrs, "moltbot.token": "output" });
|
||||
if (usage.cacheRead)
|
||||
tokensCounter.add(usage.cacheRead, { ...attrs, "moltbot.token": "cache_read" });
|
||||
if (usage.cacheWrite)
|
||||
tokensCounter.add(usage.cacheWrite, { ...attrs, "moltbot.token": "cache_write" });
|
||||
if (usage.promptTokens)
|
||||
tokensCounter.add(usage.promptTokens, { ...attrs, "moltbot.token": "prompt" });
|
||||
if (usage.total) tokensCounter.add(usage.total, { ...attrs, "moltbot.token": "total" });
|
||||
|
||||
if (evt.costUsd) costCounter.add(evt.costUsd, attrs);
|
||||
if (evt.durationMs) durationHistogram.record(evt.durationMs, attrs);
|
||||
if (evt.context?.limit)
|
||||
contextHistogram.record(evt.context.limit, {
|
||||
...attrs,
|
||||
"moltbot.context": "limit",
|
||||
});
|
||||
if (evt.context?.used)
|
||||
contextHistogram.record(evt.context.used, {
|
||||
...attrs,
|
||||
"moltbot.context": "used",
|
||||
});
|
||||
|
||||
if (!tracesEnabled) return;
|
||||
const spanAttrs: Record<string, string | number> = {
|
||||
...attrs,
|
||||
"moltbot.sessionKey": evt.sessionKey ?? "",
|
||||
"moltbot.sessionId": evt.sessionId ?? "",
|
||||
"moltbot.tokens.input": usage.input ?? 0,
|
||||
"moltbot.tokens.output": usage.output ?? 0,
|
||||
"moltbot.tokens.cache_read": usage.cacheRead ?? 0,
|
||||
"moltbot.tokens.cache_write": usage.cacheWrite ?? 0,
|
||||
"moltbot.tokens.total": usage.total ?? 0,
|
||||
};
|
||||
|
||||
const span = spanWithDuration("moltbot.model.usage", spanAttrs, evt.durationMs);
|
||||
span.end();
|
||||
};
|
||||
|
||||
const recordWebhookReceived = (
|
||||
evt: Extract<DiagnosticEventPayload, { type: "webhook.received" }>,
|
||||
) => {
|
||||
const attrs = {
|
||||
"moltbot.channel": evt.channel ?? "unknown",
|
||||
"moltbot.webhook": evt.updateType ?? "unknown",
|
||||
};
|
||||
webhookReceivedCounter.add(1, attrs);
|
||||
};
|
||||
|
||||
const recordWebhookProcessed = (
|
||||
evt: Extract<DiagnosticEventPayload, { type: "webhook.processed" }>,
|
||||
) => {
|
||||
const attrs = {
|
||||
"moltbot.channel": evt.channel ?? "unknown",
|
||||
"moltbot.webhook": evt.updateType ?? "unknown",
|
||||
};
|
||||
if (typeof evt.durationMs === "number") {
|
||||
webhookDurationHistogram.record(evt.durationMs, attrs);
|
||||
}
|
||||
if (!tracesEnabled) return;
|
||||
const spanAttrs: Record<string, string | number> = { ...attrs };
|
||||
if (evt.chatId !== undefined) spanAttrs["moltbot.chatId"] = String(evt.chatId);
|
||||
const span = spanWithDuration("moltbot.webhook.processed", spanAttrs, evt.durationMs);
|
||||
span.end();
|
||||
};
|
||||
|
||||
const recordWebhookError = (
|
||||
evt: Extract<DiagnosticEventPayload, { type: "webhook.error" }>,
|
||||
) => {
|
||||
const attrs = {
|
||||
"moltbot.channel": evt.channel ?? "unknown",
|
||||
"moltbot.webhook": evt.updateType ?? "unknown",
|
||||
};
|
||||
webhookErrorCounter.add(1, attrs);
|
||||
if (!tracesEnabled) return;
|
||||
const spanAttrs: Record<string, string | number> = {
|
||||
...attrs,
|
||||
"moltbot.error": evt.error,
|
||||
};
|
||||
if (evt.chatId !== undefined) spanAttrs["moltbot.chatId"] = String(evt.chatId);
|
||||
const span = tracer.startSpan("moltbot.webhook.error", {
|
||||
attributes: spanAttrs,
|
||||
});
|
||||
span.setStatus({ code: SpanStatusCode.ERROR, message: evt.error });
|
||||
span.end();
|
||||
};
|
||||
|
||||
const recordMessageQueued = (
|
||||
evt: Extract<DiagnosticEventPayload, { type: "message.queued" }>,
|
||||
) => {
|
||||
const attrs = {
|
||||
"moltbot.channel": evt.channel ?? "unknown",
|
||||
"moltbot.source": evt.source ?? "unknown",
|
||||
};
|
||||
messageQueuedCounter.add(1, attrs);
|
||||
if (typeof evt.queueDepth === "number") {
|
||||
queueDepthHistogram.record(evt.queueDepth, attrs);
|
||||
}
|
||||
};
|
||||
|
||||
const recordMessageProcessed = (
|
||||
evt: Extract<DiagnosticEventPayload, { type: "message.processed" }>,
|
||||
) => {
|
||||
const attrs = {
|
||||
"moltbot.channel": evt.channel ?? "unknown",
|
||||
"moltbot.outcome": evt.outcome ?? "unknown",
|
||||
};
|
||||
messageProcessedCounter.add(1, attrs);
|
||||
if (typeof evt.durationMs === "number") {
|
||||
messageDurationHistogram.record(evt.durationMs, attrs);
|
||||
}
|
||||
if (!tracesEnabled) return;
|
||||
const spanAttrs: Record<string, string | number> = { ...attrs };
|
||||
if (evt.sessionKey) spanAttrs["moltbot.sessionKey"] = evt.sessionKey;
|
||||
if (evt.sessionId) spanAttrs["moltbot.sessionId"] = evt.sessionId;
|
||||
if (evt.chatId !== undefined) spanAttrs["moltbot.chatId"] = String(evt.chatId);
|
||||
if (evt.messageId !== undefined) spanAttrs["moltbot.messageId"] = String(evt.messageId);
|
||||
if (evt.reason) spanAttrs["moltbot.reason"] = evt.reason;
|
||||
const span = spanWithDuration("moltbot.message.processed", spanAttrs, evt.durationMs);
|
||||
if (evt.outcome === "error") {
|
||||
span.setStatus({ code: SpanStatusCode.ERROR, message: evt.error });
|
||||
}
|
||||
span.end();
|
||||
};
|
||||
|
||||
const recordLaneEnqueue = (
|
||||
evt: Extract<DiagnosticEventPayload, { type: "queue.lane.enqueue" }>,
|
||||
) => {
|
||||
const attrs = { "moltbot.lane": evt.lane };
|
||||
laneEnqueueCounter.add(1, attrs);
|
||||
queueDepthHistogram.record(evt.queueSize, attrs);
|
||||
};
|
||||
|
||||
const recordLaneDequeue = (
|
||||
evt: Extract<DiagnosticEventPayload, { type: "queue.lane.dequeue" }>,
|
||||
) => {
|
||||
const attrs = { "moltbot.lane": evt.lane };
|
||||
laneDequeueCounter.add(1, attrs);
|
||||
queueDepthHistogram.record(evt.queueSize, attrs);
|
||||
if (typeof evt.waitMs === "number") {
|
||||
queueWaitHistogram.record(evt.waitMs, attrs);
|
||||
}
|
||||
};
|
||||
|
||||
const recordSessionState = (
|
||||
evt: Extract<DiagnosticEventPayload, { type: "session.state" }>,
|
||||
) => {
|
||||
const attrs: Record<string, string> = { "moltbot.state": evt.state };
|
||||
if (evt.reason) attrs["moltbot.reason"] = evt.reason;
|
||||
sessionStateCounter.add(1, attrs);
|
||||
};
|
||||
|
||||
const recordSessionStuck = (
|
||||
evt: Extract<DiagnosticEventPayload, { type: "session.stuck" }>,
|
||||
) => {
|
||||
const attrs: Record<string, string> = { "moltbot.state": evt.state };
|
||||
sessionStuckCounter.add(1, attrs);
|
||||
if (typeof evt.ageMs === "number") {
|
||||
sessionStuckAgeHistogram.record(evt.ageMs, attrs);
|
||||
}
|
||||
if (!tracesEnabled) return;
|
||||
const spanAttrs: Record<string, string | number> = { ...attrs };
|
||||
if (evt.sessionKey) spanAttrs["moltbot.sessionKey"] = evt.sessionKey;
|
||||
if (evt.sessionId) spanAttrs["moltbot.sessionId"] = evt.sessionId;
|
||||
spanAttrs["moltbot.queueDepth"] = evt.queueDepth ?? 0;
|
||||
spanAttrs["moltbot.ageMs"] = evt.ageMs;
|
||||
const span = tracer.startSpan("moltbot.session.stuck", { attributes: spanAttrs });
|
||||
span.setStatus({ code: SpanStatusCode.ERROR, message: "session stuck" });
|
||||
span.end();
|
||||
};
|
||||
|
||||
const recordRunAttempt = (evt: Extract<DiagnosticEventPayload, { type: "run.attempt" }>) => {
|
||||
runAttemptCounter.add(1, { "moltbot.attempt": evt.attempt });
|
||||
};
|
||||
|
||||
const recordHeartbeat = (
|
||||
evt: Extract<DiagnosticEventPayload, { type: "diagnostic.heartbeat" }>,
|
||||
) => {
|
||||
queueDepthHistogram.record(evt.queued, { "moltbot.channel": "heartbeat" });
|
||||
};
|
||||
|
||||
unsubscribe = onDiagnosticEvent((evt: DiagnosticEventPayload) => {
|
||||
switch (evt.type) {
|
||||
case "model.usage":
|
||||
recordModelUsage(evt);
|
||||
return;
|
||||
case "webhook.received":
|
||||
recordWebhookReceived(evt);
|
||||
return;
|
||||
case "webhook.processed":
|
||||
recordWebhookProcessed(evt);
|
||||
return;
|
||||
case "webhook.error":
|
||||
recordWebhookError(evt);
|
||||
return;
|
||||
case "message.queued":
|
||||
recordMessageQueued(evt);
|
||||
return;
|
||||
case "message.processed":
|
||||
recordMessageProcessed(evt);
|
||||
return;
|
||||
case "queue.lane.enqueue":
|
||||
recordLaneEnqueue(evt);
|
||||
return;
|
||||
case "queue.lane.dequeue":
|
||||
recordLaneDequeue(evt);
|
||||
return;
|
||||
case "session.state":
|
||||
recordSessionState(evt);
|
||||
return;
|
||||
case "session.stuck":
|
||||
recordSessionStuck(evt);
|
||||
return;
|
||||
case "run.attempt":
|
||||
recordRunAttempt(evt);
|
||||
return;
|
||||
case "diagnostic.heartbeat":
|
||||
recordHeartbeat(evt);
|
||||
return;
|
||||
}
|
||||
});
|
||||
|
||||
if (logsEnabled) {
|
||||
ctx.logger.info("diagnostics-otel: logs exporter enabled (OTLP/HTTP)");
|
||||
}
|
||||
},
|
||||
async stop() {
|
||||
unsubscribe?.();
|
||||
unsubscribe = null;
|
||||
stopLogTransport?.();
|
||||
stopLogTransport = null;
|
||||
if (logProvider) {
|
||||
await logProvider.shutdown().catch(() => undefined);
|
||||
logProvider = null;
|
||||
}
|
||||
if (sdk) {
|
||||
await sdk.shutdown().catch(() => undefined);
|
||||
sdk = null;
|
||||
}
|
||||
},
|
||||
} satisfies MoltbotPluginService;
|
||||
}
|
||||
Reference in New Issue
Block a user