diff --git a/.github/workflows/autofix.yml b/.github/workflows/autofix.yml index 2017229..5c70edb 100644 --- a/.github/workflows/autofix.yml +++ b/.github/workflows/autofix.yml @@ -2,8 +2,6 @@ name: autofix.ci on: pull_request: - push: - branches: [main] permissions: contents: read diff --git a/docs/content/docs/packages-ext/telemetry.mdx b/docs/content/docs/packages-ext/telemetry.mdx index 3b74437..2063eac 100644 --- a/docs/content/docs/packages-ext/telemetry.mdx +++ b/docs/content/docs/packages-ext/telemetry.mdx @@ -52,9 +52,9 @@ const { text } = await generateText({ ], model: 'gpt-4o', telemetry: { // [!code highlight] - metadata: { // [!code highlight] - agentId: 'weather-assistant', // [!code highlight] - instructions, // [!code highlight] + attributes: { // [!code highlight] + 'gen_ai.agent.name': 'weather-assistant', // [!code highlight] + 'gen_ai.agent.description': instructions, // [!code highlight] }, // [!code highlight] }, // [!code highlight] }) diff --git a/packages-ext/telemetry/package.json b/packages-ext/telemetry/package.json index 7cbae8b..00508cb 100644 --- a/packages-ext/telemetry/package.json +++ b/packages-ext/telemetry/package.json @@ -42,10 +42,9 @@ "xsai": "workspace:~" }, "devDependencies": { + "@langfuse/otel": "^4.5.1", "@opentelemetry/sdk-trace-base": "^2.0.1", "@opentelemetry/sdk-trace-node": "^2.0.1", - "ai": "^5.0.28", - "ollama-ai-provider-v2": "^1.2.1", "zod": "^4.1.5" } } diff --git a/packages-ext/telemetry/src/types/options.ts b/packages-ext/telemetry/src/types/options.ts index ce6ba91..08233c3 100644 --- a/packages-ext/telemetry/src/types/options.ts +++ b/packages-ext/telemetry/src/types/options.ts @@ -1,9 +1,7 @@ -import type { AttributeValue } from '@opentelemetry/api' - -export type TelemetryMetadata = Record +import type { Attributes } from '@opentelemetry/api' export interface TelemetryOptions { - metadata?: TelemetryMetadata + attributes?: Attributes // TODO // tracer?: Tracer } diff --git a/packages-ext/telemetry/src/utils/attributes.ts b/packages-ext/telemetry/src/utils/attributes.ts deleted file mode 100644 index d0e73b4..0000000 --- a/packages-ext/telemetry/src/utils/attributes.ts +++ /dev/null @@ -1,27 +0,0 @@ -import type { Attributes } from '@opentelemetry/api' - -import type { TelemetryMetadata } from '../types/options' - -export const metadataAttributes = (metadata: TelemetryMetadata = {}): Attributes => Object.fromEntries( - Object.entries(metadata) - .map(([key, value]) => [`ai.telemetry.metadata.${key}`, value]), -) - -export const idAttributes = (): Attributes => { - const id = crypto.randomUUID() - - return { - 'ai.response.id': id, - 'ai.response.timestamp': new Date().toISOString(), - 'gen_ai.response.id': id, - } -} - -export const commonAttributes = (operationId: string, model: string): Attributes => ({ - 'ai.model.id': model, - // TODO: provider name - 'ai.model.provider': 'xsai', - 'ai.operationId': operationId, - 'ai.response.providerMetadata': '{}', - 'operation.name': operationId, -}) diff --git a/packages-ext/telemetry/src/utils/chat-span.ts b/packages-ext/telemetry/src/utils/chat-span.ts new file mode 100644 index 0000000..e023037 --- /dev/null +++ b/packages-ext/telemetry/src/utils/chat-span.ts @@ -0,0 +1,30 @@ +import type { Tracer } from '@opentelemetry/api' +import type { ChatOptions } from 'xsai' + +import type { WithTelemetry } from '../types/options' +import type { RecordSpanOptions } from './record-span' + +export const chatSpan = (options: WithTelemetry, tracer: Tracer): RecordSpanOptions => ({ + attributes: { + 'gen_ai.input.messages': JSON.stringify(options.messages), + 'gen_ai.operation.name': 'chat', + 'gen_ai.provider.name': 'openai', + 'gen_ai.request.choice.count': 1, + 'gen_ai.request.frequency_penalty': options.frequencyPenalty, + 'gen_ai.request.model': options.model, + 'gen_ai.request.presence_penalty': options.presencePenalty, + 'gen_ai.request.seed': options.seed, + // eslint-disable-next-line sonarjs/no-nested-conditional + 'gen_ai.request.stop_sequences': options.stop == null ? undefined : Array.isArray(options.stop) ? options.stop : [options.stop], + 'gen_ai.request.temperature': options.temperature, + 'gen_ai.request.top_p': options.topP, // TODO: top_k + 'gen_ai.response.id': crypto.randomUUID(), + 'gen_ai.response.model': options.model, + 'gen_ai.tool.definitions': JSON.stringify(options.tools?.map(tool => ({ function: tool.function, type: tool.type }))), + 'server.address': new URL(options.baseURL).host, + ...options.telemetry?.attributes, + // TODO: gen_ai.output.type + }, + name: `chat ${options.model}`, + tracer, +}) diff --git a/packages-ext/telemetry/src/utils/generate-text-internal.ts b/packages-ext/telemetry/src/utils/generate-text-internal.ts deleted file mode 100644 index 8613179..0000000 --- a/packages-ext/telemetry/src/utils/generate-text-internal.ts +++ /dev/null @@ -1,92 +0,0 @@ -import type { CompletionStep, CompletionToolCall, CompletionToolResult, GenerateTextOptions, GenerateTextResponse, Message, ToolCall } from 'xsai' - -import { determineStepType, executeTool } from 'xsai' - -/** @internal */ -export interface RunGenerateTextStepResult { - messages: Message[] - msgToolCalls: ToolCall[] - reasoningText?: string -} - -/** @internal */ -export const extractGenerateTextStep = async ( - options: GenerateTextOptions, - res: GenerateTextResponse, -): Promise<[Omit, 'toolResults'>, RunGenerateTextStepResult]> => { - const { choices, usage } = res - - if (!choices?.length) - throw new Error(`No choices returned, response body: ${JSON.stringify(res)}`) - - const messages: Message[] = [] - - const toolCalls: CompletionToolCall[] = [] - - const { finish_reason: finishReason, message } = choices[0] - const msgToolCalls = message?.tool_calls ?? [] - - const stepType = determineStepType({ - finishReason, - maxSteps: options.maxSteps ?? 1, - stepsLength: options.steps?.length ?? 0, - toolCallsLength: msgToolCalls.length, - }) - - messages.push(message) - - if (finishReason !== 'stop' || stepType !== 'done') { - for (const toolCall of msgToolCalls) { - toolCalls.push({ - args: toolCall.function.arguments!, - toolCallId: toolCall.id, - toolCallType: toolCall.type, - toolName: toolCall.function.name!, - }) - } - } - - return [ - { - finishReason, - stepType, - text: Array.isArray(message.content) - ? message.content.filter(m => m.type === 'text').map(m => m.text).join('\n') - : message.content, - toolCalls, - usage, - }, - { - messages, - msgToolCalls, - reasoningText: message.reasoning_content, - }, - ] -} - -/** @internal */ -export const extractGenerateTextStepPost = async ( - options: GenerateTextOptions, - msgToolCalls: ToolCall[], -): Promise<[CompletionStep['toolResults'], Message[]]> => { - const inputMessages: Message[] = structuredClone(options.messages) - - const outputMessages: Message[] = [] - - const toolResults: CompletionToolResult[] = [] - for (const toolCall of msgToolCalls) { - const { completionToolResult, message } = await executeTool({ - abortSignal: options.abortSignal, - messages: inputMessages, - toolCall, - tools: options.tools, - }) - toolResults.push(completionToolResult) - outputMessages.push(message) - } - - return [ - toolResults, - outputMessages, - ] -} diff --git a/packages-ext/telemetry/src/utils/generate-text.ts b/packages-ext/telemetry/src/utils/generate-text.ts index 8e5b27c..494dc09 100644 --- a/packages-ext/telemetry/src/utils/generate-text.ts +++ b/packages-ext/telemetry/src/utils/generate-text.ts @@ -1,14 +1,12 @@ -import type { CompletionStep, GenerateTextOptions, GenerateTextResponse, GenerateTextResult, Message, TrampolineFn, WithUnknown } from 'xsai' +import type { CompletionStep, CompletionToolCall, CompletionToolResult, GenerateTextOptions, GenerateTextResponse, GenerateTextResult, Message, TrampolineFn, WithUnknown } from 'xsai' -import { chat, responseJSON, trampoline } from 'xsai' +import { chat, determineStepType, executeTool, responseJSON, trampoline } from 'xsai' import type { WithTelemetry } from '../types/options' -import { commonAttributes, idAttributes, metadataAttributes } from './attributes' -import { extractGenerateTextStep, extractGenerateTextStepPost } from './generate-text-internal' +import { chatSpan } from './chat-span' import { getTracer } from './get-tracer' import { recordSpan } from './record-span' -import { stringifyTool } from './stringify-tool' import { wrapTool } from './wrap-tool' /** @@ -18,118 +16,124 @@ import { wrapTool } from './wrap-tool' export const generateText = async (options: WithUnknown>) => { const tracer = getTracer() - const rawGenerateText = async (options: WithUnknown>): Promise> => { - const messages: Message[] = structuredClone(options.messages) - const steps: CompletionStep[] = options.steps ? structuredClone(options.steps) : [] - - const [stepWithoutToolCalls, { messages: msgs1, msgToolCalls, reasoningText }] = await recordSpan({ - attributes: { - ...idAttributes(), - ...commonAttributes('ai.generateText.doGenerate', options.model), - ...metadataAttributes(options.telemetry?.metadata), - ...(options.tools != null && options.tools.length > 0 - ? { - 'ai.prompt.toolChoice': JSON.stringify(options.toolChoice ?? { type: 'auto' }), - 'ai.prompt.tools': options.tools.map(stringifyTool), - } - : {}), - 'ai.prompt.messages': JSON.stringify(messages), - 'ai.response.model': options.model, - 'gen_ai.request.model': options.model, - 'gen_ai.response.id': crypto.randomUUID(), - 'gen_ai.response.model': options.model, - 'gen_ai.system': 'xsai', - }, - name: 'ai.generateText.doGenerate', - tracer, - }, async (span) => { - const res = await chat({ + const rawGenerateText = async (options: WithUnknown>): Promise> => + recordSpan(chatSpan(options, tracer), async span => + chat({ ...options, maxSteps: undefined, steps: undefined, stream: false, - telemetry: undefined, }) .then(responseJSON) + .then(async (res) => { + const { choices, usage } = res - const [step, { messages: msgs, msgToolCalls, reasoningText }] = await extractGenerateTextStep({ - ...options, - messages, - steps, - }, res) + if (!choices?.length) + throw new Error(`No choices returned, response body: ${JSON.stringify(res)}`) - // TODO: metrics counter - span.setAttributes({ - ...((step.text != null && step.toolCalls.length === 0) ? { 'ai.response.text': step.text } : {}), - ...(step.toolCalls.length > 0 ? { 'ai.response.toolCalls': JSON.stringify(step.toolCalls) } : {}), - 'ai.response.finishReason': step.finishReason, - 'ai.usage.completionTokens': step.usage.completion_tokens, - 'ai.usage.promptTokens': step.usage.prompt_tokens, - 'gen_ai.response.finish_reasons': [step.finishReason], - 'gen_ai.usage.input_tokens': step.usage.prompt_tokens, - 'gen_ai.usage.output_tokens': step.usage.completion_tokens, - }) + const messages: Message[] = structuredClone(options.messages) + const steps: CompletionStep[] = options.steps ? structuredClone(options.steps) : [] - return [step, { messages: msgs, msgToolCalls, reasoningText }] - }) + const toolCalls: CompletionToolCall[] = [] + const toolResults: CompletionToolResult[] = [] - const [toolResults, msgs2] = await extractGenerateTextStepPost({ - ...options, - messages, - steps, - }, msgToolCalls) + const { finish_reason: finishReason, message } = choices[0] + const msgToolCalls = message?.tool_calls ?? [] - const step = { ...stepWithoutToolCalls, toolResults } + const stepType = determineStepType({ + finishReason, + maxSteps: options.maxSteps ?? 1, + stepsLength: steps.length, + toolCallsLength: msgToolCalls.length, + }) - steps.push(step) - messages.push(...msgs1, ...msgs2) + messages.push(message) + span.setAttribute('gen_ai.output.messages', JSON.stringify([message])) - if (options.onStepFinish) - await options.onStepFinish(step) + if (finishReason !== 'stop' && stepType !== 'done') { + for (const toolCall of msgToolCalls) { + const { completionToolCall, completionToolResult, message } = await executeTool({ + abortSignal: options.abortSignal, + messages, + toolCall, + tools: options.tools, + }) + toolCalls.push(completionToolCall) + toolResults.push(completionToolResult) + messages.push(message) + } + } - if (step.finishReason === 'stop' || step.stepType === 'done') { - return { - finishReason: step.finishReason, - messages, - reasoningText, - steps, - text: step.text, - toolCalls: step.toolCalls, - toolResults: step.toolResults, - usage: step.usage, - } - } - else { - return async () => rawGenerateText({ - ...options, - messages, - steps, - }) - } - } + const step: CompletionStep = { + finishReason, + stepType, + text: Array.isArray(message.content) + // eslint-disable-next-line sonarjs/no-nested-functions + ? message.content.filter(m => m.type === 'text').map(m => m.text).join('\n') + : message.content, + toolCalls, + toolResults, + usage, + } - return recordSpan({ - attributes: { - ...commonAttributes('ai.generateText', options.model), - ...metadataAttributes(options.telemetry?.metadata), - 'ai.prompt': JSON.stringify({ messages: options.messages }), - }, - name: 'ai.generateText', - tracer, - }, async (span) => { - const result = await trampoline(async () => rawGenerateText({ - ...options, - tools: options.tools?.map(tool => wrapTool(tool, tracer)), - })) + steps.push(step) - span.setAttributes({ - ...(result.toolCalls.length > 0 ? { 'ai.response.toolCalls': JSON.stringify(result.toolCalls) } : {}), - ...(result.text != null ? { 'ai.response.text': result.text } : {}), - 'ai.response.finishReason': result.finishReason, - 'ai.usage.completionTokens': result.usage.completion_tokens, - 'ai.usage.promptTokens': result.usage.prompt_tokens, - }) + // TODO: metrics counter + span.setAttributes({ + 'gen_ai.response.finish_reasons': [step.finishReason], + 'gen_ai.usage.input_tokens': step.usage.prompt_tokens, + 'gen_ai.usage.output_tokens': step.usage.completion_tokens, + }) - return result - }) + if (options.onStepFinish) + await options.onStepFinish(step) + + if (step.finishReason === 'stop' || step.stepType === 'done') { + return { + finishReason: step.finishReason, + messages, + reasoningText: message.reasoning_content, + steps, + text: step.text, + toolCalls: step.toolCalls, + toolResults: step.toolResults, + usage: step.usage, + } + } + else { + // eslint-disable-next-line sonarjs/no-nested-functions + return async () => rawGenerateText({ + ...options, + messages, + steps, + }) + } + })) + + return trampoline(async () => rawGenerateText({ + ...options, + tools: options.tools?.map(tool => wrapTool(tool, tracer)), + })) + // return recordSpan({ + // attributes: { + // ...metadataAttributes(options.telemetry?.metadata), + // ...chatAttributes(options), + // }, + // name: `chat ${options.model}`, + // tracer, + // }, async (span) => { + // const result = await trampoline(async () => rawGenerateText({ + // ...options, + // tools: options.tools?.map(tool => wrapTool(tool, tracer)), + // })) + + // span.setAttributes({ + // 'gen_ai.output.messages': JSON.stringify(result.messages), + // 'gen_ai.response.finish_reasons': [result.finishReason], + // 'gen_ai.usage.input_tokens': result.usage.prompt_tokens, + // 'gen_ai.usage.output_tokens': result.usage.completion_tokens, + // }) + + // return result + // }) } diff --git a/packages-ext/telemetry/src/utils/get-tracer.ts b/packages-ext/telemetry/src/utils/get-tracer.ts index 94f2eef..05fc8d8 100644 --- a/packages-ext/telemetry/src/utils/get-tracer.ts +++ b/packages-ext/telemetry/src/utils/get-tracer.ts @@ -1,4 +1,6 @@ import { trace } from '@opentelemetry/api' +import pkg from '../../package.json' + export const getTracer = () => - trace.getTracer('@xsai-ext/telemetry') + trace.getTracer('@xsai-ext/telemetry', pkg.version) diff --git a/packages-ext/telemetry/src/utils/now.ts b/packages-ext/telemetry/src/utils/now.ts deleted file mode 100644 index 9b994e2..0000000 --- a/packages-ext/telemetry/src/utils/now.ts +++ /dev/null @@ -1 +0,0 @@ -export const now = (): number => globalThis?.performance?.now() ?? Date.now() diff --git a/packages-ext/telemetry/src/utils/stream-text.ts b/packages-ext/telemetry/src/utils/stream-text.ts index b5dc5b5..09b4070 100644 --- a/packages-ext/telemetry/src/utils/stream-text.ts +++ b/packages-ext/telemetry/src/utils/stream-text.ts @@ -1,22 +1,20 @@ -import type { CompletionStep, CompletionToolCall, CompletionToolResult, FinishReason, Message, StreamTextEvent, StreamTextOptions, StreamTextResult, ToolCall, Usage, WithUnknown } from 'xsai' +import type { AssistantMessage, CompletionStep, CompletionToolCall, CompletionToolResult, FinishReason, Message, StreamTextEvent, StreamTextOptions, StreamTextResult, ToolCall, Usage, WithUnknown } from 'xsai' import { chat, DelayedPromise, determineStepType, executeTool, objCamelToSnake, trampoline } from 'xsai' import type { WithTelemetry } from '../types/options' -import { commonAttributes, idAttributes, metadataAttributes } from './attributes' +import { chatSpan } from './chat-span' import { getTracer } from './get-tracer' -import { now } from './now' -import { recordSpan, recordSpanSync } from './record-span' +import { recordSpan } from './record-span' import { transformChunk } from './stream-text-internal' -import { stringifyTool } from './stringify-tool' import { wrapTool } from './wrap-tool' /** * @experimental * Streaming Text with Telemetry. */ -export const streamText = (options: WithUnknown>) => { +export const streamText = (options: WithUnknown>): StreamTextResult => { const tracer = getTracer() // state @@ -56,27 +54,7 @@ export const streamText = (options: WithUnknown ? options.tools.map(tool => wrapTool(tool, tracer)) : undefined - const doStream = async () => recordSpan({ - attributes: { - ...idAttributes(), - ...commonAttributes('ai.streamText.doStream', options.model), - ...metadataAttributes(options.telemetry?.metadata), - ...(tools != null && tools.length > 0 && { - 'ai.prompt.toolChoice': JSON.stringify(options.toolChoice ?? { type: 'auto' }), - 'ai.prompt.tools': tools.map(stringifyTool), - }), - 'ai.prompt.messages': JSON.stringify(options.messages), - 'ai.response.model': options.model, - 'gen_ai.request.model': options.model, - 'gen_ai.response.id': crypto.randomUUID(), - 'gen_ai.response.model': options.model, - 'gen_ai.system': 'xsai', - }, - name: 'ai.streamText.doStream', - tracer, - }, async (span) => { - const startMs = now() - + const doStream = async () => recordSpan(chatSpan({ ...options, messages }, tracer), async (span) => { const { body: stream } = await chat({ ...options, maxSteps: undefined, @@ -119,7 +97,6 @@ export const streamText = (options: WithUnknown const toolCalls: CompletionToolCall[] = [] const toolResults: CompletionToolResult[] = [] let finishReason: FinishReason = 'other' - let firstChunk = true await stream! .pipeThrough(transformChunk()) @@ -131,18 +108,6 @@ export const streamText = (options: WithUnknown close: () => {}, // eslint-disable-next-line sonarjs/cognitive-complexity write: (chunk) => { - // Telemetry - if (firstChunk) { - const msToFirstChunk = now() - startMs - span.addEvent('ai.stream.firstChunk', { - 'ai.response.msToFirstChunk': msToFirstChunk, - }) - span.setAttributes({ - 'ai.response.msToFirstChunk': msToFirstChunk, - }) - firstChunk = false - } - if (chunk.usage) pushUsage(chunk.usage) @@ -205,12 +170,14 @@ export const streamText = (options: WithUnknown }, })) - messages.push({ + const message: AssistantMessage = { content: text, reasoning_content: reasoningText, role: 'assistant', tool_calls: tool_calls.length > 0 ? tool_calls : undefined, - }) + } + messages.push(message) + span.setAttribute('gen_ai.output.messages', JSON.stringify([message])) if (tool_calls.length !== 0) { for (const toolCall of tool_calls) { @@ -251,19 +218,10 @@ export const streamText = (options: WithUnknown pushStep(step) // Telemetry - const msToFinish = now() - startMs - span.addEvent('ai.stream.finish') + // span.addEvent('ai.stream.finish') span.setAttributes({ - 'ai.response.msToFinish': msToFinish, - ...(step.toolCalls.length > 0 && { 'ai.response.toolCalls': JSON.stringify(step.toolCalls) }), - 'ai.response.finishReason': step.finishReason, - 'ai.response.text': step.text != null ? step.text : '', 'gen_ai.response.finish_reasons': [step.finishReason], ...step.usage && { - 'ai.response.avgOutputTokensPerSecond': (1000 * (step.usage.completion_tokens ?? 0)) / msToFinish, - 'ai.usage.inputTokens': step.usage.prompt_tokens, - 'ai.usage.outputTokens': step.usage.completion_tokens, - 'ai.usage.totalTokens': step.usage.total_tokens, 'gen_ai.usage.input_tokens': step.usage.prompt_tokens, 'gen_ai.usage.output_tokens': step.usage.completion_tokens, }, @@ -273,73 +231,69 @@ export const streamText = (options: WithUnknown return async () => doStream() }) - return recordSpanSync({ - attributes: { - ...commonAttributes('ai.streamText', options.model), - ...metadataAttributes(options.telemetry?.metadata), - 'ai.prompt': JSON.stringify({ messages: options.messages }), - }, - endWhenDone: false, - name: 'ai.streamText', - tracer, - }, (rootSpan) => { - void (async () => { - try { - await trampoline(async () => doStream()) + // return recordSpanSync({ + // attributes: { + // ...metadataAttributes(options.telemetry?.metadata), + // ...chatAttributes(options), + // }, + // name: 'xsai.streamText', // TODO: remove + // tracer, + // }, (rootSpan) => { + void (async () => { + try { + await trampoline(async () => doStream()) - eventCtrl?.close() - textCtrl?.close() - reasoningTextCtrl?.close() - } - catch (err) { - eventCtrl?.error(err) - textCtrl?.error(err) - reasoningTextCtrl?.error(err) - - resultSteps.reject(err) - resultMessages.reject(err) - resultUsage.reject(err) - resultTotalUsage.reject(err) - } - finally { - resultSteps.resolve(steps) - resultMessages.resolve(messages) - resultUsage.resolve(usage) - resultTotalUsage.resolve(totalUsage) - - const finishStep = steps.at(-1) - - if (finishStep) { - rootSpan.setAttributes({ - ...(finishStep.toolCalls.length > 0 && { 'ai.response.toolCalls': JSON.stringify(finishStep.toolCalls) }), - 'ai.response.finishReason': finishStep.finishReason, - 'ai.response.text': finishStep.text != null ? finishStep.text : '', - }) - } - - if (totalUsage) { - rootSpan.setAttributes({ - 'ai.usage.inputTokens': totalUsage.prompt_tokens, - 'ai.usage.outputTokens': totalUsage.completion_tokens, - 'ai.usage.totalTokens': totalUsage.total_tokens, - }) - } - - // eslint-disable-next-line sonarjs/void-use - void options.onFinish?.(finishStep) - - rootSpan.end() - } - })() - - return { - fullStream: eventStream, - messages: resultMessages.promise, - reasoningTextStream, - steps: resultSteps.promise, - textStream, - totalUsage: resultTotalUsage.promise, - usage: resultUsage.promise, + eventCtrl?.close() + textCtrl?.close() + reasoningTextCtrl?.close() } - }) + catch (err) { + eventCtrl?.error(err) + textCtrl?.error(err) + reasoningTextCtrl?.error(err) + + resultSteps.reject(err) + resultMessages.reject(err) + resultUsage.reject(err) + resultTotalUsage.reject(err) + } + finally { + resultSteps.resolve(steps) + resultMessages.resolve(messages) + resultUsage.resolve(usage) + resultTotalUsage.resolve(totalUsage) + + const finishStep = steps.at(-1) + + // if (finishStep) { + // rootSpan.setAttributes({ + // 'gen_ai.output.messages': JSON.stringify(messages), + // 'gen_ai.response.finish_reasons': [finishStep.finishReason], + // }) + // } + + // if (totalUsage) { + // rootSpan.setAttributes({ + // 'gen_ai.usage.input_tokens': totalUsage.prompt_tokens, + // 'gen_ai.usage.output_tokens': totalUsage.completion_tokens, + // }) + // } + + // eslint-disable-next-line sonarjs/void-use + void options.onFinish?.(finishStep) + + // rootSpan.end() + } + })() + + return { + fullStream: eventStream, + messages: resultMessages.promise, + reasoningTextStream, + steps: resultSteps.promise, + textStream, + totalUsage: resultTotalUsage.promise, + usage: resultUsage.promise, + } + // }) } diff --git a/packages-ext/telemetry/src/utils/stringify-tool.ts b/packages-ext/telemetry/src/utils/stringify-tool.ts deleted file mode 100644 index 519690e..0000000 --- a/packages-ext/telemetry/src/utils/stringify-tool.ts +++ /dev/null @@ -1,4 +0,0 @@ -import type { Tool } from 'xsai' - -export const stringifyTool = ({ function: func, type }: Tool) => - JSON.stringify({ description: func.description, inputSchema: func.parameters, name: func.name, type }) diff --git a/packages-ext/telemetry/src/utils/wrap-tool.ts b/packages-ext/telemetry/src/utils/wrap-tool.ts index c3a5c1c..a24612e 100644 --- a/packages-ext/telemetry/src/utils/wrap-tool.ts +++ b/packages-ext/telemetry/src/utils/wrap-tool.ts @@ -7,20 +7,26 @@ export const wrapTool = (tool: Tool, tracer: Tracer): Tool => ({ execute: async (input: unknown, options: ToolExecuteOptions) => recordSpan({ attributes: { - 'ai.operationId': 'ai.toolCall', - 'ai.toolCall.args': JSON.stringify(input), - 'ai.toolCall.id': options.toolCallId, - 'ai.toolCall.name': tool.function.name, - 'operation.name': 'ai.toolCall', + 'gen_ai.operation.name': 'execute_tool', + 'gen_ai.tool.call.arguments': JSON.stringify(input), + 'gen_ai.tool.call.description': tool.function.description, + 'gen_ai.tool.call.id': options.toolCallId, + 'gen_ai.tool.call.name': tool.function.name, }, - name: 'ai.toolCall', + name: `execute_tool ${tool.function.name}`, tracer, }, async (span) => { - const result = await tool.execute(input, options) + try { + const result = await tool.execute(input, options) + span.setAttribute('gen_ai.tool.call.result', JSON.stringify(result)) + return result + } + catch (err) { + if (err instanceof Error) + span.setAttribute('error.type', err.message) - span.setAttribute('ai.toolCall.result', JSON.stringify(result)) - - return result + throw err + } }), function: tool.function, type: tool.type, diff --git a/packages-ext/telemetry/test/__snapshots__/generate-text-tools.test.ts.snap b/packages-ext/telemetry/test/__snapshots__/generate-text-tools.test.ts.snap index b8558f4..0f37e74 100644 --- a/packages-ext/telemetry/test/__snapshots__/generate-text-tools.test.ts.snap +++ b/packages-ext/telemetry/test/__snapshots__/generate-text-tools.test.ts.snap @@ -1,90 +1,64 @@ // Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html exports[`generateText with tools > basic 1`] = ` -" -Okay, let me see. The user asked how many times 114514 plus 1919810 equals. Wait, that still sounds like they want the sum of those two numbers. But maybe there's confusion here. Let me check the original problem again. It says, "How many times does 114514 plus 1919810 equal?" The phrasing 'how many times does X equal Y' usually implies Y is X multiplied by something. Unless it's a code or mathematical problem, I don't think there's a multiplication involved here. But given the tools available, maybe the user expects a simple arithmetic result. Let me calculate the sum again to be sure. Adding 114514 and 1919810 gives 2034324, so the answer should be 2034324. Since the question is unclear, but the tools provided require using the add function, I'll return that result. - +"The sum of 114,514 and 1,919,810 is **2,034,324**. -The sum of 114514 and 1919810 is 2,034,324. Therefore, the answer is 2,034,324." +Therefore, the answer is $\\boxed{2034324}$." `; exports[`generateText with tools > basic 2`] = ` +[ + "execute_tool add", + "chat qwen3:0.6b", + "chat qwen3:0.6b", +] +`; + +exports[`generateText with tools > basic 3`] = ` [ { - "ai.model.id": "qwen3:0.6b", - "ai.model.provider": "xsai", - "ai.operationId": "ai.generateText.doGenerate", - "ai.prompt.messages": "[{"content":"How many times does 114514 plus 1919810 equal? Please try to call the \`add\` tool to solve the problem.","role":"user"}]", - "ai.prompt.toolChoice": "{"type":"auto"}", - "ai.prompt.tools": [ - "{"description":"Adds two numbers","inputSchema":{"$schema":"http://json-schema.org/draft-07/schema#","type":"object","properties":{"a":{"type":"string"},"b":{"type":"string"}},"required":["a","b"],"additionalProperties":false},"name":"add","type":"function"}", - ], - "ai.response.finishReason": "tool_calls", - "ai.response.id": "964e311b-3909-45f3-9f47-d661ad9cdd5f", - "ai.response.model": "qwen3:0.6b", - "ai.response.providerMetadata": "{}", - "ai.response.timestamp": "2025-10-19T09:33:20.761Z", - "ai.response.toolCalls": "[{"args":"{\\"a\\":\\"114514\\",\\"b\\":\\"1919810\\"}","toolCallId":"call_nf33i9f3","toolCallType":"function","toolName":"add"}]", - "ai.usage.completionTokens": 645, - "ai.usage.promptTokens": 180, + "gen_ai.operation.name": "execute_tool", + "gen_ai.tool.call.arguments": "{"a":"114514","b":"1919810"}", + "gen_ai.tool.call.description": "Adds two numbers", + "gen_ai.tool.call.id": "call_8xseeedt", + "gen_ai.tool.call.name": "add", + "gen_ai.tool.call.result": ""2034324"", + }, + { + "gen_ai.input.messages": "[{"content":"How many times does 114514 plus 1919810 equal? Please try to call the \`add\` tool to solve the problem.","role":"user"}]", + "gen_ai.operation.name": "chat", + "gen_ai.output.messages": "[{"role":"assistant","content":"","reasoning":"Okay, the user is asking how many times 114514 plus 1919810 equals. Hmm, wait. Let me parse this. The user wrote \\"How many times does 114514 plus 1919810 equal?\\" So they want the result of adding those two numbers and then seeing how many times that sum occurs? But the problem is phrased in a bit of confusion. Maybe they meant to ask for the sum itself or a multiple?\\n\\nBut the tools available only have the add function. So first, I need to add 114514 and 1919810 using the tools provided. Then, check if the result is a specific value or a product. Wait, but the user says \\"times\\" which probably refers to frequency. But if it's just adding them, there's no multiplication. However, maybe there's a misunderstanding. Let me think again.\\n\\nThe user might be mixing concepts. If they mean the sum of those numbers, then add them. But if they want how many times each factor is present? Wait, maybe there's a typo in the question. Let me check the numbers again. 114514 and 1919810. Let me compute their sum. Adding 114514 and 1919810: 114514 + 1919810. Let me calculate that. 114514 + 1919810 equals... well, 1000000 + 1919810 is 2919810, then add 14514: 2919810 + 14514 = 2934324. So the sum is 2,934,324. But the question says \\"how many times\\" which is unclear. If they are looking for the sum, then maybe that's the answer. However, if there's a different intent, like multiplication, but given the way it's phrased, it's likely a sum question. So the answer would be 2,934,324, which is the result of the add function call.\\n","tool_calls":[{"id":"call_8xseeedt","index":0,"type":"function","function":{"name":"add","arguments":"{\\"a\\":\\"114514\\",\\"b\\":\\"1919810\\"}"}}]}]", + "gen_ai.provider.name": "openai", + "gen_ai.request.choice.count": 1, "gen_ai.request.model": "qwen3:0.6b", + "gen_ai.request.seed": 114514, "gen_ai.response.finish_reasons": [ "tool_calls", ], - "gen_ai.response.id": "11da26ad-a287-4463-860f-ad35cd75f063", + "gen_ai.response.id": "3677c98e-4471-4f8e-ad6a-4418f82d088f", "gen_ai.response.model": "qwen3:0.6b", - "gen_ai.system": "xsai", - "gen_ai.usage.input_tokens": 180, - "gen_ai.usage.output_tokens": 645, - "operation.name": "ai.generateText.doGenerate", + "gen_ai.tool.definitions": "[{"function":{"description":"Adds two numbers","name":"add","parameters":{"$schema":"http://json-schema.org/draft-07/schema#","type":"object","properties":{"a":{"description":"First number","type":"string"},"b":{"description":"Second number","type":"string"}},"required":["a","b"],"additionalProperties":false},"strict":true},"type":"function"}]", + "gen_ai.usage.input_tokens": 186, + "gen_ai.usage.output_tokens": 533, + "server.address": "localhost:11434", }, { - "ai.operationId": "ai.toolCall", - "ai.toolCall.args": "{"a":"114514","b":"1919810"}", - "ai.toolCall.id": "call_nf33i9f3", - "ai.toolCall.name": "add", - "ai.toolCall.result": ""2034324"", - "operation.name": "ai.toolCall", - }, - { - "ai.model.id": "qwen3:0.6b", - "ai.model.provider": "xsai", - "ai.operationId": "ai.generateText.doGenerate", - "ai.prompt.messages": "[{"content":"How many times does 114514 plus 1919810 equal? Please try to call the \`add\` tool to solve the problem.","role":"user"},{"role":"assistant","content":"\\nOkay, so the user is asking how many times 114514 plus 1919810 equals. Wait, that seems like just adding those two numbers. Let me check the question again. It says \\"How many times does 114514 plus 1919810 equal?\\" So the question is phrased as a mathematical operation. But maybe the user is confused how to write it? Let me break it down.\\n\\nFirst, I need to compute 114514 + 1919810. Let's do that. Adding the two numbers: 114514 plus 1919810. Adding them gives... Let me calculate. 114514 + 1919810. Let's see, adding 114,514 to 1,919,810. Let me add the numbers step by step. 1,919,810 plus 100,000 is 2,019,810. Then plus 14,514, so total is 2,019,810 + 14,514 = 2,034,324. So the sum is 2,034,324. Then the question is how many times does that sum equal? Wait, maybe there's a typo? Because \\"how many times does something equal\\" usually implies multiplication. Unless it's a code or math problem, but the phrasing here is unclear. Or maybe it's a translation error? Let me check again.\\n\\nWait, maybe the user intended to ask for the product of 114514 and 1919810, which would be a very large number. But the original problem says \\"adds\\" and then asks \\"How many times does... equal?\\" So perhaps the user made a mistake in wording. But given that the tools provided are only the add function, maybe I need to use that. But regardless, the user is asking to call the add tool. Let me confirm: the question is to add 114514 and 1919810, which is a simple math problem. Since we can use the add function, I should call that function with a and b as the numbers given.\\n\\nSo the parameters are a=114514 and b=1919810. Then, the function will return the sum, which is 2,034,324. But the user's question is phrased as \\"how many times does... equal?\\" which might be a translation issue. But since the tools provided allow adding, I should proceed by calling the add function with the numbers provided.\\n\\n\\n","tool_calls":[{"id":"call_nf33i9f3","index":0,"type":"function","function":{"name":"add","arguments":"{\\"a\\":\\"114514\\",\\"b\\":\\"1919810\\"}"}}]},{"content":"2034324","role":"tool","tool_call_id":"call_nf33i9f3"}]", - "ai.prompt.toolChoice": "{"type":"auto"}", - "ai.prompt.tools": [ - "{"description":"Adds two numbers","inputSchema":{"$schema":"http://json-schema.org/draft-07/schema#","type":"object","properties":{"a":{"type":"string"},"b":{"type":"string"}},"required":["a","b"],"additionalProperties":false},"name":"add","type":"function"}", - ], - "ai.response.finishReason": "stop", - "ai.response.id": "d0398d64-93d6-434c-9f51-f6d02d1c59b8", - "ai.response.model": "qwen3:0.6b", - "ai.response.providerMetadata": "{}", - "ai.response.text": " Okay, let me see. The user asked how many ", - "ai.response.timestamp": "2025-10-19T09:33:56.277Z", - "ai.usage.completionTokens": 267, - "ai.usage.promptTokens": 810, + "gen_ai.input.messages": "[{"content":"How many times does 114514 plus 1919810 equal? Please try to call the \`add\` tool to solve the problem.","role":"user"},{"role":"assistant","content":"","reasoning":"Okay, the user is asking how many times 114514 plus 1919810 equals. Hmm, wait. Let me parse this. The user wrote \\"How many times does 114514 plus 1919810 equal?\\" So they want the result of adding those two numbers and then seeing how many times that sum occurs? But the problem is phrased in a bit of confusion. Maybe they meant to ask for the sum itself or a multiple?\\n\\nBut the tools available only have the add function. So first, I need to add 114514 and 1919810 using the tools provided. Then, check if the result is a specific value or a product. Wait, but the user says \\"times\\" which probably refers to frequency. But if it's just adding them, there's no multiplication. However, maybe there's a misunderstanding. Let me think again.\\n\\nThe user might be mixing concepts. If they mean the sum of those numbers, then add them. But if they want how many times each factor is present? Wait, maybe there's a typo in the question. Let me check the numbers again. 114514 and 1919810. Let me compute their sum. Adding 114514 and 1919810: 114514 + 1919810. Let me calculate that. 114514 + 1919810 equals... well, 1000000 + 1919810 is 2919810, then add 14514: 2919810 + 14514 = 2934324. So the sum is 2,934,324. But the question says \\"how many times\\" which is unclear. If they are looking for the sum, then maybe that's the answer. However, if there's a different intent, like multiplication, but given the way it's phrased, it's likely a sum question. So the answer would be 2,934,324, which is the result of the add function call.\\n","tool_calls":[{"id":"call_8xseeedt","index":0,"type":"function","function":{"name":"add","arguments":"{\\"a\\":\\"114514\\",\\"b\\":\\"1919810\\"}"}}]},{"content":"2034324","role":"tool","tool_call_id":"call_8xseeedt"}]", + "gen_ai.operation.name": "chat", + "gen_ai.output.messages": "[{"role":"assistant","content":"The sum of 114,514 and 1,919,810 is **2,034,324**. \\n\\nTherefore, the answer is $\\\\boxed{2034324}$.","reasoning":"Okay, let me figure out how the user's question is structured. They wrote, \\"How many times does 114514 plus 1919810 equal?\\" The assistant provided the sum using the add tool. But the user then gave a response of 2034324. Wait, the original problem is about counting how many times the sum occurs. But if the sum is a single value, then there's no multiplication here. Maybe the user made a mistake, or there's a different interpretation. Let me check the sum again. 114514 plus 1919810 equals 2034324? Let me verify that. Adding 114,514 and 1,919,810:\\n\\n1,919,810 + 114,514 = 2,034,324. Yes, that's correct. So the sum is 2,034,324. But the user is asking how many times this sum occurs. If the question was phrased as \\"How many times does 2,034,324 equal?\\" then the answer would just be 1, because it's a single number. But maybe the user intended to add the numbers and then the result is the answer. Wait, the original question was phrased as \\"how many times does 114514 plus 1919810 equal?\\" which might mean how many times is this expression equal (if it's a literal value). So unless there's more context, the answer is the sum value, which is 2,034,324. Therefore, the final answer should be that sum.\\n"}]", + "gen_ai.provider.name": "openai", + "gen_ai.request.choice.count": 1, "gen_ai.request.model": "qwen3:0.6b", + "gen_ai.request.seed": 114514, "gen_ai.response.finish_reasons": [ "stop", ], - "gen_ai.response.id": "9b2a0724-053a-40ab-96f7-5c832ecc4394", + "gen_ai.response.id": "24310c01-7aec-42a7-a350-af42c56e59bf", "gen_ai.response.model": "qwen3:0.6b", - "gen_ai.system": "xsai", - "gen_ai.usage.input_tokens": 810, - "gen_ai.usage.output_tokens": 267, - "operation.name": "ai.generateText.doGenerate", - }, - { - "ai.model.id": "qwen3:0.6b", - "ai.model.provider": "xsai", - "ai.operationId": "ai.generateText", - "ai.prompt": "{"messages":[{"content":"How many times does 114514 plus 1919810 equal? Please try to call the \`add\` tool to solve the problem.","role":"user"}]}", - "ai.response.finishReason": "stop", - "ai.response.providerMetadata": "{}", - "ai.response.text": " Okay, let me see. The user asked how many ", - "ai.usage.completionTokens": 267, - "ai.usage.promptTokens": 810, - "operation.name": "ai.generateText", + "gen_ai.tool.definitions": "[{"function":{"description":"Adds two numbers","name":"add","parameters":{"$schema":"http://json-schema.org/draft-07/schema#","type":"object","properties":{"a":{"description":"First number","type":"string"},"b":{"description":"Second number","type":"string"}},"required":["a","b"],"additionalProperties":false},"strict":true},"type":"function"}]", + "gen_ai.usage.input_tokens": 735, + "gen_ai.usage.output_tokens": 438, + "server.address": "localhost:11434", }, ] `; diff --git a/packages-ext/telemetry/test/__snapshots__/generate-text.test.ts.snap b/packages-ext/telemetry/test/__snapshots__/generate-text.test.ts.snap index f40d7f0..0e2eb1a 100644 --- a/packages-ext/telemetry/test/__snapshots__/generate-text.test.ts.snap +++ b/packages-ext/telemetry/test/__snapshots__/generate-text.test.ts.snap @@ -1,70 +1,31 @@ // Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html -exports[`generateText > basic 1`] = ` -" -Okay, so the user is asking why the sky is blue. Hmm, I need to explain that first. Let me start by breaking down the question. They might just be curious, but they could be interested in how the sky appears. I remember that the sky is blue because of the atmosphere. So, what causes the color to be blue? - -The primary factor is the atmosphere. When sunlight interacts with water vapor and other gases in the air to create the blue light. But maybe I should mention other factors too. Like, the refraction of light. When sunlight enters the air, it bends because of the medium it's moving through. This bending makes the blue light appear to be blue. - -Also, there's the fact that the Earth's surface is covered in water, and water vapor contributes to the blue color. But wait, in the middle of the atmosphere, the presence of greenhouse gases like CO2 could play a role, but that's a little advanced. I think the main point is the refraction and the atmosphere's composition. - -Wait, I should check if there's any other part I need to mention. Some sources say the sky is blue because of the absorption of other colors. Like, when light passes through the atmosphere and goes through the Earth's upper layers, it refracts. Also, the color we perceive as blue is a result of the way light interacts with our eyes. - -Let me structure this step by step. First, explain the refraction, then the atmosphere, and maybe add that the atmosphere is a mix of gases. Make sure to keep the explanation clear and conversational. The user might not be familiar with terms, so simplifying is better. Also, maybe compare it to other colors, but that might complicate things. Keep it straightforward. - - -The sky appears blue because of the way light interacts with the Earth's atmosphere. Here's a simple explanation: - -1. **Light Refraction**: When sunlight passes through the atmosphere, it bends due to the medium (gases like nitrogen and oxygen in the air). This bending makes the blue light appear to be blue. - -2. **Atmospheric Composition**: The mixture of gases in the air, along with water vapor, contributes to the blue color. The more light that refracts, the bluer the sky becomes. - -3. **Color Perception**: Our eyes absorb other wavelengths and reflect blue light, which we perceive as the sky's color. - -In short, the blue sky is a result of the atmosphere trapping blue light, making the color appear blue when viewed from space. 🌊" -`; +exports[`generateText > basic 1`] = `"The sky appears blue because the sunlight is scattered by the atmospheric molecules. When sunlight interacts with molecules in the air, the waves of light are dispersed - the blue light bends more than the red, creating a visible color. This scattering of light, called **Rayleigh scattering**, allows the blue light to reach our eyes, making the sky appear blue. The Sun is the main source of this light, and it’s the reason we’ve seen the color since ancient times."`; exports[`generateText > basic 2`] = ` +[ + "chat qwen3:0.6b", +] +`; + +exports[`generateText > basic 3`] = ` [ { - "ai.model.id": "qwen3:0.6b", - "ai.model.provider": "xsai", - "ai.operationId": "ai.generateText.doGenerate", - "ai.prompt.messages": "[{"content":"Why is the sky blue?","role":"user"}]", - "ai.response.finishReason": "stop", - "ai.response.id": "f7fd09d0-eae8-4d0f-9d03-41b0835e5853", - "ai.response.model": "qwen3:0.6b", - "ai.response.providerMetadata": "{}", - "ai.response.text": " Okay, so the user is asking why the sky is", - "ai.response.timestamp": "2025-10-19T09:33:20.226Z", - "ai.telemetry.metadata.agentId": "weather-assistant", - "ai.telemetry.metadata.instructions": "You are a helpful weather assistant", - "ai.usage.completionTokens": 523, - "ai.usage.promptTokens": 14, + "gen_ai.input.messages": "[{"content":"Why is the sky blue?","role":"user"}]", + "gen_ai.operation.name": "chat", + "gen_ai.output.messages": "[{"role":"assistant","content":"The sky appears blue because the sunlight is scattered by the atmospheric molecules. When sunlight interacts with molecules in the air, the waves of light are dispersed - the blue light bends more than the red, creating a visible color. This scattering of light, called **Rayleigh scattering**, allows the blue light to reach our eyes, making the sky appear blue. The Sun is the main source of this light, and it’s the reason we’ve seen the color since ancient times.","reasoning":"Okay, the user is asking why the sky is blue. I know that this is a common question with many different answers over time, mostly the atmosphere as a source of blue color. I should explain that the sky's blue color is due to sunlight, but I need to be careful not to mix different explanations. The user probably wants a clear and concise answer that covers atmosphere, maybe even a bit about scattering of light.\\n\\nFirst, I should state that the sky's color is due to the way light interacts with the atmosphere. Then, mention that the molecules in the air scatter these light waves, creating a blue hue. It's good to explain that this process is called Rayleigh scattering, and mention that the Sun is the main source of this light. Also, keep the explanation simple, avoid technical jargon, and conclude that it's a clear and concise response.\\n"}]", + "gen_ai.provider.name": "openai", + "gen_ai.request.choice.count": 1, "gen_ai.request.model": "qwen3:0.6b", + "gen_ai.request.seed": 114514, "gen_ai.response.finish_reasons": [ "stop", ], - "gen_ai.response.id": "d34cacaf-b8e0-49eb-a320-0b46e9b902a7", + "gen_ai.response.id": "11016189-fd86-497c-8ccf-a9f116c099ab", "gen_ai.response.model": "qwen3:0.6b", - "gen_ai.system": "xsai", - "gen_ai.usage.input_tokens": 14, - "gen_ai.usage.output_tokens": 523, - "operation.name": "ai.generateText.doGenerate", - }, - { - "ai.model.id": "qwen3:0.6b", - "ai.model.provider": "xsai", - "ai.operationId": "ai.generateText", - "ai.prompt": "{"messages":[{"content":"Why is the sky blue?","role":"user"}]}", - "ai.response.finishReason": "stop", - "ai.response.providerMetadata": "{}", - "ai.response.text": " Okay, so the user is asking why the sky is", - "ai.telemetry.metadata.agentId": "weather-assistant", - "ai.telemetry.metadata.instructions": "You are a helpful weather assistant", - "ai.usage.completionTokens": 523, - "ai.usage.promptTokens": 14, - "operation.name": "ai.generateText", + "gen_ai.usage.input_tokens": 16, + "gen_ai.usage.output_tokens": 274, + "server.address": "localhost:11434", }, ] `; diff --git a/packages-ext/telemetry/test/__snapshots__/stream-text-tools.test.ts.snap b/packages-ext/telemetry/test/__snapshots__/stream-text-tools.test.ts.snap index 5da5d63..e2024c7 100644 --- a/packages-ext/telemetry/test/__snapshots__/stream-text-tools.test.ts.snap +++ b/packages-ext/telemetry/test/__snapshots__/stream-text-tools.test.ts.snap @@ -1,99 +1,60 @@ // Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html -exports[`streamText with tools > basic 1`] = ` -" -Okay, so the user is asking how many times 114514 plus 1919810 equals. Let me try to parse that. The question seems to be asking for the product of those two numbers. Wait, but the way it's phrased is "How many times does X equal Y?" So it's Y divided by X? Or is it X plus Y? Let me check the exact sentence again: "How many times does 114514 plus 1919810 equal?" So, if I take the sum of 114514 and 1919810, is that the quantity we're looking for? The word "how many times" implies multiplication, but here it's a sum. Unless there's a misinterpretation. - -Wait, maybe there's a typo. Let me re-express it. If the user wants to know the product, it's 114514 multiplied by 1919810. But the question is a bit unclear. Is it possible they meant to write "how many times does X plus Y equal Z" or something else? But since the user wrote "How many times does 114514 plus 1919810 equal?" perhaps they meant the product. Alternatively, maybe it's a translation issue. Let me think. If I convert that into math, adding those two numbers is straightforward. But if the user is asking for the sum divided by something else, but no, the way it's written, it's a direct sum. - -Wait, but the provided tools only include the add function. The user says to use the add tool. So regardless of the math, I should use the add function with a and b. Let me try that. The parameters are a and b. So a is 114514, b is 1919810. But wait, the question is phrased as "How many times does ... equal?" So perhaps they want the value of (114514 + 1919810), which would be 2034324. But the user is asking how many times, which typically means division by something. Unless there's a misunderstanding here. - -Alternatively, maybe there's a specific context I'm missing. But given the tools, I can only use add. So I'll compute 114514 + 1919810 using the add function, and that's the answer. - - - -Okay, let me check the user's question again. They asked "How many times does 114514 plus 1919810 equal?" So it's a bit confusing. Adding those two numbers gives 2034324, but if they meant multiplying 114514 by 1919810, that's a huge number. But since the tools here only allow the add function, and the user's original question is phrased as asking for the result of the addition, I have to use the add tool. So, the answer is 2034324. - - -The result of adding 114514 and 1919810 is **2034324**. - -Let me know if you need further clarification!" -`; +exports[`streamText with tools > basic 1`] = `"The sum of 114514 plus 1919810 is 2034324. Therefore, the answer is 2,034,324."`; exports[`streamText with tools > basic 2`] = ` +[ + "execute_tool add", + "chat qwen3:0.6b", + "chat qwen3:0.6b", +] +`; + +exports[`streamText with tools > basic 3`] = ` [ { - "ai.operationId": "ai.toolCall", - "ai.toolCall.args": "{"a":"114514","b":"1919810"}", - "ai.toolCall.id": "call_hxg74h3m", - "ai.toolCall.name": "add", - "ai.toolCall.result": ""2034324"", - "operation.name": "ai.toolCall", + "gen_ai.operation.name": "execute_tool", + "gen_ai.tool.call.arguments": "{"a":"114514","b":"1919810"}", + "gen_ai.tool.call.description": "Adds two numbers", + "gen_ai.tool.call.id": "call_ew1w2m1s", + "gen_ai.tool.call.name": "add", + "gen_ai.tool.call.result": ""2034324"", }, { - "ai.model.id": "qwen3:0.6b", - "ai.model.provider": "xsai", - "ai.operationId": "ai.streamText.doStream", - "ai.prompt.messages": "[{"content":"How many times does 114514 plus 1919810 equal? Please try to call the \`add\` tool to solve the problem.","role":"user"}]", - "ai.prompt.toolChoice": "{"type":"auto"}", - "ai.prompt.tools": [ - "{"description":"Adds two numbers","inputSchema":{"$schema":"http://json-schema.org/draft-07/schema#","type":"object","properties":{"a":{"type":"string"},"b":{"type":"string"}},"required":["a","b"],"additionalProperties":false},"name":"add","type":"function"}", - ], - "ai.response.avgOutputTokensPerSecond": 22.489393204584776, - "ai.response.finishReason": "tool_calls", - "ai.response.id": "b1b52cfb-7075-4030-be38-f83384cb3c5f", - "ai.response.model": "qwen3:0.6b", - "ai.response.msToFinish": 25834.400898, - "ai.response.msToFirstChunk": 17141.756877, - "ai.response.providerMetadata": "{}", - "ai.response.text": " Okay, so the user is asking how many times", - "ai.response.timestamp": "2025-10-19T09:33:20.757Z", - "ai.response.toolCalls": "[{"args":"{\\"a\\":\\"114514\\",\\"b\\":\\"1919810\\"}","toolCallId":"call_hxg74h3m","toolCallType":"function","toolName":"add"}]", - "ai.usage.inputTokens": 180, - "ai.usage.outputTokens": 581, - "ai.usage.totalTokens": 761, + "gen_ai.input.messages": "[{"content":"How many times does 114514 plus 1919810 equal? Please try to call the \`add\` tool to solve the problem.","role":"user"}]", + "gen_ai.operation.name": "chat", + "gen_ai.output.messages": "[{"content":"","role":"assistant","tool_calls":[{"id":"call_ew1w2m1s","index":0,"type":"function","function":{"name":"add","arguments":"{\\"a\\":\\"114514\\",\\"b\\":\\"1919810\\"}"}}]}]", + "gen_ai.provider.name": "openai", + "gen_ai.request.choice.count": 1, "gen_ai.request.model": "qwen3:0.6b", + "gen_ai.request.seed": 114514, "gen_ai.response.finish_reasons": [ "tool_calls", ], - "gen_ai.response.id": "2d914ee0-d7b2-4c86-94be-8799e575088a", + "gen_ai.response.id": "23291262-63dc-4f00-b7dc-fb928cd562f5", "gen_ai.response.model": "qwen3:0.6b", - "gen_ai.system": "xsai", - "gen_ai.usage.input_tokens": 180, - "gen_ai.usage.output_tokens": 581, - "operation.name": "ai.streamText.doStream", + "gen_ai.tool.definitions": "[{"function":{"description":"Adds two numbers","name":"add","parameters":{"$schema":"http://json-schema.org/draft-07/schema#","type":"object","properties":{"a":{"description":"First number","type":"string"},"b":{"description":"Second number","type":"string"}},"required":["a","b"],"additionalProperties":false},"strict":true},"type":"function"}]", + "gen_ai.usage.input_tokens": 186, + "gen_ai.usage.output_tokens": 639, + "server.address": "localhost:11434", }, { - "ai.model.id": "qwen3:0.6b", - "ai.model.provider": "xsai", - "ai.operationId": "ai.streamText.doStream", - "ai.prompt.messages": "[{"content":"How many times does 114514 plus 1919810 equal? Please try to call the \`add\` tool to solve the problem.","role":"user"}]", - "ai.prompt.toolChoice": "{"type":"auto"}", - "ai.prompt.tools": [ - "{"description":"Adds two numbers","inputSchema":{"$schema":"http://json-schema.org/draft-07/schema#","type":"object","properties":{"a":{"type":"string"},"b":{"type":"string"}},"required":["a","b"],"additionalProperties":false},"name":"add","type":"function"}", - ], - "ai.response.avgOutputTokensPerSecond": 12.204896245549765, - "ai.response.finishReason": "stop", - "ai.response.id": "dd756e5d-9163-4d07-b248-f18348545c6e", - "ai.response.model": "qwen3:0.6b", - "ai.response.msToFinish": 14993.982441, - "ai.response.msToFirstChunk": 11978.728365000003, - "ai.response.providerMetadata": "{}", - "ai.response.text": " Okay, let me check the user's question aga", - "ai.response.timestamp": "2025-10-19T09:33:46.594Z", - "ai.usage.inputTokens": 746, - "ai.usage.outputTokens": 183, - "ai.usage.totalTokens": 929, + "gen_ai.input.messages": "[{"content":"How many times does 114514 plus 1919810 equal? Please try to call the \`add\` tool to solve the problem.","role":"user"},{"content":"","role":"assistant","tool_calls":[{"id":"call_ew1w2m1s","index":0,"type":"function","function":{"name":"add","arguments":"{\\"a\\":\\"114514\\",\\"b\\":\\"1919810\\"}"}}]},{"content":"2034324","role":"tool","tool_call_id":"call_ew1w2m1s"}]", + "gen_ai.operation.name": "chat", + "gen_ai.output.messages": "[{"content":"The sum of 114514 plus 1919810 is 2034324. Therefore, the answer is 2,034,324.","role":"assistant"}]", + "gen_ai.provider.name": "openai", + "gen_ai.request.choice.count": 1, "gen_ai.request.model": "qwen3:0.6b", + "gen_ai.request.seed": 114514, "gen_ai.response.finish_reasons": [ "stop", ], - "gen_ai.response.id": "647c7dc4-612e-4bcf-b97c-d8f0dd0d55e9", + "gen_ai.response.id": "22204df2-35b0-4e93-b130-bb123862dc58", "gen_ai.response.model": "qwen3:0.6b", - "gen_ai.system": "xsai", - "gen_ai.usage.input_tokens": 746, - "gen_ai.usage.output_tokens": 183, - "operation.name": "ai.streamText.doStream", + "gen_ai.tool.definitions": "[{"function":{"description":"Adds two numbers","name":"add","parameters":{"$schema":"http://json-schema.org/draft-07/schema#","type":"object","properties":{"a":{"description":"First number","type":"string"},"b":{"description":"Second number","type":"string"}},"required":["a","b"],"additionalProperties":false},"strict":true},"type":"function"}]", + "gen_ai.usage.input_tokens": 239, + "gen_ai.usage.output_tokens": 304, + "server.address": "localhost:11434", }, ] `; diff --git a/packages-ext/telemetry/test/__snapshots__/stream-text.test.ts.snap b/packages-ext/telemetry/test/__snapshots__/stream-text.test.ts.snap index 6225f39..c10438f 100644 --- a/packages-ext/telemetry/test/__snapshots__/stream-text.test.ts.snap +++ b/packages-ext/telemetry/test/__snapshots__/stream-text.test.ts.snap @@ -1,54 +1,36 @@ // Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html exports[`streamText > basic 1`] = ` -" -Okay, so the user is asking why the sky is blue. Hmm, I know that the sky's color is due to air particles scattering sunlight. But wait, where exactly does that come from? Let me think. +"The sky appears blue due to **Rayleigh scattering** of light. When sunlight interacts with the Earth's atmosphere, water vapor and other molecules scatter shorter wavelengths of light (like blue) more than longer ones (like red). As a result, the blue color of the sun shines on the earth’s surface, and since these blue wavelengths are scattered in the atmosphere, they are reflected back into our eyes, making the sky appear blue. -First, I remember that when sunlight hits the Earth's atmosphere, it gets scattered. This phenomenon is called Rayleigh scattering. Right? But I need to explain that in simpler terms. Also, maybe mention that it's part of the atmosphere's structure. Oh, right! The ozone layer in the atmosphere also plays a role. But wait, are there other reasons besides Rayleigh scattering? - -Wait, sometimes people think the sky is blue because of the way light interacts with the ground or the way light is reflected. But I think that's actually because of the atmosphere's properties. But maybe I should clarify that Rayleigh scattering is the primary reason. Also, some people find that the color is influenced by temperature or humidity, but I don't think that's the main factor here. - -I should mention that the blue light is scattered more efficiently by the air particles compared to other colors, which contributes to the overall sky color. Maybe add that it's a result of the atmosphere's natural structure, not just a physical property. Yeah, that makes sense. But wait, is there any other source of blue in the sky? I think the main one is Rayleigh scattering. But just to make sure, I should confirm that and provide a clear explanation. - - -The sky appears blue primarily due to **Rayleigh scattering** of sunlight by the Earth's atmosphere. When sunlight interacts with gas molecules (ozone, nitrogen, etc.) in the upper atmosphere, it bounces in all directions with a predominantly blue component. Since blue light is scattered more than other wavelengths by air molecules, the scattered blue light forms the color we see in the sky. - -This scattering occurs with a wavelength-dependent efficiency: blue light is scattered more than red or orange light. Additionally, the atmosphere's structure contributes to the mixing of colors, and the sunlight's path (from the sun to Earth) further shapes the blue coloration. - -So, in short: The sky is blue because the Earth's atmosphere scatters sunlight in such a way that the blue light ends up visible to us. 🌇" +A simple explanation is: +**The blue light from the sun is scattered more by the atmosphere than the red light, which is why we see the sky in hues of blue.**" `; exports[`streamText > basic 2`] = ` +[ + "chat qwen3:0.6b", +] +`; + +exports[`streamText > basic 3`] = ` [ { - "ai.model.id": "qwen3:0.6b", - "ai.model.provider": "xsai", - "ai.operationId": "ai.streamText.doStream", - "ai.prompt.messages": "[{"content":"Why is the sky blue?","role":"user"}]", - "ai.response.avgOutputTokensPerSecond": 48.36097189033391, - "ai.response.finishReason": "stop", - "ai.response.id": "2f730f80-3cdd-43fd-9c5e-010b9d44e54a", - "ai.response.model": "qwen3:0.6b", - "ai.response.msToFinish": 9615.191379000002, - "ai.response.msToFirstChunk": 2944.94823, - "ai.response.providerMetadata": "{}", - "ai.response.text": " Okay, so the user is asking why the sky is", - "ai.response.timestamp": "2025-10-19T09:33:20.226Z", - "ai.telemetry.metadata.agentId": "weather-assistant", - "ai.telemetry.metadata.instructions": "You are a helpful weather assistant", - "ai.usage.inputTokens": 14, - "ai.usage.outputTokens": 465, - "ai.usage.totalTokens": 479, + "gen_ai.input.messages": "[{"content":"Why is the sky blue?","role":"user"}]", + "gen_ai.operation.name": "chat", + "gen_ai.output.messages": "[{"content":"The sky appears blue due to **Rayleigh scattering** of light. When sunlight interacts with the Earth's atmosphere, water vapor and other molecules scatter shorter wavelengths of light (like blue) more than longer ones (like red). As a result, the blue color of the sun shines on the earth’s surface, and since these blue wavelengths are scattered in the atmosphere, they are reflected back into our eyes, making the sky appear blue. \\n\\nA simple explanation is: \\n**The blue light from the sun is scattered more by the atmosphere than the red light, which is why we see the sky in hues of blue.**","role":"assistant"}]", + "gen_ai.provider.name": "openai", + "gen_ai.request.choice.count": 1, "gen_ai.request.model": "qwen3:0.6b", + "gen_ai.request.seed": 114514, "gen_ai.response.finish_reasons": [ "stop", ], - "gen_ai.response.id": "f5c22175-23b5-4cba-80d6-af13f9180df7", + "gen_ai.response.id": "c1927e4d-9b07-4fe3-be76-08f9b4662c76", "gen_ai.response.model": "qwen3:0.6b", - "gen_ai.system": "xsai", - "gen_ai.usage.input_tokens": 14, - "gen_ai.usage.output_tokens": 465, - "operation.name": "ai.streamText.doStream", + "gen_ai.usage.input_tokens": 16, + "gen_ai.usage.output_tokens": 405, + "server.address": "localhost:11434", }, ] `; diff --git a/packages-ext/telemetry/test/generate-text-tools.test.ts b/packages-ext/telemetry/test/generate-text-tools.test.ts index af5584d..ee0e42f 100644 --- a/packages-ext/telemetry/test/generate-text-tools.test.ts +++ b/packages-ext/telemetry/test/generate-text-tools.test.ts @@ -1,9 +1,6 @@ -import type { ReadableSpan } from '@opentelemetry/sdk-trace-base' - +import { LangfuseSpanProcessor } from '@langfuse/otel' import { InMemorySpanExporter, SimpleSpanProcessor } from '@opentelemetry/sdk-trace-base' import { NodeTracerProvider } from '@opentelemetry/sdk-trace-node' -// import { generateText as aiGenerateText, tool as aiTool, stepCountIs } from 'ai' -// import { ollama } from 'ollama-ai-provider-v2' import { describe, expect, it } from 'vitest' import { tool } from 'xsai' import { z } from 'zod/v4' @@ -13,18 +10,13 @@ import { generateText } from '../src' describe.sequential('generateText with tools', () => { const memoryExporter = new InMemorySpanExporter() const tracerProvider = new NodeTracerProvider({ - spanProcessors: [new SimpleSpanProcessor(memoryExporter)], + spanProcessors: [ + new SimpleSpanProcessor(memoryExporter), + new LangfuseSpanProcessor(), + ], }) tracerProvider.register() - const getAttributes = (span: ReadableSpan) => Object.fromEntries( - Object.entries(span.attributes) - .map(([k, v]) => k === 'ai.response.text' - ? [k, (v as string).replaceAll('\n', ' ').slice(0, 50)] - : [k, v], - ), - ) - it('basic', async () => { const add = await tool({ description: 'Adds two numbers', @@ -40,6 +32,7 @@ describe.sequential('generateText with tools', () => { const { text } = await generateText({ baseURL: 'http://localhost:11434/v1', + maxSteps: 5, messages: [{ content: 'How many times does 114514 plus 1919810 equal? Please try to call the `add` tool to solve the problem.', role: 'user', @@ -49,62 +42,12 @@ describe.sequential('generateText with tools', () => { tools: [add], }) - const spans = memoryExporter.getFinishedSpans().map(getAttributes) + const spans = memoryExporter.getFinishedSpans() + const names = spans.map(s => s.name) + const attributes = spans.map(s => s.attributes) expect(text).toMatchSnapshot() - expect(spans).toMatchSnapshot() + expect(names).toMatchSnapshot() + expect(attributes).toMatchSnapshot() }, 120_000) - - // it('basic/ai', async () => { - // const add = aiTool({ - // description: 'Adds two numbers', - // // @ts-expect-error wrong types - // execute: ({ a, b }) => (Number.parseInt(a) + Number.parseInt(b)).toString(), - // // @ts-expect-error wrong types - // inputSchema: z.object({ - // a: z.string() - // .describe('First number'), - // b: z.string() - // .describe('Second number'), - // }), - // name: 'add', - // }) - - // const { text } = await aiGenerateText({ - // experimental_telemetry: { isEnabled: true }, - // model: ollama('qwen3:0.6b'), - // prompt: 'How many times does 114514 plus 1919810 equal? Please try to call the `add` tool to solve the problem.', - // seed: 114514, - // stopWhen: stepCountIs(5), - // tools: { add }, - // }) - - // const spans = memoryExporter.getFinishedSpans().slice(4).map(getAttributes) - - // expect(text).toMatchSnapshot() - // expect(spans).toMatchSnapshot() - // }, 120_000) - - // it('basic/compare', () => { - // const extractAttributeKeys = (span: ReadableSpan) => Object.keys(span.attributes) - // // eslint-disable-next-line sonarjs/no-nested-functions - // .filter(key => ['ai.settings'].every(prefix => !key.startsWith(prefix))) - // .sort((a, b) => a.localeCompare(b)) - - // const spans = memoryExporter.getFinishedSpans() - // const xsaiSpans = spans.slice(0, 4) - // const aiSpans = spans.slice(4) - - // const xsaiIds = xsaiSpans.map(span => span.attributes['ai.operationId']) - // const aiIds = aiSpans.map(span => span.attributes['ai.operationId']) - - // expect(xsaiIds).toStrictEqual(aiIds) - - // const xsaiKeys = xsaiSpans.map(extractAttributeKeys) - // const aiKeys = aiSpans.map(extractAttributeKeys) - - // xsaiKeys.forEach((attributes, i) => { - // expect(attributes).toStrictEqual(aiKeys[i]) - // }) - // }) }) diff --git a/packages-ext/telemetry/test/generate-text.test.ts b/packages-ext/telemetry/test/generate-text.test.ts index 45536fc..0645242 100644 --- a/packages-ext/telemetry/test/generate-text.test.ts +++ b/packages-ext/telemetry/test/generate-text.test.ts @@ -1,9 +1,6 @@ -import type { ReadableSpan } from '@opentelemetry/sdk-trace-base' - +import { LangfuseSpanProcessor } from '@langfuse/otel' import { InMemorySpanExporter, SimpleSpanProcessor } from '@opentelemetry/sdk-trace-base' import { NodeTracerProvider } from '@opentelemetry/sdk-trace-node' -// import { generateText as aiGenerateText } from 'ai' -// import { ollama } from 'ollama-ai-provider-v2' import { describe, expect, it } from 'vitest' import { generateText } from '../src' @@ -11,18 +8,13 @@ import { generateText } from '../src' describe.sequential('generateText', () => { const memoryExporter = new InMemorySpanExporter() const tracerProvider = new NodeTracerProvider({ - spanProcessors: [new SimpleSpanProcessor(memoryExporter)], + spanProcessors: [ + new SimpleSpanProcessor(memoryExporter), + new LangfuseSpanProcessor(), + ], }) tracerProvider.register() - const getAttributes = (span: ReadableSpan) => Object.fromEntries( - Object.entries(span.attributes) - .map(([k, v]) => k === 'ai.response.text' - ? [k, (v as string).replaceAll('\n', ' ').slice(0, 50)] - : [k, v], - ), - ) - it('basic', async () => { const { text } = await generateText({ baseURL: 'http://localhost:11434/v1', @@ -32,52 +24,20 @@ describe.sequential('generateText', () => { }], model: 'qwen3:0.6b', seed: 114514, - telemetry: { - metadata: { - agentId: 'weather-assistant', - instructions: 'You are a helpful weather assistant', - }, - }, + // telemetry: { + // metadata: { + // agentId: 'weather-assistant', + // instructions: 'You are a helpful weather assistant', + // }, + // }, }) - const spans = memoryExporter.getFinishedSpans().map(getAttributes) + const spans = memoryExporter.getFinishedSpans() + const names = spans.map(s => s.name) + const attributes = spans.map(s => s.attributes) expect(text).toMatchSnapshot() - expect(spans).toMatchSnapshot() + expect(names).toMatchSnapshot() + expect(attributes).toMatchSnapshot() }, 120_000) - - // it('basic/ai', async () => { - // const { text } = await aiGenerateText({ - // experimental_telemetry: { - // isEnabled: true, - // metadata: { - // agentId: 'weather-assistant', - // instructions: 'You are a helpful weather assistant', - // }, - // }, - // model: ollama('qwen3:0.6b'), - // prompt: 'Why is the sky blue?', - // seed: 114514, - // }) - - // const spans = memoryExporter.getFinishedSpans().slice(2).map(getAttributes) - - // expect(text).toMatchSnapshot() - // expect(spans).toMatchSnapshot() - // }, 120_000) - - // it('basic/compare', () => { - // const extractAttributeKeys = (span: ReadableSpan) => Object.keys(span.attributes) - // // eslint-disable-next-line sonarjs/no-nested-functions - // .filter(key => ['ai.settings'].every(prefix => !key.startsWith(prefix))) - // .sort((a, b) => a.localeCompare(b)) - - // const spans = memoryExporter.getFinishedSpans() - // const xsai = spans.slice(0, 2).map(extractAttributeKeys) - // const ai = spans.slice(2).map(extractAttributeKeys) - - // xsai.forEach((attributes, i) => { - // expect(attributes).toStrictEqual(ai[i]) - // }) - // }) }) diff --git a/packages-ext/telemetry/test/stream-text-tools.test.ts b/packages-ext/telemetry/test/stream-text-tools.test.ts index e23e95d..0abb636 100644 --- a/packages-ext/telemetry/test/stream-text-tools.test.ts +++ b/packages-ext/telemetry/test/stream-text-tools.test.ts @@ -1,10 +1,6 @@ -import type { AttributeValue } from '@opentelemetry/api' -import type { ReadableSpan } from '@opentelemetry/sdk-trace-base' - +import { LangfuseSpanProcessor } from '@langfuse/otel' import { InMemorySpanExporter, SimpleSpanProcessor } from '@opentelemetry/sdk-trace-base' import { NodeTracerProvider } from '@opentelemetry/sdk-trace-node' -// import { streamText as aiStreamText, tool as aiTool, stepCountIs } from 'ai' -// import { ollama } from 'ollama-ai-provider-v2' import { describe, expect, it } from 'vitest' import { tool } from 'xsai' import { z } from 'zod/v4' @@ -14,35 +10,13 @@ import { streamText } from '../src' describe.sequential('streamText with tools', () => { const memoryExporter = new InMemorySpanExporter() const tracerProvider = new NodeTracerProvider({ - spanProcessors: [new SimpleSpanProcessor(memoryExporter)], + spanProcessors: [ + new SimpleSpanProcessor(memoryExporter), + new LangfuseSpanProcessor(), + ], }) tracerProvider.register() - const trimmedAttributes = new Set([ - // Not using the following because we are testing with --update - // 'ai.response.avgOutputTokensPerSecond', // reason: time - // 'ai.response.id', // reason: id - // 'ai.response.msToFinish', // reason: time - // 'ai.response.msToFirstChunk', // reason: time - // 'ai.response.timestamp', // reason: time - // 'ai.response.toolCalls', // reason: contains id - // 'ai.toolCall.id', // reason: id - // 'gen_ai.response.id', // reason: id - ]) - - const getAttributes = (span: ReadableSpan) => Object.fromEntries( - Object.entries(span.attributes) - .reduce<[string, AttributeValue | undefined][]>((entries, [k, v]) => { - if (k === 'ai.response.text') { - entries.push([k, (v as string).replaceAll('\n', ' ').slice(0, 50)]) - } - else if (!trimmedAttributes.has(k)) { - entries.push([k, v]) - } - return entries - }, []), - ) - it('basic', async () => { const add = await tool({ description: 'Adds two numbers', @@ -66,10 +40,13 @@ describe.sequential('streamText with tools', () => { }], model: 'qwen3:0.6b', onFinish: async () => { - const spans = memoryExporter.getFinishedSpans().map(getAttributes) + const spans = memoryExporter.getFinishedSpans() + const names = spans.map(s => s.name) + const attributes = spans.map(s => s.attributes) expect(text).toMatchSnapshot() - expect(spans).toMatchSnapshot() + expect(names).toMatchSnapshot() + expect(attributes).toMatchSnapshot() }, seed: 114514, streamOptions: { @@ -82,56 +59,4 @@ describe.sequential('streamText with tools', () => { text += textDelta } }, 120_000) - - // it('basic/ai', async () => { - // const add = aiTool({ - // description: 'Adds two numbers', - // execute: ({ a, b }) => (Number.parseInt(a) + Number.parseInt(b)).toString(), - // inputSchema: z.object({ - // a: z.string() - // .describe('First number'), - // b: z.string() - // .describe('Second number'), - // }), - // name: 'add', - // }) - - // const { text } = aiStreamText({ - // experimental_telemetry: { isEnabled: true }, - // model: ollama('qwen3:0.6b'), - // prompt: 'How many times does 114514 plus 1919810 equal? Please try to call the `add` tool to solve the problem.', - // seed: 114514, - // stopWhen: stepCountIs(5), - // tools: { add }, - // }) - - // const fullText = await text - // const spans = memoryExporter.getFinishedSpans().slice(4).map(getAttributes) - - // expect(fullText).toMatchSnapshot() - // expect(spans).toMatchSnapshot() - // }, 120_000) - - // it('basic/compare', () => { - // const extractAttributeKeys = (span: ReadableSpan) => Object.keys(span.attributes) - // // eslint-disable-next-line sonarjs/no-nested-functions - // .filter(key => ['ai.settings'].every(prefix => !key.startsWith(prefix))) - // .sort((a, b) => a.localeCompare(b)) - - // const spans = memoryExporter.getFinishedSpans() - // const xsaiSpans = spans.slice(0, 4) - // const aiSpans = spans.slice(4) - - // const xsaiIds = xsaiSpans.map(span => span.attributes['ai.operationId']) - // const aiIds = aiSpans.map(span => span.attributes['ai.operationId']) - - // expect(xsaiIds).toStrictEqual(aiIds) - - // const xsaiKeys = xsaiSpans.map(extractAttributeKeys) - // const aiKeys = aiSpans.map(extractAttributeKeys) - - // xsaiKeys.forEach((attributes, i) => { - // expect(attributes).toStrictEqual(aiKeys[i]) - // }) - // }) }) diff --git a/packages-ext/telemetry/test/stream-text.test.ts b/packages-ext/telemetry/test/stream-text.test.ts index 5e081f9..c563436 100644 --- a/packages-ext/telemetry/test/stream-text.test.ts +++ b/packages-ext/telemetry/test/stream-text.test.ts @@ -1,10 +1,6 @@ -import type { AttributeValue } from '@opentelemetry/api' -import type { ReadableSpan } from '@opentelemetry/sdk-trace-base' - +import { LangfuseSpanProcessor } from '@langfuse/otel' import { InMemorySpanExporter, SimpleSpanProcessor } from '@opentelemetry/sdk-trace-base' import { NodeTracerProvider } from '@opentelemetry/sdk-trace-node' -// import { streamText as aiStreamText } from 'ai' -// import { ollama } from 'ollama-ai-provider-v2' import { describe, expect, it } from 'vitest' import { streamText } from '../src' @@ -12,32 +8,13 @@ import { streamText } from '../src' describe.sequential('streamText', () => { const memoryExporter = new InMemorySpanExporter() const tracerProvider = new NodeTracerProvider({ - spanProcessors: [new SimpleSpanProcessor(memoryExporter)], + spanProcessors: [ + new SimpleSpanProcessor(memoryExporter), + new LangfuseSpanProcessor(), + ], }) tracerProvider.register() - const trimmedAttributes = new Set([ - // Not using the following because we are testing with --update - // 'ai.response.id', // reason: id - // 'ai.response.msToFinish', // reason: time - // 'ai.response.msToFirstChunk', // reason: time - // 'ai.response.timestamp', // reason: time - // 'gen_ai.response.id', // reason: id - ]) - - const getAttributes = (span: ReadableSpan) => Object.fromEntries( - Object.entries(span.attributes) - .reduce<[string, AttributeValue | undefined][]>((entries, [k, v]) => { - if (k === 'ai.response.text') { - entries.push([k, (v as string).replaceAll('\n', ' ').slice(0, 50)]) - } - else if (!trimmedAttributes.has(k)) { - entries.push([k, v]) - } - return entries - }, []), - ) - it('basic', async () => { let text = '' @@ -49,61 +26,28 @@ describe.sequential('streamText', () => { }], model: 'qwen3:0.6b', onFinish: async () => { - const spans = memoryExporter.getFinishedSpans().map(getAttributes) + const spans = memoryExporter.getFinishedSpans() + const names = spans.map(s => s.name) + const attributes = spans.map(s => s.attributes) expect(text).toMatchSnapshot() - expect(spans).toMatchSnapshot() + expect(names).toMatchSnapshot() + expect(attributes).toMatchSnapshot() }, seed: 114514, streamOptions: { includeUsage: true, }, - telemetry: { - metadata: { - agentId: 'weather-assistant', - instructions: 'You are a helpful weather assistant', - }, - }, + // telemetry: { + // metadata: { + // agentId: 'weather-assistant', + // instructions: 'You are a helpful weather assistant', + // }, + // }, }) for await (const textDelta of textStream) { text += textDelta } }, 120_000) - - // it('basic/ai', async () => { - // const { text } = aiStreamText({ - // experimental_telemetry: { - // isEnabled: true, - // metadata: { - // agentId: 'weather-assistant', - // instructions: 'You are a helpful weather assistant', - // }, - // }, - // model: ollama('qwen3:0.6b'), - // prompt: 'Why is the sky blue?', - // seed: 114514, - // }) - - // const fullText = await text - // const spans = memoryExporter.getFinishedSpans().slice(2).map(getAttributes) - - // expect(fullText).toMatchSnapshot() - // expect(spans).toMatchSnapshot() - // }, 120_000) - - // it('basic/compare', () => { - // const extractAttributeKeys = (span: ReadableSpan) => Object.keys(span.attributes) - // // eslint-disable-next-line sonarjs/no-nested-functions - // .filter(key => ['ai.settings'].every(prefix => !key.startsWith(prefix))) - // .sort((a, b) => a.localeCompare(b)) - - // const spans = memoryExporter.getFinishedSpans() - // const xsai = spans.slice(0, 2).map(extractAttributeKeys) - // const ai = spans.slice(2).map(extractAttributeKeys) - - // xsai.forEach((attributes, i) => { - // expect(attributes).toStrictEqual(ai[i]) - // }) - // }) }) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 51e21fa..5a930bf 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -311,21 +311,18 @@ importers: specifier: workspace:~ version: link:../../packages-top/xsai devDependencies: + '@langfuse/otel': + specifier: ^4.5.1 + version: 4.5.1(@opentelemetry/api@1.9.0)(@opentelemetry/core@2.2.0(@opentelemetry/api@1.9.0))(@opentelemetry/exporter-trace-otlp-http@0.208.0(@opentelemetry/api@1.9.0))(@opentelemetry/sdk-trace-base@2.2.0(@opentelemetry/api@1.9.0)) '@opentelemetry/sdk-trace-base': specifier: ^2.0.1 - version: 2.1.0(@opentelemetry/api@1.9.0) + version: 2.2.0(@opentelemetry/api@1.9.0) '@opentelemetry/sdk-trace-node': specifier: ^2.0.1 version: 2.1.0(@opentelemetry/api@1.9.0) - ai: - specifier: ^5.0.28 - version: 5.0.59(zod@4.1.11) - ollama-ai-provider-v2: - specifier: ^1.2.1 - version: 1.4.1(zod@4.1.11) zod: specifier: ^4.1.5 - version: 4.1.11 + version: 4.2.1 packages-top/xsai: dependencies: @@ -568,22 +565,6 @@ importers: packages: - '@ai-sdk/gateway@1.0.32': - resolution: {integrity: sha512-TQRIM63EI/ccJBc7RxeB8nq/CnGNnyl7eu5stWdLwL41stkV5skVeZJe0QRvFbaOrwCkgUVE0yrUqJi4tgDC1A==} - engines: {node: '>=18'} - peerDependencies: - zod: ^3.25.76 || ^4.1.8 - - '@ai-sdk/provider-utils@3.0.10': - resolution: {integrity: sha512-T1gZ76gEIwffep6MWI0QNy9jgoybUHE7TRaHB5k54K8mF91ciGFlbtCGxDYhMH3nCRergKwYFIDeFF0hJSIQHQ==} - engines: {node: '>=18'} - peerDependencies: - zod: ^3.25.76 || ^4.1.8 - - '@ai-sdk/provider@2.0.0': - resolution: {integrity: sha512-6o7Y2SeO9vFKB8lArHXehNuusnpddKPk7xqL7T2/b+OvXMRIXUO1rR4wcv1hAFUAT9avGZshty3Wlua/XA7TvA==} - engines: {node: '>=18'} - '@alloc/quick-lru@5.2.0': resolution: {integrity: sha512-UrcABB+4bUrFABwbluTIBErXwvbsU/V7TZWfmbgJfbkwiBuziS9gxdODUyuiecfdGQ85jglMW6juS3+z5TsKLw==} engines: {node: '>=10'} @@ -1448,6 +1429,20 @@ packages: '@jridgewell/trace-mapping@0.3.31': resolution: {integrity: sha512-zzNR+SdQSDJzc8joaeP8QQoCQr8NuYx2dIIytl1QeBEZHJ9uW6hebsrYgbz8hJwUQao3TWCMtmfV8Nu1twOLAw==} + '@langfuse/core@4.5.1': + resolution: {integrity: sha512-caJ2YWcaEU+kbzxFiyzRYaCmKxGzL9DSxbrCer8HbayYo2TaFaAu67Zeili8u8qG4q7TXga4aL2+rpU5ebWdRA==} + peerDependencies: + '@opentelemetry/api': ^1.9.0 + + '@langfuse/otel@4.5.1': + resolution: {integrity: sha512-cqykMEAYmGnd9RSZW2FPCNLda5jKZpCOnHTCu0pQD8EDgxCaHbnmD16k6WyjE/jywh991BcIFXzYub2fNbbSSQ==} + engines: {node: '>=20'} + peerDependencies: + '@opentelemetry/api': ^1.9.0 + '@opentelemetry/core': ^2.0.1 + '@opentelemetry/exporter-trace-otlp-http': '>=0.202.0 <1.0.0' + '@opentelemetry/sdk-trace-base': ^2.0.1 + '@masknet/eslint-plugin@0.3.0': resolution: {integrity: sha512-JXWUhcmUY20vEr70C6JJwVw7LZ8pGjXTcXWuSc4Ogj9vbASB9QJLPbgVuss66KOGMEDnuVFa2N01wE78+nBpPA==} engines: {node: '>= 18'} @@ -1540,6 +1535,10 @@ packages: resolution: {integrity: sha512-oGB+UxlgWcgQkgwo8GcEGwemoTFt3FIO9ababBmaGwXIoBKZ+GTy0pP185beGg7Llih/NSHSV2XAs1lnznocSg==} engines: {node: '>= 8'} + '@opentelemetry/api-logs@0.208.0': + resolution: {integrity: sha512-CjruKY9V6NMssL/T1kAFgzosF1v9o6oeN+aX5JB/C/xPNtmgIJqcXHG7fA82Ou1zCpWGl4lROQUKwUNE1pMCyg==} + engines: {node: '>=8.0.0'} + '@opentelemetry/api@1.9.0': resolution: {integrity: sha512-3giAOQvZiH5F9bMlMiv8+GSPMeqg0dbaeo58/0SlA9sxSqZhnUtxzX9/2FzyhS9sWQf5S0GJE0AKBrFqjpeYcg==} engines: {node: '>=8.0.0'} @@ -1556,18 +1555,66 @@ packages: peerDependencies: '@opentelemetry/api': '>=1.0.0 <1.10.0' + '@opentelemetry/core@2.2.0': + resolution: {integrity: sha512-FuabnnUm8LflnieVxs6eP7Z383hgQU4W1e3KJS6aOG3RxWxcHyBxH8fDMHNgu/gFx/M2jvTOW/4/PHhLz6bjWw==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': '>=1.0.0 <1.10.0' + + '@opentelemetry/exporter-trace-otlp-http@0.208.0': + resolution: {integrity: sha512-jbzDw1q+BkwKFq9yxhjAJ9rjKldbt5AgIy1gmEIJjEV/WRxQ3B6HcLVkwbjJ3RcMif86BDNKR846KJ0tY0aOJA==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': ^1.3.0 + + '@opentelemetry/otlp-exporter-base@0.208.0': + resolution: {integrity: sha512-gMd39gIfVb2OgxldxUtOwGJYSH8P1kVFFlJLuut32L6KgUC4gl1dMhn+YC2mGn0bDOiQYSk/uHOdSjuKp58vvA==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': ^1.3.0 + + '@opentelemetry/otlp-transformer@0.208.0': + resolution: {integrity: sha512-DCFPY8C6lAQHUNkzcNT9R+qYExvsk6C5Bto2pbNxgicpcSWbe2WHShLxkOxIdNcBiYPdVHv/e7vH7K6TI+C+fQ==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': ^1.3.0 + '@opentelemetry/resources@2.1.0': resolution: {integrity: sha512-1CJjf3LCvoefUOgegxi8h6r4B/wLSzInyhGP2UmIBYNlo4Qk5CZ73e1eEyWmfXvFtm1ybkmfb2DqWvspsYLrWw==} engines: {node: ^18.19.0 || >=20.6.0} peerDependencies: '@opentelemetry/api': '>=1.3.0 <1.10.0' + '@opentelemetry/resources@2.2.0': + resolution: {integrity: sha512-1pNQf/JazQTMA0BiO5NINUzH0cbLbbl7mntLa4aJNmCCXSj0q03T5ZXXL0zw4G55TjdL9Tz32cznGClf+8zr5A==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': '>=1.3.0 <1.10.0' + + '@opentelemetry/sdk-logs@0.208.0': + resolution: {integrity: sha512-QlAyL1jRpOeaqx7/leG1vJMp84g0xKP6gJmfELBpnI4O/9xPX+Hu5m1POk9Kl+veNkyth5t19hRlN6tNY1sjbA==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': '>=1.4.0 <1.10.0' + + '@opentelemetry/sdk-metrics@2.2.0': + resolution: {integrity: sha512-G5KYP6+VJMZzpGipQw7Giif48h6SGQ2PFKEYCybeXJsOCB4fp8azqMAAzE5lnnHK3ZVwYQrgmFbsUJO/zOnwGw==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': '>=1.9.0 <1.10.0' + '@opentelemetry/sdk-trace-base@2.1.0': resolution: {integrity: sha512-uTX9FBlVQm4S2gVQO1sb5qyBLq/FPjbp+tmGoxu4tIgtYGmBYB44+KX/725RFDe30yBSaA9Ml9fqphe1hbUyLQ==} engines: {node: ^18.19.0 || >=20.6.0} peerDependencies: '@opentelemetry/api': '>=1.3.0 <1.10.0' + '@opentelemetry/sdk-trace-base@2.2.0': + resolution: {integrity: sha512-xWQgL0Bmctsalg6PaXExmzdedSp3gyKV8mQBwK/j9VGdCDu2fmXIb2gAehBKbkXCpJ4HPkgv3QfoJWRT4dHWbw==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': '>=1.3.0 <1.10.0' + '@opentelemetry/sdk-trace-node@2.1.0': resolution: {integrity: sha512-SvVlBFc/jI96u/mmlKm86n9BbTCbQ35nsPoOohqJX6DXH92K0kTe73zGY5r8xoI1QkjR9PizszVJLzMC966y9Q==} engines: {node: ^18.19.0 || >=20.6.0} @@ -1683,6 +1730,36 @@ packages: resolution: {integrity: sha512-4rdu8GPY9TeQwsYp5D2My74dC3dSVS3tghAvisG80ybK4lqa0gvlrglaSTBxogJbxqHRw/NjI/liEtb3+SD+Bw==} engines: {node: '>=18.12'} + '@protobufjs/aspromise@1.1.2': + resolution: {integrity: sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ==} + + '@protobufjs/base64@1.1.2': + resolution: {integrity: sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg==} + + '@protobufjs/codegen@2.0.4': + resolution: {integrity: sha512-YyFaikqM5sH0ziFZCN3xDC7zeGaB/d0IUb9CATugHWbd1FRFwWwt4ld4OYMPWu5a3Xe01mGAULCdqhMlPl29Jg==} + + '@protobufjs/eventemitter@1.1.0': + resolution: {integrity: sha512-j9ednRT81vYJ9OfVuXG6ERSTdEL1xVsNgqpkxMsbIabzSo3goCjDIveeGv5d03om39ML71RdmrGNjG5SReBP/Q==} + + '@protobufjs/fetch@1.1.0': + resolution: {integrity: sha512-lljVXpqXebpsijW71PZaCYeIcE5on1w5DlQy5WH6GLbFryLUrBD4932W/E2BSpfRJWseIL4v/KPgBFxDOIdKpQ==} + + '@protobufjs/float@1.0.2': + resolution: {integrity: sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ==} + + '@protobufjs/inquire@1.1.0': + resolution: {integrity: sha512-kdSefcPdruJiFMVSbn801t4vFK7KB/5gd2fYvrxhuJYg8ILrmn9SKSX2tZdV6V+ksulWqS7aXjBcRXl3wHoD9Q==} + + '@protobufjs/path@1.1.2': + resolution: {integrity: sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA==} + + '@protobufjs/pool@1.1.0': + resolution: {integrity: sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw==} + + '@protobufjs/utf8@1.1.0': + resolution: {integrity: sha512-Vvn3zZrhQZkkBE8LSuW3em98c0FwgO4nxzv6OdSxPKJIEKY2bGbHn+mhGIPerzI4twdxaP8/0+06HBpwf345Lw==} + '@quansync/fs@0.1.5': resolution: {integrity: sha512-lNS9hL2aS2NZgNW7BBj+6EBl4rOf8l+tQ0eRY6JWCI8jI2kc53gSoqbjojU0OnAWhzoXiOjFyGsHcDGePB3lhA==} @@ -2814,12 +2891,6 @@ packages: engines: {node: '>=0.4.0'} hasBin: true - ai@5.0.59: - resolution: {integrity: sha512-SuAFxKXt2Ha9FiXB3gaOITkOg9ek/3QNVatGVExvTT4gNXc+hJpuNe1dmuwf6Z5Op4fzc8wdbsrYP27ZCXBzlw==} - engines: {node: '>=18'} - peerDependencies: - zod: ^3.25.76 || ^4.1.8 - ajv@6.12.6: resolution: {integrity: sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==} @@ -3640,10 +3711,6 @@ packages: eventemitter3@4.0.7: resolution: {integrity: sha512-8guHBZCwKnFhYdHr2ysuRWErTwhoN2X8XELRlrRwpmfeY2jjuUN4taQMsULKUVo1K4DvZl+0pgfyoysHxvmvEw==} - eventsource-parser@3.0.6: - resolution: {integrity: sha512-Vo1ab+QXPzZ4tCa8SwIHJFaSzy4R6SHf7BY79rFBDf0idraZWAkYrDjDj8uWaSm3S2TK+hJ7/t1CEmZ7jXw+pg==} - engines: {node: '>=18.0.0'} - expect-type@1.2.1: resolution: {integrity: sha512-/kP8CAwxzLVEeFrMm4kMmy4CCDlpipyA7MYLVrdJIkV0fYF0UaigQHRsxHiuY/GEea+bh4KSv3TIlgr+2UL6bw==} engines: {node: '>=12.0.0'} @@ -4183,9 +4250,6 @@ packages: json-schema-traverse@0.4.1: resolution: {integrity: sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==} - json-schema@0.4.0: - resolution: {integrity: sha512-es94M3nTIfsEPisRafak+HDLfHXnKBhV3vU5eqPcS3flIWqcxJWgXHXiey3YrpaNsanY5ei1VoYEbOzijuq9BA==} - json-stable-stringify-without-jsonify@1.0.1: resolution: {integrity: sha512-Bdboy+l7tA3OGW6FjyFHWkP5LuByj1Tk33Ljyq0axyzdk9//JSi2u3fP1QSmd1KNwq6VOKYGlAu87CisVir6Pw==} @@ -4295,6 +4359,9 @@ packages: lodash@4.17.21: resolution: {integrity: sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==} + long@5.3.2: + resolution: {integrity: sha512-mNAgZ1GmyNhD7AuqnTG3/VQ26o760+ZYBPKjPvugO8+nLbYfX6TVpJPseBvopbdY+qpZ/lKUnmEc1LeZYS3QAA==} + longest-streak@3.1.0: resolution: {integrity: sha512-9Ri+o0JYgehTaVBBDoMqIl8GXtbWg711O3srftcHhZ0dqnETqLaoIK0x17fUw9rFSlK/0NlsKe0Ahhyl5pXE2g==} @@ -4574,6 +4641,7 @@ packages: next@15.5.4: resolution: {integrity: sha512-xH4Yjhb82sFYQfY3vbkJfgSDgXvBB6a8xPs9i35k6oZJRoQRihZH+4s9Yo2qsWpzBmZ3lPXaJ2KPXLfkvW4LnA==} engines: {node: ^18.18.0 || ^19.8.0 || >= 20.0.0} + deprecated: This version has a security vulnerability. Please upgrade to a patched version. See https://nextjs.org/blog/CVE-2025-66478 for more details. hasBin: true peerDependencies: '@opentelemetry/api': ^1.1.0 @@ -4642,12 +4710,6 @@ packages: ohash@2.0.11: resolution: {integrity: sha512-RdR9FQrFwNBNXAr4GixM8YaRZRJ5PUWbKYbE5eOsrwAjJW0q2REGcf79oYPsLyskQCZG1PLN+S/K1V00joZAoQ==} - ollama-ai-provider-v2@1.4.1: - resolution: {integrity: sha512-coo3+IvnL6O4Ku0TekUiQCEYYvLRbfLfPBcx0e30fxrfKTArEff0oinTKGZ0RRGFxvqd0LgFa6f3kxppA71f3A==} - engines: {node: '>=18'} - peerDependencies: - zod: ^4.0.16 - oniguruma-parser@0.12.1: resolution: {integrity: sha512-8Unqkvk1RYc6yq2WBYRj4hdnsAxVze8i7iPfQr8e4uSP3tRv0rpZcbGUDvxfQQcdwHt/e9PrMvGCsa8OqG9X3w==} @@ -4789,6 +4851,10 @@ packages: property-information@7.1.0: resolution: {integrity: sha512-TwEZ+X+yCJmYfL7TPUOcvBZ4QfoT5YenQiJuX//0th53DE6w0xxLEtfK3iyryQFddXuvkIk51EEgrJQ0WJkOmQ==} + protobufjs@7.5.4: + resolution: {integrity: sha512-CvexbZtbov6jW2eXAvLukXjXUW1TzFaivC46BpWc/3BpcCysb5Vffu+B3XHMm8lVEuy2Mm4XGex8hBSg1yapPg==} + engines: {node: '>=12.0.0'} + punycode@2.3.1: resolution: {integrity: sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==} engines: {node: '>=6'} @@ -5792,9 +5858,6 @@ packages: zod@4.0.5: resolution: {integrity: sha512-/5UuuRPStvHXu7RS+gmvRf4NXrNxpSllGwDnCBcJZtQsKrviYXm54yDGV2KYNLT5kq0lHGcl7lqWJLgSaG+tgA==} - zod@4.1.11: - resolution: {integrity: sha512-WPsqwxITS2tzx1bzhIKsEs19ABD5vmCVa4xBo2tq/SrV4RNZtfws1EnCWQXM6yh8bD08a1idvkB5MZSBiZsjwg==} - zod@4.2.1: resolution: {integrity: sha512-0wZ1IRqGGhMP76gLqz8EyfBXKk0J2qo2+H3fi4mcUP/KtTocoX08nmIAHl1Z2kJIZbZee8KOpBCSNPRgauucjw==} @@ -5803,23 +5866,6 @@ packages: snapshots: - '@ai-sdk/gateway@1.0.32(zod@4.1.11)': - dependencies: - '@ai-sdk/provider': 2.0.0 - '@ai-sdk/provider-utils': 3.0.10(zod@4.1.11) - zod: 4.1.11 - - '@ai-sdk/provider-utils@3.0.10(zod@4.1.11)': - dependencies: - '@ai-sdk/provider': 2.0.0 - '@standard-schema/spec': 1.1.0 - eventsource-parser: 3.0.6 - zod: 4.1.11 - - '@ai-sdk/provider@2.0.0': - dependencies: - json-schema: 0.4.0 - '@alloc/quick-lru@5.2.0': {} '@ampproject/remapping@2.3.0': @@ -6453,6 +6499,18 @@ snapshots: '@jridgewell/resolve-uri': 3.1.2 '@jridgewell/sourcemap-codec': 1.5.5 + '@langfuse/core@4.5.1(@opentelemetry/api@1.9.0)': + dependencies: + '@opentelemetry/api': 1.9.0 + + '@langfuse/otel@4.5.1(@opentelemetry/api@1.9.0)(@opentelemetry/core@2.2.0(@opentelemetry/api@1.9.0))(@opentelemetry/exporter-trace-otlp-http@0.208.0(@opentelemetry/api@1.9.0))(@opentelemetry/sdk-trace-base@2.2.0(@opentelemetry/api@1.9.0))': + dependencies: + '@langfuse/core': 4.5.1(@opentelemetry/api@1.9.0) + '@opentelemetry/api': 1.9.0 + '@opentelemetry/core': 2.2.0(@opentelemetry/api@1.9.0) + '@opentelemetry/exporter-trace-otlp-http': 0.208.0(@opentelemetry/api@1.9.0) + '@opentelemetry/sdk-trace-base': 2.2.0(@opentelemetry/api@1.9.0) + '@masknet/eslint-plugin@0.3.0(eslint@9.36.0(jiti@2.6.0))': dependencies: eslint: 9.36.0(jiti@2.6.0) @@ -6551,6 +6609,10 @@ snapshots: '@nodelib/fs.scandir': 2.1.5 fastq: 1.17.1 + '@opentelemetry/api-logs@0.208.0': + dependencies: + '@opentelemetry/api': 1.9.0 + '@opentelemetry/api@1.9.0': {} '@opentelemetry/context-async-hooks@2.1.0(@opentelemetry/api@1.9.0)': @@ -6562,12 +6624,62 @@ snapshots: '@opentelemetry/api': 1.9.0 '@opentelemetry/semantic-conventions': 1.36.0 + '@opentelemetry/core@2.2.0(@opentelemetry/api@1.9.0)': + dependencies: + '@opentelemetry/api': 1.9.0 + '@opentelemetry/semantic-conventions': 1.36.0 + + '@opentelemetry/exporter-trace-otlp-http@0.208.0(@opentelemetry/api@1.9.0)': + dependencies: + '@opentelemetry/api': 1.9.0 + '@opentelemetry/core': 2.2.0(@opentelemetry/api@1.9.0) + '@opentelemetry/otlp-exporter-base': 0.208.0(@opentelemetry/api@1.9.0) + '@opentelemetry/otlp-transformer': 0.208.0(@opentelemetry/api@1.9.0) + '@opentelemetry/resources': 2.2.0(@opentelemetry/api@1.9.0) + '@opentelemetry/sdk-trace-base': 2.2.0(@opentelemetry/api@1.9.0) + + '@opentelemetry/otlp-exporter-base@0.208.0(@opentelemetry/api@1.9.0)': + dependencies: + '@opentelemetry/api': 1.9.0 + '@opentelemetry/core': 2.2.0(@opentelemetry/api@1.9.0) + '@opentelemetry/otlp-transformer': 0.208.0(@opentelemetry/api@1.9.0) + + '@opentelemetry/otlp-transformer@0.208.0(@opentelemetry/api@1.9.0)': + dependencies: + '@opentelemetry/api': 1.9.0 + '@opentelemetry/api-logs': 0.208.0 + '@opentelemetry/core': 2.2.0(@opentelemetry/api@1.9.0) + '@opentelemetry/resources': 2.2.0(@opentelemetry/api@1.9.0) + '@opentelemetry/sdk-logs': 0.208.0(@opentelemetry/api@1.9.0) + '@opentelemetry/sdk-metrics': 2.2.0(@opentelemetry/api@1.9.0) + '@opentelemetry/sdk-trace-base': 2.2.0(@opentelemetry/api@1.9.0) + protobufjs: 7.5.4 + '@opentelemetry/resources@2.1.0(@opentelemetry/api@1.9.0)': dependencies: '@opentelemetry/api': 1.9.0 '@opentelemetry/core': 2.1.0(@opentelemetry/api@1.9.0) '@opentelemetry/semantic-conventions': 1.36.0 + '@opentelemetry/resources@2.2.0(@opentelemetry/api@1.9.0)': + dependencies: + '@opentelemetry/api': 1.9.0 + '@opentelemetry/core': 2.2.0(@opentelemetry/api@1.9.0) + '@opentelemetry/semantic-conventions': 1.36.0 + + '@opentelemetry/sdk-logs@0.208.0(@opentelemetry/api@1.9.0)': + dependencies: + '@opentelemetry/api': 1.9.0 + '@opentelemetry/api-logs': 0.208.0 + '@opentelemetry/core': 2.2.0(@opentelemetry/api@1.9.0) + '@opentelemetry/resources': 2.2.0(@opentelemetry/api@1.9.0) + + '@opentelemetry/sdk-metrics@2.2.0(@opentelemetry/api@1.9.0)': + dependencies: + '@opentelemetry/api': 1.9.0 + '@opentelemetry/core': 2.2.0(@opentelemetry/api@1.9.0) + '@opentelemetry/resources': 2.2.0(@opentelemetry/api@1.9.0) + '@opentelemetry/sdk-trace-base@2.1.0(@opentelemetry/api@1.9.0)': dependencies: '@opentelemetry/api': 1.9.0 @@ -6575,6 +6687,13 @@ snapshots: '@opentelemetry/resources': 2.1.0(@opentelemetry/api@1.9.0) '@opentelemetry/semantic-conventions': 1.36.0 + '@opentelemetry/sdk-trace-base@2.2.0(@opentelemetry/api@1.9.0)': + dependencies: + '@opentelemetry/api': 1.9.0 + '@opentelemetry/core': 2.2.0(@opentelemetry/api@1.9.0) + '@opentelemetry/resources': 2.2.0(@opentelemetry/api@1.9.0) + '@opentelemetry/semantic-conventions': 1.36.0 + '@opentelemetry/sdk-trace-node@2.1.0(@opentelemetry/api@1.9.0)': dependencies: '@opentelemetry/api': 1.9.0 @@ -6662,6 +6781,29 @@ snapshots: '@pnpm/error': 1000.0.5 find-up: 5.0.0 + '@protobufjs/aspromise@1.1.2': {} + + '@protobufjs/base64@1.1.2': {} + + '@protobufjs/codegen@2.0.4': {} + + '@protobufjs/eventemitter@1.1.0': {} + + '@protobufjs/fetch@1.1.0': + dependencies: + '@protobufjs/aspromise': 1.1.2 + '@protobufjs/inquire': 1.1.0 + + '@protobufjs/float@1.0.2': {} + + '@protobufjs/inquire@1.1.0': {} + + '@protobufjs/path@1.1.2': {} + + '@protobufjs/pool@1.1.0': {} + + '@protobufjs/utf8@1.1.0': {} + '@quansync/fs@0.1.5': dependencies: quansync: 0.2.11 @@ -7752,14 +7894,6 @@ snapshots: acorn@8.15.0: {} - ai@5.0.59(zod@4.1.11): - dependencies: - '@ai-sdk/gateway': 1.0.32(zod@4.1.11) - '@ai-sdk/provider': 2.0.0 - '@ai-sdk/provider-utils': 3.0.10(zod@4.1.11) - '@opentelemetry/api': 1.9.0 - zod: 4.1.11 - ajv@6.12.6: dependencies: fast-deep-equal: 3.1.3 @@ -8791,8 +8925,6 @@ snapshots: eventemitter3@4.0.7: {} - eventsource-parser@3.0.6: {} - expect-type@1.2.1: {} exsolve@1.0.7: {} @@ -8917,7 +9049,7 @@ snapshots: tinyglobby: 0.2.15 unified: 11.0.5 unist-util-visit: 5.0.0 - zod: 4.1.11 + zod: 4.2.1 optionalDependencies: next: 15.5.4(@opentelemetry/api@1.9.0)(react-dom@19.1.1(react@19.1.1))(react@19.1.1) react: 19.1.1 @@ -9379,8 +9511,6 @@ snapshots: json-schema-traverse@0.4.1: {} - json-schema@0.4.0: {} - json-stable-stringify-without-jsonify@1.0.1: {} jsonc-eslint-parser@2.4.0: @@ -9478,6 +9608,8 @@ snapshots: lodash@4.17.21: {} + long@5.3.2: {} + longest-streak@3.1.0: {} loose-envify@1.4.0: @@ -10100,12 +10232,6 @@ snapshots: ohash@2.0.11: {} - ollama-ai-provider-v2@1.4.1(zod@4.1.11): - dependencies: - '@ai-sdk/provider': 2.0.0 - '@ai-sdk/provider-utils': 3.0.10(zod@4.1.11) - zod: 4.1.11 - oniguruma-parser@0.12.1: {} oniguruma-to-es@4.3.3: @@ -10260,6 +10386,21 @@ snapshots: property-information@7.1.0: {} + protobufjs@7.5.4: + dependencies: + '@protobufjs/aspromise': 1.1.2 + '@protobufjs/base64': 1.1.2 + '@protobufjs/codegen': 2.0.4 + '@protobufjs/eventemitter': 1.1.0 + '@protobufjs/fetch': 1.1.0 + '@protobufjs/float': 1.0.2 + '@protobufjs/inquire': 1.1.0 + '@protobufjs/path': 1.1.2 + '@protobufjs/pool': 1.1.0 + '@protobufjs/utf8': 1.1.0 + '@types/node': 24.6.0 + long: 5.3.2 + punycode@2.3.1: {} pure-rand@6.1.0: {} @@ -11452,8 +11593,6 @@ snapshots: zod@4.0.5: {} - zod@4.1.11: {} - zod@4.2.1: {} zwitch@2.0.4: {} diff --git a/vitest.config.ts b/vitest.config.ts index b1aa35e..cec63a5 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -1,6 +1,7 @@ import { defineConfig } from 'vitest/config' export default defineConfig({ + envPrefix: ['VITE_', 'LANGFUSE_'], test: { coverage: { reporter: ['text', 'json', 'html'],