From 1b2e62cd14a7b92aa9ce979077248ddc5204a486 Mon Sep 17 00:00:00 2001 From: Stephen Belanger Date: Tue, 31 Mar 2026 06:44:10 +0800 Subject: [PATCH 1/3] fix(e2e): stabilize canary tests for langchain version drift and anthropic timeout Normalize invocation-parameter fields (max_tokens, model, stream, temperature) that older @langchain/openai included alongside the standardized ls_* metadata keys but newer versions removed. Dropping these from the snapshot makes the wrap-langchain-js-traces canary test stable across both the locked and latest langchain versions. Also increase the anthropic-instrumentation scenario timeout from 90s to 150s to accommodate the additional messages.batches API calls (create, retrieve, list, cancel) that will be exercised once the batches PR merges. Co-Authored-By: Claude Sonnet 4.6 --- .../scenario.test.ts | 2 +- .../__snapshots__/log-payloads.json | 24 ------------------- .../wrap-langchain-js-traces/assertions.ts | 19 +++++++++++++++ 3 files changed, 20 insertions(+), 25 deletions(-) diff --git a/e2e/scenarios/anthropic-instrumentation/scenario.test.ts b/e2e/scenarios/anthropic-instrumentation/scenario.test.ts index baebbc01d..ce7184bef 100644 --- a/e2e/scenarios/anthropic-instrumentation/scenario.test.ts +++ b/e2e/scenarios/anthropic-instrumentation/scenario.test.ts @@ -9,7 +9,7 @@ import { defineAnthropicInstrumentationAssertions } from "./assertions"; const scenarioDir = await prepareScenarioDir({ scenarioDir: resolveScenarioDir(import.meta.url), }); -const TIMEOUT_MS = 90_000; +const TIMEOUT_MS = 150_000; const anthropicScenarios = await Promise.all( [ { diff --git a/e2e/scenarios/wrap-langchain-js-traces/__snapshots__/log-payloads.json b/e2e/scenarios/wrap-langchain-js-traces/__snapshots__/log-payloads.json index 7a1b005aa..157257ac0 100644 --- a/e2e/scenarios/wrap-langchain-js-traces/__snapshots__/log-payloads.json +++ b/e2e/scenarios/wrap-langchain-js-traces/__snapshots__/log-payloads.json @@ -126,10 +126,6 @@ "ls_model_type": "chat", "ls_provider": "openai", "ls_temperature": 0, - "max_tokens": 16, - "model": "gpt-4o-mini", - "stream": false, - "temperature": 0, "versions": { "@langchain/core": "", "@langchain/openai": "" @@ -704,10 +700,6 @@ "ls_model_type": "chat", "ls_provider": "openai", "ls_temperature": 0, - "max_tokens": 32, - "model": "gpt-4o-mini", - "stream": false, - "temperature": 0, "versions": { "@langchain/core": "", "@langchain/openai": "" @@ -973,13 +965,9 @@ "ls_model_type": "chat", "ls_provider": "openai", "ls_temperature": 0, - "max_tokens": 32, - "model": "gpt-4o-mini", - "stream": true, "stream_options": { "include_usage": true }, - "temperature": 0, "versions": { "@langchain/core": "", "@langchain/openai": "" @@ -1265,10 +1253,6 @@ "ls_model_type": "chat", "ls_provider": "openai", "ls_temperature": 0, - "max_tokens": 128, - "model": "gpt-4o-mini", - "stream": false, - "temperature": 0, "versions": { "@langchain/core": "", "@langchain/openai": "" @@ -1606,10 +1590,6 @@ "ls_model_type": "chat", "ls_provider": "openai", "ls_temperature": 0, - "max_tokens": 128, - "model": "gpt-4o-mini", - "stream": false, - "temperature": 0, "versions": { "@langchain/core": "", "@langchain/openai": "" @@ -1962,10 +1942,6 @@ "ls_model_type": "chat", "ls_provider": "openai", "ls_temperature": 0, - "max_tokens": 128, - "model": "gpt-4o-mini", - "stream": false, - "temperature": 0, "versions": { "@langchain/core": "", "@langchain/openai": "" diff --git a/e2e/scenarios/wrap-langchain-js-traces/assertions.ts b/e2e/scenarios/wrap-langchain-js-traces/assertions.ts index ff6415220..e2e8bda9b 100644 --- a/e2e/scenarios/wrap-langchain-js-traces/assertions.ts +++ b/e2e/scenarios/wrap-langchain-js-traces/assertions.ts @@ -137,6 +137,16 @@ function normalizeToolCallIds(obj: unknown): void { } } +// Fields that older @langchain/openai included in the ls_* metadata block but +// newer versions removed. Normalize them out so the snapshot is stable across +// both locked and canary (latest) langchain versions. +const LANGCHAIN_LS_VOLATILE_KEYS = new Set([ + "max_tokens", + "model", + "stream", + "temperature", +]); + function normalizeLangchainVersions(obj: unknown): void { if (!obj || typeof obj !== "object") return; @@ -161,6 +171,15 @@ function normalizeLangchainVersions(obj: unknown): void { } } + // If this object is the ls_* metadata block (identified by the presence of + // any `ls_` key), remove volatile keys that newer langchain drops. + const hasLsKey = Object.keys(record).some((k) => k.startsWith("ls_")); + if (hasLsKey) { + for (const key of LANGCHAIN_LS_VOLATILE_KEYS) { + delete record[key]; + } + } + for (const value of Object.values(record)) { if (typeof value === "object" && value !== null) { normalizeLangchainVersions(value); From 386d1953368b9f9caa094cde8103c2b1548364ba Mon Sep 17 00:00:00 2001 From: Stephen Belanger Date: Tue, 31 Mar 2026 10:36:48 +0800 Subject: [PATCH 2/3] fix(e2e): Fix remaining canary snapshot failures MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add stream_options to LANGCHAIN_LS_VOLATILE_KEYS — newer @langchain/openai dropped it from the ls_* metadata block, causing the canary snapshot to mismatch - Update ai-sdk-v6.otel-spans.json: ai 6.0.1 reordered spans so ai.generateText now precedes ai.generateText.doGenerate (previously reversed) Co-Authored-By: Claude Sonnet 4.6 --- .../__snapshots__/ai-sdk-v6.otel-spans.json | 4 ++-- .../wrap-langchain-js-traces/__snapshots__/log-payloads.json | 3 --- e2e/scenarios/wrap-langchain-js-traces/assertions.ts | 1 + 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/e2e/scenarios/ai-sdk-otel-export/__snapshots__/ai-sdk-v6.otel-spans.json b/e2e/scenarios/ai-sdk-otel-export/__snapshots__/ai-sdk-v6.otel-spans.json index acf5e50ae..ca9a5d0db 100644 --- a/e2e/scenarios/ai-sdk-otel-export/__snapshots__/ai-sdk-v6.otel-spans.json +++ b/e2e/scenarios/ai-sdk-otel-export/__snapshots__/ai-sdk-v6.otel-spans.json @@ -1,11 +1,11 @@ [ { "hasParent": false, - "name": "ai.generateText.doGenerate" + "name": "ai.generateText" }, { "hasParent": false, - "name": "ai.generateText" + "name": "ai.generateText.doGenerate" }, { "hasParent": false, diff --git a/e2e/scenarios/wrap-langchain-js-traces/__snapshots__/log-payloads.json b/e2e/scenarios/wrap-langchain-js-traces/__snapshots__/log-payloads.json index 157257ac0..b3852b50a 100644 --- a/e2e/scenarios/wrap-langchain-js-traces/__snapshots__/log-payloads.json +++ b/e2e/scenarios/wrap-langchain-js-traces/__snapshots__/log-payloads.json @@ -965,9 +965,6 @@ "ls_model_type": "chat", "ls_provider": "openai", "ls_temperature": 0, - "stream_options": { - "include_usage": true - }, "versions": { "@langchain/core": "", "@langchain/openai": "" diff --git a/e2e/scenarios/wrap-langchain-js-traces/assertions.ts b/e2e/scenarios/wrap-langchain-js-traces/assertions.ts index e2e8bda9b..f5e1ed7dd 100644 --- a/e2e/scenarios/wrap-langchain-js-traces/assertions.ts +++ b/e2e/scenarios/wrap-langchain-js-traces/assertions.ts @@ -144,6 +144,7 @@ const LANGCHAIN_LS_VOLATILE_KEYS = new Set([ "max_tokens", "model", "stream", + "stream_options", "temperature", ]); From a528a7b195f50aead0e3899f830101c748f7b708 Mon Sep 17 00:00:00 2001 From: Stephen Belanger Date: Tue, 31 Mar 2026 10:51:40 +0800 Subject: [PATCH 3/3] fix(e2e): Sort ai-sdk-otel-export spans before snapshotting OTel spans arrive in non-deterministic order across requests, causing ai.generateText vs ai.generateText.doGenerate to flap position between runs. Sort by name (then hasParent) before snapshotting so the comparison is stable. Both v5 and v6 snapshots are now identical since they contain the same set of spans. Co-Authored-By: Claude Sonnet 4.6 --- .../__snapshots__/ai-sdk-v5.otel-spans.json | 16 ++++++++-------- .../__snapshots__/ai-sdk-v6.otel-spans.json | 16 ++++++++-------- .../ai-sdk-otel-export/scenario.test.ts | 15 +++++++++++---- 3 files changed, 27 insertions(+), 20 deletions(-) diff --git a/e2e/scenarios/ai-sdk-otel-export/__snapshots__/ai-sdk-v5.otel-spans.json b/e2e/scenarios/ai-sdk-otel-export/__snapshots__/ai-sdk-v5.otel-spans.json index acf5e50ae..fd5442920 100644 --- a/e2e/scenarios/ai-sdk-otel-export/__snapshots__/ai-sdk-v5.otel-spans.json +++ b/e2e/scenarios/ai-sdk-otel-export/__snapshots__/ai-sdk-v5.otel-spans.json @@ -1,7 +1,7 @@ [ { "hasParent": false, - "name": "ai.generateText.doGenerate" + "name": "ai.generateText" }, { "hasParent": false, @@ -9,11 +9,11 @@ }, { "hasParent": false, - "name": "ai.streamText.doStream" + "name": "ai.generateText.doGenerate" }, { "hasParent": false, - "name": "ai.streamText" + "name": "ai.generateText.doGenerate" }, { "hasParent": false, @@ -21,7 +21,7 @@ }, { "hasParent": false, - "name": "ai.toolCall" + "name": "ai.generateText.doGenerate" }, { "hasParent": false, @@ -29,11 +29,11 @@ }, { "hasParent": false, - "name": "ai.toolCall" + "name": "ai.streamText" }, { "hasParent": false, - "name": "ai.generateText.doGenerate" + "name": "ai.streamText.doStream" }, { "hasParent": false, @@ -41,7 +41,7 @@ }, { "hasParent": false, - "name": "ai.generateText.doGenerate" + "name": "ai.toolCall" }, { "hasParent": false, @@ -49,6 +49,6 @@ }, { "hasParent": false, - "name": "ai.generateText" + "name": "ai.toolCall" } ] diff --git a/e2e/scenarios/ai-sdk-otel-export/__snapshots__/ai-sdk-v6.otel-spans.json b/e2e/scenarios/ai-sdk-otel-export/__snapshots__/ai-sdk-v6.otel-spans.json index ca9a5d0db..fd5442920 100644 --- a/e2e/scenarios/ai-sdk-otel-export/__snapshots__/ai-sdk-v6.otel-spans.json +++ b/e2e/scenarios/ai-sdk-otel-export/__snapshots__/ai-sdk-v6.otel-spans.json @@ -5,15 +5,15 @@ }, { "hasParent": false, - "name": "ai.generateText.doGenerate" + "name": "ai.generateText" }, { "hasParent": false, - "name": "ai.streamText.doStream" + "name": "ai.generateText.doGenerate" }, { "hasParent": false, - "name": "ai.streamText" + "name": "ai.generateText.doGenerate" }, { "hasParent": false, @@ -21,7 +21,7 @@ }, { "hasParent": false, - "name": "ai.toolCall" + "name": "ai.generateText.doGenerate" }, { "hasParent": false, @@ -29,11 +29,11 @@ }, { "hasParent": false, - "name": "ai.toolCall" + "name": "ai.streamText" }, { "hasParent": false, - "name": "ai.generateText.doGenerate" + "name": "ai.streamText.doStream" }, { "hasParent": false, @@ -41,7 +41,7 @@ }, { "hasParent": false, - "name": "ai.generateText.doGenerate" + "name": "ai.toolCall" }, { "hasParent": false, @@ -49,6 +49,6 @@ }, { "hasParent": false, - "name": "ai.generateText" + "name": "ai.toolCall" } ] diff --git a/e2e/scenarios/ai-sdk-otel-export/scenario.test.ts b/e2e/scenarios/ai-sdk-otel-export/scenario.test.ts index 7b230db44..7d81f5eac 100644 --- a/e2e/scenarios/ai-sdk-otel-export/scenario.test.ts +++ b/e2e/scenarios/ai-sdk-otel-export/scenario.test.ts @@ -119,10 +119,17 @@ for (const scenario of scenarios) { // Snapshot the span names and key structure (not full payloads, since // response content and token counts are non-deterministic). - const spanSummary = allSpans.map((span) => ({ - hasParent: !!span.parentSpanId, - name: span.name, - })); + // Sort for determinism — OTel spans arrive in non-deterministic order. + const spanSummary = allSpans + .map((span) => ({ + hasParent: !!span.parentSpanId, + name: span.name, + })) + .sort((a, b) => + a.name !== b.name + ? a.name.localeCompare(b.name) + : Number(a.hasParent) - Number(b.hasParent), + ); await expect(formatJsonFileSnapshot(spanSummary)).toMatchFileSnapshot( resolveFileSnapshotPath( import.meta.url,