import * as uuid from "uuid"; import { AsyncCaller } from "./utils/async_caller.js"; import { convertLangChainMessageToExample, isLangChainMessage, } from "./utils/messages.js"; import { getLangChainEnvVarsMetadata, getLangSmithEnvironmentVariable, getRuntimeEnvironment, } from "./utils/env.js"; import { __version__ } from "./index.js"; import { assertUuid } from "./utils/_uuid.js"; import { warnOnce } from "./utils/warn.js"; import { isVersionGreaterOrEqual, parsePromptIdentifier, } from "./utils/prompts.js"; import { raiseForStatus } from "./utils/error.js"; import { _getFetchImplementation } from "./singletons/fetch.js"; import { stringify as stringifyForTracing } from "./utils/fast-safe-stringify/index.js"; async function mergeRuntimeEnvIntoRunCreates(runs) { const runtimeEnv = await getRuntimeEnvironment(); const envVars = getLangChainEnvVarsMetadata(); return runs.map((run) => { const extra = run.extra ?? {}; const metadata = extra.metadata; run.extra = { ...extra, runtime: { ...runtimeEnv, ...extra?.runtime, }, metadata: { ...envVars, ...(envVars.revision_id || run.revision_id ? { revision_id: run.revision_id ?? envVars.revision_id } : {}), ...metadata, }, }; return run; }); } const getTracingSamplingRate = () => { const samplingRateStr = getLangSmithEnvironmentVariable("TRACING_SAMPLING_RATE"); if (samplingRateStr === undefined) { return undefined; } const samplingRate = parseFloat(samplingRateStr); if (samplingRate < 0 || samplingRate > 1) { throw new Error(`LANGSMITH_TRACING_SAMPLING_RATE must be between 0 and 1 if set. Got: ${samplingRate}`); } return samplingRate; }; // utility functions const isLocalhost = (url) => { const strippedUrl = url.replace("http://", "").replace("https://", ""); const hostname = strippedUrl.split("/")[0].split(":")[0]; return (hostname === "localhost" || hostname === "127.0.0.1" || hostname === "::1"); }; async function toArray(iterable) { const result = []; for await (const item of iterable) { result.push(item); } return result; } function trimQuotes(str) { if (str === undefined) { return undefined; } return str .trim() .replace(/^"(.*)"$/, "$1") .replace(/^'(.*)'$/, "$1"); } const handle429 = async (response) => { if (response?.status === 429) { const retryAfter = parseInt(response.headers.get("retry-after") ?? "30", 10) * 1000; if (retryAfter > 0) { await new Promise((resolve) => setTimeout(resolve, retryAfter)); // Return directly after calling this check return true; } } // Fall back to existing status checks return false; }; export class Queue { constructor() { Object.defineProperty(this, "items", { enumerable: true, configurable: true, writable: true, value: [] }); } get size() { return this.items.length; } push(item) { // this.items.push is synchronous with promise creation: // https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Promise/Promise return new Promise((resolve) => { this.items.push([item, resolve]); }); } pop(upToN) { if (upToN < 1) { throw new Error("Number of items to pop off may not be less than 1."); } const popped = []; while (popped.length < upToN && this.items.length) { const item = this.items.shift(); if (item) { popped.push(item); } else { break; } } return [popped.map((it) => it[0]), () => popped.forEach((it) => it[1]())]; } } // 20 MB export const DEFAULT_BATCH_SIZE_LIMIT_BYTES = 20_971_520; export class Client { constructor(config = {}) { Object.defineProperty(this, "apiKey", { enumerable: true, configurable: true, writable: true, value: void 0 }); Object.defineProperty(this, "apiUrl", { enumerable: true, configurable: true, writable: true, value: void 0 }); Object.defineProperty(this, "webUrl", { enumerable: true, configurable: true, writable: true, value: void 0 }); Object.defineProperty(this, "caller", { enumerable: true, configurable: true, writable: true, value: void 0 }); Object.defineProperty(this, "batchIngestCaller", { enumerable: true, configurable: true, writable: true, value: void 0 }); Object.defineProperty(this, "timeout_ms", { enumerable: true, configurable: true, writable: true, value: void 0 }); Object.defineProperty(this, "_tenantId", { enumerable: true, configurable: true, writable: true, value: null }); Object.defineProperty(this, "hideInputs", { enumerable: true, configurable: true, writable: true, value: void 0 }); Object.defineProperty(this, "hideOutputs", { enumerable: true, configurable: true, writable: true, value: void 0 }); Object.defineProperty(this, "tracingSampleRate", { enumerable: true, configurable: true, writable: true, value: void 0 }); Object.defineProperty(this, "filteredPostUuids", { enumerable: true, configurable: true, writable: true, value: new Set() }); Object.defineProperty(this, "autoBatchTracing", { enumerable: true, configurable: true, writable: true, value: true }); Object.defineProperty(this, "batchEndpointSupported", { enumerable: true, configurable: true, writable: true, value: void 0 }); Object.defineProperty(this, "autoBatchQueue", { enumerable: true, configurable: true, writable: true, value: new Queue() }); Object.defineProperty(this, "pendingAutoBatchedRunLimit", { enumerable: true, configurable: true, writable: true, value: 100 }); Object.defineProperty(this, "autoBatchTimeout", { enumerable: true, configurable: true, writable: true, value: void 0 }); Object.defineProperty(this, "autoBatchInitialDelayMs", { enumerable: true, configurable: true, writable: true, value: 250 }); Object.defineProperty(this, "autoBatchAggregationDelayMs", { enumerable: true, configurable: true, writable: true, value: 50 }); Object.defineProperty(this, "serverInfo", { enumerable: true, configurable: true, writable: true, value: void 0 }); Object.defineProperty(this, "fetchOptions", { enumerable: true, configurable: true, writable: true, value: void 0 }); Object.defineProperty(this, "settings", { enumerable: true, configurable: true, writable: true, value: void 0 }); const defaultConfig = Client.getDefaultClientConfig(); this.tracingSampleRate = getTracingSamplingRate(); this.apiUrl = trimQuotes(config.apiUrl ?? defaultConfig.apiUrl) ?? ""; if (this.apiUrl.endsWith("/")) { this.apiUrl = this.apiUrl.slice(0, -1); } this.apiKey = trimQuotes(config.apiKey ?? defaultConfig.apiKey); this.webUrl = trimQuotes(config.webUrl ?? defaultConfig.webUrl); if (this.webUrl?.endsWith("/")) { this.webUrl = this.webUrl.slice(0, -1); } this.timeout_ms = config.timeout_ms ?? 12_000; this.caller = new AsyncCaller(config.callerOptions ?? {}); this.batchIngestCaller = new AsyncCaller({ ...(config.callerOptions ?? {}), onFailedResponseHook: handle429, }); this.hideInputs = config.hideInputs ?? config.anonymizer ?? defaultConfig.hideInputs; this.hideOutputs = config.hideOutputs ?? config.anonymizer ?? defaultConfig.hideOutputs; this.autoBatchTracing = config.autoBatchTracing ?? this.autoBatchTracing; this.pendingAutoBatchedRunLimit = config.pendingAutoBatchedRunLimit ?? this.pendingAutoBatchedRunLimit; this.fetchOptions = config.fetchOptions || {}; } static getDefaultClientConfig() { const apiKey = getLangSmithEnvironmentVariable("API_KEY"); const apiUrl = getLangSmithEnvironmentVariable("ENDPOINT") ?? "https://api.smith.langchain.com"; const hideInputs = getLangSmithEnvironmentVariable("HIDE_INPUTS") === "true"; const hideOutputs = getLangSmithEnvironmentVariable("HIDE_OUTPUTS") === "true"; return { apiUrl: apiUrl, apiKey: apiKey, webUrl: undefined, hideInputs: hideInputs, hideOutputs: hideOutputs, }; } getHostUrl() { if (this.webUrl) { return this.webUrl; } else if (isLocalhost(this.apiUrl)) { this.webUrl = "http://localhost:3000"; return this.webUrl; } else if (this.apiUrl.includes("/api") && !this.apiUrl.split(".", 1)[0].endsWith("api")) { this.webUrl = this.apiUrl.replace("/api", ""); return this.webUrl; } else if (this.apiUrl.split(".", 1)[0].includes("dev")) { this.webUrl = "https://dev.smith.langchain.com"; return this.webUrl; } else if (this.apiUrl.split(".", 1)[0].includes("eu")) { this.webUrl = "https://eu.smith.langchain.com"; return this.webUrl; } else { this.webUrl = "https://smith.langchain.com"; return this.webUrl; } } get headers() { const headers = { "User-Agent": `langsmith-js/${__version__}`, }; if (this.apiKey) { headers["x-api-key"] = `${this.apiKey}`; } return headers; } processInputs(inputs) { if (this.hideInputs === false) { return inputs; } if (this.hideInputs === true) { return {}; } if (typeof this.hideInputs === "function") { return this.hideInputs(inputs); } return inputs; } processOutputs(outputs) { if (this.hideOutputs === false) { return outputs; } if (this.hideOutputs === true) { return {}; } if (typeof this.hideOutputs === "function") { return this.hideOutputs(outputs); } return outputs; } prepareRunCreateOrUpdateInputs(run) { const runParams = { ...run }; if (runParams.inputs !== undefined) { runParams.inputs = this.processInputs(runParams.inputs); } if (runParams.outputs !== undefined) { runParams.outputs = this.processOutputs(runParams.outputs); } return runParams; } async _getResponse(path, queryParams) { const paramsString = queryParams?.toString() ?? ""; const url = `${this.apiUrl}${path}?${paramsString}`; const response = await this.caller.call(_getFetchImplementation(), url, { method: "GET", headers: this.headers, signal: AbortSignal.timeout(this.timeout_ms), ...this.fetchOptions, }); await raiseForStatus(response, `Failed to fetch ${path}`); return response; } async _get(path, queryParams) { const response = await this._getResponse(path, queryParams); return response.json(); } async *_getPaginated(path, queryParams = new URLSearchParams(), transform) { let offset = Number(queryParams.get("offset")) || 0; const limit = Number(queryParams.get("limit")) || 100; while (true) { queryParams.set("offset", String(offset)); queryParams.set("limit", String(limit)); const url = `${this.apiUrl}${path}?${queryParams}`; const response = await this.caller.call(_getFetchImplementation(), url, { method: "GET", headers: this.headers, signal: AbortSignal.timeout(this.timeout_ms), ...this.fetchOptions, }); await raiseForStatus(response, `Failed to fetch ${path}`); const items = transform ? transform(await response.json()) : await response.json(); if (items.length === 0) { break; } yield items; if (items.length < limit) { break; } offset += items.length; } } async *_getCursorPaginatedList(path, body = null, requestMethod = "POST", dataKey = "runs") { const bodyParams = body ? { ...body } : {}; while (true) { const response = await this.caller.call(_getFetchImplementation(), `${this.apiUrl}${path}`, { method: requestMethod, headers: { ...this.headers, "Content-Type": "application/json" }, signal: AbortSignal.timeout(this.timeout_ms), ...this.fetchOptions, body: JSON.stringify(bodyParams), }); const responseBody = await response.json(); if (!responseBody) { break; } if (!responseBody[dataKey]) { break; } yield responseBody[dataKey]; const cursors = responseBody.cursors; if (!cursors) { break; } if (!cursors.next) { break; } bodyParams.cursor = cursors.next; } } _filterForSampling(runs, patch = false) { if (this.tracingSampleRate === undefined) { return runs; } if (patch) { const sampled = []; for (const run of runs) { if (!this.filteredPostUuids.has(run.id)) { sampled.push(run); } else { this.filteredPostUuids.delete(run.id); } } return sampled; } else { const sampled = []; for (const run of runs) { if ((run.id !== run.trace_id && !this.filteredPostUuids.has(run.trace_id)) || Math.random() < this.tracingSampleRate) { sampled.push(run); } else { this.filteredPostUuids.add(run.id); } } return sampled; } } async drainAutoBatchQueue() { while (this.autoBatchQueue.size >= 0) { const [batch, done] = this.autoBatchQueue.pop(this.pendingAutoBatchedRunLimit); if (!batch.length) { done(); return; } try { await this.batchIngestRuns({ runCreates: batch .filter((item) => item.action === "create") .map((item) => item.item), runUpdates: batch .filter((item) => item.action === "update") .map((item) => item.item), }); } finally { done(); } } } async processRunOperation(item, immediatelyTriggerBatch) { const oldTimeout = this.autoBatchTimeout; clearTimeout(this.autoBatchTimeout); this.autoBatchTimeout = undefined; const itemPromise = this.autoBatchQueue.push(item); if (immediatelyTriggerBatch || this.autoBatchQueue.size > this.pendingAutoBatchedRunLimit) { await this.drainAutoBatchQueue().catch(console.error); } if (this.autoBatchQueue.size > 0) { this.autoBatchTimeout = setTimeout(() => { this.autoBatchTimeout = undefined; // This error would happen in the background and is uncatchable // from the outside. So just log instead. void this.drainAutoBatchQueue().catch(console.error); }, oldTimeout ? this.autoBatchAggregationDelayMs : this.autoBatchInitialDelayMs); } return itemPromise; } async _getServerInfo() { const response = await _getFetchImplementation()(`${this.apiUrl}/info`, { method: "GET", headers: { Accept: "application/json" }, signal: AbortSignal.timeout(this.timeout_ms), ...this.fetchOptions, }); await raiseForStatus(response, "get server info"); return response.json(); } async batchEndpointIsSupported() { try { this.serverInfo = await this._getServerInfo(); } catch (e) { return false; } return true; } async _getSettings() { if (!this.settings) { this.settings = this._get("/settings"); } return await this.settings; } async createRun(run) { if (!this._filterForSampling([run]).length) { return; } const headers = { ...this.headers, "Content-Type": "application/json" }; const session_name = run.project_name; delete run.project_name; const runCreate = this.prepareRunCreateOrUpdateInputs({ session_name, ...run, start_time: run.start_time ?? Date.now(), }); if (this.autoBatchTracing && runCreate.trace_id !== undefined && runCreate.dotted_order !== undefined) { void this.processRunOperation({ action: "create", item: runCreate, }).catch(console.error); return; } const mergedRunCreateParams = await mergeRuntimeEnvIntoRunCreates([ runCreate, ]); const response = await this.caller.call(_getFetchImplementation(), `${this.apiUrl}/runs`, { method: "POST", headers, body: stringifyForTracing(mergedRunCreateParams[0]), signal: AbortSignal.timeout(this.timeout_ms), ...this.fetchOptions, }); await raiseForStatus(response, "create run", true); } /** * Batch ingest/upsert multiple runs in the Langsmith system. * @param runs */ async batchIngestRuns({ runCreates, runUpdates, }) { if (runCreates === undefined && runUpdates === undefined) { return; } let preparedCreateParams = runCreates?.map((create) => this.prepareRunCreateOrUpdateInputs(create)) ?? []; let preparedUpdateParams = runUpdates?.map((update) => this.prepareRunCreateOrUpdateInputs(update)) ?? []; if (preparedCreateParams.length > 0 && preparedUpdateParams.length > 0) { const createById = preparedCreateParams.reduce((params, run) => { if (!run.id) { return params; } params[run.id] = run; return params; }, {}); const standaloneUpdates = []; for (const updateParam of preparedUpdateParams) { if (updateParam.id !== undefined && createById[updateParam.id]) { createById[updateParam.id] = { ...createById[updateParam.id], ...updateParam, }; } else { standaloneUpdates.push(updateParam); } } preparedCreateParams = Object.values(createById); preparedUpdateParams = standaloneUpdates; } const rawBatch = { post: this._filterForSampling(preparedCreateParams), patch: this._filterForSampling(preparedUpdateParams, true), }; if (!rawBatch.post.length && !rawBatch.patch.length) { return; } preparedCreateParams = await mergeRuntimeEnvIntoRunCreates(preparedCreateParams); if (this.batchEndpointSupported === undefined) { this.batchEndpointSupported = await this.batchEndpointIsSupported(); } if (!this.batchEndpointSupported) { this.autoBatchTracing = false; for (const preparedCreateParam of rawBatch.post) { await this.createRun(preparedCreateParam); } for (const preparedUpdateParam of rawBatch.patch) { if (preparedUpdateParam.id !== undefined) { await this.updateRun(preparedUpdateParam.id, preparedUpdateParam); } } return; } const sizeLimitBytes = this.serverInfo?.batch_ingest_config?.size_limit_bytes ?? DEFAULT_BATCH_SIZE_LIMIT_BYTES; const batchChunks = { post: [], patch: [], }; let currentBatchSizeBytes = 0; for (const k of ["post", "patch"]) { const key = k; const batchItems = rawBatch[key].reverse(); let batchItem = batchItems.pop(); while (batchItem !== undefined) { const stringifiedBatchItem = stringifyForTracing(batchItem); if (currentBatchSizeBytes > 0 && currentBatchSizeBytes + stringifiedBatchItem.length > sizeLimitBytes) { await this._postBatchIngestRuns(stringifyForTracing(batchChunks)); currentBatchSizeBytes = 0; batchChunks.post = []; batchChunks.patch = []; } currentBatchSizeBytes += stringifiedBatchItem.length; batchChunks[key].push(batchItem); batchItem = batchItems.pop(); } } if (batchChunks.post.length > 0 || batchChunks.patch.length > 0) { await this._postBatchIngestRuns(stringifyForTracing(batchChunks)); } } async _postBatchIngestRuns(body) { const headers = { ...this.headers, "Content-Type": "application/json", Accept: "application/json", }; const response = await this.batchIngestCaller.call(_getFetchImplementation(), `${this.apiUrl}/runs/batch`, { method: "POST", headers, body: body, signal: AbortSignal.timeout(this.timeout_ms), ...this.fetchOptions, }); await raiseForStatus(response, "batch create run", true); } async updateRun(runId, run) { assertUuid(runId); if (run.inputs) { run.inputs = this.processInputs(run.inputs); } if (run.outputs) { run.outputs = this.processOutputs(run.outputs); } // TODO: Untangle types const data = { ...run, id: runId }; if (!this._filterForSampling([data], true).length) { return; } if (this.autoBatchTracing && data.trace_id !== undefined && data.dotted_order !== undefined) { if (run.end_time !== undefined && data.parent_run_id === undefined) { // Trigger a batch as soon as a root trace ends and block to ensure trace finishes // in serverless environments. await this.processRunOperation({ action: "update", item: data }, true); return; } else { void this.processRunOperation({ action: "update", item: data }).catch(console.error); } return; } const headers = { ...this.headers, "Content-Type": "application/json" }; const response = await this.caller.call(_getFetchImplementation(), `${this.apiUrl}/runs/${runId}`, { method: "PATCH", headers, body: stringifyForTracing(run), signal: AbortSignal.timeout(this.timeout_ms), ...this.fetchOptions, }); await raiseForStatus(response, "update run", true); } async readRun(runId, { loadChildRuns } = { loadChildRuns: false }) { assertUuid(runId); let run = await this._get(`/runs/${runId}`); if (loadChildRuns && run.child_run_ids) { run = await this._loadChildRuns(run); } return run; } async getRunUrl({ runId, run, projectOpts, }) { if (run !== undefined) { let sessionId; if (run.session_id) { sessionId = run.session_id; } else if (projectOpts?.projectName) { sessionId = (await this.readProject({ projectName: projectOpts?.projectName })).id; } else if (projectOpts?.projectId) { sessionId = projectOpts?.projectId; } else { const project = await this.readProject({ projectName: getLangSmithEnvironmentVariable("PROJECT") || "default", }); sessionId = project.id; } const tenantId = await this._getTenantId(); return `${this.getHostUrl()}/o/${tenantId}/projects/p/${sessionId}/r/${run.id}?poll=true`; } else if (runId !== undefined) { const run_ = await this.readRun(runId); if (!run_.app_path) { throw new Error(`Run ${runId} has no app_path`); } const baseUrl = this.getHostUrl(); return `${baseUrl}${run_.app_path}`; } else { throw new Error("Must provide either runId or run"); } } async _loadChildRuns(run) { const childRuns = await toArray(this.listRuns({ id: run.child_run_ids })); const treemap = {}; const runs = {}; // TODO: make dotted order required when the migration finishes childRuns.sort((a, b) => (a?.dotted_order ?? "").localeCompare(b?.dotted_order ?? "")); for (const childRun of childRuns) { if (childRun.parent_run_id === null || childRun.parent_run_id === undefined) { throw new Error(`Child run ${childRun.id} has no parent`); } if (!(childRun.parent_run_id in treemap)) { treemap[childRun.parent_run_id] = []; } treemap[childRun.parent_run_id].push(childRun); runs[childRun.id] = childRun; } run.child_runs = treemap[run.id] || []; for (const runId in treemap) { if (runId !== run.id) { runs[runId].child_runs = treemap[runId]; } } return run; } /** * List runs from the LangSmith server. * @param projectId - The ID of the project to filter by. * @param projectName - The name of the project to filter by. * @param parentRunId - The ID of the parent run to filter by. * @param traceId - The ID of the trace to filter by. * @param referenceExampleId - The ID of the reference example to filter by. * @param startTime - The start time to filter by. * @param isRoot - Indicates whether to only return root runs. * @param runType - The run type to filter by. * @param error - Indicates whether to filter by error runs. * @param id - The ID of the run to filter by. * @param query - The query string to filter by. * @param filter - The filter string to apply to the run spans. * @param traceFilter - The filter string to apply on the root run of the trace. * @param limit - The maximum number of runs to retrieve. * @returns {AsyncIterable} - The runs. * * @example * // List all runs in a project * const projectRuns = client.listRuns({ projectName: "" }); * * @example * // List LLM and Chat runs in the last 24 hours * const todaysLLMRuns = client.listRuns({ * projectName: "", * start_time: new Date(Date.now() - 24 * 60 * 60 * 1000), * run_type: "llm", * }); * * @example * // List traces in a project * const rootRuns = client.listRuns({ * projectName: "", * execution_order: 1, * }); * * @example * // List runs without errors * const correctRuns = client.listRuns({ * projectName: "", * error: false, * }); * * @example * // List runs by run ID * const runIds = [ * "a36092d2-4ad5-4fb4-9c0d-0dba9a2ed836", * "9398e6be-964f-4aa4-8ae9-ad78cd4b7074", * ]; * const selectedRuns = client.listRuns({ run_ids: runIds }); * * @example * // List all "chain" type runs that took more than 10 seconds and had `total_tokens` greater than 5000 * const chainRuns = client.listRuns({ * projectName: "", * filter: 'and(eq(run_type, "chain"), gt(latency, 10), gt(total_tokens, 5000))', * }); * * @example * // List all runs called "extractor" whose root of the trace was assigned feedback "user_score" score of 1 * const goodExtractorRuns = client.listRuns({ * projectName: "", * filter: 'eq(name, "extractor")', * traceFilter: 'and(eq(feedback_key, "user_score"), eq(feedback_score, 1))', * }); * * @example * // List all runs that started after a specific timestamp and either have "error" not equal to null or a "Correctness" feedback score equal to 0 * const complexRuns = client.listRuns({ * projectName: "", * filter: 'and(gt(start_time, "2023-07-15T12:34:56Z"), or(neq(error, null), and(eq(feedback_key, "Correctness"), eq(feedback_score, 0.0))))', * }); * * @example * // List all runs where `tags` include "experimental" or "beta" and `latency` is greater than 2 seconds * const taggedRuns = client.listRuns({ * projectName: "", * filter: 'and(or(has(tags, "experimental"), has(tags, "beta")), gt(latency, 2))', * }); */ async *listRuns(props) { const { projectId, projectName, parentRunId, traceId, referenceExampleId, startTime, executionOrder, isRoot, runType, error, id, query, filter, traceFilter, treeFilter, limit, select, } = props; let projectIds = []; if (projectId) { projectIds = Array.isArray(projectId) ? projectId : [projectId]; } if (projectName) { const projectNames = Array.isArray(projectName) ? projectName : [projectName]; const projectIds_ = await Promise.all(projectNames.map((name) => this.readProject({ projectName: name }).then((project) => project.id))); projectIds.push(...projectIds_); } const default_select = [ "app_path", "child_run_ids", "completion_cost", "completion_tokens", "dotted_order", "end_time", "error", "events", "extra", "feedback_stats", "first_token_time", "id", "inputs", "name", "outputs", "parent_run_id", "parent_run_ids", "prompt_cost", "prompt_tokens", "reference_example_id", "run_type", "session_id", "start_time", "status", "tags", "total_cost", "total_tokens", "trace_id", ]; const body = { session: projectIds.length ? projectIds : null, run_type: runType, reference_example: referenceExampleId, query, filter, trace_filter: traceFilter, tree_filter: treeFilter, execution_order: executionOrder, parent_run: parentRunId, start_time: startTime ? startTime.toISOString() : null, error, id, limit, trace: traceId, select: select ? select : default_select, is_root: isRoot, }; let runsYielded = 0; for await (const runs of this._getCursorPaginatedList("/runs/query", body)) { if (limit) { if (runsYielded >= limit) { break; } if (runs.length + runsYielded > limit) { const newRuns = runs.slice(0, limit - runsYielded); yield* newRuns; break; } runsYielded += runs.length; yield* runs; } else { yield* runs; } } } async getRunStats({ id, trace, parentRun, runType, projectNames, projectIds, referenceExampleIds, startTime, endTime, error, query, filter, traceFilter, treeFilter, isRoot, dataSourceType, }) { let projectIds_ = projectIds || []; if (projectNames) { projectIds_ = [ ...(projectIds || []), ...(await Promise.all(projectNames.map((name) => this.readProject({ projectName: name }).then((project) => project.id)))), ]; } const payload = { id, trace, parent_run: parentRun, run_type: runType, session: projectIds_, reference_example: referenceExampleIds, start_time: startTime, end_time: endTime, error, query, filter, trace_filter: traceFilter, tree_filter: treeFilter, is_root: isRoot, data_source_type: dataSourceType, }; // Remove undefined values from the payload const filteredPayload = Object.fromEntries(Object.entries(payload).filter(([_, value]) => value !== undefined)); const response = await this.caller.call(_getFetchImplementation(), `${this.apiUrl}/runs/stats`, { method: "POST", headers: this.headers, body: JSON.stringify(filteredPayload), signal: AbortSignal.timeout(this.timeout_ms), ...this.fetchOptions, }); const result = await response.json(); return result; } async shareRun(runId, { shareId } = {}) { const data = { run_id: runId, share_token: shareId || uuid.v4(), }; assertUuid(runId); const response = await this.caller.call(_getFetchImplementation(), `${this.apiUrl}/runs/${runId}/share`, { method: "PUT", headers: this.headers, body: JSON.stringify(data), signal: AbortSignal.timeout(this.timeout_ms), ...this.fetchOptions, }); const result = await response.json(); if (result === null || !("share_token" in result)) { throw new Error("Invalid response from server"); } return `${this.getHostUrl()}/public/${result["share_token"]}/r`; } async unshareRun(runId) { assertUuid(runId); const response = await this.caller.call(_getFetchImplementation(), `${this.apiUrl}/runs/${runId}/share`, { method: "DELETE", headers: this.headers, signal: AbortSignal.timeout(this.timeout_ms), ...this.fetchOptions, }); await raiseForStatus(response, "unshare run", true); } async readRunSharedLink(runId) { assertUuid(runId); const response = await this.caller.call(_getFetchImplementation(), `${this.apiUrl}/runs/${runId}/share`, { method: "GET", headers: this.headers, signal: AbortSignal.timeout(this.timeout_ms), ...this.fetchOptions, }); const result = await response.json(); if (result === null || !("share_token" in result)) { return undefined; } return `${this.getHostUrl()}/public/${result["share_token"]}/r`; } async listSharedRuns(shareToken, { runIds, } = {}) { const queryParams = new URLSearchParams({ share_token: shareToken, }); if (runIds !== undefined) { for (const runId of runIds) { queryParams.append("id", runId); } } assertUuid(shareToken); const response = await this.caller.call(_getFetchImplementation(), `${this.apiUrl}/public/${shareToken}/runs${queryParams}`, { method: "GET", headers: this.headers, signal: AbortSignal.timeout(this.timeout_ms), ...this.fetchOptions, }); const runs = await response.json(); return runs; } async readDatasetSharedSchema(datasetId, datasetName) { if (!datasetId && !datasetName) { throw new Error("Either datasetId or datasetName must be given"); } if (!datasetId) { const dataset = await this.readDataset({ datasetName }); datasetId = dataset.id; } assertUuid(datasetId); const response = await this.caller.call(_getFetchImplementation(), `${this.apiUrl}/datasets/${datasetId}/share`, { method: "GET", headers: this.headers, signal: AbortSignal.timeout(this.timeout_ms), ...this.fetchOptions, }); const shareSchema = await response.json(); shareSchema.url = `${this.getHostUrl()}/public/${shareSchema.share_token}/d`; return shareSchema; } async shareDataset(datasetId, datasetName) { if (!datasetId && !datasetName) { throw new Error("Either datasetId or datasetName must be given"); } if (!datasetId) { const dataset = await this.readDataset({ datasetName }); datasetId = dataset.id; } const data = { dataset_id: datasetId, }; assertUuid(datasetId); const response = await this.caller.call(_getFetchImplementation(), `${this.apiUrl}/datasets/${datasetId}/share`, { method: "PUT", headers: this.headers, body: JSON.stringify(data), signal: AbortSignal.timeout(this.timeout_ms), ...this.fetchOptions, }); const shareSchema = await response.json(); shareSchema.url = `${this.getHostUrl()}/public/${shareSchema.share_token}/d`; return shareSchema; } async unshareDataset(datasetId) { assertUuid(datasetId); const response = await this.caller.call(_getFetchImplementation(), `${this.apiUrl}/datasets/${datasetId}/share`, { method: "DELETE", headers: this.headers, signal: AbortSignal.timeout(this.timeout_ms), ...this.fetchOptions, }); await raiseForStatus(response, "unshare dataset", true); } async readSharedDataset(shareToken) { assertUuid(shareToken); const response = await this.caller.call(_getFetchImplementation(), `${this.apiUrl}/public/${shareToken}/datasets`, { method: "GET", headers: this.headers, signal: AbortSignal.timeout(this.timeout_ms), ...this.fetchOptions, }); const dataset = await response.json(); return dataset; } /** * Get shared examples. * * @param {string} shareToken The share token to get examples for. A share token is the UUID (or LangSmith URL, including UUID) generated when explicitly marking an example as public. * @param {Object} [options] Additional options for listing the examples. * @param {string[] | undefined} [options.exampleIds] A list of example IDs to filter by. * @returns {Promise} The shared examples. */ async listSharedExamples(shareToken, options) { const params = {}; if (options?.exampleIds) { params.id = options.exampleIds; } const urlParams = new URLSearchParams(); Object.entries(params).forEach(([key, value]) => { if (Array.isArray(value)) { value.forEach((v) => urlParams.append(key, v)); } else { urlParams.append(key, value); } }); const response = await this.caller.call(_getFetchImplementation(), `${this.apiUrl}/public/${shareToken}/examples?${urlParams.toString()}`, { method: "GET", headers: this.headers, signal: AbortSignal.timeout(this.timeout_ms), ...this.fetchOptions, }); const result = await response.json(); if (!response.ok) { if ("detail" in result) { throw new Error(`Failed to list shared examples.\nStatus: ${response.status}\nMessage: ${result.detail.join("\n")}`); } throw new Error(`Failed to list shared examples: ${response.status} ${response.statusText}`); } return result.map((example) => ({ ...example, _hostUrl: this.getHostUrl(), })); } async createProject({ projectName, description = null, metadata = null, upsert = false, projectExtra = null, referenceDatasetId = null, }) { const upsert_ = upsert ? `?upsert=true` : ""; const endpoint = `${this.apiUrl}/sessions${upsert_}`; const extra = projectExtra || {}; if (metadata) { extra["metadata"] = metadata; } const body = { name: projectName, extra, description, }; if (referenceDatasetId !== null) { body["reference_dataset_id"] = referenceDatasetId; } const response = await this.caller.call(_getFetchImplementation(), endpoint, { method: "POST", headers: { ...this.headers, "Content-Type": "application/json" }, body: JSON.stringify(body), signal: AbortSignal.timeout(this.timeout_ms), ...this.fetchOptions, }); await raiseForStatus(response, "create project"); const result = await response.json(); return result; } async updateProject(projectId, { name = null, description = null, metadata = null, projectExtra = null, endTime = null, }) { const endpoint = `${this.apiUrl}/sessions/${projectId}`; let extra = projectExtra; if (metadata) { extra = { ...(extra || {}), metadata }; } const body = { name, extra, description, end_time: endTime ? new Date(endTime).toISOString() : null, }; const response = await this.caller.call(_getFetchImplementation(), endpoint, { method: "PATCH", headers: { ...this.headers, "Content-Type": "application/json" }, body: JSON.stringify(body), signal: AbortSignal.timeout(this.timeout_ms), ...this.fetchOptions, }); await raiseForStatus(response, "update project"); const result = await response.json(); return result; } async hasProject({ projectId, projectName, }) { // TODO: Add a head request let path = "/sessions"; const params = new URLSearchParams(); if (projectId !== undefined && projectName !== undefined) { throw new Error("Must provide either projectName or projectId, not both"); } else if (projectId !== undefined) { assertUuid(projectId); path += `/${projectId}`; } else if (projectName !== undefined) { params.append("name", projectName); } else { throw new Error("Must provide projectName or projectId"); } const response = await this.caller.call(_getFetchImplementation(), `${this.apiUrl}${path}?${params}`, { method: "GET", headers: this.headers, signal: AbortSignal.timeout(this.timeout_ms), ...this.fetchOptions, }); // consume the response body to release the connection // https://undici.nodejs.org/#/?id=garbage-collection try { const result = await response.json(); if (!response.ok) { return false; } // If it's OK and we're querying by name, need to check the list is not empty if (Array.isArray(result)) { return result.length > 0; } // projectId querying return true; } catch (e) { return false; } } async readProject({ projectId, projectName, includeStats, }) { let path = "/sessions"; const params = new URLSearchParams(); if (projectId !== undefined && projectName !== undefined) { throw new Error("Must provide either projectName or projectId, not both"); } else if (projectId !== undefined) { assertUuid(projectId); path += `/${projectId}`; } else if (projectName !== undefined) { params.append("name", projectName); } else { throw new Error("Must provide projectName or projectId"); } if (includeStats !== undefined) { params.append("include_stats", includeStats.toString()); } const response = await this._get(path, params); let result; if (Array.isArray(response)) { if (response.length === 0) { throw new Error(`Project[id=${projectId}, name=${projectName}] not found`); } result = response[0]; } else { result = response; } return result; } async getProjectUrl({ projectId, projectName, }) { if (projectId === undefined && projectName === undefined) { throw new Error("Must provide either projectName or projectId"); } const project = await this.readProject({ projectId, projectName }); const tenantId = await this._getTenantId(); return `${this.getHostUrl()}/o/${tenantId}/projects/p/${project.id}`; } async getDatasetUrl({ datasetId, datasetName, }) { if (datasetId === undefined && datasetName === undefined) { throw new Error("Must provide either datasetName or datasetId"); } const dataset = await this.readDataset({ datasetId, datasetName }); const tenantId = await this._getTenantId(); return `${this.getHostUrl()}/o/${tenantId}/datasets/${dataset.id}`; } async _getTenantId() { if (this._tenantId !== null) { return this._tenantId; } const queryParams = new URLSearchParams({ limit: "1" }); for await (const projects of this._getPaginated("/sessions", queryParams)) { this._tenantId = projects[0].tenant_id; return projects[0].tenant_id; } throw new Error("No projects found to resolve tenant."); } async *listProjects({ projectIds, name, nameContains, referenceDatasetId, referenceDatasetName, referenceFree, metadata, } = {}) { const params = new URLSearchParams(); if (projectIds !== undefined) { for (const projectId of projectIds) { params.append("id", projectId); } } if (name !== undefined) { params.append("name", name); } if (nameContains !== undefined) { params.append("name_contains", nameContains); } if (referenceDatasetId !== undefined) { params.append("reference_dataset", referenceDatasetId); } else if (referenceDatasetName !== undefined) { const dataset = await this.readDataset({ datasetName: referenceDatasetName, }); params.append("reference_dataset", dataset.id); } if (referenceFree !== undefined) { params.append("reference_free", referenceFree.toString()); } if (metadata !== undefined) { params.append("metadata", JSON.stringify(metadata)); } for await (const projects of this._getPaginated("/sessions", params)) { yield* projects; } } async deleteProject({ projectId, projectName, }) { let projectId_; if (projectId === undefined && projectName === undefined) { throw new Error("Must provide projectName or projectId"); } else if (projectId !== undefined && projectName !== undefined) { throw new Error("Must provide either projectName or projectId, not both"); } else if (projectId === undefined) { projectId_ = (await this.readProject({ projectName })).id; } else { projectId_ = projectId; } assertUuid(projectId_); const response = await this.caller.call(_getFetchImplementation(), `${this.apiUrl}/sessions/${projectId_}`, { method: "DELETE", headers: this.headers, signal: AbortSignal.timeout(this.timeout_ms), ...this.fetchOptions, }); await raiseForStatus(response, `delete session ${projectId_} (${projectName})`, true); } async uploadCsv({ csvFile, fileName, inputKeys, outputKeys, description, dataType, name, }) { const url = `${this.apiUrl}/datasets/upload`; const formData = new FormData(); formData.append("file", csvFile, fileName); inputKeys.forEach((key) => { formData.append("input_keys", key); }); outputKeys.forEach((key) => { formData.append("output_keys", key); }); if (description) { formData.append("description", description); } if (dataType) { formData.append("data_type", dataType); } if (name) { formData.append("name", name); } const response = await this.caller.call(_getFetchImplementation(), url, { method: "POST", headers: this.headers, body: formData, signal: AbortSignal.timeout(this.timeout_ms), ...this.fetchOptions, }); await raiseForStatus(response, "upload CSV"); const result = await response.json(); return result; } async createDataset(name, { description, dataType, inputsSchema, outputsSchema, metadata, } = {}) { const body = { name, description, extra: metadata ? { metadata } : undefined, }; if (dataType) { body.data_type = dataType; } if (inputsSchema) { body.inputs_schema_definition = inputsSchema; } if (outputsSchema) { body.outputs_schema_definition = outputsSchema; } const response = await this.caller.call(_getFetchImplementation(), `${this.apiUrl}/datasets`, { method: "POST", headers: { ...this.headers, "Content-Type": "application/json" }, body: JSON.stringify(body), signal: AbortSignal.timeout(this.timeout_ms), ...this.fetchOptions, }); await raiseForStatus(response, "create dataset"); const result = await response.json(); return result; } async readDataset({ datasetId, datasetName, }) { let path = "/datasets"; // limit to 1 result const params = new URLSearchParams({ limit: "1" }); if (datasetId !== undefined && datasetName !== undefined) { throw new Error("Must provide either datasetName or datasetId, not both"); } else if (datasetId !== undefined) { assertUuid(datasetId); path += `/${datasetId}`; } else if (datasetName !== undefined) { params.append("name", datasetName); } else { throw new Error("Must provide datasetName or datasetId"); } const response = await this._get(path, params); let result; if (Array.isArray(response)) { if (response.length === 0) { throw new Error(`Dataset[id=${datasetId}, name=${datasetName}] not found`); } result = response[0]; } else { result = response; } return result; } async hasDataset({ datasetId, datasetName, }) { try { await this.readDataset({ datasetId, datasetName }); return true; } catch (e) { if ( // eslint-disable-next-line no-instanceof/no-instanceof e instanceof Error && e.message.toLocaleLowerCase().includes("not found")) { return false; } throw e; } } async diffDatasetVersions({ datasetId, datasetName, fromVersion, toVersion, }) { let datasetId_ = datasetId; if (datasetId_ === undefined && datasetName === undefined) { throw new Error("Must provide either datasetName or datasetId"); } else if (datasetId_ !== undefined && datasetName !== undefined) { throw new Error("Must provide either datasetName or datasetId, not both"); } else if (datasetId_ === undefined) { const dataset = await this.readDataset({ datasetName }); datasetId_ = dataset.id; } const urlParams = new URLSearchParams({ from_version: typeof fromVersion === "string" ? fromVersion : fromVersion.toISOString(), to_version: typeof toVersion === "string" ? toVersion : toVersion.toISOString(), }); const response = await this._get(`/datasets/${datasetId_}/versions/diff`, urlParams); return response; } async readDatasetOpenaiFinetuning({ datasetId, datasetName, }) { const path = "/datasets"; if (datasetId !== undefined) { // do nothing } else if (datasetName !== undefined) { datasetId = (await this.readDataset({ datasetName })).id; } else { throw new Error("Must provide datasetName or datasetId"); } const response = await this._getResponse(`${path}/${datasetId}/openai_ft`); const datasetText = await response.text(); const dataset = datasetText .trim() .split("\n") .map((line) => JSON.parse(line)); return dataset; } async *listDatasets({ limit = 100, offset = 0, datasetIds, datasetName, datasetNameContains, metadata, } = {}) { const path = "/datasets"; const params = new URLSearchParams({ limit: limit.toString(), offset: offset.toString(), }); if (datasetIds !== undefined) { for (const id_ of datasetIds) { params.append("id", id_); } } if (datasetName !== undefined) { params.append("name", datasetName); } if (datasetNameContains !== undefined) { params.append("name_contains", datasetNameContains); } if (metadata !== undefined) { params.append("metadata", JSON.stringify(metadata)); } for await (const datasets of this._getPaginated(path, params)) { yield* datasets; } } /** * Update a dataset * @param props The dataset details to update * @returns The updated dataset */ async updateDataset(props) { const { datasetId, datasetName, ...update } = props; if (!datasetId && !datasetName) { throw new Error("Must provide either datasetName or datasetId"); } const _datasetId = datasetId ?? (await this.readDataset({ datasetName })).id; assertUuid(_datasetId); const response = await this.caller.call(_getFetchImplementation(), `${this.apiUrl}/datasets/${_datasetId}`, { method: "PATCH", headers: { ...this.headers, "Content-Type": "application/json" }, body: JSON.stringify(update), signal: AbortSignal.timeout(this.timeout_ms), ...this.fetchOptions, }); await raiseForStatus(response, "update dataset"); return (await response.json()); } async deleteDataset({ datasetId, datasetName, }) { let path = "/datasets"; let datasetId_ = datasetId; if (datasetId !== undefined && datasetName !== undefined) { throw new Error("Must provide either datasetName or datasetId, not both"); } else if (datasetName !== undefined) { const dataset = await this.readDataset({ datasetName }); datasetId_ = dataset.id; } if (datasetId_ !== undefined) { assertUuid(datasetId_); path += `/${datasetId_}`; } else { throw new Error("Must provide datasetName or datasetId"); } const response = await this.caller.call(_getFetchImplementation(), this.apiUrl + path, { method: "DELETE", headers: this.headers, signal: AbortSignal.timeout(this.timeout_ms), ...this.fetchOptions, }); await raiseForStatus(response, `delete ${path}`); await response.json(); } async indexDataset({ datasetId, datasetName, tag, }) { let datasetId_ = datasetId; if (!datasetId_ && !datasetName) { throw new Error("Must provide either datasetName or datasetId"); } else if (datasetId_ && datasetName) { throw new Error("Must provide either datasetName or datasetId, not both"); } else if (!datasetId_) { const dataset = await this.readDataset({ datasetName }); datasetId_ = dataset.id; } assertUuid(datasetId_); const data = { tag: tag, }; const response = await this.caller.call(_getFetchImplementation(), `${this.apiUrl}/datasets/${datasetId_}/index`, { method: "POST", headers: { ...this.headers, "Content-Type": "application/json" }, body: JSON.stringify(data), signal: AbortSignal.timeout(this.timeout_ms), ...this.fetchOptions, }); await raiseForStatus(response, "index dataset"); await response.json(); } /** * Lets you run a similarity search query on a dataset. * * Requires the dataset to be indexed. Please see the `indexDataset` method to set up indexing. * * @param inputs The input on which to run the similarity search. Must have the * same schema as the dataset. * * @param datasetId The dataset to search for similar examples. * * @param limit The maximum number of examples to return. Will return the top `limit` most * similar examples in order of most similar to least similar. If no similar * examples are found, random examples will be returned. * * @param filter A filter string to apply to the search. Only examples will be returned that * match the filter string. Some examples of filters * * - eq(metadata.mykey, "value") * - and(neq(metadata.my.nested.key, "value"), neq(metadata.mykey, "value")) * - or(eq(metadata.mykey, "value"), eq(metadata.mykey, "othervalue")) * * @returns A list of similar examples. * * * @example * dataset_id = "123e4567-e89b-12d3-a456-426614174000" * inputs = {"text": "How many people live in Berlin?"} * limit = 5 * examples = await client.similarExamples(inputs, dataset_id, limit) */ async similarExamples(inputs, datasetId, limit, { filter, } = {}) { const data = { limit: limit, inputs: inputs, }; if (filter !== undefined) { data["filter"] = filter; } assertUuid(datasetId); const response = await this.caller.call(_getFetchImplementation(), `${this.apiUrl}/datasets/${datasetId}/search`, { method: "POST", headers: { ...this.headers, "Content-Type": "application/json" }, body: JSON.stringify(data), signal: AbortSignal.timeout(this.timeout_ms), ...this.fetchOptions, }); await raiseForStatus(response, "fetch similar examples"); const result = await response.json(); return result["examples"]; } async createExample(inputs, outputs, { datasetId, datasetName, createdAt, exampleId, metadata, split, sourceRunId, }) { let datasetId_ = datasetId; if (datasetId_ === undefined && datasetName === undefined) { throw new Error("Must provide either datasetName or datasetId"); } else if (datasetId_ !== undefined && datasetName !== undefined) { throw new Error("Must provide either datasetName or datasetId, not both"); } else if (datasetId_ === undefined) { const dataset = await this.readDataset({ datasetName }); datasetId_ = dataset.id; } const createdAt_ = createdAt || new Date(); const data = { dataset_id: datasetId_, inputs, outputs, created_at: createdAt_?.toISOString(), id: exampleId, metadata, split, source_run_id: sourceRunId, }; const response = await this.caller.call(_getFetchImplementation(), `${this.apiUrl}/examples`, { method: "POST", headers: { ...this.headers, "Content-Type": "application/json" }, body: JSON.stringify(data), signal: AbortSignal.timeout(this.timeout_ms), ...this.fetchOptions, }); await raiseForStatus(response, "create example"); const result = await response.json(); return result; } async createExamples(props) { const { inputs, outputs, metadata, sourceRunIds, exampleIds, datasetId, datasetName, } = props; let datasetId_ = datasetId; if (datasetId_ === undefined && datasetName === undefined) { throw new Error("Must provide either datasetName or datasetId"); } else if (datasetId_ !== undefined && datasetName !== undefined) { throw new Error("Must provide either datasetName or datasetId, not both"); } else if (datasetId_ === undefined) { const dataset = await this.readDataset({ datasetName }); datasetId_ = dataset.id; } const formattedExamples = inputs.map((input, idx) => { return { dataset_id: datasetId_, inputs: input, outputs: outputs ? outputs[idx] : undefined, metadata: metadata ? metadata[idx] : undefined, split: props.splits ? props.splits[idx] : undefined, id: exampleIds ? exampleIds[idx] : undefined, source_run_id: sourceRunIds ? sourceRunIds[idx] : undefined, }; }); const response = await this.caller.call(_getFetchImplementation(), `${this.apiUrl}/examples/bulk`, { method: "POST", headers: { ...this.headers, "Content-Type": "application/json" }, body: JSON.stringify(formattedExamples), signal: AbortSignal.timeout(this.timeout_ms), ...this.fetchOptions, }); await raiseForStatus(response, "create examples"); const result = await response.json(); return result; } async createLLMExample(input, generation, options) { return this.createExample({ input }, { output: generation }, options); } async createChatExample(input, generations, options) { const finalInput = input.map((message) => { if (isLangChainMessage(message)) { return convertLangChainMessageToExample(message); } return message; }); const finalOutput = isLangChainMessage(generations) ? convertLangChainMessageToExample(generations) : generations; return this.createExample({ input: finalInput }, { output: finalOutput }, options); } async readExample(exampleId) { assertUuid(exampleId); const path = `/examples/${exampleId}`; return await this._get(path); } async *listExamples({ datasetId, datasetName, exampleIds, asOf, splits, inlineS3Urls, metadata, limit, offset, filter, } = {}) { let datasetId_; if (datasetId !== undefined && datasetName !== undefined) { throw new Error("Must provide either datasetName or datasetId, not both"); } else if (datasetId !== undefined) { datasetId_ = datasetId; } else if (datasetName !== undefined) { const dataset = await this.readDataset({ datasetName }); datasetId_ = dataset.id; } else { throw new Error("Must provide a datasetName or datasetId"); } const params = new URLSearchParams({ dataset: datasetId_ }); const dataset_version = asOf ? typeof asOf === "string" ? asOf : asOf?.toISOString() : undefined; if (dataset_version) { params.append("as_of", dataset_version); } const inlineS3Urls_ = inlineS3Urls ?? true; params.append("inline_s3_urls", inlineS3Urls_.toString()); if (exampleIds !== undefined) { for (const id_ of exampleIds) { params.append("id", id_); } } if (splits !== undefined) { for (const split of splits) { params.append("splits", split); } } if (metadata !== undefined) { const serializedMetadata = JSON.stringify(metadata); params.append("metadata", serializedMetadata); } if (limit !== undefined) { params.append("limit", limit.toString()); } if (offset !== undefined) { params.append("offset", offset.toString()); } if (filter !== undefined) { params.append("filter", filter); } let i = 0; for await (const examples of this._getPaginated("/examples", params)) { for (const example of examples) { yield example; i++; } if (limit !== undefined && i >= limit) { break; } } } async deleteExample(exampleId) { assertUuid(exampleId); const path = `/examples/${exampleId}`; const response = await this.caller.call(_getFetchImplementation(), this.apiUrl + path, { method: "DELETE", headers: this.headers, signal: AbortSignal.timeout(this.timeout_ms), ...this.fetchOptions, }); await raiseForStatus(response, `delete ${path}`); await response.json(); } async updateExample(exampleId, update) { assertUuid(exampleId); const response = await this.caller.call(_getFetchImplementation(), `${this.apiUrl}/examples/${exampleId}`, { method: "PATCH", headers: { ...this.headers, "Content-Type": "application/json" }, body: JSON.stringify(update), signal: AbortSignal.timeout(this.timeout_ms), ...this.fetchOptions, }); await raiseForStatus(response, "update example"); const result = await response.json(); return result; } async updateExamples(update) { const response = await this.caller.call(_getFetchImplementation(), `${this.apiUrl}/examples/bulk`, { method: "PATCH", headers: { ...this.headers, "Content-Type": "application/json" }, body: JSON.stringify(update), signal: AbortSignal.timeout(this.timeout_ms), ...this.fetchOptions, }); await raiseForStatus(response, "update examples"); const result = await response.json(); return result; } async listDatasetSplits({ datasetId, datasetName, asOf, }) { let datasetId_; if (datasetId === undefined && datasetName === undefined) { throw new Error("Must provide dataset name or ID"); } else if (datasetId !== undefined && datasetName !== undefined) { throw new Error("Must provide either datasetName or datasetId, not both"); } else if (datasetId === undefined) { const dataset = await this.readDataset({ datasetName }); datasetId_ = dataset.id; } else { datasetId_ = datasetId; } assertUuid(datasetId_); const params = new URLSearchParams(); const dataset_version = asOf ? typeof asOf === "string" ? asOf : asOf?.toISOString() : undefined; if (dataset_version) { params.append("as_of", dataset_version); } const response = await this._get(`/datasets/${datasetId_}/splits`, params); return response; } async updateDatasetSplits({ datasetId, datasetName, splitName, exampleIds, remove = false, }) { let datasetId_; if (datasetId === undefined && datasetName === undefined) { throw new Error("Must provide dataset name or ID"); } else if (datasetId !== undefined && datasetName !== undefined) { throw new Error("Must provide either datasetName or datasetId, not both"); } else if (datasetId === undefined) { const dataset = await this.readDataset({ datasetName }); datasetId_ = dataset.id; } else { datasetId_ = datasetId; } assertUuid(datasetId_); const data = { split_name: splitName, examples: exampleIds.map((id) => { assertUuid(id); return id; }), remove, }; const response = await this.caller.call(_getFetchImplementation(), `${this.apiUrl}/datasets/${datasetId_}/splits`, { method: "PUT", headers: { ...this.headers, "Content-Type": "application/json" }, body: JSON.stringify(data), signal: AbortSignal.timeout(this.timeout_ms), ...this.fetchOptions, }); await raiseForStatus(response, "update dataset splits", true); } /** * @deprecated This method is deprecated and will be removed in future LangSmith versions, use `evaluate` from `langsmith/evaluation` instead. */ async evaluateRun(run, evaluator, { sourceInfo, loadChildRuns, referenceExample, } = { loadChildRuns: false }) { warnOnce("This method is deprecated and will be removed in future LangSmith versions, use `evaluate` from `langsmith/evaluation` instead."); let run_; if (typeof run === "string") { run_ = await this.readRun(run, { loadChildRuns }); } else if (typeof run === "object" && "id" in run) { run_ = run; } else { throw new Error(`Invalid run type: ${typeof run}`); } if (run_.reference_example_id !== null && run_.reference_example_id !== undefined) { referenceExample = await this.readExample(run_.reference_example_id); } const feedbackResult = await evaluator.evaluateRun(run_, referenceExample); const [_, feedbacks] = await this._logEvaluationFeedback(feedbackResult, run_, sourceInfo); return feedbacks[0]; } async createFeedback(runId, key, { score, value, correction, comment, sourceInfo, feedbackSourceType = "api", sourceRunId, feedbackId, feedbackConfig, projectId, comparativeExperimentId, }) { if (!runId && !projectId) { throw new Error("One of runId or projectId must be provided"); } if (runId && projectId) { throw new Error("Only one of runId or projectId can be provided"); } const feedback_source = { type: feedbackSourceType ?? "api", metadata: sourceInfo ?? {}, }; if (sourceRunId !== undefined && feedback_source?.metadata !== undefined && !feedback_source.metadata["__run"]) { feedback_source.metadata["__run"] = { run_id: sourceRunId }; } if (feedback_source?.metadata !== undefined && feedback_source.metadata["__run"]?.run_id !== undefined) { assertUuid(feedback_source.metadata["__run"].run_id); } const feedback = { id: feedbackId ?? uuid.v4(), run_id: runId, key, score, value, correction, comment, feedback_source: feedback_source, comparative_experiment_id: comparativeExperimentId, feedbackConfig, session_id: projectId, }; const url = `${this.apiUrl}/feedback`; const response = await this.caller.call(_getFetchImplementation(), url, { method: "POST", headers: { ...this.headers, "Content-Type": "application/json" }, body: JSON.stringify(feedback), signal: AbortSignal.timeout(this.timeout_ms), ...this.fetchOptions, }); await raiseForStatus(response, "create feedback", true); return feedback; } async updateFeedback(feedbackId, { score, value, correction, comment, }) { const feedbackUpdate = {}; if (score !== undefined && score !== null) { feedbackUpdate["score"] = score; } if (value !== undefined && value !== null) { feedbackUpdate["value"] = value; } if (correction !== undefined && correction !== null) { feedbackUpdate["correction"] = correction; } if (comment !== undefined && comment !== null) { feedbackUpdate["comment"] = comment; } assertUuid(feedbackId); const response = await this.caller.call(_getFetchImplementation(), `${this.apiUrl}/feedback/${feedbackId}`, { method: "PATCH", headers: { ...this.headers, "Content-Type": "application/json" }, body: JSON.stringify(feedbackUpdate), signal: AbortSignal.timeout(this.timeout_ms), ...this.fetchOptions, }); await raiseForStatus(response, "update feedback", true); } async readFeedback(feedbackId) { assertUuid(feedbackId); const path = `/feedback/${feedbackId}`; const response = await this._get(path); return response; } async deleteFeedback(feedbackId) { assertUuid(feedbackId); const path = `/feedback/${feedbackId}`; const response = await this.caller.call(_getFetchImplementation(), this.apiUrl + path, { method: "DELETE", headers: this.headers, signal: AbortSignal.timeout(this.timeout_ms), ...this.fetchOptions, }); await raiseForStatus(response, `delete ${path}`); await response.json(); } async *listFeedback({ runIds, feedbackKeys, feedbackSourceTypes, } = {}) { const queryParams = new URLSearchParams(); if (runIds) { queryParams.append("run", runIds.join(",")); } if (feedbackKeys) { for (const key of feedbackKeys) { queryParams.append("key", key); } } if (feedbackSourceTypes) { for (const type of feedbackSourceTypes) { queryParams.append("source", type); } } for await (const feedbacks of this._getPaginated("/feedback", queryParams)) { yield* feedbacks; } } /** * Creates a presigned feedback token and URL. * * The token can be used to authorize feedback metrics without * needing an API key. This is useful for giving browser-based * applications the ability to submit feedback without needing * to expose an API key. * * @param runId - The ID of the run. * @param feedbackKey - The feedback key. * @param options - Additional options for the token. * @param options.expiration - The expiration time for the token. * * @returns A promise that resolves to a FeedbackIngestToken. */ async createPresignedFeedbackToken(runId, feedbackKey, { expiration, feedbackConfig, } = {}) { const body = { run_id: runId, feedback_key: feedbackKey, feedback_config: feedbackConfig, }; if (expiration) { if (typeof expiration === "string") { body["expires_at"] = expiration; } else if (expiration?.hours || expiration?.minutes || expiration?.days) { body["expires_in"] = expiration; } } else { body["expires_in"] = { hours: 3, }; } const response = await this.caller.call(_getFetchImplementation(), `${this.apiUrl}/feedback/tokens`, { method: "POST", headers: { ...this.headers, "Content-Type": "application/json" }, body: JSON.stringify(body), signal: AbortSignal.timeout(this.timeout_ms), ...this.fetchOptions, }); const result = await response.json(); return result; } async createComparativeExperiment({ name, experimentIds, referenceDatasetId, createdAt, description, metadata, id, }) { if (experimentIds.length === 0) { throw new Error("At least one experiment is required"); } if (!referenceDatasetId) { referenceDatasetId = (await this.readProject({ projectId: experimentIds[0], })).reference_dataset_id; } if (!referenceDatasetId == null) { throw new Error("A reference dataset is required"); } const body = { id, name, experiment_ids: experimentIds, reference_dataset_id: referenceDatasetId, description, created_at: (createdAt ?? new Date())?.toISOString(), extra: {}, }; if (metadata) body.extra["metadata"] = metadata; const response = await this.caller.call(_getFetchImplementation(), `${this.apiUrl}/datasets/comparative`, { method: "POST", headers: { ...this.headers, "Content-Type": "application/json" }, body: JSON.stringify(body), signal: AbortSignal.timeout(this.timeout_ms), ...this.fetchOptions, }); return await response.json(); } /** * Retrieves a list of presigned feedback tokens for a given run ID. * @param runId The ID of the run. * @returns An async iterable of FeedbackIngestToken objects. */ async *listPresignedFeedbackTokens(runId) { assertUuid(runId); const params = new URLSearchParams({ run_id: runId }); for await (const tokens of this._getPaginated("/feedback/tokens", params)) { yield* tokens; } } _selectEvalResults(results) { let results_; if ("results" in results) { results_ = results.results; } else { results_ = [results]; } return results_; } async _logEvaluationFeedback(evaluatorResponse, run, sourceInfo) { const evalResults = this._selectEvalResults(evaluatorResponse); const feedbacks = []; for (const res of evalResults) { let sourceInfo_ = sourceInfo || {}; if (res.evaluatorInfo) { sourceInfo_ = { ...res.evaluatorInfo, ...sourceInfo_ }; } let runId_ = null; if (res.targetRunId) { runId_ = res.targetRunId; } else if (run) { runId_ = run.id; } feedbacks.push(await this.createFeedback(runId_, res.key, { score: res.score, value: res.value, comment: res.comment, correction: res.correction, sourceInfo: sourceInfo_, sourceRunId: res.sourceRunId, feedbackConfig: res.feedbackConfig, feedbackSourceType: "model", })); } return [evalResults, feedbacks]; } async logEvaluationFeedback(evaluatorResponse, run, sourceInfo) { const [results] = await this._logEvaluationFeedback(evaluatorResponse, run, sourceInfo); return results; } /** * API for managing annotation queues */ /** * List the annotation queues on the LangSmith API. * @param options - The options for listing annotation queues * @param options.queueIds - The IDs of the queues to filter by * @param options.name - The name of the queue to filter by * @param options.nameContains - The substring that the queue name should contain * @param options.limit - The maximum number of queues to return * @returns An iterator of AnnotationQueue objects */ async *listAnnotationQueues(options = {}) { const { queueIds, name, nameContains, limit } = options; const params = new URLSearchParams(); if (queueIds) { queueIds.forEach((id, i) => { assertUuid(id, `queueIds[${i}]`); params.append("ids", id); }); } if (name) params.append("name", name); if (nameContains) params.append("name_contains", nameContains); params.append("limit", (limit !== undefined ? Math.min(limit, 100) : 100).toString()); let count = 0; for await (const queues of this._getPaginated("/annotation-queues", params)) { yield* queues; count++; if (limit !== undefined && count >= limit) break; } } /** * Create an annotation queue on the LangSmith API. * @param options - The options for creating an annotation queue * @param options.name - The name of the annotation queue * @param options.description - The description of the annotation queue * @param options.queueId - The ID of the annotation queue * @returns The created AnnotationQueue object */ async createAnnotationQueue(options) { const { name, description, queueId } = options; const body = { name, description, id: queueId || uuid.v4(), }; const response = await this.caller.call(_getFetchImplementation(), `${this.apiUrl}/annotation-queues`, { method: "POST", headers: { ...this.headers, "Content-Type": "application/json" }, body: JSON.stringify(Object.fromEntries(Object.entries(body).filter(([_, v]) => v !== undefined))), signal: AbortSignal.timeout(this.timeout_ms), ...this.fetchOptions, }); await raiseForStatus(response, "create annotation queue"); const data = await response.json(); return data; } /** * Read an annotation queue with the specified queue ID. * @param queueId - The ID of the annotation queue to read * @returns The AnnotationQueue object */ async readAnnotationQueue(queueId) { // TODO: Replace when actual endpoint is added const queueIteratorResult = await this.listAnnotationQueues({ queueIds: [queueId], }).next(); if (queueIteratorResult.done) { throw new Error(`Annotation queue with ID ${queueId} not found`); } return queueIteratorResult.value; } /** * Update an annotation queue with the specified queue ID. * @param queueId - The ID of the annotation queue to update * @param options - The options for updating the annotation queue * @param options.name - The new name for the annotation queue * @param options.description - The new description for the annotation queue */ async updateAnnotationQueue(queueId, options) { const { name, description } = options; const response = await this.caller.call(_getFetchImplementation(), `${this.apiUrl}/annotation-queues/${assertUuid(queueId, "queueId")}`, { method: "PATCH", headers: { ...this.headers, "Content-Type": "application/json" }, body: JSON.stringify({ name, description }), signal: AbortSignal.timeout(this.timeout_ms), ...this.fetchOptions, }); await raiseForStatus(response, "update annotation queue"); } /** * Delete an annotation queue with the specified queue ID. * @param queueId - The ID of the annotation queue to delete */ async deleteAnnotationQueue(queueId) { const response = await this.caller.call(_getFetchImplementation(), `${this.apiUrl}/annotation-queues/${assertUuid(queueId, "queueId")}`, { method: "DELETE", headers: { ...this.headers, Accept: "application/json" }, signal: AbortSignal.timeout(this.timeout_ms), ...this.fetchOptions, }); await raiseForStatus(response, "delete annotation queue"); } /** * Add runs to an annotation queue with the specified queue ID. * @param queueId - The ID of the annotation queue * @param runIds - The IDs of the runs to be added to the annotation queue */ async addRunsToAnnotationQueue(queueId, runIds) { const response = await this.caller.call(_getFetchImplementation(), `${this.apiUrl}/annotation-queues/${assertUuid(queueId, "queueId")}/runs`, { method: "POST", headers: { ...this.headers, "Content-Type": "application/json" }, body: JSON.stringify(runIds.map((id, i) => assertUuid(id, `runIds[${i}]`).toString())), signal: AbortSignal.timeout(this.timeout_ms), ...this.fetchOptions, }); await raiseForStatus(response, "add runs to annotation queue"); } /** * Get a run from an annotation queue at the specified index. * @param queueId - The ID of the annotation queue * @param index - The index of the run to retrieve * @returns A Promise that resolves to a RunWithAnnotationQueueInfo object * @throws {Error} If the run is not found at the given index or for other API-related errors */ async getRunFromAnnotationQueue(queueId, index) { const baseUrl = `/annotation-queues/${assertUuid(queueId, "queueId")}/run`; const response = await this.caller.call(_getFetchImplementation(), `${this.apiUrl}${baseUrl}/${index}`, { method: "GET", headers: this.headers, signal: AbortSignal.timeout(this.timeout_ms), ...this.fetchOptions, }); await raiseForStatus(response, "get run from annotation queue"); return await response.json(); } async _currentTenantIsOwner(owner) { const settings = await this._getSettings(); return owner == "-" || settings.tenant_handle === owner; } async _ownerConflictError(action, owner) { const settings = await this._getSettings(); return new Error(`Cannot ${action} for another tenant.\n Current tenant: ${settings.tenant_handle}\n Requested tenant: ${owner}`); } async _getLatestCommitHash(promptOwnerAndName) { const res = await this.caller.call(_getFetchImplementation(), `${this.apiUrl}/commits/${promptOwnerAndName}/?limit=${1}&offset=${0}`, { method: "GET", headers: this.headers, signal: AbortSignal.timeout(this.timeout_ms), ...this.fetchOptions, }); const json = await res.json(); if (!res.ok) { const detail = typeof json.detail === "string" ? json.detail : JSON.stringify(json.detail); const error = new Error(`Error ${res.status}: ${res.statusText}\n${detail}`); // eslint-disable-next-line @typescript-eslint/no-explicit-any error.statusCode = res.status; throw error; } if (json.commits.length === 0) { return undefined; } return json.commits[0].commit_hash; } async _likeOrUnlikePrompt(promptIdentifier, like) { const [owner, promptName, _] = parsePromptIdentifier(promptIdentifier); const response = await this.caller.call(_getFetchImplementation(), `${this.apiUrl}/likes/${owner}/${promptName}`, { method: "POST", body: JSON.stringify({ like: like }), headers: { ...this.headers, "Content-Type": "application/json" }, signal: AbortSignal.timeout(this.timeout_ms), ...this.fetchOptions, }); await raiseForStatus(response, `${like ? "like" : "unlike"} prompt`); return await response.json(); } async _getPromptUrl(promptIdentifier) { const [owner, promptName, commitHash] = parsePromptIdentifier(promptIdentifier); if (!(await this._currentTenantIsOwner(owner))) { if (commitHash !== "latest") { return `${this.getHostUrl()}/hub/${owner}/${promptName}/${commitHash.substring(0, 8)}`; } else { return `${this.getHostUrl()}/hub/${owner}/${promptName}`; } } else { const settings = await this._getSettings(); if (commitHash !== "latest") { return `${this.getHostUrl()}/prompts/${promptName}/${commitHash.substring(0, 8)}?organizationId=${settings.id}`; } else { return `${this.getHostUrl()}/prompts/${promptName}?organizationId=${settings.id}`; } } } async promptExists(promptIdentifier) { const prompt = await this.getPrompt(promptIdentifier); return !!prompt; } async likePrompt(promptIdentifier) { return this._likeOrUnlikePrompt(promptIdentifier, true); } async unlikePrompt(promptIdentifier) { return this._likeOrUnlikePrompt(promptIdentifier, false); } async *listCommits(promptOwnerAndName) { for await (const commits of this._getPaginated(`/commits/${promptOwnerAndName}/`, new URLSearchParams(), (res) => res.commits)) { yield* commits; } } async *listPrompts(options) { const params = new URLSearchParams(); params.append("sort_field", options?.sortField ?? "updated_at"); params.append("sort_direction", "desc"); params.append("is_archived", (!!options?.isArchived).toString()); if (options?.isPublic !== undefined) { params.append("is_public", options.isPublic.toString()); } if (options?.query) { params.append("query", options.query); } for await (const prompts of this._getPaginated("/repos", params, (res) => res.repos)) { yield* prompts; } } async getPrompt(promptIdentifier) { const [owner, promptName, _] = parsePromptIdentifier(promptIdentifier); const response = await this.caller.call(_getFetchImplementation(), `${this.apiUrl}/repos/${owner}/${promptName}`, { method: "GET", headers: this.headers, signal: AbortSignal.timeout(this.timeout_ms), ...this.fetchOptions, }); if (response.status === 404) { return null; } await raiseForStatus(response, "get prompt"); const result = await response.json(); if (result.repo) { return result.repo; } else { return null; } } async createPrompt(promptIdentifier, options) { const settings = await this._getSettings(); if (options?.isPublic && !settings.tenant_handle) { throw new Error(`Cannot create a public prompt without first\n creating a LangChain Hub handle. You can add a handle by creating a public prompt at:\n https://smith.langchain.com/prompts`); } const [owner, promptName, _] = parsePromptIdentifier(promptIdentifier); if (!(await this._currentTenantIsOwner(owner))) { throw await this._ownerConflictError("create a prompt", owner); } const data = { repo_handle: promptName, ...(options?.description && { description: options.description }), ...(options?.readme && { readme: options.readme }), ...(options?.tags && { tags: options.tags }), is_public: !!options?.isPublic, }; const response = await this.caller.call(_getFetchImplementation(), `${this.apiUrl}/repos/`, { method: "POST", headers: { ...this.headers, "Content-Type": "application/json" }, body: JSON.stringify(data), signal: AbortSignal.timeout(this.timeout_ms), ...this.fetchOptions, }); await raiseForStatus(response, "create prompt"); const { repo } = await response.json(); return repo; } async createCommit(promptIdentifier, object, options) { if (!(await this.promptExists(promptIdentifier))) { throw new Error("Prompt does not exist, you must create it first."); } const [owner, promptName, _] = parsePromptIdentifier(promptIdentifier); const resolvedParentCommitHash = options?.parentCommitHash === "latest" || !options?.parentCommitHash ? await this._getLatestCommitHash(`${owner}/${promptName}`) : options?.parentCommitHash; const payload = { manifest: JSON.parse(JSON.stringify(object)), parent_commit: resolvedParentCommitHash, }; const response = await this.caller.call(_getFetchImplementation(), `${this.apiUrl}/commits/${owner}/${promptName}`, { method: "POST", headers: { ...this.headers, "Content-Type": "application/json" }, body: JSON.stringify(payload), signal: AbortSignal.timeout(this.timeout_ms), ...this.fetchOptions, }); await raiseForStatus(response, "create commit"); const result = await response.json(); return this._getPromptUrl(`${owner}/${promptName}${result.commit_hash ? `:${result.commit_hash}` : ""}`); } async updatePrompt(promptIdentifier, options) { if (!(await this.promptExists(promptIdentifier))) { throw new Error("Prompt does not exist, you must create it first."); } const [owner, promptName] = parsePromptIdentifier(promptIdentifier); if (!(await this._currentTenantIsOwner(owner))) { throw await this._ownerConflictError("update a prompt", owner); } const payload = {}; if (options?.description !== undefined) payload.description = options.description; if (options?.readme !== undefined) payload.readme = options.readme; if (options?.tags !== undefined) payload.tags = options.tags; if (options?.isPublic !== undefined) payload.is_public = options.isPublic; if (options?.isArchived !== undefined) payload.is_archived = options.isArchived; // Check if payload is empty if (Object.keys(payload).length === 0) { throw new Error("No valid update options provided"); } const response = await this.caller.call(_getFetchImplementation(), `${this.apiUrl}/repos/${owner}/${promptName}`, { method: "PATCH", body: JSON.stringify(payload), headers: { ...this.headers, "Content-Type": "application/json", }, signal: AbortSignal.timeout(this.timeout_ms), ...this.fetchOptions, }); await raiseForStatus(response, "update prompt"); return response.json(); } async deletePrompt(promptIdentifier) { if (!(await this.promptExists(promptIdentifier))) { throw new Error("Prompt does not exist, you must create it first."); } const [owner, promptName, _] = parsePromptIdentifier(promptIdentifier); if (!(await this._currentTenantIsOwner(owner))) { throw await this._ownerConflictError("delete a prompt", owner); } const response = await this.caller.call(_getFetchImplementation(), `${this.apiUrl}/repos/${owner}/${promptName}`, { method: "DELETE", headers: this.headers, signal: AbortSignal.timeout(this.timeout_ms), ...this.fetchOptions, }); return await response.json(); } async pullPromptCommit(promptIdentifier, options) { const [owner, promptName, commitHash] = parsePromptIdentifier(promptIdentifier); const serverInfo = await this._getServerInfo(); const useOptimization = isVersionGreaterOrEqual(serverInfo.version, "0.5.23"); let passedCommitHash = commitHash; if (!useOptimization && commitHash === "latest") { const latestCommitHash = await this._getLatestCommitHash(`${owner}/${promptName}`); if (!latestCommitHash) { throw new Error("No commits found"); } else { passedCommitHash = latestCommitHash; } } const response = await this.caller.call(_getFetchImplementation(), `${this.apiUrl}/commits/${owner}/${promptName}/${passedCommitHash}${options?.includeModel ? "?include_model=true" : ""}`, { method: "GET", headers: this.headers, signal: AbortSignal.timeout(this.timeout_ms), ...this.fetchOptions, }); await raiseForStatus(response, "pull prompt commit"); const result = await response.json(); return { owner, repo: promptName, commit_hash: result.commit_hash, manifest: result.manifest, examples: result.examples, }; } /** * This method should not be used directly, use `import { pull } from "langchain/hub"` instead. * Using this method directly returns the JSON string of the prompt rather than a LangChain object. * @private */ async _pullPrompt(promptIdentifier, options) { const promptObject = await this.pullPromptCommit(promptIdentifier, { includeModel: options?.includeModel, }); const prompt = JSON.stringify(promptObject.manifest); return prompt; } async pushPrompt(promptIdentifier, options) { // Create or update prompt metadata if (await this.promptExists(promptIdentifier)) { if (options && Object.keys(options).some((key) => key !== "object")) { await this.updatePrompt(promptIdentifier, { description: options?.description, readme: options?.readme, tags: options?.tags, isPublic: options?.isPublic, }); } } else { await this.createPrompt(promptIdentifier, { description: options?.description, readme: options?.readme, tags: options?.tags, isPublic: options?.isPublic, }); } if (!options?.object) { return await this._getPromptUrl(promptIdentifier); } // Create a commit with the new manifest const url = await this.createCommit(promptIdentifier, options?.object, { parentCommitHash: options?.parentCommitHash, }); return url; } /** * Clone a public dataset to your own langsmith tenant. * This operation is idempotent. If you already have a dataset with the given name, * this function will do nothing. * @param {string} tokenOrUrl The token of the public dataset to clone. * @param {Object} [options] Additional options for cloning the dataset. * @param {string} [options.sourceApiUrl] The URL of the langsmith server where the data is hosted. Defaults to the API URL of your current client. * @param {string} [options.datasetName] The name of the dataset to create in your tenant. Defaults to the name of the public dataset. * @returns {Promise} */ async clonePublicDataset(tokenOrUrl, options = {}) { const { sourceApiUrl = this.apiUrl, datasetName } = options; const [parsedApiUrl, tokenUuid] = this.parseTokenOrUrl(tokenOrUrl, sourceApiUrl); const sourceClient = new Client({ apiUrl: parsedApiUrl, // Placeholder API key not needed anymore in most cases, but // some private deployments may have API key-based rate limiting // that would cause this to fail if we provide no value. apiKey: "placeholder", }); const ds = await sourceClient.readSharedDataset(tokenUuid); const finalDatasetName = datasetName || ds.name; try { if (await this.hasDataset({ datasetId: finalDatasetName })) { console.log(`Dataset ${finalDatasetName} already exists in your tenant. Skipping.`); return; } } catch (_) { // `.hasDataset` will throw an error if the dataset does not exist. // no-op in that case } // Fetch examples first, then create the dataset const examples = await sourceClient.listSharedExamples(tokenUuid); const dataset = await this.createDataset(finalDatasetName, { description: ds.description, dataType: ds.data_type || "kv", inputsSchema: ds.inputs_schema_definition ?? undefined, outputsSchema: ds.outputs_schema_definition ?? undefined, }); try { await this.createExamples({ inputs: examples.map((e) => e.inputs), outputs: examples.flatMap((e) => (e.outputs ? [e.outputs] : [])), datasetId: dataset.id, }); } catch (e) { console.error(`An error occurred while creating dataset ${finalDatasetName}. ` + "You should delete it manually."); throw e; } } parseTokenOrUrl(urlOrToken, apiUrl, numParts = 2, kind = "dataset") { // Try parsing as UUID try { assertUuid(urlOrToken); // Will throw if it's not a UUID. return [apiUrl, urlOrToken]; } catch (_) { // no-op if it's not a uuid } // Parse as URL try { const parsedUrl = new URL(urlOrToken); const pathParts = parsedUrl.pathname .split("/") .filter((part) => part !== ""); if (pathParts.length >= numParts) { const tokenUuid = pathParts[pathParts.length - numParts]; return [apiUrl, tokenUuid]; } else { throw new Error(`Invalid public ${kind} URL: ${urlOrToken}`); } } catch (error) { throw new Error(`Invalid public ${kind} URL or token: ${urlOrToken}`); } } }