56 lines
2.4 KiB
JavaScript
56 lines
2.4 KiB
JavaScript
import { ChatOpenAI } from "@langchain/openai";
|
|
import { BaseChatModel } from "@langchain/core/language_models/chat_models";
|
|
import { CriteriaEvalChain, LabeledCriteriaEvalChain, } from "./criteria/index.js";
|
|
import { LabeledPairwiseStringEvalChain, PairwiseStringEvalChain, } from "./comparison/index.js";
|
|
import { EmbeddingDistanceEvalChain, PairwiseEmbeddingDistanceEvalChain, } from "./embedding_distance/index.js";
|
|
import { TrajectoryEvalChain } from "./agents/index.js";
|
|
/**
|
|
* Load the requested evaluation chain specified by a string
|
|
* @param type The type of evaluator to load.
|
|
* @param options
|
|
* - llm The language model to use for the evaluator.
|
|
* - criteria The criteria to use for the evaluator.
|
|
* - agentTools A list of tools available to the agent,for TrajectoryEvalChain.
|
|
*/
|
|
export async function loadEvaluator(type, options) {
|
|
const { llm, chainOptions, criteria, agentTools } = options || {};
|
|
const llm_ = llm ??
|
|
new ChatOpenAI({
|
|
modelName: "gpt-4",
|
|
temperature: 0.0,
|
|
});
|
|
let evaluator;
|
|
switch (type) {
|
|
case "criteria":
|
|
evaluator = await CriteriaEvalChain.fromLLM(llm_, criteria, chainOptions);
|
|
break;
|
|
case "labeled_criteria":
|
|
evaluator = await LabeledCriteriaEvalChain.fromLLM(llm_, criteria, chainOptions);
|
|
break;
|
|
case "pairwise_string":
|
|
evaluator = await PairwiseStringEvalChain.fromLLM(llm_, criteria, chainOptions);
|
|
break;
|
|
case "labeled_pairwise_string":
|
|
evaluator = await LabeledPairwiseStringEvalChain.fromLLM(llm_, criteria, chainOptions);
|
|
break;
|
|
case "trajectory":
|
|
// eslint-disable-next-line no-instanceof/no-instanceof
|
|
if (!(llm_ instanceof BaseChatModel)) {
|
|
throw new Error("LLM must be an instance of a base chat model.");
|
|
}
|
|
evaluator = await TrajectoryEvalChain.fromLLM(llm_, agentTools, chainOptions);
|
|
break;
|
|
case "embedding_distance":
|
|
evaluator = new EmbeddingDistanceEvalChain({
|
|
embedding: options?.embedding,
|
|
distanceMetric: options?.distanceMetric,
|
|
});
|
|
break;
|
|
case "pairwise_embedding_distance":
|
|
evaluator = new PairwiseEmbeddingDistanceEvalChain({});
|
|
break;
|
|
default:
|
|
throw new Error(`Unknown type: ${type}`);
|
|
}
|
|
return evaluator;
|
|
}
|