"use strict";
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
    if (k2 === undefined) k2 = k;
    var desc = Object.getOwnPropertyDescriptor(m, k);
    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
      desc = { enumerable: true, get: function() { return m[k]; } };
    }
    Object.defineProperty(o, k2, desc);
}) : (function(o, m, k, k2) {
    if (k2 === undefined) k2 = k;
    o[k2] = m[k];
}));
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
    Object.defineProperty(o, "default", { enumerable: true, value: v });
}) : function(o, v) {
    o["default"] = v;
});
var __importStar = (this && this.__importStar) || function (mod) {
    if (mod && mod.__esModule) return mod;
    var result = {};
    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
    __setModuleDefault(result, mod);
    return result;
};
var __importDefault = (this && this.__importDefault) || function (mod) {
    return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.MatchingEngine = exports.IdDocument = void 0;
const uuid = __importStar(require("uuid"));
const flat_1 = __importDefault(require("flat"));
const google_auth_library_1 = require("google-auth-library");
const vectorstores_1 = require("@langchain/core/vectorstores");
const documents_1 = require("@langchain/core/documents");
const async_caller_1 = require("@langchain/core/utils/async_caller");
const googlevertexai_connection_js_1 = require("../utils/googlevertexai-connection.cjs");
/**
 * A Document that optionally includes the ID of the document.
 */
class IdDocument extends documents_1.Document {
    constructor(fields) {
        super(fields);
        Object.defineProperty(this, "id", {
            enumerable: true,
            configurable: true,
            writable: true,
            value: void 0
        });
        this.id = fields.id;
    }
}
exports.IdDocument = IdDocument;
class IndexEndpointConnection extends googlevertexai_connection_js_1.GoogleVertexAIConnection {
    constructor(fields, caller) {
        super(fields, caller, new google_auth_library_1.GoogleAuth(fields.authOptions));
        Object.defineProperty(this, "indexEndpoint", {
            enumerable: true,
            configurable: true,
            writable: true,
            value: void 0
        });
        this.indexEndpoint = fields.indexEndpoint;
    }
    async buildUrl() {
        const projectId = await this.client.getProjectId();
        const url = `https://${this.endpoint}/${this.apiVersion}/projects/${projectId}/locations/${this.location}/indexEndpoints/${this.indexEndpoint}`;
        return url;
    }
    buildMethod() {
        return "GET";
    }
    async request(options) {
        return this._request(undefined, options);
    }
}
class RemoveDatapointConnection extends googlevertexai_connection_js_1.GoogleVertexAIConnection {
    constructor(fields, caller) {
        super(fields, caller, new google_auth_library_1.GoogleAuth(fields.authOptions));
        Object.defineProperty(this, "index", {
            enumerable: true,
            configurable: true,
            writable: true,
            value: void 0
        });
        this.index = fields.index;
    }
    async buildUrl() {
        const projectId = await this.client.getProjectId();
        const url = `https://${this.endpoint}/${this.apiVersion}/projects/${projectId}/locations/${this.location}/indexes/${this.index}:removeDatapoints`;
        return url;
    }
    buildMethod() {
        return "POST";
    }
    async request(datapointIds, options) {
        const data = {
            datapointIds,
        };
        return this._request(data, options);
    }
}
class UpsertDatapointConnection extends googlevertexai_connection_js_1.GoogleVertexAIConnection {
    constructor(fields, caller) {
        super(fields, caller, new google_auth_library_1.GoogleAuth(fields.authOptions));
        Object.defineProperty(this, "index", {
            enumerable: true,
            configurable: true,
            writable: true,
            value: void 0
        });
        this.index = fields.index;
    }
    async buildUrl() {
        const projectId = await this.client.getProjectId();
        const url = `https://${this.endpoint}/${this.apiVersion}/projects/${projectId}/locations/${this.location}/indexes/${this.index}:upsertDatapoints`;
        return url;
    }
    buildMethod() {
        return "POST";
    }
    async request(datapoints, options) {
        const data = {
            datapoints,
        };
        return this._request(data, options);
    }
}
class FindNeighborsConnection extends googlevertexai_connection_js_1.GoogleVertexAIConnection {
    constructor(params, caller) {
        super(params, caller, new google_auth_library_1.GoogleAuth(params.authOptions));
        Object.defineProperty(this, "indexEndpoint", {
            enumerable: true,
            configurable: true,
            writable: true,
            value: void 0
        });
        Object.defineProperty(this, "deployedIndexId", {
            enumerable: true,
            configurable: true,
            writable: true,
            value: void 0
        });
        this.indexEndpoint = params.indexEndpoint;
        this.deployedIndexId = params.deployedIndexId;
    }
    async buildUrl() {
        const projectId = await this.client.getProjectId();
        const url = `https://${this.endpoint}/${this.apiVersion}/projects/${projectId}/locations/${this.location}/indexEndpoints/${this.indexEndpoint}:findNeighbors`;
        return url;
    }
    buildMethod() {
        return "POST";
    }
    async request(request, options) {
        return this._request(request, options);
    }
}
/**
 * A class that represents a connection to a Google Vertex AI Matching Engine
 * instance.
 */
class MatchingEngine extends vectorstores_1.VectorStore {
    constructor(embeddings, args) {
        super(embeddings, args);
        /**
         * Docstore that retains the document, stored by ID
         */
        Object.defineProperty(this, "docstore", {
            enumerable: true,
            configurable: true,
            writable: true,
            value: void 0
        });
        /**
         * The host to connect to for queries and upserts.
         */
        Object.defineProperty(this, "apiEndpoint", {
            enumerable: true,
            configurable: true,
            writable: true,
            value: void 0
        });
        Object.defineProperty(this, "apiVersion", {
            enumerable: true,
            configurable: true,
            writable: true,
            value: "v1"
        });
        Object.defineProperty(this, "endpoint", {
            enumerable: true,
            configurable: true,
            writable: true,
            value: "us-central1-aiplatform.googleapis.com"
        });
        Object.defineProperty(this, "location", {
            enumerable: true,
            configurable: true,
            writable: true,
            value: "us-central1"
        });
        /**
         * The id for the index endpoint
         */
        Object.defineProperty(this, "indexEndpoint", {
            enumerable: true,
            configurable: true,
            writable: true,
            value: void 0
        });
        /**
         * The id for the index
         */
        Object.defineProperty(this, "index", {
            enumerable: true,
            configurable: true,
            writable: true,
            value: void 0
        });
        /**
         * Explicitly set Google Auth credentials if you cannot get them from google auth application-default login
         * This is useful for serverless or autoscaling environments like Fargate
         */
        Object.defineProperty(this, "authOptions", {
            enumerable: true,
            configurable: true,
            writable: true,
            value: void 0
        });
        /**
         * The id for the "deployed index", which is an identifier in the
         * index endpoint that references the index (but is not the index id)
         */
        Object.defineProperty(this, "deployedIndexId", {
            enumerable: true,
            configurable: true,
            writable: true,
            value: void 0
        });
        Object.defineProperty(this, "callerParams", {
            enumerable: true,
            configurable: true,
            writable: true,
            value: void 0
        });
        Object.defineProperty(this, "callerOptions", {
            enumerable: true,
            configurable: true,
            writable: true,
            value: void 0
        });
        Object.defineProperty(this, "caller", {
            enumerable: true,
            configurable: true,
            writable: true,
            value: void 0
        });
        Object.defineProperty(this, "indexEndpointClient", {
            enumerable: true,
            configurable: true,
            writable: true,
            value: void 0
        });
        Object.defineProperty(this, "removeDatapointClient", {
            enumerable: true,
            configurable: true,
            writable: true,
            value: void 0
        });
        Object.defineProperty(this, "upsertDatapointClient", {
            enumerable: true,
            configurable: true,
            writable: true,
            value: void 0
        });
        this.embeddings = embeddings;
        this.docstore = args.docstore;
        this.apiEndpoint = args.apiEndpoint ?? this.apiEndpoint;
        this.deployedIndexId = args.deployedIndexId ?? this.deployedIndexId;
        this.apiVersion = args.apiVersion ?? this.apiVersion;
        this.endpoint = args.endpoint ?? this.endpoint;
        this.location = args.location ?? this.location;
        this.indexEndpoint = args.indexEndpoint ?? this.indexEndpoint;
        this.index = args.index ?? this.index;
        this.authOptions = args.authOptions ?? this.authOptions;
        this.callerParams = args.callerParams ?? this.callerParams;
        this.callerOptions = args.callerOptions ?? this.callerOptions;
        this.caller = new async_caller_1.AsyncCaller(this.callerParams || {});
        const indexClientParams = {
            endpoint: this.endpoint,
            location: this.location,
            apiVersion: this.apiVersion,
            indexEndpoint: this.indexEndpoint,
            authOptions: this.authOptions,
        };
        this.indexEndpointClient = new IndexEndpointConnection(indexClientParams, this.caller);
        const removeClientParams = {
            endpoint: this.endpoint,
            location: this.location,
            apiVersion: this.apiVersion,
            index: this.index,
            authOptions: this.authOptions,
        };
        this.removeDatapointClient = new RemoveDatapointConnection(removeClientParams, this.caller);
        const upsertClientParams = {
            endpoint: this.endpoint,
            location: this.location,
            apiVersion: this.apiVersion,
            index: this.index,
            authOptions: this.authOptions,
        };
        this.upsertDatapointClient = new UpsertDatapointConnection(upsertClientParams, this.caller);
    }
    _vectorstoreType() {
        return "googlevertexai";
    }
    async addDocuments(documents) {
        const texts = documents.map((doc) => doc.pageContent);
        const vectors = await this.embeddings.embedDocuments(texts);
        return this.addVectors(vectors, documents);
    }
    async addVectors(vectors, documents) {
        if (vectors.length !== documents.length) {
            throw new Error(`Vectors and metadata must have the same length`);
        }
        const datapoints = vectors.map((vector, idx) => this.buildDatapoint(vector, documents[idx]));
        const options = {};
        const response = await this.upsertDatapointClient.request(datapoints, options);
        if (Object.keys(response?.data ?? {}).length === 0) {
            // Nothing in the response in the body means we saved it ok
            const idDoc = documents;
            const docsToStore = {};
            idDoc.forEach((doc) => {
                if (doc.id) {
                    docsToStore[doc.id] = doc;
                }
            });
            await this.docstore.add(docsToStore);
        }
    }
    // TODO: Refactor this into a utility type and use with pinecone as well?
    // eslint-disable-next-line @typescript-eslint/no-explicit-any
    cleanMetadata(documentMetadata) {
        function getStringArrays(prefix, 
        // eslint-disable-next-line @typescript-eslint/no-explicit-any
        m) {
            let ret = {};
            Object.keys(m).forEach((key) => {
                const newPrefix = prefix.length > 0 ? `${prefix}.${key}` : key;
                const val = m[key];
                if (!val) {
                    // Ignore it
                }
                else if (Array.isArray(val)) {
                    // Make sure everything in the array is a string
                    ret[newPrefix] = val.map((v) => `${v}`);
                }
                else if (typeof val === "object") {
                    const subArrays = getStringArrays(newPrefix, val);
                    ret = { ...ret, ...subArrays };
                }
            });
            return ret;
        }
        const stringArrays = getStringArrays("", documentMetadata);
        const flatMetadata = (0, flat_1.default)(documentMetadata);
        Object.keys(flatMetadata).forEach((key) => {
            Object.keys(stringArrays).forEach((arrayKey) => {
                const matchKey = `${arrayKey}.`;
                if (key.startsWith(matchKey)) {
                    delete flatMetadata[key];
                }
            });
        });
        const metadata = {
            ...flatMetadata,
            ...stringArrays,
        };
        return metadata;
    }
    /**
     * Given the metadata from a document, convert it to an array of Restriction
     * objects that may be passed to the Matching Engine and stored.
     * The default implementation flattens any metadata and includes it as
     * an "allowList". Subclasses can choose to convert some of these to
     * "denyList" items or to add additional restrictions (for example, to format
     * dates into a different structure or to add additional restrictions
     * based on the date).
     * @param documentMetadata - The metadata from a document
     * @returns a Restriction[] (or an array of a subclass, from the FilterType)
     */
    metadataToRestrictions(
    // eslint-disable-next-line @typescript-eslint/no-explicit-any
    documentMetadata) {
        const metadata = this.cleanMetadata(documentMetadata);
        const restrictions = [];
        for (const key of Object.keys(metadata)) {
            // Make sure the value is an array (or that we'll ignore it)
            let valArray;
            const val = metadata[key];
            if (val === null) {
                valArray = null;
            }
            else if (Array.isArray(val) && val.length > 0) {
                valArray = val;
            }
            else {
                valArray = [`${val}`];
            }
            // Add to the restrictions if we do have a valid value
            if (valArray) {
                // Determine if this key is for the allowList or denyList
                // TODO: get which ones should be on the deny list
                const listType = "allowList";
                // Create the restriction
                const restriction = {
                    namespace: key,
                    [listType]: valArray,
                };
                // Add it to the restriction list
                restrictions.push(restriction);
            }
        }
        return restrictions;
    }
    /**
     * Create an index datapoint for the vector and document id.
     * If an id does not exist, create it and set the document to its value.
     * @param vector
     * @param document
     */
    buildDatapoint(vector, document) {
        if (!document.id) {
            // eslint-disable-next-line no-param-reassign
            document.id = uuid.v4();
        }
        const ret = {
            datapointId: document.id,
            featureVector: vector,
        };
        const restrictions = this.metadataToRestrictions(document.metadata);
        if (restrictions?.length > 0) {
            ret.restricts = restrictions;
        }
        return ret;
    }
    async delete(params) {
        const options = {};
        await this.removeDatapointClient.request(params.ids, options);
    }
    async similaritySearchVectorWithScore(query, k, filter) {
        // Format the query into the request
        const deployedIndexId = await this.getDeployedIndexId();
        const requestQuery = {
            neighborCount: k,
            datapoint: {
                datapointId: `0`,
                featureVector: query,
            },
        };
        if (filter) {
            requestQuery.datapoint.restricts = filter;
        }
        const request = {
            deployedIndexId,
            queries: [requestQuery],
        };
        // Build the connection.
        // Has to be done here, since we defer getting the endpoint until
        // we need it.
        const apiEndpoint = await this.getPublicAPIEndpoint();
        const findNeighborsParams = {
            endpoint: apiEndpoint,
            indexEndpoint: this.indexEndpoint,
            apiVersion: this.apiVersion,
            location: this.location,
            deployedIndexId,
            authOptions: this.authOptions,
        };
        const connection = new FindNeighborsConnection(findNeighborsParams, this.caller);
        // Make the call
        const options = {};
        const response = await connection.request(request, options);
        // Get the document for each datapoint id and return them
        const nearestNeighbors = response?.data?.nearestNeighbors ?? [];
        const nearestNeighbor = nearestNeighbors[0];
        const neighbors = nearestNeighbor?.neighbors ?? [];
        const ret = await Promise.all(neighbors.map(async (neighbor) => {
            const id = neighbor?.datapoint?.datapointId;
            const distance = neighbor?.distance;
            let doc;
            try {
                doc = await this.docstore.search(id);
            }
            catch (xx) {
                // Documents that are in the index are returned, even if they
                // are not in the document store, to allow for some way to get
                // the id so they can be deleted.
                console.error(xx);
                console.warn([
                    `Document with id "${id}" is missing from the backing docstore.`,
                    `This can occur if you clear the docstore without deleting from the corresponding Matching Engine index.`,
                    `To resolve this, you should call .delete() with this id as part of the "ids" parameter.`,
                ].join("\n"));
                doc = new documents_1.Document({ pageContent: `Missing document ${id}` });
            }
            doc.id ??= id;
            return [doc, distance];
        }));
        return ret;
    }
    /**
     * For this index endpoint, figure out what API Endpoint URL and deployed
     * index ID should be used to do upserts and queries.
     * Also sets the `apiEndpoint` and `deployedIndexId` property for future use.
     * @return The URL
     */
    async determinePublicAPIEndpoint() {
        const response = await this.indexEndpointClient.request(this.callerOptions);
        // Get the endpoint
        const publicEndpointDomainName = response?.data?.publicEndpointDomainName;
        this.apiEndpoint = publicEndpointDomainName;
        // Determine which of the deployed indexes match the index id
        // and get the deployed index id. The list of deployed index ids
        // contain the "index name" or path, but not the index id by itself,
        // so we need to extract it from the name
        const indexPathPattern = /projects\/.+\/locations\/.+\/indexes\/(.+)$/;
        const deployedIndexes = response?.data?.deployedIndexes ?? [];
        const deployedIndex = deployedIndexes.find((index) => {
            const deployedIndexPath = index.index;
            const match = deployedIndexPath.match(indexPathPattern);
            if (match) {
                const [, potentialIndexId] = match;
                if (potentialIndexId === this.index) {
                    return true;
                }
            }
            return false;
        });
        if (deployedIndex) {
            this.deployedIndexId = deployedIndex.id;
        }
        return {
            apiEndpoint: this.apiEndpoint,
            deployedIndexId: this.deployedIndexId,
        };
    }
    async getPublicAPIEndpoint() {
        return (this.apiEndpoint ?? (await this.determinePublicAPIEndpoint()).apiEndpoint);
    }
    async getDeployedIndexId() {
        return (this.deployedIndexId ??
            (await this.determinePublicAPIEndpoint()).deployedIndexId);
    }
    static async fromTexts(texts, metadatas, embeddings, dbConfig) {
        const docs = texts.map((text, index) => ({
            pageContent: text,
            metadata: Array.isArray(metadatas) ? metadatas[index] : metadatas,
        }));
        return this.fromDocuments(docs, embeddings, dbConfig);
    }
    static async fromDocuments(docs, embeddings, dbConfig) {
        const ret = new MatchingEngine(embeddings, dbConfig);
        await ret.addDocuments(docs);
        return ret;
    }
}
exports.MatchingEngine = MatchingEngine;