import { encodingForModel } from "js-tiktoken";
export var Tokenizers;
(function(Tokenizers) {
    Tokenizers["CL100K_BASE"] = "cl100k_base";
})(Tokenizers || (Tokenizers = {}));
/**
 * @internal Helper class singleton
 */ class GlobalsHelper {
    defaultTokenizer;
    constructor(){
        const encoding = encodingForModel("text-embedding-ada-002"); // cl100k_base
        this.defaultTokenizer = {
            encode: (text)=>{
                return new Uint32Array(encoding.encode(text));
            },
            decode: (tokens)=>{
                const numberArray = Array.from(tokens);
                const text = encoding.decode(numberArray);
                const uint8Array = new TextEncoder().encode(text);
                return new TextDecoder().decode(uint8Array);
            }
        };
    }
    tokenizer(encoding) {
        if (encoding && encoding !== "cl100k_base") {
            throw new Error(`Tokenizer encoding ${encoding} not yet supported`);
        }
        return this.defaultTokenizer.encode.bind(this.defaultTokenizer);
    }
    tokenizerDecoder(encoding) {
        if (encoding && encoding !== "cl100k_base") {
            throw new Error(`Tokenizer encoding ${encoding} not yet supported`);
        }
        return this.defaultTokenizer.decode.bind(this.defaultTokenizer);
    }
}
export const globalsHelper = new GlobalsHelper();
