agsamantha/node_modules/js-tiktoken/dist/ranks/gpt2.cjs

1 line
532 KiB
JavaScript
Raw Normal View History

2024-10-02 15:15:21 -05:00
module.exports = {"explicit_n_vocab":50257,"pat_str":"'s|'t|'re|'ve|'m|'ll|'d| ?\\p{L}+| ?\\p{N}+| ?[^\\s\\p{L}\\p{N}]+|\\s+(?!\\S)|\\s+","special_tokens":{"<|endoftext|>":50256},"bpe_ranks":"! 0 IQ== Ig== Iw== JA== JQ== Jg== Jw== KA== KQ== Kg== Kw== LA== LQ== Lg== Lw== MA== MQ== Mg== Mw== NA== NQ== Ng== Nw== OA== OQ== Og== Ow== PA== PQ== Pg== Pw== QA== QQ== Qg== Qw== RA== RQ== Rg== Rw== SA== SQ== Sg== Sw== TA== TQ== Tg== Tw== UA== UQ== Ug== Uw== VA== VQ== Vg== Vw== WA== WQ== Wg== Ww== XA== XQ== Xg== Xw== YA== YQ== Yg== Yw== ZA== ZQ== Zg== Zw== aA== aQ== ag== aw== bA== bQ== bg== bw== cA== cQ== cg== cw== dA== dQ== dg== dw== eA== eQ== eg== ew== fA== fQ== fg== oQ== og== ow== pA== pQ== pg== pw== qA== qQ== qg== qw== rA== rg== rw== sA== sQ== sg== sw== tA== tQ== tg== tw== uA== uQ== ug== uw== vA== vQ== vg== vw== wA== wQ== wg== ww== xA== xQ== xg== xw== yA== yQ== yg== yw== zA== zQ== zg== zw== 0A== 0Q== 0g== 0w== 1A== 1Q== 1g== 1w== 2A== 2Q== 2g== 2w== 3A== 3Q== 3g== 3w== 4A== 4Q== 4g== 4w== 5A== 5Q== 5g== 5w== 6A== 6Q== 6g== 6w== 7A== 7Q== 7g== 7w== 8A== 8Q== 8g== 8w== 9A== 9Q== 9g== 9w== +A== +Q== +g== +w== /A== /Q== /g== /w== AA== AQ== Ag== Aw== BA== BQ== Bg== Bw== CA== CQ== Cg== Cw== DA== DQ== Dg== Dw== EA== EQ== Eg== Ew== FA== FQ== Fg== Fw== GA== GQ== Gg== Gw== HA== HQ== Hg== Hw== IA== fw== gA== gQ== gg== gw== hA== hQ== hg== hw== iA== iQ== ig== iw== jA== jQ== jg== jw== kA== kQ== kg== kw== lA== lQ== lg== lw== mA== mQ== mg== mw== nA== nQ== ng== nw== oA== rQ== IHQ= IGE= aGU= aW4= cmU= b24= IHRoZQ== ZXI= IHM= YXQ= IHc= IG8= ZW4= IGM= aXQ= aXM= YW4= b3I= ZXM= IGI= ZWQ= IGY= aW5n IHA= b3U= IGFu YWw= YXI= IHRv IG0= IG9m IGlu IGQ= IGg= IGFuZA== aWM= YXM= bGU= IHRo aW9u b20= bGw= ZW50 IG4= IGw= c3Q= IHJl dmU= IGU= cm8= bHk= IGJl IGc= IFQ= Y3Q= IFM= aWQ= b3Q= IEk= dXQ= ZXQ= IEE= IGlz IG9u aW0= YW0= b3c= YXk= YWQ= c2U= IHRoYXQ= IEM= aWc= IGZvcg== YWM= IHk= dmVy dXI= IHU= bGQ= IHN0 IE0= J3M= IGhl IGl0 YXRpb24= aXRo aXI= Y2U= IHlvdQ== aWw= IEI= IHdo b2w= IFA= IHdpdGg= IDE= dGVy Y2g= IGFz IHdl ICg= bmQ= aWxs IEQ= aWY= IDI= YWc= ZXJz a2U= ICI= IEg= ZW0= IGNvbg== IFc= IFI= aGVy IHdhcw== IHI= b2Q= IEY= dWw= YXRl IGF0 cmk= cHA= b3Jl IFRoZQ== IHNl dXM= IHBybw== IGhh dW0= IGFyZQ== IGRl YWlu YW5k IG9y aWdo ZXN0 aXN0 YWI= cm9t IE4= dGg= IGNvbQ== IEc= dW4= b3A= MDA= IEw= IG5vdA== ZXNz IGV4 IHY= cmVz IEU= ZXc= aXR5 YW50 IGJ5 ZWw= b3M= b3J0 b2M= cXU= IGZyb20= IGhhdmU= IHN1 aXZl b3VsZA== IHNo IHRoaXM= bnQ= cmE= cGU= aWdodA== YXJ0 bWVudA== IGFs dXN0 ZW5k LS0= YWxs IE8= YWNr IGNo IGxl aWVz cmVk YXJk 4oA= b3V0 IEo= IGFi ZWFy aXY= YWxseQ== b3Vy b3N0 Z2g= cHQ= IHBs YXN0 IGNhbg== YWs= b21l dWQ= VGhl IGhpcw== IGRv IGdv IGhhcw== Z2U= J3Q= IFU= cm91 IHNh IGo= IGJ1dA== IHdvcg== IGFsbA== ZWN0 IGs= YW1l IHdpbGw= b2s= IHdoZQ== IHRoZXk= aWRl MDE= ZmY= aWNo cGw= dGhlcg== IHRy Li4= IGludA== aWU= dXJl YWdl IG5l aWFs YXA= aW5l aWNl IG1l IG91dA== YW5z b25l b25n aW9ucw== IHdobw== IEs= IHVw IHRoZWly IGFk IDM= IHVz YXRlZA== b3Vz IG1vcmU= dWU= b2c= IFN0 aW5k aWtl IHNv aW1l cGVy LiI= YmVy aXo= YWN0 IG9uZQ== IHNhaWQ= IC0= YXJl IHlvdXI= Y2M= IFRo IGNs ZXA= YWtl YWJsZQ== aXA= IGNvbnQ= IHdoaWNo aWE= IGlt IGFib3V0 IHdlcmU= dmVyeQ== dWI= IGhhZA== IGVu IGNvbXA= LCI= IElu IHVu IGFn aXJl YWNl YXU= YXJ5 IHdvdWxk YXNz cnk= IOKA Y2w= b29r ZXJl c28= IFY= aWdu aWI= IG9mZg== IHRl dmVu IFk= aWxl b3Nl aXRl b3Jt IDIwMQ== IHJlcw== IG1hbg== IHBlcg== IG90aGVy b3Jk dWx0 IGJlZW4= IGxpa2U= YXNl YW5jZQ== a3M= YXlz b3du ZW5jZQ== IGRpcw== Y3Rpb24= IGFueQ== IGFwcA== IHNw aW50 cmVzcw== YXRpb25z YWls IDQ= aWNhbA== IHRoZW0= IGhlcg== b3VudA== IENo IGFy IGlm IHRoZXJl IHBl IHllYXI= YXY= IG15 IHNvbWU= IHdoZW4= b3VnaA== YWNo IHRoYW4= cnU= b25k aWNr IG92ZXI= dmVs IHF1 Cgo= IHNj cmVhdA== cmVl IEl0 b3VuZA== cG9ydA== IGFsc28= IHBhcnQ= ZnRlcg== IGtu IGJlYw== IHRpbWU= ZW5z IDU= b3BsZQ== IHdoYXQ= IG5v ZHU= bWVy YW5n IG5ldw== LS0tLQ== IGdldA== b3J5 aXRpb24= aW5ncw== IGp1c3Q= IGludG8= IDA= ZW50cw== b3Zl dGU= IHBlb3BsZQ== IHByZQ== IGl0cw== IHJlYw== IHR3 aWFu aXJzdA== YXJr b3Jz IHdvcms= YWRl b2I= IHNoZQ== IG91cg== d24= aW5r bGlj IDE5 IEhl aXNo bmRlcg== YXVzZQ== IGhpbQ== b25z IFs= IHJv Zm9ybQ== aWxk YXRlcw== dmVycw== IG9u