agsamantha/node_modules/js-tiktoken/dist/ranks/cl100k_base.cjs

1 line
1 MiB
JavaScript
Raw Normal View History

2024-10-02 15:15:21 -05:00
module.exports = {"pat_str":"('s|'S|'t|'T|'re|'rE|'Re|'RE|'ve|'vE|'Ve|'VE|'m|'M|'ll|'lL|'Ll|'LL|'d|'D)|[^\\r\\n\\p{L}\\p{N}]?\\p{L}+|\\p{N}{1,3}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+","special_tokens":{"<|endoftext|>":100257,"<|fim_prefix|>":100258,"<|fim_middle|>":100259,"<|fim_suffix|>":100260,"<|endofprompt|>":100276},"bpe_ranks":"! 0 IQ== Ig== Iw== JA== JQ== Jg== Jw== KA== KQ== Kg== Kw== LA== LQ== Lg== Lw== MA== MQ== Mg== Mw== NA== NQ== Ng== Nw== OA== OQ== Og== Ow== PA== PQ== Pg== Pw== QA== QQ== Qg== Qw== RA== RQ== Rg== Rw== SA== SQ== Sg== Sw== TA== TQ== Tg== Tw== UA== UQ== Ug== Uw== VA== VQ== Vg== Vw== WA== WQ== Wg== Ww== XA== XQ== Xg== Xw== YA== YQ== Yg== Yw== ZA== ZQ== Zg== Zw== aA== aQ== ag== aw== bA== bQ== bg== bw== cA== cQ== cg== cw== dA== dQ== dg== dw== eA== eQ== eg== ew== fA== fQ== fg== oQ== og== ow== pA== pQ== pg== pw== qA== qQ== qg== qw== rA== rg== rw== sA== sQ== sg== sw== tA== tQ== tg== tw== uA== uQ== ug== uw== vA== vQ== vg== vw== wA== wQ== wg== ww== xA== xQ== xg== xw== yA== yQ== yg== yw== zA== zQ== zg== zw== 0A== 0Q== 0g== 0w== 1A== 1Q== 1g== 1w== 2A== 2Q== 2g== 2w== 3A== 3Q== 3g== 3w== 4A== 4Q== 4g== 4w== 5A== 5Q== 5g== 5w== 6A== 6Q== 6g== 6w== 7A== 7Q== 7g== 7w== 8A== 8Q== 8g== 8w== 9A== 9Q== 9g== 9w== +A== +Q== +g== +w== /A== /Q== /g== /w== AA== AQ== Ag== Aw== BA== BQ== Bg== Bw== CA== CQ== Cg== Cw== DA== DQ== Dg== Dw== EA== EQ== Eg== Ew== FA== FQ== Fg== Fw== GA== GQ== Gg== Gw== HA== HQ== Hg== Hw== IA== fw== gA== gQ== gg== gw== hA== hQ== hg== hw== iA== iQ== ig== iw== jA== jQ== jg== jw== kA== kQ== kg== kw== lA== lQ== lg== lw== mA== mQ== mg== mw== nA== nQ== ng== nw== oA== rQ== ICA= ICAgIA== aW4= IHQ= ICAgICAgICA= ZXI= ICAg b24= IGE= cmU= YXQ= c3Q= ZW4= b3I= IHRo Cgo= IGM= bGU= IHM= aXQ= YW4= YXI= YWw= IHRoZQ== Owo= IHA= IGY= b3U= ID0= aXM= ICAgICAgIA== aW5n ZXM= IHc= aW9u ZWQ= aWM= IGI= IGQ= ZXQ= IG0= IG8= CQk= cm8= YXM= ZWw= Y3Q= bmQ= IGlu IGg= ZW50 aWQ= IG4= YW0= ICAgICAgICAgICA= IHRv IHJl LS0= IHs= IG9m b20= KTsK aW0= DQo= ICg= aWw= Ly8= IGFuZA== dXI= c2U= IGw= ZXg= IFM= YWQ= ICI= Y2g= dXQ= aWY= Kio= IH0= ZW0= b2w= ICAgICAgICAgICAgICAgIA== dGg= KQo= IHsK IGc= aWc= aXY= LAo= Y2U= b2Q= IHY= YXRl IFQ= YWc= YXk= ICo= b3Q= dXM= IEM= IHN0 IEk= dW4= dWw= dWU= IEE= b3c= ICc= ZXc= IDw= YXRpb24= KCk= IGZvcg== YWI= b3J0 dW0= YW1l IGlz cGU= dHI= Y2s= 4oA= IHk= aXN0 LS0tLQ== LgoK aGU= IGU= bG8= IE0= IGJl ZXJz IG9u IGNvbg== YXA= dWI= IFA= ICAgICAgICAgICAgICAg YXNz aW50 Pgo= bHk= dXJu ICQ= OwoK YXY= cG9ydA== aXI= LT4= bnQ= Y3Rpb24= ZW5k IGRl MDA= aXRo b3V0 dHVybg== b3Vy ICAgICA= bGlj cmVz cHQ= PT0= IHRoaXM= IHdo IGlm IEQ= dmVy YWdl IEI= aHQ= ZXh0 PSI= IHRoYXQ= KioqKg== IFI= IGl0 ZXNz IEY= IHI= b3M= YW5k IGFz ZWN0 a2U= cm9t IC8v Y29u IEw= KCI= cXU= bGFzcw== IHdpdGg= aXo= ZGU= IE4= IGFs b3A= dXA= Z2V0 IH0K aWxl IGFu YXRh b3Jl cmk= IHBybw== Ow0K CQkJCQ== dGVy YWlu IFc= IEU= IGNvbQ== IHJldHVybg== YXJ0 IEg= YWNr aW1wb3J0 dWJsaWM= IG9y ZXN0 bWVudA== IEc= YWJsZQ== IC0= aW5l aWxs aW5k ZXJl Ojo= aXR5 ICs= IHRy ZWxm aWdodA== KCc= b3Jt dWx0 c3Ry Li4= Iiw= IHlvdQ== eXBl cGw= IG5ldw== IGo= ICAgICAgICAgICAgICAgICAgIA== IGZyb20= IGV4 IE8= MjA= bGQ= IFs= b2M= Ogo= IHNl IGxl LS0tLS0tLS0= LnM= ewo= Jyw= YW50 IGF0 YXNl LmM= IGNo PC8= YXZl YW5n IGFyZQ== IGludA== 4oCZ X3Q= ZXJ0 aWFs YWN0 fQo= aXZl b2Rl b3N0 IGNsYXNz IG5vdA== b2c= b3Jk YWx1ZQ== YWxs ZmY= KCk7Cg== b250 aW1l YXJl IFU= IHBy IDo= aWVz aXpl dXJl IGJ5 aXJl IH0KCg== LnA= IHNo aWNl YXN0 cHRpb24= dHJpbmc= b2s= X18= Y2w= IyM= IGhl YXJk KS4= IEA= aWV3 CQkJ IHdhcw== aXA= dGhpcw== IHU= IFRoZQ== aWRl YWNl aWI= YWM= cm91 IHdl amVjdA== IHB1YmxpYw== YWs= dmU= YXRo b2lk ID0+ dXN0 cXVl IHJlcw== KSk= J3M= IGs= YW5z eXN0 dW5jdGlvbg== KioqKioqKio= IGk= IHVz cHA= MTA= b25l YWls PT09PQ== bmFtZQ== IHN0cg== IC8= ICY= YWNo ZGl2 eXN0ZW0= ZWxs IGhhdmU= ZXJy b3VsZA== dWxs cG9u IEo= X3A= ID09 aWdu U3Q= Lgo= IHBs KTsKCg== Zm9ybQ== cHV0 b3VudA== fQoK ZGQ= aXRl IGdldA== cnI= b21l IOKA YXJhbQ== Y2M= ICov RVI= SW4= bGVz X3M= b25n aWU= IGNhbg== IFY= ZXJ2 cHI= IHVu cm93 YmVy IGRv bGw= IGVs IHNlbGY= YXRlZA== YXJ5 IC4= J10= dWQ= IGVu IFRo ICAgICAgICAgICAgICAgICAgICAgICA= dGU= X2M= dWN0