1.0.19 • Published 8 months ago

js-tiktoken v1.0.19

Weekly downloads
-
License
MIT
Repository
github
Last release
8 months ago

⏳ js-tiktoken

tiktoken is a BPE tokeniser for use with OpenAI's models. This is a pure JS port of the original tiktoken library.

Install the library from NPM:

npm install js-tiktoken

Lite

You can only load the ranks you need, which will significantly reduce the bundle size:

import { Tiktoken } from "js-tiktoken/lite";
import o200k_base from "js-tiktoken/ranks/o200k_base";

const enc = new Tiktoken(o200k_base);
assert(enc.decode(enc.encode("hello world")) === "hello world");

Alternatively, encodings can be loaded dynamically from our CDN hosted on Cloudflare Pages.

import { Tiktoken } from "js-tiktoken/lite";

const res = await fetch(`https://tiktoken.pages.dev/js/o200k_base.json`);
const o200k_base = await res.json();

const enc = new Tiktoken(o200k_base);
assert(enc.decode(enc.encode("hello world")) === "hello world");

Full usage

If you need all the OpenAI tokenizers, you can import the entire library:

!CAUTION This will include all the OpenAI tokenizers, which may significantly increase the bundle size. See

import assert from "node:assert";
import { getEncoding, encodingForModel } from "js-tiktoken";

const enc = getEncoding("gpt2");
assert(enc.decode(enc.encode("hello world")) === "hello world");
@dingyi222666/chathub-llm-core@everything-registry/sub-chunk-1978@chainlit/clientcustom-gpt-usertalkpilesparkai-jssparkaiv2-jssparkaiv3-jssparkaiv4-jssparkaiv5-jssparkaiv6-jssparkaiv7-jsjoplin-plugin-jarvislanggraph-hitllangchainlangchain-corelangchain-edge-fixi18next-ai-keyscanhalbotlightrag-jslitellmllama-flowllamaflowjslocale-file-managerllm-apillm-api-clientllmscubiellmosaickoishi-plugin-chatluna@wfatec/chatgpt@unit-mesh/autodev@traceloop/instrumentation-openai@traceloop/instrumentation-together@unsync/ai-tools@vm-x-ai/completion-provider@zenghongtu/chatgpt@zhengxs/langchain-tools@zhengxs/wechaty-plugin-assistantandy-chatgptadeptus@zhujunsan/chatgpt-apiagent-formationagent-dexterabccorea-jsabcaskcorea-jsagnaiai-utils.jsai-dumperai-digestadccorea-jsadcorea-jsai-jsxalinagptaskopsbcorea-jsaskopscorea-js0xble@cli-upkaran/dataprep-core@graviton-inc/core@gritlab/core@hanzo/openai@ai-jsx/openai@ai16z/eliza@earnkit-ai-sdk/runtime-env@aeonsule/flow-insight@cognum-factory/langchain@coderabbitai/openai-chat-tokens@codebit-programando-solucoes/llamaindex@aisynapze/traces-typescript-sdk@aisenatus/core@continuedev/core@colibryx/langchain@copilotkit/backendfastgpt-global-xyqsparkaiv8-jssosc-ai-testingty-sdktranslation-agenttokensizets-chat@basproul/core@chatgptweb/chatgpt-api@chat-bot/core@chat1/chatgpt@chiefry/chatgpt@chatluna/coreyilangchainyhh_chatgptunitokenuniversal-llm-completionundetermini@langchain/textsplitters@llamaindex/edge@logicleai/llmosaic@langtail/openai-chat-tokens@langtrase/trace-attributes@langtrase/typescript-sdk@langchain/core@langchain/openai@langchain/azure-openai@leikeduntech/leiai-js
1.0.19

8 months ago

1.0.18

9 months ago

1.0.17

9 months ago

1.0.16

10 months ago

1.0.15

1 year ago

1.0.14

1 year ago

1.0.13

1 year ago

1.0.12

1 year ago

1.0.11

2 years ago

1.0.9

2 years ago

1.0.10

2 years ago

1.0.8

2 years ago

1.0.2

2 years ago

1.0.1

2 years ago

1.0.7

2 years ago

1.0.6

2 years ago

1.0.5

2 years ago

1.0.4

2 years ago

1.0.3

2 years ago

1.0.0

2 years ago