@eklmv/jscpd-tokenizer v4.0.3

@eklmv/jscpd-tokenizer

tokenizer is package from @jscpd used for convert programming code to list of tokens

Installation

npm install @eklmv/jscpd-tokenizer --save

Usage

import {IOptions,  ITokensMap} from '@eklmv/jscpd-core';
import {Tokenizer} from '@eklmv/jscpd-tokenizer';

const tokenizer = new Tokenizer();
const options: IOptions = {};

const maps: ITokensMap[] = tokenizer.generateMaps('source_id', 'let a = "11"', 'javascript', options);

License

MIT © Andrey Kucherenko