2.0.0 • Published 4 years ago
@kessler/tokenize v2.0.0
tokenize
personal tokenize util
example
npm i -S @kessler/tokenize
const tokenize = require('@kessler/tokenize')
let tokens1 = tokenize('text')
// with non default tokenization rules
const rules = {
tokenChars: ['(', ')', ',', '[', ']'],
globbingChars: ['"', '\''],
separators: [' ']
}
let tokens2 = tokenize('(text) "asd" blabla', rules)
license
MIT © Yaniv Kessler