Source-code-tokenizer NPM

Source code tokenizer powerd by vscode

Usage

Install

npm install -g source-code-tokenizer

Make sample source code

console.log("Hello World");

Execute Tokenizer

tokenizer target.js

1:1:8: console
[ 'source.js', 'meta.function-call.js', 'support.class.console.js' ]
1:8:9: .
[ 'source.js', 'meta.function-call.js', 'punctuation.accessor.js' ]
1:9:12: log
[ 'source.js', 'meta.function-call.js', 'support.function.console.js' ]
1:12:13: (
[ 'source.js', 'meta.brace.round.js' ]
1:13:14: "
[
  'source.js',
  'string.quoted.double.js',
  'punctuation.definition.string.begin.js'
]
1:14:25: Hello World
[ 'source.js', 'string.quoted.double.js' ]
1:25:26: "
[
  'source.js',
  'string.quoted.double.js',
  'punctuation.definition.string.end.js'
]
1:26:27: )
[ 'source.js', 'meta.brace.round.js' ]
1:27:28: ;
[ 'source.js', 'punctuation.terminator.statement.js' ]