String Tokenization

Function to convert a string into an array of its tokens.

Install

npm i string-tokenization

usage

import { tokenize } from 'string-tokenization'

// Tokenize a string
tokenize('Hello world! This is an example.', false);
// Output: ["hello", "world!", "this", "is", "an", "example."]

// Also tokenize individual words in a string
tokenize('Hello world! This is an example.', true);
// Output: [
"h", "he", "o", "hel", "lo", "hell", "llo", "hello",
"ello", "", "w", "wo","!", "wor", "d!", "worl",
"ld!", "world", "rld!", "world!","orld!", "t", "th",
"s","thi", "is", "this", "his","i", "a", "an", "n",
"e", "ex", ".", "exa","e.",  "exam", "le.", "examp",
"ple.", "exampl", "mple.", "example","ample.", "example.", "xample."
]