Created mappings

2 files changed, 2124 insertions, 4 deletions

diff --git a/notebooks/03-look-at-iam-paragraphs.ipynb b/notebooks/03-look-at-iam-paragraphs.ipynb
index cfa0ba5..7599e11 100644
--- a/notebooks/03-look-at-iam-paragraphs.ipynb
+++ b/notebooks/03-look-at-iam-paragraphs.ipynb
@@ -2,7 +2,7 @@
  "cells": [
   {
    "cell_type": "code",
-   "execution_count": 1,
+   "execution_count": 2,
    "id": "6ce2519f",
    "metadata": {},
    "outputs": [],
@@ -212,7 +212,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 5,
+   "execution_count": 3,
    "id": "45649194",
    "metadata": {},
    "outputs": [],
@@ -223,7 +223,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 6,
+   "execution_count": 11,
    "id": "0fc13f9f",
    "metadata": {},
    "outputs": [],
@@ -241,6 +241,1105 @@
   },
   {
    "cell_type": "code",
+   "execution_count": 12,
+   "id": "c70e8873",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "['!',\n",
+       " '\"',\n",
+       " '&',\n",
+       " \"'\",\n",
+       " '(',\n",
+       " ')',\n",
+       " '*',\n",
+       " '+',\n",
+       " ',',\n",
+       " '-',\n",
+       " '.',\n",
+       " '/',\n",
+       " '0',\n",
+       " '1',\n",
+       " '2',\n",
+       " '3',\n",
+       " '4',\n",
+       " '5',\n",
+       " '6',\n",
+       " '7',\n",
+       " '8',\n",
+       " '9',\n",
+       " ':',\n",
+       " ';',\n",
+       " '?',\n",
+       " 'a',\n",
+       " 'b',\n",
+       " 'c',\n",
+       " 'd',\n",
+       " 'e',\n",
+       " 'f',\n",
+       " 'g',\n",
+       " 'h',\n",
+       " 'i',\n",
+       " 'j',\n",
+       " 'k',\n",
+       " 'l',\n",
+       " 'm',\n",
+       " 'n',\n",
+       " 'o',\n",
+       " 'p',\n",
+       " 'q',\n",
+       " 'r',\n",
+       " 's',\n",
+       " 't',\n",
+       " 'u',\n",
+       " 'v',\n",
+       " 'w',\n",
+       " 'x',\n",
+       " 'y',\n",
+       " 'z',\n",
+       " '▁',\n",
+       " '<s>',\n",
+       " '<e>',\n",
+       " '<p>',\n",
+       " '\\n']"
+      ]
+     },
+     "execution_count": 12,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "processor.graphemes"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 13,
+   "id": "6def52f6",
+   "metadata": {
+    "scrolled": true
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "['/',\n",
+       " '▁',\n",
+       " 's',\n",
+       " '▁the',\n",
+       " ',',\n",
+       " '▁.',\n",
+       " '▁a',\n",
+       " 'e',\n",
+       " '▁to',\n",
+       " 'ed',\n",
+       " '▁of',\n",
+       " '▁in',\n",
+       " 'ing',\n",
+       " 'y',\n",
+       " '▁and',\n",
+       " 't',\n",
+       " 'm',\n",
+       " 'd',\n",
+       " '▁be',\n",
+       " 'at',\n",
+       " 'a',\n",
+       " 'n',\n",
+       " '▁th',\n",
+       " 'i',\n",
+       " '▁he',\n",
+       " 'p',\n",
+       " 'o',\n",
+       " 'en',\n",
+       " 'al',\n",
+       " '▁is',\n",
+       " '-',\n",
+       " 'ly',\n",
+       " '.',\n",
+       " '▁w',\n",
+       " 'c',\n",
+       " '▁for',\n",
+       " 'l',\n",
+       " '\"',\n",
+       " '▁it',\n",
+       " 'er',\n",
+       " '▁was',\n",
+       " 'u',\n",
+       " '▁on',\n",
+       " '▁with',\n",
+       " \"'\",\n",
+       " '▁his',\n",
+       " 'g',\n",
+       " 'h',\n",
+       " '▁had',\n",
+       " '▁mr',\n",
+       " 'in',\n",
+       " '▁i',\n",
+       " 'or',\n",
+       " '▁f',\n",
+       " 'r',\n",
+       " \"▁'\",\n",
+       " '▁but',\n",
+       " '▁by',\n",
+       " 'ar',\n",
+       " '▁as',\n",
+       " '▁not',\n",
+       " 'b',\n",
+       " 'it',\n",
+       " 'ir',\n",
+       " 'v',\n",
+       " '▁re',\n",
+       " 'st',\n",
+       " '▁an',\n",
+       " '▁has',\n",
+       " '▁are',\n",
+       " 'this',\n",
+       " 'f',\n",
+       " 'on',\n",
+       " 'hich',\n",
+       " 'ce',\n",
+       " '▁who',\n",
+       " '▁have',\n",
+       " 'ion',\n",
+       " 'ch',\n",
+       " '▁p',\n",
+       " 're',\n",
+       " '▁at',\n",
+       " '▁de',\n",
+       " '▁m',\n",
+       " 'w',\n",
+       " 'le',\n",
+       " 'ri',\n",
+       " '▁so',\n",
+       " '▁fro',\n",
+       " 'ation',\n",
+       " 'ic',\n",
+       " '▁do',\n",
+       " 'ould',\n",
+       " '▁will',\n",
+       " '▁b',\n",
+       " 'k',\n",
+       " 'is',\n",
+       " 'ment',\n",
+       " 'il',\n",
+       " '▁st',\n",
+       " '▁no',\n",
+       " 'an',\n",
+       " 'ter',\n",
+       " 'se',\n",
+       " '▁we',\n",
+       " 'ur',\n",
+       " 'ies',\n",
+       " '▁un',\n",
+       " '▁her',\n",
+       " 'th',\n",
+       " 'es',\n",
+       " 'ent',\n",
+       " 'et',\n",
+       " '▁-',\n",
+       " '▁all',\n",
+       " '▁c',\n",
+       " '▁there',\n",
+       " 'un',\n",
+       " 've',\n",
+       " '▁can',\n",
+       " 'ne',\n",
+       " '▁up',\n",
+       " '▁one',\n",
+       " '▁were',\n",
+       " '▁me',\n",
+       " '▁br',\n",
+       " '▁com',\n",
+       " '▁more',\n",
+       " '▁se',\n",
+       " 'ro',\n",
+       " '▁per',\n",
+       " 'ld',\n",
+       " 'ge',\n",
+       " '▁man',\n",
+       " 'est',\n",
+       " '▁pa',\n",
+       " 'ul',\n",
+       " 'el',\n",
+       " 'if',\n",
+       " '▁government',\n",
+       " '▁or',\n",
+       " '▁she',\n",
+       " 'ill',\n",
+       " '▁co',\n",
+       " '▁year',\n",
+       " 'ad',\n",
+       " '▁bo',\n",
+       " 'te',\n",
+       " '▁than',\n",
+       " '▁r',\n",
+       " 'll',\n",
+       " 'able',\n",
+       " 'ous',\n",
+       " '▁what',\n",
+       " '▁out',\n",
+       " 'ck',\n",
+       " '!',\n",
+       " '▁new',\n",
+       " '▁dis',\n",
+       " 'lo',\n",
+       " 'ra',\n",
+       " 'uch',\n",
+       " '▁him',\n",
+       " 'age',\n",
+       " 'ct',\n",
+       " '▁go',\n",
+       " '▁con',\n",
+       " 'day',\n",
+       " 'z',\n",
+       " 'op',\n",
+       " 'ther',\n",
+       " '▁you',\n",
+       " '▁dr',\n",
+       " 'us',\n",
+       " '▁1',\n",
+       " '▁may',\n",
+       " 'ers',\n",
+       " ':',\n",
+       " '▁first',\n",
+       " 'li',\n",
+       " '0',\n",
+       " 'end',\n",
+       " 'ng',\n",
+       " '▁people',\n",
+       " '▁some',\n",
+       " 'our',\n",
+       " '▁anglesey',\n",
+       " '▁two',\n",
+       " '▁wh',\n",
+       " 'tion',\n",
+       " '▁lo',\n",
+       " '▁sir',\n",
+       " 'ig',\n",
+       " 'ain',\n",
+       " 'tic',\n",
+       " '▁now',\n",
+       " 'ive',\n",
+       " 'and',\n",
+       " '▁whe',\n",
+       " ';',\n",
+       " 'ry',\n",
+       " 'um',\n",
+       " '▁other',\n",
+       " '▁su',\n",
+       " '▁ho',\n",
+       " '▁ha',\n",
+       " '▁ra',\n",
+       " '▁even',\n",
+       " 'ish',\n",
+       " 'ard',\n",
+       " 'wn',\n",
+       " '▁li',\n",
+       " 'ten',\n",
+       " 'ok',\n",
+       " '▁k',\n",
+       " 'ver',\n",
+       " 'into',\n",
+       " '▁na',\n",
+       " '▁vi',\n",
+       " '▁po',\n",
+       " '▁common',\n",
+       " 'ight',\n",
+       " '▁over',\n",
+       " '▁like',\n",
+       " '▁mad',\n",
+       " 'ud',\n",
+       " 'ow',\n",
+       " '▁any',\n",
+       " '▁again',\n",
+       " '▁great',\n",
+       " '▁last',\n",
+       " 'ough',\n",
+       " 'ose',\n",
+       " 'ant',\n",
+       " '▁under',\n",
+       " 'ful',\n",
+       " 'self',\n",
+       " '▁how',\n",
+       " '▁sai',\n",
+       " 'x',\n",
+       " '▁most',\n",
+       " 'ine',\n",
+       " '▁give',\n",
+       " '▁about',\n",
+       " 'ance',\n",
+       " 'me',\n",
+       " 'fore',\n",
+       " 'cu',\n",
+       " '(',\n",
+       " '▁minister',\n",
+       " '▁labour',\n",
+       " '▁talk',\n",
+       " '▁work',\n",
+       " '▁take',\n",
+       " '▁writ',\n",
+       " '▁whil',\n",
+       " '▁well',\n",
+       " '▁plan',\n",
+       " '▁la',\n",
+       " '▁pri',\n",
+       " '▁play',\n",
+       " '▁house',\n",
+       " 'mber',\n",
+       " '▁ex',\n",
+       " '▁good',\n",
+       " 'ern',\n",
+       " 'ple',\n",
+       " 'ble',\n",
+       " '▁german',\n",
+       " 'ak',\n",
+       " 'ever',\n",
+       " '▁time',\n",
+       " '▁pro',\n",
+       " '▁ga',\n",
+       " '▁germany',\n",
+       " '▁britain',\n",
+       " '▁should',\n",
+       " '▁polic',\n",
+       " '▁found',\n",
+       " '▁part',\n",
+       " 'ood',\n",
+       " '▁g',\n",
+       " '▁west',\n",
+       " 'thing',\n",
+       " '▁cur',\n",
+       " '▁know',\n",
+       " '▁back',\n",
+       " '▁ma',\n",
+       " '▁show',\n",
+       " ')',\n",
+       " 'ure',\n",
+       " 'ian',\n",
+       " 'app',\n",
+       " '▁ever',\n",
+       " '▁use',\n",
+       " 'ally',\n",
+       " 'led',\n",
+       " '▁miss',\n",
+       " '▁car',\n",
+       " '▁nation',\n",
+       " 'ttle',\n",
+       " '▁end',\n",
+       " '▁gr',\n",
+       " '▁must',\n",
+       " '▁tr',\n",
+       " '▁du',\n",
+       " '▁get',\n",
+       " '▁unit',\n",
+       " 'less',\n",
+       " '▁agree',\n",
+       " 'ie',\n",
+       " 'tom',\n",
+       " '..',\n",
+       " '▁di',\n",
+       " '▁men',\n",
+       " '4',\n",
+       " '▁my',\n",
+       " '▁today',\n",
+       " 'ves',\n",
+       " 'hap',\n",
+       " 'tory',\n",
+       " 'ence',\n",
+       " 'out',\n",
+       " '▁its',\n",
+       " '8',\n",
+       " '▁believe',\n",
+       " '▁say',\n",
+       " '▁make',\n",
+       " 'for',\n",
+       " '▁them',\n",
+       " '▁president',\n",
+       " '▁return',\n",
+       " '▁british',\n",
+       " '▁after',\n",
+       " '▁earl',\n",
+       " '▁here',\n",
+       " 'ton',\n",
+       " 'ext',\n",
+       " '▁life',\n",
+       " '▁put',\n",
+       " '▁war',\n",
+       " '▁2',\n",
+       " '▁cont',\n",
+       " '▁every',\n",
+       " '▁god',\n",
+       " '▁strong',\n",
+       " 'ren',\n",
+       " 'ty',\n",
+       " '▁mon',\n",
+       " 'man',\n",
+       " 'very',\n",
+       " '5',\n",
+       " '6',\n",
+       " '▁yester',\n",
+       " '▁week',\n",
+       " '▁film',\n",
+       " 'ual',\n",
+       " '▁need',\n",
+       " 'lso',\n",
+       " 'aid',\n",
+       " 'law',\n",
+       " 'ward',\n",
+       " '▁ask',\n",
+       " 'bo',\n",
+       " '▁pre',\n",
+       " '▁conference',\n",
+       " '▁real',\n",
+       " '▁roy',\n",
+       " 'ince',\n",
+       " '▁us',\n",
+       " 'ke',\n",
+       " '▁franc',\n",
+       " '▁direct',\n",
+       " '▁child',\n",
+       " 'cause',\n",
+       " '▁long',\n",
+       " '▁action',\n",
+       " '▁hope',\n",
+       " '▁live',\n",
+       " '▁see',\n",
+       " '▁point',\n",
+       " '▁char',\n",
+       " '▁way',\n",
+       " 'val',\n",
+       " '▁add',\n",
+       " '▁ba',\n",
+       " 'tt',\n",
+       " 'qui',\n",
+       " 'ism',\n",
+       " 'other',\n",
+       " '▁character',\n",
+       " '&',\n",
+       " '▁kennedy',\n",
+       " '▁macmillan',\n",
+       " '▁service',\n",
+       " '▁open',\n",
+       " '▁tri',\n",
+       " '▁country',\n",
+       " 'orn',\n",
+       " 'abl',\n",
+       " '▁off',\n",
+       " '▁fac',\n",
+       " '▁effective',\n",
+       " 'rote',\n",
+       " '▁let',\n",
+       " 'each',\n",
+       " '▁cent',\n",
+       " '▁party',\n",
+       " '▁act',\n",
+       " '▁prim',\n",
+       " '▁str',\n",
+       " 'amp',\n",
+       " '▁3',\n",
+       " '▁describe',\n",
+       " '▁europe',\n",
+       " '▁through',\n",
+       " '▁medic',\n",
+       " '▁home',\n",
+       " 'ug',\n",
+       " '▁african',\n",
+       " 'ind',\n",
+       " '▁prop',\n",
+       " 'by',\n",
+       " '▁thre',\n",
+       " 'ndon',\n",
+       " 'ative',\n",
+       " '▁sit',\n",
+       " ',000',\n",
+       " '▁high',\n",
+       " '▁secret',\n",
+       " '▁night',\n",
+       " '▁fr',\n",
+       " 'health',\n",
+       " '▁call',\n",
+       " '▁north',\n",
+       " '▁mean',\n",
+       " '▁seem',\n",
+       " '▁lea',\n",
+       " 'tire',\n",
+       " 'lway',\n",
+       " 'ness',\n",
+       " '▁rhodesia',\n",
+       " '▁195',\n",
+       " '▁hand',\n",
+       " '▁federa',\n",
+       " '▁place',\n",
+       " 'round',\n",
+       " '▁far',\n",
+       " 'ship',\n",
+       " 'low',\n",
+       " '▁exa',\n",
+       " 'ip',\n",
+       " '▁change',\n",
+       " '▁patient',\n",
+       " '▁lord',\n",
+       " '▁arriv',\n",
+       " '▁trem',\n",
+       " '▁pear',\n",
+       " 'ring',\n",
+       " '▁near',\n",
+       " 'land',\n",
+       " 'ress',\n",
+       " '▁clo',\n",
+       " '▁did',\n",
+       " 'later',\n",
+       " 'ius',\n",
+       " 'tho',\n",
+       " 'well',\n",
+       " '7',\n",
+       " '▁enjoy',\n",
+       " '▁success',\n",
+       " '▁appear',\n",
+       " '▁personal',\n",
+       " '▁move',\n",
+       " '▁treat',\n",
+       " '▁better',\n",
+       " 'case',\n",
+       " '▁never',\n",
+       " '▁look',\n",
+       " 'ture',\n",
+       " 'way',\n",
+       " 'mark',\n",
+       " 'ought',\n",
+       " 'tu',\n",
+       " '▁hu',\n",
+       " 'twe',\n",
+       " '▁foreign',\n",
+       " '▁number',\n",
+       " '▁fall',\n",
+       " '▁month',\n",
+       " '▁young',\n",
+       " 'ound',\n",
+       " '▁find',\n",
+       " '▁american',\n",
+       " '▁thought',\n",
+       " '▁state',\n",
+       " 'ilit',\n",
+       " '▁cit',\n",
+       " '▁side',\n",
+       " '▁centr',\n",
+       " 'lace',\n",
+       " 'ash',\n",
+       " '▁old',\n",
+       " 'rome',\n",
+       " '▁subject',\n",
+       " '▁support',\n",
+       " '▁world',\n",
+       " '▁word',\n",
+       " 'different',\n",
+       " '▁report',\n",
+       " '1',\n",
+       " '▁tru',\n",
+       " '▁sent',\n",
+       " '▁best',\n",
+       " '▁symp',\n",
+       " '▁just',\n",
+       " '▁richards',\n",
+       " '▁produc',\n",
+       " '▁left',\n",
+       " '▁clear',\n",
+       " '▁power',\n",
+       " 'ber',\n",
+       " '▁few',\n",
+       " 'solut',\n",
+       " 'tack',\n",
+       " '▁right',\n",
+       " '▁mar',\n",
+       " '▁might',\n",
+       " '3',\n",
+       " 'not',\n",
+       " '▁full',\n",
+       " '▁union',\n",
+       " '▁base',\n",
+       " '▁head',\n",
+       " 'ealth',\n",
+       " 'many',\n",
+       " '▁happen',\n",
+       " '▁mauro',\n",
+       " 'vision',\n",
+       " '▁inter',\n",
+       " '▁ye',\n",
+       " 'where',\n",
+       " '▁sail',\n",
+       " '▁past',\n",
+       " '▁your',\n",
+       " '▁want',\n",
+       " '▁stor',\n",
+       " 'went',\n",
+       " '▁eas',\n",
+       " '▁rem',\n",
+       " '▁nuclear',\n",
+       " '▁small',\n",
+       " '▁discuss',\n",
+       " 'spect',\n",
+       " '▁girl',\n",
+       " '▁view',\n",
+       " '▁south',\n",
+       " '▁mod',\n",
+       " 'elect',\n",
+       " '▁usual',\n",
+       " 'evi',\n",
+       " 'peration',\n",
+       " '▁engl',\n",
+       " 'train',\n",
+       " '▁mind',\n",
+       " 'acle',\n",
+       " 'riv',\n",
+       " 'side',\n",
+       " 'aris',\n",
+       " '▁event',\n",
+       " '▁lei',\n",
+       " '▁favour',\n",
+       " '▁problem',\n",
+       " '▁large',\n",
+       " 'ordina',\n",
+       " '▁feel',\n",
+       " 'search',\n",
+       " '▁wife',\n",
+       " '▁remain',\n",
+       " 'ready',\n",
+       " 'stone',\n",
+       " '▁cla',\n",
+       " '▁stand',\n",
+       " '▁big',\n",
+       " '▁bus',\n",
+       " 'ox',\n",
+       " '▁four',\n",
+       " 'port',\n",
+       " '▁main',\n",
+       " '▁nor',\n",
+       " '▁va',\n",
+       " 'mann',\n",
+       " '▁pot',\n",
+       " 'rry',\n",
+       " '▁fan',\n",
+       " '▁fig',\n",
+       " '▁attempt',\n",
+       " '▁follow',\n",
+       " '▁particular',\n",
+       " '▁church',\n",
+       " '▁wonder',\n",
+       " '▁doctor',\n",
+       " '▁control',\n",
+       " '▁half',\n",
+       " '▁politic',\n",
+       " 'ciple',\n",
+       " '▁trans',\n",
+       " '▁extra',\n",
+       " '▁start',\n",
+       " '▁death',\n",
+       " '▁tell',\n",
+       " '▁eye',\n",
+       " '▁own',\n",
+       " '▁allow',\n",
+       " '▁offer',\n",
+       " '▁quit',\n",
+       " '▁easi',\n",
+       " 'ular',\n",
+       " '▁che',\n",
+       " '▁cri',\n",
+       " 'bit',\n",
+       " '▁bar',\n",
+       " 'ption',\n",
+       " 'rate',\n",
+       " '▁system',\n",
+       " 'crochet',\n",
+       " '▁certain',\n",
+       " '▁taste',\n",
+       " '▁method',\n",
+       " '▁course',\n",
+       " '▁leader',\n",
+       " 'million',\n",
+       " '▁19',\n",
+       " '▁fu',\n",
+       " '▁help',\n",
+       " 'lady',\n",
+       " 'super',\n",
+       " 'date',\n",
+       " '▁fact',\n",
+       " '▁amo',\n",
+       " '▁sho',\n",
+       " 'nson',\n",
+       " 'lin',\n",
+       " '2',\n",
+       " '▁advi',\n",
+       " 'mitt',\n",
+       " '▁line',\n",
+       " 'effect',\n",
+       " '▁school',\n",
+       " '▁doubt',\n",
+       " '▁consult',\n",
+       " '▁answer',\n",
+       " 'special',\n",
+       " '▁burde',\n",
+       " '▁room',\n",
+       " '▁probabl',\n",
+       " '▁delight',\n",
+       " '▁bank',\n",
+       " '▁six',\n",
+       " '▁brown',\n",
+       " '▁studie',\n",
+       " '▁human',\n",
+       " '▁committee',\n",
+       " 'cial',\n",
+       " '▁creat',\n",
+       " '▁distin',\n",
+       " '▁cut',\n",
+       " '▁mother',\n",
+       " 'morr',\n",
+       " 'ground',\n",
+       " '▁free',\n",
+       " '▁five',\n",
+       " '▁arm',\n",
+       " '▁pol',\n",
+       " 'done',\n",
+       " '▁late',\n",
+       " '▁basi',\n",
+       " 'nato',\n",
+       " '9',\n",
+       " '▁econom',\n",
+       " '▁opposit',\n",
+       " '▁public',\n",
+       " '▁chief',\n",
+       " '▁become',\n",
+       " '▁record',\n",
+       " '▁peace',\n",
+       " '▁consider',\n",
+       " '▁understand',\n",
+       " '▁numer',\n",
+       " '▁slight',\n",
+       " '▁turn',\n",
+       " '▁duke',\n",
+       " '▁homo',\n",
+       " '▁grow',\n",
+       " '▁grant',\n",
+       " '▁office',\n",
+       " 'msel',\n",
+       " 'ainful',\n",
+       " '▁famil',\n",
+       " '▁ob',\n",
+       " '▁advise',\n",
+       " '▁cr',\n",
+       " '▁love',\n",
+       " 'ches',\n",
+       " '▁tea',\n",
+       " 'tist',\n",
+       " 'ivate',\n",
+       " '▁america',\n",
+       " 'aff',\n",
+       " 'author',\n",
+       " '▁gaitskell',\n",
+       " '▁industry',\n",
+       " '▁john',\n",
+       " '▁mistake',\n",
+       " '▁pleasant',\n",
+       " '▁temperature',\n",
+       " '▁difficult',\n",
+       " '▁brief',\n",
+       " '▁defence',\n",
+       " '▁gentle',\n",
+       " '▁piece',\n",
+       " '▁argu',\n",
+       " '▁irrita',\n",
+       " '▁berlin',\n",
+       " '▁father',\n",
+       " '▁cult',\n",
+       " '▁despa',\n",
+       " '▁market',\n",
+       " '▁white',\n",
+       " '▁negotiation',\n",
+       " '▁energ',\n",
+       " '▁fear',\n",
+       " '▁english',\n",
+       " '▁tele',\n",
+       " '▁interest',\n",
+       " 'same',\n",
+       " 'body',\n",
+       " '▁chap',\n",
+       " '▁table',\n",
+       " '▁mass',\n",
+       " '▁press',\n",
+       " '▁spi',\n",
+       " 'lay',\n",
+       " 'hall',\n",
+       " 'most',\n",
+       " '▁journ',\n",
+       " '▁diploma',\n",
+       " '▁expect',\n",
+       " '▁1960',\n",
+       " '▁join',\n",
+       " '▁provide',\n",
+       " '▁russell',\n",
+       " '▁speak',\n",
+       " '▁decide',\n",
+       " '▁complete',\n",
+       " '▁senator',\n",
+       " '▁tax',\n",
+       " '▁earth',\n",
+       " '▁colour',\n",
+       " 'armament',\n",
+       " '▁sevent',\n",
+       " '▁learn',\n",
+       " '▁honey',\n",
+       " '▁militar',\n",
+       " '▁claim',\n",
+       " '▁chamb',\n",
+       " '▁master',\n",
+       " '▁peel',\n",
+       " '▁firm',\n",
+       " 'eason',\n",
+       " '▁trade',\n",
+       " '▁washing',\n",
+       " '▁statement',\n",
+       " 'range',\n",
+       " '▁rol',\n",
+       " 'guis',\n",
+       " 'hei',\n",
+       " 'ago',\n",
+       " '▁pla',\n",
+       " '▁cam',\n",
+       " 'eopa',\n",
+       " '▁africa',\n",
+       " '▁convention',\n",
+       " '▁council',\n",
+       " '▁decision',\n",
+       " '▁experience',\n",
+       " '▁increasing',\n",
+       " '▁majorit',\n",
+       " '▁suffer',\n",
+       " '▁suggest',\n",
+       " '▁position',\n",
+       " '▁resource',\n",
+       " '▁soldier',\n",
+       " 'temper',\n",
+       " '▁general',\n",
+       " '▁financi',\n",
+       " '▁express',\n",
+       " '▁short',\n",
+       " '▁compan',\n",
+       " 'field',\n",
+       " '▁think',\n",
+       " 'cross',\n",
+       " '▁odd',\n",
+       " '▁1830',\n",
+       " '▁reason',\n",
+       " '▁hard',\n",
+       " '▁fright',\n",
+       " '▁final',\n",
+       " '▁thread',\n",
+       " 'heart',\n",
+       " '▁nea',\n",
+       " '▁assu',\n",
+       " '▁step',\n",
+       " '▁countr',\n",
+       " 'did',\n",
+       " '▁infa',\n",
+       " 'struct',\n",
+       " '▁question',\n",
+       " 'advantage',\n",
+       " '▁atmospher',\n",
+       " '▁screw',\n",
+       " '▁square',\n",
+       " '▁welensky',\n",
+       " '▁cinema',\n",
+       " '▁weapon',\n",
+       " '▁importan',\n",
+       " '▁apart',\n",
+       " '▁prepar',\n",
+       " '▁payment',\n",
+       " '▁present',\n",
+       " '▁persever',\n",
+       " 'simpl',\n",
+       " '▁idea',\n",
+       " '▁expl',\n",
+       " '▁chance',\n",
+       " '▁fast',\n",
+       " '▁notice',\n",
+       " 'rade',\n",
+       " '▁giv',\n",
+       " '▁leave',\n",
+       " 'finger',\n",
+       " 'eppe',\n",
+       " '▁gla',\n",
+       " '▁psycholog',\n",
+       " '▁robertson',\n",
+       " '▁accept',\n",
+       " '▁improv',\n",
+       " '▁philip',\n",
+       " '▁technical',\n",
+       " '▁trouble',\n",
+       " '▁premier',\n",
+       " '▁progress',\n",
+       " 'minute',\n",
+       " '▁scot',\n",
+       " '▁couple',\n",
+       " '▁concern',\n",
+       " '▁schem',\n",
+       " '▁weaver',\n",
+       " '▁inform',\n",
+       " '▁cap',\n",
+       " 'stead',\n",
+       " '▁heav',\n",
+       " '▁propos',\n",
+       " '▁order',\n",
+       " 'ensure',\n",
+       " 'worth',\n",
+       " '▁former',\n",
+       " '▁sense',\n",
+       " '▁clu',\n",
+       " 'nder',\n",
+       " 'easing',\n",
+       " '▁possib',\n",
+       " 'ulate',\n",
+       " 'polit',\n",
+       " 'equa',\n",
+       " 'communist',\n",
+       " '▁adenauer',\n",
+       " '▁delaney',\n",
+       " '▁demonstrat',\n",
+       " '▁independen',\n",
+       " '▁welcome',\n",
+       " '▁speech',\n",
+       " '▁includ',\n",
+       " '▁oppose',\n",
+       " 'timate',\n",
+       " '▁picture',\n",
+       " '▁share',\n",
+       " '▁result',\n",
+       " '▁receiv',\n",
+       " '▁garden',\n",
+       " '▁appeal',\n",
+       " '▁chair',\n",
+       " '▁keep',\n",
+       " '▁regard',\n",
+       " '▁chines',\n",
+       " '▁sister',\n",
+       " '▁prince',\n",
+       " '▁scene',\n",
+       " '▁spend',\n",
+       " '▁congo',\n",
+       " '▁issue',\n",
+       " '▁authorit',\n",
+       " '▁memor',\n",
+       " '▁scar',\n",
+       " '▁moral',\n",
+       " '▁matter',\n",
+       " 'aught',\n",
+       " '▁wait',\n",
+       " 'street',\n",
+       " '▁concept',\n",
+       " '▁destructi',\n",
+       " 'sol',\n",
+       " 'committe',\n",
+       " 'house',\n",
+       " 'older',\n",
+       " 'ference',\n",
+       " '▁adjust',\n",
+       " '▁associat',\n",
+       " '▁campaign',\n",
+       " '▁conservative',\n",
+       " '▁immediate',\n",
+       " '▁industri',\n",
+       " '▁james',\n",
+       " '▁lucas',\n",
+       " '▁practic',\n",
+       " '▁publish',\n",
+       " '▁purpose',\n",
+       " '▁queer',\n",
+       " '▁representative',\n",
+       " '▁relief',\n",
+       " '▁detail',\n",
+       " '▁official',\n",
+       " '▁faith',\n",
+       " '▁design',\n",
+       " '▁sacr',\n",
+       " '▁household',\n",
+       " '▁variet',\n",
+       " '▁village',\n",
+       " '▁foot',\n",
+       " '▁russia',\n",
+       " '▁fifty',\n",
+       " '▁block',\n",
+       " '▁inevitab',\n",
+       " '▁prevent',\n",
+       " '▁belgi',\n",
+       " '▁type',\n",
+       " '▁cheap',\n",
+       " '▁protest',\n",
+       " '▁along',\n",
+       " '▁demand',\n",
+       " '▁earn',\n",
+       " '▁sick',\n",
+       " '▁eviden',\n",
+       " '▁someone',\n",
+       " '▁peers',\n",
+       " '▁suitabl',\n",
+       " '▁sympath',\n",
+       " 'brid',\n",
+       " '▁discr',\n",
+       " 'flu',\n",
+       " 'bomb',\n",
+       " 'rator',\n",
+       " 'coming',\n",
+       " 'famili',\n",
+       " '▁183',\n",
+       " 'employ',\n",
+       " '▁catherine',\n",
+       " '▁correspond',\n",
+       " '▁material',\n",
+       " '▁rubb',\n",
+       " '▁measure',\n",
+       " '▁sharp',\n",
+       " '▁wicki',\n",
+       " '▁marriage',\n",
+       " '▁size',\n",
+       " '▁liberal',\n",
+       " '▁level',\n",
+       " '▁surround',\n",
+       " '▁delegat',\n",
+       " 'script',\n",
+       " '▁suppl',\n",
+       " '▁reserve',\n",
+       " '▁refus',\n",
+       " '▁gift',\n",
+       " '▁promis',\n",
+       " '▁safe',\n",
+       " '▁exist',\n",
+       " '▁alto',\n",
+       " '▁wide',\n",
+       " '▁arrang',\n",
+       " '▁relations',\n",
+       " 'shab',\n",
+       " '▁intelligen',\n",
+       " 'emphasi',\n",
+       " '▁advertisement',\n",
+       " '▁beautiful',\n",
+       " '+',\n",
+       " '?',\n",
+       " 'q',\n",
+       " 'j',\n",
+       " ...]"
+      ]
+     },
+     "execution_count": 13,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "processor.tokens"
+   ]
+  },
+  {
+   "cell_type": "code",
    "execution_count": 8,
    "id": "fb0afccf",
    "metadata": {},
@@ -262,6 +1361,1027 @@
   },
   {
    "cell_type": "code",
+   "execution_count": 14,
+   "id": "fb600959",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "{'!': ['▁', '!'],\n",
+       " '\"': ['▁', '\"'],\n",
+       " '&': ['▁', '&'],\n",
+       " \"'\": [\"▁'\"],\n",
+       " \"'30s\": [\"▁'\", '3', '0', 's'],\n",
+       " \"'61\": [\"▁'\", '6', '1'],\n",
+       " \"'d\": [\"▁'\", 'd'],\n",
+       " \"'ll\": [\"▁'\", 'll'],\n",
+       " \"'m\": [\"▁'\", 'm'],\n",
+       " \"'re\": [\"▁'\", 're'],\n",
+       " \"'s\": [\"▁'\", 's'],\n",
+       " \"'ve\": [\"▁'\", 've'],\n",
+       " '(': ['▁', '('],\n",
+       " ')': ['▁', ')'],\n",
+       " '+2.8': ['▁', '+', '2', '.', '8'],\n",
+       " '+3.6': ['▁', '+', '3', '.', '6'],\n",
+       " ',': ['▁', ','],\n",
+       " '-': ['▁-'],\n",
+       " '-2.6': ['▁-', '2', '.', '6'],\n",
+       " '-5.4': ['▁-', '5', '.', '4'],\n",
+       " '.': ['▁.'],\n",
+       " '...': ['▁.', '..'],\n",
+       " '0m': ['▁', '0', 'm'],\n",
+       " '1': ['▁1'],\n",
+       " '1,157': ['▁1', ',', '1', '5', '7'],\n",
+       " '1.8': ['▁1', '.', '8'],\n",
+       " '1/2': ['▁1', '/', '2'],\n",
+       " '1/2-in.-long': ['▁1', '/', '2', '-', 'in', '.', '-', 'lo', 'ng'],\n",
+       " '1/4': ['▁1', '/', '4'],\n",
+       " '10': ['▁1', '0'],\n",
+       " '10,000': ['▁1', '0', ',000'],\n",
+       " '100': ['▁1', '0', '0'],\n",
+       " '100,000,000': ['▁1', '0', '0', ',000', ',000'],\n",
+       " '11': ['▁1', '1'],\n",
+       " '12': ['▁1', '2'],\n",
+       " '125': ['▁1', '2', '5'],\n",
+       " '13': ['▁1', '3'],\n",
+       " '13,000': ['▁1', '3', ',000'],\n",
+       " '14': ['▁1', '4'],\n",
+       " '15': ['▁1', '5'],\n",
+       " '15,000,000': ['▁1', '5', ',000', ',000'],\n",
+       " '15-17': ['▁1', '5', '-', '1', '7'],\n",
+       " '15-year-olds': ['▁1', '5', '-', 'y', 'e', 'ar', '-', 'o', 'ld', 's'],\n",
+       " '150,000,000': ['▁1', '5', '0', ',000', ',000'],\n",
+       " '16': ['▁1', '6'],\n",
+       " '160': ['▁1', '6', '0'],\n",
+       " '163,000,000': ['▁1', '6', '3', ',000', ',000'],\n",
+       " '167': ['▁1', '6', '7'],\n",
+       " '17': ['▁1', '7'],\n",
+       " '18.1': ['▁1', '8', '.', '1'],\n",
+       " '1830': ['▁1830'],\n",
+       " \"1830's\": ['▁1830', \"'\", 's'],\n",
+       " '1834': ['▁183', '4'],\n",
+       " '1897': ['▁1', '8', '9', '7'],\n",
+       " '19': ['▁19'],\n",
+       " '19.5': ['▁19', '.', '5'],\n",
+       " '1910': ['▁19', '1', '0'],\n",
+       " '1913': ['▁19', '1', '3'],\n",
+       " '1914': ['▁19', '1', '4'],\n",
+       " '1914-18': ['▁19', '1', '4', '-', '1', '8'],\n",
+       " '1920': ['▁19', '2', '0'],\n",
+       " '1931': ['▁19', '3', '1'],\n",
+       " '1932': ['▁19', '3', '2'],\n",
+       " '1934': ['▁19', '3', '4'],\n",
+       " '1939': ['▁19', '3', '9'],\n",
+       " '1943': ['▁19', '4', '3'],\n",
+       " '1944': ['▁19', '4', '4'],\n",
+       " '1950': ['▁195', '0'],\n",
+       " '1951': ['▁195', '1'],\n",
+       " '1954': ['▁195', '4'],\n",
+       " '1956': ['▁195', '6'],\n",
+       " '1957': ['▁195', '7'],\n",
+       " '1958': ['▁195', '8'],\n",
+       " '1959': ['▁195', '9'],\n",
+       " '1960': ['▁1960'],\n",
+       " '1960s': ['▁1960', 's'],\n",
+       " '1961': ['▁19', '6', '1'],\n",
+       " '1963': ['▁19', '6', '3'],\n",
+       " '19th': ['▁19', 'th'],\n",
+       " '1superceded': ['▁1', 'super', 'c', 'ed', 'ed'],\n",
+       " \"1tho'\": ['▁1', 'tho', \"'\"],\n",
+       " '2': ['▁2'],\n",
+       " '2,000': ['▁2', ',000'],\n",
+       " '2,415,000,000': ['▁2', ',', '4', '1', '5', ',000', ',000'],\n",
+       " '20': ['▁2', '0'],\n",
+       " '20-month-old': ['▁2', '0', '-', 'm', 'on', 'th', '-', 'o', 'ld'],\n",
+       " '200': ['▁2', '0', '0'],\n",
+       " '20th-century': ['▁2', '0', 'th', '-', 'c', 'ent', 'ur', 'y'],\n",
+       " '210million': ['▁2', '1', '0', 'million'],\n",
+       " '22': ['▁2', '2'],\n",
+       " '23.1': ['▁2', '3', '.', '1'],\n",
+       " '24-strong': ['▁2', '4', '-', 'st', 'ro', 'ng'],\n",
+       " '25': ['▁2', '5'],\n",
+       " '27': ['▁2', '7'],\n",
+       " '28.5': ['▁2', '8', '.', '5'],\n",
+       " '280,000': ['▁2', '8', '0', ',000'],\n",
+       " '287': ['▁2', '8', '7'],\n",
+       " '288': ['▁2', '8', '8'],\n",
+       " '2bhoys': ['▁2', 'b', 'h', 'o', 'y', 's'],\n",
+       " '2ole': ['▁2', 'o', 'le'],\n",
+       " '2skint': ['▁2', 's', 'k', 'in', 't'],\n",
+       " '3': ['▁3'],\n",
+       " '3,000': ['▁3', ',000'],\n",
+       " '3.6': ['▁3', '.', '6'],\n",
+       " '3/0': ['▁3', '/', '0'],\n",
+       " '30': ['▁3', '0'],\n",
+       " '30-minute': ['▁3', '0', '-', 'minute'],\n",
+       " '300,000': ['▁3', '0', '0', ',000'],\n",
+       " '32': ['▁3', '2'],\n",
+       " '34': ['▁3', '4'],\n",
+       " '35': ['▁3', '5'],\n",
+       " '357million': ['▁3', '5', '7', 'million'],\n",
+       " '36': ['▁3', '6'],\n",
+       " '37,000,000': ['▁3', '7', ',000', ',000'],\n",
+       " '37.2': ['▁3', '7', '.', '2'],\n",
+       " '38': ['▁3', '8'],\n",
+       " '4': ['▁', '4'],\n",
+       " '4.8': ['▁', '4', '.', '8'],\n",
+       " '40': ['▁', '4', '0'],\n",
+       " '400': ['▁', '4', '0', '0'],\n",
+       " '400,000': ['▁', '4', '0', '0', ',000'],\n",
+       " '43': ['▁', '4', '3'],\n",
+       " '450': ['▁', '4', '5', '0'],\n",
+       " '5': ['▁', '5'],\n",
+       " '5,000': ['▁', '5', ',000'],\n",
+       " '5.30': ['▁', '5', '.', '3', '0'],\n",
+       " '5/8': ['▁', '5', '/', '8'],\n",
+       " '50,000': ['▁', '5', '0', ',000'],\n",
+       " '500': ['▁', '5', '0', '0'],\n",
+       " '550,000': ['▁', '5', '5', '0', ',000'],\n",
+       " '58': ['▁', '5', '8'],\n",
+       " '6': ['▁', '6'],\n",
+       " '6,000': ['▁', '6', ',000'],\n",
+       " '60': ['▁', '6', '0'],\n",
+       " '600': ['▁', '6', '0', '0'],\n",
+       " '61-year-old': ['▁', '6', '1', '-', 'y', 'e', 'ar', '-', 'o', 'ld'],\n",
+       " '6al': ['▁', '6', 'al'],\n",
+       " '6tic': ['▁', '6', 'tic'],\n",
+       " '7.42': ['▁', '7', '.', '4', '2'],\n",
+       " '70': ['▁', '7', '0'],\n",
+       " '70,000,000': ['▁', '7', '0', ',000', ',000'],\n",
+       " '707': ['▁', '7', '0', '7'],\n",
+       " '750': ['▁', '7', '5', '0'],\n",
+       " '8': ['▁', '8'],\n",
+       " '80': ['▁', '8', '0'],\n",
+       " '800': ['▁', '8', '0', '0'],\n",
+       " '800,000': ['▁', '8', '0', '0', ',000'],\n",
+       " '86': ['▁', '8', '6'],\n",
+       " '88': ['▁', '8', '8'],\n",
+       " '88-year-old': ['▁', '8', '8', '-', 'y', 'e', 'ar', '-', 'o', 'ld'],\n",
+       " '89': ['▁', '8', '9'],\n",
+       " '89-year-old': ['▁', '8', '9', '-', 'y', 'e', 'ar', '-', 'o', 'ld'],\n",
+       " '90-day': ['▁', '9', '0', '-', 'day'],\n",
+       " '90-minute': ['▁', '9', '0', '-', 'minute'],\n",
+       " '91': ['▁', '9', '1'],\n",
+       " '950': ['▁', '9', '5', '0'],\n",
+       " '97.5': ['▁', '9', '7', '.', '5'],\n",
+       " ':': ['▁', ':'],\n",
+       " ';': ['▁', ';'],\n",
+       " '?': ['▁', '?'],\n",
+       " 'a': ['▁a'],\n",
+       " 'abandon': ['▁a', 'b', 'a', 'ndon'],\n",
+       " 'abandoned': ['▁a', 'b', 'a', 'ndon', 'ed'],\n",
+       " 'abashed': ['▁a', 'b', 'ash', 'ed'],\n",
+       " 'ability': ['▁a', 'b', 'ilit', 'y'],\n",
+       " 'able': ['▁', 'able'],\n",
+       " 'able-bodied': ['▁', 'able', '-', 'bo', 'd', 'i', 'ed'],\n",
+       " 'abolish': ['▁a', 'bo', 'l', 'ish'],\n",
+       " 'abolished': ['▁a', 'bo', 'l', 'ish', 'ed'],\n",
+       " 'abolition': ['▁a', 'bo', 'li', 'tion'],\n",
+       " 'abou': ['▁a', 'bo', 'u'],\n",
+       " 'about': ['▁about'],\n",
+       " 'about-': ['▁about', '-'],\n",
+       " 'above': ['▁a', 'bo', 've'],\n",
+       " 'abreast': ['▁a', 'b', 're', 'a', 'st'],\n",
+       " 'abroad': ['▁a', 'b', 'ro', 'ad'],\n",
+       " 'absence': ['▁a', 'b', 's', 'ence'],\n",
+       " 'absent': ['▁a', 'b', 's', 'ent'],\n",
+       " 'absolutely': ['▁a', 'b', 'solut', 'e', 'ly'],\n",
+       " 'abstraction': ['▁a', 'b', 'st', 'ra', 'c', 'tion'],\n",
+       " 'abundance': ['▁a', 'b', 'un', 'd', 'ance'],\n",
+       " 'ac-': ['▁a', 'c', '-'],\n",
+       " 'academic': ['▁a', 'c', 'ad', 'e', 'm', 'ic'],\n",
+       " 'accent': ['▁a', 'c', 'c', 'ent'],\n",
+       " 'accents': ['▁a', 'c', 'c', 'ent', 's'],\n",
+       " 'accept': ['▁accept'],\n",
+       " 'acceptable': ['▁accept', 'able'],\n",
+       " 'accepted': ['▁accept', 'ed'],\n",
+       " 'accepting': ['▁accept', 'ing'],\n",
+       " 'accessories': ['▁a', 'c', 'ce', 's', 's', 'or', 'ies'],\n",
+       " 'accident': ['▁a', 'c', 'c', 'i', 'd', 'ent'],\n",
+       " 'accidental': ['▁a', 'c', 'c', 'i', 'd', 'ent', 'al'],\n",
+       " 'accommodate': ['▁a', 'c', 'c', 'o', 'm', 'm', 'o', 'date'],\n",
+       " 'accommodation': ['▁a', 'c', 'c', 'o', 'm', 'm', 'o', 'd', 'ation'],\n",
+       " 'accompanied': ['▁a', 'c', 'c', 'o', 'm', 'p', 'an', 'i', 'ed'],\n",
+       " 'accompanist': ['▁a', 'c', 'c', 'o', 'm', 'p', 'an', 'is', 't'],\n",
+       " 'accompany': ['▁a', 'c', 'c', 'o', 'm', 'p', 'an', 'y'],\n",
+       " 'accomplished': ['▁a', 'c', 'c', 'o', 'm', 'p', 'l', 'ish', 'ed'],\n",
+       " 'accomplishments': ['▁a', 'c', 'c', 'o', 'm', 'p', 'l', 'ish', 'ment', 's'],\n",
+       " 'according': ['▁a', 'c', 'c', 'or', 'd', 'ing'],\n",
+       " 'account': ['▁a', 'c', 'c', 'o', 'un', 't'],\n",
+       " 'accountancy': ['▁a', 'c', 'c', 'o', 'un', 't', 'an', 'c', 'y'],\n",
+       " 'accra': ['▁a', 'c', 'c', 'ra'],\n",
+       " \"accra's\": ['▁a', 'c', 'c', 'ra', \"'\", 's'],\n",
+       " 'accuracy': ['▁a', 'c', 'cu', 'ra', 'c', 'y'],\n",
+       " 'accurate': ['▁a', 'c', 'cu', 'rate'],\n",
+       " 'accurately': ['▁a', 'c', 'cu', 'rate', 'ly'],\n",
+       " 'accused': ['▁a', 'c', 'cu', 's', 'ed'],\n",
+       " 'achieved': ['▁a', 'ch', 'ie', 'v', 'ed'],\n",
+       " 'achievement': ['▁a', 'ch', 'ie', 've', 'ment'],\n",
+       " 'acquaintance': ['▁a', 'c', 'q', 'u', 'ain', 't', 'ance'],\n",
+       " 'acquaintances': ['▁a', 'c', 'q', 'u', 'ain', 't', 'ance', 's'],\n",
+       " 'acres': ['▁a', 'c', 're', 's'],\n",
+       " 'across': ['▁a', 'cross'],\n",
+       " 'act': ['▁act'],\n",
+       " 'acting': ['▁act', 'ing'],\n",
+       " 'action': ['▁action'],\n",
+       " 'actions': ['▁action', 's'],\n",
+       " 'active': ['▁act', 'ive'],\n",
+       " 'activists': ['▁act', 'i', 'v', 'is', 't', 's'],\n",
+       " 'activities': ['▁act', 'i', 'v', 'it', 'ies'],\n",
+       " 'activity': ['▁act', 'i', 'v', 'it', 'y'],\n",
+       " 'acton': ['▁act', 'on'],\n",
+       " 'actor': ['▁act', 'or'],\n",
+       " 'actress': ['▁act', 'ress'],\n",
+       " 'acts': ['▁act', 's'],\n",
+       " 'actual': ['▁act', 'ual'],\n",
+       " 'actually': ['▁act', 'ual', 'ly'],\n",
+       " 'adamafio': ['▁a', 'd', 'a', 'm', 'a', 'f', 'i', 'o'],\n",
+       " 'adaptation': ['▁a', 'd', 'a', 'p', 't', 'ation'],\n",
+       " 'adapted': ['▁a', 'd', 'a', 'p', 't', 'ed'],\n",
+       " 'adapting': ['▁a', 'd', 'a', 'p', 't', 'ing'],\n",
+       " 'add': ['▁add'],\n",
+       " 'added': ['▁add', 'ed'],\n",
+       " 'adding': ['▁add', 'ing'],\n",
+       " 'addition': ['▁add', 'it', 'ion'],\n",
+       " 'additions': ['▁add', 'it', 'ion', 's'],\n",
+       " 'address': ['▁add', 'ress'],\n",
+       " 'addressed': ['▁add', 'ress', 'ed'],\n",
+       " 'addresses': ['▁add', 'ress', 'es'],\n",
+       " 'addressing': ['▁add', 'ress', 'ing'],\n",
+       " 'adenauer': ['▁adenauer'],\n",
+       " \"adenauer's\": ['▁adenauer', \"'\", 's'],\n",
+       " 'adequate': ['▁a', 'd', 'equa', 'te'],\n",
+       " 'adhem': ['▁a', 'd', 'h', 'e', 'm'],\n",
+       " 'adjust': ['▁adjust'],\n",
+       " 'adjustment': ['▁adjust', 'ment'],\n",
+       " 'administration': ['▁a', 'd', 'm', 'in', 'is', 't', 'r', 'ation'],\n",
+       " \"administration's\": ['▁a', 'd', 'm', 'in', 'is', 't', 'r', 'ation', \"'\", 's'],\n",
+       " 'administrative': ['▁a', 'd', 'm', 'in', 'is', 't', 'r', 'ative'],\n",
+       " 'admiralty': ['▁a', 'd', 'm', 'ir', 'al', 'ty'],\n",
+       " 'admire': ['▁a', 'd', 'm', 'ir', 'e'],\n",
+       " 'admit': ['▁a', 'd', 'm', 'it'],\n",
+       " 'admitted': ['▁a', 'd', 'mitt', 'ed'],\n",
+       " 'admitting': ['▁a', 'd', 'mitt', 'ing'],\n",
+       " 'adopted': ['▁a', 'd', 'op', 't', 'ed'],\n",
+       " 'adoption': ['▁a', 'd', 'o', 'ption'],\n",
+       " 'adult': ['▁a', 'd', 'ul', 't'],\n",
+       " 'advance': ['▁a', 'd', 'v', 'ance'],\n",
+       " 'advanced': ['▁a', 'd', 'v', 'ance', 'd'],\n",
+       " 'advancing': ['▁a', 'd', 'v', 'an', 'c', 'ing'],\n",
+       " 'advantage': ['▁', 'advantage'],\n",
+       " 'advantages': ['▁', 'advantage', 's'],\n",
+       " 'advertisement': ['▁advertisement'],\n",
+       " 'advertisements': ['▁advertisement', 's'],\n",
+       " 'advice': ['▁advi', 'ce'],\n",
+       " 'advisability': ['▁advi', 's', 'a', 'b', 'ilit', 'y'],\n",
+       " 'advise': ['▁advise'],\n",
+       " 'advised': ['▁advise', 'd'],\n",
+       " 'advisers': ['▁advise', 'r', 's'],\n",
+       " 'advocate': ['▁a', 'd', 'v', 'o', 'c', 'at', 'e'],\n",
+       " 'af-': ['▁a', 'f', '-'],\n",
+       " 'affairs': ['▁', 'aff', 'a', 'ir', 's'],\n",
+       " 'affected': ['▁', 'aff', 'e', 'ct', 'ed'],\n",
+       " 'affection': ['▁', 'aff', 'e', 'c', 'tion'],\n",
+       " 'affilia-': ['▁', 'aff', 'il', 'i', 'a', '-'],\n",
+       " 'affiliations': ['▁', 'aff', 'il', 'i', 'ation', 's'],\n",
+       " 'affluence': ['▁a', 'f', 'flu', 'ence'],\n",
+       " 'affluent': ['▁a', 'f', 'flu', 'ent'],\n",
+       " 'afraid': ['▁a', 'f', 'r', 'aid'],\n",
+       " 'africa': ['▁africa'],\n",
+       " \"africa's\": ['▁africa', \"'\", 's'],\n",
+       " 'african': ['▁african'],\n",
+       " 'africans': ['▁african', 's'],\n",
+       " 'after': ['▁after'],\n",
+       " 'afternoon': ['▁after', 'n', 'o', 'on'],\n",
+       " 'afterwards': ['▁after', 'ward', 's'],\n",
+       " 'again': ['▁again'],\n",
+       " 'against': ['▁again', 'st'],\n",
+       " 'age': ['▁', 'age'],\n",
+       " 'age-structure': ['▁', 'age', '-', 'struct', 'ure'],\n",
+       " 'aged': ['▁a', 'g', 'ed'],\n",
+       " 'ageing': ['▁', 'age', 'ing'],\n",
+       " 'agent': ['▁a', 'g', 'ent'],\n",
+       " 'agents': ['▁a', 'g', 'ent', 's'],\n",
+       " 'ages': ['▁', 'age', 's'],\n",
+       " 'agitation': ['▁a', 'g', 'it', 'ation'],\n",
+       " 'ago': ['▁', 'ago'],\n",
+       " 'agree': ['▁agree'],\n",
+       " 'agreed': ['▁agree', 'd'],\n",
+       " 'agreement': ['▁agree', 'ment'],\n",
+       " 'agreements': ['▁agree', 'ment', 's'],\n",
+       " 'agriculture': ['▁a', 'g', 'ri', 'c', 'ul', 'ture'],\n",
+       " 'ahead': ['▁a', 'h', 'e', 'ad'],\n",
+       " 'aid': ['▁', 'aid'],\n",
+       " 'aide': ['▁', 'aid', 'e'],\n",
+       " 'aided': ['▁', 'aid', 'ed'],\n",
+       " 'aides': ['▁', 'aid', 'es'],\n",
+       " 'aim': ['▁a', 'i', 'm'],\n",
+       " 'aimed': ['▁a', 'i', 'm', 'ed'],\n",
+       " 'aiming': ['▁a', 'i', 'm', 'ing'],\n",
+       " 'air': ['▁a', 'ir'],\n",
+       " 'aircraft': ['▁a', 'ir', 'c', 'ra', 'f', 't'],\n",
+       " 'aired': ['▁a', 'ir', 'ed'],\n",
+       " \"airliner's\": ['▁a', 'ir', 'lin', 'er', \"'\", 's'],\n",
+       " 'airmen': ['▁a', 'ir', 'm', 'en'],\n",
+       " 'airport': ['▁a', 'ir', 'port'],\n",
+       " 'akin': ['▁', 'ak', 'in'],\n",
+       " \"aladdin's\": ['▁', 'al', 'ad', 'd', 'in', \"'\", 's'],\n",
+       " 'alan': ['▁', 'al', 'an'],\n",
+       " 'alarmed': ['▁', 'al', 'ar', 'm', 'ed'],\n",
+       " 'alas': ['▁', 'al', 'a', 's'],\n",
+       " 'alcoholic': ['▁', 'al', 'c', 'o', 'h', 'o', 'l', 'ic'],\n",
+       " 'algeria': ['▁', 'al', 'ge', 'ri', 'a'],\n",
+       " 'alike': ['▁a', 'li', 'ke'],\n",
+       " 'alive': ['▁', 'al', 'ive'],\n",
+       " 'all': ['▁all'],\n",
+       " 'all-regular': ['▁all', '-', 're', 'g', 'ular'],\n",
+       " 'alleged': ['▁all', 'e', 'g', 'ed'],\n",
+       " 'allen': ['▁all', 'en'],\n",
+       " 'alleviation': ['▁all', 'evi', 'ation'],\n",
+       " 'alley': ['▁all', 'e', 'y'],\n",
+       " 'alliance': ['▁all', 'i', 'ance'],\n",
+       " 'alliances': ['▁all', 'i', 'ance', 's'],\n",
+       " 'allied': ['▁all', 'i', 'ed'],\n",
+       " 'allies': ['▁all', 'ies'],\n",
+       " 'allow': ['▁allow'],\n",
+       " 'allowance': ['▁allow', 'ance'],\n",
+       " 'allowances': ['▁allow', 'ance', 's'],\n",
+       " 'allowed': ['▁allow', 'ed'],\n",
+       " 'allowing': ['▁allow', 'ing'],\n",
+       " 'ally': ['▁', 'ally'],\n",
+       " 'almost': ['▁', 'al', 'most'],\n",
+       " 'alone': ['▁a', 'lo', 'ne'],\n",
+       " 'along': ['▁along'],\n",
+       " 'alongside': ['▁along', 'side'],\n",
+       " 'already': ['▁', 'al', 'ready'],\n",
+       " 'also': ['▁a', 'lso'],\n",
+       " 'alter': ['▁', 'al', 'ter'],\n",
+       " 'alternative': ['▁', 'al', 'ter', 'n', 'ative'],\n",
+       " 'alternatively': ['▁', 'al', 'ter', 'n', 'ative', 'ly'],\n",
+       " 'alternatives': ['▁', 'al', 'ter', 'n', 'ative', 's'],\n",
+       " 'although': ['▁', 'al', 'th', 'ough'],\n",
+       " 'altogether': ['▁alto', 'ge', 'ther'],\n",
+       " 'altos': ['▁alto', 's'],\n",
+       " 'always': ['▁a', 'lway', 's'],\n",
+       " 'am': ['▁a', 'm'],\n",
+       " 'amateur': ['▁a', 'm', 'at', 'e', 'ur'],\n",
+       " 'amazed': ['▁a', 'm', 'a', 'z', 'ed'],\n",
+       " 'amazing': ['▁a', 'm', 'a', 'z', 'ing'],\n",
+       " 'ambassador': ['▁a', 'm', 'b', 'a', 's', 's', 'ad', 'or'],\n",
+       " 'amber': ['▁a', 'mber'],\n",
+       " 'ambition': ['▁a', 'm', 'bit', 'ion'],\n",
+       " 'ambitious': ['▁a', 'm', 'bit', 'i', 'ous'],\n",
+       " 'ambulance': ['▁a', 'm', 'b', 'ul', 'ance'],\n",
+       " 'ambulances': ['▁a', 'm', 'b', 'ul', 'ance', 's'],\n",
+       " 'america': ['▁america'],\n",
+       " \"america's\": ['▁america', \"'\", 's'],\n",
+       " 'american': ['▁american'],\n",
+       " 'american-born': ['▁american', '-', 'b', 'orn'],\n",
+       " 'americans': ['▁american', 's'],\n",
+       " 'ammunition': ['▁a', 'm', 'm', 'un', 'it', 'ion'],\n",
+       " 'among': ['▁amo', 'ng'],\n",
+       " 'amount': ['▁amo', 'un', 't'],\n",
+       " 'ample': ['▁a', 'm', 'ple'],\n",
+       " 'amusement': ['▁a', 'm', 'us', 'e', 'ment'],\n",
+       " 'amusing': ['▁a', 'm', 'us', 'ing'],\n",
+       " 'an': ['▁an'],\n",
+       " 'analogy': ['▁an', 'al', 'o', 'g', 'y'],\n",
+       " 'analysed': ['▁an', 'al', 'y', 's', 'ed'],\n",
+       " 'anchor': ['▁an', 'ch', 'or'],\n",
+       " 'ancient': ['▁an', 'c', 'i', 'ent'],\n",
+       " 'and': ['▁and'],\n",
+       " 'andrei': ['▁and', 're', 'i'],\n",
+       " 'andrew': ['▁and', 're', 'w'],\n",
+       " 'anecdotal': ['▁an', 'e', 'c', 'd', 'o', 't', 'al'],\n",
+       " 'angel': ['▁a', 'ng', 'el'],\n",
+       " 'angeles': ['▁an', 'ge', 'le', 's'],\n",
+       " 'angelo': ['▁an', 'ge', 'lo'],\n",
+       " 'anger': ['▁a', 'ng', 'er'],\n",
+       " 'anglais': ['▁a', 'ng', 'l', 'a', 'is'],\n",
+       " 'angle': ['▁a', 'ng', 'le'],\n",
+       " 'anglesey': ['▁anglesey'],\n",
+       " \"anglesey's\": ['▁anglesey', \"'\", 's'],\n",
+       " 'anglesey-road': ['▁anglesey', '-', 'ro', 'ad'],\n",
+       " 'angola': ['▁a', 'ng', 'o', 'l', 'a'],\n",
+       " 'angry': ['▁a', 'ng', 'ry'],\n",
+       " 'ann': ['▁an', 'n'],\n",
+       " 'anna': ['▁an', 'n', 'a'],\n",
+       " 'announced': ['▁an', 'n', 'o', 'un', 'c', 'ed'],\n",
+       " 'announcement': ['▁an', 'n', 'o', 'un', 'ce', 'ment'],\n",
+       " 'announcing': ['▁an', 'n', 'o', 'un', 'c', 'ing'],\n",
+       " 'annual': ['▁an', 'n', 'ual'],\n",
+       " 'another': ['▁an', 'other'],\n",
+       " 'answer': ['▁answer'],\n",
+       " 'answered': ['▁answer', 'ed'],\n",
+       " 'answering': ['▁answer', 'ing'],\n",
+       " 'antagonism': ['▁', 'ant', 'ago', 'n', 'ism'],\n",
+       " 'anthony': ['▁an', 'th', 'on', 'y'],\n",
+       " 'anti-apartheid': ['▁', 'ant', 'i', '-', 'a', 'p', 'ar', 't', 'hei', 'd'],\n",
+       " 'anti-bomb': ['▁', 'ant', 'i', '-', 'bomb'],\n",
+       " 'anti-german': ['▁', 'ant', 'i', '-', 'g', 'er', 'man'],\n",
+       " 'anti-nato': ['▁', 'ant', 'i', '-', 'nato'],\n",
+       " 'anti-negro': ['▁', 'ant', 'i', '-', 'ne', 'g', 'ro'],\n",
+       " 'anti-nuclear': ['▁', 'ant', 'i', '-', 'n', 'u', 'c', 'le', 'ar'],\n",
+       " 'anti-soviet': ['▁', 'ant', 'i', '-', 's', 'o', 'v', 'i', 'et'],\n",
+       " 'anti-tory': ['▁', 'ant', 'i', '-', 'tory'],\n",
+       " 'anticipation': ['▁an', 'tic', 'ip', 'ation'],\n",
+       " 'antonioni': ['▁an', 'ton', 'ion', 'i'],\n",
+       " \"antonioni's\": ['▁an', 'ton', 'ion', 'i', \"'\", 's'],\n",
+       " 'any': ['▁any'],\n",
+       " 'any-': ['▁any', '-'],\n",
+       " 'anybody': ['▁any', 'body'],\n",
+       " \"anybody's\": ['▁any', 'body', \"'\", 's'],\n",
+       " 'anyone': ['▁any', 'on', 'e'],\n",
+       " 'anything': ['▁any', 'thing'],\n",
+       " 'apart': ['▁apart'],\n",
+       " 'apartheid': ['▁apart', 'hei', 'd'],\n",
+       " 'apathetic': ['▁a', 'p', 'at', 'h', 'e', 'tic'],\n",
+       " 'apathy': ['▁a', 'p', 'at', 'h', 'y'],\n",
+       " 'apex': ['▁a', 'p', 'e', 'x'],\n",
+       " 'apocalypse': ['▁a', 'p', 'o', 'c', 'al', 'y', 'p', 'se'],\n",
+       " 'apologising': ['▁a', 'p', 'o', 'lo', 'g', 'is', 'ing'],\n",
+       " 'appalled': ['▁', 'app', 'al', 'led'],\n",
+       " 'appalling': ['▁', 'app', 'al', 'l', 'ing'],\n",
+       " 'apparatus': ['▁', 'app', 'ar', 'at', 'us'],\n",
+       " 'apparent': ['▁', 'app', 'ar', 'ent'],\n",
+       " 'apparently': ['▁', 'app', 'ar', 'ent', 'ly'],\n",
+       " 'appeal': ['▁appeal'],\n",
+       " 'appealing': ['▁appeal', 'ing'],\n",
+       " 'appeals': ['▁appeal', 's'],\n",
+       " 'appear': ['▁appear'],\n",
+       " 'appearance': ['▁appear', 'ance'],\n",
+       " 'appeared': ['▁appear', 'ed'],\n",
+       " 'appears': ['▁appear', 's'],\n",
+       " 'appeasement': ['▁', 'app', 'e', 'a', 'se', 'ment'],\n",
+       " 'applauding': ['▁', 'app', 'l', 'a', 'ud', 'ing'],\n",
+       " 'appliances': ['▁', 'app', 'li', 'ance', 's'],\n",
+       " 'application': ['▁', 'app', 'l', 'ic', 'ation'],\n",
+       " 'applications': ['▁', 'app', 'l', 'ic', 'ation', 's'],\n",
+       " 'applied': ['▁', 'app', 'li', 'ed'],\n",
+       " 'apply': ['▁', 'app', 'ly'],\n",
+       " 'appointed': ['▁', 'app', 'o', 'in', 't', 'ed'],\n",
+       " 'appointment': ['▁', 'app', 'o', 'in', 't', 'ment'],\n",
+       " 'appreciable': ['▁', 'app', 're', 'c', 'i', 'able'],\n",
+       " 'appreciably': ['▁', 'app', 're', 'c', 'i', 'abl', 'y'],\n",
+       " 'appreciated': ['▁', 'app', 're', 'c', 'i', 'at', 'ed'],\n",
+       " 'apprenticeships': ['▁', 'app', 'ren', 'tic', 'e', 'ship', 's'],\n",
+       " 'approach': ['▁', 'app', 'ro', 'a', 'ch'],\n",
+       " 'approached': ['▁', 'app', 'ro', 'a', 'ch', 'ed'],\n",
+       " 'approaches': ['▁', 'app', 'ro', 'a', 'ches'],\n",
+       " 'appropriate': ['▁', 'app', 'ro', 'p', 'ri', 'at', 'e'],\n",
+       " 'appropriated': ['▁', 'app', 'ro', 'p', 'ri', 'at', 'ed'],\n",
+       " 'approval': ['▁', 'app', 'ro', 'val'],\n",
+       " 'approximately': ['▁', 'app', 'ro', 'x', 'i', 'm', 'at', 'e', 'ly'],\n",
+       " 'april': ['▁a', 'p', 'ri', 'l'],\n",
+       " 'archbishop': ['▁', 'ar', 'ch', 'b', 'ish', 'op'],\n",
+       " 'arches': ['▁', 'ar', 'ches'],\n",
+       " 'archipelago': ['▁', 'ar', 'ch', 'ip', 'el', 'ago'],\n",
+       " 'architect': ['▁', 'ar', 'ch', 'it', 'e', 'ct'],\n",
+       " 'architecture': ['▁', 'ar', 'ch', 'it', 'e', 'c', 'ture'],\n",
+       " 'are': ['▁are'],\n",
+       " 'area': ['▁are', 'a'],\n",
+       " 'areas': ['▁are', 'a', 's'],\n",
+       " 'arguably': ['▁argu', 'abl', 'y'],\n",
+       " 'argued': ['▁argu', 'ed'],\n",
+       " 'argues': ['▁argu', 'es'],\n",
+       " 'arguing': ['▁argu', 'ing'],\n",
+       " 'argument': ['▁argu', 'ment'],\n",
+       " 'arguments': ['▁argu', 'ment', 's'],\n",
+       " 'arise': ['▁', 'aris', 'e'],\n",
+       " 'arises': ['▁', 'aris', 'es'],\n",
+       " 'arm': ['▁arm'],\n",
+       " 'armament': ['▁', 'armament'],\n",
+       " 'armaments': ['▁', 'armament', 's'],\n",
+       " 'armed': ['▁arm', 'ed'],\n",
+       " 'armoured': ['▁arm', 'our', 'ed'],\n",
+       " 'arms': ['▁arm', 's'],\n",
+       " \"arms'\": ['▁arm', 's', \"'\"],\n",
+       " 'army': ['▁arm', 'y'],\n",
+       " 'arose': ['▁', 'ar', 'ose'],\n",
+       " 'around': ['▁a', 'round'],\n",
+       " 'aroused': ['▁', 'ar', 'ous', 'ed'],\n",
+       " 'arrange': ['▁arrang', 'e'],\n",
+       " 'arranged': ['▁arrang', 'ed'],\n",
+       " 'arrangement': ['▁arrang', 'e', 'ment'],\n",
+       " 'arrangements': ['▁arrang', 'e', 'ment', 's'],\n",
+       " 'arranging': ['▁arrang', 'ing'],\n",
+       " 'arrears': ['▁', 'ar', 're', 'ar', 's'],\n",
+       " 'arrested': ['▁', 'ar', 're', 'st', 'ed'],\n",
+       " 'arrival': ['▁arriv', 'al'],\n",
+       " 'arrive': ['▁arriv', 'e'],\n",
+       " 'arrived': ['▁arriv', 'ed'],\n",
+       " 'arrives': ['▁arriv', 'es'],\n",
+       " 'arrogant': ['▁', 'ar', 'ro', 'g', 'ant'],\n",
+       " 'art': ['▁', 'ar', 't'],\n",
+       " 'arthur': ['▁', 'ar', 'th', 'ur'],\n",
+       " 'article': ['▁', 'ar', 'tic', 'le'],\n",
+       " 'articles': ['▁', 'ar', 'tic', 'le', 's'],\n",
+       " 'articulation': ['▁', 'ar', 'tic', 'ul', 'ation'],\n",
+       " 'artistic': ['▁', 'ar', 'tist', 'ic'],\n",
+       " 'artists': ['▁', 'ar', 'tist', 's'],\n",
+       " 'as': ['▁as'],\n",
+       " 'ascents': ['▁as', 'c', 'ent', 's'],\n",
+       " 'ash': ['▁', 'ash'],\n",
+       " 'ashen': ['▁', 'ash', 'en'],\n",
+       " 'ask': ['▁ask'],\n",
+       " 'asked': ['▁ask', 'ed'],\n",
+       " 'asking': ['▁ask', 'ing'],\n",
+       " 'aspect': ['▁a', 'spect'],\n",
+       " 'aspects': ['▁a', 'spect', 's'],\n",
+       " 'assault': ['▁as', 's', 'a', 'ul', 't'],\n",
+       " 'assembler': ['▁as', 'se', 'm', 'ble', 'r'],\n",
+       " 'assembly': ['▁as', 'se', 'm', 'b', 'ly'],\n",
+       " 'assess': ['▁as', 'se', 's', 's'],\n",
+       " 'assessment': ['▁as', 'se', 's', 's', 'ment'],\n",
+       " 'assistance': ['▁as', 's', 'is', 't', 'ance'],\n",
+       " 'assistant': ['▁as', 's', 'is', 't', 'ant'],\n",
+       " 'assistants': ['▁as', 's', 'is', 't', 'ant', 's'],\n",
+       " 'associate': ['▁associat', 'e'],\n",
+       " 'associated': ['▁associat', 'ed'],\n",
+       " 'associates': ['▁associat', 'es'],\n",
+       " 'association': ['▁associat', 'ion'],\n",
+       " 'assortment': ['▁as', 's', 'or', 't', 'ment'],\n",
+       " 'assumption': ['▁assu', 'm', 'ption'],\n",
+       " 'assurance': ['▁assu', 'r', 'ance'],\n",
+       " 'astute': ['▁as', 'tu', 'te'],\n",
+       " 'at': ['▁at'],\n",
+       " 'ately': ['▁at', 'e', 'ly'],\n",
+       " 'atlantic': ['▁at', 'l', 'an', 'tic'],\n",
+       " 'atmosphere': ['▁atmospher', 'e'],\n",
+       " 'atmospheric': ['▁atmospher', 'ic'],\n",
+       " 'atomic': ['▁a', 'tom', 'ic'],\n",
+       " 'atoms': ['▁a', 'tom', 's'],\n",
+       " 'attach': ['▁at', 't', 'a', 'ch'],\n",
+       " 'attached': ['▁at', 't', 'a', 'ch', 'ed'],\n",
+       " 'attack': ['▁at', 'tack'],\n",
+       " 'attacked': ['▁at', 'tack', 'ed'],\n",
+       " 'attacks': ['▁at', 'tack', 's'],\n",
+       " 'attainable': ['▁at', 't', 'ain', 'able'],\n",
+       " 'attempt': ['▁attempt'],\n",
+       " 'attempted': ['▁attempt', 'ed'],\n",
+       " 'attempting': ['▁attempt', 'ing'],\n",
+       " 'attempts': ['▁attempt', 's'],\n",
+       " 'atten-': ['▁at', 'ten', '-'],\n",
+       " 'attend': ['▁at', 't', 'end'],\n",
+       " 'attendance': ['▁at', 't', 'end', 'ance'],\n",
+       " 'attended': ['▁at', 't', 'end', 'ed'],\n",
+       " 'attending': ['▁at', 't', 'end', 'ing'],\n",
+       " 'attention': ['▁at', 'ten', 'tion'],\n",
+       " 'attitude': ['▁at', 't', 'it', 'ud', 'e'],\n",
+       " 'attitudes': ['▁at', 't', 'it', 'ud', 'es'],\n",
+       " 'attracted': ['▁at', 't', 'ra', 'ct', 'ed'],\n",
+       " 'attractive': ['▁at', 't', 'ra', 'ct', 'ive'],\n",
+       " 'aubrey': ['▁a', 'u', 'b', 're', 'y'],\n",
+       " 'audacity': ['▁a', 'ud', 'a', 'c', 'it', 'y'],\n",
+       " 'auden': ['▁a', 'ud', 'en'],\n",
+       " 'audience': ['▁a', 'ud', 'i', 'ence'],\n",
+       " 'audited': ['▁a', 'ud', 'it', 'ed'],\n",
+       " 'august': ['▁a', 'ug', 'u', 'st'],\n",
+       " 'auntie': ['▁a', 'un', 't', 'ie'],\n",
+       " 'austerity': ['▁a', 'us', 'ter', 'it', 'y'],\n",
+       " 'australia': ['▁a', 'u', 'st', 'ra', 'li', 'a'],\n",
+       " 'austria': ['▁a', 'u', 'st', 'ri', 'a'],\n",
+       " 'austrian': ['▁a', 'u', 'st', 'ri', 'an'],\n",
+       " 'authentic': ['▁a', 'u', 'th', 'en', 'tic'],\n",
+       " 'author': ['▁', 'author'],\n",
+       " 'authorised': ['▁', 'author', 'is', 'ed'],\n",
+       " 'authorities': ['▁authorit', 'ies'],\n",
+       " 'authority': ['▁authorit', 'y'],\n",
+       " 'automatically': ['▁a', 'u', 'tom', 'at', 'ic', 'ally'],\n",
+       " 'automation': ['▁a', 'u', 'tom', 'ation'],\n",
+       " 'autumn': ['▁a', 'u', 't', 'um', 'n'],\n",
+       " 'available': ['▁a', 'v', 'a', 'il', 'able'],\n",
+       " 'avenue': ['▁a', 'v', 'en', 'u', 'e'],\n",
+       " 'average': ['▁a', 'ver', 'age'],\n",
+       " 'averages': ['▁a', 'ver', 'age', 's'],\n",
+       " 'avert': ['▁a', 'ver', 't'],\n",
+       " 'aviation': ['▁a', 'v', 'i', 'ation'],\n",
+       " 'avoid': ['▁a', 'v', 'o', 'i', 'd'],\n",
+       " 'avoided': ['▁a', 'v', 'o', 'i', 'd', 'ed'],\n",
+       " 'avon': ['▁a', 'v', 'on'],\n",
+       " 'awake': ['▁a', 'w', 'ak', 'e'],\n",
+       " 'awarded': ['▁a', 'ward', 'ed'],\n",
+       " 'awards': ['▁a', 'ward', 's'],\n",
+       " 'aware': ['▁a', 'w', 'ar', 'e'],\n",
+       " 'awareness': ['▁a', 'w', 'ar', 'e', 'ness'],\n",
+       " 'away': ['▁a', 'way'],\n",
+       " 'awful': ['▁a', 'w', 'ful'],\n",
+       " 'awfully': ['▁a', 'w', 'ful', 'ly'],\n",
+       " 'b': ['▁b'],\n",
+       " 'b.': ['▁b', '.'],\n",
+       " 'b.b.c.': ['▁b', '.', 'b', '.', 'c', '.'],\n",
+       " 'babe': ['▁ba', 'b', 'e'],\n",
+       " 'babel': ['▁ba', 'b', 'el'],\n",
+       " 'bably': ['▁b', 'abl', 'y'],\n",
+       " 'baby': ['▁ba', 'by'],\n",
+       " 'back': ['▁back'],\n",
+       " 'backbone': ['▁back', 'bo', 'ne'],\n",
+       " 'backed': ['▁back', 'ed'],\n",
+       " 'backers': ['▁back', 'ers'],\n",
+       " 'background': ['▁back', 'ground'],\n",
+       " 'backing': ['▁back', 'ing'],\n",
+       " 'backstage': ['▁back', 'st', 'age'],\n",
+       " 'backward': ['▁back', 'ward'],\n",
+       " 'bad': ['▁ba', 'd'],\n",
+       " 'badly': ['▁ba', 'd', 'ly'],\n",
+       " 'baffled': ['▁b', 'aff', 'led'],\n",
+       " 'bag': ['▁ba', 'g'],\n",
+       " 'bagaya': ['▁ba', 'g', 'a', 'y', 'a'],\n",
+       " 'baker': ['▁b', 'ak', 'er'],\n",
+       " 'balance': ['▁b', 'al', 'ance'],\n",
+       " 'balance-sheet': ['▁b', 'al', 'ance', '-', 's', 'h', 'e', 'et'],\n",
+       " 'balances': ['▁b', 'al', 'ance', 's'],\n",
+       " 'bald': ['▁ba', 'ld'],\n",
+       " 'ball': ['▁ba', 'll'],\n",
+       " 'balloon': ['▁b', 'al', 'lo', 'on'],\n",
+       " 'ballyhoo': ['▁b', 'ally', 'h', 'o', 'o'],\n",
+       " 'baltic': ['▁b', 'al', 'tic'],\n",
+       " 'ban': ['▁ba', 'n'],\n",
+       " 'ban-': ['▁ba', 'n', '-'],\n",
+       " 'ban-the-': ['▁ba', 'n', '-', 'th', 'e', '-'],\n",
+       " 'bank': ['▁bank'],\n",
+       " \"bank's\": ['▁bank', \"'\", 's'],\n",
+       " 'banking': ['▁bank', 'ing'],\n",
+       " 'bankrupt': ['▁bank', 'r', 'u', 'p', 't'],\n",
+       " 'banks': ['▁bank', 's'],\n",
+       " \"banks'\": ['▁bank', 's', \"'\"],\n",
+       " 'banned': ['▁ba', 'n', 'n', 'ed'],\n",
+       " 'banzie': ['▁ba', 'n', 'z', 'ie'],\n",
+       " 'bar': ['▁bar'],\n",
+       " 'barb': ['▁bar', 'b'],\n",
+       " 'barbarously': ['▁bar', 'b', 'ar', 'ous', 'ly'],\n",
+       " 'barclay': ['▁bar', 'c', 'lay'],\n",
+       " 'bare': ['▁bar', 'e'],\n",
+       " 'bargain': ['▁bar', 'g', 'ain'],\n",
+       " 'bargaining': ['▁bar', 'g', 'ain', 'ing'],\n",
+       " 'bark': ['▁bar', 'k'],\n",
+       " 'barrier': ['▁bar', 'ri', 'er'],\n",
+       " 'barriers': ['▁bar', 'ri', 'ers'],\n",
+       " 'barry': ['▁bar', 'ry'],\n",
+       " 'base': ['▁base'],\n",
+       " 'based': ['▁base', 'd'],\n",
+       " 'bases': ['▁base', 's'],\n",
+       " 'basic': ['▁basi', 'c'],\n",
+       " 'basin': ['▁basi', 'n'],\n",
+       " 'basing': ['▁ba', 's', 'ing'],\n",
+       " 'basis': ['▁basi', 's'],\n",
+       " 'baskerville': ['▁ba', 's', 'k', 'er', 'v', 'ill', 'e'],\n",
+       " 'basses': ['▁ba', 's', 'se', 's'],\n",
+       " 'basting': ['▁ba', 'st', 'ing'],\n",
+       " 'bats': ['▁b', 'at', 's'],\n",
+       " 'batsman': ['▁b', 'at', 's', 'man'],\n",
+       " 'battalions': ['▁b', 'at', 't', 'al', 'ion', 's'],\n",
+       " 'batting': ['▁b', 'at', 't', 'ing'],\n",
+       " 'battle': ['▁ba', 'ttle'],\n",
+       " 'bavaria': ['▁ba', 'v', 'ar', 'i', 'a'],\n",
+       " 'bavarian': ['▁ba', 'v', 'ar', 'ian'],\n",
+       " 'bavarians': ['▁ba', 'v', 'ar', 'ian', 's'],\n",
+       " 'bay': ['▁ba', 'y'],\n",
+       " 'be': ['▁be'],\n",
+       " 'beach': ['▁b', 'each'],\n",
+       " 'beaches': ['▁be', 'a', 'ches'],\n",
+       " 'beacon': ['▁be', 'a', 'c', 'on'],\n",
+       " 'beaks': ['▁be', 'ak', 's'],\n",
+       " 'bean': ['▁be', 'an'],\n",
+       " 'bear': ['▁be', 'ar'],\n",
+       " 'bearer': ['▁be', 'ar', 'er'],\n",
+       " 'bears': ['▁be', 'ar', 's'],\n",
+       " 'beastly': ['▁be', 'a', 'st', 'ly'],\n",
+       " 'beaten': ['▁be', 'at', 'en'],\n",
+       " 'beautiful': ['▁beautiful'],\n",
+       " 'beautifully': ['▁beautiful', 'ly'],\n",
+       " 'beauty': ['▁be', 'a', 'u', 'ty'],\n",
+       " 'became': ['▁be', 'c', 'a', 'me'],\n",
+       " 'because': ['▁be', 'cause'],\n",
+       " 'beckoning': ['▁be', 'ck', 'on', 'ing'],\n",
+       " 'become': ['▁become'],\n",
+       " 'becomes': ['▁become', 's'],\n",
+       " 'becoming': ['▁be', 'coming'],\n",
+       " 'bedlam': ['▁be', 'd', 'l', 'a', 'm'],\n",
+       " 'beds': ['▁be', 'd', 's'],\n",
+       " 'bedspreads': ['▁be', 'd', 's', 'p', 're', 'ad', 's'],\n",
+       " 'beech': ['▁be', 'e', 'ch'],\n",
+       " 'been': ['▁be', 'en'],\n",
+       " 'before': ['▁be', 'fore'],\n",
+       " 'befriended': ['▁be', 'f', 'ri', 'end', 'ed'],\n",
+       " 'began': ['▁be', 'g', 'an'],\n",
+       " 'begin': ['▁be', 'g', 'in'],\n",
+       " 'beginner': ['▁be', 'g', 'in', 'n', 'er'],\n",
+       " 'beginning': ['▁be', 'g', 'in', 'n', 'ing'],\n",
+       " 'begun': ['▁be', 'g', 'un'],\n",
+       " 'behan': ['▁be', 'h', 'an'],\n",
+       " 'behaviour': ['▁be', 'h', 'a', 'v', 'i', 'our'],\n",
+       " 'behind': ['▁be', 'h', 'ind'],\n",
+       " 'beier': ['▁be', 'i', 'er'],\n",
+       " 'being': ['▁be', 'ing'],\n",
+       " 'belgian': ['▁belgi', 'an'],\n",
+       " 'belgium': ['▁belgi', 'um'],\n",
+       " 'belgrade': ['▁be', 'l', 'g', 'rade'],\n",
+       " 'belief': ['▁be', 'li', 'e', 'f'],\n",
+       " 'believe': ['▁believe'],\n",
+       " 'believed': ['▁believe', 'd'],\n",
+       " 'believes': ['▁believe', 's'],\n",
+       " 'bell': ['▁be', 'll'],\n",
+       " \"bell's\": ['▁be', 'll', \"'\", 's'],\n",
+       " 'belonged': ['▁be', 'lo', 'ng', 'ed'],\n",
+       " 'belongs': ['▁be', 'lo', 'ng', 's'],\n",
+       " 'below': ['▁be', 'low'],\n",
+       " 'belt': ['▁be', 'l', 't'],\n",
+       " 'ben': ['▁be', 'n'],\n",
+       " 'bench': ['▁be', 'n', 'ch'],\n",
+       " 'benches': ['▁be', 'n', 'ches'],\n",
+       " 'bend': ['▁b', 'end'],\n",
+       " 'bending': ['▁b', 'end', 'ing'],\n",
+       " 'benefits': ['▁be', 'ne', 'f', 'it', 's'],\n",
+       " 'bent': ['▁b', 'ent'],\n",
+       " 'ber': ['▁be', 'r'],\n",
+       " 'berlin': ['▁berlin'],\n",
+       " \"berlin's\": ['▁berlin', \"'\", 's'],\n",
+       " 'bernhard': ['▁b', 'ern', 'h', 'ard'],\n",
+       " 'berry': ['▁be', 'rry'],\n",
+       " 'bertrand': ['▁be', 'r', 't', 'r', 'and'],\n",
+       " 'beset': ['▁be', 's', 'et'],\n",
+       " 'beside': ['▁be', 'side'],\n",
+       " 'best': ['▁best'],\n",
+       " 'best-seller': ['▁best', '-', 'se', 'll', 'er'],\n",
+       " 'bet': ['▁be', 't'],\n",
+       " 'betjeman': ['▁be', 't', 'j', 'e', 'man'],\n",
+       " 'betrayal': ['▁be', 't', 'ra', 'y', 'al'],\n",
+       " 'betrayed': ['▁be', 't', 'ra', 'y', 'ed'],\n",
+       " 'better': ['▁better'],\n",
+       " 'better-': ['▁better', '-'],\n",
+       " \"betti's\": ['▁be', 'tt', 'i', \"'\", 's'],\n",
+       " 'between': ['▁be', 'twe', 'en'],\n",
+       " 'bevel': ['▁be', 've', 'l'],\n",
+       " 'bevelled': ['▁be', 've', 'll', 'ed'],\n",
+       " 'beware': ['▁be', 'w', 'ar', 'e'],\n",
+       " 'bewildered': ['▁be', 'w', 'il', 'd', 'er', 'ed'],\n",
+       " 'beyond': ['▁be', 'y', 'on', 'd'],\n",
+       " 'bidet': ['▁b', 'i', 'd', 'et'],\n",
+       " 'big': ['▁big'],\n",
+       " 'bigger': ['▁big', 'g', 'er'],\n",
+       " 'biggest': ['▁big', 'g', 'est'],\n",
+       " 'bill': ['▁b', 'ill'],\n",
+       " 'bills': ['▁b', 'ill', 's'],\n",
+       " 'binding': ['▁b', 'ind', 'ing'],\n",
+       " 'biological': ['▁b', 'i', 'o', 'lo', 'g', 'ic', 'al'],\n",
+       " 'bird': ['▁b', 'ir', 'd'],\n",
+       " 'birds': ['▁b', 'ir', 'd', 's'],\n",
+       " 'bishop': ['▁b', 'ish', 'op'],\n",
+       " 'bit': ['▁', 'bit'],\n",
+       " 'bite': ['▁', 'bit', 'e'],\n",
+       " 'bits': ['▁', 'bit', 's'],\n",
+       " 'bitter-sweet': ['▁', 'bit', 'ter', '-', 's', 'w', 'e', 'et'],\n",
+       " 'bitterest': ['▁', 'bit', 'ter', 'est'],\n",
+       " 'bitterly': ['▁', 'bit', 'ter', 'ly'],\n",
+       " 'bituminized': ['▁', 'bit', 'um', 'in', 'i', 'z', 'ed'],\n",
+       " 'black': ['▁b', 'l', 'a', 'ck'],\n",
+       " 'black-': ['▁b', 'l', 'a', 'ck', '-'],\n",
+       " 'blackbird': ['▁b', 'l', 'a', 'ck', 'b', 'ir', 'd'],\n",
+       " 'blacks': ['▁b', 'l', 'a', 'ck', 's'],\n",
+       " 'blame': ['▁b', 'l', 'a', 'me'],\n",
+       " 'blamed': ['▁b', 'l', 'a', 'm', 'ed'],\n",
+       " 'blander': ['▁b', 'land', 'er'],\n",
+       " 'blank': ['▁b', 'l', 'an', 'k'],\n",
+       " 'blight': ['▁b', 'l', 'ight'],\n",
+       " 'blind': ['▁b', 'lin', 'd'],\n",
+       " 'blinked': ['▁b', 'lin', 'k', 'ed'],\n",
+       " 'block': ['▁block'],\n",
+       " 'blocks': ['▁block', 's'],\n",
+       " 'bloem-': ['▁b', 'lo', 'e', 'm', '-'],\n",
+       " 'blond': ['▁b', 'l', 'on', 'd'],\n",
+       " 'blood': ['▁b', 'l', 'ood'],\n",
+       " 'bloodstained': ['▁b', 'l', 'ood', 'st', 'ain', 'ed'],\n",
+       " 'bloody': ['▁b', 'l', 'ood', 'y'],\n",
+       " 'blouse': ['▁b', 'l', 'ous', 'e'],\n",
+       " 'blouses': ['▁b', 'l', 'ous', 'es'],\n",
+       " 'blow': ['▁b', 'low'],\n",
+       " 'blowflies': ['▁b', 'low', 'f', 'l', 'ies'],\n",
+       " 'blown': ['▁b', 'low', 'n'],\n",
+       " 'blue': ['▁b', 'l', 'u', 'e'],\n",
+       " 'blunt': ['▁b', 'l', 'un', 't'],\n",
+       " 'bluntly': ['▁b', 'l', 'un', 't', 'ly'],\n",
+       " 'bluster': ['▁b', 'l', 'us', 'ter'],\n",
+       " 'board': ['▁bo', 'ard'],\n",
+       " 'boat': ['▁bo', 'at'],\n",
+       " 'boat-train': ['▁bo', 'at', '-', 'train'],\n",
+       " 'bobby': ['▁bo', 'b', 'by'],\n",
+       " 'bodies': ['▁bo', 'd', 'ies'],\n",
+       " 'body': ['▁', 'body'],\n",
+       " 'boeing': ['▁bo', 'e', 'ing'],\n",
+       " 'bogy': ['▁bo', 'g', 'y'],\n",
+       " 'boiled': ['▁bo', 'il', 'ed'],\n",
+       " 'boils': ['▁bo', 'il', 's'],\n",
+       " 'bold': ['▁bo', 'ld'],\n",
+       " 'boldly': ['▁bo', 'ld', 'ly'],\n",
+       " 'bolted': ['▁bo', 'l', 't', 'ed'],\n",
+       " 'bomb': ['▁', 'bomb'],\n",
+       " 'bombay': ['▁', 'bomb', 'a', 'y'],\n",
+       " 'bombed': ['▁', 'bomb', 'ed'],\n",
+       " 'bombers': ['▁bo', 'mber', 's'],\n",
+       " 'bonded': ['▁bo', 'n', 'd', 'ed'],\n",
+       " 'bone': ['▁bo', 'ne'],\n",
+       " 'bones': ['▁bo', 'ne', 's'],\n",
+       " 'bonn': ['▁bo', 'n', 'n'],\n",
+       " 'book': ['▁bo', 'ok'],\n",
+       " 'booklet': ['▁bo', 'ok', 'le', 't'],\n",
+       " 'books': ['▁bo', 'ok', 's'],\n",
+       " 'booming': ['▁bo', 'o', 'm', 'ing'],\n",
+       " 'border': ['▁b', 'or', 'd', 'er'],\n",
+       " 'bore': ['▁bo', 're'],\n",
+       " 'bored': ['▁b', 'or', 'ed'],\n",
+       " 'boredom': ['▁b', 'or', 'ed', 'o', 'm'],\n",
+       " 'bores': ['▁bo', 're', 's'],\n",
+       " 'borough': ['▁b', 'or', 'ough'],\n",
+       " 'borrow': ['▁b', 'or', 'ro', 'w'],\n",
+       " 'borstal': ['▁b', 'or', 'st', 'al'],\n",
+       " 'bosoms': ['▁bo', 's', 'o', 'm', 's'],\n",
+       " 'bossed': ['▁bo', 's', 's', 'ed'],\n",
+       " 'bosses': ['▁bo', 's', 'se', 's'],\n",
+       " 'both': ['▁bo', 'th'],\n",
+       " 'bottle': ['▁bo', 'ttle'],\n",
+       " 'bottom': ['▁bo', 't', 'tom'],\n",
+       " 'bought': ['▁b', 'ought'],\n",
+       " 'boun': ['▁bo', 'un'],\n",
+       " 'bound': ['▁b', 'ound'],\n",
+       " 'boutiques': ['▁b', 'out', 'i', 'q', 'u', 'es'],\n",
+       " 'bow': ['▁bo', 'w'],\n",
+       " 'bow-street': ['▁bo', 'w', '-', 'street'],\n",
+       " 'bowed': ['▁bo', 'w', 'ed'],\n",
+       " 'bowing': ['▁bo', 'w', 'ing'],\n",
+       " 'bows': ['▁bo', 'w', 's'],\n",
+       " 'box': ['▁bo', 'x'],\n",
+       " 'boxes': ['▁bo', 'x', 'es'],\n",
+       " 'boxing': ['▁bo', 'x', 'ing'],\n",
+       " 'boy': ['▁bo', 'y'],\n",
+       " 'boycotted': ['▁bo', 'y', 'c', 'o', 'tt', 'ed'],\n",
+       " 'boycotting': ['▁bo', 'y', 'c', 'o', 'tt', 'ing'],\n",
+       " 'boyle': ['▁bo', 'y', 'le'],\n",
+       " 'boys': ['▁bo', 'y', 's'],\n",
+       " 'braces': ['▁br', 'a', 'ce', 's'],\n",
+       " 'brain': ['▁br', 'ain'],\n",
+       " 'brain-activity': ['▁br', 'ain', '-', 'a', 'ct', 'i', 'v', 'it', 'y'],\n",
+       " 'brain-children': ['▁br', 'ain', '-', 'ch', 'il', 'd', 'ren'],\n",
+       " 'brains': ['▁br', 'ain', 's'],\n",
+       " 'brandy': ['▁br', 'and', 'y'],\n",
+       " 'brash': ['▁br', 'ash'],\n",
+       " 'brass': ['▁br', 'a', 's', 's'],\n",
+       " 'brauchitsch': ['▁br', 'a', 'uch', 'it', 's', 'ch'],\n",
+       " 'breach': ['▁br', 'each'],\n",
+       " 'bread-and-butter': ['▁br', 'e', 'ad', '-', 'and', '-', 'b', 'u', 't', 'ter'],\n",
+       " 'break': ['▁br', 'e', 'ak'],\n",
+       " 'breaking': ['▁br', 'e', 'ak', 'ing'],\n",
+       " 'breaks': ['▁br', 'e', 'ak', 's'],\n",
+       " 'breath': ['▁br', 'e', 'at', 'h'],\n",
+       " 'breathing': ['▁br', 'e', 'a', 'thing'],\n",
+       " 'breathless': ['▁br', 'e', 'at', 'h', 'less'],\n",
+       " 'breeding': ['▁br', 'e', 'ed', 'ing'],\n",
+       " 'breezily': ['▁br', 'e', 'e', 'z', 'i', 'ly'],\n",
+       " 'brenda': ['▁br', 'end', 'a'],\n",
+       " 'brendan': ['▁br', 'end', 'an'],\n",
+       " \"brendan's\": ['▁br', 'end', 'an', \"'\", 's'],\n",
+       " 'brentano': ['▁br', 'ent', 'an', 'o'],\n",
+       " 'brezhnev': ['▁br', 'e', 'z', 'h', 'ne', 'v'],\n",
+       " 'brian': ['▁br', 'ian'],\n",
+       " 'bridal': ['▁', 'brid', 'al'],\n",
+       " 'bride': ['▁', 'brid', 'e'],\n",
+       " 'brief': ['▁brief'],\n",
+       " 'briefcase': ['▁brief', 'case'],\n",
+       " 'briefing': ['▁brief', 'ing'],\n",
+       " 'brigadiers': ['▁br', 'ig', 'ad', 'i', 'ers'],\n",
+       " 'bright': ['▁br', 'ight'],\n",
+       " 'brighter': ['▁br', 'ight', 'er'],\n",
+       " 'brightly': ['▁br', 'ight', 'ly'],\n",
+       " \"brighton's\": ['▁br', 'ight', 'on', \"'\", 's'],\n",
+       " 'brilliant': ['▁br', 'ill', 'ian', 't'],\n",
+       " 'brilliantly': ['▁br', 'ill', 'ian', 't', 'ly'],\n",
+       " 'bring': ['▁br', 'ing'],\n",
+       " 'brings': ['▁br', 'ing', 's'],\n",
+       " 'bristled': ['▁br', 'is', 't', 'led'],\n",
+       " 'britain': ['▁britain'],\n",
+       " \"britain's\": ['▁britain', \"'\", 's'],\n",
+       " 'british': ['▁british'],\n",
+       " 'british-owned': ['▁british', '-', 'ow', 'n', 'ed'],\n",
+       " 'britishers': ['▁british', 'ers'],\n",
+       " 'brittle': ['▁br', 'i', 'ttle'],\n",
+       " 'broad': ['▁br', 'o', 'ad'],\n",
+       " 'broadcast': ['▁br', 'o', 'ad', 'c', 'a', 'st'],\n",
+       " 'broadcasting': ['▁br', 'o', 'ad', 'c', 'a', 'st', 'ing'],\n",
+       " 'broke': ['▁br', 'ok', 'e'],\n",
+       " 'broken': ['▁br', 'ok', 'en'],\n",
+       " \"brook's\": ['▁br', 'o', 'ok', \"'\", 's'],\n",
+       " 'brother': ['▁br', 'other'],\n",
+       " 'brother-': ['▁br', 'other', '-'],\n",
+       " 'brother-in-law': ['▁br', 'other', '-', 'in', '-', 'law'],\n",
+       " 'brought': ['▁br', 'ought'],\n",
+       " 'brown': ['▁brown'],\n",
+       " \"brown's\": ['▁brown', \"'\", 's'],\n",
+       " 'bru\"cke': ['▁br', 'u', '\"', 'ck', 'e'],\n",
+       " 'bruce': ['▁br', 'u', 'ce'],\n",
+       " 'bruno': ['▁br', 'un', 'o'],\n",
+       " 'brunswick': ['▁br', 'un', 's', 'w', 'i', 'ck'],\n",
+       " 'brussels': ['▁br', 'us', 's', 'el', 's'],\n",
+       " 'brutal': ['▁br', 'u', 't', 'al'],\n",
+       " 'bryan': ['▁br', 'y', 'an'],\n",
+       " 'buck': ['▁b', 'u', 'ck'],\n",
+       " 'buckingham': ['▁b', 'u', 'ck', 'ing', 'h', 'a', 'm'],\n",
+       " 'buckley': ['▁b', 'u', 'ck', 'le', 'y'],\n",
+       " 'budge': ['▁b', 'ud', 'ge'],\n",
+       " 'budgerigar': ['▁b', 'ud', 'g', 'er', 'ig', 'ar'],\n",
+       " 'budget': ['▁b', 'ud', 'ge', 't'],\n",
+       " 'budgetary': ['▁b', 'ud', 'ge', 't', 'ar', 'y'],\n",
+       " 'budgette': ['▁b', 'ud', 'ge', 't', 'te'],\n",
+       " 'buganda': ['▁b', 'ug', 'and', 'a'],\n",
+       " 'build': ['▁b', 'u', 'il', 'd'],\n",
+       " 'building': ['▁b', 'u', 'il', 'd', 'ing'],\n",
+       " 'built': ['▁b', 'u', 'il', 't'],\n",
+       " 'bulk': ['▁b', 'ul', 'k'],\n",
+       " 'bulky': ['▁b', 'ul', 'k', 'y'],\n",
+       " 'bulletins': ['▁b', 'ul', 'le', 't', 'in', 's'],\n",
+       " 'bullying': ['▁b', 'ul', 'ly', 'ing'],\n",
+       " 'bumps': ['▁b', 'um', 'p', 's'],\n",
+       " 'bundle': ['▁b', 'un', 'd', 'le'],\n",
+       " 'bundy': ['▁b', 'un', 'd', 'y'],\n",
+       " 'bunk': ['▁b', 'un', 'k'],\n",
+       " 'burden': ['▁burde', 'n'],\n",
+       " 'burdens': ['▁burde', 'n', 's'],\n",
+       " 'burdett': ['▁burde', 'tt'],\n",
+       " 'bureau': ['▁b', 'ure', 'a', 'u'],\n",
+       " 'burge': ['▁b', 'ur', 'ge'],\n",
+       " 'burning': ['▁b', 'ur', 'n', 'ing'],\n",
+       " 'burst': ['▁b', 'ur', 'st'],\n",
+       " 'bush': ['▁bus', 'h'],\n",
+       " 'busi-': ['▁bus', 'i', '-'],\n",
+       " 'business': ['▁bus', 'i', 'ness'],\n",
+       " 'bustling': ['▁bus', 't', 'l', 'ing'],\n",
+       " 'busy': ['▁bus', 'y'],\n",
+       " 'but': ['▁but'],\n",
+       " 'butagas': ['▁but', 'a', 'g', 'a', 's'],\n",
+       " 'butler': ['▁but', 'l', 'er'],\n",
+       " 'butt': ['▁but', 't'],\n",
+       " 'buttery': ['▁but', 'ter', 'y'],\n",
+       " 'button': ['▁but', 'ton'],\n",
+       " 'buy': ['▁b', 'u', 'y'],\n",
+       " 'buying': ['▁b', 'u', 'y', 'ing'],\n",
+       " 'by': ['▁by'],\n",
+       " 'by-': ['▁by', '-'],\n",
+       " 'by-elections': ['▁by', '-', 'elect', 'ion', 's'],\n",
+       " 'by-passed': ['▁by', '-', 'p', 'a', 's', 's', 'ed'],\n",
+       " 'byrne': ['▁by', 'r', 'ne'],\n",
+       " 'byron': ['▁by', 'ro', 'n'],\n",
+       " 'c': ['▁c'],\n",
+       " 'c.c.f.': ['▁c', '.', 'c', '.', 'f', '.'],\n",
+       " 'c.i.g.s.': ['▁c', '.', 'i', '.', 'g', '.', 's', '.'],\n",
+       " 'cabinet': ['▁c', 'a', 'b', 'ine', 't'],\n",
+       " 'cable': ['▁c', 'able'],\n",
+       " 'cables': ['▁c', 'able', 's'],\n",
+       " 'cagnes': ['▁c', 'a', 'g', 'ne', 's'],\n",
+       " 'call': ['▁call'],\n",
+       " 'call-out': ['▁call', '-', 'out'],\n",
+       " 'callaghan': ['▁call', 'a', 'g', 'h', 'an'],\n",
+       " 'called': ['▁call', 'ed'],\n",
+       " 'calls': ['▁call', 's'],\n",
+       " 'cally': ['▁call', 'y'],\n",
+       " 'calm': ['▁c', 'al', 'm'],\n",
+       " 'calorie': ['▁c', 'al', 'or', 'ie'],\n",
+       " 'cambodia': ['▁cam', 'bo', 'd', 'i', 'a'],\n",
+       " 'cambridge': ['▁cam', 'brid', 'ge'],\n",
+       " 'came': ['▁cam', 'e'],\n",
+       " 'camp': ['▁cam', 'p'],\n",
+       " 'campaign': ['▁campaign'],\n",
+       " 'campaigns': ['▁campaign', 's'],\n",
+       " 'can': ['▁can'],\n",
+       " \"can't\": ['▁can', \"'\", 't'],\n",
+       " 'can-': ['▁can', '-'],\n",
+       " 'canada': ['▁can', 'ad', 'a'],\n",
+       " 'canadian': ['▁can', 'ad', 'ian'],\n",
+       " 'canadians': ['▁can', 'ad', 'ian', 's'],\n",
+       " 'canals': ['▁can', 'al', 's'],\n",
+       " 'canard': ['▁can', 'ard'],\n",
+       " 'cancelled': ['▁can', 'ce', 'll', 'ed'],\n",
+       " 'cancerous': ['▁can', 'c', 'er', 'ous'],\n",
+       " 'candi-': ['▁can', 'd', 'i', '-'],\n",
+       " 'candidate': ['▁can', 'did', 'at', 'e'],\n",
+       " 'candles': ['▁can', 'd', 'le', 's'],\n",
+       " 'cannot': ['▁can', 'not'],\n",
+       " 'canon': ['▁can', 'on'],\n",
+       " 'cantabile': ['▁can', 't', 'a', 'b', 'il', 'e'],\n",
+       " 'canteen': ['▁can', 'te', 'en'],\n",
+       " ...}"
+      ]
+     },
+     "execution_count": 14,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "processor.lexicon"
+   ]
+  },
+  {
+   "cell_type": "code",
    "execution_count": 167,
    "id": "d08a0259",
    "metadata": {},
diff --git a/notebooks/04b-look-at-iam-paragraphs.ipynb b/notebooks/04b-look-at-iam-paragraphs.ipynb
index dc0aef6..11ebddf 100644
--- a/notebooks/04b-look-at-iam-paragraphs.ipynb
+++ b/notebooks/04b-look-at-iam-paragraphs.ipynb
@@ -256,7 +256,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.8.2"
+   "version": "3.9.1"
   }
  },
  "nbformat": 4,