diff options
| author | aktersnurra <gustaf.rydholm@gmail.com> | 2021-02-24 22:00:29 +0100 |
|---|---|---|
| committer | aktersnurra <gustaf.rydholm@gmail.com> | 2021-02-24 22:00:29 +0100 |
| commit | 905eeeb4c3c0ba54b5414eb8f435e2e9870b7307 (patch) | |
| tree | 91dab598a94911e6147b996237e786dd47f11f2f /src/tasks/make_wordpieces.py | |
| parent | 4a54d7e690897dd6e6c719fb908fd371a44c2952 (diff) | |
updates
Diffstat (limited to 'src/tasks/make_wordpieces.py')
| -rw-r--r-- | src/tasks/make_wordpieces.py | 2 |
1 files changed, 1 insertions, 1 deletions
diff --git a/src/tasks/make_wordpieces.py b/src/tasks/make_wordpieces.py index f605920..2ac0e2c 100644 --- a/src/tasks/make_wordpieces.py +++ b/src/tasks/make_wordpieces.py @@ -30,7 +30,7 @@ def iamdb_pieces( user_symbols=["/"], # added so token is in the output set ) - vocab = sorted(set(w for t in text for w in t.split("_") if w)) + vocab = sorted(set(w for t in text for w in t.split("▁") if w)) if "move" not in vocab: raise RuntimeError("`MOVE` not in vocab") |