Add support for Split pretokenizer w/ behavior=removed & `invert=fa…

…lse`
huggingface · Nov 16, 2024 · 7c1bde9 · 7c1bde9
1 parent 7b1ce3c
commit 7c1bde9
Show file tree

Hide file tree

Showing 2 changed files with 12 additions and 0 deletions.
diff --git a/src/tokenizers.js b/src/tokenizers.js
@@ -1518,6 +1518,8 @@ class SplitPreTokenizer extends PreTokenizer {
 
         if (this.config.invert) {
             return text.match(this.pattern) || [];
+        } else if (this.config.behavior?.toLowerCase() === 'removed') {
+            return text.split(this.pattern).filter(x => x);
         } else {
             return regexSplit(text, this.pattern);
         }

diff --git a/tests/models/roberta/tokenization.js b/tests/models/roberta/tokenization.js
@@ -691,4 +691,14 @@ export const TEST_CONFIG = {
       decoded: "<s> \tH\u00e4LLo!how  \n Are yoU?  </s>",
     },
   },
+
+  // Split tokenizer with behavior="Removed" and invert=false
+  "onnx-community/camembertv2-base": {
+    SIMPLE: {
+      text: BASE_TEST_STRINGS.SIMPLE,
+      tokens: ['How', 'are', 'you', 'doi', '##ng', '?'],
+      ids: [1, 14473, 9556, 10577, 6471, 9274, 38, 2],
+      decoded: "[CLS] How are you doing? [SEP]",
+    }
+  },
 };