feat: Qwen2 support

2024-03-03 22:26:28 +08:00
parent 7b746cb89c
commit 66d9202360
7 changed files with 291 additions and 2 deletions
--- a/Libraries/LLM/Tokenizer.swift
+++ b/Libraries/LLM/Tokenizer.swift
@@ -116,4 +116,5 @@ public func discardUnhandledMerges(tokenizerData: Config) -> Config {
 let replacementTokenizers = [
    "CodeLlamaTokenizer": "LlamaTokenizer",
    "GemmaTokenizer": "PreTrainedTokenizer",
+    "Qwen2Tokenizer": "PreTrainedTokenizer",
 ]