zhangshuai/mlx-swift-examples
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add command r model support (#35)

Browse Source 
* feat: add command r model support
This commit is contained in:
Anchen
2024-03-27 06:43:10 +11:00
committed by GitHub
parent c37018d7d2
commit 63531bc5fa
5 changed files with 254 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
Libraries/LLM/Tokenizer.swift
View File

@@ -28,5 +28,6 @@ public func loadTokenizer(configuration: ModelConfiguration) async throws -> Tok
/// overrides for TokenizerModel/knownTokenizers
let replacementTokenizers = [
"Qwen2Tokenizer": "PreTrainedTokenizer"
"Qwen2Tokenizer": "PreTrainedTokenizer",
"CohereTokenizer": "PreTrainedTokenizer",
]