Remove Tokenizer wrapper (#28)

* Remove Tokenizer wrapper Depends on https://github.com/huggingface/swift-transformers/pull/70 * Update swift-transformers revision
2024-03-18 17:07:52 +01:00
parent 0af2efe8a0
commit 6c270a9d12
3 changed files with 4 additions and 50 deletions
--- a/Tools/llm-tool/LLMTool.swift
+++ b/Tools/llm-tool/LLMTool.swift
@@ -5,6 +5,7 @@ import Foundation
 import LLM
 import MLX
 import MLXRandom
+import Tokenizers

@main
 struct LLMTool: AsyncParsableCommand {