allow alternate location for tokenizer

2024-03-01 23:27:03 -08:00
parent fe116f857d
commit 7b746cb89c
2 changed files with 8 additions and 2 deletions
--- a/Libraries/LLM/Tokenizer.swift
+++ b/Libraries/LLM/Tokenizer.swift
@@ -51,7 +51,8 @@ public struct Tokenizer: Tokenizers.Tokenizer {

 public func loadTokenizer(configuration: ModelConfiguration) async throws -> Tokenizer {
    // from AutoTokenizer.from() -- this lets us override parts of the configuration
-    let config = LanguageModelConfigurationFromHub(modelName: configuration.id)
+    let config = LanguageModelConfigurationFromHub(
+        modelName: configuration.tokenizerId ?? configuration.id)
    guard var tokenizerConfig = try await config.tokenizerConfig else {
        throw LLMError(message: "missing config")
    }