allow alternate location for tokenizer

Libraries/LLM/Models.swift

@@ -11,6 +11,9 @@ import Foundation
 public struct ModelConfiguration {
     public let id: String
 
+    /// pull the tokenizer from an alternate id
+    public let tokenizerId: String?
+
     /// overrides for TokenizerModel/knownTokenizers -- useful before swift-transformers is updated
     public let overrideTokenizer: String?
 
@@ -20,9 +23,11 @@ public struct ModelConfiguration {
     private let preparePrompt: ((String) -> String)?
 
     public init(
-        id: String, overrideTokenizer: String? = nil, preparePrompt: ((String) -> String)? = nil
+        id: String, tokenizerId: String? = nil, overrideTokenizer: String? = nil,
+        preparePrompt: ((String) -> String)? = nil
     ) {
         self.id = id
+        self.tokenizerId = tokenizerId
         self.overrideTokenizer = overrideTokenizer
         self.preparePrompt = preparePrompt
     }

Libraries/LLM/Tokenizer.swift

@@ -51,7 +51,8 @@ public struct Tokenizer: Tokenizers.Tokenizer {
 
 public func loadTokenizer(configuration: ModelConfiguration) async throws -> Tokenizer {
     // from AutoTokenizer.from() -- this lets us override parts of the configuration
-    let config = LanguageModelConfigurationFromHub(modelName: configuration.id)
+    let config = LanguageModelConfigurationFromHub(
+        modelName: configuration.tokenizerId ?? configuration.id)
     guard var tokenizerConfig = try await config.tokenizerConfig else {
         throw LLMError(message: "missing config")
     }