zhangshuai/mlx-swift-examples
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

allow alternate location for tokenizer

Browse Source
This commit is contained in:
David Koski
2024-03-01 23:27:03 -08:00
parent fe116f857d
commit 7b746cb89c
2 changed files with 8 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
Libraries/LLM/Models.swift
View File

@@ -11,6 +11,9 @@ import Foundation
public struct ModelConfiguration { public struct ModelConfiguration {
public let id: String public let id: String
/// pull the tokenizer from an alternate id
public let tokenizerId: String?
/// overrides for TokenizerModel/knownTokenizers -- useful before swift-transformers is updated /// overrides for TokenizerModel/knownTokenizers -- useful before swift-transformers is updated
public let overrideTokenizer: String? public let overrideTokenizer: String?
@@ -20,9 +23,11 @@ public struct ModelConfiguration {
private let preparePrompt: ((String) -> String)? private let preparePrompt: ((String) -> String)?
public init( public init(
id: String, overrideTokenizer: String? = nil, preparePrompt: ((String) -> String)? = nil id: String, tokenizerId: String? = nil, overrideTokenizer: String? = nil,
preparePrompt: ((String) -> String)? = nil
) { ) {
self.id = id self.id = id
self.tokenizerId = tokenizerId
self.overrideTokenizer = overrideTokenizer self.overrideTokenizer = overrideTokenizer
self.preparePrompt = preparePrompt self.preparePrompt = preparePrompt
} }

3
Libraries/LLM/Tokenizer.swift
View File

@@ -51,7 +51,8 @@ public struct Tokenizer: Tokenizers.Tokenizer {
public func loadTokenizer(configuration: ModelConfiguration) async throws -> Tokenizer { public func loadTokenizer(configuration: ModelConfiguration) async throws -> Tokenizer {
// from AutoTokenizer.from() -- this lets us override parts of the configuration // from AutoTokenizer.from() -- this lets us override parts of the configuration
let config = LanguageModelConfigurationFromHub(modelName: configuration.id) let config = LanguageModelConfigurationFromHub(
modelName: configuration.tokenizerId ?? configuration.id)
guard var tokenizerConfig = try await config.tokenizerConfig else { guard var tokenizerConfig = try await config.tokenizerConfig else {
throw LLMError(message: "missing config") throw LLMError(message: "missing config")
} }