add reference to filed issue

2024-02-26 13:31:33 -08:00
parent c86d1c195e
commit a2ff291608
1 changed files with 1 additions and 0 deletions
--- a/Libraries/LLM/Load.swift
+++ b/Libraries/LLM/Load.swift
@@ -93,6 +93,7 @@ public func loadTokenizer(name: String) async throws -> Tokenizer {

 public func discardUnhandledMerges(tokenizerData: Config) -> Config {
    // see https://github.com/ml-explore/mlx-swift-examples/issues/1
+    // and https://github.com/huggingface/swift-transformers/issues/51

    if let model = tokenizerData.model {
        if let merges = model.dictionary["merges"] as? [String] {