Fix extra EOS tokens (#91)

* Fix extra EOS tokens

* Fix pre-commit error

.pre-commit-config.yaml

@@ -1,6 +1,6 @@
 repos:
 - repo: https://github.com/slessans/pre-commit-swift-format
-  rev: ""
+  rev: "fd627de92bdf84a75c924ed95691336d14e94cf1"
   hooks:
     - id: swift-format
       args: ["--configuration", ".swift-format"]

Libraries/LLM/Evaluate.swift

@@ -183,19 +183,10 @@ public func generate(
     var start = Date.timeIntervalSinceReferenceDate
     var promptTime: TimeInterval = 0
 
-    // build a set of additional stop tokens
     let additionalEOSTokenIds = Set(
         (extraEOSTokens ?? [])
-            .map {
+            .compactMap {
-                tokenizer.encode(text: $0)
+                tokenizer.convertTokenToId($0)
-            }
-            .filter {
-                // discard anything that is not a single token.  sometimes
-                // the tokenizer will insert a <s> token, so accept that too
-                $0.count == 1 || ($0.count == 2 && $0[0] == 1)
-            }
-            .map {
-                $0.last!
             })
 
     var tokens = [Int]()