feat: add missing gpt-5 models

Author: dqbd

.changeset/pink-apes-decide.md

@@ -0,0 +1,7 @@
+---
+"tiktoken": patch
+"js-tiktoken": patch
+"@dqbd/tiktoken": patch
+---
+
+Add gpt-5 models

js/src/core.ts

@@ -318,7 +318,14 @@ export function getEncodingNameForModel(model: TiktokenModel) {
     case "gpt-4.1-nano":
     case "gpt-4.1-nano-2025-04-14":
     case "gpt-4.5-preview":
-    case "gpt-4.5-preview-2025-02-27": {
+    case "gpt-4.5-preview-2025-02-27":
+    case "gpt-5":
+    case "gpt-5-2025-08-07":
+    case "gpt-5-nano":
+    case "gpt-5-nano-2025-08-07":
+    case "gpt-5-mini":
+    case "gpt-5-mini-2025-08-07":
+    case "gpt-5-chat-latest": {
       return "o200k_base";
     }
     default:

tiktoken/model_to_encoding.json

@@ -97,5 +97,12 @@
     "gpt-4.1-nano": "o200k_base",
     "gpt-4.1-nano-2025-04-14": "o200k_base",
     "gpt-4.5-preview": "o200k_base",
-    "gpt-4.5-preview-2025-02-27": "o200k_base"
+    "gpt-4.5-preview-2025-02-27": "o200k_base",
+    "gpt-5": "o200k_base",
+    "gpt-5-2025-08-07": "o200k_base",
+    "gpt-5-nano": "o200k_base",
+    "gpt-5-nano-2025-08-07": "o200k_base",
+    "gpt-5-mini": "o200k_base",
+    "gpt-5-mini-2025-08-07": "o200k_base",
+    "gpt-5-chat-latest": "o200k_base"
 }

wasm/src/lib.rs

@@ -462,7 +462,14 @@ export type TiktokenModel =
     | "gpt-4.1-nano"
     | "gpt-4.1-nano-2025-04-14"
     | "gpt-4.5-preview"
-    | "gpt-4.5-preview-2025-02-27";
+    | "gpt-4.5-preview-2025-02-27"
+    | "gpt-5"
+    | "gpt-5-2025-08-07"
+    | "gpt-5-nano"
+    | "gpt-5-nano-2025-08-07"
+    | "gpt-5-mini"
+    | "gpt-5-mini-2025-08-07"
+    | "gpt-5-chat-latest";
 
 /**
  * @param {TiktokenModel} encoding
@@ -578,6 +585,13 @@ pub fn encoding_for_model(
         "gpt-4.1-nano-2025-04-14" => Ok("o200k_base"),
         "gpt-4.5-preview" => Ok("o200k_base"),
         "gpt-4.5-preview-2025-02-27" => Ok("o200k_base"),
+        "gpt-5" => Ok("o200k_base"),
+        "gpt-5-2025-08-07" => Ok("o200k_base"),
+        "gpt-5-nano" => Ok("o200k_base"),
+        "gpt-5-nano-2025-08-07" => Ok("o200k_base"),
+        "gpt-5-mini" => Ok("o200k_base"),
+        "gpt-5-mini-2025-08-07" => Ok("o200k_base"),
+        "gpt-5-chat-latest" => Ok("o200k_base"),
         model => Err(JsError::new(
             format!("Invalid model: {}", model.to_string()).as_str(),
         )),