Api: Add token endpoints

Support for encoding and decoding with various parameters.

Signed-off-by: kingbri <bdashore3@proton.me>

OAI/types/token.py

@@ -0,0 +1,30 @@
+from pydantic import BaseModel
+from typing import List
+
+class CommonTokenRequest(BaseModel):
+    add_bos: bool = True
+    encode_special_tokens: bool = True
+    decode_special_tokens: bool = True
+
+    def get_params(self):
+        return {
+            "add_bos": self.add_bos,
+            "encode_special_tokens": self.encode_special_tokens,
+            "decode_special_tokens": self.decode_special_tokens
+        }
+
+class TokenEncodeRequest(CommonTokenRequest):
+    text: str
+
+class TokenEncodeResponse(BaseModel):
+    tokens: List[int]
+    length: int
+
+class TokenDecodeRequest(CommonTokenRequest):
+    tokens: List[int]
+
+class TokenDecodeResponse(BaseModel):
+    text: str
+
+class TokenCountResponse(BaseModel):
+    length: int