Refactoring step one

ai_functions.py

@@ -0,0 +1,26 @@
+
+import tiktoken
+
+
+def num_tokens_from_string(message, model):
+    """
+    The function takes a string message and a model as input and returns the number of tokens in the
+    message according to the given model.
+
+    :param message: A string containing the message or text from which you want to count the number of
+    tokens
+    :param model: The model parameter refers to a language model or tokenizer that can be used to
+    tokenize the input string. It could be a pre-trained model or a custom tokenizer
+    """
+    tokens_per_message = 3
+    tokens_per_name = 1
+    chat_gpt_encoding = tiktoken.encoding_for_model(model)
+
+    num_tokens = 0
+    num_tokens += tokens_per_message
+    for keys, values in message.items():
+        num_tokens += len(chat_gpt_encoding.encode(values))
+        if keys == "role":
+            num_tokens += tokens_per_name
+    num_tokens += 3  # every reply is primed with <|start|>assistant<|message|>
+    return num_tokens