create_model.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "163f12fd",
   "metadata": {},
   "source": [
    "# Creating Personalized Text Chatbot\n",
    "\n",
    "This jupyter notebook creates a Long short-term memory (LSTM) network for a ChatBot trained on my texts. The goal is to create a ChatBot that would replicate my behaviors, texting patterns, and colloqialisms. Our model is a seq2seq model that utilizes and an encoder and decoder to decipher text messages."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "2e20a138",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "import pickle\n",
    "from tensorflow.keras import layers , activations , models , preprocessing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "ddd03208",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2.11.0\n"
     ]
    }
   ],
   "source": [
    "print(tf.__version__)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e3502d54",
   "metadata": {},
   "source": [
    "Some preprocessing was done on my text dataset. I will include instructions on how to download your iMessage dataset below. For privacy reasons, I will not be sharing my \"texts.txt\" file. But the general look of it is as follows:\n",
    "Mateen (me): Text\n",
    "Responder: Response\n",
    "\n",
    "https://www.switchingtomac.com/tutorials/how-to-download-your-imessage-chat-history/\n",
    "\n",
    "Read in the text file and split the texts into questions and answers and then tag+tokenize them so the model is able to understand them."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "69553c52",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "24054 24055\n",
      "VOCAB SIZE : 2840\n"
     ]
    }
   ],
   "source": [
    "from tensorflow.keras import preprocessing , utils\n",
    "import os\n",
    "import yaml\n",
    "\n",
    "questions = []\n",
    "answers = []\n",
    "with open('texts.txt', 'r', encoding=\"utf-8\") as f:\n",
    "    for line in f:\n",
    "        if line.startswith('Text'):\n",
    "            questions.append(line[6:].rstrip('\\n'))\n",
    "        else:\n",
    "            if line.startswith('Mateen'):\n",
    "                answers.append(line[8:].rstrip('\\n'))\n",
    "                \n",
    "\n",
    "answers_with_tags = list()\n",
    "for i in range( len( answers ) ):\n",
    "    if type( answers[i] ) == str:\n",
    "        answers_with_tags.append( answers[i] )\n",
    "    else:\n",
    "        questions.pop( i )\n",
    "\n",
    "answers = list()\n",
    "for i in range( len( answers_with_tags ) ) :\n",
    "    answers.append( '<START> ' + answers_with_tags[i] + ' <END>' )\n",
    "\n",
    "tokenizer = preprocessing.text.Tokenizer()\n",
    "print(len(questions), len(answers))\n",
    "questions = questions[:2000]\n",
    "answers = answers[:2000]\n",
    "tokenizer.fit_on_texts( questions + answers )\n",
    "VOCAB_SIZE = len( tokenizer.word_index )+1\n",
    "print( 'VOCAB SIZE : {}'.format( VOCAB_SIZE ))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "41c11e4b",
   "metadata": {},
   "source": [
    "Tokenize our input and output training sets."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "22b08c77",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(2000, 174) 174\n",
      "(2000, 63) 63\n",
      "(2000, 63, 2840)\n",
      "done\n"
     ]
    }
   ],
   "source": [
    "from gensim.models import Word2Vec\n",
    "import re\n",
    "\n",
    "vocab = []\n",
    "for word in tokenizer.word_index:\n",
    "    vocab.append( word )\n",
    "\n",
    "def tokenize( sentences ):\n",
    "    tokens_list = []\n",
    "    vocabulary = []\n",
    "    for sentence in sentences:\n",
    "        sentence = sentence.lower()\n",
    "        sentence = re.sub( '[^a-zA-Z]', ' ', sentence )\n",
    "        tokens = sentence.split()\n",
    "        vocabulary += tokens\n",
    "        tokens_list.append( tokens )\n",
    "    return tokens_list , vocabulary\n",
    "\n",
    "# encoder_input_data\n",
    "tokenized_questions = tokenizer.texts_to_sequences( questions )\n",
    "maxlen_questions = max( [ len(x) for x in tokenized_questions ] )\n",
    "padded_questions = preprocessing.sequence.pad_sequences( tokenized_questions , maxlen=maxlen_questions , padding='post' )\n",
    "encoder_input_data = np.array( padded_questions )\n",
    "print( encoder_input_data.shape , maxlen_questions )\n",
    "\n",
    "# decoder_input_data\n",
    "tokenized_answers = tokenizer.texts_to_sequences( answers )\n",
    "maxlen_answers = max( [ len(x) for x in tokenized_answers ] )\n",
    "padded_answers = preprocessing.sequence.pad_sequences( tokenized_answers , maxlen=maxlen_answers , padding='post' )\n",
    "decoder_input_data = np.array( padded_answers )\n",
    "print( decoder_input_data.shape , maxlen_answers )\n",
    "\n",
    "# decoder_output_data\n",
    "tokenized_answers = tokenizer.texts_to_sequences( answers )\n",
    "for i in range(len(tokenized_answers)) :\n",
    "    tokenized_answers[i] = tokenized_answers[i][1:]\n",
    "padded_answers = preprocessing.sequence.pad_sequences( tokenized_answers , maxlen=maxlen_answers , padding='post' )\n",
    "onehot_answers = utils.to_categorical( padded_answers , VOCAB_SIZE )\n",
    "decoder_output_data = np.array( onehot_answers )\n",
    "print( decoder_output_data.shape )\n",
    "print('done')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d6e98d2a",
   "metadata": {},
   "source": [
    "Create our model which uses embeddings to map the input tokens to continuous vectors before processing them through the LSTM layers."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "5062264a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"model\"\n",
      "__________________________________________________________________________________________________\n",
      " Layer (type)                   Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      " input_1 (InputLayer)           [(None, 174)]        0           []                               \n",
      "                                                                                                  \n",
      " input_2 (InputLayer)           [(None, 63)]         0           []                               \n",
      "                                                                                                  \n",
      " embedding (Embedding)          (None, 174, 200)     568000      ['input_1[0][0]']                \n",
      "                                                                                                  \n",
      " embedding_1 (Embedding)        (None, 63, 200)      568000      ['input_2[0][0]']                \n",
      "                                                                                                  \n",
      " lstm (LSTM)                    [(None, 200),        320800      ['embedding[0][0]']              \n",
      "                                 (None, 200),                                                     \n",
      "                                 (None, 200)]                                                     \n",
      "                                                                                                  \n",
      " lstm_1 (LSTM)                  [(None, 63, 200),    320800      ['embedding_1[0][0]',            \n",
      "                                 (None, 200),                     'lstm[0][1]',                   \n",
      "                                 (None, 200)]                     'lstm[0][2]']                   \n",
      "                                                                                                  \n",
      " dense (Dense)                  (None, 63, 2840)     570840      ['lstm_1[0][0]']                 \n",
      "                                                                                                  \n",
      "==================================================================================================\n",
      "Total params: 2,348,440\n",
      "Trainable params: 2,348,440\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "encoder_inputs = tf.keras.layers.Input(shape=( maxlen_questions , ))\n",
    "encoder_embedding = tf.keras.layers.Embedding( VOCAB_SIZE, 200 , mask_zero=True ) (encoder_inputs)\n",
    "encoder_outputs , state_h , state_c = tf.keras.layers.LSTM( 200 , return_state=True )( encoder_embedding )\n",
    "encoder_states = [ state_h , state_c ]\n",
    "\n",
    "decoder_inputs = tf.keras.layers.Input(shape=( maxlen_answers ,  ))\n",
    "decoder_embedding = tf.keras.layers.Embedding( VOCAB_SIZE, 200 , mask_zero=True) (decoder_inputs)\n",
    "decoder_lstm = tf.keras.layers.LSTM( 200 , return_state=True , return_sequences=True )\n",
    "decoder_outputs , _ , _ = decoder_lstm ( decoder_embedding , initial_state=encoder_states )\n",
    "decoder_dense = tf.keras.layers.Dense( VOCAB_SIZE , activation=tf.keras.activations.softmax ) \n",
    "output = decoder_dense ( decoder_outputs )\n",
    "\n",
    "model = tf.keras.models.Model([encoder_inputs, decoder_inputs], output )\n",
    "model.compile(optimizer=tf.keras.optimizers.RMSprop(), loss='categorical_crossentropy')\n",
    "\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "62bc1fb9",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/150\n",
      "40/40 [==============================] - 24s 483ms/step - loss: 6.8438\n",
      "Epoch 2/150\n",
      "40/40 [==============================] - 20s 490ms/step - loss: 5.4911\n",
      "Epoch 3/150\n",
      "40/40 [==============================] - 18s 457ms/step - loss: 5.2729\n",
      "Epoch 4/150\n",
      "40/40 [==============================] - 19s 467ms/step - loss: 5.0357\n",
      "Epoch 5/150\n",
      "40/40 [==============================] - 19s 473ms/step - loss: 4.8900\n",
      "Epoch 6/150\n",
      "40/40 [==============================] - 19s 485ms/step - loss: 4.8112\n",
      "Epoch 7/150\n",
      "40/40 [==============================] - 19s 478ms/step - loss: 4.7788\n",
      "Epoch 8/150\n",
      "40/40 [==============================] - 19s 476ms/step - loss: 4.7430\n",
      "Epoch 9/150\n",
      "40/40 [==============================] - 19s 475ms/step - loss: 4.7124\n",
      "Epoch 10/150\n",
      "40/40 [==============================] - 19s 479ms/step - loss: 4.6852\n",
      "Epoch 11/150\n",
      "40/40 [==============================] - 20s 489ms/step - loss: 4.6536\n",
      "Epoch 12/150\n",
      "40/40 [==============================] - 20s 487ms/step - loss: 4.6228\n",
      "Epoch 13/150\n",
      "40/40 [==============================] - 21s 518ms/step - loss: 4.6016\n",
      "Epoch 14/150\n",
      "40/40 [==============================] - 21s 519ms/step - loss: 4.5735\n",
      "Epoch 15/150\n",
      "40/40 [==============================] - 22s 544ms/step - loss: 4.5401\n",
      "Epoch 16/150\n",
      "40/40 [==============================] - 20s 496ms/step - loss: 4.5189\n",
      "Epoch 17/150\n",
      "40/40 [==============================] - 21s 531ms/step - loss: 4.4902\n",
      "Epoch 18/150\n",
      "40/40 [==============================] - 22s 545ms/step - loss: 4.4650\n",
      "Epoch 19/150\n",
      "40/40 [==============================] - 21s 533ms/step - loss: 4.4449\n",
      "Epoch 20/150\n",
      "40/40 [==============================] - 20s 512ms/step - loss: 4.4193\n",
      "Epoch 21/150\n",
      "40/40 [==============================] - 20s 510ms/step - loss: 4.3977\n",
      "Epoch 22/150\n",
      "40/40 [==============================] - 21s 515ms/step - loss: 4.3816\n",
      "Epoch 23/150\n",
      "40/40 [==============================] - 21s 522ms/step - loss: 4.3506\n",
      "Epoch 24/150\n",
      "40/40 [==============================] - 22s 563ms/step - loss: 4.3268\n",
      "Epoch 25/150\n",
      "40/40 [==============================] - 23s 570ms/step - loss: 4.3104\n",
      "Epoch 26/150\n",
      "40/40 [==============================] - 21s 524ms/step - loss: 4.2808\n",
      "Epoch 27/150\n",
      "40/40 [==============================] - 21s 513ms/step - loss: 4.2586\n",
      "Epoch 28/150\n",
      "40/40 [==============================] - 22s 548ms/step - loss: 4.2476\n",
      "Epoch 29/150\n",
      "40/40 [==============================] - 22s 548ms/step - loss: 4.2152\n",
      "Epoch 30/150\n",
      "40/40 [==============================] - 18s 460ms/step - loss: 4.2019\n",
      "Epoch 31/150\n",
      "40/40 [==============================] - 19s 467ms/step - loss: 4.1743\n",
      "Epoch 32/150\n",
      "40/40 [==============================] - 21s 517ms/step - loss: 4.1641\n",
      "Epoch 33/150\n",
      "40/40 [==============================] - 19s 488ms/step - loss: 4.1411\n",
      "Epoch 34/150\n",
      "40/40 [==============================] - 20s 487ms/step - loss: 4.1243\n",
      "Epoch 35/150\n",
      "40/40 [==============================] - 20s 506ms/step - loss: 4.1008\n",
      "Epoch 36/150\n",
      "40/40 [==============================] - 20s 500ms/step - loss: 4.0820\n",
      "Epoch 37/150\n",
      "40/40 [==============================] - 20s 490ms/step - loss: 4.0653\n",
      "Epoch 38/150\n",
      "40/40 [==============================] - 20s 489ms/step - loss: 4.0466\n",
      "Epoch 39/150\n",
      "40/40 [==============================] - 20s 505ms/step - loss: 4.0236\n",
      "Epoch 40/150\n",
      "40/40 [==============================] - 20s 511ms/step - loss: 4.0138\n",
      "Epoch 41/150\n",
      "40/40 [==============================] - 20s 495ms/step - loss: 3.9950\n",
      "Epoch 42/150\n",
      "40/40 [==============================] - 20s 503ms/step - loss: 3.9688\n",
      "Epoch 43/150\n",
      "40/40 [==============================] - 20s 489ms/step - loss: 3.9593\n",
      "Epoch 44/150\n",
      "40/40 [==============================] - 20s 495ms/step - loss: 3.9313\n",
      "Epoch 45/150\n",
      "40/40 [==============================] - 18s 440ms/step - loss: 3.9222\n",
      "Epoch 46/150\n",
      "40/40 [==============================] - 17s 421ms/step - loss: 3.9050\n",
      "Epoch 47/150\n",
      "40/40 [==============================] - 17s 423ms/step - loss: 3.8784\n",
      "Epoch 48/150\n",
      "40/40 [==============================] - 17s 424ms/step - loss: 3.8605\n",
      "Epoch 49/150\n",
      "40/40 [==============================] - 17s 421ms/step - loss: 3.8427\n",
      "Epoch 50/150\n",
      "40/40 [==============================] - 17s 422ms/step - loss: 3.8182\n",
      "Epoch 51/150\n",
      "40/40 [==============================] - 17s 419ms/step - loss: 3.8009\n",
      "Epoch 52/150\n",
      "40/40 [==============================] - 17s 425ms/step - loss: 3.7865\n",
      "Epoch 53/150\n",
      "40/40 [==============================] - 17s 426ms/step - loss: 3.7725\n",
      "Epoch 54/150\n",
      "40/40 [==============================] - 17s 424ms/step - loss: 3.7501\n",
      "Epoch 55/150\n",
      "40/40 [==============================] - 17s 425ms/step - loss: 3.7256\n",
      "Epoch 56/150\n",
      "40/40 [==============================] - 17s 424ms/step - loss: 3.7067\n",
      "Epoch 57/150\n",
      "40/40 [==============================] - 17s 421ms/step - loss: 3.6948\n",
      "Epoch 58/150\n",
      "40/40 [==============================] - 17s 422ms/step - loss: 3.6714\n",
      "Epoch 59/150\n",
      "40/40 [==============================] - 17s 420ms/step - loss: 3.6558\n",
      "Epoch 60/150\n",
      "40/40 [==============================] - 17s 423ms/step - loss: 3.6329\n",
      "Epoch 61/150\n",
      "40/40 [==============================] - 17s 424ms/step - loss: 3.6090\n",
      "Epoch 62/150\n",
      "40/40 [==============================] - 17s 435ms/step - loss: 3.5936\n",
      "Epoch 63/150\n",
      "40/40 [==============================] - 17s 424ms/step - loss: 3.5736\n",
      "Epoch 64/150\n",
      "40/40 [==============================] - 17s 423ms/step - loss: 3.5554\n",
      "Epoch 65/150\n",
      "40/40 [==============================] - 17s 424ms/step - loss: 3.5392\n",
      "Epoch 66/150\n",
      "40/40 [==============================] - 17s 423ms/step - loss: 3.5210\n",
      "Epoch 67/150\n",
      "40/40 [==============================] - 17s 425ms/step - loss: 3.4941\n",
      "Epoch 68/150\n",
      "40/40 [==============================] - 17s 428ms/step - loss: 3.4746\n",
      "Epoch 69/150\n",
      "40/40 [==============================] - 17s 428ms/step - loss: 3.4565\n",
      "Epoch 70/150\n",
      "40/40 [==============================] - 17s 434ms/step - loss: 3.4296\n",
      "Epoch 71/150\n",
      "40/40 [==============================] - 17s 430ms/step - loss: 3.4139\n",
      "Epoch 72/150\n",
      "40/40 [==============================] - 17s 434ms/step - loss: 3.3927\n",
      "Epoch 73/150\n",
      "40/40 [==============================] - 18s 441ms/step - loss: 3.3771\n",
      "Epoch 74/150\n",
      "40/40 [==============================] - 17s 433ms/step - loss: 3.3514\n",
      "Epoch 75/150\n",
      "40/40 [==============================] - 18s 451ms/step - loss: 3.3405\n",
      "Epoch 76/150\n",
      "40/40 [==============================] - 18s 457ms/step - loss: 3.3207\n",
      "Epoch 77/150\n",
      "40/40 [==============================] - 17s 436ms/step - loss: 3.2926\n",
      "Epoch 78/150\n",
      "40/40 [==============================] - 17s 433ms/step - loss: 3.2756\n",
      "Epoch 79/150\n",
      "40/40 [==============================] - 17s 433ms/step - loss: 3.2551\n",
      "Epoch 80/150\n",
      "40/40 [==============================] - 17s 435ms/step - loss: 3.2359\n",
      "Epoch 81/150\n",
      "40/40 [==============================] - 17s 436ms/step - loss: 3.2158\n",
      "Epoch 82/150\n",
      "40/40 [==============================] - 17s 433ms/step - loss: 3.1989\n",
      "Epoch 83/150\n",
      "40/40 [==============================] - 19s 472ms/step - loss: 3.1719\n",
      "Epoch 84/150\n",
      "40/40 [==============================] - 20s 499ms/step - loss: 3.1572\n",
      "Epoch 85/150\n",
      "40/40 [==============================] - 18s 458ms/step - loss: 3.1362\n",
      "Epoch 86/150\n",
      "40/40 [==============================] - 18s 448ms/step - loss: 3.1156\n",
      "Epoch 87/150\n",
      "40/40 [==============================] - 18s 444ms/step - loss: 3.0917\n",
      "Epoch 88/150\n",
      "40/40 [==============================] - 18s 441ms/step - loss: 3.0687\n",
      "Epoch 89/150\n",
      "40/40 [==============================] - 18s 439ms/step - loss: 3.0554\n",
      "Epoch 90/150\n",
      "40/40 [==============================] - 18s 446ms/step - loss: 3.0360\n",
      "Epoch 91/150\n",
      "40/40 [==============================] - 18s 437ms/step - loss: 3.0109\n",
      "Epoch 92/150\n",
      "40/40 [==============================] - 17s 433ms/step - loss: 2.9953\n",
      "Epoch 93/150\n",
      "40/40 [==============================] - 17s 433ms/step - loss: 2.9716\n",
      "Epoch 94/150\n",
      "40/40 [==============================] - 17s 433ms/step - loss: 2.9533\n",
      "Epoch 95/150\n",
      "40/40 [==============================] - 18s 440ms/step - loss: 2.9278\n",
      "Epoch 96/150\n",
      "40/40 [==============================] - 18s 457ms/step - loss: 2.9089\n",
      "Epoch 97/150\n",
      "40/40 [==============================] - 18s 446ms/step - loss: 2.8903\n",
      "Epoch 98/150\n",
      "40/40 [==============================] - 18s 446ms/step - loss: 2.8664\n",
      "Epoch 99/150\n",
      "40/40 [==============================] - 18s 446ms/step - loss: 2.8477\n",
      "Epoch 100/150\n",
      "40/40 [==============================] - 18s 462ms/step - loss: 2.8287\n",
      "Epoch 101/150\n",
      "40/40 [==============================] - 19s 469ms/step - loss: 2.8087\n",
      "Epoch 102/150\n",
      "40/40 [==============================] - 18s 457ms/step - loss: 2.7819\n",
      "Epoch 103/150\n",
      "40/40 [==============================] - 22s 549ms/step - loss: 2.7634\n",
      "Epoch 104/150\n",
      "40/40 [==============================] - 26s 660ms/step - loss: 2.7427\n",
      "Epoch 105/150\n",
      "40/40 [==============================] - 27s 674ms/step - loss: 2.7223\n",
      "Epoch 106/150\n",
      "40/40 [==============================] - 20s 499ms/step - loss: 2.6990\n",
      "Epoch 107/150\n",
      "40/40 [==============================] - 21s 521ms/step - loss: 2.6743\n",
      "Epoch 108/150\n",
      "40/40 [==============================] - 18s 461ms/step - loss: 2.6563\n",
      "Epoch 109/150\n",
      "40/40 [==============================] - 18s 459ms/step - loss: 2.6336\n",
      "Epoch 110/150\n",
      "40/40 [==============================] - 17s 431ms/step - loss: 2.6117\n",
      "Epoch 111/150\n",
      "40/40 [==============================] - 17s 432ms/step - loss: 2.5893\n",
      "Epoch 112/150\n",
      "40/40 [==============================] - 18s 445ms/step - loss: 2.5666\n",
      "Epoch 113/150\n",
      "40/40 [==============================] - 18s 450ms/step - loss: 2.5417\n",
      "Epoch 114/150\n",
      "40/40 [==============================] - 18s 453ms/step - loss: 2.5202\n",
      "Epoch 115/150\n",
      "40/40 [==============================] - 19s 467ms/step - loss: 2.5050\n",
      "Epoch 116/150\n",
      "40/40 [==============================] - 18s 446ms/step - loss: 2.4807\n",
      "Epoch 117/150\n",
      "40/40 [==============================] - 18s 459ms/step - loss: 2.4523\n",
      "Epoch 118/150\n",
      "40/40 [==============================] - 19s 470ms/step - loss: 2.4349\n",
      "Epoch 119/150\n",
      "40/40 [==============================] - 19s 470ms/step - loss: 2.4086\n",
      "Epoch 120/150\n",
      "40/40 [==============================] - 19s 473ms/step - loss: 2.3846\n",
      "Epoch 121/150\n",
      "40/40 [==============================] - 21s 514ms/step - loss: 2.3678\n",
      "Epoch 122/150\n",
      "40/40 [==============================] - 20s 512ms/step - loss: 2.3446\n",
      "Epoch 123/150\n",
      "40/40 [==============================] - 21s 529ms/step - loss: 2.3200\n",
      "Epoch 124/150\n",
      "40/40 [==============================] - 25s 617ms/step - loss: 2.2925\n",
      "Epoch 125/150\n",
      "40/40 [==============================] - 23s 585ms/step - loss: 2.2728\n",
      "Epoch 126/150\n",
      "40/40 [==============================] - 22s 561ms/step - loss: 2.2536\n",
      "Epoch 127/150\n",
      "40/40 [==============================] - 19s 476ms/step - loss: 2.2262\n",
      "Epoch 128/150\n",
      "40/40 [==============================] - 20s 496ms/step - loss: 2.2026\n",
      "Epoch 129/150\n",
      "40/40 [==============================] - 24s 598ms/step - loss: 2.1863\n",
      "Epoch 130/150\n",
      "40/40 [==============================] - 23s 576ms/step - loss: 2.1621\n",
      "Epoch 131/150\n",
      "40/40 [==============================] - 21s 537ms/step - loss: 2.1343\n",
      "Epoch 132/150\n",
      "40/40 [==============================] - 24s 598ms/step - loss: 2.1150\n",
      "Epoch 133/150\n",
      "40/40 [==============================] - 23s 564ms/step - loss: 2.0914\n",
      "Epoch 134/150\n",
      "40/40 [==============================] - 23s 567ms/step - loss: 2.0641\n",
      "Epoch 135/150\n",
      "40/40 [==============================] - 22s 561ms/step - loss: 2.0465\n",
      "Epoch 136/150\n",
      "40/40 [==============================] - 22s 552ms/step - loss: 2.0269\n",
      "Epoch 137/150\n",
      "40/40 [==============================] - 22s 549ms/step - loss: 2.0015\n",
      "Epoch 138/150\n",
      "40/40 [==============================] - 22s 548ms/step - loss: 1.9746\n",
      "Epoch 139/150\n",
      "40/40 [==============================] - 21s 515ms/step - loss: 1.9557\n",
      "Epoch 140/150\n",
      "40/40 [==============================] - 19s 484ms/step - loss: 1.9279\n",
      "Epoch 141/150\n",
      "40/40 [==============================] - 19s 485ms/step - loss: 1.9104\n",
      "Epoch 142/150\n",
      "40/40 [==============================] - 19s 482ms/step - loss: 1.8861\n",
      "Epoch 143/150\n",
      "40/40 [==============================] - 20s 493ms/step - loss: 1.8644\n",
      "Epoch 144/150\n",
      "40/40 [==============================] - 19s 484ms/step - loss: 1.8377\n",
      "Epoch 145/150\n",
      "40/40 [==============================] - 19s 477ms/step - loss: 1.8219\n",
      "Epoch 146/150\n",
      "40/40 [==============================] - 19s 481ms/step - loss: 1.7968\n",
      "Epoch 147/150\n",
      "40/40 [==============================] - 20s 494ms/step - loss: 1.7680\n",
      "Epoch 148/150\n",
      "40/40 [==============================] - 20s 489ms/step - loss: 1.7504\n",
      "Epoch 149/150\n",
      "40/40 [==============================] - 20s 491ms/step - loss: 1.7236\n",
      "Epoch 150/150\n",
      "40/40 [==============================] - 20s 507ms/step - loss: 1.7046\n"
     ]
    }
   ],
   "source": [
    "model.fit([encoder_input_data , decoder_input_data], decoder_output_data, batch_size=50, epochs=150 ) \n",
    "model.save( 'model_small.h5' ) "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e9be72c1",
   "metadata": {},
   "source": [
    "Take our model that we just created and create 2 models from it, one encoder and one decoder, that will be able to decipher our messages that we input."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "1f1c1f0d",
   "metadata": {},
   "outputs": [],
   "source": [
    "def make_inference_models():\n",
    "    \n",
    "    encoder_model = tf.keras.models.Model(encoder_inputs, encoder_states)\n",
    "    \n",
    "    decoder_state_input_h = tf.keras.layers.Input(shape=( 200 ,))\n",
    "    decoder_state_input_c = tf.keras.layers.Input(shape=( 200 ,))\n",
    "    \n",
    "    decoder_states_inputs = [decoder_state_input_h, decoder_state_input_c]\n",
    "    \n",
    "    decoder_outputs, state_h, state_c = decoder_lstm(decoder_embedding , initial_state=decoder_states_inputs)\n",
    "    decoder_states = [state_h, state_c]\n",
    "    decoder_outputs = decoder_dense(decoder_outputs)\n",
    "    decoder_model = tf.keras.models.Model(\n",
    "        [decoder_inputs] + decoder_states_inputs,\n",
    "        [decoder_outputs] + decoder_states)\n",
    "    \n",
    "    return encoder_model , decoder_model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "57999788",
   "metadata": {},
   "outputs": [],
   "source": [
    "def str_to_tokens( sentence : str ):\n",
    "    words = sentence.lower().split()\n",
    "    tokens_list = list()\n",
    "    for word in words:\n",
    "        tokens_list.append( tokenizer.word_index[ word ] ) \n",
    "    return preprocessing.sequence.pad_sequences( [tokens_list] , maxlen=maxlen_questions , padding='post')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "f4a0025c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Enter question : Hello\n",
      "1/1 [==============================] - 1s 896ms/step\n",
      "WARNING:tensorflow:Model was constructed with shape (None, 63) for input KerasTensor(type_spec=TensorSpec(shape=(None, 63), dtype=tf.float32, name='input_2'), name='input_2', description=\"created by layer 'input_2'\"), but it was called on an input with incompatible shape (None, 1).\n",
      "1/1 [==============================] - 1s 894ms/step\n",
      "1/1 [==============================] - 0s 18ms/step\n",
      " sorry end\n",
      "Enter question : Hi\n",
      "1/1 [==============================] - 0s 30ms/step\n",
      "1/1 [==============================] - 0s 19ms/step\n",
      "1/1 [==============================] - 0s 19ms/step\n",
      "1/1 [==============================] - 0s 20ms/step\n",
      "1/1 [==============================] - 0s 20ms/step\n",
      "1/1 [==============================] - 0s 19ms/step\n",
      "1/1 [==============================] - 0s 18ms/step\n",
      " i’m down to the family end\n",
      "Enter question : Whats up\n",
      "I am not sure\n",
      "Enter question : wyd\n",
      "1/1 [==============================] - 0s 30ms/step\n",
      "1/1 [==============================] - 0s 22ms/step\n",
      "1/1 [==============================] - 0s 19ms/step\n",
      "1/1 [==============================] - 0s 18ms/step\n",
      "1/1 [==============================] - 0s 19ms/step\n",
      "1/1 [==============================] - 0s 19ms/step\n",
      "1/1 [==============================] - 0s 20ms/step\n",
      "1/1 [==============================] - 0s 19ms/step\n",
      " wait i’m down for the night end\n",
      "Enter question : whats the hw\n",
      "I am not sure\n",
      "Enter question : Bye\n",
      "1/1 [==============================] - 0s 32ms/step\n",
      "1/1 [==============================] - 0s 19ms/step\n",
      "1/1 [==============================] - 0s 19ms/step\n",
      " yes end\n",
      "Enter question : goodnight\n",
      "1/1 [==============================] - 0s 31ms/step\n",
      "1/1 [==============================] - 0s 19ms/step\n",
      " end\n",
      "Enter question : gn\n",
      "1/1 [==============================] - 0s 30ms/step\n",
      "1/1 [==============================] - 0s 19ms/step\n",
      "1/1 [==============================] - 0s 20ms/step\n",
      " lol end\n",
      "Enter question : lol\n",
      "1/1 [==============================] - 0s 31ms/step\n",
      "1/1 [==============================] - 0s 19ms/step\n",
      "1/1 [==============================] - 0s 19ms/step\n",
      " lol end\n",
      "Enter question : lmao\n",
      "1/1 [==============================] - 0s 31ms/step\n",
      "1/1 [==============================] - 0s 19ms/step\n",
      "1/1 [==============================] - 0s 19ms/step\n",
      " ahaha end\n",
      "Enter question : How are you\n",
      "1/1 [==============================] - 0s 30ms/step\n",
      "1/1 [==============================] - 0s 19ms/step\n",
      "1/1 [==============================] - 0s 20ms/step\n",
      "1/1 [==============================] - 0s 20ms/step\n",
      " i’m not end\n",
      "Enter question : Why\n",
      "1/1 [==============================] - 0s 31ms/step\n",
      "1/1 [==============================] - 0s 19ms/step\n",
      "1/1 [==============================] - 0s 19ms/step\n",
      " ok end\n",
      "Enter question : where\n",
      "1/1 [==============================] - 0s 30ms/step\n",
      "1/1 [==============================] - 0s 21ms/step\n",
      "1/1 [==============================] - 0s 19ms/step\n",
      "1/1 [==============================] - 0s 19ms/step\n",
      "1/1 [==============================] - 0s 20ms/step\n",
      "1/1 [==============================] - 0s 19ms/step\n",
      " i’m down to open end\n",
      "Enter question : end\n",
      "1/1 [==============================] - 0s 30ms/step\n",
      "1/1 [==============================] - 0s 19ms/step\n",
      "1/1 [==============================] - 0s 20ms/step\n",
      "1/1 [==============================] - 0s 18ms/step\n",
      "1/1 [==============================] - 0s 19ms/step\n",
      "1/1 [==============================] - 0s 20ms/step\n",
      "1/1 [==============================] - 0s 20ms/step\n",
      "1/1 [==============================] - 0s 19ms/step\n",
      "1/1 [==============================] - 0s 19ms/step\n",
      "1/1 [==============================] - 0s 19ms/step\n",
      " wait i have a pic of the lab end\n",
      "Enter question : q\n",
      "Quitting...Goodbye\n",
      "1/1 [==============================] - 0s 31ms/step\n",
      "1/1 [==============================] - 0s 20ms/step\n",
      "1/1 [==============================] - 0s 20ms/step\n",
      " yes end\n",
      "Enter question : q\n",
      "Quitting...Goodbye\n",
      "1/1 [==============================] - 0s 32ms/step\n",
      "1/1 [==============================] - 0s 19ms/step\n",
      "1/1 [==============================] - 0s 19ms/step\n",
      " yes end\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "Interrupted by user",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "Cell \u001b[1;32mIn[10], line 5\u001b[0m\n\u001b[0;32m      3\u001b[0m \u001b[38;5;28;01mwhile\u001b[39;00m \u001b[38;5;28;01mTrue\u001b[39;00m:\n\u001b[0;32m      4\u001b[0m     \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m----> 5\u001b[0m         user_input \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43minput\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43mEnter question : \u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m      6\u001b[0m         \u001b[38;5;28;01mif\u001b[39;00m user_input \u001b[38;5;241m==\u001b[39m \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mq\u001b[39m\u001b[38;5;124m'\u001b[39m:\n\u001b[0;32m      7\u001b[0m             \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mQuitting...Goodbye\u001b[39m\u001b[38;5;124m'\u001b[39m)\n",
      "File \u001b[1;32m~\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\ipykernel\\kernelbase.py:1187\u001b[0m, in \u001b[0;36mKernel.raw_input\u001b[1;34m(self, prompt)\u001b[0m\n\u001b[0;32m   1185\u001b[0m     msg \u001b[38;5;241m=\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mraw_input was called, but this frontend does not support input requests.\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m   1186\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m StdinNotImplementedError(msg)\n\u001b[1;32m-> 1187\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_input_request\u001b[49m\u001b[43m(\u001b[49m\n\u001b[0;32m   1188\u001b[0m \u001b[43m    \u001b[49m\u001b[38;5;28;43mstr\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43mprompt\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m   1189\u001b[0m \u001b[43m    \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_parent_ident\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mshell\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m]\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m   1190\u001b[0m \u001b[43m    \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mget_parent\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mshell\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m   1191\u001b[0m \u001b[43m    \u001b[49m\u001b[43mpassword\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mFalse\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[0;32m   1192\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[1;32m~\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\ipykernel\\kernelbase.py:1230\u001b[0m, in \u001b[0;36mKernel._input_request\u001b[1;34m(self, prompt, ident, parent, password)\u001b[0m\n\u001b[0;32m   1227\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mKeyboardInterrupt\u001b[39;00m:\n\u001b[0;32m   1228\u001b[0m     \u001b[38;5;66;03m# re-raise KeyboardInterrupt, to truncate traceback\u001b[39;00m\n\u001b[0;32m   1229\u001b[0m     msg \u001b[38;5;241m=\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mInterrupted by user\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m-> 1230\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mKeyboardInterrupt\u001b[39;00m(msg) \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;28mNone\u001b[39m\n\u001b[0;32m   1231\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mException\u001b[39;00m:\n\u001b[0;32m   1232\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mlog\u001b[38;5;241m.\u001b[39mwarning(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mInvalid Message:\u001b[39m\u001b[38;5;124m\"\u001b[39m, exc_info\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m)\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: Interrupted by user"
     ]
    }
   ],
   "source": [
    "\n",
    "enc_model , dec_model = make_inference_models()\n",
    "\n",
    "while True:\n",
    "    try:\n",
    "        user_input = input( 'Enter question : ' )\n",
    "        if user_input == 'q':\n",
    "            print('Quitting...Goodbye')\n",
    "            exit()\n",
    "        states_values = enc_model.predict( str_to_tokens( user_input ) )\n",
    "        empty_target_seq = np.zeros( ( 1 , 1 ) )\n",
    "        empty_target_seq[0, 0] = tokenizer.word_index['start']\n",
    "        stop_condition = False\n",
    "        decoded_translation = ''\n",
    "        while not stop_condition :\n",
    "            dec_outputs , h , c = dec_model.predict([ empty_target_seq ] + states_values )\n",
    "            sampled_word_index = np.argmax( dec_outputs[0, -1, :] )\n",
    "            sampled_word = None\n",
    "            for word , index in tokenizer.word_index.items() :\n",
    "                if sampled_word_index == index :\n",
    "                    decoded_translation += ' {}'.format( word )\n",
    "                    sampled_word = word\n",
    "\n",
    "            if sampled_word == 'end' or len(decoded_translation.split()) > maxlen_answers:\n",
    "                stop_condition = True\n",
    "\n",
    "            empty_target_seq = np.zeros( ( 1 , 1 ) )  \n",
    "            empty_target_seq[ 0 , 0 ] = sampled_word_index\n",
    "            states_values = [ h , c ] \n",
    "\n",
    "        print( decoded_translation )\n",
    "    except KeyError:\n",
    "        print(\"I am not sure\")\n",
    "        continue\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}