TextGeneration.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Code Reference: https://github.com/nfmcclure/tensorflow_cookbook/blob/master/09_Recurrent_Neural_Networks/03_Implementing_LSTM/03_implementing_lstm.py"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Importing the libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "from tensorflow.python.framework import ops\n",
    "import numpy as np\n",
    "import string, re, os\n",
    "import requests\n",
    "import collections\n",
    "import random"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Sample data:\n",
      "From fairest creatures we desire increase,  That thereby beauty's rose might never die,  But as the riper should by time decease,  His tender heir might bear his memory:  But thou contracted to thine \n"
     ]
    }
   ],
   "source": [
    "data_dir = 'data'\n",
    "data_file = 'shakespeare.txt'\n",
    "if not os.path.isfile(os.path.join(data_dir, data_file)):\n",
    "    if not os.path.exists(data_dir):\n",
    "        os.makedirs(data_dir)\n",
    "    print('Data file not found, downloading the dataset')\n",
    "    shakespeare_url = 'http://www.gutenberg.org/cache/epub/100/pg100.txt'\n",
    "    response = requests.get(shakespeare_url)\n",
    "    shakespeare_file = response.content\n",
    "    # Decode binary into string\n",
    "    s_text = shakespeare_file.decode('utf-8')\n",
    "    # Drop first few descriptive paragraphs.\n",
    "    s_text = s_text[7675:]\n",
    "    # Remove newlines\n",
    "    s_text = s_text.replace('\\r\\n', '')\n",
    "    s_text = s_text.replace('\\n', '')\n",
    "    # Write to file\n",
    "    with open(os.path.join(data_dir, data_file), 'w') as out_conn:\n",
    "        out_conn.write(s_text)\n",
    "else:\n",
    "    with open(os.path.join(data_dir, data_file), 'r') as file_conn:\n",
    "        s_text = file_conn.read().replace('\\n', '')\n",
    "print('Sample data:\\n'+ s_text[:200])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Clean data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Declare punctuation to remove, everything except hyphens and apostrophes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "punctuation = string.punctuation\n",
    "punctuation = ''.join([x for x in punctuation if x not in ['-', \"'\"]])\n",
    "s_text = re.sub(r'[{}]'.format(punctuation), ' ', s_text)\n",
    "s_text = re.sub('\\s+', ' ', s_text).strip().lower()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Build Vocabulary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('word counts: ', 31953, 'text len: ', 901088)\n",
      "Vocabulary Length = 8009\n"
     ]
    }
   ],
   "source": [
    "def build_vocab(text, min_word_freq):\n",
    "    word_counts = collections.Counter(text.split(' '))\n",
    "    print ('word counts: ', len(word_counts), 'text len: ', len(text.split(' ')))\n",
    "    # limit word counts to those more frequent than cutoff\n",
    "    word_counts = {key: val for key, val in word_counts.items() if val > min_word_freq}\n",
    "    # Create vocab --> index mapping\n",
    "    words = word_counts.keys()\n",
    "    vocab_to_ix_dict = {key: (ix + 1) for ix, key in enumerate(words)}\n",
    "    # Add unknown key --> 0 index\n",
    "    vocab_to_ix_dict['unknown'] = 0\n",
    "    # Create index --> vocab mapping\n",
    "    ix_to_vocab_dict = {val: key for key, val in vocab_to_ix_dict.items()}\n",
    "    return (ix_to_vocab_dict, vocab_to_ix_dict)\n",
    "\n",
    "# Build Shakespeare vocabulary\n",
    "min_word_freq = 5  # Trim the less frequent words off\n",
    "ix2vocab, vocab2ix = build_vocab(s_text, min_word_freq)\n",
    "vocab_size = len(ix2vocab) + 1\n",
    "print('Vocabulary Length = {}'.format(vocab_size))\n",
    "# Sanity Check\n",
    "assert (len(ix2vocab) == len(vocab2ix))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Convert text to word Vectors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "s_text_words = s_text.split(' ')\n",
    "s_text_ix = []\n",
    "for ix, x in enumerate(s_text_words):\n",
    "    try:\n",
    "        s_text_ix.append(vocab2ix[x])\n",
    "    except:\n",
    "        s_text_ix.append(0)\n",
    "s_text_ix = np.array(s_text_ix)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# LSTM RNN Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class LSTM_Model():\n",
    "    def __init__(self, rnn_size, batch_size, learning_rate,\n",
    "                 training_seq_len, vocab_size, infer_sample=False):\n",
    "        self.rnn_size = rnn_size\n",
    "        self.vocab_size = vocab_size\n",
    "        self.infer_sample = infer_sample\n",
    "        self.learning_rate = learning_rate\n",
    "\n",
    "        if infer_sample:\n",
    "            self.batch_size = 1\n",
    "            self.training_seq_len = 1\n",
    "        else:\n",
    "            self.batch_size = batch_size\n",
    "            self.training_seq_len = training_seq_len\n",
    "\n",
    "        self.lstm_cell = tf.contrib.rnn.core_rnn_cell.BasicLSTMCell(rnn_size)\n",
    "        self.initial_state = self.lstm_cell.zero_state(self.batch_size, tf.float32)\n",
    "\n",
    "        self.x_data = tf.placeholder(tf.int32, [self.batch_size, self.training_seq_len])\n",
    "        self.y_output = tf.placeholder(tf.int32, [self.batch_size, self.training_seq_len])\n",
    "\n",
    "        with tf.variable_scope('lstm_vars'):\n",
    "            # Softmax Output Weights\n",
    "            W = tf.get_variable('W', [self.rnn_size, self.vocab_size], tf.float32, tf.random_normal_initializer())\n",
    "            b = tf.get_variable('b', [self.vocab_size], tf.float32, tf.constant_initializer(0.0))\n",
    "\n",
    "            # Define Embedding\n",
    "            embedding_mat = tf.get_variable('embedding_mat', [self.vocab_size, self.rnn_size],\n",
    "                                            tf.float32, tf.random_normal_initializer())\n",
    "            print('xdata:', self.x_data.get_shape())\n",
    "            print('emb_mat: ', embedding_mat.get_shape())\n",
    "            embedding_output = tf.nn.embedding_lookup(embedding_mat, self.x_data)\n",
    "            print('emb_output: ', embedding_output.get_shape())\n",
    "            rnn_inputs = tf.split(axis=1, num_or_size_splits=self.training_seq_len, value=embedding_output)\n",
    "            print('rnninputs: ', len(rnn_inputs), rnn_inputs[0].get_shape())\n",
    "            rnn_inputs_trimmed = [tf.squeeze(x, [1]) for x in rnn_inputs]\n",
    "            print('rnninput trimmed:', len(rnn_inputs_trimmed), rnn_inputs_trimmed[0].get_shape())\n",
    "\n",
    "            # If we are inferring (generating text), we add a 'loop' function\n",
    "            # Define how to get the i+1 th input from the i th output\n",
    "            def inferred_loop(prev, count):\n",
    "                # Apply hidden layer\n",
    "                prev_transformed = tf.matmul(prev, W) + b\n",
    "                # Get the index of the output (also don't run the gradient)\n",
    "                prev_symbol = tf.stop_gradient(tf.argmax(prev_transformed, 1))\n",
    "                # Get embedded vector\n",
    "                output = tf.nn.embedding_lookup(embedding_mat, prev_symbol)\n",
    "                return (output)\n",
    "\n",
    "            decoder = tf.contrib.legacy_seq2seq.rnn_decoder\n",
    "            outputs, last_state = decoder(rnn_inputs_trimmed,\n",
    "                                          self.initial_state,\n",
    "                                          self.lstm_cell,\n",
    "                                          loop_function=inferred_loop if infer_sample else None)\n",
    "            # Non inferred outputs\n",
    "            output = tf.reshape(tf.concat(axis=1, values=outputs), [-1, self.rnn_size])\n",
    "            # Logits and output\n",
    "            self.logit_output = tf.matmul(output, W) + b\n",
    "            self.model_output = tf.nn.softmax(self.logit_output)\n",
    "\n",
    "            loss_fun = tf.contrib.legacy_seq2seq.sequence_loss_by_example\n",
    "            loss = loss_fun([self.logit_output], [tf.reshape(self.y_output, [-1])],\n",
    "                            [tf.ones([self.batch_size * self.training_seq_len])],\n",
    "                            self.vocab_size)\n",
    "            self.cost = tf.reduce_sum(loss) / (self.batch_size * self.training_seq_len)\n",
    "            self.final_state = last_state\n",
    "            gradients, _ = tf.clip_by_global_norm(tf.gradients(self.cost, tf.trainable_variables()), 4.5)\n",
    "            optimizer = tf.train.AdamOptimizer(self.learning_rate)\n",
    "            self.train_op = optimizer.apply_gradients(zip(gradients, tf.trainable_variables()))\n",
    "\n",
    "    def sample(self, sess, words=ix2vocab, vocab=vocab2ix, num=10, prime_text='thou art'):\n",
    "        state = sess.run(self.lstm_cell.zero_state(1, tf.float32))\n",
    "        word_list = prime_text.split()\n",
    "        for word in word_list[:-1]:\n",
    "            x = np.zeros((1, 1))\n",
    "            x[0, 0] = vocab[word]\n",
    "            feed_dict = {self.x_data: x, self.initial_state: state}\n",
    "            [state] = sess.run([self.final_state], feed_dict=feed_dict)\n",
    "\n",
    "        out_sentence = prime_text\n",
    "        word = word_list[-1]\n",
    "        for n in range(num):\n",
    "            x = np.zeros((1, 1))\n",
    "            x[0, 0] = vocab[word]\n",
    "            feed_dict = {self.x_data: x, self.initial_state: state}\n",
    "            [model_output, state] = sess.run([self.model_output, self.final_state], feed_dict=feed_dict)\n",
    "            sample = np.argmax(model_output[0])\n",
    "            if sample == 0:\n",
    "                break\n",
    "            word = words[sample]\n",
    "            out_sentence = out_sentence + ' ' + word\n",
    "        return (out_sentence)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('xdata:', TensorShape([Dimension(32), Dimension(11)]))\n",
      "('emb_mat: ', TensorShape([Dimension(8009), Dimension(1024)]))\n",
      "('emb_output: ', TensorShape([Dimension(32), Dimension(11), Dimension(1024)]))\n",
      "('rnninputs: ', 11, TensorShape([Dimension(32), Dimension(1), Dimension(1024)]))\n",
      "('rnninput trimmed:', 11, TensorShape([Dimension(32), Dimension(1024)]))\n",
      "('xdata:', TensorShape([Dimension(1), Dimension(1)]))\n",
      "('emb_mat: ', TensorShape([Dimension(8009), Dimension(1024)]))\n",
      "('emb_output: ', TensorShape([Dimension(1), Dimension(1), Dimension(1024)]))\n",
      "('rnninputs: ', 1, TensorShape([Dimension(1), Dimension(1), Dimension(1024)]))\n",
      "('rnninput trimmed:', 1, TensorShape([Dimension(1), Dimension(1024)]))\n"
     ]
    }
   ],
   "source": [
    "rnn_size = 1024  # RNN Model size, has to equal embedding size\n",
    "epochs = 10  # Number of epochs to cycle through data\n",
    "batch_size = 32  # Train on this many examples at once\n",
    "learning_rate = 0.001  # Learning rate\n",
    "training_seq_len = 11  # how long of a word group to consider\n",
    "embedding_size = rnn_size\n",
    "eval_every = 50  # How often to evaluate the test sentences\n",
    "prime_texts = ['thou art more', 'to be or not to', 'wherefore art thou']\n",
    "\n",
    "sess = tf.Session()\n",
    "\n",
    "# Define LSTM RNN Model\n",
    "with tf.variable_scope('lstm_model') as scope:\n",
    "    # Define LSTM Model\n",
    "    lstm_model = LSTM_Model(rnn_size, batch_size, learning_rate,\n",
    "                            training_seq_len, vocab_size)\n",
    "    scope.reuse_variables()\n",
    "    test_lstm_model = LSTM_Model(rnn_size, batch_size, learning_rate,\n",
    "                                 training_seq_len, vocab_size, infer_sample=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Train Model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Create batches"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "num_batches = int(len(s_text_ix) / (batch_size * training_seq_len)) + 1\n",
    "# Split up text indices into subarrays, of equal size\n",
    "batches = np.array_split(s_text_ix, num_batches)\n",
    "# Reshape each split into [batch_size, training_seq_len]\n",
    "batches = [np.resize(x, [batch_size, training_seq_len]) for x in batches]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Initialize variables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "init = tf.global_variables_initializer()\n",
    "sess.run(init)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Start Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Starting Epoch #1 of 10.\n",
      "Iteration: 10, Epoch: 1, Batch: 10 out of 2561, Loss: 18.15\n",
      "Iteration: 20, Epoch: 1, Batch: 20 out of 2561, Loss: 14.81\n",
      "Iteration: 30, Epoch: 1, Batch: 30 out of 2561, Loss: 10.78\n",
      "Iteration: 40, Epoch: 1, Batch: 40 out of 2561, Loss: 9.44\n",
      "Iteration: 50, Epoch: 1, Batch: 50 out of 2561, Loss: 9.27\n",
      "thou art more than than thy sword for\n",
      "to be or not to the\n",
      "wherefore art thou art thy phrase is worth a man in i have\n",
      "Iteration: 60, Epoch: 1, Batch: 60 out of 2561, Loss: 8.85\n",
      "Iteration: 70, Epoch: 1, Batch: 70 out of 2561, Loss: 8.44\n",
      "Iteration: 80, Epoch: 1, Batch: 80 out of 2561, Loss: 8.00\n",
      "Iteration: 90, Epoch: 1, Batch: 90 out of 2561, Loss: 8.28\n",
      "Iteration: 100, Epoch: 1, Batch: 100 out of 2561, Loss: 7.85\n",
      "thou art more thy hand prince john we we will be\n",
      "to be or not to my\n",
      "wherefore art thou art thou art thou art thou art thou art thou\n",
      "Iteration: 110, Epoch: 1, Batch: 110 out of 2561, Loss: 7.69\n",
      "Iteration: 120, Epoch: 1, Batch: 120 out of 2561, Loss: 7.22\n",
      "Iteration: 130, Epoch: 1, Batch: 130 out of 2561, Loss: 7.33\n",
      "Iteration: 140, Epoch: 1, Batch: 140 out of 2561, Loss: 7.31\n",
      "Iteration: 150, Epoch: 1, Batch: 150 out of 2561, Loss: 7.62\n",
      "thou art more than for my life and i had rather than do\n",
      "to be or not to\n",
      "wherefore art thou wilt be not\n",
      "Iteration: 160, Epoch: 1, Batch: 160 out of 2561, Loss: 7.29\n",
      "Iteration: 170, Epoch: 1, Batch: 170 out of 2561, Loss: 7.64\n",
      "Iteration: 180, Epoch: 1, Batch: 180 out of 2561, Loss: 7.56\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-11-d6be9ae094ad>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     17\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     18\u001b[0m         temp_loss, state, _ = sess.run([lstm_model.cost, lstm_model.final_state, lstm_model.train_op],\n\u001b[0;32m---> 19\u001b[0;31m                                        feed_dict=training_dict)\n\u001b[0m\u001b[1;32m     20\u001b[0m         \u001b[0mtrain_loss\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtemp_loss\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     21\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Python/2.7/site-packages/tensorflow/python/client/session.pyc\u001b[0m in \u001b[0;36mrun\u001b[0;34m(self, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m    765\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    766\u001b[0m       result = self._run(None, fetches, feed_dict, options_ptr,\n\u001b[0;32m--> 767\u001b[0;31m                          run_metadata_ptr)\n\u001b[0m\u001b[1;32m    768\u001b[0m       \u001b[0;32mif\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    769\u001b[0m         \u001b[0mproto_data\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf_session\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTF_GetBuffer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrun_metadata_ptr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Python/2.7/site-packages/tensorflow/python/client/session.pyc\u001b[0m in \u001b[0;36m_run\u001b[0;34m(self, handle, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m    963\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mfinal_fetches\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mfinal_targets\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    964\u001b[0m       results = self._do_run(handle, final_targets, final_fetches,\n\u001b[0;32m--> 965\u001b[0;31m                              feed_dict_string, options, run_metadata)\n\u001b[0m\u001b[1;32m    966\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    967\u001b[0m       \u001b[0mresults\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Python/2.7/site-packages/tensorflow/python/client/session.pyc\u001b[0m in \u001b[0;36m_do_run\u001b[0;34m(self, handle, target_list, fetch_list, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m   1013\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mhandle\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1014\u001b[0m       return self._do_call(_run_fn, self._session, feed_dict, fetch_list,\n\u001b[0;32m-> 1015\u001b[0;31m                            target_list, options, run_metadata)\n\u001b[0m\u001b[1;32m   1016\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1017\u001b[0m       return self._do_call(_prun_fn, self._session, handle, feed_dict,\n",
      "\u001b[0;32m/Library/Python/2.7/site-packages/tensorflow/python/client/session.pyc\u001b[0m in \u001b[0;36m_do_call\u001b[0;34m(self, fn, *args)\u001b[0m\n\u001b[1;32m   1020\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0m_do_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1021\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1022\u001b[0;31m       \u001b[0;32mreturn\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1023\u001b[0m     \u001b[0;32mexcept\u001b[0m \u001b[0merrors\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mOpError\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1024\u001b[0m       \u001b[0mmessage\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcompat\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mas_text\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmessage\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Python/2.7/site-packages/tensorflow/python/client/session.pyc\u001b[0m in \u001b[0;36m_run_fn\u001b[0;34m(session, feed_dict, fetch_list, target_list, options, run_metadata)\u001b[0m\n\u001b[1;32m   1002\u001b[0m         return tf_session.TF_Run(session, options,\n\u001b[1;32m   1003\u001b[0m                                  \u001b[0mfeed_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtarget_list\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1004\u001b[0;31m                                  status, run_metadata)\n\u001b[0m\u001b[1;32m   1005\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1006\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_prun_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msession\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhandle\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "train_loss = []\n",
    "iteration_count = 1\n",
    "for epoch in range(epochs):\n",
    "    # Shuffle word indices\n",
    "    random.shuffle(batches)\n",
    "    # Create targets from shuffled batches\n",
    "    targets = [np.roll(x, -1, axis=1) for x in batches]\n",
    "    # Run a through one epoch\n",
    "    print('Starting Epoch #{} of {}.'.format(epoch + 1, epochs))\n",
    "    # Reset initial LSTM state every epoch\n",
    "    state = sess.run(lstm_model.initial_state)\n",
    "    for ix, batch in enumerate(batches):\n",
    "        training_dict = {lstm_model.x_data: batch, lstm_model.y_output: targets[ix]}\n",
    "        c, h = lstm_model.initial_state\n",
    "        training_dict[c] = state.c\n",
    "        training_dict[h] = state.h\n",
    "\n",
    "        temp_loss, state, _ = sess.run([lstm_model.cost, lstm_model.final_state, lstm_model.train_op],\n",
    "                                       feed_dict=training_dict)\n",
    "        train_loss.append(temp_loss)\n",
    "\n",
    "        # Print status every 10 gens\n",
    "        if iteration_count % 10 == 0:\n",
    "            summary_nums = (iteration_count, epoch + 1, ix + 1, num_batches + 1, temp_loss)\n",
    "            print('Iteration: {}, Epoch: {}, Batch: {} out of {}, Loss: {:.2f}'.format(*summary_nums))\n",
    "\n",
    "        if iteration_count % eval_every == 0:\n",
    "            for sample in prime_texts:\n",
    "                print(test_lstm_model.sample(sess, ix2vocab, vocab2ix, num=10, prime_text=sample))\n",
    "\n",
    "        iteration_count += 1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Plotting the training error"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Library/Python/2.7/site-packages/matplotlib/font_manager.py:273: UserWarning: Matplotlib is building the font cache using fc-list. This may take a moment.\n",
      "  warnings.warn('Matplotlib is building the font cache using fc-list. This may take a moment.')\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYYAAAEZCAYAAACTsIJzAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xd8VGX2+PHPoQVIqKEmgYRQhVAExAVZiAgLUhYRRFeq\nveuKXXd/wqqr+F3XglgQqSsqoCAWFClREJEmJXRIaAmGjjQlJOf3x9yMMymQkExmkpz36zUvZu5z\ny7mXyZz7PM+9zxVVxRhjjMlQyt8BGGOMCSyWGIwxxnixxGCMMcaLJQZjjDFeLDEYY4zxYonBGGOM\nF0sMxhhjvFhiKEFEpLOI/CAix0XksIgsFZF2/o6rMIhIuohE52P520Rki4icEJEDIvK5iAQXZIyB\nSES6isg+f8dhClcZfwdgCoeIVAI+B+4CZgHlgD8Dv/szrkJ0yXdyikhX4AXgL6q6QUSqAv0KLLLA\nZ3fBljBWYyg5mgCqqjPV5XdVXaiq8RkziMitIrJZRI6IyHwRqe9R1sM5Yz4mIuNEJE5EbnXKnhWR\n6R7zRjpn6KWcz5VFZKKIJIvIPhF5TkTEKRvh1Fz+T0SOisguEenlsa5qIjJJRJKcuD71KOsrIj87\nMS0TkZbZ7biIfAcIsEFEfhWRG5zpd4jIDqf2NFdE6uZw7NoDy1V1A66DeFxVp6vqaWc95UTkPyKy\nx6lNvCUiQR7bf8zZ9/0icotn7UVElmQcR8/j4fG5mYgscPZ9S0bsTtlkEXlTRL5w9utHEWngUd7C\nY9kDIvKkM11E5EkR2Skih0TkIyfZ5Ynz/zpNRA6KSKKIPONR1tD5jhx3yj/0KHtVRFKcsnUi0jyv\n2za+ZYmh5NgOpInIFBHplfmHQESuA54ErgNqAkuBD52yGsBs4GmgBrAL6JRp/ZnPKj0/TwPOAdHA\n5UAP4HaP8g7AFiAU+D/gfY+y/wEVgMuAWsCrTkxtnfnuAKoD7wLzRKRs5h1X1a7O25aqWllVZ4lI\nN+DfwCCgLrAX+Cjzso6fgJ4iMlpEOolIuUzlLwONgFbOv+HA/3Pi7AWMAq4BGgPdufgZuDrLVgQW\nOMegBvA34C0Rucxj3puAZ4GquP5fXnCWDQG+Bb5y9q8RsMhZ5iHgr7hqjGHAMeCti8SUnTeBSkAU\nEAsMF5FbnLLngG9UtSoQAYxz4voL0Blo5JTdCBy5hG0bX1JVe5WQF9AUmITrR/Ac8BlQ0yn7CrjF\nY95SwGmgHjAM1xmz57r2Abc6758FpnmURQJpzjpqA78BQR7lNwGLnfcjgO0eZRWAdFxJoA5wHqic\nzb68BYzJNG0r8Occ9j0diPb4PBF4yeNzsHNM6uewfE/neB0FfgVeAcQpOwU08Ji3I5DgvH8f+LdH\nWWPn2EQ7n5dkHEeP4/G9834w8F2mON4B/um8nwxM8Ci7FtjsvP8bsCaHfdkMXO3xua6z76Wymbcr\nsDeb6aWc/9emHtPu9Ph/nerEGp5puaud/6crM46fvQLvZTWGEkRVt6nqrapaH4jBdbb4mlMcCbzu\nNOccxXUWp7jOfsNwJQJPue2QrA+UBQ446z6G6wejhsc8v3jEeNZ5G4IrKR1V1V+zWW8k8EhGvM56\nI5xYcyMM2OOx3dO49jk8u5lV9RtV7a+q1YH+wEjgdhGpCVQE1ngcu/m4aj8Z2/E8VntwNWvlRiTw\np0z7eDOuZJvhF4/3Z3AdN3Adi10XWO8cj3g3A6mZ1nsxNXD9v+71mLaHP47f47iSx0oR2ZhRk1DV\nJbhqGuOBX0TkHad2YwKIJYYSSlW3A1NwJQhw/XjdparVnVc1VQ1R1RXAAVw/8J7qebw/jevHMYNn\nW/0+XGeWoR7rraqqrXIR5j6guohUzqHshWzi/TgX6wVIxvUDCYC4rjAKBZIutqDz47YY17E7jOsH\nuYVHLFVVtYoz+wG8j1Uk3k1JmY9dnUz7GJdpHyur6v252L99uJqPsrMXuDbTeoNV9UAu1pvhMK5k\nEukxLRLn+KlqiqreqarhwN24msCinbI3VbU90AJXLfaxPGzXFAJLDCWEiDQVkVEiEu58roerueFH\nZ5Z3gKczOgJFpIqIDHLKvgSai8h1IlJaRB7C++xyHdBFROqJSBVcfRUAqOovuNrJXxWRSk7HZ7SI\ndLlYzM6y83H9qFQVkTIi8men+D3gbhHp4MQbLCK9JedLSH/B1ceRYQZwi4i0cjqK/w2sUNW9mRcU\nkb+KyI0Z/TLONrsCP6qqOrG85tQeEJFwpy0dYCYwUkQuc/oM/l+m1a8DrheRCiLSCLjNo+wLoImI\nDHX2vayItBeRphc7ds6ytUXkQXF1jodkHCtc/TH/FufiAhGpKSJ/vcC6RESCPF+qmu7s2wvOuiOB\nh4HpzgKDMr5rwHFcTXlpTvwdRKQMcBbXSUNaLvbHFCJLDCXHSVztuj+JyElgObABeBRAVecCLwEf\nichxp6yXU3YEuAEYi+tMsSHwQ8aKVXUh8LGzzCpcl8V6Go7r8tjNuNroZ+F9ZpyZ5xn1MFz9DFuB\nFFwdp6jqGlwdz286zSHbcbXP52Q0MM1pPhmkqouBfwKf4jrLbYCr7yM7x5xtbReRE7g608eqakZn\n9RPATmCFc+wW4LoKDFX9Gldz3WInxkWZ1v0qrjPvX3D1GfzPfRBUTwF/ceJKdl4vAUFchLNsD1yd\nzL842451il/H1V+ywNmf5bguAMhJGK5a0RlcP+ZnnLP/B51pCcD3wP9UdbKzzBW4vmu/AnOBB1V1\nD1AZVyI9CiTi+j7952L7YwpXRueZb1YuEoHrj6gOrrOCCao6TkRexnUd+O+42kFvyaEd2QQoEVkC\nTFfVSf6OpagRkXRcV+Uk+DsWY7Lj6xrDeWCUqjbHdaXG/SLSDNcZVQtVbQPsAJ7ycRzGGGNyyaeJ\nQVV/UdV1zvtTuK5VD1fXjVXpzmwrcF1BYYoWuxv20tmxMwGt0IbEEJEooA2um4U83UrONxaZAKWq\n3fwdQ1GlqqX9HYMxF1Ionc/OdcqzgYecmkPG9GeAVFWdURhxGGOMuTif1xicy9Jm4+qo/Mxj+gig\nN5DjmaeIWJXbGGMugarm9kbKLAqjxjAJ1236r2dMcMaPeRz4q6pecHRPf98aXpxezz77rN9jKC4v\nO5Z2PAP5lV8+rTGIyFXAEGCjiPyMq9PtGeANXNe1fyuuQTZXqOq9vozFGGNM7vg0MajqD0B2HW2N\nfbldY4wxl87ufC5BYmNj/R1CsWHHsmDZ8QwsPr3zOb9ERAM5PmOMCUQiggZ457MxxpgixBKDMcYY\nL5YYjDHGeLHEYIwxxkuRSAxr1qyhe/fu/g7DGGNKhCKRGObMmcOyZctIS7MHPRljjK8VicTw1Vdf\ncf78eXbtyunZ5sYYYwpKwCeG5ORkdu/eTc+ePYmPj/d3OMYYU+wFfGKYP38+f/nLX2jTpg0bN270\ndzjGGFPsBXxi+Oqrr+jduzcxMTFWYzDGmEIQ8ENi1KhRg3Xr1nHs2DFuuOEGtmzZ4u+wjDEmoOV3\nSIyATwyVKlXixIkTpKamUqVKFY4dO0b58uX9HZoxxgSsYj9WUpMmTRARypUrR8OGDdm6dau/QzLG\nmGKtSCSGDNbPYIwxvmeJwRhjjJcilRhatmxpicEYY3ysSCWGmJiYHO9lePvttzl58mRhhWWMMcWW\nTxODiESIyGIR2SwiG0XkQWd6NRFZICLbROQbEamS0zoaN/7j8dANGjTg8OHD/Prrr17znDlzhr//\n/e+sW7fOV7tijDElhq9rDOeBUaraHOgI3CcizYAngYWq2hRYDDyV0wqqVPkjZ5QqVYrmzZuzadMm\nr3mWL1/OuXPn2Lt3rw92wRhjShafJgZV/UVV1znvTwFbgAigPzDVmW0qcF1u15nRAb1z505GjhyJ\nqrJ48WLKli3Lvn37CnoXjDGmxClTWBsSkSigDbACqK2qKeBKHiJSM7fryehn2LJlC1OnTuWWW25h\nyZIl9OvXzxKDMcYUgELpfBaREGA28JBTc7jk261btmzJ6tWrmT59Oo899hhjxowhPj6ewYMHW1OS\nMcYUAJ/XGESkDK6kMF1VP3Mmp4hIbVVNEZE6wMGclh89erT7fWxsLDExMfz444/06NGD0aNHU79+\nfTp06EDjxo2txmCMKZHi4uKIi4srsPX5fKwkEZkGHFbVUR7TxgJHVXWsiDwBVFPVJ7NZVjPHp6qE\nhoYyYcIEBg0axPjx46lYsSJ9+/aladOmHD161Kf7Y4wxgS6gB9ETkauA74GNuJqPFHgaWAnMBOoB\ne4EbVPV4NstnSQwAK1asoH379pQp80eFR1WpWLEihw8fJjg42Be7Y4wxRUJAJ4b8yikx5KRJkybM\nmzePZs2a+TAqY4wJbMV+dNW8qFevnnVAG2NMPhWrxFC/fn3rgDbGmHwqVonBagzGGJN/xS4xWI3B\nGGPyxxKDMcYYL8UqMdSvX9+akowxJp+KVWIIDw8nKSmJQL4E1xhjAl2xSgyVK1cGyPK8BmOMMblX\nrBKDiLhrDQDdunVj8+bNfo7KGGOKlmKVGMC7OWnVqlWsXr3a3yEZY0yRUuwSQ0REBElJSRw+fJhT\np05ledqbMcaYCyt2iSGjxpCYmAhgicEYY/Ko2CaGhIQEWrdubYnBGGPyqNgmhsTERLp3705KSgqn\nTp3yd1jGGFNkFNvEkJCQQKNGjWjatClbtmzxd1jGGFNkFNvEkJiYSHR0NC1atLDmJGOMyYNilxjq\n1KnDkSNH2LZtGw0aNLDEYIwxeVTsEkPp0qWpVasWSUlJ1K9f3xKDMcbkUZmLz1L0hIeHU6pUKYKC\ngmjWrBnbtm3zd0jGGFNk+LTGICLvi0iKiGzwmNZaRH4UkZ9FZKWItC/o7YaHh9OgQQPA1bSUkpJS\n0Jswxphiy9dNSZOBnpmmvQw8q6qXA88C/1fQGw0PDyc6OhqASpUqkZqaytmzZwt6M8YYUyz5tClJ\nVZeJSGSmyelAFed9VSCpoLfbvXt3zp07B7gG1qtZsyaHDh2ifv36Bb0pY4wpdvzRx/Aw8I2IvAII\n0KmgN9C/f3+vz5YYjDEm9/yRGO4BHlLVuSIyCJgE9Mhp5tGjR7vfx8bGEhsbm+cNZiQGY4wpjuLi\n4oiLiyuw9Ymvn3bmNCV9rqqtnM/HVbWqR/kJVa2Sw7JaEPENGTKEXr16MWzYsHyvyxhjAp2IoKpy\nqcsXxn0M4rwyJIlIVwARuQbY7usArMZgjDG559OmJBGZAcQCoSKyF9dVSHcAb4hIaeA34E5fxgCW\nGIwxJi98fVXSzTkUFfi9CxdSs2ZN9/MZjDHGXFixGxIjO1ZjMMaY3LPEYIwxxoslBmOMMV4sMRhj\njPHi8/sY8qOg7mNIT08nKCiI06dPU65cuQKIzBhjAldRuI/B70qVKkVoaCiHDx/2dyjGGBPwSkRi\nAGtOMsaY3LLEYIwxxoslBmOMMV4sMRhjjPFiicEYY4yXEpMY6taty4EDB/wdhjHGBLwSkxjCw8NJ\nSirwp4gaY0yxU2ISQ1hYmCUGY4zJhRKTGMLDw0lOTvZ3GMYYE/BKxJAY4BoWo0KFCpw4cYLy5csX\nyDqNMSYQ2ZAYuVSqVCnq1q1rtQZjjLmIEpMYwDqgjTEmN0pUYrAOaGOMuTifJgYReV9EUkRkQ6bp\nD4jIVhHZKCIv+TIGTzl1QG/atInU1NTCCsMYYwKar2sMk4GenhNEJBboB8SoakvgPz6OwS2npqSb\nb76Z5cuXF1YYxhgT0HyaGFR1GXAs0+R7gJdU9bwzT6E9JCG7xKCqJCQkcOrUqcIKwxhjApo/+hia\nAF1EZIWILBGR9oW14ewSw5EjRzh16hSnT58urDCMMSaglfHTNquq6p9E5ApgJhCd08yjR492v4+N\njSU2NvaSN5xd53NiYiKAJQZjTJEVFxdHXFxcga3PH4lhH/ApgKquEpF0EQlV1SPZzeyZGPIro/NZ\nVRFx3fuRkJAAWGIwxhRdmU+ax4wZk6/1FUZTkjivDHOBawBEpAlQNqekUNAqVqxIhQoVOHr0qHua\n1RiMMcabry9XnQEsB5qIyF4RuQWYBESLyEZgBjDclzFkFh4ezu7du92fExMTqV27tiUGY4xx+LQp\nSVVvzqFomC+3eyEDBgzg7bffZuLEiYCrKSkmJsYSgzHGOErUnc8Ao0aNYu7cuezatQtw1Rhatmxp\nicEYYxwlLjFUq1aNBx54gH/961+kpaWxb98+mjdvbonBGGMc/rgqye/+/ve/07RpU7744gtq1qxJ\n9erVLTEYY4yjxNUYAKpUqcKYMWO44447aNCgAcHBwZYYjDHGUSITA8Dtt99O3bp1iY6OJiQkxIbE\nMMYYR4lsSgIoXbo0H3/8MadPn6ZUqVJWYzDGGEeJTQwAzZo1A2D79u2WGIwxxlFim5I8WR+DMcb8\nwRIDlhiMMcaTJQb+SAyq6u9QjDHG7ywxAGXLlqVUqVKcO3fO36EYY4zfWWJwZG5O+vDDDzlypFAG\nfTXGmIBiicHhmRji4+MZOnQoS5Ys8XNUxhhT+CwxODwTw6OPPkr16tXZt2+fn6MyxpjCZ4nBkZEY\nFi5cSGJiIg8//LAlBmNMiZSrxCAiDUUkyHkfKyIPikhV34ZWuDISw48//sigQYOIjo62xGCMKZFy\nW2P4BEgTkUbABKAerqevFRshISGcPn2apKQkwsPDqVevniUGY0yJlNvEkK6q54EBwDhVfQyo67uw\nCl9GjSEpKYmwsDDq169vicEYUyLlNjGkisjfgBHAF860sr4JyT88E0N4eDh169bl0KFDpKam+js0\nY4wpVLlNDLcAHYEXVDVRRBoA/7vYQiLyvoikiMiGbMoeFZF0Eamet5B9I3NiKFOmDLVr1yY5Odnf\noRljTKHKVWJQ1c2q+qCqfigi1YBKqvpSLhadDPTMPFFEIoDuwJ48RetDwcHBHD9+nGPHjlG7dm0A\n6tWrx969e/0cmTHGFK7cXpUUJyKVnbP7tcB7IvLfiy2nqsuAY9kUvQo8lqdIfSw4OJidO3dSq1Yt\nSpcuDWAd0MaYEim3TUlVVPVX4HpgmqpeieuMP89EpB+wT1U3XsryvhIcHMyOHTsICwtzT7PEYIwp\niXL7oJ4yIlIXGAw8c6kbE5EKzvI9PCdfaJnRo0e738fGxhIbG3upm7+g4OBgtm3bxlVXXeWeVq9e\nPXbs2OGT7RljTEGJi4sjLi6uwNaX28TwL+Ab4AdVXSUi0cCl/GI2BKKA9SIiQASwRkQ6qOrB7Bbw\nTAy+FBwczKFDhwgPD3dPq1evHosXLy6U7RtjzKXKfNI8ZsyYfK0vV4lBVWcBszw+JwADc7kNcV6o\najxQx10gkgi0VdXs+iEKVXBwMECWxGBNScaYkia3nc8RIjJHRA46l59+4lxZdLHlZgDLgSYisldE\nbsk0i3KRpqTCkpEYPPsYIiMjSUxMJC0tzV9hGWNMoctt5/NkYB4QBoQDnzvTLkhVb1bVMFUNUtX6\nqjo5U3m0qh7Na9C+kF2NoVatWkRFRbFs2TJ/hWWMMYUut4mhpqpOVtXzzmsKUNOHcRW67BIDwODB\ng5k5c6Y/QjLGGL/IbWI4LCJDRaS08xoKFKvHm4WEhABZE8MNN9zAJ598Ys1JxpgSI7eJ4VZcl6r+\nAhwABuEaJqPYCAkJISQkhMqVK3tNb9SoEeHh4Xz//fd+iswYYwpXbofE2Kuqf1XVmqpaS1Wvw3Wz\nW7ERFhbGt99+m23ZjTfeyJQpUwo3IGOM8RNR1UtbUGSvqtYv4Hgyb0MvNb6CdPz4cZo2bcq3335L\nq1at/B2OMcZckIigqpd8xWd+Hu0ZEJeZFoaqVavyz3/+k8ceC6jhnYwxxifykxj8fypfiO666y4S\nEhJYsWKFv0MxxhifumBiEJGTIvJrNq+TuO5pKDHKli1L165dWb9+vb9DMcYYn7rgkBiqWqmwAikK\nGjZsyK5du/wdhjHG+FR+mpJKnIYNG5KQkODvMIwxxqcsMeRBdHS01RiMMcXeJV+uWhgC5XLVDMeO\nHSMyMpITJ07gGjXcGGMCjz8vVy1xqlWrRpkyZTh8+LC/QzHGGJ+xxJBH1gFtjCnuLDHkUXR0tHVA\nG2OKNUsMeWQ1BmNMcWeJIY8sMRhjijtLDHlkicEYU9xZYsijhg0b8vPPP9OhQwcbVM8YUyz59D4G\nEXkf6AukqGorZ9rLQD/gd2AXcIuq/prD8gF1HwOAqjJjxgyqVavGkCFD2L9/v/uxoMYYEwjyex+D\nrxNDZ+AUMM0jMXQHFqtquoi8BKiqPpXD8gGXGDz16dOHv/3tbwwdOtTfoRhjjFtA3+CmqsuAY5mm\nLVTVdOfjCiDClzH40vDhw5k6daq/wzDGmALl7z6GW4H5fo7hkv31r39lzZo17Nu3z9+hGGNMgbng\nsNu+JCLPAKmqOuNC840ePdr9PjY2ltjYWN8GlgcVKlTg2muvZcGCBdx2223+DscYU0LFxcURFxdX\nYOvz+SB6IhIJfJ7Rx+BMGwHcCXRT1d8vsGxA9zEAvP7662zbto233nrL36EYYwwQ4H0MDsHj+dAi\n0gt4HPjrhZJCUdG+fXtWr17t7zCMMabA+PqqpBlALBAKpADPAk8D5YAjzmwrVPXeHJYP+BrD6dOn\nqVmzJsePH+fYsWPs3buXK664wt9hGWNKsIC+XDW/ikJiAGjZsiVTp05l2rRpbNu2jfnzi2x/ujGm\nGMhvYvBb53Nx0r59e3744QemT59OtWrV/B2OMcbki78vVy0W2rdvz0svvUTz5s3Zt28fqamp/g7J\nGGMumSWGAtC+fXuSk5O55557CAsLY/fu3f4OyRhjLpklhgLQunVr+vfvz4ABA2jcuDE7d+70d0jG\nGHPJLDEUgPLlyzN37lwqVKhAo0aN2LFjh79DMsaYS2aJoYBlrjEsW7aMFi1acNlll7F06VI/RmaM\nMbljiaGANW7c2KvG8Mknn9C7d2/+9Kc/sXLlSj9GZowxuWOJoYA1atTIq8awaNEiBg0aRJs2baxT\n2hhTJFhiKGANGjRg7969pKamcvDgQfbs2UO7du2IjIy0xGCMKRLsBrcCFhQURFhYGHv27GHt2rV0\n6dKFMmXKEBUVxZ49e/wdnjHGXJQlBh9o0qQJixYt4ueff6Zbt24AREVFsXv3blQVkUu+U90YY3zO\nxkrygTVr1tC/f39OnDjBsmXLaN26NQBVqlQhMTGR6tWr52l9J06cICgoiPLly/siXGNMMVMUht0u\ncdq1a8fKlSu57777aNmypXv6pTYnPf7440yYMKEgQzTGmBxZYvCRsLAwXnrpJUqV+uMQZzQn5dX+\n/fvZvn17AUZnjDE5sz6GQpRxZdKpU6fYt28fl112Wa6WS0lJIT093cfRGWOMi9UYClFGU9LLL7/M\nsGHDspTn9OOfkpJCQkKCr8MzxhjAEkOhioqKYseOHUycOJH4+HhOnTrlLvv2229p1aoVJ06c8FpG\nVd33Q6SlpRV2yMaYEsgSQyGKjIzkm2++oXHjxrRt29ZriIy4uDhSUlIYMWKEV83h+PHjBAUFUaNG\nDfbv3++PsI0xJYwlhkIUFRVFWload999N1dddRU//PCDu2z16tW88847HDhwgA8++MA9PSUlhTp1\n6tCwYUN27drlj7CNMSWMTxODiLwvIikissFjWjURWSAi20TkGxGp4ssYAkn16tV56KGHuP766+nc\nubM7Magqq1evplOnTtx3333MnTvXvUxKSgq1a9emYcOG1s9gjCkUvq4xTAZ6Zpr2JLBQVZsCi4Gn\nfBxDwBARXnvtNYKCgujUqRMrVqwgLS2NxMREKlSoQN26denVqxeLFi3i3LlzwB+JITo62moMxphC\n4dPEoKrLgGOZJvcHpjrvpwLX+TKGQFWzZk1q165NfHw8q1at4oorrgCgVq1aNG7cmOXLlwPwyy+/\nuGsMlhiMMYXBH30MtVQ1BUBVfwFq+iGGgHDttdcybtw4Vq1aRfv27b2mz58/H/BuSrLEYIwpDAF/\ng9vo0aPd72NjY4mNjfVbLAXtueeeo3379hw+fJgPP/zQPb13797ccccdjB07lpSUFK644gqio6Pz\n1cfw6aef0qdPH4KCggoidGNMAImLiyMuLq7A1ufzQfREJBL4XFVbOZ+3ALGqmiIidYAlqprtLcBF\ndRC9vFi/fj3du3dn69athIaGApCWlkbt2rVZt24d99xzD7fddhv9+/enatWqJCQkuOfLrfT0dKpU\nqcKCBQvo2LGjL3bDGBNAisIgeuK8MswDRjrvRwCfFUIMAat169YcOHDA68e+dOnSxMbGsmTJEvfl\nqiJC8+bN2bx5c563kZiYyKlTp6wpyhiTK76+XHUGsBxoIiJ7ReQW4CWgh4hsA7o7n0u0MmWytuhd\nc801LFq0yN3HABATE0N8fDwAX3zxBU8++SRffPEFF6tVrV+/HsASgzEmV3x9VdLNqhqmqkGqWl9V\nJ6vqMVXtrqpNVbWHqh73ZQxFVbdu3S6YGN544w327dvHk08+yYMPPnjB5LB+/XoiIiK8nkVtjDE5\nsTufA1STJk1QVcqWLUvFihWBPxJDeno6K1eu5LXXXuOHH35gxYoVPPPMMzmua/369QwYMMBqDMaY\nXLHEEKBEhGuuucZdW4A/EsPmzZupWbMmNWvWpEqVKsyfP58JEybk+KyH9evXM3DgQHdiUNWLNj8Z\nY0ouSwwBrFu3btSpU8f9uVatWpQqVYpPP/3U6+qiGjVqcNdddzF27Ngs6zhx4gSHDh2ic+fOnDx5\nkpMnT/L888/z9NNPF8o+GGOKHksMAeymm27ivffec38WEVq2bMn777+f5bLTv//973z88cdMnTqV\ne+65h4MHDwKwYcMGYmJiKF26tPsmuY8++ojx48dz/Lire+fs2bMsXbrURm81xgCWGAJaUFBQlqe8\nxcTEsHfv3iyJoWbNmjz88MNMmzaNAwcO8MADDwCwdu1aWrduDUDDhg1ZuHAhhw8fpm/fvkycOJGZ\nM2dSq1YtbrzxRl56yfsCsW3btvHzzz/7cA+NMYEo4O98Nt5iYmIIDg4mJiYmS9k///lP/vnPf3L2\n7Flat26jkbh2AAAfrUlEQVTNPffcw+zZs5k1axbgSgzjx4+nd+/e3H///fTo0YOgoCB++OEH9u/f\nz+uvv+61vrfeeovdu3fz2Wcl+lYTY0ocqzEUMZ07d2bYsGHZ3vuQoUKFCkyaNIn4+Hi+//579zAi\njRo1Yvfu3fTp04d27dpx7733smDBAlq1akWTJk3Yvn2713rWrFnD4sWL3SO9GmNKBp8PiZEfJWFI\njMK0YMEC+vTpw+HDh6lSxfsxGOfPnyckJITjx49Tvnx50tLSqFq1KmFhYUyYMIGuXbuSlpZG6dKl\n/RS9MSa3isKQGCZAdOjQgZdffjlLUgDX3ddRUVHuS1q3bdtG7dq1GTRoEAsWLGDx4sU0bdrULnM1\npgSwxFCCVK1alYcffjjH8saNG7Njxw7A1Wndrl07evbsyeeff85dd91FUlJSluYmX1NVRo0aRVpa\nWqFu15iSzBKDcfPsZ1izZg1t27alY8eOJCYm0rp1awYMGOD1nOrCcPDgQV599VWSkpIKdbvGlGSW\nGIxb5sTQrl07ypYty4QJE3jzzTe56qqr3InhzJkzpKeney0/duxYYmNjGTBgQIE1OWXEk5iYWCDr\nM8ZcnCUG45aRGNLT01m3bh1t27YF4G9/+xt16tRxJ4bU1FRat25NWFiY1xhNH3zwAcOHD2fZsmUk\nJycXSEwXSgw//PADd999d4FsxxjzB0sMxq1Jkybs2LGD2bNnU69ePapXr+5V3rJlSw4cOMB///tf\nIiMj+f777/nPf/7jrjkkJyfTt29frrjiClatWlUgMW3fvp0KFSpkOw7UnDlz+OqrrwpkO8aYP1hi\nMG5hYWH8+uuv3HfffUyfPj1LeenSpbnyyiv5xz/+wZgxY2jSpAlVqlTh4MGD/P777/z666/UqFGD\n9u3bs3r16hy3s3fvXlJTU3MV0/bt24mNjc22xrBo0SL27dvnHtqjqEtPT+f8+fP+DsMYSwzmDxkj\nur711lvuZqTMYmNjueaaa7jqqqsAqFevHvv37yc5OZm6detSqlQprrjiimwTw+nTp3n22Wdp2LAh\n48aNy1VMO3bs4C9/+UuWxHDo0CESEhK4/PLL2bRpUx73NDBNmzaN2267zd9hGGOJwXibN28eN9xw\nQ47ljz32GHPmzHF/joiIYN++fSQnJxMWFgZA+/btWbVqlbsDOi0tjcGDB1OnTh02b97MBx98wIQJ\nE1BVJk6cyI033pjtttLS0ti1axc9evTI0pS0ePFiunTpwuWXX+5+eJEvqKp7QEJf27hxI19//bXd\nK2L8zhKDyZOyZctSoUIF9+eIiAj2799PUlIS4eHhANStW9erX+DQoUMsWrSI/fv3M2vWLG644QZK\nly7NnDlzePrpp1m6dCmLFy92r/P8+fP8+uuv7Nu3jxo1atC0aVNSUlK8huZYtGgR3bt3p2XLlmzc\nuDHbWNPT03nooYfyVaNYu3Ytbdu2LZQf6507d3Lw4EGfJjpjcsNviUFEHhaReBHZICIfiEg5f8Vi\nLl1GYvCsMQBe/QzHjx+nRo0a7juuRYS7776bm2++mcGDB/Pqq6/y+OOPk56ezu7du+ncuTNXXnkl\n8fHxNGnShDJlyhAWFsbevXsB16Wy33zzDd27d/d63GlmX375JZ9++imxsbH85z//YcyYMWzYsCFP\n+7dp0yaSkpIKpblq586ddOzYkYULF/p8W8ZciF8Sg4iEAQ8AbVW1Fa5RXm/yRywmf+rVq+duSsqo\nMYB3Yjh27BhVq1b1Wm7YsGF07NiRf/3rXwwePJigoCBCQ0OJiYlh8ODBNGjQgMcee4wmTZoAEBUV\nxe7duzlx4gQ9e/YkNjaW5s2bExMTw8aNGzl27BidOnXi+++/d29j7NixvPLKK8ybN49Nmzaxdu1a\nXn755Tzt39atWylTpgzffvvtpR6iXElPTychIYE777yTRYsW+XRbxlyMP5uSSgPBIlIGqAgUzIXv\nplB5NiV51hiio6PdZ/jHjx+nWrVqXstVrVqVJUuWUL16dUSEJUuWuJtSRo0axbhx40hMTHQnhgYN\nGrBr1y4GDBhAy5YtmTx5MiJC7dq1KVWqFCNGjCAkJIRBgwYxY8YMPvroI3755RcGDhxIx44dmTx5\nMu+++y5ffvklv/32W673b8uWLQwcONArMezcuZPBgwfn57BlkZSURLVq1ejbty9Lly5l/PjxREdH\nc+rUqQLdjjG54ZfEoKrJwCvAXiAJOK6qVn8ugjybkjxrDKGhoRw5cgTIvsaQWbly5QgNDaVixYqA\n69kRH3zwAQMGDABcieGFF14gNTWVcePGUaqU66srIsTExLB69WpmzZrF7NmzmThxIs899xwvvvii\n12iwderUoVWrVtme/Z88eTLbuLZu3cq9997LsmXL+P333wGYOHEis2bNcie+3Lr++utZvnx5tmU7\nd+6kUaNG1KhRg4YNG/Luu+9Sp04dPvnkkxzXd+TIkTwlOWNyy19NSVWB/kAkEAaEiMjN/ojF5E9E\nRARJSUns37/fq8aQOTFkrjHkxsCBA4mKigJcieH48eNMnz49y9Dft956K9OmTaNKlSp06dKFxYsX\ns2nTpmyvrrrhhhvcDy7KcObMGSIjI1m2bJnX9NTUVBITE+nQoQPNmjXjxx9/5Pz580ybNo127dqx\nYMGCHGM/cOBAlg7rn376idmzZ2c7/44dO2jcuDEAn332GT/99BOPPPIIU6ZMyXEbt9xyC5dffnmB\n3UxYGMaPH5/lgVAm8PirKak7kKCqR1U1DfgU6JTdjKNHj3a/4uLiCjNGkwvly5encuXK7Nq1K8fE\nkF1TUl4NHDiQ5cuXuxOFp2HDhtG9e/dcref666/n888/58yZM+5ps2fP5syZM7z//vte8yYkJBAR\nEUH58uXp168fL7zwAnPmzKFevXrcf//9OSaGhIQEmjZt6pVozp49S3JyMl9++WW2y2TUGMDVb1Oh\nQgX69u3Lxo0bs73rG1zNXEOHDqVPnz451ixyeyNhYVBVXn31Vetc94G4uDiv38p8U9VCfwEdgI1A\neUCAKcB92cynJvC1adNGQ0JCvKadOHFCK1WqpKqqjzzyiI4dO9YfoWVryJAhOnLkSE1PT1dV1S5d\nuuj48eO1SpUqeuLECfd8c+bM0T59+qiq6rlz53TkyJFaunRpffvttzUpKUmrV6+u58+f91p3amqq\nduzYUatWrapTpkxxT9+yZYs2bNhQ69atqzt27MgS04ABA3TmzJlZpt933306ZMgQ3b59u9f0c+fO\nably5fS3337Tn3/+WWvXrq2ffvqp1zwzZszQHj165PHo5E/GMc3Od999pzVr1tT69esXYkT5l5aW\npocPH/Z3GHni/HZe8m+0v/oYVgKzgZ+B9U5ymOCPWEz+1atXz6u2AFCpUiXOnj3LuXPnCqTGUJDe\neecdVq1axcsvv8y6devYunUrt99+O1dffTUzZ850z7d161aaNWsGuO7fmDRpEpMmTWLo0KGEhYUR\nFhbGTz/9xKZNm9zPi3jhhRcICQnh3nvvZc+ePe51JSQk0LBhQ3r37p1trcGzxuDpqaeeIiQkhE6d\nOnnVCnbv3k14eDhBQUG0adOG6dOn8/zzz3stO2nSJBYtWuSuuRWGP/3pTzkOhzJp0iQef/xxjhw5\nwokTJwotpvyaOHEi1157rb/DKFR+uypJVceo6mWq2kpVR6hq4NR5TZ5ERER4dTyDq1O4evXqHD16\nNFedz4UpJCSEOXPm8M0339ChQweGDx9OuXLluPXWW5k8ebJ7Ps/EAK59Gj58OCEhIQD06tWLLl26\ncPXVVzNy5EiWLVvG22+/zZQpU4iKivJKDImJiURHR9OnT58siUFV2bVrV7aJITw8nHfeeYfx48fz\n1ltvuad79kkAdO3ale3bt7vHjUpOTmbNmjV0796d+fPn5+t4paenM2LECK9hSTSbG/7279/PypUr\nGT9+fJaykydP8tlnnzF8+HD3JcZFxeTJk1mzZg2//vqrv0MpNHbns8m3iIiILDUG+KOf4VI7n32p\ncePGLF68mKNHj/LCCy8A0LNnTzZv3uweMnzLli1cdtllOa5jzJgxJCcns2fPHpKTk7nmmmt49913\nCQsLIzIyMkuNITo6mh49erBq1SqvYcnj4+OpVasWlSpVynFb/fv3Z8OGDSQkJABZE0O5cuW48sor\n3f0aH330Eddddx2DBw9m3rx5eT4+Z8+eZenSpYBrFNvZs2dz55138vvvv9OvXz9eeeWVLMssXbqU\nzp07M2fOHI4dO+ZVNnPmTGJjY6lVqxYtW7Zkw4YNpKenF9pd3ufOnePo0aPZli1evDjHWs7WrVvZ\nvXs3nTp1cl9RlvH42+LMEoPJt8GDB3P//fdnmZ6RGAKtKclTSEgI5cq5brovV64cvXv35rPPPmP3\n7t3s2rWLyy+/PMdlK1asSK1atahQoQLz5s1j9uzZ9O/fH4D69etnSQwNGjQgJCSEm266iffee89d\nNmvWLAYOHHjBOIOCghgyZIi7RpM5MYCr1vDdd9+hqvzvf/9zd0wvWLDAaziRDOnp6fzrX//K9pLX\nL774gquvvpolS5bw73//m2nTpnHs2DHatm1LcnIyU6dOzbLM0qVLGTBgAL1792batGleZZMnT+bW\nW28FoFWrVmzYsIFPP/2UK664grNnz7rnS05OJiYmJl9DkHzyySe0bdvW62mDs2fPZtCgQdnO/+ij\nj9KtW7csTXEAU6ZMYejQoXTr1o3vvvuOPXv2uIen9/TFF19k6ejPnByLlPx0UPj6hXU+F2n9+/fX\nTz/9VKOionTnzp3+DidXZs2apT169NAnnnhCH3744Utez+nTpzUoKEjT0tJUVbV169a6Zs0aVVXd\nuHGjhoWF6blz5zQ9PV2bNm2qP/3000XXuWHDBo2IiNDU1FTt0aOHfvnll17l33//vbZv315nz56t\nzZs3d3eMX3nllfrJJ5+oquqOHTvc72fPnq2Azp8/P8u2Hn/8ce3Ro4dWqlRJW7RooWlpabpx40a9\n88479cyZM1q3bl3dsmWL1zIxMTG6cuVKjYuL09atW7unb926VevUqaOpqamqqhoXF6edOnXSTp06\naUhIiH799dfueWfOnKmAbt68+aLHIycjRozQG2+8UWvVqqXfffedqqq++OKLWrp0aT1y5IjXvIcO\nHdLKlSvr7t27tWrVqnrgwAF3WXp6ukZERGh8fLwuWrRIO3XqpE888YQC+vHHH3utQ0S8juOMGTO0\nbNmyGhcXd8n7kR/ks/PZ7z/+FwzOEkORduutt+rEiRO1atWqWf4gA9XJkye1UqVKWqNGjSxXAuVV\nzZo19cCBA5qenq6VKlXSY8eOucu6dOmiH374oa5fv14jIyMveDWPp44dO+rcuXM1KioqS3y//fab\nhoSEaHh4uPsHUVV14cKFWqNGDX3xxRe1du3aGhoaqgsXLtRWrVpply5d9MEHH8yynauvvlrnz5+v\n7733ni5YsCBL+f3336/PP/+8+/ORI0e0UqVKmpqaqqmpqRoSEqJHjx5VVdUnnnhCH3vsMa95y5Qp\no1FRUTpmzBh96KGH3GWPPPKI+8qvzL788ksdM2ZMlunnzp3Tvn376uLFi1VVNTIyUrds2aJ33323\nvvnmm6qq+sADD6iI6LRp07yW/eijj7Rfv36qqtq5c2ddsmSJu+zAgQMaGhqq6enpevr0aQ0ODtYa\nNWrokCFD9Mknn3TP98EHH2jZsmX19ttvV1XVqVOnakREhD7yyCPau3fvLPEWBksMJmA9+uij+uKL\nL2qpUqWyXNYZyPr166c9e/bM93rat2+vK1as0MOHD2vVqlW9yhYuXKihoaHarl07rx/Ni5k+fbp2\n6dJFy5Urp+fOnctSfvXVV+vw4cOzTF+5cqW2atVK58+fr99++60GBwdru3btdM2aNdq4cWOvedPS\n0rRy5cp66NChHOP47rvvvGoF8+bN0+7du3vF8dVXX2l6errWq1dP4+PjvZaPiIjQV155RdesWaNN\nmzZ1T+/cubMOGTJEb7rppizbvP3227VcuXKalJSkqq5EqKr61FNPaWhoqN54442amJiotWvX1vT0\ndH3++efdP+ADBw7Ua6+9Vq+//no9ePCgDh8+XI8dO6a33Xabvv766+71v/XWW+7tLVy4ULt27er+\n3LFjR7322mt17ty52qtXL/f0YcOG6RNPPKE1atTQ48ePa+3atfXnn3/Ws2fPap06dXTjxo05Hkdf\nscRgAtaLL76od955p1auXNnfoeTJli1bsr3XIK+uv/56/fjjj3XlypXatm3bLOX79u3TO+64Q7du\n3Zrrdf72229ao0YNbdSoUbble/fu1VOnTl10Pe+++64uW7ZM09PTtU6dOl77u23bNo2Kirrg8ufP\nn9fIyEjt2rWrDhkyRGvVqqWvvfaau/wf//iHPvPMM7p27Vpt3LhxlhrRsmXL9PTp05qWlqa1a9fW\nhIQEPXfunAYHB+vatWs1LCxM09PT9cEHH9Rt27apqmqrVq30z3/+s44aNUqnTJmipUuX1iZNmmhY\nWJhu2bJFK1eurG+88YYOHjxYVVWnTJmiQ4cOVVXXj/qcOXO0cuXK2qlTJ23YsKGOGDFC69ev7262\neuWVV7xqT6+99pred9997s9ffPGFrlu3Tvfs2aO1a9dWVVcSrVWrliYmJmq7du20d+/eesMNN7iX\nef7553XEiBEX/f8oaPlNDGX80K1hSojQ0FAWLlwYsB3POfG8RDU/Mq5MUlWio6OzlEdERDBhQt5u\n3wkKCuL2229n/fr12ZbXq1cvV+u588473e979erFpEmTSEpKomfPnoBrdNwLKV26NBs2bGD58uXs\n3buX0aNHe11ue9VVVzF27FjKli1Lv379EBGv5TOeAJix/S+//JJOnToRGRlJmzZtEBHGjBnDG2+8\nQWhoKKNGjWLnzp3Ex8fTpk0bQkJC2LBhA7/99hvVq1cnKiqKrl27MmbMGJ577jngj3G8wNWp3bp1\na9q0aUOtWrX4+uuvadOmDWlpae7/72bNmvH111+744qPj/d6kmGfPn0A18l0amoqBw4cIDk52b39\ngQMH8swzz3hdinvPPffQqFEjr+eVFAWWGIzPhIaGsnPnzoC6h6EwRUZGsnPnTtauXXvRH9q8ePrp\np0lJSSmw9fXt25ebb76ZO+64g4ceeohrrrkmV/FWrlyZXr16ZVuWcaPbsWPHLjo20p133snAgQO5\n/fbbufLKKxERunTpwvPPP8/YsWPdz9Ro1aoVDRo04M0336R9+/ZZLiUeOnQon3/+ObGxscAfQ8Kr\nKgcOHKBu3brMnTuXkJAQypYty4wZM1i3bp07aV122WVs3brVvb74+HiGDx+eJV4R4fLLL+fnn39m\nxYoV7mMwcuRIqlSpQosWLdzzVq9enWHDhvH666/nech3v8pPdcPXL6wpqUiLi4tTEdHY2Fh/h+IX\nc+bM0aioKI2IiMhV846/pKen68GDB1XV1QQE6MKFC/O93piYGK1WrZr7aqQLee655xTQd999V1VV\nv/rqK/2///s//f3337VKlSr6+OOPZ9tJ7unMmTM6atQod7PVqVOnNCgoSA8ePKjVq1e/aAznz5/X\nChUq6MmTJ90XDOR00cSjjz6q/fr105o1a7qbunKSmJio1atX1+PHj180hoKC9TGYQLVhwwYF9Lrr\nrvN3KH6xdu1aBfR///ufv0PJtbNnz+rIkSO9xoy6VHfddZcOGTIkV/OeP39e77//ft29e3eWsgED\nBmhwcPAlHcdq1arpokWLNCYmJlfzt27dWlevXq179uzRsLCwHOf74IMPtFSpUtle6pudm266SceN\nG5ereQtCfhOD3eBmfCY0NBSgyPUxFJRmzZrx3HPPcfPNRWdE+fLlyzN58mQqV66c73U988wz/Pvf\n/87VvKVLl2bcuHFERkZmKevduzenT5+mQ4cOeY4hIiKClStXZntnfnaaNWvGli1biI+P92oSyqx/\n//58++23OTalZda3b98iNTq0JQbjMyU9MVSoUIF//OMfWTpeS4p69epRv379fK+nd+/eNG/ePNux\npHITw08//ZTrxJDRzxAfH09MTEyO8wUHB9OtW7dcx9GxY0d+/PHHjJaQgGeJwfhMUFAQwcHBJbbz\n2RSMsLAwNm3adEkJNiIiIk+JoUWLFrz++us8++yzdOnSJc/by0mDBg04f/48+/btK7B1+pJdlWR8\nKjQ0tMTWGIz/RUREcODAgVwnhv79+7NixQrq169/wUEN80pE3LWGgqhF+ZrVGIxPhYaGWo3B+E1E\nRARArhND2bJladGiRYEmhQwZiaEosMRgfKpu3brUqVPH32GYEirjhr/cJgZf6tixo3vo7kAngdwZ\nIiIayPGZizt58iTBwcGUKmXnIKbwbd26lcsuu4y9e/fm+q5wXzlz5gw1a9bk8OHDVKhQwafbEhFU\n9ZKverC/VuNTlSpVsqRg/CYiIoKgoKCAqLVWrFiRe++9t0g8p8FqDMaYYq2ojVNUEIpsjUFEqojI\nLBHZIiKbRORKf8VijCm+SlpSKAj+rOO/DnylqpcBrYEtfoylRChKd14GOjuWBcuOZ2DxS2IQkUrA\nn1V1MoCqnlfVX/0RS0lif3wFx45lwbLjGVj8VWOIBg6LyGQRWSsiE0TEt930xhhjcsVfiaEM0BYY\nr6ptgTPAk36KxRhjjAe/XJUkIrWBH1U12vncGXhCVftlms8uSTLGmEuQn6uS/DJWkqqmiMg+EWmi\nqtuBa4DN2cxXMoelNMYYP/LbfQwi0hqYCJQFEoBbVPWEX4IxxhjjFtA3uBljjCl8ATlWgYj0EpGt\nIrJdRJ7wdzxFkYjsFpH1IvKziKx0plUTkQUisk1EvhGRKv6OM1CJyPsikiIiGzym5Xj8ROQNEdkh\nIutEpI1/og5cORzPZ0Vkv3Nl4loR6eVR9pRzPLeIyF/8E3VgEpEIEVksIptFZKOIPOhML7DvZ8Al\nBhEpBbwJ9ARaAH8TkWb+japISgdiVfVyVc14JuKTwEJVbQosBp7yW3SBbzKu76CnbI+fiFwLNFTV\nxsBdwDuFGWgRkd3xBPivqrZ1Xl8DiMhlwGDgMuBa4C0pqY/By955YJSqNgc6Avc5v5EF9v0MuMQA\ndAB2qOoeVU0FPgL6+zmmokjI+v/bH5jqvJ8KXFeoERUhqroMyDzaWebj199j+jRnuZ+AKs6Vd8aR\nw/EE1/c0s/7AR86Nr7uBHbh+Fwygqr+o6jrn/Slco0ZEUIDfz0BMDOGA5/Pv9jvTTN4o8I2IrBKR\n251ptVU1BVxfLqCm36IrmmplOn61nOmZv7NJ2Hc2t+5zmjcmejR92PHMJRGJAtoAK8j6933J389A\nTAzZnUFYD3nedVLV9kBvXH98f8aOo6/Yd/bSvIWriaMN8AvwijPdjmcuiEgIMBt4yKk55HSM8nw8\nAzEx7Ac8H4oaAST7KZYiyzljQFUPAXNxVcVTMqqQIlIHOOi/CIuknI7ffsDzKTD2nc0FVT3kMa7+\ne/zRXGTH8yJEpAyupDBdVT9zJhfY9zMQE8MqoJGIRIpIOeAmYJ6fYypSRKSiczaBiAQDfwE24jqO\nI53ZRgCfZbsCk0HwPtvyPH4j+eP4zQOGA4jIn4DjGVV648XreDo/XhmuB+Kd9/OAm0SknIg0ABoB\nKwstyqJhErBZVV/3mFZg38+AvI/BuWztdVyJ631VfcnPIRUpzh/THFzVxTLAB6r6kohUB2biOnvY\nC9ygqsf9F2ngEpEZQCwQCqQAz+Kqec0im+MnIm8CvYDTuG7WXOuHsANWDsfzalzt4+nAbuCujB8s\nEXkKuA1IxdVUsqDwow5MInIV8D2ukz11Xk/jSp7Z/n3n9fsZkInBGGOM/wRiU5Ixxhg/ssRgjDHG\niyUGY4wxXiwxGGOM8WKJwRhjjBdLDMYYY7xYYjDFiojUEpEPRGSnM07UDyLil0EYRaSriHT0+HyX\niAz1RyzG5IVfHu1pjA/NBSar6hAAEakH/NVXGxOR0qqalkNxLHAK+BFAVd/1VRzGFCS7wc0UGyLS\nDfinql6dTVkp4CWgKxAEjFfV90SkKzAaOAzEAKtVdZizTFvgv0CwUz7SeV75EmAdcBXwIa5hof+B\n6zG1R4AhQEVcI16eBw4BDwDdgZOq+l/nYSlvAxWAXcCtqnrCWfdPuO4KrgLcpqo/FOiBMuYirCnJ\nFCctgJxu9b8N1xgxV+IarO1OEYl0ytoADwLNgYYi0skZpGwcMFBVr8D1oJl/e6yvrKp2UNVXgaWq\n+idVbQd8DDyuqntwPRDlVechNJl/3KcCjzkji8bjGiIiQ2knzodxJS1jCpU1JZliyxkfpjNwDtgD\ntBSRG5ziykBjXGPxrFTVA84y64Ao4ASuGsS3ztPDSuE9IuXHHu/richMoC6uWkPiReKqDFRxHl4D\nriQx02OWT51/1wCRGFPILDGY4mQTMDDjg6re7wwcuAZXYnhAVb/1XMBpSvrdY1Iarr8LAeJV9aoc\ntnXa4/044D+q+qWzvmdzWMZr0xcoy4gnIxZjCpU1JZliQ1UXA0EicpfH5BCcp9kB9zpNRIhIYxGp\neIHVbQNqOsMUIyJlRKR5DvNW5o/axAiP6Sedssxx/gocdUbJBBgGfJfDuu1Zx6bQ2dmIKW6uA14T\nkcdxdfqextXmP9sZjnyt0zR0kOyfea0AqpoqIoOAcc4jJ0sDrwGbyfr0qzHAbBE5iush7FHO9M+d\n6X/F1fnsudxI4B0RqQAkALd4bj9zPMYUJrsqyRhjjBdrSjLGGOPFEoMxxhgvlhiMMcZ4scRgjDHG\niyUGY4wxXiwxGGOM8WKJwRhjjBdLDMYYY7z8f49hzOzRKgeQAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x115ffa4d0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "plt.plot(train_loss, 'k-')\n",
    "plt.title('Sequence to Sequence Loss')\n",
    "plt.xlabel('Generation')\n",
    "plt.ylabel('Loss')\n",
    "plt.show()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}