Main_scratch.ipynb

{
 "nbformat": 4,
 "nbformat_minor": 0,
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "name": "Main_colab.ipynb",
   "provenance": [],
   "collapsed_sections": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "name": "python3"
  }
 },
 "cells": [
  {
   "cell_type": "code",
   "metadata": {
    "id": "G2N-uWl7pPJg",
    "executionInfo": {
     "status": "ok",
     "timestamp": 1605547205224,
     "user_tz": -60,
     "elapsed": 150276,
     "user": {
      "displayName": "Francesco Marchetti",
      "photoUrl": "",
      "userId": "18126159302682886539"
     }
    },
    "outputId": "8f76151d-fc3b-470f-8965-ab00073bed27",
    "colab": {
     "base_uri": "https://localhost:8080/"
    }
   },
   "source": [
    "# ------ Copyright (C) 2020 University of Strathclyde and Author ------\n",
    "# ---------------- Author: Francesco Marchetti ------------------------\n",
    "# ----------- e-mail: francesco.marchetti@strath.ac.uk ----------------\n"
   ],
   "execution_count": 1,
   "outputs": [
    {
     "output_type": "stream",
     "text": [
      "Mounted at /content/gdrive/\n",
      "Mounted at /content/gdrive/\n"
     ],
     "name": "stdout"
    }
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "ybIUbjTTrTvG",
    "executionInfo": {
     "status": "ok",
     "timestamp": 1605547257812,
     "user_tz": -60,
     "elapsed": 40280,
     "user": {
      "displayName": "Francesco Marchetti",
      "photoUrl": "",
      "userId": "18126159302682886539"
     }
    },
    "outputId": "095753d5-a912-4a22-e78c-5cee7ed5b329",
    "colab": {
     "base_uri": "https://localhost:8080/"
    }
   },
   "source": [
    "!pip install -q SharedArray\n",
    "!pip install -q pypianoroll==0.5.3\n",
    "#!pip install wandb\n",
    "\n",
    "import sys\n",
    "import os\n",
    "import sys\n",
    "\n",
    "\n",
    "import numpy as np\n",
    "\n",
    "%tensorflow_version 1.x\n",
    "import tensorflow as tf\n",
    "\n",
    "import SharedArray as sa\n",
    "from musegan.bmusegan.models import GAN, RefineGAN, End2EndGAN\n",
    "from config import EXP_CONFIG, DATA_CONFIG, MODEL_CONFIG, TRAIN_CONFIG\n",
    "from config import TF_CONFIG\n",
    "#import wandb\n",
    "#wandb.init()\n"
   ],
   "execution_count": 2,
   "outputs": [
    {
     "output_type": "stream",
     "text": [
      "  Building wheel for SharedArray (setup.py) ... \u001B[?25l\u001B[?25hdone\n",
      "\u001B[K     |████████████████████████████████| 5.6MB 24.2MB/s \n",
      "\u001B[K     |████████████████████████████████| 61kB 7.6MB/s \n",
      "\u001B[?25h  Building wheel for pypianoroll (setup.py) ... \u001B[?25l\u001B[?25hdone\n",
      "  Building wheel for pretty-midi (setup.py) ... \u001B[?25l\u001B[?25hdone\n",
      "TensorFlow 1.x selected.\n",
      "Found GPU at: /device:GPU:0\n",
      "Imageio: 'ffmpeg-linux64-v3.3.1' was not found on your computer; downloading it now.\n",
      "Try 1. Download from https://github.com/imageio/imageio-binaries/raw/master/ffmpeg/ffmpeg-linux64-v3.3.1 (43.8 MB)\n",
      "Downloading: 8192/45929032 bytes (0.0%)\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b3719168/45929032 bytes (8.1%)\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b7872512/45929032 bytes (17.1%)\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b12091392/45929032 bytes (26.3%)\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b16318464/45929032 bytes (35.5%)\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b20529152/45929032 bytes (44.7%)\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b24674304/45929032 bytes (53.7%)\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b28549120/45929032 bytes (62.2%)\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b32718848/45929032 bytes (71.2%)\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b36872192/45929032 bytes (80.3%)\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b41041920/45929032 bytes (89.4%)\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b45129728/45929032 bytes (98.3%)\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b45929032/45929032 bytes (100.0%)\n",
      "  Done\n",
      "File saved as /root/.imageio/ffmpeg/ffmpeg-linux64-v3.3.1.\n",
      "WARNING:tensorflow:From /content/gdrive/My Drive/GAN_test/bmusegan-master/config.py:111: The name tf.ConfigProto is deprecated. Please use tf.compat.v1.ConfigProto instead.\n",
      "\n"
     ],
     "name": "stdout"
    }
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "SWw2mdNhrc3R",
    "executionInfo": {
     "status": "ok",
     "timestamp": 1605549958613,
     "user_tz": -60,
     "elapsed": 2698790,
     "user": {
      "displayName": "Francesco Marchetti",
      "photoUrl": "",
      "userId": "18126159302682886539"
     }
    },
    "outputId": "6e29a551-66dd-4e09-8160-a0646b6752df",
    "colab": {
     "base_uri": "https://localhost:8080/"
    }
   },
   "source": [
    "def load_data():\n",
    "    \"\"\"Load and return the training data.\"\"\"\n",
    "    print('[*] Loading data...')\n",
    "    DATA_CONFIG['training_data'] = '/bmusegan/training_data/final-balanced-scottish-midi-data.npy'\n",
    "    if DATA_CONFIG['training_data_location'] == 'sa':\n",
    "        x_train = sa.attach(DATA_CONFIG['training_data'])\n",
    "    elif DATA_CONFIG['training_data_location'] == 'hd':\n",
    "        x_train = np.load(DATA_CONFIG['training_data'])\n",
    "    x_train = x_train.reshape(\n",
    "        -1, MODEL_CONFIG['num_bar'], MODEL_CONFIG['num_timestep'],\n",
    "        MODEL_CONFIG['num_pitch'], MODEL_CONFIG['num_track']\n",
    "    )\n",
    "    print('Training set size:', len(x_train))\n",
    "    return x_train\n",
    "\n",
    "def pretrain():\n",
    "    \"\"\"Create and pretrain a two-stage model\"\"\"\n",
    "    x_train = load_data()\n",
    "    with tf.Session(config=TF_CONFIG) as sess:\n",
    "        gan = GAN(sess, MODEL_CONFIG)\n",
    "        gan.init_all()\n",
    "        if EXP_CONFIG['pretrained_dir'] is not None:\n",
    "            gan.load_latest(EXP_CONFIG['pretrained_dir'])\n",
    "        gan.train(x_train, TRAIN_CONFIG)\n",
    "\n",
    "def train():\n",
    "    \"\"\"Load the pretrained model and run the second-stage training\"\"\"\n",
    "    x_train = load_data()\n",
    "    with tf.Session(config=TF_CONFIG) as sess:\n",
    "        gan = GAN(sess, MODEL_CONFIG)\n",
    "        gan.init_all()\n",
    "        gan.load_latest(EXP_CONFIG['first_stage_dir'])\n",
    "\n",
    "        refine_gan = RefineGAN(sess, MODEL_CONFIG, gan)\n",
    "        refine_gan.init_all()\n",
    "        if EXP_CONFIG['pretrained_dir'] is not None:\n",
    "            refine_gan.load_latest(EXP_CONFIG['pretrained_dir'])\n",
    "        refine_gan.train(x_train, TRAIN_CONFIG)\n",
    "\n",
    "def train_end2end():\n",
    "    \"\"\"Create and train an end-to-end model\"\"\"\n",
    "    x_train = load_data()\n",
    "    with tf.Session(config=TF_CONFIG) as sess:\n",
    "        end2end_gan = End2EndGAN(sess, MODEL_CONFIG)\n",
    "        end2end_gan.init_all()\n",
    "        if EXP_CONFIG['pretrained_dir'] is not None:\n",
    "            end2end_gan.load_latest(EXP_CONFIG['pretrained_dir'])\n",
    "        end2end_gan.train(x_train, TRAIN_CONFIG)\n",
    "\n",
    "if __name__ == '__main__':\n",
    "    print(\"Start experiment: {}\".format(EXP_CONFIG['exp_name']))\n",
    "    if not EXP_CONFIG['two_stage_training']:\n",
    "        train_end2end()\n",
    "    elif TRAIN_CONFIG['training_phase'] == 'pretrain':\n",
    "        pretrain()\n",
    "    elif TRAIN_CONFIG['training_phase'] == 'train':\n",
    "        train()"
   ],
   "execution_count": 3,
   "outputs": [
    {
     "output_type": "stream",
     "text": [
      "Start experiment: haggis_balancedDataset_10Epochs\n",
      "[*] Loading data...\n",
      "Training set size: 1047\n",
      "[*] Building GAN...\n",
      "WARNING:tensorflow:From /content/gdrive/My Drive/GAN_test/bmusegan-master/musegan/bmusegan/models.py:213: The name tf.variable_scope is deprecated. Please use tf.compat.v1.variable_scope instead.\n",
      "\n",
      "WARNING:tensorflow:From /content/gdrive/My Drive/GAN_test/bmusegan-master/musegan/bmusegan/models.py:213: The name tf.AUTO_REUSE is deprecated. Please use tf.compat.v1.AUTO_REUSE instead.\n",
      "\n",
      "WARNING:tensorflow:From /content/gdrive/My Drive/GAN_test/bmusegan-master/musegan/bmusegan/models.py:222: The name tf.placeholder is deprecated. Please use tf.compat.v1.placeholder instead.\n",
      "\n",
      "WARNING:tensorflow:From /content/gdrive/My Drive/GAN_test/bmusegan-master/musegan/utils/neuralnet.py:113: dense (from tensorflow.python.layers.core) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use keras.layers.Dense instead.\n",
      "WARNING:tensorflow:From /tensorflow-1.15.2/python3.6/tensorflow_core/python/layers/core.py:187: Layer.apply (from tensorflow.python.keras.engine.base_layer) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use `layer.__call__` method instead.\n",
      "WARNING:tensorflow:From /content/gdrive/My Drive/GAN_test/bmusegan-master/musegan/utils/neuralnet.py:156: batch_normalization (from tensorflow.python.layers.normalization) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use keras.layers.BatchNormalization instead.  In particular, `tf.control_dependencies(tf.GraphKeys.UPDATE_OPS)` should not be used (consult the `tf.keras.layers.batch_normalization` documentation).\n",
      "WARNING:tensorflow:From /content/gdrive/My Drive/GAN_test/bmusegan-master/musegan/utils/neuralnet.py:29: The name tf.get_collection is deprecated. Please use tf.compat.v1.get_collection instead.\n",
      "\n",
      "WARNING:tensorflow:From /content/gdrive/My Drive/GAN_test/bmusegan-master/musegan/utils/neuralnet.py:29: The name tf.GraphKeys is deprecated. Please use tf.compat.v1.GraphKeys instead.\n",
      "\n",
      "WARNING:tensorflow:From /content/gdrive/My Drive/GAN_test/bmusegan-master/musegan/utils/neuralnet.py:142: conv3d_transpose (from tensorflow.python.layers.convolutional) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use `tf.keras.layers.Conv3DTranspose` instead.\n",
      "WARNING:tensorflow:From /content/gdrive/My Drive/GAN_test/bmusegan-master/musegan/bmusegan/models.py:235: The name tf.random_uniform is deprecated. Please use tf.random.uniform instead.\n",
      "\n",
      "WARNING:tensorflow:From /content/gdrive/My Drive/GAN_test/bmusegan-master/musegan/utils/neuralnet.py:138: conv3d (from tensorflow.python.layers.convolutional) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use `tf.keras.layers.Conv3D` instead.\n",
      "WARNING:tensorflow:From /content/gdrive/My Drive/GAN_test/bmusegan-master/musegan/bmusegan/models.py:76: The name tf.train.AdamOptimizer is deprecated. Please use tf.compat.v1.train.AdamOptimizer instead.\n",
      "\n",
      "WARNING:tensorflow:From /content/gdrive/My Drive/GAN_test/bmusegan-master/musegan/bmusegan/models.py:267: The name tf.train.Saver is deprecated. Please use tf.compat.v1.train.Saver instead.\n",
      "\n",
      "=============================== Model Statistics ===============================\n",
      "WARNING:tensorflow:From /content/gdrive/My Drive/GAN_test/bmusegan-master/musegan/bmusegan/models.py:89: The name tf.trainable_variables is deprecated. Please use tf.compat.v1.trainable_variables instead.\n",
      "\n",
      "Number of parameters: 3735737\n",
      "Number of parameters in G: 1580440\n",
      "Number of parameters in D: 2155297\n",
      "================================ Model Summary =================================\n",
      "----------------------------------- < GAN > ------------------------------------\n",
      "------------------------------------ GAN/G -------------------------------------\n",
      "Input                                             (32, 128)\n",
      "--------------------------------------------------------------------------------\n",
      "[GAN/G/shared]\n",
      "Input                                             (32, 128)\n",
      "GAN/G/shared/Layer_0                 dense        (32, 1536)                    \n",
      "GAN/G/shared/Layer_1                 reshape      (32, 3, 1, 1, 512)            \n",
      "GAN/G/shared/Layer_2                 transconv3d  (32, 4, 1, 1, 256)            \n",
      "GAN/G/shared/Layer_3                 transconv3d  (32, 4, 4, 1, 128)            \n",
      "GAN/G/shared/Layer_4                 transconv3d  (32, 4, 4, 3, 128)            \n",
      "GAN/G/shared/Layer_5                 transconv3d  (32, 4, 16, 3, 64)            \n",
      "GAN/G/shared/Layer_6                 transconv3d  (32, 4, 16, 7, 64)            \n",
      "--------------------------------------------------------------------------------\n",
      "[GAN/G/pt_0]\n",
      "Input                                             (32, 4, 16, 7, 64)\n",
      "GAN/G/pt_0/Layer_0                   transconv3d  (32, 4, 16, 84, 64)           \n",
      "GAN/G/pt_0/Layer_1                   transconv3d  (32, 4, 96, 84, 32)           \n",
      "--------------------------------------------------------------------------------\n",
      "[GAN/G/tp_0]\n",
      "Input                                             (32, 4, 16, 7, 64)\n",
      "GAN/G/tp_0/Layer_0                   transconv3d  (32, 4, 96, 7, 64)            \n",
      "GAN/G/tp_0/Layer_1                   transconv3d  (32, 4, 96, 84, 32)           \n",
      "--------------------------------------------------------------------------------\n",
      "[GAN/G/merged_0]\n",
      "Input                                             (32, 4, 96, 84, 64)\n",
      "GAN/G/merged_0/Layer_0               transconv3d  (32, 4, 96, 84, 1)            \n",
      "--------------------------------------------------------------------------------\n",
      "------------------------------------ GAN/D -------------------------------------\n",
      "Input                                             (32, 4, 96, 84, 8)\n",
      "--------------------------------------------------------------------------------\n",
      "[GAN/D/pt_0]\n",
      "Input                                             (32, 4, 96, 84, 1)\n",
      "GAN/D/pt_0/Layer_0                   conv3d       (32, 4, 96, 7, 32)            \n",
      "GAN/D/pt_0/Layer_1                   conv3d       (32, 4, 16, 7, 64)            \n",
      "--------------------------------------------------------------------------------\n",
      "[GAN/D/tp_0]\n",
      "Input                                             (32, 4, 96, 84, 1)\n",
      "GAN/D/tp_0/Layer_0                   conv3d       (32, 4, 16, 84, 32)           \n",
      "GAN/D/tp_0/Layer_1                   conv3d       (32, 4, 16, 7, 64)            \n",
      "--------------------------------------------------------------------------------\n",
      "[GAN/D/merged_0]\n",
      "Input                                             (32, 4, 16, 7, 128)\n",
      "GAN/D/merged_0/Layer_0               conv3d       (32, 4, 16, 7, 64)            \n",
      "--------------------------------------------------------------------------------\n",
      "[GAN/D/shared]\n",
      "Input                                             (32, 4, 16, 7, 512)\n",
      "GAN/D/shared/Layer_0                 conv3d       (32, 4, 4, 3, 128)            \n",
      "GAN/D/shared/Layer_1                 conv3d       (32, 4, 1, 1, 256)            \n",
      "--------------------------------------------------------------------------------\n",
      "[GAN/D/chroma]\n",
      "Input                                             (32, 4, 4, 12, 8)\n",
      "GAN/D/chroma/Layer_0                 conv3d       (32, 4, 4, 1, 64)             \n",
      "GAN/D/chroma/Layer_1                 conv3d       (32, 4, 1, 1, 128)            \n",
      "--------------------------------------------------------------------------------\n",
      "[GAN/D/onset]\n",
      "Input                                             (32, 4, 96, 84, 8)\n",
      "GAN/D/onset/Layer_0                  sum          (32, 4, 96, 1, 8)             \n",
      "GAN/D/onset/Layer_1                  conv3d       (32, 4, 16, 1, 32)            \n",
      "GAN/D/onset/Layer_2                  conv3d       (32, 4, 4, 1, 64)             \n",
      "GAN/D/onset/Layer_3                  conv3d       (32, 4, 1, 1, 128)            \n",
      "--------------------------------------------------------------------------------\n",
      "[GAN/D/merged]\n",
      "Input                                             (32, 4, 1, 1, 512)\n",
      "GAN/D/merged/Layer_0                 conv3d       (32, 3, 1, 1, 512)            \n",
      "GAN/D/merged/Layer_1                 reshape      (32, 1536)                    \n",
      "GAN/D/merged/Layer_2                 dense        (32, 1)                       \n",
      "--------------------------------------------------------------------------------\n",
      "[*] Initializing variables...\n",
      "WARNING:tensorflow:From /content/gdrive/My Drive/GAN_test/bmusegan-master/musegan/bmusegan/models.py:34: The name tf.variables_initializer is deprecated. Please use tf.compat.v1.variables_initializer instead.\n",
      "\n",
      "WARNING:tensorflow:From /content/gdrive/My Drive/GAN_test/bmusegan-master/musegan/bmusegan/models.py:34: The name tf.global_variables is deprecated. Please use tf.compat.v1.global_variables instead.\n",
      "\n",
      "================================ Training Start ================================\n",
      "-------------------------------- Epoch 0 Start ---------------------------------\n",
      "WARNING:tensorflow:From /content/gdrive/My Drive/GAN_test/bmusegan-master/musegan/bmusegan/models.py:105: The name tf.train.global_step is deprecated. Please use tf.compat.v1.train.global_step instead.\n",
      "\n",
      "epoch |   batch   |  time  |    - D_loss    |     G_loss\n",
      "   0  |    0/  32 |  71.14 |    1741.007690 |    2202.505615\n",
      "   0  |    1/  32 |  59.47 |    1763.919678 |    2642.469482\n",
      "   0  |    2/  32 |  59.06 |    1744.817139 |    3487.379395\n",
      "   0  |    3/  32 |  59.39 |    1799.325928 |    3479.042969\n",
      "   0  |    4/  32 |  59.51 |    1741.270874 |    3153.053223\n",
      "   0  |    5/  32 |  59.52 |    1547.209473 |    3038.545898\n",
      "   0  |    6/  32 |  59.50 |     820.548828 |    1548.107056\n",
      "   0  |    7/  32 |  59.49 |     126.481461 |     214.059830\n",
      "   0  |    8/  32 |  59.52 |      56.398952 |      28.410345\n",
      "   0  |    9/  32 |  59.50 |      60.729984 |    -131.879730\n",
      "   0  |   10/  32 |  59.40 |      57.274185 |     -62.253090\n",
      "   0  |   11/  32 |  59.39 |      64.972229 |      26.281744\n",
      "   0  |   12/  32 |  59.44 |      74.566071 |      -8.866362\n",
      "   0  |   13/  32 |  59.40 |      72.450859 |       0.894538\n",
      "   0  |   14/  32 |  59.28 |      82.153671 |      -3.031345\n",
      "   0  |   15/  32 |  59.20 |      91.794159 |      -2.211947\n",
      "   0  |   16/  32 |  59.31 |      89.354362 |      -3.434157\n",
      "   0  |   17/  32 |  59.41 |      90.414772 |      -4.181215\n",
      "   0  |   18/  32 |  59.45 |      90.005478 |      -7.204871\n",
      "   0  |   19/  32 |  59.34 |      91.291260 |      -1.431459\n",
      "   0  |   20/  32 |  59.35 |      89.744286 |      -0.948381\n",
      "   0  |   21/  32 |  59.18 |      91.472610 |      -3.345336\n",
      "   0  |   22/  32 |  59.23 |      87.843407 |      -1.646707\n",
      "   0  |   23/  32 |  59.22 |      92.511230 |      -1.751754\n",
      "   0  |   24/  32 |  59.27 |      91.144897 |      -1.629350\n",
      "   0  |   25/  32 |   3.65 |      88.073807 |      -2.341540\n",
      "   0  |   26/  32 |   3.66 |      81.939392 |      -1.228130\n",
      "   0  |   27/  32 |   3.65 |      86.648865 |      -2.616988\n",
      "   0  |   28/  32 |   3.64 |      77.652115 |      -0.180356\n",
      "   0  |   29/  32 |   3.66 |      91.509430 |      -1.645667\n",
      "   0  |   30/  32 |   3.65 |      92.378830 |      -1.434709\n",
      "   0  |   31/  32 |   3.66 |      89.407326 |      -2.060602\n",
      "[*] Saving checkpoint...\n",
      "-------------------------------- Epoch 1 Start ---------------------------------\n",
      "epoch |   batch   |  time  |    - D_loss    |     G_loss\n",
      "   1  |    0/  32 |   3.69 |      89.767365 |      -1.747568\n",
      "   1  |    1/  32 |   3.65 |      91.219193 |      -2.059584\n",
      "   1  |    2/  32 |   3.65 |      85.193069 |      -1.837170\n",
      "   1  |    3/  32 |   3.65 |      92.747009 |      -1.446746\n",
      "   1  |    4/  32 |   3.66 |      89.237518 |      -2.162577\n",
      "   1  |    5/  32 |   3.65 |      84.121078 |      -1.313362\n",
      "   1  |    6/  32 |   3.67 |      93.732841 |      -1.683048\n",
      "   1  |    7/  32 |   3.65 |      95.357315 |      -1.880099\n",
      "   1  |    8/  32 |   3.67 |      91.899940 |      -2.055449\n",
      "   1  |    9/  32 |   3.68 |      87.728989 |      -1.601676\n",
      "   1  |   10/  32 |   3.67 |      78.459778 |      -2.830202\n",
      "   1  |   11/  32 |   3.66 |      92.398781 |      -1.161643\n",
      "   1  |   12/  32 |   3.67 |      91.770287 |      -1.067274\n",
      "   1  |   13/  32 |   3.66 |      93.156906 |      -1.333415\n",
      "   1  |   14/  32 |   3.66 |      90.760208 |      -1.277920\n",
      "   1  |   15/  32 |   3.67 |      89.854294 |      -1.052817\n",
      "   1  |   16/  32 |   3.67 |      79.930450 |      -0.813307\n",
      "   1  |   17/  32 |   3.67 |      87.898285 |      -0.079120\n",
      "   1  |   18/  32 |   3.65 |      86.907341 |      -0.198245\n",
      "   1  |   19/  32 |   3.66 |      86.098633 |      -0.948279\n",
      "   1  |   20/  32 |   3.65 |      86.317490 |      -0.454356\n",
      "   1  |   21/  32 |   3.66 |      90.131531 |      -1.820017\n",
      "   1  |   22/  32 |   3.66 |      85.364975 |      -2.208996\n",
      "   1  |   23/  32 |   3.66 |      88.781883 |      -1.890040\n",
      "   1  |   24/  32 |   3.65 |      94.887512 |      -2.470233\n",
      "   1  |   25/  32 |   3.65 |      93.415894 |      -1.533518\n",
      "   1  |   26/  32 |   3.65 |      79.934677 |      -2.729671\n",
      "   1  |   27/  32 |   3.65 |      86.753403 |      -1.970712\n",
      "   1  |   28/  32 |   3.66 |      83.630051 |      -1.667167\n",
      "   1  |   29/  32 |   3.66 |      88.612732 |      -0.684257\n",
      "   1  |   30/  32 |   3.66 |      88.783577 |      -2.924523\n",
      "   1  |   31/  32 |   3.66 |      91.275345 |      -0.883486\n",
      "[*] Saving checkpoint...\n",
      "-------------------------------- Epoch 2 Start ---------------------------------\n",
      "epoch |   batch   |  time  |    - D_loss    |     G_loss\n",
      "   2  |    0/  32 |   3.66 |      76.364319 |      -5.662345\n",
      "   2  |    1/  32 |   3.67 |      86.444855 |       2.546493\n",
      "   2  |    2/  32 |   3.67 |      78.809608 |      -1.457162\n",
      "   2  |    3/  32 |   3.66 |      86.960617 |      -1.955630\n",
      "   2  |    4/  32 |   3.66 |      89.329720 |      -2.707402\n",
      "   2  |    5/  32 |   3.65 |      86.887733 |      -4.708935\n",
      "   2  |    6/  32 |   3.67 |      88.101616 |      -5.318454\n",
      "   2  |    7/  32 |   3.66 |      89.373428 |      -4.278707\n",
      "   2  |    8/  32 |   3.67 |      74.864609 |      -7.272538\n",
      "   2  |    9/  32 |   3.68 |      92.362465 |       8.120932\n",
      "   2  |   10/  32 |   3.67 |      87.205429 |      -8.449512\n",
      "   2  |   11/  32 |   3.67 |      85.227936 |      -7.287391\n",
      "   2  |   12/  32 |   3.67 |      85.203255 |     -10.688547\n",
      "   2  |   13/  32 |   3.67 |      87.214043 |       2.514792\n",
      "   2  |   14/  32 |   3.66 |      85.040237 |      -7.074753\n",
      "   2  |   15/  32 |   3.67 |      79.035507 |      -7.158422\n",
      "   2  |   16/  32 |   3.65 |      89.936745 |      -3.632833\n",
      "   2  |   17/  32 |   3.66 |      86.062042 |      -6.010236\n",
      "   2  |   18/  32 |   3.65 |      80.199539 |      -4.068734\n",
      "   2  |   19/  32 |   3.65 |      83.569756 |       5.708080\n",
      "   2  |   20/  32 |   3.66 |      85.206360 |      -5.598304\n",
      "   2  |   21/  32 |   3.65 |      84.301498 |      -3.732240\n",
      "   2  |   22/  32 |   3.67 |      82.805481 |      -4.443225\n",
      "   2  |   23/  32 |   3.66 |      84.191071 |      -2.230517\n",
      "   2  |   24/  32 |   3.66 |      80.326141 |       1.250413\n",
      "   2  |   25/  32 |   3.66 |      78.636147 |      -1.088345\n",
      "   2  |   26/  32 |   3.66 |      78.439079 |      -3.110039\n",
      "   2  |   27/  32 |   3.67 |      70.026749 |      -6.333650\n",
      "   2  |   28/  32 |   3.67 |      86.821976 |      -0.745943\n",
      "   2  |   29/  32 |   3.67 |      80.632614 |      -4.372145\n",
      "   2  |   30/  32 |   3.67 |      81.876373 |      -1.691554\n",
      "   2  |   31/  32 |   3.67 |      78.299385 |       1.129828\n",
      "[*] Saving checkpoint...\n",
      "-------------------------------- Epoch 3 Start ---------------------------------\n",
      "epoch |   batch   |  time  |    - D_loss    |     G_loss\n",
      "   3  |    0/  32 |   3.66 |      88.127335 |      -0.876305\n",
      "   3  |    1/  32 |   3.65 |      81.767097 |      -2.410223\n",
      "   3  |    2/  32 |   3.65 |      88.860153 |      -2.944027\n",
      "   3  |    3/  32 |   3.65 |      87.068832 |       5.735107\n",
      "   3  |    4/  32 |   3.66 |      86.393105 |      -4.029305\n",
      "   3  |    5/  32 |   3.65 |      82.277252 |      -3.716414\n",
      "   3  |    6/  32 |   3.65 |      81.492661 |      -3.371916\n",
      "   3  |    7/  32 |   3.65 |      84.919777 |      -1.624420\n",
      "   3  |    8/  32 |   3.65 |      80.130966 |      -4.029711\n",
      "   3  |    9/  32 |   3.66 |      88.715515 |      -3.085809\n",
      "   3  |   10/  32 |   3.66 |      83.936684 |      -2.399436\n",
      "   3  |   11/  32 |   3.66 |      80.706116 |       7.190752\n",
      "   3  |   12/  32 |   3.66 |      83.973801 |      -1.928015\n",
      "   3  |   13/  32 |   3.66 |      82.622620 |      -2.868993\n",
      "   3  |   14/  32 |   3.65 |      87.414894 |      -3.851111\n",
      "   3  |   15/  32 |   3.66 |      83.249527 |      -3.280489\n",
      "   3  |   16/  32 |   3.67 |      84.914948 |       2.855549\n",
      "   3  |   17/  32 |   3.67 |      81.975296 |      -2.758830\n",
      "   3  |   18/  32 |   3.66 |      84.853645 |      -3.531683\n",
      "   3  |   19/  32 |   3.66 |      84.473145 |      -3.869451\n",
      "   3  |   20/  32 |   3.65 |      79.478806 |       4.694384\n",
      "   3  |   21/  32 |   3.66 |      85.851990 |      -2.536718\n",
      "   3  |   22/  32 |   3.65 |      80.810806 |      -3.478861\n",
      "   3  |   23/  32 |   3.65 |      86.558182 |      -1.193163\n",
      "   3  |   24/  32 |   3.66 |      86.129135 |      -1.239800\n",
      "   3  |   25/  32 |   3.66 |      85.954163 |      -1.537014\n",
      "   3  |   26/  32 |   3.67 |      89.233017 |      -0.628764\n",
      "   3  |   27/  32 |   3.66 |      84.015030 |       2.265518\n",
      "   3  |   28/  32 |   3.66 |      88.279724 |      -4.523076\n",
      "   3  |   29/  32 |   3.67 |      86.675949 |      -1.359999\n",
      "   3  |   30/  32 |   3.66 |      81.094513 |       2.078846\n",
      "   3  |   31/  32 |   3.65 |      84.910629 |      -0.804752\n",
      "[*] Saving checkpoint...\n",
      "-------------------------------- Epoch 4 Start ---------------------------------\n",
      "epoch |   batch   |  time  |    - D_loss    |     G_loss\n",
      "   4  |    0/  32 |   3.66 |      88.559563 |      -0.134077\n",
      "   4  |    1/  32 |   3.65 |      83.058517 |       2.691485\n",
      "   4  |    2/  32 |   3.65 |      79.343170 |      -2.065204\n",
      "   4  |    3/  32 |   3.65 |      84.929939 |      -2.698899\n",
      "   4  |    4/  32 |   3.65 |      83.685333 |      -0.243781\n",
      "   4  |    5/  32 |   3.66 |      79.724350 |      -5.966107\n",
      "   4  |    6/  32 |   3.67 |      84.780243 |       4.635174\n",
      "   4  |    7/  32 |   3.67 |      83.077576 |      -5.789540\n",
      "   4  |    8/  32 |   3.67 |      81.944260 |       8.350121\n",
      "   4  |    9/  32 |   3.66 |      78.796722 |      -3.567818\n",
      "   4  |   10/  32 |   3.66 |      81.128403 |      -5.661411\n",
      "   4  |   11/  32 |   3.65 |      84.311203 |       4.845872\n",
      "   4  |   12/  32 |   3.67 |      81.731758 |      19.675985\n",
      "   4  |   13/  32 |   3.65 |      77.962837 |      -7.665399\n",
      "   4  |   14/  32 |   3.65 |      81.917358 |      -7.213752\n",
      "   4  |   15/  32 |   3.66 |     100.752640 |      22.161087\n",
      "   4  |   16/  32 |   3.66 |      81.710251 |      -4.368706\n",
      "   4  |   17/  32 |   3.67 |      77.057175 |     -10.350624\n",
      "   4  |   18/  32 |   3.68 |      87.833069 |      -0.328649\n",
      "   4  |   19/  32 |   3.67 |      79.791992 |       0.319821\n",
      "   4  |   20/  32 |   3.66 |      79.550568 |      -6.761851\n",
      "   4  |   21/  32 |   3.65 |      80.571205 |      -7.861778\n",
      "   4  |   22/  32 |   3.67 |      83.682419 |      -2.559418\n",
      "   4  |   23/  32 |   3.65 |      77.204506 |      -2.443186\n",
      "   4  |   24/  32 |   3.67 |      75.678009 |       7.071592\n",
      "   4  |   25/  32 |   3.67 |      75.023613 |     -13.594807\n",
      "   4  |   26/  32 |   3.66 |      85.884857 |       2.693257\n",
      "   4  |   27/  32 |   3.67 |      82.434052 |     -10.013304\n",
      "   4  |   28/  32 |   3.66 |      84.770653 |       0.564966\n",
      "   4  |   29/  32 |   3.67 |      79.382767 |       5.068781\n",
      "   4  |   30/  32 |   3.66 |      84.092499 |      -1.340194\n",
      "   4  |   31/  32 |   3.66 |      88.786789 |      -5.149456\n",
      "[*] Saving checkpoint...\n",
      "-------------------------------- Epoch 5 Start ---------------------------------\n",
      "epoch |   batch   |  time  |    - D_loss    |     G_loss\n",
      "   5  |    0/  32 |   3.67 |      85.234573 |      -0.629067\n",
      "   5  |    1/  32 |   3.65 |      81.296646 |      -2.395100\n",
      "   5  |    2/  32 |   3.67 |      83.942833 |       9.021949\n",
      "   5  |    3/  32 |   3.65 |      75.895660 |       5.294479\n",
      "   5  |    4/  32 |   3.67 |      70.775368 |     -13.700379\n",
      "   5  |    5/  32 |   3.66 |      82.547951 |       6.723219\n",
      "   5  |    6/  32 |   3.65 |      83.729706 |       3.673934\n",
      "   5  |    7/  32 |   3.66 |      81.145599 |       4.066042\n",
      "   5  |    8/  32 |   3.66 |      82.686920 |       3.560672\n",
      "   5  |    9/  32 |   3.65 |      91.288773 |      -4.600375\n",
      "   5  |   10/  32 |   3.67 |      79.541824 |      -2.324508\n",
      "   5  |   11/  32 |   3.66 |      78.651390 |      -2.626952\n",
      "   5  |   12/  32 |   3.65 |      79.530563 |      -3.732165\n",
      "   5  |   13/  32 |   3.65 |      79.593094 |      -3.666543\n",
      "   5  |   14/  32 |   3.65 |      86.363914 |       1.282927\n",
      "   5  |   15/  32 |   3.65 |      78.698959 |       1.238382\n",
      "   5  |   16/  32 |   3.66 |      81.358238 |       1.333580\n",
      "   5  |   17/  32 |   3.65 |      81.756775 |      -4.866244\n",
      "   5  |   18/  32 |   3.65 |      83.377663 |       7.148738\n",
      "   5  |   19/  32 |   3.66 |      88.453476 |       2.480733\n",
      "   5  |   20/  32 |   3.65 |      85.112915 |      -2.493279\n",
      "   5  |   21/  32 |   3.65 |      88.183708 |       2.249455\n",
      "   5  |   22/  32 |   3.64 |      79.771645 |       4.527341\n",
      "   5  |   23/  32 |   3.65 |      85.188103 |       0.380072\n",
      "   5  |   24/  32 |   3.66 |      87.241821 |      -1.243259\n",
      "   5  |   25/  32 |   3.65 |      92.165710 |      -3.578205\n",
      "   5  |   26/  32 |   3.65 |      84.993401 |      -1.275202\n",
      "   5  |   27/  32 |   3.65 |      82.168953 |       4.018966\n",
      "   5  |   28/  32 |   3.65 |      80.835175 |      -0.436427\n",
      "   5  |   29/  32 |   3.65 |      84.105011 |       0.617393\n",
      "   5  |   30/  32 |   3.65 |      83.255966 |      -2.047556\n",
      "   5  |   31/  32 |   3.67 |      85.639252 |      -2.641564\n",
      "[*] Saving checkpoint...\n",
      "WARNING:tensorflow:From /tensorflow-1.15.2/python3.6/tensorflow_core/python/training/saver.py:963: remove_checkpoint (from tensorflow.python.training.checkpoint_management) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use standard file APIs to delete files with this prefix.\n",
      "-------------------------------- Epoch 6 Start ---------------------------------\n",
      "epoch |   batch   |  time  |    - D_loss    |     G_loss\n",
      "   6  |    0/  32 |   3.65 |      87.613373 |       0.340512\n",
      "   6  |    1/  32 |   3.65 |      75.896317 |      12.501965\n",
      "   6  |    2/  32 |   3.64 |      80.438187 |       4.933489\n",
      "   6  |    3/  32 |   3.64 |      75.867813 |      -0.351960\n",
      "   6  |    4/  32 |   3.65 |      81.948227 |      -0.037326\n",
      "   6  |    5/  32 |   3.66 |      77.785309 |      -0.673600\n",
      "   6  |    6/  32 |   3.65 |      84.142647 |      -2.034627\n",
      "   6  |    7/  32 |   3.65 |      82.030212 |      -5.555498\n",
      "   6  |    8/  32 |   3.65 |      88.999702 |      -4.568639\n",
      "   6  |    9/  32 |   3.66 |      81.212372 |      -6.688892\n",
      "   6  |   10/  32 |   3.66 |      87.674232 |      30.191967\n",
      "   6  |   11/  32 |   3.67 |      79.425842 |      -5.673069\n",
      "   6  |   12/  32 |   3.67 |      74.761612 |      -8.116127\n",
      "   6  |   13/  32 |   3.67 |      88.533012 |      -2.692422\n",
      "   6  |   14/  32 |   3.66 |      76.752319 |      26.105064\n",
      "   6  |   15/  32 |   3.67 |      81.416504 |      -6.767049\n",
      "   6  |   16/  32 |   3.67 |      79.810425 |      -4.391788\n",
      "   6  |   17/  32 |   3.65 |      83.268845 |      -6.323143\n",
      "   6  |   18/  32 |   3.65 |      82.393219 |      -7.701467\n",
      "   6  |   19/  32 |   3.67 |      81.046272 |       2.257077\n",
      "   6  |   20/  32 |   3.66 |      77.625778 |      -4.655918\n",
      "   6  |   21/  32 |   3.66 |      78.344994 |       1.771993\n",
      "   6  |   22/  32 |   3.65 |      85.772827 |      -0.101814\n",
      "   6  |   23/  32 |   3.66 |      79.652962 |       7.870957\n",
      "   6  |   24/  32 |   3.65 |      84.265701 |      -2.989175\n",
      "   6  |   25/  32 |   3.67 |      85.460831 |      -2.579028\n",
      "   6  |   26/  32 |   3.66 |      76.996399 |      -2.250805\n",
      "   6  |   27/  32 |   3.65 |      86.225517 |      -1.658131\n",
      "   6  |   28/  32 |   3.66 |      75.290169 |      -6.274929\n",
      "   6  |   29/  32 |   3.66 |      75.460754 |      36.503944\n",
      "   6  |   30/  32 |   3.66 |      81.860016 |      -5.362385\n",
      "   6  |   31/  32 |   3.65 |      80.135040 |      -1.001852\n",
      "[*] Saving checkpoint...\n",
      "-------------------------------- Epoch 7 Start ---------------------------------\n",
      "epoch |   batch   |  time  |    - D_loss    |     G_loss\n",
      "   7  |    0/  32 |   3.64 |      86.953079 |      -1.579947\n",
      "   7  |    1/  32 |   3.65 |      76.619446 |       5.580633\n",
      "   7  |    2/  32 |   3.65 |      81.817154 |      -2.933463\n",
      "   7  |    3/  32 |   3.65 |      79.487450 |      -5.898402\n",
      "   7  |    4/  32 |   3.67 |      85.021957 |      -1.561860\n",
      "   7  |    5/  32 |   3.65 |      85.664993 |      -4.438582\n",
      "   7  |    6/  32 |   3.65 |      79.280037 |      -4.779115\n",
      "   7  |    7/  32 |   3.65 |      79.632538 |      -0.545451\n",
      "   7  |    8/  32 |   3.64 |      79.271332 |      -4.597102\n",
      "   7  |    9/  32 |   3.65 |      84.946663 |      -5.751140\n",
      "   7  |   10/  32 |   3.66 |      83.501312 |      -6.670973\n",
      "   7  |   11/  32 |   3.66 |      80.296463 |       2.234775\n",
      "   7  |   12/  32 |   3.65 |      82.618279 |       0.847277\n",
      "   7  |   13/  32 |   3.65 |      80.086700 |      -4.615443\n",
      "   7  |   14/  32 |   3.65 |      88.078865 |      -4.544448\n",
      "   7  |   15/  32 |   3.66 |      80.899536 |      -4.009295\n",
      "   7  |   16/  32 |   3.66 |      79.670807 |      -6.297738\n",
      "   7  |   17/  32 |   3.65 |      73.933609 |      -1.309173\n",
      "   7  |   18/  32 |   3.66 |      79.172478 |      -3.763010\n",
      "   7  |   19/  32 |   3.66 |      81.682083 |      -5.855809\n",
      "   7  |   20/  32 |   3.66 |      88.493576 |      27.913742\n",
      "   7  |   21/  32 |   3.66 |      81.829559 |       0.219344\n",
      "   7  |   22/  32 |   3.67 |      85.744026 |      -5.233106\n",
      "   7  |   23/  32 |   3.66 |      69.919273 |      -7.032838\n",
      "   7  |   24/  32 |   3.67 |      80.455246 |      -0.449552\n",
      "   7  |   25/  32 |   3.67 |      81.695236 |      -5.264624\n",
      "   7  |   26/  32 |   3.67 |      78.700500 |      -3.538061\n",
      "   7  |   27/  32 |   3.67 |      82.030701 |      -3.439373\n",
      "   7  |   28/  32 |   3.66 |      85.975662 |      -2.372652\n",
      "   7  |   29/  32 |   3.65 |      83.506058 |      -2.703575\n",
      "   7  |   30/  32 |   3.66 |      88.073418 |      -1.848795\n",
      "   7  |   31/  32 |   3.67 |      86.971962 |      -1.467483\n",
      "[*] Saving checkpoint...\n",
      "-------------------------------- Epoch 8 Start ---------------------------------\n",
      "epoch |   batch   |  time  |    - D_loss    |     G_loss\n",
      "   8  |    0/  32 |   3.66 |      81.594009 |       4.131750\n",
      "   8  |    1/  32 |   3.67 |      82.234955 |      -0.933492\n",
      "   8  |    2/  32 |   3.66 |      86.801163 |      -3.216640\n",
      "   8  |    3/  32 |   3.66 |      80.024696 |       7.209454\n",
      "   8  |    4/  32 |   3.67 |      85.733109 |      -2.804069\n",
      "   8  |    5/  32 |   3.66 |      90.646217 |      -7.473364\n",
      "   8  |    6/  32 |   3.67 |      80.912003 |      -7.852852\n",
      "   8  |    7/  32 |   3.66 |      84.100273 |      -8.471001\n",
      "   8  |    8/  32 |   3.67 |      84.105347 |      -6.324570\n",
      "   8  |    9/  32 |   3.67 |      78.653549 |      11.715100\n",
      "   8  |   10/  32 |   3.67 |      85.774529 |       1.352578\n",
      "   8  |   11/  32 |   3.67 |      81.813744 |       4.851215\n",
      "   8  |   12/  32 |   3.67 |      77.566353 |       8.776732\n",
      "   8  |   13/  32 |   3.67 |      87.203430 |      -6.752254\n",
      "   8  |   14/  32 |   3.67 |      85.098404 |      -5.285067\n",
      "   8  |   15/  32 |   3.66 |      85.711273 |      -2.929366\n",
      "   8  |   16/  32 |   3.65 |      80.769272 |      -3.932233\n",
      "   8  |   17/  32 |   3.66 |      88.538605 |      -4.308562\n",
      "   8  |   18/  32 |   3.67 |      83.727753 |      -4.447146\n",
      "   8  |   19/  32 |   3.65 |      85.262642 |      -0.299977\n",
      "   8  |   20/  32 |   3.65 |      87.052383 |      -4.632779\n",
      "   8  |   21/  32 |   3.67 |      85.061264 |      -3.288054\n",
      "   8  |   22/  32 |   3.66 |      79.705009 |      -0.687101\n",
      "   8  |   23/  32 |   3.65 |      80.436806 |      -1.799353\n",
      "   8  |   24/  32 |   3.65 |      83.448174 |       8.683298\n",
      "   8  |   25/  32 |   3.66 |      77.326599 |       4.090655\n",
      "   8  |   26/  32 |   3.67 |      82.387764 |       1.096370\n",
      "   8  |   27/  32 |   3.66 |      84.939301 |       8.501831\n",
      "   8  |   28/  32 |   3.65 |      86.715675 |      -3.061217\n",
      "   8  |   29/  32 |   3.67 |      83.366348 |       3.951255\n",
      "   8  |   30/  32 |   3.65 |      81.585983 |      -2.814652\n",
      "   8  |   31/  32 |   3.65 |      85.910645 |      17.235779\n",
      "[*] Saving checkpoint...\n",
      "-------------------------------- Epoch 9 Start ---------------------------------\n",
      "epoch |   batch   |  time  |    - D_loss    |     G_loss\n",
      "   9  |    0/  32 |   3.65 |      83.774078 |      -9.029389\n",
      "   9  |    1/  32 |   3.65 |      86.111481 |      -1.466620\n",
      "   9  |    2/  32 |   3.65 |      78.346413 |      -0.052599\n",
      "   9  |    3/  32 |   3.66 |      86.439430 |       3.100657\n",
      "   9  |    4/  32 |   3.65 |      83.672394 |       6.306314\n",
      "   9  |    5/  32 |   3.65 |      84.977089 |       2.033346\n",
      "   9  |    6/  32 |   3.65 |      86.428848 |      -2.233200\n",
      "   9  |    7/  32 |   3.67 |      78.244370 |      -4.065166\n",
      "   9  |    8/  32 |   3.65 |      84.506714 |      -3.515913\n",
      "   9  |    9/  32 |   3.65 |      88.414337 |      -0.838052\n",
      "   9  |   10/  32 |   3.65 |      82.465645 |      -0.119444\n",
      "   9  |   11/  32 |   3.65 |      75.619682 |       4.804214\n",
      "   9  |   12/  32 |   3.65 |      87.629860 |      -5.017964\n",
      "   9  |   13/  32 |   3.65 |      84.008499 |       0.756030\n",
      "   9  |   14/  32 |   3.64 |      81.308548 |       2.008878\n",
      "   9  |   15/  32 |   3.66 |      81.666237 |      -1.547859\n",
      "   9  |   16/  32 |   3.65 |      81.539551 |      -8.909411\n",
      "   9  |   17/  32 |   3.66 |      81.637695 |      -1.175961\n",
      "   9  |   18/  32 |   3.65 |      81.486465 |       5.335352\n",
      "   9  |   19/  32 |   3.66 |      81.331284 |      -0.784929\n",
      "   9  |   20/  32 |   3.66 |      83.032837 |      -4.465617\n",
      "   9  |   21/  32 |   3.66 |      83.336716 |      -1.563384\n",
      "   9  |   22/  32 |   3.66 |      79.175644 |       1.473362\n",
      "   9  |   23/  32 |   3.65 |      83.807526 |       0.078810\n",
      "   9  |   24/  32 |   3.65 |      79.237091 |      -1.242120\n",
      "   9  |   25/  32 |   3.66 |      78.783218 |      13.208389\n",
      "   9  |   26/  32 |   3.67 |      78.983734 |       2.382369\n",
      "   9  |   27/  32 |   3.65 |      83.477867 |      -5.879448\n",
      "   9  |   28/  32 |   3.65 |      86.216110 |       5.998336\n",
      "   9  |   29/  32 |   3.65 |      86.367821 |      -6.063594\n",
      "   9  |   30/  32 |   3.65 |      82.215286 |      -0.757096\n",
      "   9  |   31/  32 |   3.65 |      85.319489 |      -6.057401\n",
      "[*] Saving checkpoint...\n",
      "================================= Training End =================================\n"
     ],
     "name": "stdout"
    }
   ]
  }
 ]
}