playground.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "ny_path = \"/home/fuest/EnData/data/pecanstreet/15minute_data_newyork.csv\"\n",
    "austin_path = \"/home/fuest/EnData/data/pecanstreet/15minute_data_austin.csv\"\n",
    "cali_path = \"/home/fuest/EnData/data/pecanstreet/15minute_data_california.csv\"\n",
    "metadata_path = \"/home/fuest/EnData/data/pecanstreet/metadata.csv\"\n",
    "\n",
    "ny_data = pd.read_csv(ny_path)\n",
    "austin_data = pd.read_csv(austin_path)\n",
    "cali_data = pd.read_csv(cali_path)\n",
    "metadata = pd.read_csv(metadata_path)\n",
    "\n",
    "ny_user_ids = ny_data.dataid.unique().tolist()\n",
    "austin_user_ids = austin_data.dataid.unique().tolist()\n",
    "cali_user_ids = cali_data.dataid.unique().tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "ename": "OSError",
     "evalue": "Cannot save file into a non-existent directory: 'tests/testdata'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mOSError\u001b[0m                                   Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[5], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m ny_data[ny_data\u001b[39m.\u001b[39;49mdataid \u001b[39m==\u001b[39;49m \u001b[39m27\u001b[39;49m]\u001b[39m.\u001b[39;49mto_csv(\u001b[39m\"\u001b[39;49m\u001b[39mtests/testdata/15minute_data_newyork.csv\u001b[39;49m\u001b[39m\"\u001b[39;49m)\n",
      "File \u001b[0;32m~/EnData/venv/lib/python3.8/site-packages/pandas/util/_decorators.py:211\u001b[0m, in \u001b[0;36mdeprecate_kwarg.<locals>._deprecate_kwarg.<locals>.wrapper\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m    209\u001b[0m     \u001b[39melse\u001b[39;00m:\n\u001b[1;32m    210\u001b[0m         kwargs[new_arg_name] \u001b[39m=\u001b[39m new_arg_value\n\u001b[0;32m--> 211\u001b[0m \u001b[39mreturn\u001b[39;00m func(\u001b[39m*\u001b[39;49margs, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mkwargs)\n",
      "File \u001b[0;32m~/EnData/venv/lib/python3.8/site-packages/pandas/core/generic.py:3720\u001b[0m, in \u001b[0;36mNDFrame.to_csv\u001b[0;34m(self, path_or_buf, sep, na_rep, float_format, columns, header, index, index_label, mode, encoding, compression, quoting, quotechar, lineterminator, chunksize, date_format, doublequote, escapechar, decimal, errors, storage_options)\u001b[0m\n\u001b[1;32m   3709\u001b[0m df \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m \u001b[39mif\u001b[39;00m \u001b[39misinstance\u001b[39m(\u001b[39mself\u001b[39m, ABCDataFrame) \u001b[39melse\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mto_frame()\n\u001b[1;32m   3711\u001b[0m formatter \u001b[39m=\u001b[39m DataFrameFormatter(\n\u001b[1;32m   3712\u001b[0m     frame\u001b[39m=\u001b[39mdf,\n\u001b[1;32m   3713\u001b[0m     header\u001b[39m=\u001b[39mheader,\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m   3717\u001b[0m     decimal\u001b[39m=\u001b[39mdecimal,\n\u001b[1;32m   3718\u001b[0m )\n\u001b[0;32m-> 3720\u001b[0m \u001b[39mreturn\u001b[39;00m DataFrameRenderer(formatter)\u001b[39m.\u001b[39;49mto_csv(\n\u001b[1;32m   3721\u001b[0m     path_or_buf,\n\u001b[1;32m   3722\u001b[0m     lineterminator\u001b[39m=\u001b[39;49mlineterminator,\n\u001b[1;32m   3723\u001b[0m     sep\u001b[39m=\u001b[39;49msep,\n\u001b[1;32m   3724\u001b[0m     encoding\u001b[39m=\u001b[39;49mencoding,\n\u001b[1;32m   3725\u001b[0m     errors\u001b[39m=\u001b[39;49merrors,\n\u001b[1;32m   3726\u001b[0m     compression\u001b[39m=\u001b[39;49mcompression,\n\u001b[1;32m   3727\u001b[0m     quoting\u001b[39m=\u001b[39;49mquoting,\n\u001b[1;32m   3728\u001b[0m     columns\u001b[39m=\u001b[39;49mcolumns,\n\u001b[1;32m   3729\u001b[0m     index_label\u001b[39m=\u001b[39;49mindex_label,\n\u001b[1;32m   3730\u001b[0m     mode\u001b[39m=\u001b[39;49mmode,\n\u001b[1;32m   3731\u001b[0m     chunksize\u001b[39m=\u001b[39;49mchunksize,\n\u001b[1;32m   3732\u001b[0m     quotechar\u001b[39m=\u001b[39;49mquotechar,\n\u001b[1;32m   3733\u001b[0m     date_format\u001b[39m=\u001b[39;49mdate_format,\n\u001b[1;32m   3734\u001b[0m     doublequote\u001b[39m=\u001b[39;49mdoublequote,\n\u001b[1;32m   3735\u001b[0m     escapechar\u001b[39m=\u001b[39;49mescapechar,\n\u001b[1;32m   3736\u001b[0m     storage_options\u001b[39m=\u001b[39;49mstorage_options,\n\u001b[1;32m   3737\u001b[0m )\n",
      "File \u001b[0;32m~/EnData/venv/lib/python3.8/site-packages/pandas/util/_decorators.py:211\u001b[0m, in \u001b[0;36mdeprecate_kwarg.<locals>._deprecate_kwarg.<locals>.wrapper\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m    209\u001b[0m     \u001b[39melse\u001b[39;00m:\n\u001b[1;32m    210\u001b[0m         kwargs[new_arg_name] \u001b[39m=\u001b[39m new_arg_value\n\u001b[0;32m--> 211\u001b[0m \u001b[39mreturn\u001b[39;00m func(\u001b[39m*\u001b[39;49margs, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mkwargs)\n",
      "File \u001b[0;32m~/EnData/venv/lib/python3.8/site-packages/pandas/io/formats/format.py:1189\u001b[0m, in \u001b[0;36mDataFrameRenderer.to_csv\u001b[0;34m(self, path_or_buf, encoding, sep, columns, index_label, mode, compression, quoting, quotechar, lineterminator, chunksize, date_format, doublequote, escapechar, errors, storage_options)\u001b[0m\n\u001b[1;32m   1168\u001b[0m     created_buffer \u001b[39m=\u001b[39m \u001b[39mFalse\u001b[39;00m\n\u001b[1;32m   1170\u001b[0m csv_formatter \u001b[39m=\u001b[39m CSVFormatter(\n\u001b[1;32m   1171\u001b[0m     path_or_buf\u001b[39m=\u001b[39mpath_or_buf,\n\u001b[1;32m   1172\u001b[0m     lineterminator\u001b[39m=\u001b[39mlineterminator,\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m   1187\u001b[0m     formatter\u001b[39m=\u001b[39m\u001b[39mself\u001b[39m\u001b[39m.\u001b[39mfmt,\n\u001b[1;32m   1188\u001b[0m )\n\u001b[0;32m-> 1189\u001b[0m csv_formatter\u001b[39m.\u001b[39;49msave()\n\u001b[1;32m   1191\u001b[0m \u001b[39mif\u001b[39;00m created_buffer:\n\u001b[1;32m   1192\u001b[0m     \u001b[39massert\u001b[39;00m \u001b[39misinstance\u001b[39m(path_or_buf, StringIO)\n",
      "File \u001b[0;32m~/EnData/venv/lib/python3.8/site-packages/pandas/io/formats/csvs.py:241\u001b[0m, in \u001b[0;36mCSVFormatter.save\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    237\u001b[0m \u001b[39m\u001b[39m\u001b[39m\"\"\"\u001b[39;00m\n\u001b[1;32m    238\u001b[0m \u001b[39mCreate the writer & save.\u001b[39;00m\n\u001b[1;32m    239\u001b[0m \u001b[39m\"\"\"\u001b[39;00m\n\u001b[1;32m    240\u001b[0m \u001b[39m# apply compression and byte/text conversion\u001b[39;00m\n\u001b[0;32m--> 241\u001b[0m \u001b[39mwith\u001b[39;00m get_handle(\n\u001b[1;32m    242\u001b[0m     \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mfilepath_or_buffer,\n\u001b[1;32m    243\u001b[0m     \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mmode,\n\u001b[1;32m    244\u001b[0m     encoding\u001b[39m=\u001b[39;49m\u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mencoding,\n\u001b[1;32m    245\u001b[0m     errors\u001b[39m=\u001b[39;49m\u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49merrors,\n\u001b[1;32m    246\u001b[0m     compression\u001b[39m=\u001b[39;49m\u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mcompression,\n\u001b[1;32m    247\u001b[0m     storage_options\u001b[39m=\u001b[39;49m\u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mstorage_options,\n\u001b[1;32m    248\u001b[0m ) \u001b[39mas\u001b[39;00m handles:\n\u001b[1;32m    249\u001b[0m \n\u001b[1;32m    250\u001b[0m     \u001b[39m# Note: self.encoding is irrelevant here\u001b[39;00m\n\u001b[1;32m    251\u001b[0m     \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mwriter \u001b[39m=\u001b[39m csvlib\u001b[39m.\u001b[39mwriter(\n\u001b[1;32m    252\u001b[0m         handles\u001b[39m.\u001b[39mhandle,\n\u001b[1;32m    253\u001b[0m         lineterminator\u001b[39m=\u001b[39m\u001b[39mself\u001b[39m\u001b[39m.\u001b[39mlineterminator,\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m    258\u001b[0m         quotechar\u001b[39m=\u001b[39m\u001b[39mself\u001b[39m\u001b[39m.\u001b[39mquotechar,\n\u001b[1;32m    259\u001b[0m     )\n\u001b[1;32m    261\u001b[0m     \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_save()\n",
      "File \u001b[0;32m~/EnData/venv/lib/python3.8/site-packages/pandas/io/common.py:734\u001b[0m, in \u001b[0;36mget_handle\u001b[0;34m(path_or_buf, mode, encoding, compression, memory_map, is_text, errors, storage_options)\u001b[0m\n\u001b[1;32m    732\u001b[0m \u001b[39m# Only for write methods\u001b[39;00m\n\u001b[1;32m    733\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39m\"\u001b[39m\u001b[39mr\u001b[39m\u001b[39m\"\u001b[39m \u001b[39mnot\u001b[39;00m \u001b[39min\u001b[39;00m mode \u001b[39mand\u001b[39;00m is_path:\n\u001b[0;32m--> 734\u001b[0m     check_parent_directory(\u001b[39mstr\u001b[39;49m(handle))\n\u001b[1;32m    736\u001b[0m \u001b[39mif\u001b[39;00m compression:\n\u001b[1;32m    737\u001b[0m     \u001b[39mif\u001b[39;00m compression \u001b[39m!=\u001b[39m \u001b[39m\"\u001b[39m\u001b[39mzstd\u001b[39m\u001b[39m\"\u001b[39m:\n\u001b[1;32m    738\u001b[0m         \u001b[39m# compression libraries do not like an explicit text-mode\u001b[39;00m\n",
      "File \u001b[0;32m~/EnData/venv/lib/python3.8/site-packages/pandas/io/common.py:597\u001b[0m, in \u001b[0;36mcheck_parent_directory\u001b[0;34m(path)\u001b[0m\n\u001b[1;32m    595\u001b[0m parent \u001b[39m=\u001b[39m Path(path)\u001b[39m.\u001b[39mparent\n\u001b[1;32m    596\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mnot\u001b[39;00m parent\u001b[39m.\u001b[39mis_dir():\n\u001b[0;32m--> 597\u001b[0m     \u001b[39mraise\u001b[39;00m \u001b[39mOSError\u001b[39;00m(\u001b[39mrf\u001b[39m\u001b[39m\"\u001b[39m\u001b[39mCannot save file into a non-existent directory: \u001b[39m\u001b[39m'\u001b[39m\u001b[39m{\u001b[39;00mparent\u001b[39m}\u001b[39;00m\u001b[39m'\u001b[39m\u001b[39m\"\u001b[39m)\n",
      "\u001b[0;31mOSError\u001b[0m: Cannot save file into a non-existent directory: 'tests/testdata'"
     ]
    }
   ],
   "source": [
    "ny_data[ny_data.dataid == 27].to_csv(\"tests/testdata/15minute_data_newyork.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_user_ids = cali_user_ids + austin_user_ids + ny_user_ids"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "yes    15\n",
       "Name: car1, dtype: int64"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "metadata = metadata.loc[metadata[\"dataid\"].isin(all_user_ids)].copy()\n",
    "metadata[\"car1\"].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "USER_IDs = [27,  142,  203,  387,  558,  661,  914,  950, 1222, 1240, 1417,\n",
    "       1450, 1524, 1642, 1731, 2096, 2318, 2335, 2358, 2361, 2606, 2818,\n",
    "       3000, 3039, 3456, 3488, 3517, 3538, 3687, 3700, 3864, 3938, 3996,\n",
    "       4031, 4283, 4373, 4495, 4550, 4767, 4934, 5058, 5587, 5679, 5746,\n",
    "       5938, 5982, 5997, 6139, 6377, 6547, 7062, 7114, 7536, 7719, 7800,\n",
    "       7901, 7951, 8061, 8156, 8342, 8386, 8565, 8574, 8733, 9019, 9053,\n",
    "       9160, 9213, 9278, 9612, 9775, 9836, 9922]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from datasets.pecanstreet import PecanStreetDataManager\n",
    "\n",
    "pv = PecanStreetDataManager(geography=\"austin\", include_generation=True).create_all_pv_user_dataset()\n",
    "non_pv = PecanStreetDataManager(geography=\"austin\",include_generation=True).create_non_pv_user_dataset()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 1:  62%|██████▏   | 133/213 [00:06<00:03, 20.05it/s]\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[14], line 25\u001b[0m\n\u001b[1;32m      9\u001b[0m conditioning_dict \u001b[39m=\u001b[39m {\n\u001b[1;32m     10\u001b[0m     \u001b[39m\"\u001b[39m\u001b[39mmonth\u001b[39m\u001b[39m\"\u001b[39m: \u001b[39m12\u001b[39m,\n\u001b[1;32m     11\u001b[0m     \u001b[39m\"\u001b[39m\u001b[39mweekday\u001b[39m\u001b[39m\"\u001b[39m: \u001b[39m7\u001b[39m,\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m     18\u001b[0m     \u001b[39m\"\u001b[39m\u001b[39mhouse_construction_year\u001b[39m\u001b[39m\"\u001b[39m: \u001b[39m5\u001b[39m\n\u001b[1;32m     19\u001b[0m }\n\u001b[1;32m     24\u001b[0m dataset \u001b[39m=\u001b[39m TimeSeriesDataset(df, \u001b[39m\"\u001b[39m\u001b[39mtimeseries\u001b[39m\u001b[39m\"\u001b[39m, conditioning_dict)\n\u001b[0;32m---> 25\u001b[0m generator\u001b[39m.\u001b[39;49mfit(dataset, \u001b[39m\"\u001b[39;49m\u001b[39mtimeseries\u001b[39;49m\u001b[39m\"\u001b[39;49m)\n",
      "File \u001b[0;32m~/EnData/generator/data_generator.py:86\u001b[0m, in \u001b[0;36mDataGenerator.fit\u001b[0;34m(self, X, timeseries_colname)\u001b[0m\n\u001b[1;32m     80\u001b[0m expected_input_dim \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mmodel\u001b[39m.\u001b[39mopt\u001b[39m.\u001b[39minput_dim\n\u001b[1;32m     81\u001b[0m \u001b[39massert\u001b[39;00m sample_timeseries\u001b[39m.\u001b[39mshape \u001b[39m==\u001b[39m (\n\u001b[1;32m     82\u001b[0m     expected_seq_len,\n\u001b[1;32m     83\u001b[0m     expected_input_dim,\n\u001b[1;32m     84\u001b[0m ), \u001b[39mf\u001b[39m\u001b[39m\"\u001b[39m\u001b[39mExpected timeseries shape (\u001b[39m\u001b[39m{\u001b[39;00mexpected_seq_len\u001b[39m}\u001b[39;00m\u001b[39m, \u001b[39m\u001b[39m{\u001b[39;00mexpected_input_dim\u001b[39m}\u001b[39;00m\u001b[39m), but got \u001b[39m\u001b[39m{\u001b[39;00msample_timeseries\u001b[39m.\u001b[39mshape\u001b[39m}\u001b[39;00m\u001b[39m\"\u001b[39m\n\u001b[0;32m---> 86\u001b[0m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mmodel\u001b[39m.\u001b[39;49mtrain_model(dataset)\n",
      "File \u001b[0;32m~/EnData/generator/gan/acgan.py:301\u001b[0m, in \u001b[0;36mACGAN.train_model\u001b[0;34m(self, dataset)\u001b[0m\n\u001b[1;32m    299\u001b[0m _lambda \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39msparse_conditioning_loss_weight\n\u001b[1;32m    300\u001b[0m N_r \u001b[39m=\u001b[39m rare_mask\u001b[39m.\u001b[39msum()\u001b[39m.\u001b[39mitem()\n\u001b[0;32m--> 301\u001b[0m N_nr \u001b[39m=\u001b[39m (torch\u001b[39m.\u001b[39;49mlogical_not(rare_mask))\u001b[39m.\u001b[39;49msum()\u001b[39m.\u001b[39;49mitem()\n\u001b[1;32m    302\u001b[0m N \u001b[39m=\u001b[39m current_batch_size\n\u001b[1;32m    303\u001b[0m g_loss \u001b[39m=\u001b[39m (\n\u001b[1;32m    304\u001b[0m     _lambda \u001b[39m*\u001b[39m (N_r \u001b[39m/\u001b[39m N) \u001b[39m*\u001b[39m g_loss_rare\n\u001b[1;32m    305\u001b[0m     \u001b[39m+\u001b[39m (\u001b[39m1\u001b[39m \u001b[39m-\u001b[39m _lambda) \u001b[39m*\u001b[39m (N_nr \u001b[39m/\u001b[39m N) \u001b[39m*\u001b[39m g_loss_non_rare\n\u001b[1;32m    306\u001b[0m )\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "from datasets.timeseries_dataset import TimeSeriesDataset\n",
    "from generator.data_generator import DataGenerator\n",
    "\n",
    "df = pv.data\n",
    "df\n",
    "\n",
    "generator = DataGenerator(\"acgan\", model_params=None)\n",
    "\n",
    "conditioning_dict = {\n",
    "    \"month\": 12,\n",
    "    \"weekday\": 7,\n",
    "    \"city\": 6,\n",
    "    \"building_type\": 3,\n",
    "    \"has_solar\": 2,\n",
    "    \"car1\": 2,\n",
    "    \"state\": 3,\n",
    "    \"total_square_footage\": 5,\n",
    "    \"house_construction_year\": 5\n",
    "}\n",
    "\n",
    "dataset = TimeSeriesDataset(df, \"timeseries\", conditioning_dict)\n",
    "generator.fit(dataset, \"timeseries\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "venv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.5 (default, Nov 23 2021, 15:27:38) \n[GCC 9.3.0]"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "5ea03d6e3b6d6622a3c4a21d472827b30621b9677ee232f2748a7c51f6a29b0a"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}