develop.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 50,
   "id": "ad49df01",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "from collections import defaultdict\n",
    "\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "id": "402a57a2-c86d-4698-97b2-71b19694f6cb",
   "metadata": {},
   "outputs": [],
   "source": [
    "RAW_VER = \"202202\"\n",
    "CLS_VER = 48"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "id": "c9a25d16",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[' 1 Agric  Agriculture\\n',\n",
       " '          0100-0199 Agricultural production - crops\\n',\n",
       " '          0200-0299 Agricultural production - livestock\\n',\n",
       " '          0700-0799 Agricultural services\\n',\n",
       " '          0910-0919 Commercial fishing\\n']"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "with open('raw' + os.sep + RAW_VER + os.sep + f'Siccodes{CLS_VER}.txt', 'r+', encoding='utf-8') as f:\n",
    "    t = f.readlines()\n",
    "t[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "id": "50bfd87e-bcef-41aa-86c6-2240ab766aa6",
   "metadata": {},
   "outputs": [],
   "source": [
    "range_dct = defaultdict(list)\n",
    "name_dct = defaultdict(str)\n",
    "for l in t:\n",
    "    if len(l) == 1: continue\n",
    "    if l[:2] != '  ':\n",
    "        # print(l)\n",
    "        ind = int(l[:2].strip())\n",
    "        name_dct[ind] = l[3:10].strip()\n",
    "    else:\n",
    "        range_dct[ind].append([int(x) for x in l[10:19].split('-')])\n",
    "        \n",
    "# name_dct\n",
    "# range_dct"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "id": "ddf52a7f-61a7-4476-b536-1b67ad2f480e",
   "metadata": {},
   "outputs": [],
   "source": [
    "def query(sic):\n",
    "    for ind in range_dct:\n",
    "        for lo, hi in range_dct[ind]:\n",
    "            if lo <= sic <= hi:\n",
    "                return ind\n",
    "    return CLS_VER  # Default: Other industry."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "id": "7bcd793c-6165-43b7-89a5-e8a7902452d3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(9999, 1)"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sic</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   sic\n",
       "0    1\n",
       "1    2\n",
       "2    3"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.DataFrame({'sic': range(1, 10000)})\n",
    "df.shape\n",
    "df.head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "id": "7eb2bf37-54ca-4192-b794-77a571fbd720",
   "metadata": {},
   "outputs": [],
   "source": [
    "df[f'ff_{cls_ver}ind'] = df['sic'].apply(query)\n",
    "df[f'ff_{cls_ver}ind_desc'] = df[f'ff_{cls_ver}ind'].apply(lambda x: name_dct[x])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "id": "602b40e6-19aa-4e32-bda1-048d85cd435c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sic</th>\n",
       "      <th>ff_48ind</th>\n",
       "      <th>ff_48ind_desc</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>48</td>\n",
       "      <td>Other</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>48</td>\n",
       "      <td>Other</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>48</td>\n",
       "      <td>Other</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   sic  ff_48ind ff_48ind_desc\n",
       "0    1        48         Other\n",
       "1    2        48         Other\n",
       "2    3        48         Other"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "id": "9462e183-ad1e-420a-baa0-589ebbfb96b5",
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_csv(f'ff_{CLS_VER}ind.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "10cb0782-2a91-481f-b930-69ac4fe5475c",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}