scripts/rajasthan/parse_rajasthan_hindi.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import pdf2image\n",
    "from PIL import Image\n",
    "import pytesseract\n",
    "import difflib\n",
    "import re\n",
    "import pandas as pd\n",
    "import sys\n",
    "sys.path.append('../')\n",
    "from helper import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "DATA_PATH = '../../data/'\n",
    "STATE = 'rajasthan/'\n",
    "\n",
    "PARSE_DATA_PAGES = \"../../parseData/images/\"+STATE\n",
    "create_path(PARSE_DATA_PAGES)\n",
    "\n",
    "PARSE_DATA_BLOCKS = \"../../parseData/blocks/\"+STATE\n",
    "create_path(PARSE_DATA_BLOCKS)\n",
    "\n",
    "PARSE_DATA_CSVS = \"../../parseData/csvs/\"+STATE\n",
    "create_path(PARSE_DATA_CSVS)\n",
    "\n",
    "COLUMNS = [\"number\",\"id\", \"elector_name\", \"father_or_husband_name\", \"relationship\", \"house_no\", \"age\", \"sex\", \"ac_name\", \"parl_constituency\", \"part_no\", \"year\", \"state\", \"filename\", \"main_town\", \"police_station\", \"mandal\", \"revenue_division\", \"district\", \"pin_code\", \"polling_station_name\", \"polling_station_address\", \"net_electors_male\", \"net_electors_female\", \"net_electors_third_gender\", \"net_electors_total\",\"original_or_amendment\"]\n",
    "\n",
    "state_pdfs_path = DATA_PATH+STATE\n",
    "state_pdfs_files = os.listdir(state_pdfs_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def split_data(data):\n",
    "    seps = [\":\",\">\",\"-\",\".\"]\n",
    "    \n",
    "    for s in seps:\n",
    "        if s in data:\n",
    "            break\n",
    "\n",
    "    data = data.split(s)\n",
    "    data = [ i for i in data if i.strip()!='']\n",
    "    if len(data)>1:\n",
    "        data = data[1].strip()\n",
    "        return data\n",
    "    else:\n",
    "        data = \"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def extract_4_numbers(crop_stat_path):\n",
    "    \n",
    "    text = (pytesseract.image_to_string(crop_stat_path, config='--psm 6', lang='eng+hin')) #config='--psm 4' config='-c preserve_interword_spaces=1'\n",
    "\n",
    "    text = re.findall(r'\\d+', text)    \n",
    "    if len(text)==4:\n",
    "        if int(text[0]) + int(text[1]) == int(text[2]):\n",
    "            net_electors_male,net_electors_female,net_electors_third_gender,net_electors_total = text[0],text[1],\"0\",text[2]\n",
    "        elif int(text[0]) + int(text[1]) == int(text[3]):\n",
    "            net_electors_male,net_electors_female,net_electors_third_gender,net_electors_total = text[0],text[1],\"0\",text[3]\n",
    "        else:\n",
    "            net_electors_male,net_electors_female,net_electors_third_gender,net_electors_total = text[0],text[1],text[2],text[3]\n",
    "    elif len(text) == 3 and int(text[2])>=int(text[1]) and int(text[2])>=int(text[0]):\n",
    "        net_electors_male,net_electors_female,net_electors_third_gender,net_electors_total = text[0],text[1],\"0\",text[2]\n",
    "    elif len(text) == 2 and int(text[0])*2-100<int(text[1]):\n",
    "        net_electors_male,net_electors_female,net_electors_third_gender,net_electors_total = text[0],int(text[1])-int(text[0]),\"0\",text[1]\n",
    "    else:\n",
    "        net_electors_male,net_electors_female,net_electors_third_gender,net_electors_total = \"\",\"\",\"\",\"\"\n",
    "    \n",
    "    return net_electors_male,net_electors_female,net_electors_third_gender,net_electors_total\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "def extract_3rd_pin_code(path):\n",
    "    \n",
    "    path = path.replace('1.jpg','3.jpg')\n",
    "    \n",
    "    img = Image.open(path)\n",
    "    \n",
    "    a,b,c,d = 2890,50,1132,389 # pin_code\n",
    "    crop_img = crop_section(a,b,c,d,img)\n",
    "    \n",
    "    crop_path = input_images_blocks_path+\"page/\"\n",
    "    create_path(crop_path)\n",
    "    \n",
    "    crop_part_path = crop_path+\"pincode.jpg\"\n",
    "    crop_img.save(crop_part_path)\n",
    "\n",
    "    text = (pytesseract.image_to_string(crop_part_path, config='--psm 6', lang='eng+hin')) #config='--psm 4' config='-c preserve_interword_spaces=1'\n",
    "    text = text.split('\\n')\n",
    "    text = [ i for i in text if i!='' and i!='\\x0c']\n",
    "    text = text[-1]\n",
    "    text = re.findall(r'\\d+', text)\n",
    "    \n",
    "    if len(text)>0:\n",
    "        pin_code = text[-1]\n",
    "    else:\n",
    "        pin_code = \"\"\n",
    "    \n",
    "    return pin_code\n",
    "\n",
    "def extract_first_page_details(path):\n",
    "    \n",
    "    img = Image.open(path)\n",
    "        \n",
    "    a,b,c,d = 1450,5394,2420,100  # stats for male and female\n",
    "    crop_img = crop_section(a,b,c,d,img)\n",
    "\n",
    "    crop_path = input_images_blocks_path+\"page/\"\n",
    "    create_path(crop_path)\n",
    "    \n",
    "    crop_stat_path = crop_path+\"stat.jpg\"\n",
    "    crop_img.save(crop_stat_path)\n",
    "    \n",
    "    a_n,b_n,c_n,d_n = extract_4_numbers(crop_stat_path)\n",
    "    \n",
    "    a,b,c,d = 2380,3482,1320,531  # mandal block\n",
    "    crop_img = crop_section(a,b,c,d,img)\n",
    "    \n",
    "    crop_det_path = crop_path+\"det.jpg\"\n",
    "    crop_img.save(crop_det_path)\n",
    "\n",
    "    text = (pytesseract.image_to_string(crop_det_path, config='--psm 6', lang='eng+hin')) #config='--psm 4' config='-c preserve_interword_spaces=1'\n",
    "    text = text.split('\\n')\n",
    "    text = [ i for i in text if i!='' and i!='\\x0c']\n",
    "        \n",
    "    if len(text) == 5:\n",
    "        \n",
    "        main_town = split_data(text[0])\n",
    "        police_station = split_data(text[1])\n",
    "        revenue_division = split_data(text[2])\n",
    "        mandal = split_data(text[3])\n",
    "        district = split_data(text[4])\n",
    "        \n",
    "        \n",
    "        if district is None:\n",
    "            district = text[4].split(\"जिला\")\n",
    "            if len(district)>1:\n",
    "                district = district[1]\n",
    "            else:\n",
    "                district = ''\n",
    "        \n",
    "        if main_town is None:\n",
    "            main_town = text[0].split(\"गांव\")\n",
    "            if len(main_town)>1:\n",
    "                main_town = main_town[1]\n",
    "            else:\n",
    "                main_town = ''   \n",
    "    else:\n",
    "        main_town,police_station,revenue_division,mandal,district,pin_code = extract_detail_section(text)\n",
    "    \n",
    "    district = district.replace(\"&\",\"\")\n",
    "    district = district.replace(\"£\",\"\")\n",
    "        \n",
    "    pin_code = extract_3rd_pin_code(path)\n",
    "\n",
    "    a,b,c,d = 3325,320,585,125 # part no\n",
    "    crop_img = crop_section(a,b,c,d,img)\n",
    "    \n",
    "    crop_part_path = crop_path+\"part.jpg\"\n",
    "    crop_img.save(crop_part_path)\n",
    "\n",
    "    text = (pytesseract.image_to_string(crop_part_path, config='--psm 6', lang='eng+hin')) #config='--psm 4' config='-c preserve_interword_spaces=1'\n",
    "    text = re.findall(r'\\d+', text)\n",
    "    \n",
    "    if len(text)>0:\n",
    "        part_no = text[0]\n",
    "    else:\n",
    "        part_no = \"\"\n",
    "        \n",
    "    a,b,c,d = 400,4256,2029,665 # police name name and address\n",
    "    crop_img = crop_section(a,b,c,d,img)\n",
    "    \n",
    "    crop_police_path = crop_path+\"police.jpg\"\n",
    "    crop_img.save(crop_police_path)\n",
    "    \n",
    "    text = (pytesseract.image_to_string(crop_police_path, config='--psm 6', lang='eng+hin')) #config='--psm 4' config='-c preserve_interword_spaces=1'\n",
    "    text = text.split('\\n')\n",
    "    text = [ i for i in text if i!='' and i!='\\x0c']\n",
    "        \n",
    "    if len(text) == 3:\n",
    "        polling_station_name = text[0].split(\":\")[1]\n",
    "        polling_station_address = text[2]\n",
    "    elif len(text) == 4:\n",
    "        polling_station_name = text[1]\n",
    "        polling_station_address = text[3]\n",
    "    else:\n",
    "        polling_station_name, polling_station_address = text_police_compare(text)        \n",
    "    \n",
    "    a,b,c,d = 350,340,2706,345 # ac name and parl\n",
    "    crop_img = crop_section(a,b,c,d,img)\n",
    "    \n",
    "    crop_ac_path = crop_path+\"ac.jpg\"\n",
    "    crop_img.save(crop_ac_path)\n",
    "    \n",
    "    text = (pytesseract.image_to_string(crop_ac_path, config='--psm 6', lang='eng+hin')) #config='--psm 4' config='-c preserve_interword_spaces=1'\n",
    "    text = text.split('\\n')\n",
    "    text = [ i for i in text if i!='' and i!='\\x0c']\n",
    "    \n",
    "    if len(text) == 2:\n",
    "        try:\n",
    "            seps = [\":\",\",\"]\n",
    "\n",
    "            for s in seps:\n",
    "                if s in text[0]:\n",
    "                    break\n",
    "            ac_name = text[0].split(s)[1]\n",
    "        except:\n",
    "            ac_name = text[0].split(\"-\")[1]\n",
    "        \n",
    "        try:\n",
    "            seps = [\":\",\",\"]\n",
    "\n",
    "            for s in seps:\n",
    "                if s in text[1]:\n",
    "                    break\n",
    "            parl_constituency = text[1].split(s)[1]\n",
    "        except:\n",
    "            parl_constituency = text[1].split(\"-\")[1]\n",
    "        \n",
    "    else:\n",
    "        ac_name,parl_constituency = '',''\n",
    "    \n",
    "    print_null(\"ac_name\", ac_name)\n",
    "    print_null(\"parl_constituency\", parl_constituency)\n",
    "    print_null(\"part_no\", part_no)\n",
    "    print_null(\"main_town\", main_town)\n",
    "    print_null(\"police_station\", police_station)\n",
    "    print_null(\"polling_station_name\", polling_station_name)\n",
    "    print_null(\"polling_station_address\", polling_station_address)\n",
    "    print_null(\"revenue_division\", revenue_division)\n",
    "    print_null(\"mandal\", mandal)\n",
    "    print_null(\"district\", district)\n",
    "    print_null(\"pin_code\", pin_code)\n",
    "    print()\n",
    "    print()\n",
    "        \n",
    "    return [ac_name,parl_constituency,part_no,main_town,police_station,polling_station_name,polling_station_address,revenue_division,mandal,district,pin_code,a_n,b_n,c_n,d_n]\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "def print_null(name, value):\n",
    "    if value == \"\" or value == '':\n",
    "        print(name,\" = empty\")\n",
    "    else:\n",
    "        print(name, \" = \", value)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "def generate_poll_blocks_from_page(page_full_path,page_blocks_path,amend_page):\n",
    "    \n",
    "    img = Image.open(page_full_path)\n",
    "\n",
    "    amend = False\n",
    "    \n",
    "    def generate(intial_width,a,b,gap):\n",
    "        count = 0\n",
    "        crop_width = 1210\n",
    "        crop_height = 495\n",
    "\n",
    "        for col in range(1,11):\n",
    "\n",
    "            for row in range(1,4):\n",
    "                c = a+crop_width\n",
    "                d = b+crop_height\n",
    "                area = (a, b, c, d)\n",
    "                cropped_img = img.crop(area)\n",
    "                count = count+1\n",
    "                \n",
    "                new_area = (800,100, 1200, 470)\n",
    "                region = Image.new(\"RGB\", (400, 370), (255, 255, 255))\n",
    "                cropped_img.paste(region,new_area)\n",
    "                \n",
    "                cropped_img.save(page_blocks_path+str(count)+\".jpg\")\n",
    "\n",
    "                a = c\n",
    "\n",
    "            a = intial_width\n",
    "            b = b+crop_height+gap\n",
    "    \n",
    "    page_type,intial_height = check_page_type(img,amend_page)\n",
    "    \n",
    "    if page_type == 1:\n",
    "        intial_width = 230\n",
    "        generate(intial_width,intial_width,intial_height,6)\n",
    "        amend_page = False\n",
    "    else:\n",
    "        intial_width = 230\n",
    "        generate(intial_width,intial_width,intial_height,40)\n",
    "        amend_page = True\n",
    "        \n",
    "    return amend_page\n",
    "\n",
    "def check_page_type(img,amend_page):\n",
    "    \n",
    "    return 1,515\n",
    "\n",
    "    if amend_page:\n",
    "        return 2,295\n",
    "    \n",
    "    a,b,c,d = 130, 280,800,155  # amend page check\n",
    "    crop_img = crop_section(a,b,c,d,img)\n",
    "\n",
    "    crop_temp_path = \"temp.jpg\"\n",
    "    crop_img.save(crop_temp_path)\n",
    "    \n",
    "    text = (pytesseract.image_to_string(crop_temp_path, config='--psm 6', lang='eng+hin')) #config='--psm 4' config='-c preserve_interword_spaces=1'\n",
    "    text = text.split('\\n')\n",
    "    text = [ i for i in text if i!='' and i!='\\x0c']\n",
    "    \n",
    "    if len(text)>0:\n",
    "        for key in ['घटक','परिवर्धन']:\n",
    "            for t in text:\n",
    "                if key in t:\n",
    "                    return 2,460                  \n",
    "            \n",
    "    return 1,270\n",
    "    \n",
    "        \n",
    "def step2_order(params):\n",
    "    \n",
    "    if len(params) == 6:\n",
    "        new_params_list = []\n",
    "        \n",
    "        new_params_list.append(params[0] + \" \" + params[1])\n",
    "       \n",
    "        for i in params[2:]:\n",
    "            new_params_list.append(i)\n",
    "            \n",
    "        return new_params_list\n",
    "\n",
    "    return params\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "def extract_name(name):\n",
    "    \n",
    "    row = name.split(\":\")\n",
    "    if len(row)!=2:\n",
    "        return \"\"\n",
    "    else:\n",
    "        return row[1].strip()\n",
    "    \n",
    "def extract_vid(v_id):\n",
    "    row = v_id.split(\"|\")\n",
    "    if len(row)>=2:\n",
    "        number = re.findall(r'\\d+', row[0].strip())\n",
    "        if len(number)>0:\n",
    "            return number[0],row[1]\n",
    "        else:\n",
    "            if len(row[1])<4:\n",
    "                return row[1], \"\"\n",
    "    \n",
    "    if len(row)==1:\n",
    "        if len(row[0])>3:\n",
    "            return \"\", row[-1]\n",
    "        else:\n",
    "            return row[-1],\"\"\n",
    "    else:\n",
    "        row = v_id.split(\" \")\n",
    "        if len(row)>=2:    \n",
    "            return row[0],row[-1]\n",
    "        else:\n",
    "            return \"\",\"\"\n",
    "       \n",
    "        \n",
    "\n",
    "def extract_house_no(house_no):\n",
    "    row = house_no.split(\":\")\n",
    "    if len(row)==2:\n",
    "        house_no = re.findall(r'\\d+', row[1].strip())\n",
    "        if len(house_no)>0:\n",
    "            return house_no[0]\n",
    "        else:\n",
    "            return \"\"\n",
    "    else:\n",
    "        house_no = re.findall(r'\\d+', row[0].strip())\n",
    "        if len(house_no)>0:\n",
    "            return house_no[0]\n",
    "        else:\n",
    "            return \"\"\n",
    "    \n",
    "def extract_age_gender(age_gender):\n",
    "    row = age_gender.split(\":\")\n",
    "    \n",
    "    if len(row)!=3:\n",
    "        return \"\",\"\"\n",
    "    else:    \n",
    "        age = re.findall(r'\\d+', row[1].strip())\n",
    "        if len(age)>0:\n",
    "            age =  age[0]\n",
    "        else:\n",
    "            age = \"\"\n",
    "        \n",
    "        if 'पुरूष' in row[2].strip() or 'पुरुष' in row[2].strip():\n",
    "            gender = 'पुरूष'\n",
    "        elif 'स्त्री' in row[2].strip():\n",
    "            gender = 'स्त्री'\n",
    "        else:\n",
    "            gender =''\n",
    "\n",
    "    return age, gender\n",
    "\n",
    "def extract_rel_name(rel_name):\n",
    "    row = rel_name.split(\":\")\n",
    "    if len(row)!=2:\n",
    "        \n",
    "        row = rel_name.split(\";\")\n",
    "        if len(row)!=2:\n",
    "            return \"\",\"\"\n",
    "        else:\n",
    "            rel_type = extract_rel_type(row[0].strip())\n",
    "            return row[1].strip(),rel_type\n",
    "    else:\n",
    "        rel_type = extract_rel_type(row[0].strip())\n",
    "        \n",
    "        return row[1].strip(),rel_type\n",
    "    \n",
    "def extract_rel_type(rel_type):\n",
    "    line = rel_type\n",
    "    if line.startswith(\"पति\") :\n",
    "        rel_type = 'husband'\n",
    "    elif line.startswith(\"पिता\") or 'ता' in line :\n",
    "        rel_type = 'father'\n",
    "    elif line.startswith(\"माता\") :\n",
    "        rel_type = 'mother'\n",
    "    elif line.startswith(\"अन्य\") :\n",
    "        rel_type = 'other'\n",
    "    else:\n",
    "        rel_type = \"\"\n",
    "    \n",
    "    return rel_type \n",
    "\n",
    "\n",
    "def extract_details_from_block(block):\n",
    "    \n",
    "    v_id = block[0]\n",
    "    name = block[1]\n",
    "    rel_name = block[2]\n",
    "    house_no = block[3]\n",
    "    age_gender = block[4]\n",
    "    \n",
    "    name = extract_name(name)\n",
    "    rel_name,rel_type = extract_rel_name(rel_name)\n",
    "    house_no = extract_house_no(house_no)\n",
    "    age, gender = extract_age_gender(age_gender)\n",
    "    number,voter_id = extract_vid(v_id)\n",
    "    \n",
    "    return [name,rel_name,rel_type,house_no,age,gender,voter_id,number]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "def arrange_columns(first_page_list,block_list,filename):\n",
    "    \n",
    "    year = 2018\n",
    "    state = 'rajasthan'\n",
    "    \n",
    "    ac_name,parl_constituency,part_no,main_town,police_station,polling_station_name,polling_station_address,revenue_division,mandal,district,pin_code,net_electors_male,net_electors_female,net_electors_third_gender,net_electors_total = first_page_list\n",
    "    \n",
    "    name,rel_name,rel_type,house_no,age,gender,voter_id,number = block_list\n",
    "    \n",
    " \n",
    "    final_list = [stripe(number),stripe(voter_id),stripe(name),stripe(rel_name),stripe(rel_type),stripe(house_no),age,stripe(gender),stripe(ac_name),\n",
    "                 stripe(parl_constituency),part_no,year,state,filename,stripe(main_town),stripe(police_station),stripe(mandal),\n",
    "                 stripe(revenue_division),stripe(district),pin_code,stripe(polling_station_name),stripe(polling_station_address),\n",
    "                 net_electors_male,net_electors_female,net_electors_third_gender,net_electors_total]\n",
    "\n",
    "    return final_list\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "def stripe(data):\n",
    "    if type(data) == str:\n",
    "        return data.strip()\n",
    "    else:\n",
    "        return data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "A082_A082209.pdf\n",
      "ac_name  =   82 - हिण्डौन (अ.जा.)\n",
      "parl_constituency  =   10, करौली-धौलपुर (अ.जा.)\n",
      "part_no  =  209\n",
      "main_town  =   ० हिण्डौन\n",
      "police_station  =  Hindaun Municipality\n",
      "polling_station_name  =   209 हिण्डौन\n",
      "polling_station_address  =  राजकीय उच्च प्राथमिक विद्यालय नं01 हिण्डौन उत्तरी भाग\n",
      "revenue_division  =  हिण्डौन\n",
      "mandal  =  हिण्डौन\n",
      "district  =    करौली\n",
      "pin_code  =  322230\n",
      "\n",
      "\n",
      "A082_A082209.csv already exists\n",
      "CSV saved\n",
      "\n",
      "A038_A038034.pdf\n",
      "ac_name  =   38 - नीम का थाना (सामान्य)\n",
      "parl_constituency  =   5, सीकर (सामान्य)\n",
      "part_no  =  34\n",
      "main_town  =   ० मावण्डा कलां\n",
      "police_station  =  मावण्डा कलां\n",
      "polling_station_name  =   34 मावण्डा कलां\n",
      "polling_station_address  =  राजकीय आदर्श उच्च माध्यमिक विद्यालय, मावण्डा कलां (दायां भाग)\n",
      "revenue_division  =  नाथा की नांगल\n",
      "mandal  =  None\n",
      "district  =   सीकर\n",
      "pin_code  =  332711\n",
      "\n",
      "\n",
      "A038_A038034.csv already exists\n",
      "CSV saved\n",
      "\n",
      "A163_A163007.pdf\n",
      "ac_name  =   163 - गढ़ी (अ.ज.जा.)\n",
      "parl_constituency  =   20, बांसवाड़ा (अ.ज.जा.)\n",
      "part_no  =  7\n",
      "main_town  =  LOHARIYA\n",
      "police_station  =  लोहारिया\n",
      "polling_station_name  =   7 लोहारिया\n",
      "polling_station_address  =  आदर्श राजकीय उच्च माध्यमिक विधालय, लोहारिया बांया भाग\n",
      "revenue_division  =  पालोदा\n",
      "mandal  =  गढी\n",
      "district  =    बांसवाड़ा\n",
      "pin_code  =  327801\n",
      "\n",
      "\n",
      "page done :  3.jpg\n",
      "page done :  4.jpg\n",
      "page done :  5.jpg\n",
      "page done :  6.jpg\n",
      "page done :  7.jpg\n",
      "page done :  8.jpg\n",
      "page done :  9.jpg\n",
      "page done :  10.jpg\n",
      "page done :  11.jpg\n",
      "page done :  12.jpg\n",
      "page done :  13.jpg\n",
      "page done :  14.jpg\n",
      "page done :  15.jpg\n",
      "page done :  16.jpg\n",
      "page done :  17.jpg\n",
      "page done :  18.jpg\n",
      "page done :  19.jpg\n",
      "page done :  20.jpg\n",
      "page done :  21.jpg\n",
      "page done :  22.jpg\n",
      "page done :  23.jpg\n",
      "page done :  24.jpg\n",
      "page done :  25.jpg\n",
      "page done :  26.jpg\n",
      "page done :  27.jpg\n",
      "page done :  28.jpg\n",
      "page done :  29.jpg\n",
      "page done :  30.jpg\n",
      "page done :  31.jpg\n",
      "page done :  32.jpg\n",
      "page done :  33.jpg\n",
      "page done :  34.jpg\n",
      "page done :  35.jpg\n",
      "page done :  36.jpg\n",
      "page done :  37.jpg\n",
      "page done :  38.jpg\n",
      "page done :  39.jpg\n",
      "CSV saved\n",
      "\n",
      "A122_A122149.pdf\n",
      "ac_name  =   122 - फलौदी (सामान्य)\n",
      "parl_constituency  =   16, जोधपुर (सामान्य)\n",
      "part_no  =  149\n",
      "main_town  =  गोदरली\n",
      "police_station  =  मलार\n",
      "polling_station_name  =   149 गोदरली\n",
      "polling_station_address  =  राजकीय प्राथमिक विद्यालय गोदरली\n",
      "revenue_division  =  बाप\n",
      "mandal  =  बाप\n",
      "district  =    जोधपुर\n",
      "pin_code  = empty\n",
      "\n",
      "\n",
      "page done :  3.jpg\n",
      "page done :  4.jpg\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-44-e2c41cca36d7>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     71\u001b[0m                 \u001b[0;32mif\u001b[0m \u001b[0mjpg_file\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mendswith\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'.jpg'\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     72\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 73\u001b[0;31m                     \u001b[0mtext\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mpytesseract\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mimage_to_string\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpage_blocks_path\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0mjpg_file\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mconfig\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'--psm 6'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlang\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'eng+hin'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;31m#config='--psm 4' config='-c preserve_interword_spaces=1'\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     74\u001b[0m                     \u001b[0mparams_list\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtext\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msplit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'\\n'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     75\u001b[0m                     \u001b[0mnew_params_list\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m \u001b[0mi\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mparams_list\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mi\u001b[0m\u001b[0;34m!=\u001b[0m\u001b[0;34m''\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mi\u001b[0m\u001b[0;34m!=\u001b[0m\u001b[0;34m'\\x0c'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/Desktop/GauravFreelanceing/env/lib/python3.8/site-packages/pytesseract/pytesseract.py\u001b[0m in \u001b[0;36mimage_to_string\u001b[0;34m(image, lang, config, nice, output_type, timeout)\u001b[0m\n\u001b[1;32m    352\u001b[0m     \u001b[0margs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mimage\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'txt'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlang\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mconfig\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnice\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtimeout\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    353\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 354\u001b[0;31m     return {\n\u001b[0m\u001b[1;32m    355\u001b[0m         \u001b[0mOutput\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mBYTES\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;32mlambda\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mrun_and_get_output\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0margs\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    356\u001b[0m         \u001b[0mOutput\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mDICT\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;32mlambda\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;34m{\u001b[0m\u001b[0;34m'text'\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mrun_and_get_output\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/Desktop/GauravFreelanceing/env/lib/python3.8/site-packages/pytesseract/pytesseract.py\u001b[0m in \u001b[0;36m<lambda>\u001b[0;34m()\u001b[0m\n\u001b[1;32m    355\u001b[0m         \u001b[0mOutput\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mBYTES\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;32mlambda\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mrun_and_get_output\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0margs\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    356\u001b[0m         \u001b[0mOutput\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mDICT\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;32mlambda\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;34m{\u001b[0m\u001b[0;34m'text'\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mrun_and_get_output\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 357\u001b[0;31m         \u001b[0mOutput\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mSTRING\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;32mlambda\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mrun_and_get_output\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    358\u001b[0m     }[output_type]()\n\u001b[1;32m    359\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/Desktop/GauravFreelanceing/env/lib/python3.8/site-packages/pytesseract/pytesseract.py\u001b[0m in \u001b[0;36mrun_and_get_output\u001b[0;34m(image, extension, lang, config, nice, timeout, return_bytes)\u001b[0m\n\u001b[1;32m    264\u001b[0m         }\n\u001b[1;32m    265\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 266\u001b[0;31m         \u001b[0mrun_tesseract\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    267\u001b[0m         \u001b[0mfilename\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'output_filename_base'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mextsep\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mextension\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    268\u001b[0m         \u001b[0;32mwith\u001b[0m \u001b[0mopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfilename\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'rb'\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0moutput_file\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/Desktop/GauravFreelanceing/env/lib/python3.8/site-packages/pytesseract/pytesseract.py\u001b[0m in \u001b[0;36mrun_tesseract\u001b[0;34m(input_filename, output_filename_base, extension, lang, config, nice, timeout)\u001b[0m\n\u001b[1;32m    238\u001b[0m         \u001b[0;32mraise\u001b[0m \u001b[0mTesseractNotFoundError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    239\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 240\u001b[0;31m     \u001b[0;32mwith\u001b[0m \u001b[0mtimeout_manager\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mproc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtimeout\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0merror_string\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    241\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mproc\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreturncode\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    242\u001b[0m             \u001b[0;32mraise\u001b[0m \u001b[0mTesseractError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mproc\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreturncode\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mget_errors\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0merror_string\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/opt/python@3.8/Frameworks/Python.framework/Versions/3.8/lib/python3.8/contextlib.py\u001b[0m in \u001b[0;36m__enter__\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    111\u001b[0m         \u001b[0;32mdel\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mkwds\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfunc\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    112\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 113\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mnext\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgen\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    114\u001b[0m         \u001b[0;32mexcept\u001b[0m \u001b[0mStopIteration\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    115\u001b[0m             \u001b[0;32mraise\u001b[0m \u001b[0mRuntimeError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"generator didn't yield\"\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/Desktop/GauravFreelanceing/env/lib/python3.8/site-packages/pytesseract/pytesseract.py\u001b[0m in \u001b[0;36mtimeout_manager\u001b[0;34m(proc, seconds)\u001b[0m\n\u001b[1;32m    101\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    102\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mseconds\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 103\u001b[0;31m             \u001b[0;32myield\u001b[0m \u001b[0mproc\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcommunicate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    104\u001b[0m             \u001b[0;32mreturn\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    105\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/opt/python@3.8/Frameworks/Python.framework/Versions/3.8/lib/python3.8/subprocess.py\u001b[0m in \u001b[0;36mcommunicate\u001b[0;34m(self, input, timeout)\u001b[0m\n\u001b[1;32m   1022\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1023\u001b[0m             \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1024\u001b[0;31m                 \u001b[0mstdout\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstderr\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_communicate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mendtime\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtimeout\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1025\u001b[0m             \u001b[0;32mexcept\u001b[0m \u001b[0mKeyboardInterrupt\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1026\u001b[0m                 \u001b[0;31m# https://bugs.python.org/issue25942\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/opt/python@3.8/Frameworks/Python.framework/Versions/3.8/lib/python3.8/subprocess.py\u001b[0m in \u001b[0;36m_communicate\u001b[0;34m(self, input, endtime, orig_timeout)\u001b[0m\n\u001b[1;32m   1864\u001b[0m                             'failed to raise TimeoutExpired.')\n\u001b[1;32m   1865\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1866\u001b[0;31m                     \u001b[0mready\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mselector\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mselect\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtimeout\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1867\u001b[0m                     \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_check_timeout\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mendtime\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0morig_timeout\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstdout\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstderr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1868\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/opt/python@3.8/Frameworks/Python.framework/Versions/3.8/lib/python3.8/selectors.py\u001b[0m in \u001b[0;36mselect\u001b[0;34m(self, timeout)\u001b[0m\n\u001b[1;32m    413\u001b[0m         \u001b[0mready\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    414\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 415\u001b[0;31m             \u001b[0mfd_event_list\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_selector\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpoll\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtimeout\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    416\u001b[0m         \u001b[0;32mexcept\u001b[0m \u001b[0mInterruptedError\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    417\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0mready\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "# for pdf_file_name in ['A159_A159073.pdf']:\n",
    "for pdf_file_name in state_pdfs_files:\n",
    "    \n",
    "    print(pdf_file_name)\n",
    "    \n",
    "    #create images,blocks and csvs paths for each file\n",
    "    pdf_file_name_without_ext = pdf_file_name.split('.pdf')[0]\n",
    "    input_pdf_images_path = PARSE_DATA_PAGES+pdf_file_name_without_ext+\"/\"\n",
    "    create_path(input_pdf_images_path)\n",
    "\n",
    "    input_images_blocks_path = PARSE_DATA_BLOCKS+pdf_file_name_without_ext+\"/\"\n",
    "    create_path(input_images_blocks_path)\n",
    "    \n",
    "    #convert pdf into bunch of images\n",
    "    #pdf_2_images_list = pdf_to_img(state_pdfs_path+pdf_file_name, input_pdf_images_path,dpi=500)\n",
    "    \n",
    "    \n",
    "    #sort pages for looping\n",
    "    input_images = os.listdir(input_pdf_images_path)\n",
    "    sort_nicely(input_images)\n",
    "    \n",
    "    #empty intial data\n",
    "    df = pd.DataFrame(columns = COLUMNS)\n",
    "    order_problem = []\n",
    "    \n",
    "    amend_page = False\n",
    "    \n",
    "    #for each page, parse the data\n",
    "    for page in input_images:\n",
    "    \n",
    "        page_full_path = input_pdf_images_path+page\n",
    "        \n",
    "        #extract first page content\n",
    "        if page == '1.jpg':\n",
    "            first_page_list = extract_first_page_details(page_full_path)\n",
    "            continue\n",
    "            \n",
    "\n",
    "        #ingnore 2nd page and last page\n",
    "        if page == '2.jpg' or input_images[-1] == page:\n",
    "            continue\n",
    "        \n",
    "        if os.path.exists(PARSE_DATA_CSVS+pdf_file_name_without_ext+\".csv\"):\n",
    "            print(pdf_file_name_without_ext+\".csv\", \"already exists\")\n",
    "            break\n",
    "            \n",
    "\n",
    "        #loop from 3 page onwards\n",
    "        if page.endswith('.jpg'):\n",
    "            \n",
    "            final_invidual_blocks = []\n",
    "            blocks_path = input_images_blocks_path+\"blocks/\"\n",
    "            create_path(blocks_path)\n",
    "\n",
    "            page_idx = page.split(\".jpg\")[0] + \"/\"\n",
    "            page_blocks_path = blocks_path+page_idx\n",
    "            create_path(page_blocks_path)\n",
    "                \n",
    "            amend_page = generate_poll_blocks_from_page(page_full_path,page_blocks_path,amend_page)\n",
    "\n",
    "            if amend_page:\n",
    "                page_type = 'amendment'\n",
    "            else:\n",
    "                page_type = 'original'\n",
    "                \n",
    "            sorted_blocks = os.listdir(page_blocks_path)\n",
    "            sort_nicely(sorted_blocks)\n",
    "            \n",
    "            for jpg_file in sorted_blocks:\n",
    "                \n",
    "                if jpg_file.endswith('.jpg') :\n",
    "\n",
    "                    text = (pytesseract.image_to_string(page_blocks_path+jpg_file, config='--psm 6', lang='eng+hin')) #config='--psm 4' config='-c preserve_interword_spaces=1'\n",
    "                    params_list = text.split('\\n')\n",
    "                    new_params_list = [ i for i in params_list if i!='' and i!='\\x0c']\n",
    "                    \n",
    "                    if len(new_params_list) !=5:\n",
    "                        if len(new_params_list)>5:\n",
    "                            new_params_list = step2_order(new_params_list)\n",
    "                            if len(new_params_list)!=5:\n",
    "                                order_problem.append((page, jpg_file,new_params_list))\n",
    "                            else:\n",
    "                                final_invidual_blocks.append(new_params_list)\n",
    "                        else:\n",
    "                            order_problem.append((page, jpg_file,new_params_list))\n",
    "                    else:\n",
    "                        final_invidual_blocks.append(new_params_list)\n",
    "        \n",
    "        #put the data into dataframe\n",
    "        for block in final_invidual_blocks:\n",
    "            block_list = extract_details_from_block(block)\n",
    "                        \n",
    "            final_list = arrange_columns(first_page_list,block_list,pdf_file_name_without_ext)\n",
    "            final_list.append(page_type)\n",
    "            \n",
    "            df_length = len(df)\n",
    "            df.loc[df_length] = final_list\n",
    "        \n",
    "        print(\"page done : \",page)\n",
    "                \n",
    "    #save the dataframe(pdf) data into csv\n",
    "    save_to_csv(df,PARSE_DATA_CSVS+pdf_file_name_without_ext+\".csv\")\n",
    "    print(\"CSV saved\")\n",
    "    print()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#combine all state files into one csv\n",
    "combine_all_csvs(\"rajasthan_final.csv\",PARSE_DATA_CSVS)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}