1-Web_Scrapping-Car_Price.ipynb

{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":["!pip install bs4"],"execution_count":1,"outputs":[{"output_type":"stream","text":"Collecting bs4\n  Downloading bs4-0.0.1.tar.gz (1.1 kB)\nCollecting beautifulsoup4\n  Downloading beautifulsoup4-4.9.3-py3-none-any.whl (115 kB)\n\u001b[K     |████████████████████████████████| 115 kB 842 kB/s eta 0:00:01\n\u001b[?25hCollecting soupsieve>1.2\n  Downloading soupsieve-2.2-py3-none-any.whl (33 kB)\nBuilding wheels for collected packages: bs4\n  Building wheel for bs4 (setup.py) ... \u001b[?25ldone\n\u001b[?25h  Created wheel for bs4: filename=bs4-0.0.1-py3-none-any.whl size=1273 sha256=a4ae74daae2ab51ebda119f08fcfd25149d5f14031d712a600092e5aa024a282\n  Stored in directory: /root/.cache/pip/wheels/0a/9e/ba/20e5bbc1afef3a491f0b3bb74d508f99403aabe76eda2167ca\nSuccessfully built bs4\nInstalling collected packages: soupsieve, beautifulsoup4, bs4\nSuccessfully installed beautifulsoup4-4.9.3 bs4-0.0.1 soupsieve-2.2\n","name":"stdout"}]},{"metadata":{"trusted":true},"cell_type":"code","source":["from bs4 import BeautifulSoup\n","import requests\n","import pandas as pd\n","import numpy as np\n","pd.set_option('display.max_columns', None)\n","pd.set_option('display.max_rows',None)\n","from IPython.display import FileLink"],"execution_count":3,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":["def check_status_code(url):\n","    r=requests.get(url)\n","    print(r.status_code)"],"execution_count":4,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":["check_status_code(\"https://www.truecar.com/used-cars-for-sale/listings/\")"],"execution_count":5,"outputs":[{"output_type":"stream","text":"200\n","name":"stdout"}]},{"metadata":{"trusted":true},"cell_type":"code","source":["def get_page_links(page_start,page_end):\n","    pages=np.arange(page_start,page_end)\n","    page_links=[]\n","    \n","    for page in pages:\n","        page_link=\"https://www.truecar.com/used-cars-for-sale/listings/?page={page}&sort[]=best_match\".format(page=page)\n","        page_links.append(page_link)\n","    return page_links"],"execution_count":6,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":["def getCarInfo(url):\n","    r=requests.get(url)\n","    soup=BeautifulSoup(r.text,\"html.parser\")\n","    #print(soup.prettify())\n","    return soup "],"execution_count":7,"outputs":[]},{"metadata":{"tags":[],"trusted":true},"cell_type":"code","source":["page_links=get_page_links(1,332)\n","announs=[]\n","for i in range(len(page_links)):\n","    url=page_links[i]\n","    soup=getCarInfo(url)\n","    \n","    for j in range(29):\n","        link=\"https://www.truecar.com\"+soup.find_all(class_=\"linkable order-2 vehicle-card-overlay\")[j].get(\"href\")\n","        announs.append(link)"],"execution_count":8,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":["len(announs)"],"execution_count":9,"outputs":[{"output_type":"execute_result","execution_count":9,"data":{"text/plain":"9599"},"metadata":{}}]},{"metadata":{"trusted":true},"cell_type":"code","source":["prices=[]\n","ext_colors=[]\n","int_colors=[] \n","MPGs=[]\n","engines=[] \n","transmissions=[] \n","drive_types=[]\n","fuel_types=[]\n","mileages=[]\n","opt_levels=[] \n","styles=[]  \n","locations=[]\n","\n","for i in announs: \n","    url=i\n","    r=requests.get(url)\n","    soup=BeautifulSoup(r.content,\"html.parser\")\n","    \n","    try:\n","           \n","            price=soup.find(class_=\"label-block-text\").text\n","            prices.append(price)\n","            \n","            exterior_col=soup.find_all(class_=\"_1l9na3c align-self-center col\")[0].find(\"li\").text\n","            ext_colors.append(exterior_col)\n","            \n","            int_col=soup.find_all(class_=\"_1l9na3c align-self-center col\")[2].find(\"li\").text\n","            int_colors.append(int_col)\n","            \n","            mpg=soup.find_all(class_=\"_1l9na3c align-self-center col\")[3].find(\"li\").text\n","            MPGs.append(mpg)\n","            \n","            eng=soup.find_all(class_=\"_1l9na3c align-self-center col\")[4].find(\"li\").text\n","            engines.append(eng)\n","            \n","            transmission=soup.find_all(class_=\"_1l9na3c align-self-center col\")[5].find(\"li\").text\n","            transmissions.append(transmission)\n","            \n","            drive_type=soup.find_all(class_=\"_1l9na3c align-self-center col\")[6].find(\"li\").text\n","            drive_types.append(drive_type)\n","            \n","            fuel_type=soup.find_all(class_=\"_1l9na3c align-self-center col\")[7].find(\"li\").text\n","            fuel_types.append(fuel_type)\n","            \n","            mileage=soup.find_all(class_=\"_1l9na3c align-self-center col\")[8].find(\"li\").text\n","            mileages.append(mileage)\n","            \n","            opt_level=soup.find_all(class_=\"_1l9na3c align-self-center col\")[9].find(\"li\").text\n","            opt_levels.append(opt_level)\n","            \n","            style=soup.find_all(class_=\"_1l9na3c align-self-center col\")[1].find(\"li\").text\n","            styles.append(style)\n","        \n","            loc=soup.find(class_=\"_6c9ih0 _b9xhv wrappable-2 text-md margin-top-1\").text\n","            locations.append(loc)\n","              \n","\n","    except (IndexError,TypeError,AttributeError):\n","            pass; \n"],"execution_count":10,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":["d={\"price\":prices,\"exterior_col\":ext_colors,\"interior_col\":int_colors,\"MPG\":MPGs,\n","    \"engine\":engines,\"transmission\":transmissions,\"drive_type\":drive_types,\"fuel_type\":fuel_types,\n","    \"mileage\":mileages,\"option_level\":opt_levels,\"style\":styles,\"location\":locations}\n","\n","\n","data=pd.DataFrame(dict([ (k,pd.Series(v)) for k,v in d.items() ]))   "],"execution_count":11,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":["len(data)"],"execution_count":27,"outputs":[{"output_type":"execute_result","execution_count":27,"data":{"text/plain":"9503"},"metadata":{}}]},{"metadata":{"trusted":true},"cell_type":"code","source":["#data_csv=data.to_csv(\"data_car.csv\",index=False)\n","#from IPython.display import FileLink\n","#FileLink(r'data_car.csv')"],"execution_count":1,"outputs":[]}],"metadata":{"kernelspec":{"name":"python3","display_name":"Python 3","language":"python"},"language_info":{"name":"python","version":"3.7.7-final","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":4}