{"cells":[{"metadata":{"collapsed":true,"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","trusted":false},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"8ab677c0-96b6-4395-8660-7dafdc5ac789","_uuid":"6ee107beabac4ee6308c0878bc2da419d461844c"},"cell_type":"markdown","source":"This may not directly be availble on Kaggle kernal (because it seems like kaggle kernal does not allow web scraping). So I scraped this on my own kernal, and re-loaded the csv file. I attatch the scraped result. "},{"metadata":{"collapsed":true,"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":false},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nfrom bs4 import BeautifulSoup\nimport urllib\nfrom urllib import request\nimport re\n\nWiki_url = ['https://en.wikipedia.org/wiki/Sverdlovsk_Oblast',\n 'https://en.wikipedia.org/wiki/Samara_Oblast',\n 'https://en.wikipedia.org/wiki/Rostov_Oblast',\n 'https://en.wikipedia.org/wiki/Tatarstan',\n 'https://en.wikipedia.org/wiki/Volgograd_Oblast',\n 'https://en.wikipedia.org/wiki/Nizhny Novgorod_Oblast',\n 'https://en.wikipedia.org/wiki/Perm_Krai',\n 'https://en.wikipedia.org/wiki/Orenburg_Oblast',\n 'https://en.wikipedia.org/wiki/Khanty-Mansi Autonomous_Okrug',\n 'https://en.wikipedia.org/wiki/Bashkortostan',\n 'https://en.wikipedia.org/wiki/Krasnodar_Krai',\n 'https://en.wikipedia.org/wiki/Novosibirsk_Oblast',\n 'https://en.wikipedia.org/wiki/Omsk_Oblast',\n 'https://en.wikipedia.org/wiki/Chelyabinsk_Oblast',\n 'https://en.wikipedia.org/wiki/Voronezh_Oblast',\n 'https://en.wikipedia.org/wiki/Kemerovo_Oblast',\n 'https://en.wikipedia.org/wiki/Saratov_Oblast',\n 'https://en.wikipedia.org/wiki/Vladimir_Oblast',\n 'https://en.wikipedia.org/wiki/Krasnoyarsk_Krai',\n 'https://en.wikipedia.org/wiki/Belgorod_Oblast',\n 'https://en.wikipedia.org/wiki/Yaroslavl_Oblast',\n 'https://en.wikipedia.org/wiki/Kaliningrad_Oblast',\n 'https://en.wikipedia.org/wiki/Tyumen_Oblast',\n 'https://en.wikipedia.org/wiki/Udmurtia',\n 'https://en.wikipedia.org/wiki/Altai_Krai',\n 'https://en.wikipedia.org/wiki/Irkutsk_Oblast',\n 'https://en.wikipedia.org/wiki/Stavropol_Krai',\n 'https://en.wikipedia.org/wiki/Tula_Oblast']\n\nwant=[\"Density\", \"Time zone\", \"Rural\", \"Urban\", \"Total\"]\ndictionary = {}\n\ndef Scrape_info(url):\n    wiki_url = url\n    udr = {'User-Agent': 'Mozilla/5.0'}\n    try:\n        page = urllib.request.urlopen(wiki_url).read()\n        soup = BeautifulSoup(page, \"html.parser\")\n        table = soup.find('table', class_='infobox geography vcard')\n    except:\n        table = None\n    \n    country = url.split(\"/\")[-1]\n    \n    if table is not None:\n        result = {}\n    \n        exceptional_row_count = 0\n        for tr in table.find_all('tr'):\n            if tr.find('th'):\n                if tr.find(\"td\"):\n                    result[tr.find('th').text] = tr.find('td').text\n    \n \n        dictionary[country]={}\n        for i in result.keys():\n            for j in want:\n                if j in i:\n                    dictionary[country][j] = result[i]\n    elif  table is None:\n        dictionary[country] = {}\n        for i in want:\n            dictionary[country][i] = {np.nan}\n                \nfor i in Wiki_url:\n    Scrape_info(i)\n\nreg = list(dictionary.keys())\nvals = [dictionary[reg[i]] for i in range(len(reg)) ]\n\nregional_data = pd.DataFrame(vals, index = reg)\n\ndens = np.array([float(i.split()[0].split(\"/\")[0]) for i in regional_data[\"Density\"].values])\nrul = np.array([float(i.split(\"%\")[0]) for i in regional_data[\"Rural\"]])\ntim = np.array([i.split()[0] for i in regional_data[\"Time zone\"]])\ntot = np.array([int(\"\".join(i.split()[0].split(\"[\")[0].split(\",\"))) for i in regional_data[\"Total\"].values])\nurb = np.array([float(i.split(\"%\")[0]) for i in regional_data[\"Urban\"]])\n\nregional_data[\"Density_of_region(km2)\"] = dens\nregional_data[\"Rural_%\"] = rul\nregional_data[\"Time_zone\"] = tim\nregional_data[\"Total_population\"] = tot\nregional_data[\"Urban%\"] = urb\n\ndel regional_data[\"Density\"]\ndel regional_data[\"Rural\"]\ndel regional_data[\"Time zone\"]\ndel regional_data[\"Total\"]\ndel regional_data[\"Urban\"]\n\nregional_data.to_csv(\"/Users/HongSukhyun/Desktop/Python/Kaggle competition/Avito/regional.csv\", encoding = \"utf-8\")\n","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.5","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}