{"metadata":{"language_info":{"name":"python","pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py","version":"3.6.5","codemirror_mode":{"version":3,"name":"ipython"},"mimetype":"text/x-python"},"kernelspec":{"name":"python3","language":"python","display_name":"Python 3"}},"cells":[{"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","collapsed":true,"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","trusted":false},"execution_count":null,"source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","outputs":[],"cell_type":"code"},{"metadata":{"_cell_guid":"c7899f03-2264-4f66-b614-9012bbc628b8","_uuid":"1d9ccd368d95cb517a12c32375e09b680517f4f8"},"cell_type":"markdown","source":"This one is how I scraped regional information - population, time zone, density, etc - from Wikipedia. Two things to note: first to do this, I converted all the region names into English. (I referred to one of the public Kernal to do this). Second, this code acually does not work on kaggle kernal (seems like web scraping is not available on kaggle kernal) so I scraped it through my own kernal, sent it to csv, and re-loaded it on kaggle kernal. (I attatch the result as a csv file too.)  Help it would be useful to everyone!"},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","collapsed":true,"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":false},"execution_count":null,"source":"import pandas as pd\nimport numpy as np\nfrom bs4 import BeautifulSoup\nimport urllib\nfrom urllib import request\nimport re\n\nWiki_url = ['https://en.wikipedia.org/wiki/Sverdlovsk_Oblast',\n 'https://en.wikipedia.org/wiki/Samara_Oblast',\n 'https://en.wikipedia.org/wiki/Rostov_Oblast',\n 'https://en.wikipedia.org/wiki/Tatarstan',\n 'https://en.wikipedia.org/wiki/Volgograd_Oblast',\n 'https://en.wikipedia.org/wiki/Nizhny Novgorod_Oblast',\n 'https://en.wikipedia.org/wiki/Perm_Krai',\n 'https://en.wikipedia.org/wiki/Orenburg_Oblast',\n 'https://en.wikipedia.org/wiki/Khanty-Mansi Autonomous_Okrug',\n 'https://en.wikipedia.org/wiki/Bashkortostan',\n 'https://en.wikipedia.org/wiki/Krasnodar_Krai',\n 'https://en.wikipedia.org/wiki/Novosibirsk_Oblast',\n 'https://en.wikipedia.org/wiki/Omsk_Oblast',\n 'https://en.wikipedia.org/wiki/Chelyabinsk_Oblast',\n 'https://en.wikipedia.org/wiki/Voronezh_Oblast',\n 'https://en.wikipedia.org/wiki/Kemerovo_Oblast',\n 'https://en.wikipedia.org/wiki/Saratov_Oblast',\n 'https://en.wikipedia.org/wiki/Vladimir_Oblast',\n 'https://en.wikipedia.org/wiki/Krasnoyarsk_Krai',\n 'https://en.wikipedia.org/wiki/Belgorod_Oblast',\n 'https://en.wikipedia.org/wiki/Yaroslavl_Oblast',\n 'https://en.wikipedia.org/wiki/Kaliningrad_Oblast',\n 'https://en.wikipedia.org/wiki/Tyumen_Oblast',\n 'https://en.wikipedia.org/wiki/Udmurtia',\n 'https://en.wikipedia.org/wiki/Altai_Krai',\n 'https://en.wikipedia.org/wiki/Irkutsk_Oblast',\n 'https://en.wikipedia.org/wiki/Stavropol_Krai',\n 'https://en.wikipedia.org/wiki/Tula_Oblast']\n\nwant=[\"Density\", \"Time zone\", \"Rural\", \"Urban\", \"Total\"]\ndictionary = {}\n\ndef Scrape_info(url):\n    wiki_url = url\n    udr = {'User-Agent': 'Mozilla/5.0'}\n    try:\n        page = urllib.request.urlopen(wiki_url).read()\n        soup = BeautifulSoup(page, \"html.parser\")\n        table = soup.find('table', class_='infobox geography vcard')\n    except:\n        table = None\n    \n    country = url.split(\"/\")[-1]\n    \n    if table is not None:\n        result = {}\n    \n        exceptional_row_count = 0\n        for tr in table.find_all('tr'):\n            if tr.find('th'):\n                if tr.find(\"td\"):\n                    result[tr.find('th').text] = tr.find('td').text\n    \n \n        dictionary[country]={}\n        for i in result.keys():\n            for j in want:\n                if j in i:\n                    dictionary[country][j] = result[i]\n    elif  table is None:\n        dictionary[country] = {}\n        for i in want:\n            dictionary[country][i] = {np.nan}\n                \nfor i in Wiki_url:\n    Scrape_info(i)\n\nreg = list(dictionary.keys())\nvals = [dictionary[reg[i]] for i in range(len(reg)) ]\n\nregional_data = pd.DataFrame(vals, index = reg)\n\ndens = np.array([float(i.split()[0].split(\"/\")[0]) for i in regional_data[\"Density\"].values])\nrul = np.array([float(i.split(\"%\")[0]) for i in regional_data[\"Rural\"]])\ntim = np.array([i.split()[0] for i in regional_data[\"Time zone\"]])\ntot = np.array([int(\"\".join(i.split()[0].split(\"[\")[0].split(\",\"))) for i in regional_data[\"Total\"].values])\nurb = np.array([float(i.split(\"%\")[0]) for i in regional_data[\"Urban\"]])\n\nregional_data[\"Density_of_region(km2)\"] = dens\nregional_data[\"Rural_%\"] = rul\nregional_data[\"Time_zone\"] = tim\nregional_data[\"Total_population\"] = tot\nregional_data[\"Urban%\"] = urb\n\ndel regional_data[\"Density\"]\ndel regional_data[\"Rural\"]\ndel regional_data[\"Time zone\"]\ndel regional_data[\"Total\"]\ndel regional_data[\"Urban\"]\n\nregional_data.to_csv(\"/Users/HongSukhyun/Desktop/Python/Kaggle competition/Avito/regional.csv\", encoding = \"utf-8\")\n","outputs":[],"cell_type":"code"}],"nbformat_minor":1,"nbformat":4}