{"cells":[{"metadata":{"_cell_guid":"5f9b1649-d4ff-4870-8edb-d3f69a4a4835","_uuid":"30bca7702ffa3d745e6ed724e8e96a7948490035"},"cell_type":"markdown","source":"

Import Data Set

\n"},{"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","collapsed":true,"scrolled":true,"trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nfrom transliterate import translit, get_available_language_codes\nimport os\n\n\ntrain_data = pd.read_csv(\"../input/train.csv\")\ntest_data = pd.read_csv(\"../input/test.csv\")\n\n#train_data.head(n=5)\n#print(os.listdir(\"../input/\"))\n","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"e8871bba-24f5-4d67-b1a0-c020fdb2e1c1","_uuid":"4229eed20c603a61250b79c3980b4a1002df1aa0"},"cell_type":"markdown","source":"

Data Preparation

\n\n

Translate the Russian characters into English using transliterate

"},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","collapsed":true,"trusted":true},"cell_type":"code","source":"region = (train_data['region']).apply(translit, 'ru', reversed=True)\ncity = (train_data['city']).apply(translit, 'ru', reversed=True)\nparent_category_name = (train_data['parent_category_name']).apply(translit, 'ru', reversed=True)\ncategory_name = (train_data['category_name']).apply(translit, 'ru', reversed=True)\nparam_1 = train_data['param_1'].apply(lambda row:row if pd.isnull(row) else translit(row, 'ru', reversed=True))\nparam_2 = train_data['param_2'].apply(lambda row:row if pd.isnull(row) else translit(row, 'ru', reversed=True))\nparam_3 = train_data['param_3'].apply(lambda row:row if pd.isnull(row) else translit(row, 'ru', reversed=True))\ntitle = train_data['title'].apply(lambda row:row if pd.isnull(row) else translit(row, 'ru', reversed=True))\ndescription = train_data['description'].apply(lambda row:row if pd.isnull(row) else translit(row, 'ru', reversed=True))\n\n\nregion_test = (test_data['region']).apply(translit, 'ru', reversed=True)\ncity_test = (test_data['city']).apply(translit, 'ru', reversed=True)\nparent_category_name_test = (test_data['parent_category_name']).apply(translit, 'ru', reversed=True)\ncategory_name_test = (test_data['category_name']).apply(translit, 'ru', reversed=True)\nparam_1_test = test_data['param_1'].apply(lambda row:row if pd.isnull(row) else translit(row, 'ru', reversed=True))\nparam_2_test = test_data['param_2'].apply(lambda row:row if pd.isnull(row) else translit(row, 'ru', reversed=True))\nparam_3_test = test_data['param_3'].apply(lambda row:row if pd.isnull(row) else translit(row, 'ru', reversed=True))\ntitle_test = test_data['title'].apply(lambda row:row if pd.isnull(row) else translit(row, 'ru', reversed=True))\ndescription_test = test_data['description'].apply(lambda row:row if pd.isnull(row) else translit(row, 'ru', reversed=True))\n","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"e58baa84-af63-4bec-9922-fe70639da5d8","_uuid":"bfc205a9a50d694d620a019b84dd9fa39bc9d337"},"cell_type":"markdown","source":"**Create the translated DataFrame**"},{"metadata":{"_cell_guid":"5a02fe58-82a8-4d25-a7c1-e0a2a413addd","_uuid":"d422f8df7fcb748ed385eb0815c7868b7923fe2d","collapsed":true,"trusted":true},"cell_type":"code","source":"train_data_translated = train_data\ntrain_data_translated['region'] = region\ntrain_data_translated['city'] = city\ntrain_data_translated['parent_category_name'] = parent_category_name\ntrain_data_translated['category_name'] = category_name\ntrain_data_translated['param_1'] = param_1\ntrain_data_translated['param_2'] = param_2\ntrain_data_translated['param_3'] = param_3\ntrain_data_translated['title'] = title\ntrain_data_translated['description'] = description\n\ntest_data_translated = test_data\ntest_data_translated['region'] = region_test\ntest_data_translated['city'] = city_test\ntest_data_translated['parent_category_name'] = parent_category_name_test\ntest_data_translated['category_name'] = category_name_test\ntest_data_translated['param_1'] = param_1_test\ntest_data_translated['param_2'] = param_2_test\ntest_data_translated['param_3'] = param_3_test\ntest_data_translated['title'] = title_test\ntest_data_translated['description'] = description_test","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"8576a094-fe1d-4865-bc19-5ca50ba9b915","_uuid":"0a3c3b7432a40fe9bf2b9c4d0179fc144ecdf01b"},"cell_type":"markdown","source":"**Output the translated DataFrame**"},{"metadata":{"_cell_guid":"ab7a038d-b437-4a11-b2df-272fc92aab4e","_uuid":"37aff29962e53c0b358825a1ba9459cbd1200f48","collapsed":true,"scrolled":true,"trusted":true},"cell_type":"code","source":"## @hidden_cell\n#train_data_translated.to_csv(\"train_translated.csv\")\n#train_data_translated = pd.read_csv(\"../input/translated/train_translated.csv\")\n#train_data_translated.head(n=5)\n","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"833aec60-6913-46cb-89f4-1c721685b872","_uuid":"6610bdeaa8366f3b04c60b81f220b31ce7e028ef"},"cell_type":"markdown","source":"

Data Analysis

\n\nDigging into the data to extract some inner insights of the data"},{"metadata":{"_cell_guid":"ed1ed38f-9cb6-402a-95a7-59f8d8828d21","_uuid":"57dfbe4af35b82fade1607c4298c9970be1fff6c"},"cell_type":"markdown","source":"

Feature Engineering

\n\nThe columns such as City, Region, Categories, Params, Title, Description has descriptive data in String format. So using feature engineering to classify the features of the Columns like City Region, Category. For the other fields like Title, Params, description we need a different kind of approach."},{"metadata":{"_cell_guid":"2c4a7c75-df88-43d4-8556-1ff2140b63b2","_uuid":"2b38f6970252b1eff5c11bc2ddd402f496feae0a","collapsed":true,"trusted":true},"cell_type":"code","source":"train_data_transformed_feature = train_data_translated\nsliced_data_for_feature_engg = train_data_translated.iloc[:, [2,3,4,5,6,7,8,13,14]]\nsliced_data_for_feature_engg = sliced_data_for_feature_engg.apply(lambda s: s.map({k:i for i,k in enumerate(s.unique())}))\ntrain_data_transformed_feature.iloc[:, [2,3,4,5,6,7,8,13,14]] = sliced_data_for_feature_engg\n#train_data_transformed_feature.head(5)\n\ntest_data_transformed_feature = test_data_translated\nsliced_data_for_feature_engg = test_data_translated.iloc[:, [2,3,4,5,6,7,8,13,14]]\nsliced_data_for_feature_engg = sliced_data_for_feature_engg.apply(lambda s: s.map({k:i for i,k in enumerate(s.unique())}))\ntest_data_transformed_feature.iloc[:, [2,3,4,5,6,7,8,13,14]] = sliced_data_for_feature_engg\ntest_data_transformed_feature.head(5)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"d3848122-6a01-4082-9fac-d92757e1be87","_uuid":"57fa11ecf937eea130cadc1e37c29effa43d09eb"},"cell_type":"markdown","source":"**Show correlation plot on quantified fileds.**"},{"metadata":{"_cell_guid":"cdefd977-ed0c-49b5-a97f-78a8539acf61","_uuid":"d1190c6feda47195f923296686afc7afbe2a3770","collapsed":true,"scrolled":true,"trusted":true},"cell_type":"code","source":"from string import ascii_letters\nimport seaborn as sns\nimport matplotlib.pyplot as plt\n\nsns.set(style=\"white\")\ncorr = train_data_transformed_feature.iloc[:, [3,4,5,6,7,8,11,12,13,14,16,17]].corr()\n\nmask = np.zeros_like(corr, dtype=np.bool)\nmask[np.triu_indices_from(mask)] = True\n\n# Set up the matplotlib figure\nf, ax = plt.subplots(figsize=(11, 9))\n\n# Generate a custom diverging colormap\ncmap = sns.diverging_palette(220, 10, as_cmap=True)\n\n# Draw the heatmap with the mask and correct aspect ratio\nsns.heatmap(corr, mask=mask, cmap=cmap, vmax=1, vmin=-1, center=0,\n square=True, linewidths=.5, cbar_kws={\"shrink\": .5})","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"b35fea3c-f7b4-4e74-8887-09ffd8ea7513","_uuid":"949ab092b516a356788f3912305d0267e0cacb8f","collapsed":true,"trusted":true},"cell_type":"code","source":"sns.pairplot(train_data_transformed_feature.iloc[:, [3,4,5,6,7,8,13,14]])\n","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.5","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}