{"cells":[{"metadata":{"_uuid":"b048389258cd3ae64de98594b42112ccceba7a25"},"cell_type":"markdown","source":" THIS NOTEBOOK IS BASED ON UNIVARIATE ANALYSIS."},{"metadata":{"trusted":false,"collapsed":true,"_uuid":"920ee0ceff11d8d2a5725b0863214251e288b1a2"},"cell_type":"code","source":"#Importing libraries\nimport pandas as pd\npd.set_option('display.max_columns', None)\nimport numpy as np\nimport scipy as sci\nimport seaborn as sns\nimport matplotlib.pyplot as plt\n%matplotlib inline\nfrom tqdm import tqdm","execution_count":1,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"90145918f22f0919170e2a67f8de4a1ab0d9216b"},"cell_type":"code","source":"test = pd.read_csv('../input/test.csv')\ntrain = pd.read_csv('../input/train.csv')","execution_count":2,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"d0c3d57419ea17edb6e3fdf882112b96216e68c7"},"cell_type":"code","source":"train.shape","execution_count":3,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"d2751543a4169c5f64576bfc3a535e8f495e1be1"},"cell_type":"code","source":"test.shape","execution_count":4,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"0c8f4c9795e5b48c6c78c2480fb8e17e83f77d3e"},"cell_type":"code","source":"train.head()","execution_count":5,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"e3da9c536ef8c382fcb3fa1700b15d0d2de2e9dc"},"cell_type":"code","source":"test.head()","execution_count":6,"outputs":[]},{"metadata":{"_uuid":"43b24baa550e2de538c6f736bb5dcd6a9234f1a8"},"cell_type":"markdown","source":"# MISSING VALUES"},{"metadata":{"trusted":false,"collapsed":true,"_uuid":"dd72ef304895dae7bbd4686412e4c96467032e20"},"cell_type":"code","source":"import missingno as msno","execution_count":7,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"24ff8caf706dbaae90734d88bb188d1c679618db"},"cell_type":"code","source":"msno.bar(train,figsize=(10,5))","execution_count":8,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"35f259dc356b9d645f8c72a8ce1614770976f990"},"cell_type":"code","source":"msno.bar(test,figsize=(10,5))","execution_count":9,"outputs":[]},{"metadata":{"_uuid":"90a694d20a46f50edddb1c5d01f5a505d230a9c1"},"cell_type":"markdown","source":"param_1 has some NaN's. Approximately, half of param_2 is NaN. 60% of param_3 is also NaN"},{"metadata":{"_uuid":"74fdc1929297eefd5e8040145ca36c110650c287"},"cell_type":"markdown","source":"# REGION"},{"metadata":{"trusted":false,"_uuid":"49c1c6ccb8438e478855c30d2ab23bf8e0000cf6"},"cell_type":"code","source":"print(\"no of unique values in region column of train data = \", len(set(train['region'])))","execution_count":10,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"c350b15cc2d109518d864d904dfc49ea072ded40"},"cell_type":"code","source":"print(\"no of unique values in region column of test data = \", len(set(test['region'])))","execution_count":11,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"32fe7de00d4dfca9493f1ba8d4fb870feeaef063"},"cell_type":"code","source":"f,ax=plt.subplots(1,2,figsize=(16,8))\ntrain['region'].value_counts().sort_values(ascending=True).plot.barh(width=0.9,color=sns.color_palette('afmhot',15),ax=ax[0])\ntest['region'].value_counts().sort_values(ascending=True).plot.barh(width=0.9,color=sns.color_palette('afmhot',15),ax=ax[1])\nplt.subplots_adjust(wspace=0.8)\nax[0].set_title('Train')\nax[1].set_title('Test')\nplt.show()","execution_count":12,"outputs":[]},{"metadata":{"trusted":false,"collapsed":true,"_uuid":"fcf6752e8b6e7754593f700339694ce1aedeaab4"},"cell_type":"code","source":"#Let's check the distribution of train and test in region column","execution_count":13,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"64474e6048f240cae985c21b3d4e2e4c218f65f6"},"cell_type":"code","source":"pd.Series(((train['region'].value_counts())/len(train))/((test['region'].value_counts())/len(test))).plot(kind='barh',title = 'Ratio of region column in train / Ratio of region column in test',figsize=(14,7))","execution_count":14,"outputs":[]},{"metadata":{"_uuid":"06e60be433e45e5cada49fe706d029ef435a4a59"},"cell_type":"markdown","source":" The distributions of train and test in region column look pretty similar"},{"metadata":{"_uuid":"d9e15e5da8bb84e686ea186ecfed500f805bdf16"},"cell_type":"markdown","source":"# CITY"},{"metadata":{"trusted":false,"_uuid":"3faf86cd04ecf9050bd34756a6b9b3a7dadc7665"},"cell_type":"code","source":"print(\"no of unique values in city column of train data = \", len(set(train['city'])))","execution_count":15,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"10599510c9e5d701673f37743b46bdfa7d5f15dc"},"cell_type":"code","source":"print(\"no of unique values in city column of test data = \", len(set(test['city'])))","execution_count":16,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"322f15935295b2d2a14bcfbf639138e8b585c443"},"cell_type":"code","source":"f,ax=plt.subplots(1,2,figsize=(16,8))\ntrain['city'].value_counts()[:50].sort_values(ascending=True).plot.barh(width=0.9,color=sns.color_palette('cool',15),ax=ax[0])\ntest['city'].value_counts()[:50].sort_values(ascending=True).plot.barh(width=0.9,color=sns.color_palette('cool',15),ax=ax[1])\nplt.subplots_adjust(wspace=0.8)\nax[0].set_title('Train')\nax[1].set_title('Test')\nplt.show()","execution_count":17,"outputs":[]},{"metadata":{"_uuid":"f89b152f8e2f1f9672537a50e344be30bdaefce3"},"cell_type":"markdown","source":"# Parent Cateogry Name"},{"metadata":{"trusted":false,"_uuid":"f7087a0fe67efa65ddc84ca830edb1073130e9fc"},"cell_type":"code","source":"print(\"no of unique values in parent_category_name column of train data = \", len(set(train['parent_category_name'])))","execution_count":18,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"e7a59d276cd517e3ee82e82f74ad687d80e88802"},"cell_type":"code","source":"print(\"no of unique values in parent_category_name column of test data = \", len(set(test['parent_category_name'])))","execution_count":19,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"731d8b22ea1553fe16163db309b3f2dc5fcf8a80"},"cell_type":"code","source":"f,ax=plt.subplots(1,2,figsize=(16,8))\ntrain['parent_category_name'].value_counts().sort_values(ascending=True).plot.barh(width=0.9,color=sns.color_palette('prism',15),ax=ax[0])\ntest['parent_category_name'].value_counts().sort_values(ascending=True).plot.barh(width=0.9,color=sns.color_palette('prism',15),ax=ax[1])\nplt.subplots_adjust(wspace=0.8)\nax[0].set_title('Train')\nax[1].set_title('Test')\nplt.show()","execution_count":20,"outputs":[]},{"metadata":{"_uuid":"a5b84db4a4ba51610175ffe5759cb9ae106cb5c7"},"cell_type":"markdown","source":"# Category Name"},{"metadata":{"trusted":false,"_uuid":"4471b2d2405a385ce6b87ce13d7b4419e770f2cf"},"cell_type":"code","source":"print(\"no of unique values in category_name column of train data = \", len(set(train['category_name'])))","execution_count":21,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"c9b59999e016ff2e92acc8c3ed31120eff6dde9c"},"cell_type":"code","source":"print(\"no of unique values in category_name column of test data = \", len(set(train['category_name'])))","execution_count":22,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"dda66c7514e4c4d83ee8e6e37d647824101c5029"},"cell_type":"code","source":"f,ax=plt.subplots(1,2,figsize=(16,8))\ntrain['category_name'].value_counts().sort_values(ascending=True).plot.barh(width=0.9,color=sns.color_palette('gist_heat',15),ax=ax[0])\ntest['category_name'].value_counts().sort_values(ascending=True).plot.barh(width=0.9,color=sns.color_palette('gist_heat',15),ax=ax[1])\nplt.subplots_adjust(wspace=0.8)\nax[0].set_title('Train')\nax[1].set_title('Test')\nplt.show()","execution_count":23,"outputs":[]},{"metadata":{"_uuid":"08dd51764eb423c6a5288ddf7c1fe7a49c319834"},"cell_type":"markdown","source":"# PARAM_1"},{"metadata":{"trusted":false,"_uuid":"bc7e795d0052cff11ee92c04a8e920af45ad81c9"},"cell_type":"code","source":"print(\"no of unique values in param_1 column of train data = \", len(set(train['param_1'])))","execution_count":24,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"89c11bc26b94f42a3afb6bed4820595addf88d41"},"cell_type":"code","source":"print(\"no of unique values in param_2 column of test data = \", len(set(test['param_1'])))","execution_count":25,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"b9e556b230223da1d6a77ca2449c52818dac0adb"},"cell_type":"code","source":"len([x for x in list(train['param_1'].value_counts()[:9].index) if x in list(test['param_1'].value_counts()[:8].index)])","execution_count":26,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"31b51cf6aa7a918642597a16b0fb7b92c33a6a03"},"cell_type":"code","source":"print(\"percentage of nan values in param_1 column in train data\", round(train['param_1'].isnull().sum()*100/len(train),3))","execution_count":27,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"2a29785246a601e0ab718b33e31b6410ed74ca9c"},"cell_type":"code","source":"print(\"percentage of nan values in param_1 column in test data\", round(test['param_1'].isnull().sum()*100/len(test),3))","execution_count":28,"outputs":[]},{"metadata":{"_uuid":"d056fe742fe400ce10422bc2ae7e31e04aa6f53e"},"cell_type":"markdown","source":" Percentage of NaN values in train and test is similar"},{"metadata":{"trusted":false,"_uuid":"9810ab8def7abaaad230cb46ab51c239bc5ec158"},"cell_type":"code","source":"f,ax=plt.subplots(1,2,figsize=(14,7))\ntrain['param_1'].value_counts()[:8].sort_values(ascending=True).plot.barh(width=0.9,color=sns.color_palette('gist_heat',15),ax=ax[0])\ntest['param_1'].value_counts()[:8].sort_values(ascending=True).plot.barh(width=0.9,color=sns.color_palette('gist_heat',15),ax=ax[1])\nplt.subplots_adjust(wspace=0.8)\nax[0].set_title('Train')\nax[1].set_title('Test')\nplt.show()","execution_count":29,"outputs":[]},{"metadata":{"_uuid":"87c948645efc1e10aad18ffd5d4a03194c5c610b"},"cell_type":"markdown","source":"# PARAM_2"},{"metadata":{"trusted":false,"_uuid":"6f0dbf12bcf3cf350fb11cd7ea565d204f0432aa"},"cell_type":"code","source":"pd.Series([len(set(train['param_2'])),len(set(test['param_2']))],index=['Number of unique values in train','Number of unique values in test']).plot(kind='bar',title='Counts of Uniques in train vs Counts of Uniques in test')","execution_count":30,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"e5d4593bde460b535b0d2db7159181e06d9d7ca7"},"cell_type":"code","source":"f,ax=plt.subplots(1,2,figsize=(14,7))\npd.Series([train['param_2'].isnull().sum(),train['param_2'].notnull().sum()],index=['Number of Nulls','Number of Not-Nulls']).plot(kind='pie',title='Counts of Nulls vs Counts of Not-Nulls in param_2 column in train',ax=ax[0])\npd.Series([test['param_2'].isnull().sum(),test['param_2'].notnull().sum()],index=['Number of Nulls','Number of Not-Nulls']).plot(kind='pie',title='Counts of Nulls vs Counts of Not-Nulls in param_2 column in test',ax=ax[1])\nplt.subplots_adjust(wspace=0.8)\nax[0].set_title('Train')\nax[1].set_title('Test')\nplt.show()","execution_count":31,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"0749532bb2389f50884d0f42942a90662cbbdfb9"},"cell_type":"code","source":"f,ax=plt.subplots(1,2,figsize=(16,8))\ntrain['param_2'].value_counts()[:10].sort_values(ascending=True).plot.barh(width=0.9,color=sns.color_palette('summer',15),ax=ax[0])\ntest['param_2'].value_counts()[:10].sort_values(ascending=True).plot.barh(width=0.9,color=sns.color_palette('summer',15),ax=ax[1])\nplt.subplots_adjust(wspace=0.8)\nax[0].set_title('Train')\nax[1].set_title('Test')\nplt.show()","execution_count":32,"outputs":[]},{"metadata":{"_uuid":"3dbed82e59d6aaa3c3af04e3ff0564affe43167b"},"cell_type":"markdown","source":"# Param_3"},{"metadata":{"trusted":false,"_uuid":"b0ba28cb08bd2260afc43b5d2086baae6433c7d4"},"cell_type":"code","source":"pd.Series([len(set(train['param_3'])),len(set(test['param_3']))],index=['Number of unique values in train','Number of unique values in test']).plot(kind='bar',title='Counts of Uniques in train vs Counts of Uniques in test')","execution_count":33,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"e4d4b1f2748ce4bea1e5497d3bd23521f6d1aa29"},"cell_type":"code","source":"f,ax=plt.subplots(1,2,figsize=(14,7))\npd.Series([train['param_3'].isnull().sum(),train['param_3'].notnull().sum()],index=['Number of Nulls','Number of Not-Nulls']).plot(kind='pie',title='Counts of Nulls vs Counts of Not-Nulls in param_2 column in train',ax=ax[0])\npd.Series([test['param_3'].isnull().sum(),test['param_3'].notnull().sum()],index=['Number of Nulls','Number of Not-Nulls']).plot(kind='pie',title='Counts of Nulls vs Counts of Not-Nulls in param_2 column in test',ax=ax[1])\nplt.subplots_adjust(wspace=0.8)\nax[0].set_title('Train')\nax[1].set_title('Test')\nplt.show()","execution_count":34,"outputs":[]},{"metadata":{"_uuid":"9318439641d77b9b52a342e68c2353a610ded6d7"},"cell_type":"markdown","source":"Most of param_3 column is NULL"},{"metadata":{"trusted":false,"_uuid":"5104b4a26969467f046599aedc9c53d94adcbb8e"},"cell_type":"code","source":"f,ax=plt.subplots(1,2,figsize=(16,8))\ntrain['param_3'].value_counts()[:10].sort_values(ascending=True).plot.barh(width=0.9,color=sns.color_palette('spring',15),ax=ax[0])\ntest['param_3'].value_counts()[:10].sort_values(ascending=True).plot.barh(width=0.9,color=sns.color_palette('spring',15),ax=ax[1])\nplt.subplots_adjust(wspace=0.8)\nax[0].set_title('Train')\nax[1].set_title('Test')\nplt.show()","execution_count":35,"outputs":[]},{"metadata":{"_uuid":"e6cf6ac74a2baea895548943e500442b12d1e267"},"cell_type":"markdown","source":"# PRICE"},{"metadata":{"trusted":false,"_uuid":"71daa3142c0fec720dec22bb6d7c05caa8434142"},"cell_type":"code","source":"#Checking NaN's\nf,ax=plt.subplots(1,2,figsize=(14,7))\npd.Series([train['price'].isnull().sum(),train['price'].notnull().sum()],index=['Number of Nulls','Number of Not-Nulls']).plot(kind='pie',title='Counts of Nulls vs Counts of Not-Nulls in param_2 column in train',ax=ax[0])\npd.Series([test['price'].isnull().sum(),test['price'].notnull().sum()],index=['Number of Nulls','Number of Not-Nulls']).plot(kind='pie',title='Counts of Nulls vs Counts of Not-Nulls in param_2 column in test',ax=ax[1])\nplt.subplots_adjust(wspace=0.8)\nax[0].set_title('Train')\nax[1].set_title('Test')\nplt.show()","execution_count":36,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"31e3c0e5fff60fd6d6858fea8527fc38664771a9"},"cell_type":"code","source":"sns.distplot(np.log(train['price'].dropna()+0.000001),kde=True,color='g',bins=20)","execution_count":59,"outputs":[]},{"metadata":{"_uuid":"ff8197bf60f83e4fa37d158ed737ec25f6a2f3be"},"cell_type":"markdown","source":"# ITEM SEQ NUMBER"},{"metadata":{"trusted":false,"_uuid":"6d560533ba4865e6c120a914ceed226ff98cd749"},"cell_type":"code","source":"train['item_seq_number'].describe()","execution_count":56,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"3850f0ffa5fe403fd61896a7c82b65891a8d74e8"},"cell_type":"code","source":"np.log(pd.concat([train['item_seq_number'],test['item_seq_number']])+0.0001).hist(bins=20)\nplt.title('Log-transformed Item SEQ Number Distribution')\nplt.show()","execution_count":65,"outputs":[]},{"metadata":{"_uuid":"02656455eb0ab3b969dd2f0f914355d3907ff94a"},"cell_type":"markdown","source":"# ACTIVATION DATE"},{"metadata":{"trusted":false,"_uuid":"f6feb08ee0d0d576a87718f7744e9ef987effc86"},"cell_type":"code","source":"plt.plot(train.groupby('activation_date').count()[['region']], 'o-', label='train')\nplt.plot(test.groupby('activation_date').count()[['region']], 'o-', label='test')\nplt.title('Train and test period not overlapping.')\nplt.legend(loc=0)\nplt.ylabel('number of records')\nplt.show()","execution_count":69,"outputs":[]},{"metadata":{"_uuid":"f7dfe9bbcc238fe2ebf1f39dfdac4818f72a8cf3"},"cell_type":"markdown","source":"Look interesting . Test period is after train data."},{"metadata":{"_uuid":"04092bb5d4271e9b6d3787766305bf3a85f28bb0"},"cell_type":"markdown","source":"# USER TYPE"},{"metadata":{"trusted":false,"_uuid":"6f418230b5efba3239dbf98c7c5176653e15cd70"},"cell_type":"code","source":"f,ax=plt.subplots(1,2,figsize=(16,8))\ntrain['user_type'].value_counts().sort_values(ascending=True).plot.barh(width=0.9,color=sns.color_palette('afmhot',15),ax=ax[0])\ntest['user_type'].value_counts().sort_values(ascending=True).plot.barh(width=0.9,color=sns.color_palette('afmhot',15),ax=ax[1])\nplt.subplots_adjust(wspace=0.8)\nax[0].set_title('Train')\nax[1].set_title('Test')\nplt.show()","execution_count":72,"outputs":[]},{"metadata":{"_uuid":"45f33ce651c33002298d834165e980463bd73870"},"cell_type":"markdown","source":"Look similar. Very Good."},{"metadata":{"_uuid":"333f44fc973c52eaf93fdc98694ed0c63e698a9f"},"cell_type":"markdown","source":"# USER_TOP_1"},{"metadata":{"trusted":false,"_uuid":"b4318763d1d9fedb0e0e83b262db3293821040fa"},"cell_type":"code","source":"train['image_top_1'].describe()","execution_count":75,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"562bfd10857a930a5f993d1cb6a70df31dddb128"},"cell_type":"code","source":"pd.concat([train[train['image_top_1'].notnull()]['image_top_1'],test[test['image_top_1'].notnull()]['image_top_1']]).hist(bins=20)\nplt.title('Histogram of image_top_1 Distribution')\nplt.show()","execution_count":81,"outputs":[]},{"metadata":{"_uuid":"c169ae0b0da51a69966d7ab22cf872068b552589"},"cell_type":"markdown","source":"# Deal Probability"},{"metadata":{"trusted":false,"_uuid":"ee041be2c03255ad4b5f1c69044954b53722c54c"},"cell_type":"code","source":"train['deal_probability'].describe()","execution_count":82,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"7e177fbfa1b4468cb4cc7a3c62aa890e7a986c92"},"cell_type":"code","source":"train['deal_probability'].hist(bins=5)","execution_count":86,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.6.3"}},"nbformat":4,"nbformat_minor":1}