{"cells":[{"metadata":{"_uuid":"e290ab48b2d40e590a1484dfb6ae12bde7b51dd5"},"cell_type":"markdown","source":"# Avito Demand Prediction - Data Exploration\n*Basis notebook covering the training and test data based on distributions, time series and relationships with the target variable.*"},{"metadata":{"trusted":true,"_uuid":"b67f64aa333b370eed0cb1d640ff1332b3916c48"},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\n# from dataninja import kick\nimport altair as alt\nimport seaborn as sns\nimport cufflinks as cf\nimport plotly.offline as plotly\n%matplotlib inline\n\ncf.go_offline()","execution_count":1,"outputs":[]},{"metadata":{"_uuid":"b8bb313193b9c7764e13442a9aa6ba02c47601d1"},"cell_type":"markdown","source":"### Import Data"},{"metadata":{"trusted":true,"_uuid":"2e71ab4a2eb1e268f24f0b3c7d2a0657db5076c3"},"cell_type":"code","source":"%%time\ndtypes = {'user_type':'category',\n          'category_name':'category',\n          'parent_category_name':'category',\n          'region':'category',\n          'city':'category'}\n\ntrain = pd.read_csv(\"../input/train.csv\",\n                    parse_dates=['activation_date'],\n                    dtype=dtypes)\ntest = pd.read_csv(\"../input/test.csv\",\n                   parse_dates=['activation_date'],\n                   dtype=dtypes)\n\ntarget = 'deal_probability'","execution_count":3,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"41fb621b29bee6424dda5b92e3a97e601e46fd69"},"cell_type":"code","source":"all_data = pd.concat([train.assign(dataset='train'),test.assign(dataset='test')],sort=False).reset_index(drop=True)","execution_count":4,"outputs":[]},{"metadata":{"_uuid":"718df7b872f9abc8e9017f7c99694b6a08b2afbd"},"cell_type":"markdown","source":"### Exploration"},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"beb6157fa58fa51b71c5170f5befb2315309a3a6"},"cell_type":"code","source":"dataset_colors = ['red','orange']","execution_count":5,"outputs":[]},{"metadata":{"_uuid":"1bc26b70615b621bf8afd27230c3eb3fe42f443a"},"cell_type":"markdown","source":"#### Unique Values"},{"metadata":{"trusted":true,"_uuid":"9d7b0dba488c4165d26923ade2354f283fe60877"},"cell_type":"code","source":"(all_data\n .groupby('dataset')\n .nunique()\n .unstack()\n .sort_values(ascending=False)\n .unstack()\n .sort_values('test',ascending=False)\n .iloc[8:]\n .drop([target]+['dataset'])\n .iplot(title='Unique Value Counts for Categoricals by Dataset',kind='bar',colors=dataset_colors)\n)","execution_count":6,"outputs":[]},{"metadata":{"_uuid":"a4e9b50b52a4811cc7c23247c0ed66f4dfc08871"},"cell_type":"markdown","source":"#### Missing Values "},{"metadata":{"trusted":true,"_uuid":"307cde099e463f403e0effc02425a4b6325c7921"},"cell_type":"code","source":"#features with missing values\nmissing_vals = all_data.isnull().sum()\nmissing_vals = missing_vals[missing_vals>0]\nmissing_vals = missing_vals.index.tolist()\nmissing_vals.remove(target)\n\nobs_counts = all_data.groupby('dataset').size().reset_index(name='obs_count')\n\n(all_data\n .groupby('dataset')\n .apply(lambda x: x.isnull().sum())\n .merge(obs_counts,left_index=True,right_on='dataset')\n .set_index('dataset')\n .loc[:,missing_vals+['obs_count']]\n .transform(lambda x: x/x.max(),axis=1)\n .drop('obs_count',axis=1)\n .T\n .iplot(kind='bar',title='Missing Values Comparison - Train vs. Test',colors=dataset_colors)\n)","execution_count":7,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2075001f5cf750295baeb9e6c95e5a7119c33e5f"},"cell_type":"code","source":"(all_data\n .set_index('dataset')\n .loc[:,missing_vals]\n .isnull().sum(axis=1)\n .reset_index(name='item_missing_val_count')\n .groupby(['dataset','item_missing_val_count'])\n .size().reset_index(name='item_missing_val_count_count')\n .merge(obs_counts,on='dataset')\n .assign(item_missing_val_count_scaled = lambda x: x.item_missing_val_count_count/x.obs_count)\n .set_index(['dataset','item_missing_val_count']).item_missing_val_count_scaled\n .unstack(0)\n .iplot(title='Missing Values per Item Comparisons',kind='bar',colors=dataset_colors)\n)","execution_count":8,"outputs":[]},{"metadata":{"_uuid":"982025e124c6b9ad8c2199744bdacf57aad845cf"},"cell_type":"markdown","source":"#### Target Variable Averages by Features"},{"metadata":{"trusted":true,"_uuid":"e8cbb49662774a7b1674dc492abb7bdd09e14bd0"},"cell_type":"code","source":"train.groupby('user_type')[target].mean().sort_values().iplot(mode='markers+lines',title='Average Deal Probability by User Type',color='blue')","execution_count":9,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"939313eaf56616b80947fecc6fcd1f1932082ccb"},"cell_type":"code","source":"train.groupby('parent_category_name')[target].mean().sort_values().iplot(mode='markers+lines',title='Average Deal Probability by Parent Category',color='green')","execution_count":10,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"83d46c6adfb7f7c0b6829943d014994fcf0eee54"},"cell_type":"code","source":"train.groupby('category_name')[target].mean().sort_values().iplot(mode='markers+lines',title='Average Deal Probability by Category',color='gold',margin={'b':120})","execution_count":11,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"afb31cbbec3c8dc86b60b69dec76e7da48e78976"},"cell_type":"code","source":"train.groupby('region')[target].mean().sort_values().iplot(mode='markers+lines',title='Average Deal Probability by Region',color='magenta')","execution_count":12,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9b015eb8708d12f15eacbe7085762d8703d61ffa"},"cell_type":"code","source":"train.query('price<5000000').groupby('price')[target].mean().rolling(1000,min_periods=50).mean().iplot(title='Deal Probabilty by Price',color='purple')","execution_count":13,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a3261c63ac4190093e5eab73f577abf6394dde22"},"cell_type":"code","source":"train.groupby('image_top_1')[target].mean().rolling(1000,min_periods=50).mean().iplot(title='Deal Probabilty by Image Top 1',color='black')","execution_count":14,"outputs":[]},{"metadata":{"_uuid":"f58a0714aef78edf90d338d6a729d4601ab3576d"},"cell_type":"markdown","source":"#### Time Series Analysis"},{"metadata":{"trusted":true,"_uuid":"fd066fc76dc632b218ee0dda8915374cb246b0c0"},"cell_type":"code","source":"all_data.groupby(['dataset','activation_date']).size().unstack(0).iplot(title='Train vs. Test by Activation Date',colors=dataset_colors)","execution_count":15,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ed3982a5d90244a2cbf4a6a26e734da016e21b26"},"cell_type":"code","source":"fig, ax = plt.subplots(figsize=(6,6))\nsns.heatmap(data = all_data\n            .assign(weekday = lambda x: x.activation_date.dt.weekday,\n                    week = lambda x: x.activation_date.dt.week)\n            .groupby(['weekday','week'])\n            .size()\n            .unstack(0),\n            cmap='viridis',\n            ax=ax\n            )\nplt.title('Listings by Week and Day');","execution_count":16,"outputs":[]},{"metadata":{"_uuid":"71872f768d725efacee0c4d52297a48d1932d6ee"},"cell_type":"markdown","source":"#### Feature Distribution"},{"metadata":{"trusted":true,"_uuid":"9c8daff95049f02285d963ad0a5a17a423bcd80a"},"cell_type":"code","source":"all_data.description.str.len().hist(bins=50,color='black',figsize=(10,6))\nplt.title('Description Length Distribution');","execution_count":17,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e6c077aad6c228ed268185ccd500d381d4c960a3"},"cell_type":"code","source":"all_data.title.str.len().hist(bins=50,color='gold',figsize=(10,6))\nplt.title('Title Length Distribution');","execution_count":18,"outputs":[]},{"metadata":{"_uuid":"9c1553326505d4af6780c2d20df2766d8131ed0f"},"cell_type":"markdown","source":"#### Target Distribution "},{"metadata":{"trusted":true,"_uuid":"a7c201e42739f7aa887f4db6a5ffcab5159dad9a"},"cell_type":"code","source":"train[target].iplot(kind='hist',bins=20,title='Deal Probability Distribution')","execution_count":19,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"873eb1348c77a9a39d6f0218e43a7ba4e67f3e92"},"cell_type":"code","source":"train[target].value_counts().transform(lambda x: x/x.sum()).head(10).round(2)","execution_count":20,"outputs":[]},{"metadata":{"_uuid":"39e885ae9a37ceb6e1633c310b00de0bc24edc43"},"cell_type":"markdown","source":"Deal probabilities **potentially created with a non-continuous model such as a GBM** as many listings share duplicate deal probabilities indicating they **likely belong in the same prediction bin**."}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.6.5"}},"nbformat":4,"nbformat_minor":1}