{"cells":[
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": true
  },
  "outputs": [],
  "source": "%matplotlib inline"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "%matplotlib inline\n\nfrom __future__ import division\nimport datetime\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom scipy.stats import pearsonr, spearmanr, probplot\nfrom sklearn.cross_validation import StratifiedKFold\nfrom sklearn.feature_selection import VarianceThreshold, SelectKBest, chi2, f_classif, RFECV\nfrom sklearn.svm import LinearSVC, SVC\nfrom sklearn import preprocessing\n\npd.set_option('display.max_columns', 1800)\npd.set_option('display.width', 1800)"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "train = pd.read_csv('../input/train.csv')\n#train = DataSet(train_data, target_column='y', id_column='id')\nprint(train[:5])\n\nstore = pd.read_csv('../input/store.csv')\nprint(store[:5])\n\nall_data = pd.merge(train, store, on='Store', how='left')\n\ntest = pd.read_csv('../input/test.csv')"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "# clean StateHoliday column\nall_data['StateHoliday'][all_data['StateHoliday'] == 0 ] = '0'"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "# a couple of statistics\nprint('number of stores: {}'.format(train.Store.size))\n\n# count open stores by week day\nprint(train.groupby(['DayOfWeek']).sum())\nprint(all_data[['DayOfWeek', 'Open', 'Sales', 'Customers']].groupby(['DayOfWeek', 'Open']).agg([np.sum, np.mean, np.std]))"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "avg_per_store = all_data[['Sales', 'Store']].groupby('Store').mean()\navg_per_store.reset_index().plot(kind='scatter', x='Store', y='Sales')"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "avg_per_weekday = all_data[['Sales', 'DayOfWeek']].groupby('DayOfWeek').mean()\navg_per_weekday.reset_index().plot(kind='bar', x='DayOfWeek', y='Sales')"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "all_data[['Customers', 'Sales']].plot(kind='scatter', x='Customers', y='Sales')"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "np.log(all_data[['Customers', 'Sales']]).plot(kind='scatter', x='Customers', y='Sales')"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "avg_promotion = all_data[['Sales', 'Customers', 'Promo']].groupby('Promo').mean()\navg_promotion.plot(kind='bar')"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "avg_stateholiday = all_data[['Sales', 'Customers', 'StateHoliday']].groupby('StateHoliday').mean()\navg_stateholiday.plot(kind='bar')"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "avg_stateholiday = all_data[['Sales', 'Customers', 'SchoolHoliday']].groupby('SchoolHoliday').mean()\navg_stateholiday.plot(kind='bar')"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "avg_stateholiday = all_data[['Sales', 'Customers', 'StoreType']].groupby('StoreType').mean()\navg_stateholiday.plot(kind='bar')"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "avg_stateholiday = all_data[['Sales', 'Customers', 'Assortment']].groupby('Assortment').mean()\navg_stateholiday.plot(kind='bar')"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "all_data[['CompetitionDistance', 'Sales']].plot(kind='scatter', x='CompetitionDistance', y='Sales')"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "all_data.hist('CompetitionDistance')"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "# Bin the competition distance with 10 bins...\nbins = np.linspace(all_data['CompetitionDistance'].min(), all_data.CompetitionDistance.max(), 10)\n\ncompetition_bins = all_data[['Sales', 'Customers']].groupby(np.digitize(all_data['CompetitionDistance'], bins))\ncompetition_avg = competition_bins.mean()\ncompetition_avg.plot(kind='bar')"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "competition_bins = all_data[['Sales', 'Customers']].groupby(pd.cut(all_data['CompetitionDistance'], bins))\ncompetition_avg = competition_bins.mean()\ncompetition_avg.plot(kind='bar')"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "competition_bins = all_data[['Sales', 'Customers']].groupby(np.digitize(all_data['CompetitionDistance'], bins))\ncompetition_avg = competition_bins.count()\ncompetition_avg.plot(kind='bar')"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "competition_bins = all_data[['Sales', 'Customers']].groupby(pd.cut(all_data['CompetitionDistance'], bins))\ncompetition_avg = competition_bins.count()\ncompetition_avg.plot(kind='bar')"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "# average sales and customers by month \nall_data['MMYYYY'] = all_data['Date'].map(lambda x: x[:7])\n\n\navg_hist_by_month = all_data[['Sales', 'Customers', 'MMYYYY']].groupby('MMYYYY').mean()\navg_hist_by_month.plot(kind='bar')"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "# average sales and customers by month\nall_data['Months'] = all_data['Date'].map(lambda x: x[5:7])\n\navg_hist_by_month = all_data[['Sales', 'Customers', 'Months']].groupby('Months').mean()\navg_hist_by_month.plot(kind='bar')"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "# average sales and customers by month day\nall_data['MonthDay'] = all_data['Date'].map(lambda x: x[8:])\n\navg_hist_by_month = all_data[['Sales', 'Customers', 'MonthDay']].groupby('MonthDay').mean()\navg_hist_by_month.plot(kind='bar')"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "# average sales and customers by week day\nall_data['WeekDay'] = all_data['Date'].apply(lambda x: datetime.datetime.strptime(x, \"%Y-%m-%d\").date().weekday())\navg_hist_by_month = all_data[['Sales', 'Customers', 'WeekDay']].groupby('WeekDay').mean()\navg_hist_by_month.plot(kind='bar')"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "# average sales by week day by promo\n\navg_hist_by_month = all_data[['Sales', 'Customers', 'Promo', 'WeekDay']].groupby(['WeekDay', 'Promo']).mean()\nsns.barplot(x=\"WeekDay\", y=\"Sales\", hue=\"Promo\", order=[0, 1, 2, 3, 4, 5, 6], data=all_data)"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "# unique values in train\ntrain.apply(lambda x: len(x.unique()))"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "# unique values in test\ntest.apply(lambda x: len(x.unique()))"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "# since there's 3 open unique values\n# let's check which store has the extra value\ntest[pd.isnull(test.Open)]"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "print('''\n        unique train stores: {} \n        unique test stores: {} \n        are alltest stores in train stores: {}\n        extra train stores: {}\n      '''.format(\n        len(pd.unique(train.Store)),\n        len(pd.unique(test.Store)),\n        'yes' if set(pd.unique(test.Store)) - set(pd.unique(train.Store)) else 'no',\n        len(set(pd.unique(train.Store)) - set(pd.unique(test.Store)))\n    ))"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "# the fraction of open/closed stores train\nopen_close_frac = pd.value_counts(train.Open) / train.Open.size\nopen_close_frac.plot(kind='bar')\nopen_close_frac"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "# the fraction of open/closed stores test\nopen_close_frac = pd.value_counts(test.Open) / test.Open.size\nopen_close_frac.plot(kind='bar')\nopen_close_frac"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "# the fraction of promo/no promo stores\npromo_frac = pd.value_counts(train.Promo) / train.Promo.size\npromo_frac.plot(kind='bar')\npromo_frac"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "# the fraction of promo/no promo stores\npromo_frac = pd.value_counts(test.Promo) / test.Promo.size\npromo_frac.plot(kind='bar')\npromo_frac"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "print ('sales when stores are open')\nprint (pd.value_counts(train[train.Open == 1].Sales > 0))\nprint ('----')\nprint ('sales when stores are closes')\nprint (pd.value_counts(train[train.Open == 0].Sales > 0))"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "print ('sales when stores are on promo')\nprint (pd.value_counts(train[train.Promo == 1].Sales > 0))\nprint ('----')\nprint ('sales when stores have no promo')\nprint (pd.value_counts(train[train.Promo == 0].Sales > 0))"
 }
],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"}}, "nbformat": 4, "nbformat_minor": 0}