{"cells":[{"metadata":{"_cell_guid":"7be592ad-b017-45b9-bf40-4650297932f3","_uuid":"2c255f9a7fc421162a8f84ff50e9d7b6ed4fcbb7"},"cell_type":"markdown","source":"# It starts with wanting to get an accurate score without having to submit to the LB everytime.\n\nLike all of you I'd love to train my model with scores that will correspond to the LB, reasonably enough to be somewhat between the private and public one.\nI thought I'd use the last day of training data for (the 9th) for validation, but [this post](https://www.kaggle.com/cpmpml/ip-download-rates) opened my eyes on the tricks the organizers have pulled. \n\nIf you're like me, you thought : \"shit just got even more exciting!\" and investigated further. (if you do get excited, don't hesitate to fork the kernel, the graphs will be much bigger in the editing mode)\n\nTo make a validation set of the 9th resembling the test data, I selected the same hours and ip range. We'll call this dataset **validation_set**. Let's analyse the difference between this and the real test data. We wrote a class called **test_val_compare** that has many functions to analyse the two sets. If you want to see the code, just expand the windows below:"},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_kg_hide-input":true,"_kg_hide-output":true,"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\n%matplotlib inline\nimport matplotlib.pyplot as plt\nfrom matplotlib.colors import ListedColormap\nimport seaborn as sns\nsns.set()\nimport plotly.offline as py\npy.init_notebook_mode(connected=True)\n\n# set-up\nlocal_path='../input/8th-as-train-data-9th-as-test-data/'\nkaggle_path='../input/talkingdata-adtracking-fraud-detection/'\n\ndef load_data(name,rows=None):\n    ''' Load the csv files into a TimeSeries dataframe with minimal data types to reduce the used RAM space.\n    Arg:\n        -name (str): train, train_sample or test\n    Returns:\n        pd.DataFrame\n    '''\n    \n    # Defining dtypes\n    types = {'ip':np.uint32,'app': np.uint16,'os': np.uint16,'device': np.uint16,'channel':np.uint16,'click_time': object}\n    \n    if name=='test':\n        types['click_id']= np.uint32\n    else:\n        types['is_attributed']='bool'\n    \n    # Defining reading arguments\n    read_args={'nrows':rows,'parse_dates':['click_time'],'infer_datetime_format':True,'index_col':'click_time','usecols':list(types.keys()),'dtype':types,\n        'engine':'c',\n        'sep':','\n        }\n    \n    # Setting file path and compression type\n    if name in['train','test']:\n        file_path='{}{}.csv'.format(kaggle_path,name)\n    else:\n        file_path='{}{}.csv.zip'.format(local_path,name)\n        read_args['compression']='gzip'\n    \n    # Reading file and setting the timezone \n    with open(file_path,'rb') as File:\n        data=(pd\n            .read_csv(File,**read_args)\n            .tz_localize('UTC')\n            .tz_convert('Asia/Shanghai')\n            .reset_index()\n        )\n\n    return data","execution_count":1,"outputs":[]},{"metadata":{"_cell_guid":"48608b29-c6ce-48f3-9453-c0544391f7e9","_kg_hide-input":true,"_kg_hide-output":true,"_uuid":"5853bfba83b3c1715c03d3302a21584dda7e185b","collapsed":true,"trusted":true},"cell_type":"code","source":"class test_val_compare(object):\n    def __init__(self,validation,test):\n        self.val=load_data(validation)\n        self.val.name='validation'\n        self.test=load_data(test)\n        self.test.name='test'\n        self.actors=[]\n        self.val_cap=[]\n        self.capped=False\n    \n    def info(self,**kwargs):\n        print('Validation data:')\n        print(self.val.info(**kwargs))\n        print('Test data:')\n        print(self.test.info(**kwargs))\n        \n    def def_actors(self,cols,merge=False):\n        self.actors=(pd.concat([self.val[cols].drop_duplicates().assign(left_data=True,right_data=False).astype({'left_data':'bool'}),\n                         self.test[cols].drop_duplicates().assign(left_data=False,right_data=True).astype({'left_data':'bool'})])\n                 .reset_index(drop=True)\n                 .assign(actor=lambda x: x.index.values.astype(np.uint32))\n                )\n        ##if Merge:\n            ## add the actors columns to the frames\n            \n    def cap(self):\n        if not self.capped:\n            real_max = self.test.describe().loc['max',:]\n            self.val_cap=self.val.loc[(self.val['ip']<real_max['ip'])&(self.val['app']<real_max['app'])&(self.val['device']<real_max['device'])&(self.val['os']<real_max['os'])&(self.val['channel']<real_max['channel']),:]\n            self.val_cap.name='capped_validation'\n            self.capped=True\n        else:\n            print('capped validation set already created, use .val_cap')\n\n    def overlap(self,var,cap=False):\n        if not cap:\n            frames=[self.val,self.test]\n        else:\n            frames=[self.val_cap,self.test]\n        intersection=list(set.intersection(*[set(x[var]) for x in frames]))\n        overlap=pd.DataFrame({'val':list(set.union(*[set(x[var]) for x in frames]))})\n        for frame in frames:\n            overlap[frame.name]=overlap.val.isin(frame[var])\n        overlap=(overlap\n                 .set_index(['val'])\n                 .stack()\n                 .reset_index()\n                 .rename(columns={'level_1':'set',0:'keep'})\n                 .loc[lambda x:x.keep,['val','set']]\n                 .assign(is_shared=lambda x:x.val.isin(intersection))\n                )\n        return overlap\n\n    def plot_overlap(self,var,with_cap=False,dual=False):\n        if dual:\n            if not self.capped:\n                self.cap()\n            frame_list=[frame.rename(columns={var:frame.name})[frame.name] for frame in [self.val,self.val_cap]]\n            frame2=self.test.rename(columns={var:self.test.name})[self.test.name]\n            fig, axs = plt.subplots(ncols=2,figsize=(30,6))\n            for i in [False,True]:\n                (sns.stripplot(data=self.overlap(var,i),y='set',x='val',hue='is_shared',jitter=True,dodge=True,palette='Set1',ax=axs[i])\n                .set(title='Difference in {} values'.format(var),xlabel='',ylabel=''));\n        else:\n            plt.figure(figsize=(15,6))\n            if with_cap:\n                if not self.capped:\n                    self.cap()\n                (sns.stripplot(data=self.overlap(var,True),y='set',x='val',hue='is_shared',jitter=True,dodge=True,palette='Set1')\n                    .set(title='Difference in {} values'.format(var),xlabel='',ylabel=''));\n            else:\n                (sns.stripplot(data=self.overlap(var),y='set',x='val',hue='is_shared',jitter=True,dodge=True,palette='Set1')\n                    .set(title='Difference in {} values'.format(var),xlabel='',ylabel=''));\n            \n    def heat_compare(self,var1,var2,with_cap=False,combine=False,respace=False,black=False):\n        data1=self.val\n        data2=self.test\n        if with_cap:\n            data1=self.val_cap\n        if combine & respace:\n            min_var1=min([min(data1[var1]),min(data2[var1])])\n            max_var1=max([max(data1[var1]),max(data2[var1])])\n            min_var2=min([min(data1[var2]),min(data2[var2])])\n            max_var2=max([max(data1[var2]),max(data2[var2])])\n            new_range=[range(min_var1,max_var1),range(min_var2,max_var2)]\n            new_index=pd.MultiIndex.from_product(new_range,names=[var1,var2])\n\n            fig,ax=plt.subplots(figsize=(30,30))\n            sns.heatmap(data1.groupby([var1,var2])['click_time'].count().pipe(np.log1p).reindex(index=new_index).unstack(),\n                cbar=False,\n                ax=ax)\n            sns.heatmap(data2.groupby([var1,var2])['click_time'].count().pipe(np.log1p).reindex(index=new_index).unstack(),\n                cmap='YlGnBu_r',\n                cbar=False,\n                ax=ax)\n        elif combine:\n            var1_union=np.union1d(data1[var1],data2[var1])\n            var2_union=np.union1d(data1[var2],data2[var2])\n            new_range=[var1_union,var2_union]\n            new_index=pd.MultiIndex.from_product(new_range,names=[var1,var2])\n            fig,ax=plt.subplots(figsize=(30,30))\n            sns.heatmap(data1.groupby([var1,var2])['click_time'].count().pipe(np.log1p).reindex(index=new_index).unstack(),\n                cbar=False,\n                ax=ax)\n            sns.heatmap(data2.groupby([var1,var2])['click_time'].count().pipe(np.log1p).reindex(index=new_index).unstack(),\n                cmap='YlGnBu_r',\n                cbar=False,\n                ax=ax)\n        else:\n            if black:\n                fig,axes=plt.subplots(ncols=2,figsize=(30,20))\n                sns.heatmap(data1.groupby([var1,var2])['click_time'].count().pipe(np.log1p).unstack().fillna(0),\n                            cbar=False,\n                            ax=axes[0]).set(title=data1.name)\n                sns.heatmap(data2.groupby([var1,var2])['click_time'].count().pipe(np.log1p).unstack().fillna(0),\n                            cbar=False,\n                            ax=axes[1]).set(title=data2.name)\n            else:\n                fig,axes=plt.subplots(ncols=2,figsize=(30,20))\n                sns.heatmap(data1.groupby([var1,var2])['click_time'].count().pipe(np.log1p).unstack(),\n                            cbar=False,\n                            ax=axes[0]).set(title=data1.name)\n                sns.heatmap(data2.groupby([var1,var2])['click_time'].count().pipe(np.log1p).unstack(),\n                            cbar=False,\n                            ax=axes[1]).set(title=data2.name)","execution_count":2,"outputs":[]},{"metadata":{"_cell_guid":"cfd0d249-43d9-44fb-b768-5b02f85dc993","_uuid":"ce4856f138e59205e56eefbb6ae4494d6273db29","collapsed":true,"trusted":true},"cell_type":"code","source":"%%time\nanalysis=test_val_compare('last_day','test')\nanalysis.info()","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"92379d57-0b8a-44f5-843b-e74b55dbfbdc","_uuid":"074c290f491dcedac6556445a96cc07328c251f0"},"cell_type":"markdown","source":"This class has a  funtion **plot_overlap** which takes a sequence of series and returns a long dataframe with the following columns:\n - 'val' - all values present in the series\n -  'serie' - series in which they are present\n -  'intersection' - a boolean indicating the presence of the value in the intersection of all series.\n\nWe use it to see plot the differences betweein the unique values present in the fake test set from the 9th and the real test set. Note that the IP values were already truncated.\n\nIn the graphs below, the <font color=red>**red**</font> denotes the unique values that are **not** shared between the sets, the  <font color=blue>**blue**</font> ones are the common ones."},{"metadata":{"_cell_guid":"94d3f080-6b73-4c2a-803b-f4b3001b5c67","_kg_hide-input":true,"_uuid":"8562b8b2e7373ce6577cce28344d51b1c1ef82a1","collapsed":true,"scrolled":false,"trusted":true},"cell_type":"code","source":"analysis.plot_overlap('ip')\nanalysis.plot_overlap('app')\nanalysis.plot_overlap('channel')\nanalysis.plot_overlap('device')\nanalysis.plot_overlap('os')","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"77d6b919-74e8-4495-8c20-1609bf903e44","_uuid":"ea6e04499714cc2a95c8b461bebb8be764afa6f2"},"cell_type":"markdown","source":"We see that **ip** was not only truncated, but the amount of ips not shared is far greater in the train data than the test data.\nMoreover it was far from the only feature that got truncated. **Device, os and app** also were filtered out past 3031, 604 and 521 respectively. There is also something a little *funky* going on with device and os : the less dense the common values become in the test data, the more dense it becomes in the unique ones. \n\nLet us now check if truncating the fake test set removes the values not present in the test set."},{"metadata":{"_cell_guid":"3781d981-c0f5-44f7-8b31-67e8376e84f2","_uuid":"9b1bf8dcb6892eb198bb32db498de45bb1701405","collapsed":true,"trusted":true},"cell_type":"code","source":"analysis.cap()","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"28d43bc7-8c70-4ad2-be74-cec9b5d88fa6","_uuid":"98b10f8da3f00fda02c26b2adba88964192827b3","collapsed":true,"scrolled":false,"trusted":true},"cell_type":"code","source":"analysis.plot_overlap('ip',with_cap=True)\nanalysis.plot_overlap('app',with_cap=True)\nanalysis.plot_overlap('channel',with_cap=True)\nanalysis.plot_overlap('device',with_cap=True)\nanalysis.plot_overlap('os',with_cap=True,)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"0d4e6b3b-2e7d-44d1-980e-b4c83c8cea70","_uuid":"6355934a086f24aada707b762c3df83e64667055"},"cell_type":"markdown","source":"We see that  there has **not been a significant change**, aside from matching the maxima. I had hoped it would indeed clean up a reduce, if not remove, the amount of red in the fake test dataset."},{"metadata":{"_cell_guid":"c38c6b2a-5864-4afb-bb38-91d909fd761e","_uuid":"fc7611401a26d1dd2463b9fab46eb868a3607341"},"cell_type":"markdown","source":"## Click count analysis of the difference\n\nWe will now compare the sets a little more closely, to see if the click count is distributed similarly in the different features. We'll do this by juxtaposing heatmaps of the real and fake test set."},{"metadata":{"_cell_guid":"bd56d8ea-99cf-446c-a42a-d11e545b846f","_uuid":"c1035986db3b9495a8af8f8618dcfe7d235d166e","collapsed":true,"trusted":true},"cell_type":"code","source":"analysis.heat_compare('ip','app',True)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"93a81aef-2d0e-4735-8a59-ab3b909cb35d","_uuid":"129292281e2a6face045a137d4fc38eb6c5cd0dd","collapsed":true,"trusted":true},"cell_type":"code","source":"# the kernel dies with this one, it takes too much memory (exit code 137)\n#heat_compare(validation_data_cap,test_data,'ip','device')","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"8d5d849b-0323-499b-b48f-4b8ecae3a901","_uuid":"07ca17f7c66e9fde989ca9c961645fb255085f15","collapsed":true,"trusted":true},"cell_type":"code","source":"analysis.heat_compare('ip','os',True)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"6cb8dd39-1602-4492-a25c-e8e9077788fe","_uuid":"6ebf9678c89d488e9403b11a723a57ca50be8a67","collapsed":true,"trusted":true},"cell_type":"code","source":"analysis.heat_compare('ip','channel',True)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"835f2a29-a078-4ac4-b4ae-b4542f3e7791","_uuid":"855249712e34b6c5c76e28b62b0578e700fb92d4","collapsed":true,"trusted":true},"cell_type":"code","source":"analysis.heat_compare('app','device',True)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"9d555145-0ed3-4e61-8000-fc2c17b284eb","_uuid":"541c19de0491cc6f482390ded4b0708411eca96c","collapsed":true,"trusted":true},"cell_type":"code","source":"analysis.heat_compare('app','os',True)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"7ba00c01-1c84-422a-a1c9-f926079d75b8","_uuid":"96a5d5efd329ba3664abb24a6ff5d2f654d43fda","collapsed":true,"trusted":true},"cell_type":"code","source":"analysis.heat_compare('app','channel',True)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"f00cb55a-27a8-415c-8783-198541b271f6","_uuid":"653a08a290c054cc30764d3f6198f986b69aef0b","collapsed":true,"trusted":true},"cell_type":"code","source":"analysis.heat_compare('device','os',True)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"a02f41e8-0f10-43e3-bbb2-cec4549ba7f1","_uuid":"08107c4a7e9711fb18cef34c16bb48468277461f","collapsed":true,"trusted":true},"cell_type":"code","source":"analysis.heat_compare('device','channel',True)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"54e2e44d-b4e5-466c-94f6-98c9bb243e70","_uuid":"5e155326c8f8074622ad631908a25461e2c891b8","collapsed":true,"trusted":true},"cell_type":"code","source":"analysis.heat_compare('os','channel',True)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"b1284372-3b81-4bee-bb45-bb1dd26db7f0","_uuid":"5555f858f5f8d6746dfed4287912c32461fe8420","collapsed":true},"cell_type":"markdown","source":"We see that the distributions are **extremely** similar, except for  app 159 and os 35 in the os vs app plot, and a. It is also showing the features' independence, apart from app and os, which seem to be somewhat correlated. The plots strongly suggest not to use the variables as categorical. Not only the unique values of the test set make it tricky, but there seem to be a fairly good nothion of closeness on the set. It is tempting to say that the labels given to the features come from a topology on the feature space.\n\nThere is one **very strange** thing though - the \"waterfall\" edge of the ip vs device plot in the test data. It very clearly breaks the space into two sides, which doesn't happen in fake one coming from the training data. If any of you have an idea what it could be.... I should plot it with evenly spaced axes to see if it is a linear or not.\n\nWe would like to now plot the \"is attributed\" distribution of the training set is consistent with the click count distribution.\n\n...coming soon."},{"metadata":{"_cell_guid":"be3ceeee-b9ab-4ea7-b800-a836adea2f51","_uuid":"553383e6299291a7ec9bc9ed1b872bb35a4d2b38"},"cell_type":"markdown","source":"# Tracking actors' behavior through time.\n\nI wanted to toy with sankey diagrams to try to determine specific behaviors of what I'd like to call the actors of the dataset. Indeed, what is the best notion of user we can get with the given data? Let's just start with visualizing the flow of unique combination of all 5 \"categorical\" features with respect to activity grouped by hours and see what transpires."},{"metadata":{"_cell_guid":"0ccb4bfc-85d7-4bd4-8df8-eb44ddc02eeb","_uuid":"400da6058277c43ac7fc9a6fb172a75520d42b80","collapsed":true,"trusted":true},"cell_type":"code","source":"# defining our actors\nactors_factors=['ip','app','device','os','channel']\nanalysis.def_actors(actors_factors)\n\n# defining the nodes of our graphs\ntimes=analysis.val.click_time.dt.hour.unique()\nstates=[0,1,2]\ncolor_dict={0 : 'red', 1 : 'grey', 2 : 'green'}\nstate_dict={0 : 'no activity', 1 : 'click only', 2 : 'download'}\nnodes=(pd.DataFrame(index=pd.MultiIndex.from_product([times,states],names=['hour','state']))\n       .reset_index()\n       .assign(code=lambda x:10*x.hour+x.state,\n               label=lambda x: x.state.replace(state_dict).str.cat(x.hour.astype(str),sep='_'),\n              color=lambda x: x.state.replace(color_dict))\n      )\nnode_codes=nodes.reset_index().set_index(['code'])['index'].to_dict()\nnodes.head()","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"de1bfcfd-8a03-425f-9d21-09caa0dcafb3","_uuid":"055805b1bd7b53fa29aa2b00cd264285d37bb4f5","collapsed":true,"trusted":true},"cell_type":"code","source":"# defining the links\nlinks=(analysis.val.assign(hour=lambda x: x.click_time.dt.hour.astype(np.uint8))\n         .groupby(actors_factors+['hour'])\n         .max()\n         .reset_index()\n         .merge(analysis.actors[actors_factors+['actor']],on=actors_factors,how='left')\n         .reset_index(drop=True)\n         .loc[:,['hour','actor','is_attributed']]\n         .assign(code=lambda x: 10*x.hour+x.is_attributed+1)\n         .pivot(index='actor',columns='hour',values='code')\n         .fillna({x:10*x for x in times})\n)\nlinks=pd.concat([links.iloc[:,k].T.reset_index(drop=True).T for k in [[i,i+1]for i in range(len(links.columns)-1)]]).reset_index()\nlinks.columns=['actor','source','target']\nlinks.head()","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"dce09f2b-688b-4bd1-aa5d-4bb7227c0395","_uuid":"7cf9582204402d709396c0c8a14d4350103c782a"},"cell_type":"markdown","source":"There are a humongous number of actors, so we should first plot them grouped by their behavior, hour to hour. We can look closer into certain ranges of actors later."},{"metadata":{"_cell_guid":"1c6fb88f-b611-42b3-ac7a-5c2d1b0a9585","_uuid":"9af549a334d2c2c43a272850becd674a2c76fed8","collapsed":true,"trusted":true},"cell_type":"code","source":"links_grouped=links.groupby(['source','target']).count().reset_index()","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"fdec171f-225e-499a-b0be-a38de51efd0e","_uuid":"7942da223800735887c40798282ef60d0fa63ad2","collapsed":true,"trusted":true},"cell_type":"code","source":"data = {\n    'type': 'sankey',\n     'domain' : {'x': [0,1], 'y': [0,1]},\n    'orientation': 'h',\n    'node' : {\n      'pad':  15,\n      'thickness' : 20,\n      'line' : {\n            'color' : \"black\",\n            'width' : 0.5\n          },\n      'label':  nodes.label,\n      'color' : nodes.color\n    },\n    'link' : {\n          'source' : links_grouped.loc[:,'source'].replace(node_codes),\n          'target': links_grouped.loc[:,'target'].replace(node_codes),\n          'value' : links_grouped.loc[:,'actor'],\n          'label' : links_grouped.loc[:,'actor']\n            }\n}\n\nlayout =  {\n    'title' : \"Flow of activity between hour slices for the validation set\",\n    'font' : dict(size = 10)\n}\n\nfig = dict(data=[data], layout=layout)\npy.iplot(fig, validate=False)","execution_count":null,"outputs":[]},{"metadata":{"collapsed":true,"trusted":true,"_uuid":"52abe6a32308cf11c967704e26254f1616a44712"},"cell_type":"code","source":"links=(analysis.test.assign(hour=lambda x: x.click_time.dt.hour.astype(np.uint8))\n         .groupby(actors_factors+['hour'])\n         .max()\n         .reset_index()\n         .merge(analysis.actors[actors_factors+['actor']],on=actors_factors,how='left')\n         .reset_index(drop=True)\n         .loc[:,['hour','actor']]\n         .assign(code=lambda x: 10*x.hour+1)\n         .pivot(index='actor',columns='hour',values='code')\n         .fillna({x:10*x for x in times})\n)\nlinks=pd.concat([links.iloc[:,k].T.reset_index(drop=True).T for k in [[i,i+1]for i in range(len(links.columns)-1)]]).reset_index()\nlinks.columns=['actor','source','target']\nlinks_grouped=links.groupby(['source','target']).count().reset_index()","execution_count":null,"outputs":[]},{"metadata":{"collapsed":true,"trusted":true,"_uuid":"7890e1689cea891bc7ed0eef9216a1d7b442f695"},"cell_type":"code","source":"data = {\n    'type': 'sankey',\n     'domain' : {'x': [0,1], 'y': [0,1]},\n    'orientation': 'h',\n    'node' : {\n      'pad':  15,\n      'thickness' : 20,\n      'line' : {\n            'color' : \"black\",\n            'width' : 0.5\n          },\n      'label':  nodes.label,\n      'color' : nodes.color\n    },\n    'link' : {\n          'source' : links_grouped.loc[:,'source'].replace(node_codes),\n          'target': links_grouped.loc[:,'target'].replace(node_codes),\n          'value' : links_grouped.loc[:,'actor'],\n          'label' : links_grouped.loc[:,'actor']\n            }\n}\n\nlayout =  {\n    'title' : \"Flow of activity between hour slices for the \",\n    'font' : dict(size = 10)\n}\n\nfig = dict(data=[data], layout=layout)\npy.iplot(fig, validate=False)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"bf3c7cdc-b149-489e-9ac9-32f46c2f377f","_uuid":"97bf86971bcce7f24272ebc67f7517e054f56a78"},"cell_type":"markdown","source":"## appendix\n\nFor the sake of completeness,  we wish to compare with the full training data to see if there is a significant difference with the slice we picked of the 9th. We will do this below soon."}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}