{"cells":[{"metadata":{"_uuid":"445b163388a54ea0332920a9e3ccb1f7243aaba4"},"cell_type":"markdown","source":"# [Tabular learning](https://docs.fast.ai/tabular.html) \nI wanted to go a slightly different route from the LGBM that seems to be favorite here.\nI don't expect too great results but maybe I can average with LGBM results later to get a few precent more out of my other models.\n## Deep in this case means embeddings+2 layers\nNot very deep I know I know, but lets see maybe Ill add some if that increases performance.\nThe Idea is to use the continuous variables as they are. \nThe categorical variables are in a first layer transformed in a lower dimensional space via embeddings with dropout.\nThe embeddings also use droput and the continuous variables batchnorm.\nAfter that blocks of BatchNorm, Dropout, Linear and ReLU.\nThe Embedding sizes I chose to sqrt(n_cat) or a maximum of 100.\n\nThe preprocessing is done in a different kernel. Just standard stuff and I added a column for the date that can be decoded from OsBuildLab.\nI plan to add new features when I set the pipeline for my lgbm and this model up.\nIn this kernel, I still need to add the prediction for the test case so I can submit this.\n\nI would additionally like to explore the embeddings a bit more since one can make pretty plots of the relationship of for instance dates of weekdays vs weekends, or gaming vs non gaming.\n"},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\nfrom pathlib import Path\n\nfrom fastai import *\nfrom fastai.tabular import *\n\nfrom sklearn.metrics import roc_auc_score\n\nfrom tqdm import tqdm","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c29d9b686f6035dd85640f0c2d096f64417d853e"},"cell_type":"code","source":"import numpy as np\nfrom sklearn.manifold import TSNE\nfrom sklearn.decomposition import PCA","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6606ac7020ca788aec24de746b3be80acc8541fb"},"cell_type":"code","source":"path=Path('../input')\nfe_path=path/'feature-engineering-msmw'\ncomp_path=path/'microsoft-malware-prediction'\nhp_path=path/'lgbm-hyperopt'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6606ac7020ca788aec24de746b3be80acc8541fb"},"cell_type":"code","source":"cats=np.load(fe_path/'categories.npy').item()\nmeans,stds=np.load(fe_path/'means.npy').item(),np.load(fe_path/'stds.npy').item()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9282d2736e67a2acf631ba56fb2afd2d7798a99f"},"cell_type":"code","source":"index='MachineIdentifier'\ndep_var='HasDetections'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9132afd735ee0ff4377bc94673975ab7ba691099"},"cell_type":"code","source":"cont_names=list(means.keys())\ncat_names=list(cats.keys())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"076e181bbb83a3faf269695340ca1e8f91dcf802"},"cell_type":"code","source":"feature_imp=pd.read_pickle(hp_path/'feature_importance')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1b14de1e425dfb2451345b42b33e5b5fbc70c543"},"cell_type":"code","source":"idx=np.argsort(feature_imp.mean(axis=1))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"73688e97749e6f040931ef5b6150c9c19575b0d1"},"cell_type":"code","source":"frac=0.01\nfrac_important=np.cumsum(feature_imp.mean(axis=1)[idx])/np.max(np.sum(feature_imp.mean(axis=1)[idx]))\nidx_important=np.where(frac_important>frac)[0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2bc1163403c9f2e97478f9c2726158c2b3519a52"},"cell_type":"code","source":"feature_cols=feature_imp.iloc[idx[idx_important]].index\ncat_names=[cat for cat in cat_names if cat in feature_cols]\ncont_names=[cont for cont in cont_names if cont in feature_cols]\n\nfeatures=[index]+cat_names+cont_names+[dep_var]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"dae30b7346cff904942177cd4a25d428427bcdcf"},"cell_type":"code","source":"import matplotlib.pyplot as plt\nplt.figure(figsize=(16,16))\np1 = plt.barh( idx_important,feature_imp.mean(axis=1)[idx[idx_important]], xerr=feature_imp.std(axis=1)[idx[idx_important]],orientation ='horizontal')\nplt.yticks(idx_important, feature_imp.mean(axis=1).index[idx[idx_important]]);\nplt.xscale('log')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6606ac7020ca788aec24de746b3be80acc8541fb"},"cell_type":"code","source":"df_trn=pd.read_hdf(fe_path/'train.h5',columns=features)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9b138a6dd4b0b8a8990d1ab1e57e5486354be3f9"},"cell_type":"code","source":"df_trn.set_index('MachineIdentifier',inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"76602383b90f2d3a5dc876306079c4f2703bf71c"},"cell_type":"markdown","source":"Validation set is therefore about 10% of the whole dataset"},{"metadata":{"trusted":true,"_uuid":"cbebce5d67c89abe32e8c20a9aed820b2ae18f2c"},"cell_type":"code","source":"#df_trn=df_trn.iloc[:int(0.2*len(df_trn))]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1193eb427e8ec4ed56aac7be5806cc0c93089375"},"cell_type":"code","source":"N_trn=int(0.9*len(df_trn))\nvalid_idx =range(N_trn,len(df_trn))# np.argsort(df_trn.OSVersion_Elapsed)[N_trn:]\nbs=1024","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e426d460fda9b52c2ad6ed4e9b6b39557c5baa8b"},"cell_type":"markdown","source":"This part creates the dataset as well as a fancy pytorch dataloaderfor training and validation set."},{"metadata":{"trusted":true,"_uuid":"ed4f2330253b739718974bd20728a6d2d4596ca0"},"cell_type":"code","source":"data = TabularDataBunch.from_df('.', df_trn, dep_var, valid_idx=valid_idx, cat_names=cat_names,bs=bs)#,test_df=df_test)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a5d71c795c8fa7eec36b567b9df1c2fc382d3726"},"cell_type":"markdown","source":"Based on the categorical and continuous variables a model of a certain size is created. Although its a 0 or 1 case out_sz has to be 2. "},{"metadata":{"trusted":true,"_uuid":"3b135e5c50b9af4065c910037eb35c48ffbcaa82"},"cell_type":"code","source":"model = TabularModel(data.get_emb_szs(), n_cont=len(data.cont_names), out_sz=2, layers=[2000,4000,1000], ps=None, emb_drop=0.5,\n                         y_range=[0,1], use_bn=True)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ccc93410cc053fbf68ac2691522c26db1faea632"},"cell_type":"markdown","source":"Pretty stupid way of defining the metric but I couldn't make it work otherwise to work with the fastai library"},{"metadata":{"trusted":true,"_uuid":"9907259a23450fdae6227b00c2e111d6c34e517f"},"cell_type":"code","source":"def auc_score(y_pred,y_true,tens=True):\n    score=roc_auc_score(y_true,torch.sigmoid(y_pred)[:,1])\n    if tens:\n        score=tensor(score)\n    else:\n        score=score\n    return score","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"bca9e8bee378323410a1783e34fd787c54630551"},"cell_type":"markdown","source":"Now combining the data and model with a learner object.\nThis one defines metric, loss function, a learning rate scheduler and handles all details about model saving including handling the predictions on a specific dataset (for now validation but I'll add test set soon)"},{"metadata":{"trusted":true,"_uuid":"c70036dd68353c3d5a55adffef54284425c1c73f"},"cell_type":"code","source":"data.show_batch()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f42bfc356623c79e998ad9e1e63bc521ea3915f5"},"cell_type":"code","source":"learn=Learner(data, model,metrics=[auc_score,accuracy])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2efa9b9af6581161be265da51c947a4926f77c0c"},"cell_type":"code","source":"learn.lr_find(num_it=1000)\nlearn.recorder.plot()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bf958b3f3c0409cb70e5b93922aa6500b90b654f"},"cell_type":"code","source":"learn.save('untrained')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"eac01f56038b2e7b1e0c7b0f866e948afe117327"},"cell_type":"code","source":"learn.fit_one_cycle(10, 1e-2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b4eb55133a5d981b7114e9107de07d4d51b73d7d"},"cell_type":"code","source":"learn.save('embeddings-stage-1')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b4eb55133a5d981b7114e9107de07d4d51b73d7d"},"cell_type":"code","source":"learn.recorder.plot_losses()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b4eb55133a5d981b7114e9107de07d4d51b73d7d"},"cell_type":"code","source":"learn.recorder.plot_metrics()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b4eb55133a5d981b7114e9107de07d4d51b73d7d"},"cell_type":"code","source":"plt.plot(learn.recorder.losses)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b4eb55133a5d981b7114e9107de07d4d51b73d7d"},"cell_type":"code","source":"plt.plot(learn.recorder.lrs)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fda4d3adf2df9cee29d8fdfc140377e20021b173"},"cell_type":"code","source":"(x_cat,x_cont),y=data.one_batch(detach=False)\n\nimport matplotlib.gridspec as gridspec\n\nn_plot=int(np.sqrt(len(cat_names)))-1\nlearn.load('untrained')\nplt.figure(figsize=(16,16))\ngs = gridspec.GridSpec(n_plot,n_plot)\nfor i in range(n_plot):\n    for j in range(n_plot):\n        ax = plt.subplot(gs[i,j])\n        x_embed=to_np(learn.model.embeds[j+i*n_plot](x_cat[:,j+i*n_plot]))\n        X_embedded = PCA(n_components=2).fit_transform(x_embed)\n        color=to_np(x_cat[:,j+i*n_plot])\n        color=color/np.max(color)\n        ax.scatter(X_embedded[:,0],X_embedded[:,1],c=color)\n        ax.set_title(cat_names[j+i*n_plot])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2f4829648230c9b79cd294f9b959946787209386"},"cell_type":"code","source":"(x_cat,x_cont),y=data.one_batch(detach=False)\n\nimport matplotlib.gridspec as gridspec\n\nn_plot=int(np.sqrt(len(cat_names)))-1\nlearn.load('embeddings-stage-1')\n\nplt.figure(figsize=(16,16))\ngs = gridspec.GridSpec(n_plot,n_plot)\nfor i in range(n_plot):\n    for j in range(n_plot):\n        ax = plt.subplot(gs[i,j])\n        x_embed=to_np(learn.model.embeds[j+i*n_plot](x_cat[:,j+i*n_plot]))\n        X_embedded = PCA(n_components=2).fit_transform(x_embed)\n        color=to_np(x_cat[:,j+i*n_plot])\n        color=color/np.max(color)\n        ax.scatter(X_embedded[:,0],X_embedded[:,1],c=color)\n        ax.set_title(cat_names[j+i*n_plot])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1dc9b1f37584fb1dfa496fe7740fa03d57ad94ed"},"cell_type":"code","source":"y_pred,y_true=learn.get_preds()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1dc9b1f37584fb1dfa496fe7740fa03d57ad94ed"},"cell_type":"code","source":"scr=to_np(auc_score(y_pred,y_true))\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1dc9b1f37584fb1dfa496fe7740fa03d57ad94ed"},"cell_type":"code","source":"f\"{scr:0.3}\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"311a939878a8dbd733ebfa9d37a9e5777b6c9d21"},"cell_type":"code","source":"learn.show_results(rows=20)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b096e4745c80e21db63de35bf0427c615bbe1d1e"},"cell_type":"code","source":"df_sub=pd.read_csv(comp_path/'sample_submission.csv').set_index('MachineIdentifier')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1df6dcc46f1ee6b42db2f336bf8e4234391f4bfa"},"cell_type":"code","source":"features.remove(dep_var)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b096e4745c80e21db63de35bf0427c615bbe1d1e"},"cell_type":"code","source":"chk_size=1000*1024\ndf_test_iter=pd.read_hdf(fe_path/'test.h5',iterator=True, chunksize=chk_size, columns=features)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b096e4745c80e21db63de35bf0427c615bbe1d1e","scrolled":true},"cell_type":"code","source":"for df_test in tqdm(df_test_iter):\n    df_test.set_index('MachineIdentifier',inplace=True)\n    for col in cat_names:\n        df_test[col]=df_test[col].cat.codes\n    x_cat_np=df_test.loc[:,cat_names].values.astype(np.int64)+1\n    x_cont_np=df_test.loc[:,cont_names].values.astype(np.float32)\n    for idxs in np.array_split(range(df_test.shape[0]),chk_size//bs):\n        x_cat=to_device(tensor(x_cat_np[idxs,:]),'cuda')\n        x_cont=to_device(tensor(x_cont_np[idxs,:]),'cuda')\n        pred=learn.model(x_cat,x_cont)\n        df_sub.loc[df_test.index[idxs],'HasDetections']=to_np(pred)[:,1]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"82396c1e3d9294765eef79d99bfcf8242c40e8bf"},"cell_type":"code","source":"import seaborn as sns","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"be28fce2f2c95e75338fb65a7fc550224120cdaf"},"cell_type":"code","source":"sns.distplot(df_sub)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7f7db9b9ea68ca73be3c7e80022009d16b9ec9e6"},"cell_type":"code","source":"df_sub.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e2c870841be8131449bb9c5b76a97b7d170753b0"},"cell_type":"code","source":"df_sub.to_csv('submission')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"90e268811a7305e17c1d9a1ec181eb81a5dd2a20"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0eac74878e680dc3cd981c732235dc96bae320ac"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7ca54c6a23710940d6c9e8bf59fa3c7d12e8b8b0"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f4eb51c8eda2053f5e72125500bc50dbbdfef4cb"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}