{"cells":[{"metadata":{"_uuid":"d17b30c0c6d95a2e976840575fcdfdf95409ac72","_cell_guid":"020152c8-a7f8-4f7a-9d49-9ef6bff56d10"},"cell_type":"markdown","source":"Simple example of the use of Neural Networks. The Neural Net has performed particularly well with only 3 epochs of training despite the very heavily imbalanced dataset. Enjoy and extend!\n*For some reason that I cannot reckon, the exact same code outputs 97.2% AUC score in my PC - if anyone has any hint on that I would like to hear it :)"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"#Import modules\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom sklearn.model_selection import train_test_split\nfrom keras.models import Sequential\nfrom keras.layers import Activation, Dense, Dropout\nfrom keras import optimizers\nfrom sklearn.metrics import confusion_matrix,accuracy_score, roc_curve, auc\n%matplotlib inline\nsns.set_style(\"whitegrid\")\nnp.random.seed(697)","execution_count":1,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true,"collapsed":true},"cell_type":"code","source":"#Import data\ndf = pd.read_csv('../input/train_sample.csv', header = 0)","execution_count":2,"outputs":[]},{"metadata":{"_uuid":"bd280281339907523dfa70a6b41eb2379a920e5f","_cell_guid":"a8182432-9305-410f-b10d-cf5f8dce3655","trusted":true},"cell_type":"code","source":"#Check num of cases in label \nprint(df.is_attributed.value_counts()) #very imbalanced data set","execution_count":3,"outputs":[]},{"metadata":{"collapsed":true,"_uuid":"7b4a285041ead35f4d10771e721dcf995fc8d59e","_cell_guid":"e6435e15-875b-4f02-a70c-e6a879ad1bfa","trusted":true},"cell_type":"code","source":"#---------------------------Pre-processing-------------------------\n#Create new variables\ndf['ip_cut'] = pd.cut(df.ip,15)\ndf['time_interval'] = df.click_time.str[11:13]\n\n#Drop unneeded variables\ndf = df.drop(['ip', 'attributed_time', 'click_time'], axis = 1)","execution_count":4,"outputs":[]},{"metadata":{"_uuid":"e7842fbd04788ccabcfbf585cd2d2383a5eb740e","_cell_guid":"5a713365-22f8-4912-b55d-4f271d3eb2f6","trusted":true,"collapsed":true},"cell_type":"code","source":"#Encode categorical variables to ONE-HOT\ncategorical_columns = ['app', 'device', 'os', 'channel', 'ip_cut', 'time_interval']\n\ndf = pd.get_dummies(df, columns = categorical_columns)     ","execution_count":5,"outputs":[]},{"metadata":{"collapsed":true,"_uuid":"8931760d86e080c88cd844235ac2fe2e41f83bbf","_cell_guid":"733b1e9f-df00-48be-8b60-72942bd80f7b","trusted":true},"cell_type":"code","source":"#Split in 75% train and 25% test set\ntrain_df, test_df = train_test_split(df, test_size = 0.25, random_state= 1984)\n\n#Make sure labels are equally distributed in train and test set\ntrain_df.is_attributed.sum()/train_df.shape[0] #0.2233\ntest_df.is_attributed.sum()/test_df.shape[0] #0.2148\n\n#Get the data ready for the Neural Network\ntrain_y = train_df.is_attributed\ntest_y = test_df.is_attributed\n\ntrain_x = train_df.drop(['is_attributed'], axis = 1)\ntest_x = test_df.drop(['is_attributed'], axis = 1)\n\ntrain_x =np.array(train_x)\ntest_x = np.array(test_x)\n\ntrain_y = np.array(train_y)\ntest_y = np.array(test_y)","execution_count":6,"outputs":[]},{"metadata":{"_uuid":"c0dc9717c5b228e881efeadf8433e42370bed5e6","_cell_guid":"3a1db286-4549-4edf-9ce0-071a0c9de68e","trusted":true},"cell_type":"code","source":"#-------------------Build the Neural Network model-------------------\nprint('Building Neural Network model...')\nadam = optimizers.adam(lr = 0.005, decay = 0.0000001)\n\nmodel = Sequential()\nmodel.add(Dense(48, input_dim=train_x.shape[1],\n                kernel_initializer='normal',\n                #kernel_regularizer=regularizers.l2(0.02),\n                activation=\"relu\"))\nmodel.add(Dropout(0.2))\nmodel.add(Dense(24,\n                #kernel_regularizer=regularizers.l2(0.02),\n                activation=\"tanh\"))\nmodel.add(Dropout(0.3))\nmodel.add(Dense(1))\nmodel.add(Activation(\"sigmoid\"))\nmodel.compile(loss=\"binary_crossentropy\", optimizer='adam')\n\nhistory = model.fit(train_x, train_y, validation_split=0.2, epochs=3, batch_size=64)","execution_count":7,"outputs":[]},{"metadata":{"_uuid":"74333d282410413309bbf5592572b7dc950c58b3","_cell_guid":"e7e97866-c345-4060-b75a-103493fc5c31","trusted":true},"cell_type":"code","source":"# summarize history for loss\nplt.figure()\nplt.plot(history.history['loss'])\nplt.plot(history.history['val_loss'])\nplt.title('model loss')\nplt.ylabel('loss')\nplt.xlabel('epoch')\nplt.legend(['train', 'test'], loc='upper right')\nplt.show()","execution_count":8,"outputs":[]},{"metadata":{"collapsed":true,"_uuid":"e2c4ff3bb88a213ca27f5a91dae7b8e22eef61d2","_cell_guid":"afee2436-a875-4f55-bf84-e04c9bd71b17","trusted":true},"cell_type":"code","source":"#Predict on test set\npredictions_NN_prob = model.predict(test_x)\npredictions_NN_prob = predictions_NN_prob[:,0]\n\npredictions_NN_01 = np.where(predictions_NN_prob > 0.5, 1, 0) #Turn probability to 0-1 binary output","execution_count":9,"outputs":[]},{"metadata":{"_uuid":"3a5383f855f061816ae0b2d51d09b1e96eef674f","_cell_guid":"2b15a6de-dabc-47ff-a8ca-934cbb0f292e","trusted":true},"cell_type":"code","source":"#Print accuracy\nacc_NN = accuracy_score(test_y, predictions_NN_01)\nprint('Overall accuracy of Neural Network model:', acc_NN)","execution_count":10,"outputs":[]},{"metadata":{"_uuid":"80bdf03f267cfd80676f32f55205bb34c09a3d4b","_cell_guid":"e5bea720-9095-4e68-9bf9-76836a6eddac","trusted":true},"cell_type":"code","source":"#Print Area Under Curve\nfalse_positive_rate, recall, thresholds = roc_curve(test_y, predictions_NN_prob)\nroc_auc = auc(false_positive_rate, recall)\nplt.figure()\nplt.title('Receiver Operating Characteristic (ROC)')\nplt.plot(false_positive_rate, recall, 'b', label = 'AUC = %0.3f' %roc_auc)\nplt.legend(loc='lower right')\nplt.plot([0,1], [0,1], 'r--')\nplt.xlim([0.0,1.0])\nplt.ylim([0.0,1.0])\nplt.ylabel('Recall')\nplt.xlabel('Fall-out (1-Specificity)')\nplt.show()","execution_count":11,"outputs":[]},{"metadata":{"_uuid":"961a51c740957591c0810d421e729da7f94661c5","_cell_guid":"f8900057-eb12-4c52-8ad0-0cd9cafcebeb","trusted":true},"cell_type":"code","source":"#Print Confusion Matrix\ncm = confusion_matrix(test_y, predictions_NN_01)\nlabels = ['No Default', 'Default']\nplt.figure(figsize=(8,6))\nsns.heatmap(cm,xticklabels=labels, yticklabels=labels, annot=True, fmt='d', cmap=\"Blues\", vmin = 0.2);\nplt.title('Confusion Matrix')\nplt.ylabel('True Class')\nplt.xlabel('Predicted Class')\nplt.show()","execution_count":12,"outputs":[]}],"metadata":{"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"}},"nbformat":4,"nbformat_minor":1}