{"cells":[{"metadata":{},"cell_type":"markdown","source":"## Overview\n\nThe purpose of this kernel is to take a look at the data, come up with some insights, and attempt to create a predictive model or two. This notebook is still **very** raw. I will work on it as my very limited time permits, and hope to expend it in the upcoming days and weeks.\n\n\n## Packages\n\nFirst, let's load a few useful Python packages. This section will keep growing in subsequent versions of this EDA.","execution_count":null},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport gc\nimport json\nimport math\nimport cv2\nimport PIL\nfrom PIL import Image\n\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom sklearn.model_selection import KFold\nfrom sklearn.metrics import roc_auc_score\nfrom sklearn.linear_model import LogisticRegression, Ridge\nfrom sklearn.preprocessing import OneHotEncoder\nfrom tqdm import tqdm\nfrom sklearn.decomposition import PCA\nimport os\nimport imagesize\n\n%matplotlib inline","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Let's see what files we have in the input directory:","execution_count":null},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import os\nprint(os.listdir(\"../input/siim-isic-melanoma-classification\"))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Loading Train and Test Data\ntrain = pd.read_csv(\"../input/siim-isic-melanoma-classification/train.csv\")\ntest = pd.read_csv(\"../input/siim-isic-melanoma-classification/test.csv\")\nprint(\"{} images in train set.\".format(train.shape[0]))\nprint(\"{} images in test set.\".format(test.shape[0]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Let's look at the distribution of teh target:","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"np.mean(train.target)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"So this is a binary classification problem with highly imbalanced data.","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"Let's now look at the distributions of various \"features\"","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(12, 5))\nplt.hist(train['age_approx'].values, bins=200)\nplt.title('Histogram age_approx counts in train')\nplt.xlabel('Value')\nplt.ylabel('Count')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Let's take a look at a few images.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"images = []\nfor i, image_id in enumerate(tqdm(train['image_name'].head(10))):\n    im = Image.open(f'../input/siim-isic-melanoma-classification/jpeg/train/{image_id}.jpg')\n    im = im.resize((128, )*2, resample=Image.LANCZOS)\n    images.append(im)\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"images[0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"images[1]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"images[3]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(12, 5))\nplt.hist(test['age_approx'].values, bins=200)\nplt.title('Histogram age_approx counts in test')\nplt.xlabel('Value')\nplt.ylabel('Count')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Now we will load some of the resized images (32x32 for now) and try to build some simple models. ","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"x_train_32 = np.load('../input/siimisic-melanoma-resized-images/x_train_32.npy')\nx_test_32 = np.load('../input/siimisic-melanoma-resized-images/x_test_32.npy')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"x_train_32.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"x_train_32 = x_train_32.reshape((x_train_32.shape[0], 32*32*3))\nx_train_32.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"x_test_32 = x_test_32.reshape((x_test_32.shape[0], 32*32*3))\nx_test_32.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y = train.target.values","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_oof = np.zeros((x_train_32.shape[0], ))\ntest_preds = 0\ntrain_oof.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"n_splits = 5\nkf = KFold(n_splits=n_splits, random_state=137, shuffle=True)\n\nfor jj, (train_index, val_index) in enumerate(kf.split(x_train_32)):\n    print(\"Fitting fold\", jj+1)\n    train_features = x_train_32[train_index]\n    train_target = y[train_index]\n    \n    val_features = x_train_32[val_index]\n    val_target = y[val_index]\n    \n    model = LogisticRegression(C=1, solver='lbfgs', multi_class='multinomial', max_iter=60)\n    model.fit(train_features, train_target)\n    val_pred = model.predict_proba(val_features)[:,1]\n    train_oof[val_index] = val_pred\n    print(\"Fold AUC:\", roc_auc_score(val_target, val_pred))\n    test_preds += model.predict_proba(x_test_32)[:,1]/n_splits\n    del train_features, train_target, val_features, val_target\n    gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(roc_auc_score(y, train_oof))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_oof_0_2 = np.zeros((x_train_32.shape[0], ))\ntest_preds_0_2 = 0\n\nn_splits = 5\nkf = KFold(n_splits=n_splits, random_state=137, shuffle=True)\n\nfor jj, (train_index, val_index) in enumerate(kf.split(x_train_32)):\n    print(\"Fitting fold\", jj+1)\n    train_features = x_train_32[train_index]\n    train_target = y[train_index]\n    \n    val_features = x_train_32[val_index]\n    val_target = y[val_index]\n    \n    model = LogisticRegression(C=5, solver='lbfgs', multi_class='multinomial', max_iter=80)\n    model.fit(train_features, train_target)\n    val_pred = model.predict_proba(val_features)[:,1]\n    train_oof_0_2[val_index] = val_pred\n    print(\"Fold AUC:\", roc_auc_score(val_target, val_pred))\n    test_preds_0_2 += model.predict_proba(x_test_32)[:,1]/n_splits\n    del train_features, train_target, val_features, val_target\n    gc.collect()\n    \nprint(roc_auc_score(y, train_oof_0_2))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(roc_auc_score(y, 0.95*train_oof+0.05*train_oof_0_2))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train['age_approx'].unique()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Wow, so we get an 0.82 AUC with just unravelled resized images and a simple Logistic Regression!","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"Let's now add some non-image features. We can start with sex, and one-hot encode it.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train['sex'] = (train['sex'].values == 'male')*1\ntest['sex'] = (test['sex'].values == 'male')*1\ntrain.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train['sex'].mean()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test['sex'].mean()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train['age_approx'].mean()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test['age_approx'].mean()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train['age_approx'] = train['age_approx'].fillna(train['age_approx'].mean())\ntest['age_approx'] = test['age_approx'].fillna(test['age_approx'].mean())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"x_train_32 = np.hstack([x_train_32, train['sex'].values.reshape(-1,1), train['age_approx'].values.reshape(-1,1)])\nx_test_32 = np.hstack([x_test_32, test['sex'].values.reshape(-1,1), test['age_approx'].values.reshape(-1,1)])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_oof_2 = np.zeros((x_train_32.shape[0], ))\ntest_preds_2 = 0\n\n\nn_splits = 5\nkf = KFold(n_splits=n_splits, random_state=137, shuffle=True)\n\nfor jj, (train_index, val_index) in enumerate(kf.split(x_train_32)):\n    print(\"Fitting fold\", jj+1)\n    train_features = x_train_32[train_index]\n    train_target = y[train_index]\n    \n    val_features = x_train_32[val_index]\n    val_target = y[val_index]\n    \n    model = LogisticRegression(C=1, solver='lbfgs', multi_class='multinomial', max_iter=50)\n    model.fit(train_features, train_target)\n    val_pred = model.predict_proba(val_features)[:,1]\n    train_oof_2[val_index] = val_pred\n    print(\"Fold AUC:\", roc_auc_score(val_target, val_pred))\n    test_preds_2 += model.predict_proba(x_test_32)[:,1]/n_splits\n    del train_features, train_target, val_features, val_target\n    gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(roc_auc_score(y, train_oof_2))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(roc_auc_score(y, 0.8*train_oof_2+0.2*train_oof))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(roc_auc_score(y, 0.5*train_oof_2+0.5*train_oof))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_preds.max()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_preds_2.max()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_oof_2_2 = np.zeros((x_train_32.shape[0], ))\ntest_preds_2_2 = 0\n\n\nn_splits = 5\nkf = KFold(n_splits=n_splits, random_state=137, shuffle=True)\n\nfor jj, (train_index, val_index) in enumerate(kf.split(x_train_32)):\n    print(\"Fitting fold\", jj+1)\n    train_features = x_train_32[train_index]\n    train_target = y[train_index]\n    \n    val_features = x_train_32[val_index]\n    val_target = y[val_index]\n    \n    model = LogisticRegression(C=5, solver='lbfgs', multi_class='multinomial', max_iter=80)\n    model.fit(train_features, train_target)\n    val_pred = model.predict_proba(val_features)[:,1]\n    train_oof_2_2[val_index] = val_pred\n    print(\"Fold AUC:\", roc_auc_score(val_target, val_pred))\n    test_preds_2_2 += model.predict_proba(x_test_32)[:,1]/n_splits\n    del train_features, train_target, val_features, val_target\n    gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(roc_auc_score(y, train_oof_2_2))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train['anatom_site_general_challenge'].unique()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test['anatom_site_general_challenge'].unique()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train['anatom_site_general_challenge'].mode()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test['anatom_site_general_challenge'].mode()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train['anatom_site_general_challenge'].fillna(train['anatom_site_general_challenge'].mode(), inplace=True)\ntest['anatom_site_general_challenge'].fillna(test['anatom_site_general_challenge'].mode(), inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train['anatom_site_general_challenge'] = train['anatom_site_general_challenge'].astype(str)\ntest['anatom_site_general_challenge'] = test['anatom_site_general_challenge'].astype(str)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test['anatom_site_general_challenge'].isnull().sum()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"x_train_32 = np.hstack([x_train_32, pd.get_dummies(train['anatom_site_general_challenge']).values])\nx_test_32 = np.hstack([x_test_32, pd.get_dummies(test['anatom_site_general_challenge']).values])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_oof_3 = np.zeros((x_train_32.shape[0], ))\ntest_preds_3 = 0\n\n\nn_splits = 5\nkf = KFold(n_splits=n_splits, random_state=137, shuffle=True)\n\nfor jj, (train_index, val_index) in enumerate(kf.split(x_train_32)):\n    print(\"Fitting fold\", jj+1)\n    train_features = x_train_32[train_index]\n    train_target = y[train_index]\n    \n    val_features = x_train_32[val_index]\n    val_target = y[val_index]\n    \n    model = LogisticRegression(C=1, solver='lbfgs', multi_class='multinomial', max_iter=60)\n    model.fit(train_features, train_target)\n    val_pred = model.predict_proba(val_features)[:,1]\n    train_oof_3[val_index] = val_pred\n    print(\"Fold AUC:\", roc_auc_score(val_target, val_pred))\n    test_preds_3 += model.predict_proba(x_test_32)[:,1]/n_splits\n    del train_features, train_target, val_features, val_target\n    gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(roc_auc_score(y, train_oof_3))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_oof_4 = np.zeros((x_train_32.shape[0], ))\ntest_preds_4 = 0\n\n\nn_splits = 5\nkf = KFold(n_splits=n_splits, random_state=137, shuffle=True)\n\nfor jj, (train_index, val_index) in enumerate(kf.split(x_train_32)):\n    print(\"Fitting fold\", jj+1)\n    train_features = x_train_32[train_index]\n    train_target = y[train_index]\n    \n    val_features = x_train_32[val_index]\n    val_target = y[val_index]\n    \n    model = LogisticRegression(C=5, max_iter=80)\n    model.fit(train_features, train_target)\n    val_pred = model.predict_proba(val_features)[:,1]\n    train_oof_4[val_index] = val_pred\n    print(\"Fold AUC:\", roc_auc_score(val_target, val_pred))\n    test_preds_4 += model.predict_proba(x_test_32)[:,1]/n_splits\n    del train_features, train_target, val_features, val_target\n    gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(roc_auc_score(y, train_oof_4))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(roc_auc_score(y, 0.25*train_oof_2+0.25*train_oof+0.25*train_oof_3+0.25*train_oof_4))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Taken from raddar's notebook: https://www.kaggle.com/raddar/simple-baseline-revamped","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"im_shape_test = []\nim_shape_train = []\n\nfor i in range(train.shape[0]):\n    im_shape_train.append(imagesize.get('../input/siim-isic-melanoma-classification/jpeg/train/'+train['image_name'][i]+'.jpg'))\nfor i in range(test.shape[0]):\n    im_shape_test.append(imagesize.get('../input/siim-isic-melanoma-classification/jpeg/test/'+test['image_name'][i]+'.jpg'))\n    \n\ntrain['dim'] = im_shape_train\ntest['dim'] = im_shape_test","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train['dim'] == (6000,4000)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train['dim'] == (1872,1053)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"(train['dim'] != (6000,4000)) & (train['dim'] != (1872,1053))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train['dim_1'] = (train['dim'] == (6000,4000))\ntrain['dim_1'] = train['dim_1'].values*1\ntrain['dim_2'] = (train['dim'] == (1872,1053))\ntrain['dim_2'] = train['dim_2'].values*1\ntrain['dim_3'] = (train['dim'] != (6000,4000)) & (train['dim'] != (1872,1053))\ntrain['dim_3'] = train['dim_3'].values*1\ntrain['dim_3']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test['dim_1'] = (test['dim'] == (6000,4000))\ntest['dim_1'] = test['dim_1'].values*1\ntest['dim_2'] = (test['dim'] == (1872,1053))\ntest['dim_2'] = test['dim_2'].values*1\ntest['dim_3'] = (test['dim'] != (6000,4000)) & (test['dim'] != (1872,1053))\ntest['dim_3'] = test['dim_3'].values*1\ntest['dim_3']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"x_train_32 = np.hstack([x_train_32, train['dim_1'].values.reshape(-1,1), train['dim_2'].values.reshape(-1,1), train['dim_3'].values.reshape(-1,1)])\nx_test_32 = np.hstack([x_test_32, test['dim_1'].values.reshape(-1,1), test['dim_2'].values.reshape(-1,1), test['dim_3'].values.reshape(-1,1)])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_oof_4_2 = np.zeros((x_train_32.shape[0], ))\ntest_preds_4_2 = 0\n\n\nn_splits = 5\nkf = KFold(n_splits=n_splits, random_state=137, shuffle=True)\n\nfor jj, (train_index, val_index) in enumerate(kf.split(x_train_32)):\n    print(\"Fitting fold\", jj+1)\n    train_features = x_train_32[train_index]\n    train_target = y[train_index]\n    \n    val_features = x_train_32[val_index]\n    val_target = y[val_index]\n    \n    model = LogisticRegression(C=0.9, max_iter=50)\n    model.fit(train_features, train_target)\n    val_pred = model.predict_proba(val_features)[:,1]\n    train_oof_4_2[val_index] = val_pred\n    print(\"Fold AUC:\", roc_auc_score(val_target, val_pred))\n    test_preds_4_2 += model.predict_proba(x_test_32)[:,1]/n_splits\n    del train_features, train_target, val_features, val_target\n    gc.collect()\n    \nprint(roc_auc_score(y, train_oof_4_2))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"0.8262080489449671","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\npca = PCA(n_components=0.99)\npca.fit(x_train_32)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pca.n_components_","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"x_train_32 = pca.transform(x_train_32)\nx_test_32 = pca.transform(x_test_32)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_oof_5 = np.zeros((x_train_32.shape[0], ))\ntest_preds_5 = 0\n\n\nn_splits = 5\nkf = KFold(n_splits=n_splits, random_state=137, shuffle=True)\n\nfor jj, (train_index, val_index) in enumerate(kf.split(x_train_32)):\n    print(\"Fitting fold\", jj+1)\n    train_features = x_train_32[train_index]\n    train_target = y[train_index]\n    \n    val_features = x_train_32[val_index]\n    val_target = y[val_index]\n    \n    model = LogisticRegression(C=0.1, max_iter=6)\n    model.fit(train_features, train_target)\n    val_pred = model.predict_proba(val_features)[:,1]\n    train_oof_5[val_index] = val_pred\n    print(\"Fold AUC:\", roc_auc_score(val_target, val_pred))\n    test_preds_5 += model.predict_proba(x_test_32)[:,1]/n_splits\n    del train_features, train_target, val_features, val_target\n    gc.collect()\n    \nprint(roc_auc_score(y, train_oof_5))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"0.7910534268464863","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(roc_auc_score(y, 0.988*(0.27*train_oof_2+0.27*train_oof+0.27*train_oof_3+0.19*train_oof_4)+0.012*train_oof_5))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(roc_auc_score(y, 1.082*(0.99*(0.25*train_oof_2+0.25*train_oof+0.25*train_oof_3+0.25*train_oof_4)+0.01*train_oof_5)-0.082*(train_oof_0_2+train_oof_2_2)/2))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"How about 64x64 images?","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"x_train_64 = np.load('../input/siimisic-melanoma-resized-images/x_train_64.npy')\nx_test_64 = np.load('../input/siimisic-melanoma-resized-images/x_test_64.npy')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"x_train_64 = x_train_64.reshape((x_train_64.shape[0], 64*64*3))\nx_train_64.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"x_test_64 = x_test_64.reshape((x_test_64.shape[0], 64*64*3))\nx_test_64.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_oof_6 = np.zeros((x_train_64.shape[0], ))\ntest_preds_6 = 0\n\n\nn_splits = 5\nkf = KFold(n_splits=n_splits, random_state=137, shuffle=True)\n\nfor jj, (train_index, val_index) in enumerate(kf.split(x_train_64)):\n    print(\"Fitting fold\", jj+1)\n    train_features = x_train_64[train_index]\n    train_target = y[train_index]\n    \n    val_features = x_train_64[val_index]\n    val_target = y[val_index]\n    \n    model = LogisticRegression(C=0.1, max_iter=45)\n    model.fit(train_features, train_target)\n    val_pred = model.predict_proba(val_features)[:,1]\n    train_oof_6[val_index] = val_pred\n    print(\"Fold AUC:\", roc_auc_score(val_target, val_pred))\n    test_preds_6 += model.predict_proba(x_test_64)[:,1]/n_splits\n    del train_features, train_target, val_features, val_target\n    gc.collect()\n    \nprint(roc_auc_score(y, train_oof_6))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"0.8213209504598062","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(roc_auc_score(y, 0.73*(1.1*(0.99*(0.25*train_oof_2+0.25*train_oof+0.25*train_oof_3+0.25*train_oof_4)+0.01*train_oof_5)-0.1*(train_oof_0_2+train_oof_2_2)/2)+0.27*train_oof_6))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(roc_auc_score(y, 0.9*(0.73*(1.1*(0.99*(0.25*train_oof_2+0.25*train_oof+0.25*train_oof_3+0.25*train_oof_4)+0.01*train_oof_5)-0.1*(train_oof_0_2+train_oof_2_2)/2)+0.27*train_oof_6)+\n                   0.1*train_oof_4_2))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"0.8285058171399994","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Now let's make a submission.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"sample_submission = pd.read_csv('../input/siim-isic-melanoma-classification/sample_submission.csv')\nsample_submission.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sample_submission['target'] = 0.9*(0.73*(1.1*(0.99*(0.25*test_preds+0.25*test_preds_2+0.25*test_preds_3+0.25*test_preds_4)+0.015*test_preds_5)- 0.1*(0.5*test_preds_0_2+0.5*test_preds_2_2))+0.27*test_preds_6)+0.1*test_preds_4_2\nsample_submission.to_csv('submission_32x32_64x64_lr.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sample_submission['target'].max()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sample_submission['target'].min()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}