{"cells":[{"metadata":{},"cell_type":"markdown","source":"Hi,\nThank to:\n    + https://www.kaggle.com/anshuls235/melanoma-eda-and-prediction/notebook\nTry:\n    + Stacking model: https://www.kaggle.com/truonghoang/stacking-ensemble-on-my-submissions?scriptVersionId=37425177\n    + Using feature by other methods:\n        DenseNet169      : 0.869 n_estimators=1000\n        DenseNet201      : 0.872 n_estimators=1000\n        InceptionResNetV2: 0.836 n_estimators=10000\n        VGG19            : 0.858 n_estimators=10000\n        VGG19            : 0.850 n_estimators=1000\n        VGG16            : 0.841 n_estimators=10000\n        NASNetLarge      : 0.847 n_estimators=1000\n        ResNet50         : 0.873 n_estimators=1000\n    + Turning lightgbm params\n        Dense121 n_estimators=1000, learning_rate=0.03, max_depth=8, colsample_bytree=0.9, num_leaves=50, early_stopping_rounds=20 256: 0.884\n        Dense121 n_estimators=1000, learning_rate=0.03, max_depth=8, colsample_bytree=0.9, num_leaves=50, early_stopping_rounds=20 384: 0.856\n        Dense121 n_estimators=1000, learning_rate=0.03, max_depth=8, colsample_bytree=0.9, num_leaves=50, early_stopping_rounds=20 512: 0.861\n        Dense121 n_estimators=10000, learning_rate=0.1, max_depth=8, colsample_bytree=0.8,early_stopping_rounds=200                   : 0.876","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"# Import Libraries","execution_count":null},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport cv2\nimport os, gc, random\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n\nfrom tqdm import tqdm, tqdm_notebook\nfrom lightgbm import LGBMClassifier\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.model_selection import StratifiedKFold\nimport tensorflow as tf\nfrom sklearn.metrics import roc_auc_score\nfrom tensorflow.keras.models import Model\nfrom tensorflow.keras.layers import GlobalAveragePooling2D, Input, Lambda, AveragePooling1D\nfrom tensorflow.keras import backend as K\nfrom sklearn.preprocessing import MinMaxScaler\n\n\nfrom tensorflow.keras.applications.densenet import preprocess_input, DenseNet121 as ModelPredict\n# from tensorflow.keras.applications.densenet import preprocess_input, DenseNet169 as ModelPredict\n# from tensorflow.keras.applications.densenet import preprocess_input, DenseNet201 as ModelPredict\n# from tensorflow.keras.applications.nasnet import preprocess_input, NASNetLarge as ModelPredict\n# from tensorflow.keras.applications.inception_resnet_v2 import preprocess_input, InceptionResNetV2 as ModelPredict\n# from tensorflow.keras.applications.vgg19 import preprocess_input, VGG19 as ModelPredict\n# from tensorflow.keras.applications.resnet import preprocess_input, ResNet50 as ModelPredict","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Configure","execution_count":null},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"# Data access\nfrom kaggle_datasets import KaggleDatasets\nGCS_DS_PATH = KaggleDatasets().get_gcs_path('siim-isic-melanoma-classification')\n# GCS_DS_PATH = './'\n\nimg_size = 256\nbatch_size = 16\nSEED = 2020\n\ndf_train = pd.read_csv(GCS_DS_PATH + '/train.csv')\ndf_test = pd.read_csv(GCS_DS_PATH + '/test.csv')\n\ntrain_img_path = '/kaggle/input/siim-isic-melanoma-classification/jpeg/train/'\ntest_img_path = '/kaggle/input/siim-isic-melanoma-classification/jpeg/test/'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def seed_everything(seed=0):\n    np.random.seed(seed)\n    tf.random.set_seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    os.environ['TF_DETERMINISTIC_OPS'] = '1'\n    random.seed(SEED)\n\nseed_everything(SEED)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Create feature","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def resize_image(img):\n    old_size = img.shape[:2]\n    if old_size[1] == img_size:\n        return img\n    ratio = float(img_size)/max(old_size)\n    new_size = tuple([int(x*ratio) for x in old_size])\n    img = cv2.resize(img, (new_size[1],new_size[0]))\n    delta_w = img_size - new_size[1]\n    delta_h = img_size - new_size[0]\n    top, bottom = delta_h//2, delta_h-(delta_h//2)\n    left, right = delta_w//2, delta_w-(delta_w//2)\n    color = [0,0,0]\n    new_img = cv2.copyMakeBorder(img, top, bottom, left, right, cv2.BORDER_CONSTANT, value=color)\n    return new_img\n\ndef load_image(path, img_id):\n    path = os.path.join(path,img_id+'.jpg')\n    img = cv2.imread(path)\n    new_img = resize_image(img)\n    new_img = preprocess_input(new_img)\n    return new_img\n\ndef get_model_feature():\n    inp = Input((img_size, img_size, 3))\n    backbone = ModelPredict(input_tensor=inp, include_top=False)\n    x = backbone.output\n    x = GlobalAveragePooling2D()(x)\n    x = Lambda(lambda x: K.expand_dims(x, axis=-1))(x)\n    x = AveragePooling1D(4)(x)\n    out = Lambda(lambda x: x[:,:,0])(x)\n    return Model(inp, out)\n\ndef create_feature(model_feature, df, img_path, out=\"features.csv\"):\n    img_ids = df.image_name.values\n    n_batches = len(img_ids)//batch_size + 1\n    features = {}\n    for b in tqdm_notebook(range(n_batches)):\n        start = b*batch_size\n        end = (b+1)*batch_size\n        batch_ids = img_ids[start:end]\n        batch_images = np.zeros((len(batch_ids),img_size,img_size,3))\n        for i, img_id in enumerate(batch_ids):\n            batch_images[i] = load_image(img_path, img_id)\n        batch_preds = model_feature.predict(batch_images)\n        for i, img_id in enumerate(batch_ids):\n            features[img_id] = batch_preds[i]\n\n    feats = pd.DataFrame.from_dict(features, orient='index')\n    feats.to_csv(out)\n    return feats","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# train_feats = pd.read_csv('/kaggle/input/siimisic-256x256/train_img_features_dense121.csv')\n# test_feats = pd.read_csv('/kaggle/input/siimisic-256x256/test_img_features_dense121.csv')\n\nmf = get_model_feature()\ncreate_feature(mf, df_train, train_img_path, out='train_img_features.csv')\ncreate_feature(mf, df_test, test_img_path, out='test_img_features.csv')\ntrain_feats = pd.read_csv('train_img_features.csv')\ntest_feats = pd.read_csv('test_img_features.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_feats.set_index(train_feats.columns[0], inplace=True)\ntest_feats.set_index(test_feats.columns[0], inplace=True)\n\n#Combine the image and tabular data\ndf_train_full = pd.merge(df_train, train_feats, how='inner', left_on='image_name', right_index=True)\ndf_test_full = pd.merge(df_test, test_feats, how='inner', left_on='image_name', right_index=True)\n\n#Drop the unwanted columns\ntrain = df_train_full.drop(['image_name','patient_id','diagnosis','benign_malignant'],axis=1)\ntest = df_test_full.drop(['image_name','patient_id'],axis=1)\n\n#Label Encode categorical features\ntrain.sex.fillna('NaN',inplace=True)\ntest.sex.fillna('NaN',inplace=True)\ntrain.anatom_site_general_challenge.fillna('NaN',inplace=True)\ntest.anatom_site_general_challenge.fillna('NaN',inplace=True)\nle_sex = LabelEncoder()\nle_site = LabelEncoder()\ntrain.sex = le_sex.fit_transform(train.sex)\ntest.sex = le_sex.transform(test.sex)\ntrain.anatom_site_general_challenge = le_site.fit_transform(train.anatom_site_general_challenge)\ntest.anatom_site_general_challenge = le_site.transform(test.anatom_site_general_challenge)\ntrain['age_approx'] = train['age_approx'].fillna(0)\ntest['age_approx'] = test['age_approx'].fillna(0)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Train, Inference","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"Since, the data is imbalanced meaning there are a lot of records for benign tumours while very less records for malign tumour. So, I'll be using a Stratified K-fold for validation.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"folds = StratifiedKFold(n_splits=5, shuffle=True)\noof_preds = np.zeros(train.shape[0])\nsub_preds = np.zeros(test.shape[0])\nfeature_importance_df = pd.DataFrame()\nfeatures = [f for f in train.columns if f != 'target']\nfor n_fold, (train_idx, valid_idx) in enumerate(folds.split(train[features], train['target'])):\n    train_X, train_y = train[features].iloc[train_idx], train['target'].iloc[train_idx]\n    valid_X, valid_y = train[features].iloc[valid_idx], train['target'].iloc[valid_idx]\n    clf = LGBMClassifier(\n        #device='gpu',\n        n_estimators=1000,\n        learning_rate=0.03,\n        max_depth=8,\n        colsample_bytree=0.9,\n        num_leaves=50\n    )\n    print('*****Fold: {}*****'.format(n_fold))\n    clf.fit(train_X, train_y, eval_set=[(train_X, train_y), (valid_X, valid_y)],\n            eval_metric= 'auc', verbose= 20, early_stopping_rounds= 20)\n\n    oof_preds[valid_idx] = clf.predict_proba(valid_X, num_iteration=clf.best_iteration_)[:, 1]\n    sub_preds += clf.predict_proba(test[features], num_iteration=clf.best_iteration_)[:, 1] / folds.n_splits\n\n    fold_importance_df = pd.DataFrame()\n    fold_importance_df[\"feature\"] = features\n    fold_importance_df[\"importance\"] = clf.feature_importances_\n    fold_importance_df[\"fold\"] = n_fold + 1\n    feature_importance_df = pd.concat([feature_importance_df, fold_importance_df], axis=0)\n    print('Fold %2d AUC : %.6f' % (n_fold + 1, roc_auc_score(valid_y, oof_preds[valid_idx])))\n    del clf, train_X, train_y, valid_X, valid_y\n    gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def display_importances(feature_importance_df_):\n    cols = feature_importance_df_[[\"feature\", \"importance\"]].groupby(\"feature\").mean().sort_values(by=\"importance\", ascending=False)[:40].index\n    best_features = feature_importance_df_.loc[feature_importance_df_.feature.isin(cols)]\n    plt.figure(figsize=(8, 10))\n    sns.barplot(x=\"importance\", y=\"feature\", data=best_features.sort_values(by=\"importance\", ascending=False))\n    plt.title('LightGBM Features (averaged over folds)')\n    plt.tight_layout()\n    plt.savefig('lgbm_importances.png')\n    \ndisplay_importances(feature_importance_df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission = pd.DataFrame({\n    \"image_name\": df_test.image_name, \n    \"target\": sub_preds\n})\nsubmission.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}