{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import sys\n!cp ../input/rapids/rapids.0.16.0 /opt/conda/envs/rapids.tar.gz\n!cd /opt/conda/envs/ && tar -xzvf rapids.tar.gz > /dev/null\nsys.path = [\"/opt/conda/envs/rapids/lib/python3.7/site-packages\"] + sys.path\nsys.path = [\"/opt/conda/envs/rapids/lib/python3.7\"] + sys.path\nsys.path = [\"/opt/conda/envs/rapids/lib\"] + sys.path\n!cp /opt/conda/envs/rapids/lib/libxgboost.so /opt/conda/lib/","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport cudf\nimport cupy as cp\nfrom cuml.neighbors import KNeighborsClassifier\nfrom cuml.linear_model import LogisticRegression\nfrom cuml import RandomForestClassifier\nfrom cuml.svm import SVC\n\nfrom sklearn.model_selection import KFold, train_test_split\nfrom sklearn.metrics import roc_auc_score, log_loss\nimport gc","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_InceptionResNetV2 = np.load('../input/melanoma-pretrained-embeddings/Pretrained/test_InceptionResNetV2.npy')\ntrain_InceptionResNetV2 = np.load('../input/melanoma-pretrained-embeddings/Pretrained/train_InceptionResNetV2.npy')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_0 = pd.read_csv('../input/siim-isic-melanoma-classification/train.csv')\ntarget = train_0.target.values","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_lr_oof_0 = np.zeros((train_InceptionResNetV2.shape[0], ))\ntest_lr_preds_0 = 0\n\nn_splits = 8\nn_seeds = 5\n\nfor ii in range(n_seeds):\n    kf = KFold(n_splits=n_splits, random_state=137, shuffle=True)\n\n    for jj, (train_index, val_index) in enumerate(kf.split(train_InceptionResNetV2)):\n        print(\"Fitting fold\", jj+1)\n        train_features = train_InceptionResNetV2[train_index]\n        train_target = target[train_index]\n\n        val_features = train_InceptionResNetV2[val_index]\n        val_target = target[val_index]\n\n        model = LogisticRegression(C=1.7, max_iter=120)\n        model.fit(train_features, train_target)\n        \n\n        val_pred = model.predict_proba(val_features)[:,1]\n        test_lr_preds_0 += model.predict_proba(test_InceptionResNetV2)[:,1]/(n_splits*n_seeds)\n        train_lr_oof_0[val_index] += val_pred/n_seeds\n        print(\"Fold AUC:\", roc_auc_score(val_target, val_pred))\n        del train_features, train_target, val_features, val_target\n        gc.collect()\n    \nprint(roc_auc_score(target, train_lr_oof_0))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_knn_oof_0 = np.zeros((train_InceptionResNetV2.shape[0], ))\ntest_knn_preds_0 = 0\n\nn_splits = 8\n\nn_seeds = 5\n\nfor ii in range(n_seeds):\n    kf = KFold(n_splits=n_splits, random_state=137, shuffle=True)\n\n    for jj, (train_index, val_index) in enumerate(kf.split(train_InceptionResNetV2)):\n        print(\"Fitting fold\", jj+1)\n        train_features = train_InceptionResNetV2[train_index]\n        train_target = target[train_index]\n\n        val_features = train_InceptionResNetV2[val_index]\n        val_target = target[val_index]\n\n        model = KNeighborsClassifier(n_neighbors=170)\n        model.fit(train_features, train_target)\n        val_pred = model.predict_proba(val_features)[:,1]\n        test_knn_preds_0 += model.predict_proba(test_InceptionResNetV2)[:,1]/(n_splits*n_seeds)\n        train_knn_oof_0[val_index] += val_pred/n_seeds\n        print(\"Fold AUC:\", roc_auc_score(val_target, val_pred))\n        del train_features, train_target, val_features, val_target\n        gc.collect()\n\nprint(roc_auc_score(target, train_knn_oof_0))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\ntrain_svc_oof_0 = np.zeros((train_InceptionResNetV2.shape[0], ))\ntest_svc_preds_0 = 0\n\nn_splits = 8\n\nn_seeds = 1\n\nfor ii in range(n_seeds):\n    kf = KFold(n_splits=n_splits, random_state=137, shuffle=True)\n\n    for jj, (train_index, val_index) in enumerate(kf.split(train_InceptionResNetV2)):\n        print(\"Fitting fold\", jj+1)\n        train_features = train_InceptionResNetV2[train_index]\n        train_target = target[train_index]\n\n        val_features = train_InceptionResNetV2[val_index]\n        val_target = target[val_index]\n\n        model = SVC(C=0.5, probability=True)\n        model.fit(train_features, train_target)\n        val_pred = model.predict_proba(val_features)[:,1]\n        test_svc_preds_0 += model.predict_proba(test_InceptionResNetV2)[:,1]/(n_splits*n_seeds)\n        train_svc_oof_0[val_index] += val_pred/n_seeds\n        print(\"Fold AUC:\", roc_auc_score(val_target, val_pred))\n        del train_features, train_target, val_features, val_target\n        gc.collect()\n\nprint(roc_auc_score(target, train_svc_oof_0))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"0.8216398218361436","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\ntrain_rfc_oof_0 = np.zeros((train_InceptionResNetV2.shape[0], ))\ntest_rfc_preds_0 = 0\n\ncu_rf_params = {'n_estimators': 1000,\n    'max_depth': 17,\n    'n_bins': 15,\n    'n_streams': 8\n}\n\nn_splits = 8\nkf = KFold(n_splits=n_splits, random_state=137, shuffle=True)\n\nfor jj, (train_index, val_index) in enumerate(kf.split(train_InceptionResNetV2)):\n    print(\"Fitting fold\", jj+1)\n    train_features = train_InceptionResNetV2[train_index]\n    train_target = target[train_index]\n    \n    val_features = train_InceptionResNetV2[val_index]\n    val_target = target[val_index]\n    \n    model = RandomForestClassifier(**cu_rf_params)\n    model.fit(train_features, train_target)\n    val_pred = model.predict_proba(val_features)[:,1]\n    test_rfc_preds_0 += model.predict_proba(test_InceptionResNetV2)[:,1]/n_splits\n    train_rfc_oof_0[val_index] = val_pred\n    print(\"Fold AUC:\", roc_auc_score(val_target, val_pred))\n    del train_features, train_target, val_features, val_target\n    gc.collect()\n    \nprint(roc_auc_score(target, train_rfc_oof_0))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"0.8370434403843127","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sample_submission = pd.read_csv('../input/siim-isic-melanoma-classification/sample_submission.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sample_submission['target'] = test_lr_preds_0\nsample_submission.to_csv('InceptionResNetV2_pretrained_LR.csv', index=False)\nsample_submission.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sample_submission['target'] = test_knn_preds_0\nsample_submission.to_csv('InceptionResNetV2_pretrained_KNN.csv', index=False)\nsample_submission.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sample_submission['target'] = test_svc_preds_0\nsample_submission.to_csv('InceptionResNetV2_pretrained_SVC.csv', index=False)\nsample_submission.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sample_submission['target'] = test_rfc_preds_0\nsample_submission.to_csv('InceptionResNetV2_pretrained_RFC.csv', index=False)\nsample_submission.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(roc_auc_score(target, 0.9*train_knn_oof_0+0.1*train_lr_oof_0))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(roc_auc_score(target, 0.5*train_knn_oof_0+0.5*train_lr_oof_0))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(roc_auc_score(target, (0.45*train_knn_oof_0+0.45*train_lr_oof_0+0.1*train_rfc_oof_0)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(roc_auc_score(target, (0.35*train_knn_oof_0+0.35*train_lr_oof_0+0.1*train_rfc_oof_0+\n                             0.2*train_svc_oof_0)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(roc_auc_score(target, (0.225*train_knn_oof_0+0.225*train_lr_oof_0+0.05*train_rfc_oof_0+\n                             0.5*train_svc_oof_0)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sample_submission['target'] = 0.9*test_knn_preds_0+0.1*test_lr_preds_0\nsample_submission.to_csv('InceptionResNetV2_pretrained.csv', index=False)\nsample_submission.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sample_submission['target'] = 0.5*test_knn_preds_0+0.5*test_lr_preds_0\nsample_submission.to_csv('InceptionResNetV2_pretrained_2.csv', index=False)\nsample_submission.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sample_submission['target'] = (0.45*test_knn_preds_0+0.45*test_lr_preds_0+0.1*test_rfc_preds_0)\nsample_submission.to_csv('InceptionResNetV2_pretrained_3.csv', index=False)\nsample_submission.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sample_submission['target'] = (0.35*test_knn_preds_0+0.35*test_lr_preds_0+\n                               0.1*test_rfc_preds_0+0.2*test_svc_preds_0)\nsample_submission.to_csv('InceptionResNetV2_pretrained_4.csv', index=False)\nsample_submission.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sample_submission['target'] = (0.225*test_knn_preds_0+0.225*test_lr_preds_0+\n                               0.05*test_rfc_preds_0+0.5*test_svc_preds_0)\nsample_submission.to_csv('InceptionResNetV2_pretrained_4.csv', index=False)\nsample_submission.head()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}