{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\nimport seaborn as sns\nimport os\nimport cv2\nimport glob2\nfrom tqdm import tqdm\nfrom PIL import Image\nimport matplotlib.pyplot as plt\nimport lightgbm as lgb\nprint(os.listdir(\"../input\"))\n\nfrom sklearn.model_selection import train_test_split,StratifiedKFold\nfrom sklearn.metrics import roc_auc_score\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torchvision.models import resnet50, resnet34, densenet201, densenet121\nfrom torch.utils.data import Dataset, DataLoader\n","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"train_df = pd.read_csv('../input/aptos2019-blindness-detection/train.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df = pd.read_csv('../input/aptos2019-blindness-detection/test.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sns.countplot(x='diagnosis',data=train_df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"len(os.listdir('../input/aptos2019-blindness-detection/train_images'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"len(os.listdir('../input/aptos2019-blindness-detection/test_images'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = glob2.glob('../input/aptos2019-blindness-detection/train_images/*.png')\ntest = glob2.glob('../input/aptos2019-blindness-detection/test_images/*.png')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def read_image(filename):\n    img = cv2.imread(str(filename))\n    \n    x_tot = img.mean() #image statistics\n    x_rot2 = img.std()\n    return x_tot, x_rot2\n\ndef get_stats(stats): # get dataset statistics \n    x_tot, x2_tot = 0.0, 0.0\n    for x, x2 in stats:\n        x_tot += x\n        x2_tot += x2\n    \n    img_avr =  x_tot/len(stats)\n    img_std = x2_tot/len(stats)\n    print('mean:',img_avr, ', std:', img_std)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"trn_stats = []\nfor fname in tqdm(train, total=len(train)):\n    trn_stats.append(read_image(fname))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_stats = []        \nfor fname in tqdm(test, total=len(test)):\n    test_stats.append(read_image(fname))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"get_stats(trn_stats)\nget_stats(test_stats)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"IMG_SIZE = 512\nBATCH_SIZE = 16","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\ndef img_to_torch(image):\n    return torch.from_numpy(np.transpose(image, (2, 0, 1)))\n\ndef pad_to_square(image):\n    h, w = image.shape[0:2]\n    new_size = max(h, w)\n    delta_top = (new_size-h)//2\n    delta_bottom = new_size-h-delta_top\n    delta_left = (new_size-w)//2\n    delta_right = new_size-delta_left-w\n    new_im = cv2.copyMakeBorder(image, delta_top, delta_bottom, delta_left, delta_right, \n                                cv2.BORDER_CONSTANT,  value=[0,0,0])\n    return new_im\n\nclass AptosDataset(Dataset):\n    def __init__(self, df,datatype='train'):\n        self.df = df\n        self.datatype = datatype\n        self.image_files_list = [f'../input/aptos2019-blindness-detection/{self.datatype}_images/{i}.png' for i in df['id_code'].values]\n        self.cache = {}\n        \n    def __len__(self):\n        return len(self.df)\n    \n    def __getitem__(self, index):\n        if index not in range(0, len(self.df)):\n            return self.__getitem__(np.random.randint(0, self.__len__()))\n        \n        # only take on channel\n#         if index not in self.cache:\n        image = cv2.imread(self.image_files_list[index])\n        image = pad_to_square(image)\n        image = cv2.resize(image, (IMG_SIZE, IMG_SIZE))\n#             self.cache[index] = img_to_torch(image)\n\n        return img_to_torch(image)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_image = AptosDataset(train_df,datatype='train')\ntrain_image_loader = DataLoader(train_image, batch_size=BATCH_SIZE, shuffle=False, \n                       num_workers=1, pin_memory=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class ResnetModel(nn.Module):\n    def __init__(self, resnet_fun=resnet50, freeze_basenet = True):\n        super(ResnetModel, self).__init__()\n        self.resnet = resnet_fun(pretrained=False)\n        if freeze_basenet:\n            for p in self.resnet.parameters():\n                p.requires_grad = False\n       \n    def init_resnet(self, path):\n        state = torch.load(path)\n        self.resnet.load_state_dict(state)\n        \n    def forward(self, x):\n        batch_size = x.shape[0]\n        x = x/255.0\n        mean = [0.485, 0.456, 0.406]\n        std = [0.229, 0.224, 0.225]\n        x = torch.cat([\n            (x[:, [0]] - mean[0]) / std[0],\n            (x[:, [1]] - mean[1]) / std[1],\n            (x[:, [2]] - mean[2]) / std[2],\n        ], 1)\n        x = self.resnet.conv1(x)\n        x = self.resnet.bn1(x)\n        x = self.resnet.relu(x)\n        x = self.resnet.maxpool(x)\n        x = self.resnet.layer1(x)\n        x = self.resnet.layer2(x)\n        x = self.resnet.layer3(x)\n        x = self.resnet.layer4(x)\n        x = F.adaptive_avg_pool2d(x, output_size=1).view(batch_size, -1)\n        return x","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"resnet50_feature = []\nmodel = ResnetModel()\nmodel.init_resnet('../input/pytorch-pretrained-image-models/resnet50.pth')\nmodel.cuda()\nmodel.eval()\nwith torch.no_grad():\n    for img_batch in tqdm(train_image_loader):\n        img_batch = img_batch.float().cuda()\n        y_pred = model(img_batch)\n        resnet50_feature.append(y_pred.cpu().numpy()) \nresnet50_feature = np.vstack(resnet50_feature)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"RES50_IMG_FEATURE_DIM = resnet50_feature.shape[1]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"resnet50_feature_df = pd.DataFrame(resnet50_feature, dtype=np.float32,\n                                   columns=['resnet50_%d'%i for i in range(RES50_IMG_FEATURE_DIM)])\nresnet50_feature_df['id_code'] = train_df['id_code'].values","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"resnet50_feature_df_avg = resnet50_feature_df.groupby('id_code').agg('mean').reset_index()\nresnet50_feature_df_avg.columns = ['id_code']+['resnet50_mean_%d'%i for i in range(RES50_IMG_FEATURE_DIM)]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"resnet50_feature_df_avg.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"resnet50_feature_train = train_df[['id_code','diagnosis']].merge(resnet50_feature_df_avg, on='id_code', how='left')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"resnet50_feature_train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_image = AptosDataset(test_df,datatype='test')\ntest_image_loader = DataLoader(test_image, batch_size=BATCH_SIZE, shuffle=False, \n                       num_workers=1, pin_memory=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"resnet50_feature = []\nmodel = ResnetModel()\nmodel.init_resnet('../input/pytorch-pretrained-image-models/resnet50.pth')\nmodel.cuda()\nmodel.eval()\nwith torch.no_grad():\n    for img_batch in tqdm(test_image_loader):\n        img_batch = img_batch.float().cuda()\n        y_pred = model(img_batch)\n        resnet50_feature.append(y_pred.cpu().numpy()) \nresnet50_feature = np.vstack(resnet50_feature)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"RES50_IMG_FEATURE_DIM = resnet50_feature.shape[1]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"resnet50_feature_df = pd.DataFrame(resnet50_feature, dtype=np.float32,\n                                   columns=['resnet50_%d'%i for i in range(RES50_IMG_FEATURE_DIM)])\nresnet50_feature_df['id_code'] = test_df['id_code'].values\n#resnet50_feature_df['PicID'] = image_df['PicID'].values\nresnet50_feature_df_avg = resnet50_feature_df.groupby('id_code').agg('mean').reset_index()\nresnet50_feature_df_avg.columns = ['id_code']+['resnet50_mean_%d'%i for i in range(RES50_IMG_FEATURE_DIM)]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"resnet50_feature_test = test_df[['id_code']].merge(resnet50_feature_df_avg, on='id_code', how='left')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"resnet50_feature_test.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"lgb_params = {\n    'boosting_type': 'gbdt',\n    'objective': 'binary',\n    'metric': 'auc',\n    'verbose': 1,\n    'learning_rate': 0.05,\n    'num_leaves': 31,\n    'feature_fraction': 0.7,\n    'min_data_in_leaf': 200,\n    'bagging_fraction': 0.8,\n    'bagging_freq': 20,\n    'min_hessian': 0.01,\n    'feature_fraction_seed': 2,\n    'bagging_seed': 3,\n    \"seed\": 1234\n}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"features = [c for c in resnet50_feature_train.columns if c not in ['id_code', 'diagnosis']]\n\nlen_train = len(resnet50_feature_train)\nresnet50_feature_train['target'] = 1\nresnet50_feature_train = resnet50_feature_train.append(resnet50_feature_test).reset_index(drop = True)\nresnet50_feature_train['target'] = resnet50_feature_train['target'].fillna(0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"resnet50_feature_train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"skf = StratifiedKFold(n_splits=5, shuffle=True, random_state=1234)\noof = resnet50_feature_train[['id_code', 'target']]\noof['predict'] = 0\nval_aucs = []","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for fold, (trn_idx, val_idx) in enumerate(skf.split(resnet50_feature_train, resnet50_feature_train['target'])):\n    X_train, y_train = resnet50_feature_train.iloc[trn_idx][features], resnet50_feature_train.iloc[trn_idx]['target']\n    X_valid, y_valid = resnet50_feature_train.iloc[val_idx][features], resnet50_feature_train.iloc[val_idx]['target']\n    trn_data = lgb.Dataset(X_train, label=y_train)\n    val_data = lgb.Dataset(X_valid, label=y_valid)\n    evals_result = {}\n    lgb_clf = lgb.train(lgb_params,\n                        trn_data,\n                        7500,\n                        valid_sets=[val_data],\n                        early_stopping_rounds=100,\n                        verbose_eval=50,\n                        evals_result=evals_result)\n\n    p_valid = lgb_clf.predict(X_valid[features], num_iteration=lgb_clf.best_iteration)\n\n    oof['predict'][val_idx] = p_valid\n    val_score = roc_auc_score(y_valid, p_valid)\n    val_aucs.append(val_score)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"mean_auc = np.mean(val_aucs)\nstd_auc = np.std(val_aucs)\nall_auc = roc_auc_score(oof['target'], oof['predict'])\nprint(\"Mean auc: %.9f, std: %.9f. All auc: %.9f.\" % (mean_auc, std_auc, all_auc))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Since AUC is ` 0.950535932`, LGB can easily differentiate between `train` and `test` set. This means that they come from different distribution. So expect mismatch between CV and public LB. Try making the `train` and `test` set have similar distribution.  "}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}