{"cells":[{"metadata":{},"cell_type":"markdown","source":"# submissionを行う  \ndatasetに作成したモデルのパラメータをnotebook内で読み込み、提出する。  \nこのnotebookはmelanoma_tachyon_simplemodel_v1の続きとなっている。  "},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"local = False\nDEBUG = False","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#before import process\nimport sys\nif local == True:\n    package_path = 'Q:/kaggle/efficientnet-pytorch/EfficientNet-PyTorch/EfficientNet-PyTorch-master'\nelse:\n    package_path = '../input/efficientnet-pytorch/EfficientNet-PyTorch/EfficientNet-PyTorch-master'\nsys.path.append(package_path)\n\n#imports\nimport os, warnings, random, time\nimport pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport cv2\nfrom IPython.display import display\nfrom tqdm import tqdm_notebook as tqdm\n\nimport torch\nimport albumentations\nfrom torch import nn, optim\nfrom torch.functional import F \nfrom torch.utils.data import Dataset, DataLoader\nfrom efficientnet_pytorch import model as enet\n\n#Data Augmentation用ライブラリ\nimport albumentations as A\n\n%matplotlib inline\nwarnings.filterwarnings('ignore')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"SEED = 69\ndef seed_everything(seed):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = True\n\nseed_everything(SEED)\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\nprint(device)\n\nProgress_Bar = True","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#params\nenet_type = 'efficientnet-b0'\nmodel_name = 'v1'\npretrained_models = \"../input/melanoma-tachyon-simplemodel-v1-models\"\nn_fold = 4\nTTA = 1\nbatch_size = 64\nimage_size = 224\n\nnum_workers = 2","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = pd.read_csv('/kaggle/input/jpeg-melanoma-256x256/train.csv')\ntest_df = pd.read_csv('/kaggle/input/jpeg-melanoma-256x256/test.csv')\ntrain_images = '../input/jpeg-melanoma-256x256/train'\ntest_images = '../input/jpeg-melanoma-256x256/test'\n\nif DEBUG:\n    train_df = train_df[:1000]\n    test_df = test_df[:500]\n    \n# One-hot encoding of anatom_site_general_challenge feature\nconcat = pd.concat([train_df['anatom_site_general_challenge'], test_df['anatom_site_general_challenge']], ignore_index=True)\ndummies = pd.get_dummies(concat, dummy_na=True, dtype=np.uint8, prefix='site')\ntrain_df = pd.concat([train_df, dummies.iloc[:train_df.shape[0]]], axis=1)\ntest_df = pd.concat([test_df, dummies.iloc[train_df.shape[0]:].reset_index(drop=True)], axis=1)\n\n# Sex features\ntrain_df['sex'] = train_df['sex'].map({'male': 1, 'female': 0})\ntest_df['sex'] = test_df['sex'].map({'male': 1, 'female': 0})\ntrain_df['sex'] = train_df['sex'].fillna(-1)\ntest_df['sex'] = test_df['sex'].fillna(-1)\n\n# Age features\ntrain_df['age_approx'] /= train_df['age_approx'].max()\ntest_df['age_approx'] /= test_df['age_approx'].max()\ntrain_df['age_approx'] = train_df['age_approx'].fillna(0)\ntest_df['age_approx'] = test_df['age_approx'].fillna(0)\n\ntrain_df['patient_id'] = train_df['patient_id'].fillna(0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"meta_features = ['sex', 'age_approx'] + [col for col in train_df.columns if 'site_' in col]\nmeta_features.remove('anatom_site_general_challenge')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pretrained_model = {\n        'efficientnet-b0': '../input/efficientnet-pytorch/efficientnet-b0-08094119.pth',\n        'efficientnet-b1': '../input/efficientnet-pytorch/efficientnet-b1-dbc7070a.pth',\n        'efficientnet-b2': '../input/efficientnet-pytorch/efficientnet-b2-27687264.pth',\n        'efficientnet-b3': '../input/efficientnet-pytorch/efficientnet-b3-c8376fa2.pth',\n        'efficientnet-b4': '../input/efficientnet-pytorch/efficientnet-b4-e116e8b3.pth',\n        'efficientnet-b5': '../input/efficientnet-pytorch/efficientnet-b5-586e6cc6.pth',\n        \n    }\nmodel_save_path = False\n\n\nclass enetv2(nn.Module):\n    def __init__(self, backbone, out_dim=1):\n        super(enetv2, self).__init__()\n        self.enet = enet.EfficientNet.from_name(backbone)\n        self.enet.load_state_dict(torch.load(pretrained_model[backbone]))\n\n        self.myfc = nn.Linear(self.enet._fc.in_features, out_dim)\n        self.enet._fc = nn.Identity()\n        self.sigmoid = nn.Sigmoid()\n\n    def extract(self, x):\n        return self.enet(x)\n\n    def forward(self, x):\n        x = self.extract(x)\n        x = self.myfc(x)\n        #x = self.sigmoid(x)\n        return x","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# モデルの読み込み  \nまずモデルの構造自体(enetv2)を読み込み、その後に保存しておいたパラメータをはめ込んでいく。  \nここのパラメータ数が合わなかったりするとエラーが出るので注意。  \n(ありがちなのが、output sizeが合わなかったり。)  "},{"metadata":{"trusted":true},"cell_type":"code","source":"def load_models(model_files,backbone):\n    models = []\n    for model_f in model_files:\n        model_f = os.path.join(pretrained_models, model_f)\n        model = enetv2(backbone, out_dim=1)\n        model.load_state_dict(torch.load(model_f, map_location=lambda storage, loc: storage), strict=True)\n        model.eval()\n        model.to(device)\n        models.append(model)\n        print(f'{model_f} loaded!')\n    return models","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"res = os.listdir(pretrained_models)\nprint(\"all:\")\nprint(res)\nmodel_files_path = []\nmodel_files_pickle_path = []\n\n#pytorchとpickleを分別\nfor i in res:\n    if i[-3:] == \".pt\":\n        model_files_path.append(i)\n    elif i[-7:] == \".pickle\":\n        model_files_pickle_path.append(i)\nprint(\"pytorch:\")\nprint(model_files_path)\nprint(\"pickle:\")\nprint(model_files_pickle_path)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#今回、pytorch側で出力したモデルはbestとfinalがある、finalだけに絞る。\nfor i in model_files_path:\n    if \"best\" in i:\n        model_files_path.remove(i)\nprint(\"pytorch:\")\nprint(model_files_path)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#pytorchで、パラメータの読み込み\nmodels = load_models(model_files_path,enet_type) #enet_type = 'efficientnet-b0'\n#print(models)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# オマケ pickleでの読み込み  \n今回、pickleはパラメータのみでなくモデル全体を読み込んでいるので、それを読み出してみる。  \nモデル構造を読み込んで、そこにパラメータを当てはめていくという過程が無い分、ちょっと楽。  "},{"metadata":{"trusted":true},"cell_type":"code","source":"import pickle\n\nmodels_from_pickle = []\n\nfor i in model_files_pickle_path:\n    i = os.path.join(pretrained_models, i)\n    \n    with open(i,mode=\"rb\") as fp:\n        model = pickle.load(fp)\n    model.eval()\n    model.to(device)\n    models.append(model)\n    print(f'{i} loaded!')\n    models_from_pickle.append(model)\n#print(models_from_pickle)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"以上。pickleの方のモデルはここで削除する。  \nメモリ解放にはgcを用いる。"},{"metadata":{"trusted":true},"cell_type":"code","source":"import gc\ndel models_from_pickle\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class MelanomaDataset(Dataset):\n    def __init__(self, df: pd.DataFrame, imfolder: str, train: bool = True, transforms = None):\n\n        self.df = df\n        self.imfolder = imfolder\n        self.transforms = transforms\n        self.train = train\n        \n    def __getitem__(self, index):\n        if self.train:\n            im_path = os.path.join(self.imfolder, self.df.iloc[index]['image_id'] + '.jpg')\n        else:\n            im_path = os.path.join(self.imfolder, self.df.iloc[index]['image_name'] + '.jpg')\n\n        x = cv2.imread(im_path)\n\n        if self.transforms:\n            x = self.transforms(image = x) #albumentationsに画像を投げます\n            x = x['image'].astype(np.float32) #帰ってきたデータから画像データを取り出します(SSDのような場合には矩形領域データも合わせて帰ってきたりするため、このような仕様になっています)\n            \n        x = x.transpose(2, 0, 1) #channel first\n        \n        if self.train:\n            y = self.df.iloc[index]['target']\n            return x, y\n        else:\n            return x\n    \n    def __len__(self):\n        return len(self.df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"transforms_train = A.Compose([\n    A.Transpose(p=0.5),\n    A.VerticalFlip(p=0.5),\n    A.HorizontalFlip(p=0.5),\n    A.Resize(image_size, image_size), \n    #A.Normalize()\n])\n\ntest = MelanomaDataset(df=test_df,\n                       imfolder=test_images, \n                       train=False,\n                       transforms=transforms_train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#(model数*TTA数)回すので注意\ndef get_predictions(model, iterator, device):\n    \n    preds = np.array([0.]*len(test_df))\n    model.eval()\n    bar = tqdm(iterator) if Progress_Bar else iterator\n    \n    with torch.no_grad():\n        for tta in range(TTA):\n            res = np.array([])\n            for x in bar:\n                x = torch.tensor(x, device=device, dtype=torch.float32)\n                y_pred = model(x)\n                y_pred = torch.sigmoid(y_pred)\n                res = np.append(res, y_pred.detach().cpu().numpy())\n            preds += res\n    preds /= TTA\n    return preds","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"prediction = np.array([0.]*len(test_df))\nfor i in range(len(models)):\n    test_iterator = DataLoader(dataset=test, batch_size=16, shuffle=False, num_workers=num_workers)\n    preds = get_predictions(models[i], test_iterator, device)\n    prediction += preds\nprediction /= len(models)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub_df = pd.read_csv('../input/siim-isic-melanoma-classification/sample_submission.csv')\nsub_df = sub_df[:50] if DEBUG else sub_df\nsub_df['target'] = prediction\n\nsub_df.to_csv('submission.csv', index=False) #indexをfalseにしないと、先頭列にindex情報が付加されたcsvファイルが出力されるので注意\nsub_df.head()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}