{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Introduction\n\nif this notebook is helpful for you, please, upvote!\n\nもし役に立ったらupvoteしてくださいね。\n\nBased on [this great notebook](https://www.kaggle.com/yasufuminakama/cassava-resnext50-32x4d-starter-inference).\n\n元にしたもの： [notebook](https://www.kaggle.com/yasufuminakama/cassava-resnext50-32x4d-starter-inference).\n\nThis is a notebook for studying parameter tuning using Optuna in the Cassava classification competition.\n\nこれはCassava クラス分けコンペでの、Optunaを使ったパラメータチューニング勉強用notebookです。\n\n- resnext50_32x4d_fold0.pth\n- resnext50_32x4d_fold1.pth  \n- resnext50_32x4d_fold2.pth\n- resnext50_32x4d_fold3.pth\n- resnext50_32x4d_fold4.pth\n\nOptimize the blend ratio of Resnext 5 model with optuna.\n\nResnext 5 modelのブレンド比率をoptunaで最適化します。\n\nThe probability table for the Resnext 5 and Effcientnet 5 models is in this [notebook](https://www.kaggle.com/marutama/cassava-rnxt-effn-prob-list). You can also enter it in Optuna or Light GBM to get feature importance.\n\nResnext 5 model, Effcientnet 5 model の確率表はこちらの[notebook](https://www.kaggle.com/marutama/cassava-rnxt-effn-prob-list)です。OptunaやLight GBMに入力してfeature importance出してもいいでしょう。"},{"metadata":{},"cell_type":"markdown","source":"# Library import"},{"metadata":{"trusted":true},"cell_type":"code","source":"# ====================================================\n# Library\n# ====================================================\nimport sys\nimport os\nimport math\nimport time\nimport random\nimport shutil\nfrom pathlib import Path\nfrom contextlib import contextmanager\n\nfrom collections import defaultdict, Counter\n\nimport scipy as sp\nimport numpy as np\nimport pandas as pd\n\nimport matplotlib.pyplot as plt\n\nfrom sklearn import preprocessing\nfrom sklearn.metrics import accuracy_score\nfrom sklearn.model_selection import StratifiedKFold\n\nfrom tqdm.auto import tqdm\nfrom functools import partial\n\nimport cv2\nfrom PIL import Image\n\nimport optuna","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Optuna example"},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\noptuna.logging.disable_default_handler() # not display log\n#optuna.logging.enable_default_handler() # display log\n\ndef f(x):\n    #y = (x - 2) ** 2\n    y = x ** 2 + np.sin(x/5)*2000 + 2000\n    return y\n\ndef objective(trial):\n    x = trial.suggest_uniform('x', -50, 50)\n    score = f(x)\n    #print('x: %1.3f, score: %1.3f' % (x, score))\n    return score\n\n#study = optuna.create_study()\n# TPESampler is default sampler\nstudy = optuna.create_study(sampler=optuna.samplers.TPESampler(seed=5678))\nstudy.optimize(objective, n_trials=100)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"study.best_value","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"study.trials[10].params","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"study.best_params","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"values = [each.value for each in study.trials]\nbest_values = [np.min(values[:k+1]) for k in range(len(values))]\nt = [k for k in range(len(values))]\nx = [each.params['x'] for each in study.trials]\nx_seq = [x for x in range(-50, 50)] \nfx    = [f(x) for x in range(-50, 50)]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(12,4.5))\nax = plt.subplot(131)\nax.set_xlim(np.min(x)-0.5, np.max(x)+0.5)\nax.set_ylim(np.min(values)-500, np.max(values)+500)\nax.set_title('graph plot')\nax.plot(x_seq, fx, alpha=0.3, color='red')\nax.scatter(x, values, alpha=0.3)\n\nax = plt.subplot(132)\nax.set_title('best score')\nax.plot(best_values)\nax.scatter(t, values, alpha=0.3)\n\nax = plt.subplot(133)\nax.set_title('best score:log scale')\nax.plot(best_values)\nax.scatter(t, values, alpha=0.3)\nax.set_yscale('log')\n\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Resnext50_32x4d Inference"},{"metadata":{},"cell_type":"markdown","source":"# Settings"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# ====================================================\n# Directory settings\n# ====================================================\nOUTPUT_DIR = './'\nMODEL_DIR = '../input/cassava-resnext50-32x4d-starter-training/'\nif not os.path.exists(OUTPUT_DIR):\n    os.makedirs(OUTPUT_DIR)\n    \nTRAIN_PATH = '../input/cassava-leaf-disease-classification/train_images'\nMERGED_PATH = '../input/cassava-leaf-disease-merged/train'\n\nTEST_PATH = MERGED_PATH\nMERGED_CSV = '../input/cassava-leaf-disease-merged/merged.csv'\n\nFINAL_TEST_PATH = '../input/cassava-leaf-disease-classification/test_images'\nFINAL_TEST_CSV= '../input/cassava-leaf-disease-classification/sample_submission.csv'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#BATCH_SIZE(min) : in case of 26337(MERGED num)\n#01 ... 67\n#02 ... 35\n#04 ... 23\n#08 ... 20\n#16 ... 16\n#32 ... 15\n#64 ... 15\nBATCH_SIZE = 16\ntotal_file_nums = len(os.listdir(TEST_PATH))\nd = total_file_nums // BATCH_SIZE\nmax_file_nums = d * BATCH_SIZE\nprint(max_file_nums)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"inp_imgs= []\nfor dirname, _, filenames in os.walk(TEST_PATH):\n    for filename in filenames[:max_file_nums]: #ここで入力画像数を絞る、バッチサイズの整数倍\n        #print(os.path.join(dirname, filename))\n        #print(filename)\n        inp_imgs.append(filename)\ninp_imgs.sort()\n#print(len(inp_imgs))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# CFG"},{"metadata":{"trusted":true},"cell_type":"code","source":"# ====================================================\n# CFG\n# ====================================================\nclass CFG:\n    debug=False\n    num_workers=0 # original is 4\n    model_name='resnext50_32x4d'\n    size=256\n    batch_size=BATCH_SIZE # original is 32\n    seed=42\n    target_size=5\n    target_col='label'\n    n_fold=5\n    trn_fold=[0, 1, 2, 3, 4]\n    train=False\n    inference=True","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Library for Pytorch and GPU"},{"metadata":{"trusted":true},"cell_type":"code","source":"# ====================================================\n# Library for Pytorch and GPU\n# ====================================================\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.optim import Adam, SGD\nimport torchvision.models as models\nfrom torch.nn.parameter import Parameter\nfrom torch.utils.data import DataLoader, Dataset\nfrom torch.optim.lr_scheduler import CosineAnnealingWarmRestarts, CosineAnnealingLR, ReduceLROnPlateau\n\nfrom albumentations import (\n    Compose, OneOf, Normalize, Resize, RandomResizedCrop, RandomCrop, HorizontalFlip, VerticalFlip, \n    RandomBrightness, RandomContrast, RandomBrightnessContrast, Rotate, ShiftScaleRotate, Cutout, \n    IAAAdditiveGaussianNoise, Transpose\n    )\nfrom albumentations.pytorch import ToTensorV2\nfrom albumentations import ImageOnlyTransform\n\nsys.path.append('../input/pytorch-image-models/pytorch-image-models-master')\nimport timm\n\nimport warnings \nwarnings.filterwarnings('ignore')\n\n\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\nprint(device)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Utils"},{"metadata":{"trusted":true},"cell_type":"code","source":"# ====================================================\n# Utils\n# ====================================================\ndef get_score(y_true, y_pred):\n    return accuracy_score(y_true, y_pred)\n\n\n@contextmanager\ndef timer(name):\n    t0 = time.time()\n    LOGGER.info(f'[{name}] start')\n    yield\n    LOGGER.info(f'[{name}] done in {time.time() - t0:.0f} s.')\n\n\ndef init_logger(log_file=OUTPUT_DIR+'inference.log'):\n    from logging import getLogger, INFO, FileHandler,  Formatter,  StreamHandler\n    logger = getLogger(__name__)\n    logger.setLevel(INFO)\n    handler1 = StreamHandler()\n    handler1.setFormatter(Formatter(\"%(message)s\"))\n    handler2 = FileHandler(filename=log_file)\n    handler2.setFormatter(Formatter(\"%(message)s\"))\n    logger.addHandler(handler1)\n    logger.addHandler(handler2)\n    return logger\n\n#LOGGER = init_logger()\n\n\ndef seed_torch(seed=1234):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n\nseed_torch(seed=CFG.seed)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Dataset"},{"metadata":{"trusted":true},"cell_type":"code","source":"# ====================================================\n# Dataset\n# ====================================================\nclass TestDataset(Dataset):\n    def __init__(self, df, transform=None):\n        self.df = df\n        self.file_names = df['image_id'].values\n        self.transform = transform\n        \n    def __len__(self):\n        return len(self.df)\n\n    def __getitem__(self, idx):\n        file_name = self.file_names[idx]\n        file_path = f'{TEST_PATH}/{file_name}'\n        image = cv2.imread(file_path)\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n        if self.transform:\n            augmented = self.transform(image=image)\n            image = augmented['image']\n        return image","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Transforms"},{"metadata":{"trusted":true},"cell_type":"code","source":"# ====================================================\n# Transforms\n# ====================================================\ndef get_transforms(*, data):\n    \n    if data == 'train':\n        return Compose([\n            #Resize(CFG.size, CFG.size),\n            RandomResizedCrop(CFG.size, CFG.size),\n            Transpose(p=0.5),\n            HorizontalFlip(p=0.5),\n            VerticalFlip(p=0.5),\n            ShiftScaleRotate(p=0.5),\n            Normalize(\n                mean=[0.485, 0.456, 0.406],\n                std=[0.229, 0.224, 0.225],\n            ),\n            ToTensorV2(),\n        ])\n\n    elif data == 'valid':\n        return Compose([\n            Resize(CFG.size, CFG.size),\n            Normalize(\n                mean=[0.485, 0.456, 0.406],\n                std=[0.229, 0.224, 0.225],\n            ),\n            ToTensorV2(),\n        ])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# MODEL"},{"metadata":{"trusted":true},"cell_type":"code","source":"# ====================================================\n# MODEL\n# ====================================================\nclass CustomResNext(nn.Module):\n    def __init__(self, model_name='resnext50_32x4d', pretrained=False):\n        super().__init__()\n        self.model = timm.create_model(model_name, pretrained=pretrained)\n        n_features = self.model.fc.in_features\n        self.model.fc = nn.Linear(n_features, CFG.target_size)\n\n    def forward(self, x):\n        x = self.model(x)\n        return x","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Helper functions"},{"metadata":{"trusted":true},"cell_type":"code","source":"# ====================================================\n# Helper functions\n# ====================================================\ndef inference(model, states, test_loader, device):\n    model.to(device)\n    tk0 = tqdm(enumerate(test_loader), total=len(test_loader))\n    probs = []\n    for i, (images) in tk0:\n        images = images.to(device)\n        avg_preds = []\n        for state in states:\n            model.load_state_dict(state['model'])\n            model.eval()\n            with torch.no_grad():\n                y_preds = model(images)\n            avg_preds.append(y_preds.softmax(1).to('cpu').numpy())\n        #avg_preds = np.mean(avg_preds, axis=0)\n        probs.append(avg_preds)\n    probs = np.concatenate(probs)\n    return probs","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Data Loading"},{"metadata":{"trusted":true},"cell_type":"code","source":"merged = pd.read_csv(MERGED_CSV)\nmerged = merged.set_index('image_id')\n#merged","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"len(inp_imgs)\n# 21397 TRAIN num\n# 26337 MERGED num","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"merged","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tmp = merged.loc[inp_imgs]\ntest = tmp.rename(columns={'label':'correct'}).drop('source', axis=1)\ntest = test.reset_index()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# inference(all images)"},{"metadata":{"trusted":true},"cell_type":"code","source":"# ====================================================\n# inference\n# ====================================================\nmodel = CustomResNext(CFG.model_name, pretrained=False)\nstates = [torch.load(MODEL_DIR+f'{CFG.model_name}_fold{fold}_best.pth') for fold in CFG.trn_fold]\ntest_dataset = TestDataset(test, transform=get_transforms(data='valid'))\ntest_loader = DataLoader(test_dataset, batch_size=CFG.batch_size, shuffle=False, \n                         num_workers=CFG.num_workers, pin_memory=True)\np = inference(model, states, test_loader, device)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Make DataFrame for 5 models inference results"},{"metadata":{"trusted":true},"cell_type":"code","source":"MODELS = 5\nfor i in range(len(p)//MODELS):\n    if i==0:\n        p0=p[0]\n        p1=p[1]\n        p2=p[2]\n        p3=p[3]\n        p4=p[4]\n    else:\n        p0 = np.concatenate([p0, p[i*5+0]])\n        p1 = np.concatenate([p1, p[i*5+1]])\n        p2 = np.concatenate([p2, p[i*5+2]])\n        p3 = np.concatenate([p3, p[i*5+3]])\n        p4 = np.concatenate([p4, p[i*5+4]])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"col = ['p0_0', 'p0_1', 'p0_2', 'p0_3', 'p0_4']\ndf0 = pd.DataFrame(data=p0, columns=col, dtype='float32')\ncol = ['p1_0', 'p1_1', 'p1_2', 'p1_3', 'p1_4']\ndf1 = pd.DataFrame(data=p1, columns=col, dtype='float32')\ncol = ['p2_0', 'p2_1', 'p2_2', 'p2_3', 'p2_4']\ndf2 = pd.DataFrame(data=p2, columns=col, dtype='float32')\ncol = ['p3_0', 'p3_1', 'p3_2', 'p3_3', 'p3_4']\ndf3 = pd.DataFrame(data=p3, columns=col, dtype='float32')\ncol = ['p4_0', 'p4_1', 'p4_2', 'p4_3', 'p4_4']\ndf4 = pd.DataFrame(data=p4, columns=col, dtype='float32')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# 参考として元のmeanでのinefence結果を作っておく\n#avg_p = np.mean(p, axis=0)\navg_p = p0*0.2 + p1*0.2 + p2*0.2 + p3*0.2 + p4*0.2","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"avg_p","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#test=test.drop('label', axis=1)\ntest['label']=9999 # 9999 is NaN, to keep integer\n#test","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for i, name in enumerate(tqdm(test['image_id'].values)):\n    #print(i, name)\n    #print(predictions[i])\n    #print(predictions[i].argmax())\n    test.loc[test['image_id']==name, 'label'] = avg_p[i].argmax()\ntest['label'] = test['label'].astype('int')\ntest","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# acc_scoreで検算チェック\nacc_score = accuracy_score(test['correct'], test['label'])\nprint(acc_score)\n# 0.909753703790251 ... TRAIN score\n# 0.8880282492311197 ... MERGED score","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"all_df = pd.merge(test,   df0, left_index=True, right_index=True)\nall_df = pd.merge(all_df, df1, left_index=True, right_index=True)\nall_df = pd.merge(all_df, df2, left_index=True, right_index=True)\nall_df = pd.merge(all_df, df3, left_index=True, right_index=True)\nall_df = pd.merge(all_df, df4, left_index=True, right_index=True)\nall_df.to_csv(\"all.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# You can also use this csv file created in advance.\n#all_df = pd.read_csv('../input/cassava-resnext50-5-inference-results/all.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"all_df","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Define calculation function"},{"metadata":{"trusted":true},"cell_type":"code","source":"def calc_p(df, a0, a1, a2, a3, a4):\n    l = []\n    for n in range(len(df)):\n        p0 = np.array([df['p0_0'][n], df['p0_1'][n], df['p0_2'][n], df['p0_3'][n], df['p0_4'][n]])\n        p1 = np.array([df['p1_0'][n], df['p1_1'][n], df['p1_2'][n], df['p1_3'][n], df['p1_4'][n]])\n        p2 = np.array([df['p2_0'][n], df['p2_1'][n], df['p2_2'][n], df['p2_3'][n], df['p2_4'][n]])\n        p3 = np.array([df['p3_0'][n], df['p3_1'][n], df['p3_2'][n], df['p3_3'][n], df['p3_4'][n]])\n        p4 = np.array([df['p4_0'][n], df['p4_1'][n], df['p4_2'][n], df['p4_3'][n], df['p4_4'][n]])\n        p=p0*a0+p1*a1+p2*a2+p3*a3+p4*a4\n        c = p.argmax()\n        l.append(c)\n    return l","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def calc_p_pd(df, a0, a1, a2, a3, a4):\n    df['pf_0'] = df['p0_0']*a0 + df['p1_0']*a1 + df['p2_0']*a2 + df['p3_0']*a3 + df['p4_0']*a4\n    df['pf_1'] = df['p0_1']*a0 + df['p1_1']*a1 + df['p2_1']*a2 + df['p3_1']*a3 + df['p4_1']*a4\n    df['pf_2'] = df['p0_2']*a0 + df['p1_2']*a1 + df['p2_2']*a2 + df['p3_2']*a3 + df['p4_2']*a4\n    df['pf_3'] = df['p0_3']*a0 + df['p1_3']*a1 + df['p2_3']*a2 + df['p3_3']*a3 + df['p4_3']*a4\n    df['pf_4'] = df['p0_4']*a0 + df['p1_4']*a1 + df['p2_4']*a2 + df['p3_4']*a3 + df['p4_4']*a4\n    l = []\n    for n in range(len(df)):\n        p = np.array([df['pf_0'][n] , df['pf_1'][n] , df['pf_2'][n] , df['pf_3'][n] , df['pf_4'][n]])\n        c = p.argmax()\n        #df['pl'][n] = c\n        l.append(c)\n    return l","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df = all_df\ncorrect = df['correct']\nlabel = df['label']","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## calc_p : normal version"},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\npred = calc_p(df, 0.2, 0.2, 0.2, 0.2, 0.2)\nacc_score = accuracy_score(correct, pred)\nprint(acc_score)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## calc_p_pd : pandas version"},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\npred = calc_p_pd(df, 0.2, 0.2, 0.2, 0.2, 0.2)\nacc_score = accuracy_score(correct, pred)\nprint(acc_score)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"calc_p_pd() is faster than calc_p()."},{"metadata":{},"cell_type":"markdown","source":"# Optuna optimization"},{"metadata":{"trusted":true},"cell_type":"code","source":"r_min = 0\nr_max = 1\n# You can increase iteration number.\niteration = 50\n\noptuna.logging.disable_default_handler() # not display log\n#optuna.logging.enable_default_handler() # display log","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## For Merged(2019 and 2020) data"},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\ncorrect = df['correct']\n\ndef objective(trial):\n    a = trial.suggest_uniform('a', r_min, r_max)\n    b = trial.suggest_uniform('b', r_min, r_max)\n    c = trial.suggest_uniform('c', r_min, r_max)\n    d = trial.suggest_uniform('d', r_min, r_max)\n    e = trial.suggest_uniform('e', r_min, r_max)\n\n    pred = calc_p_pd(df, a, b, c, d, e)\n    score = accuracy_score(correct, pred)\n    #print('a:%1.3f,b:%1.3f,c:%1.3f,d:%1.3f,e:%1.3f,score %1.3f' % (a,b,c,d,e,score))\n    return score\nSEED=1234\n#study = optuna.create_study(direction='maximize')\nstudy = optuna.create_study(direction='maximize', sampler=optuna.samplers.TPESampler(seed=SEED))\nstudy.optimize(objective, n_trials=iteration)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"study.best_value","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"study.best_params","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.plot([trial.value for trial in study.trials])\nplt.grid()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.plot([trial.params['a'] for trial in study.trials], label='a')\nplt.plot([trial.params['b'] for trial in study.trials], label='b')\nplt.plot([trial.params['c'] for trial in study.trials], label='c')\nplt.plot([trial.params['d'] for trial in study.trials], label='d')\nplt.plot([trial.params['e'] for trial in study.trials], label='e')\nplt.legend()\nplt.grid()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from optuna.visualization import plot_optimization_history\nplot_optimization_history(study)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from optuna.visualization import plot_param_importances\nplot_param_importances(study)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from optuna.visualization import plot_contour\nplot_contour(study)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## For 2020 train data"},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train = df[:21396].copy()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\ncorrect = df_train['correct']\nlabel = df_train['label']\npred = calc_p_pd(df_train, 0.2, 0.2, 0.2, 0.2, 0.2)\nacc_score = accuracy_score(correct, pred)\nprint(acc_score)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n#correct = df_train['correct']\n\ndef objective(trial):\n    a = trial.suggest_uniform('a', r_min, r_max)\n    b = trial.suggest_uniform('b', r_min, r_max)\n    c = trial.suggest_uniform('c', r_min, r_max)\n    d = trial.suggest_uniform('d', r_min, r_max)\n    e = trial.suggest_uniform('e', r_min, r_max)\n    pred = calc_p_pd(df_train, a, b, c, d, e)\n    score = accuracy_score(correct, pred)\n    print('a:%1.3f,b:%1.3f,c:%1.3f,d:%1.3f,e:%1.3f,score %1.3f' % (a,b,c,d,e,score))\n    return score\n\nSEED=1234\nstudy = optuna.create_study(direction='maximize', sampler=optuna.samplers.TPESampler(seed=SEED))\nstudy.optimize(objective, n_trials=iteration)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"study.best_value","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"study.best_params","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.plot([trial.value for trial in study.trials])\nplt.grid()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.plot([trial.params['a'] for trial in study.trials], label='a')\nplt.plot([trial.params['b'] for trial in study.trials], label='b')\nplt.plot([trial.params['c'] for trial in study.trials], label='c')\nplt.plot([trial.params['d'] for trial in study.trials], label='d')\nplt.plot([trial.params['e'] for trial in study.trials], label='e')\nplt.legend()\nplt.grid()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from optuna.visualization import plot_optimization_history\nplot_optimization_history(study)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from optuna.visualization import plot_param_importances\nplot_param_importances(study)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from optuna.visualization import plot_contour\nplot_contour(study)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Final inference"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Use last study : 2020 train data\na = study.best_params['a']\nb = study.best_params['b']\nc = study.best_params['c']\nd = study.best_params['d']\ne = study.best_params['e']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# ====================================================\n# Helper functions\n# ====================================================\ndef final_inference(model, states, test_loader, device):\n    model.to(device)\n    tk0 = tqdm(enumerate(test_loader), total=len(test_loader))\n    probs = []\n    for i, (images) in tk0:\n        images = images.to(device)\n        avg_preds = []\n        for state in states:\n            model.load_state_dict(state['model'])\n            model.eval()\n            with torch.no_grad():\n                y_preds = model(images)\n            avg_preds.append(y_preds.softmax(1).to('cpu').numpy())\n        #avg_preds = np.mean(avg_preds, axis=0)\n        avg_preds = a*avg_preds[0]+b*avg_preds[1]+c*avg_preds[2]+d*avg_preds[3]+e*avg_preds[4]\n        probs.append(avg_preds)\n    probs = np.concatenate(probs)\n    return probs","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"TEST_PATH=FINAL_TEST_PATH \ntest = pd.read_csv(FINAL_TEST_CSV)\ntest.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# ====================================================\n# inference\n# ====================================================\n\nmodel = CustomResNext(CFG.model_name, pretrained=False)\nstates = [torch.load(MODEL_DIR+f'{CFG.model_name}_fold{fold}_best.pth') for fold in CFG.trn_fold]\ntest_dataset = TestDataset(test, transform=get_transforms(data='valid'))\ntest_loader = DataLoader(test_dataset, batch_size=CFG.batch_size, shuffle=False, \n                         num_workers=CFG.num_workers, pin_memory=True)\np = final_inference(model, states, test_loader, device)\n\n# submission\ntest['label'] = p.argmax(1)\ntest[['image_id', 'label']].to_csv(OUTPUT_DIR+'submission.csv', index=False)\ntest.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}