{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport os\nimport numpy as np\nimport random\nimport cv2\nimport sys\nimport shutil\nfrom PIL import Image","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","papermill":{"duration":6.123694,"end_time":"2022-05-08T07:07:20.685937","exception":false,"start_time":"2022-05-08T07:07:14.562243","status":"completed"},"pycharm":{"name":"#%%\n"},"tags":[],"execution":{"iopub.status.busy":"2022-05-29T09:13:44.736765Z","iopub.execute_input":"2022-05-29T09:13:44.737003Z","iopub.status.idle":"2022-05-29T09:13:45.066211Z","shell.execute_reply.started":"2022-05-29T09:13:44.736930Z","shell.execute_reply":"2022-05-29T09:13:45.065519Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import tensorflow as tf\nimport math\nimport matplotlib.pyplot as plt\nimport tensorflow_addons as tfa\nimport albumentations as A\nfrom albumentations.core.composition import Compose, OneOf\nfrom tensorflow import keras\n\nfrom sklearn.model_selection import StratifiedKFold, train_test_split\nfrom tensorflow.keras.models import Model, load_model\nfrom tensorflow.keras.layers import GlobalAveragePooling2D, Flatten, Dense, Dropout, BatchNormalization, Input\nfrom tensorflow.keras.losses import categorical_crossentropy\nfrom tensorflow.keras.optimizers import Adam\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\nfrom tensorflow.keras.callbacks import EarlyStopping, ReduceLROnPlateau, ModelCheckpoint\nfrom tensorflow.keras.applications import EfficientNetB3, InceptionV3\nfrom tensorflow.keras.utils import array_to_img, load_img, img_to_array","metadata":{"papermill":{"duration":0.042945,"end_time":"2022-05-08T07:07:20.765795","exception":false,"start_time":"2022-05-08T07:07:20.72285","status":"completed"},"pycharm":{"name":"#%%\n"},"tags":[],"execution":{"iopub.status.busy":"2022-05-29T11:50:24.988247Z","iopub.execute_input":"2022-05-29T11:50:24.988538Z","iopub.status.idle":"2022-05-29T11:50:24.999629Z","shell.execute_reply.started":"2022-05-29T11:50:24.988494Z","shell.execute_reply":"2022-05-29T11:50:24.998828Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Configuration","metadata":{}},{"cell_type":"code","source":"train_data_dir = './train'\ntardet_input_size = (256,256)\ninit_lr = 1e-4\nreduce_lr_in = 3\nepoch = 40\nbatch_size = 32\nsetps_in_epoch = 822\nvalidation_steps = 245\nbest_ck_point = 'best_checkpoint.hdf5'\nfinal_model = 'model.hdf5'","metadata":{"execution":{"iopub.status.busy":"2022-05-29T15:35:34.869326Z","iopub.execute_input":"2022-05-29T15:35:34.869597Z","iopub.status.idle":"2022-05-29T15:35:34.875125Z","shell.execute_reply.started":"2022-05-29T15:35:34.869565Z","shell.execute_reply":"2022-05-29T15:35:34.874114Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Basic croping of all training set provided to increase sample sizes\n\ncropping strategy is done in this notebook: https://www.kaggle.com/code/asheniranga/128-128-sorghum-cultivar-pre-process","metadata":{}},{"cell_type":"markdown","source":"## Pre-processing Pipeline","metadata":{"papermill":{"duration":0.031041,"end_time":"2022-05-08T07:07:20.899353","exception":false,"start_time":"2022-05-08T07:07:20.868312","status":"completed"},"pycharm":{"name":"#%% md\n"},"tags":[]}},{"cell_type":"code","source":"def resize(image, size):\n    return tf.image.resize(image, size)\n\n\ndef blur(img, blur_limit):\n    return cv2.blur(img, ksize=[blur_limit, blur_limit])\n\n\ndef gaussian_blur(img, blur_limit=(3, 7), sigma_limit=0):\n    return cv2.GaussianBlur(img, ksize=blur_limit, sigmaX=sigma_limit)\n\n\ndef motion_blur(img, blur_limit=7):\n    kmb = np.zeros((blur_limit, blur_limit))\n    kmb[(blur_limit - 1) // 2, :] = np.ones(blur_limit)\n    kmb = kmb / blur_limit\n    return cv2.filter2D(img, -1, kernel=kmb)\n\n\ndef gaussian_noise(img):\n    x = tf.compat.v1.placeholder(dtype=tf.float32, shape=[512, 512, 3])\n    noise = tf.random.normal(shape=tf.shape(x), mean=0.0, stddev=1, dtype=tf.float32)\n    return tf.add(img, noise)\n\n\ndef iso_noise(img, color_shift=0.05, intensity=0.5):\n    one_over_255 = float(1.0 / 255.0)\n    image = np.multiply(img, one_over_255, dtype=np.float32)\n    hls = cv2.cvtColor(image, cv2.COLOR_RGB2HLS)\n    _, stddev = cv2.meanStdDev(hls)\n\n    luminance_noise = np.random.poisson(stddev[1] * intensity * 255, hls.shape[:2])\n    color_noise = np.random.normal(0, color_shift * 360 * intensity, hls.shape[:2])\n\n    hue = hls[..., 0]\n    hue += color_noise\n    hue[hue < 0] += 360\n    hue[hue > 360] -= 360\n\n    luminance = hls[..., 1]\n    luminance += (luminance_noise / 255) * (1.0 - luminance)\n\n    image = cv2.cvtColor(hls, cv2.COLOR_HLS2RGB) * 255\n    return image.astype(np.uint8)\n\n\ndef random_cut_out(images):\n    return tfa.image.random_cutout(images, (32, 32), constant_values=0)","metadata":{"papermill":{"duration":0.066608,"end_time":"2022-05-08T07:07:20.997664","exception":false,"start_time":"2022-05-08T07:07:20.931056","status":"completed"},"pycharm":{"name":"#%%\n"},"tags":[],"execution":{"iopub.status.busy":"2022-05-29T09:14:03.680858Z","iopub.execute_input":"2022-05-29T09:14:03.681128Z","iopub.status.idle":"2022-05-29T09:14:03.694021Z","shell.execute_reply.started":"2022-05-29T09:14:03.681098Z","shell.execute_reply":"2022-05-29T09:14:03.693356Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def aug_fn(image):\n    data = {\"image\":image}\n    aug_data = get_transform(**data)\n    aug_img = aug_data[\"image\"]\n    aug_img = tf.cast(aug_img/255.0, tf.float32)\n    aug_img = tf.image.resize(aug_img, size=[256, 256])\n    return aug_img\n\nget_transform = Compose([A.CoarseDropout(max_holes=16, min_holes=8, max_height=16, max_width=16, min_height=8, min_width=8, p=0.2)])\n\ndef get_transforms_train(image):\n    # get random crop of random crop window size\n    crop_side = int(256*random.uniform(0.33, 1))\n    temp = tf.image.random_crop(image, size=(crop_side, crop_side, 3)).numpy()\n    temp = resize(temp, size=(256, 256)).numpy()\n\n    # random flip (vertically)\n    temp = tf.image.random_flip_left_right(temp).numpy()\n\n    if np.random.choice([True, False], p=[0.6, 0.4]):\n        # rotate randomly by N*90 deg\n        k = random.randint(1, 4)\n        temp = tf.image.rot90(temp, k=k).numpy()\n\n    if np.random.choice([True, False], p=[0.45, 0.55]):\n        if random.choice([True, False]):\n            delta = random.uniform(-0.4, 0.4)\n            cf = random.uniform(-1.5, 2.5)\n            temp = tf.image.adjust_brightness(temp, delta=delta).numpy()\n            temp = tf.image.adjust_contrast(temp, contrast_factor=cf).numpy()\n        else:\n            gamma = random.uniform(0.33, 2.2)\n            temp = tf.image.adjust_gamma(temp, gamma=gamma).numpy()\n\n    if np.random.choice([True, False], p=[0.25, 0.75]):\n        delta = random.uniform(-0.2, 0.4)\n        temp = tf.image.adjust_hue(temp, delta=delta).numpy()\n\n    if np.random.choice([True, False], p=[0.2, 0.8]):\n        sf = random.uniform(-0.2, 0.8)\n        temp = tf.image.adjust_saturation(temp, saturation_factor=sf).numpy()\n\n    if np.random.choice([True, False], p=[0.4, 0.6]):\n        one_of_blur = random.choice([1, 2, 3])\n\n        if one_of_blur == 1:\n            temp = blur(temp, blur_limit=7)\n        elif one_of_blur == 2:\n            temp = gaussian_blur(temp)\n        elif one_of_blur == 3:\n            temp = motion_blur(temp)\n\n    if np.random.choice([True, False], p=[0.35, 0.65]):\n        temp = iso_noise(temp)\n\n    if np.random.choice([True, False], p=[0.3, 0.7]):\n        temp = temp.reshape([1,temp.shape[0], temp.shape[1], 3])\n        temp = random_cut_out(temp).numpy()\n\n        return tf.convert_to_tensor(temp[0]/255.0, dtype=tf.float32)\n\n    temp = aug_fn(temp).numpy()\n\n    return tf.convert_to_tensor(temp/255.0, dtype=tf.float32)","metadata":{"execution":{"iopub.status.busy":"2022-05-29T09:14:06.632821Z","iopub.execute_input":"2022-05-29T09:14:06.633352Z","iopub.status.idle":"2022-05-29T09:14:06.650794Z","shell.execute_reply.started":"2022-05-29T09:14:06.633312Z","shell.execute_reply":"2022-05-29T09:14:06.650136Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, axes = plt.subplots(nrows=1, ncols=5, figsize=[18, 6], dpi=300)\naxes = axes.ravel()\n\nfor i in range(5):\n    axes[i].imshow(array_to_img(get_transforms_train(img_to_array(load_img('../input/sorghum-cultivar-identification-512512/train/2017-06-01__10-26-27-479.png', target_size=(256,256))))))\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-05-29T09:15:27.016361Z","iopub.execute_input":"2022-05-29T09:15:27.016611Z","iopub.status.idle":"2022-05-29T09:15:29.570135Z","shell.execute_reply.started":"2022-05-29T09:15:27.016582Z","shell.execute_reply":"2022-05-29T09:15:29.569079Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Split data","metadata":{"papermill":{"duration":0.031973,"end_time":"2022-05-08T07:07:21.131491","exception":false,"start_time":"2022-05-08T07:07:21.099518","status":"completed"},"pycharm":{"name":"#%% md\n"},"tags":[]}},{"cell_type":"code","source":"df_train = pd.read_csv('../input/128128-sorghum-cultivar/train_meta.csv')\ndf_valid = pd.read_csv('../input/128128-sorghum-cultivar/valid_meta.csv')\n    \ndf_valid, df_test = train_test_split(df_valid, test_size=0.1)\n\nprint(f\"train size: {len(df_train)}\")\nprint(f\"valid size: {len(df_valid)}\")\nprint(f\"test size: {len(df_test)}\")\n\nprint(df_train.cultivar.value_counts())\nprint(df_valid.cultivar.value_counts())\nprint(df_test.cultivar.value_counts())","metadata":{"collapsed":false,"jupyter":{"outputs_hidden":false},"papermill":{"duration":0.178646,"end_time":"2022-05-08T07:07:21.342011","exception":false,"start_time":"2022-05-08T07:07:21.163365","status":"completed"},"pycharm":{"name":"#%%\n"},"tags":[],"execution":{"iopub.status.busy":"2022-05-29T14:35:35.686833Z","iopub.execute_input":"2022-05-29T14:35:35.687109Z","iopub.status.idle":"2022-05-29T14:35:35.884876Z","shell.execute_reply.started":"2022-05-29T14:35:35.687077Z","shell.execute_reply":"2022-05-29T14:35:35.883479Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if not os.path.isdir('train'):\n    os.mkdir('train')\n    \nfor i, file in enumerate(os.listdir('../input/128128-sorghum-cultivar/train')):\n    src = os.path.join('../input/128128-sorghum-cultivar/train', file)\n    dst = os.path.join('./train', file)\n    \n    shutil.copyfile(src, dst)\n    \n    print(f\"{i}/{len(os.listdir('../input/128128-sorghum-cultivar/train'))}\", end='\\r')","metadata":{"execution":{"iopub.status.busy":"2022-05-29T09:17:02.267537Z","iopub.execute_input":"2022-05-29T09:17:02.267853Z","iopub.status.idle":"2022-05-29T09:43:40.085995Z","shell.execute_reply.started":"2022-05-29T09:17:02.267811Z","shell.execute_reply":"2022-05-29T09:43:40.084536Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"c = 0\nnew_train_meta = []\n\nfor filename, label in df_train.sample(n=int(df_train.shape[0]*0.65)).values:\n    if filename in os.listdir('train'):\n        if random.choice([True, False]):\n            image = tf.keras.utils.img_to_array(Image.open(os.path.join('train', filename)))\n            \n            c += 1\n            print(f'{c}~{int(df_train.shape[0]*0.65)*0.5}', end='\\r')\n            \n            # apply aumentations\n            for i in range(5):\n                process = get_transforms_train(image=image)\n                \n                dst_file = f'{i}-{filename}'\n                tf.keras.utils.array_to_img(process).save(f'train/{dst_file}')\n                new_train_meta.append([dst_file, label])\n\n\ntrain_df_1 = pd.DataFrame(new_train_meta, columns=['image', 'cultivar'])\ntrain_df_1","metadata":{"execution":{"iopub.status.busy":"2022-05-29T09:44:56.880285Z","iopub.execute_input":"2022-05-29T09:44:56.880532Z","iopub.status.idle":"2022-05-29T10:58:32.932879Z","shell.execute_reply.started":"2022-05-29T09:44:56.880506Z","shell.execute_reply":"2022-05-29T10:58:32.931373Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train = pd.concat([df_train, train_df_1], ignore_index=True)\ndf_train","metadata":{"execution":{"iopub.status.busy":"2022-05-29T15:30:06.829584Z","iopub.execute_input":"2022-05-29T15:30:06.829880Z","iopub.status.idle":"2022-05-29T15:30:06.849771Z","shell.execute_reply.started":"2022-05-29T15:30:06.829847Z","shell.execute_reply":"2022-05-29T15:30:06.848938Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"temp = df_train['image'].sample(n=36).tolist()\nfig, axes = plt.subplots(nrows=6, ncols=6, figsize=[18, 18], dpi=300)\naxes = axes.ravel()\n\nfor i in range(36):\n    axes[i].imshow(Image.open(f'train/{temp[i]}'))\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-05-29T15:30:11.180406Z","iopub.execute_input":"2022-05-29T15:30:11.180678Z","iopub.status.idle":"2022-05-29T15:31:10.891304Z","shell.execute_reply.started":"2022-05-29T15:30:11.180647Z","shell.execute_reply":"2022-05-29T15:31:10.890235Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Training\n","metadata":{"pycharm":{"name":"#%% md\n"}}},{"cell_type":"code","source":"model_base = EfficientNetB3(include_top=False, input_shape=(256, 256, 3), weights='imagenet')","metadata":{"pycharm":{"name":"#%%\n"},"execution":{"iopub.status.busy":"2022-05-29T15:31:10.893804Z","iopub.execute_input":"2022-05-29T15:31:10.894197Z","iopub.status.idle":"2022-05-29T15:31:13.555373Z","shell.execute_reply.started":"2022-05-29T15:31:10.894160Z","shell.execute_reply":"2022-05-29T15:31:13.554618Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"input_layer = Input(shape=(256, 256, 3))\nx_ = model_base(input_layer)\nx_ = GlobalAveragePooling2D()(x_)\noutput_layer = Dense(units=100, activation='softmax')(x_)\n\nmodel = Model(input_layer, output_layer)","metadata":{"collapsed":false,"pycharm":{"name":"#%%\n"},"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2022-05-29T15:31:13.556673Z","iopub.execute_input":"2022-05-29T15:31:13.556926Z","iopub.status.idle":"2022-05-29T15:31:14.493922Z","shell.execute_reply.started":"2022-05-29T15:31:13.556891Z","shell.execute_reply":"2022-05-29T15:31:14.493220Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.compile(optimizer=Adam(learning_rate=init_lr),\n              loss=categorical_crossentropy,\n              metrics=['accuracy'])","metadata":{"papermill":{"duration":0.063606,"end_time":"2022-05-08T07:09:10.291814","exception":false,"start_time":"2022-05-08T07:09:10.228208","status":"completed"},"tags":[],"pycharm":{"name":"#%%\n"},"execution":{"iopub.status.busy":"2022-05-29T15:31:14.496118Z","iopub.execute_input":"2022-05-29T15:31:14.496381Z","iopub.status.idle":"2022-05-29T15:31:14.513208Z","shell.execute_reply.started":"2022-05-29T15:31:14.496345Z","shell.execute_reply":"2022-05-29T15:31:14.512392Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.summary()","metadata":{"collapsed":false,"pycharm":{"name":"#%%\n"},"jupyter":{"outputs_hidden":false},"_kg_hide-output":true,"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-05-29T15:31:14.514353Z","iopub.execute_input":"2022-05-29T15:31:14.514593Z","iopub.status.idle":"2022-05-29T15:31:14.541158Z","shell.execute_reply.started":"2022-05-29T15:31:14.514560Z","shell.execute_reply":"2022-05-29T15:31:14.540470Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_generator = ImageDataGenerator(samplewise_center=False,\n                                     samplewise_std_normalization=False,\n                                     width_shift_range=0.35,\n                                     height_shift_range=0.45,\n                                     channel_shift_range=0.65,\n                                     fill_mode='reflect').flow_from_dataframe(dataframe=df_train,\n                                                           directory=train_data_dir,\n                                                           x_col='image',\n                                                           y_col='cultivar',\n                                                           batch_size=batch_size,\n                                                           target_size=tardet_input_size)\n\nvalid_generator = ImageDataGenerator().flow_from_dataframe(dataframe=df_valid,\n                                                           directory=train_data_dir,\n                                                           x_col='image',\n                                                           y_col='cultivar',\n                                                           batch_size=batch_size,\n                                                           target_size=tardet_input_size)\n\ntest_generator = ImageDataGenerator().flow_from_dataframe(dataframe=df_test,\n                                                          directory=train_data_dir,\n                                                          x_col='image',\n                                                          y_col='cultivar',\n                                                          batch_size=batch_size,\n                                                          target_size=tardet_input_size)","metadata":{"papermill":{"duration":0.047536,"end_time":"2022-05-08T07:09:10.381034","exception":false,"start_time":"2022-05-08T07:09:10.333498","status":"completed"},"pycharm":{"name":"#%%\n"},"tags":[],"execution":{"iopub.status.busy":"2022-05-29T15:35:51.081774Z","iopub.execute_input":"2022-05-29T15:35:51.082068Z","iopub.status.idle":"2022-05-29T15:35:53.736147Z","shell.execute_reply.started":"2022-05-29T15:35:51.082012Z","shell.execute_reply":"2022-05-29T15:35:53.735381Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"K = keras.backend\n\n\nclass OneCycleLr(keras.callbacks.Callback):\n    def __init__(self,\n                 max_lr: float,\n                 total_steps: int = None,\n                 epochs: int = None,\n                 steps_per_epoch: int = None,\n                 pct_start: float = 0.2,\n                 anneal_strategy: str = \"cos\",\n                 cycle_momentum: bool = True,\n                 base_momentum: float = 0.85,\n                 max_momentum: float = 0.95,\n                 div_factor: float = 1.0e+3,\n                 final_div_factor: float = 1e4,\n                 ) -> None:\n\n        super(OneCycleLr, self).__init__()\n\n        # validate total steps:\n        if total_steps is None and epochs is None and steps_per_epoch is None:\n            raise ValueError(\n                \"You must define either total_steps OR (epochs AND steps_per_epoch)\"\n            )\n        elif total_steps is not None:\n            if total_steps <= 0 or not isinstance(total_steps, int):\n                raise ValueError(\n                    \"Expected non-negative integer total_steps, but got {}\".format(\n                        total_steps\n                    )\n                )\n            self.total_steps = total_steps\n        else:\n            if epochs <= 0 or not isinstance(epochs, int):\n                raise ValueError(\n                    \"Expected non-negative integer epochs, but got {}\".format(\n                        epochs)\n                )\n            if steps_per_epoch <= 0 or not isinstance(steps_per_epoch, int):\n                raise ValueError(\n                    \"Expected non-negative integer steps_per_epoch, but got {}\".format(\n                        steps_per_epoch\n                    )\n                )\n            # Compute total steps\n            self.total_steps = epochs * steps_per_epoch\n\n        self.step_num = 0\n        self.step_size_up = float(pct_start * self.total_steps) - 1\n        self.step_size_down = float(self.total_steps - self.step_size_up) - 1\n\n        # Validate pct_start\n        if pct_start < 0 or pct_start > 1 or not isinstance(pct_start, float):\n            raise ValueError(\n                \"Expected float between 0 and 1 pct_start, but got {}\".format(\n                    pct_start)\n            )\n\n        # Validate anneal_strategy\n        if anneal_strategy not in [\"cos\", \"linear\"]:\n            raise ValueError(\n                \"anneal_strategy must by one of 'cos' or 'linear', instead got {}\".format(\n                    anneal_strategy\n                )\n            )\n        elif anneal_strategy == \"cos\":\n            self.anneal_func = self._annealing_cos\n        elif anneal_strategy == \"linear\":\n            self.anneal_func = self._annealing_linear\n\n        # Initialize learning rate variables\n        self.initial_lr = max_lr / div_factor\n        self.max_lr = max_lr\n        self.min_lr = self.initial_lr / final_div_factor\n\n        # Initial momentum variables\n        self.cycle_momentum = cycle_momentum\n        if self.cycle_momentum:\n            self.m_momentum = max_momentum\n            self.momentum = max_momentum\n            self.b_momentum = base_momentum\n\n        # Initialize variable to learning_rate & momentum\n        self.track_lr = []\n        self.track_mom = []\n\n    def _annealing_cos(self, start, end, pct) -> float:\n        \"Cosine anneal from `start` to `end` as pct goes from 0.0 to 1.0.\"\n        cos_out = math.cos(math.pi * pct) + 1\n        return end + (start - end) / 2.0 * cos_out\n\n    def _annealing_linear(self, start, end, pct) -> float:\n        \"Linearly anneal from `start` to `end` as pct goes from 0.0 to 1.0.\"\n        return (end - start) * pct + start\n\n    def set_lr_mom(self) -> None:\n        \"\"\"Update the learning rate and momentum\"\"\"\n        if self.step_num <= self.step_size_up:\n            # update learining rate\n            computed_lr = self.anneal_func(\n                self.initial_lr, self.max_lr, self.step_num / self.step_size_up\n            )\n            K.set_value(self.model.optimizer.lr, computed_lr)\n            # update momentum if cycle_momentum\n            if self.cycle_momentum:\n                computed_momentum = self.anneal_func(\n                    self.m_momentum, self.b_momentum, self.step_num / self.step_size_up\n                )\n                try:\n                    K.set_value(self.model.optimizer.momentum,\n                                computed_momentum)\n                except:\n                    K.set_value(self.model.optimizer.beta_1, computed_momentum)\n        else:\n            down_step_num = self.step_num - self.step_size_up\n            # update learning rate\n            computed_lr = self.anneal_func(\n                self.max_lr, self.min_lr, down_step_num / self.step_size_down\n            )\n            K.set_value(self.model.optimizer.lr, computed_lr)\n            # update momentum if cycle_momentum\n            if self.cycle_momentum:\n                computed_momentum = self.anneal_func(\n                    self.b_momentum,\n                    self.m_momentum,\n                    down_step_num / self.step_size_down,\n                )\n                try:\n                    K.set_value(self.model.optimizer.momentum,\n                                computed_momentum)\n                except:\n                    K.set_value(self.model.optimizer.beta_1, computed_momentum)\n\n    def on_train_begin(self, logs=None) -> None:\n        # Set initial learning rate & momentum values\n        K.set_value(self.model.optimizer.lr, self.initial_lr)\n        if self.cycle_momentum:\n            try:\n                K.set_value(self.model.optimizer.momentum, self.momentum)\n            except:\n                K.set_value(self.model.optimizer.beta_1, self.momentum)\n\n    def on_train_batch_end(self, batch, logs=None) -> None:\n        # Grab the current learning rate & momentum\n        lr = float(K.get_value(self.model.optimizer.lr))\n        try:\n            mom = float(K.get_value(self.model.optimizer.momentum))\n        except:\n            mom = float(K.get_value(self.model.optimizer.beta_1))\n        # Append to the list\n        self.track_lr.append(lr)\n        self.track_mom.append(mom)\n        # Update learning rate & momentum\n        self.set_lr_mom()\n        # increment step_num\n        self.step_num += 1\n\n    def plot_lrs_moms(self, axes=None) -> None:\n        if axes == None:\n            _, (ax1, ax2) = plt.subplots(1, 2, figsize=(15, 5))\n        else:\n            try:\n                ax1, ax2 = axes\n            except:\n                ax1, ax2 = axes[0], axes[1]\n        ax1.plot(self.track_lr)\n        ax1.set_title(\"Learning Rate vs Steps\")\n        ax2.plot(self.track_mom)\n        ax2.set_title(\"Momentum (or beta_1) vs Steps\")\n\n\nmodel_checkpoint_callback = ModelCheckpoint(filepath=best_ck_point,\n                                            save_weights_only=False,\n                                            monitor='val_accuracy',\n                                            mode='max',\n                                            verbose=1,\n                                            save_best_only=True)\n\nearly_stop = EarlyStopping(monitor='val_loss',\n                           patience=10,\n                           verbose=1,\n                           restore_best_weights=True)\n\nreduce_lr = ReduceLROnPlateau(monitor='val_loss',\n                              patience=reduce_lr_in,\n                              verbose=1)\n\none_cycle = OneCycleLr(max_lr=1e-3, steps_per_epoch=setps_in_epoch, epochs=epoch)","metadata":{"papermill":{"duration":0.048165,"end_time":"2022-05-08T07:09:44.707843","exception":false,"start_time":"2022-05-08T07:09:44.659678","status":"completed"},"pycharm":{"name":"#%%\n"},"tags":[],"execution":{"iopub.status.busy":"2022-05-29T15:35:57.185155Z","iopub.execute_input":"2022-05-29T15:35:57.185639Z","iopub.status.idle":"2022-05-29T15:35:57.230623Z","shell.execute_reply.started":"2022-05-29T15:35:57.185605Z","shell.execute_reply":"2022-05-29T15:35:57.229776Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.optimizer.lr","metadata":{"execution":{"iopub.status.busy":"2022-05-29T15:35:58.757617Z","iopub.execute_input":"2022-05-29T15:35:58.758251Z","iopub.status.idle":"2022-05-29T15:35:58.765318Z","shell.execute_reply.started":"2022-05-29T15:35:58.758211Z","shell.execute_reply":"2022-05-29T15:35:58.764486Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history = model.fit(train_generator,\n                    validation_data=valid_generator,\n                    epochs=epoch,\n                    steps_per_epoch=821,\n                    validation_steps=370,\n                    callbacks=[early_stop, model_checkpoint_callback, one_cycle])","metadata":{"papermill":{"duration":6791.786561,"end_time":"2022-05-08T09:02:56.533572","exception":false,"start_time":"2022-05-08T07:09:44.747011","status":"completed"},"pycharm":{"name":"#%%\n"},"tags":[],"execution":{"iopub.status.busy":"2022-05-29T15:36:00.367521Z","iopub.execute_input":"2022-05-29T15:36:00.367807Z","iopub.status.idle":"2022-05-29T17:34:23.908269Z","shell.execute_reply.started":"2022-05-29T15:36:00.367773Z","shell.execute_reply":"2022-05-29T17:34:23.901920Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.evaluate(valid_generator)","metadata":{"papermill":{"duration":36.727198,"end_time":"2022-05-08T09:03:45.114069","exception":false,"start_time":"2022-05-08T09:03:08.386871","status":"completed"},"pycharm":{"name":"#%%\n"},"tags":[],"execution":{"iopub.status.busy":"2022-05-28T17:29:43.203469Z","iopub.execute_input":"2022-05-28T17:29:43.204056Z","iopub.status.idle":"2022-05-28T17:30:30.911279Z","shell.execute_reply.started":"2022-05-28T17:29:43.204016Z","shell.execute_reply":"2022-05-28T17:30:30.910583Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"temp_1 = pd.DataFrame(history.history)\ntemp_1.to_pickle('history.pkl')\ntemp_1","metadata":{"papermill":{"duration":11.8875,"end_time":"2022-05-08T09:04:09.551403","exception":false,"start_time":"2022-05-08T09:03:57.663903","status":"completed"},"pycharm":{"name":"#%%\n"},"tags":[],"execution":{"iopub.status.busy":"2022-05-28T17:30:30.913021Z","iopub.execute_input":"2022-05-28T17:30:30.913254Z","iopub.status.idle":"2022-05-28T17:30:30.940245Z","shell.execute_reply.started":"2022-05-28T17:30:30.913219Z","shell.execute_reply":"2022-05-28T17:30:30.939559Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.save(final_model)","metadata":{"papermill":{"duration":13.392409,"end_time":"2022-05-08T09:04:34.715153","exception":false,"start_time":"2022-05-08T09:04:21.322744","status":"completed"},"pycharm":{"name":"#%%\n"},"tags":[],"execution":{"iopub.status.busy":"2022-05-28T17:30:30.941427Z","iopub.execute_input":"2022-05-28T17:30:30.941695Z","iopub.status.idle":"2022-05-28T17:30:32.109641Z","shell.execute_reply.started":"2022-05-28T17:30:30.941646Z","shell.execute_reply":"2022-05-28T17:30:32.108861Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"load_model(final_model).evaluate(valid_generator)","metadata":{"execution":{"iopub.status.busy":"2022-05-28T17:32:30.556266Z","iopub.execute_input":"2022-05-28T17:32:30.556526Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"valid_generator.class_indices","metadata":{"papermill":{"duration":12.06445,"end_time":"2022-05-08T09:04:58.858379","exception":false,"start_time":"2022-05-08T09:04:46.793929","status":"completed"},"pycharm":{"name":"#%%\n"},"tags":[],"execution":{"iopub.status.busy":"2022-05-28T17:29:13.355285Z","iopub.status.idle":"2022-05-28T17:29:13.355996Z","shell.execute_reply.started":"2022-05-28T17:29:13.355731Z","shell.execute_reply":"2022-05-28T17:29:13.355759Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import json\n\nwith open('class_indices.json', 'w') as file:\n    json.dump(train_generator.class_indices, file)","metadata":{"papermill":{"duration":12.41886,"end_time":"2022-05-08T09:05:23.674097","exception":false,"start_time":"2022-05-08T09:05:11.255237","status":"completed"},"pycharm":{"name":"#%%\n"},"tags":[],"execution":{"iopub.status.busy":"2022-05-28T17:29:13.357166Z","iopub.status.idle":"2022-05-28T17:29:13.357807Z","shell.execute_reply.started":"2022-05-28T17:29:13.357554Z","shell.execute_reply":"2022-05-28T17:29:13.357578Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_pred = pd.read_csv('../input/sorghum-id-fgvc-9/sample_submission.csv')","metadata":{"execution":{"iopub.status.busy":"2022-05-28T17:29:13.358913Z","iopub.status.idle":"2022-05-28T17:29:13.3595Z","shell.execute_reply.started":"2022-05-28T17:29:13.359276Z","shell.execute_reply":"2022-05-28T17:29:13.3593Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds_generator = ImageDataGenerator().flow_from_dataframe(dataframe=df_pred,\n                                                           directory='../input/sorghum-id-fgvc-9/test',\n                                                           x_col='filename',\n                                                           y_col='cultivar',\n                                                           batch_size=batch_size,\n                                                           target_size=tardet_input_size)\n\npreds = model.predict(preds_generator)","metadata":{"execution":{"iopub.status.busy":"2022-05-28T17:29:13.360618Z","iopub.status.idle":"2022-05-28T17:29:13.361219Z","shell.execute_reply.started":"2022-05-28T17:29:13.360996Z","shell.execute_reply":"2022-05-28T17:29:13.36102Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"key = []\n\nfor i in range(len(preds)):\n    key.append(list(valid_generator.class_indices.keys())[list(valid_generator.class_indices.values()).index(np.argmax(preds[i]))])\n","metadata":{"execution":{"iopub.status.busy":"2022-05-28T17:29:13.362314Z","iopub.status.idle":"2022-05-28T17:29:13.36292Z","shell.execute_reply.started":"2022-05-28T17:29:13.362697Z","shell.execute_reply":"2022-05-28T17:29:13.362721Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_pred = df_pred.drop('cultivar', axis=1)\ndf_pred['cultivar'] = key\ndf_pred","metadata":{"execution":{"iopub.status.busy":"2022-05-28T17:29:13.364034Z","iopub.status.idle":"2022-05-28T17:29:13.364633Z","shell.execute_reply.started":"2022-05-28T17:29:13.364399Z","shell.execute_reply":"2022-05-28T17:29:13.364423Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_pred.to_csv('submission_15.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2022-05-28T17:29:13.366442Z","iopub.status.idle":"2022-05-28T17:29:13.367051Z","shell.execute_reply.started":"2022-05-28T17:29:13.366827Z","shell.execute_reply":"2022-05-28T17:29:13.36685Z"},"trusted":true},"execution_count":null,"outputs":[]}]}