{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# !pip install torch==1.10.1+cu111 torchvision==0.11.2+cu111 torchaudio==0.10.1 -f https://download.pytorch.org/whl/torch_stable.html\n!pip install timm # install pytorch image models\n!pip install torchmetrics","metadata":{"papermill":{"duration":19.586589,"end_time":"2022-04-30T09:16:46.27058","exception":false,"start_time":"2022-04-30T09:16:26.683991","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-05-04T07:16:43.311481Z","iopub.execute_input":"2022-05-04T07:16:43.312465Z","iopub.status.idle":"2022-05-04T07:17:00.155589Z","shell.execute_reply.started":"2022-05-04T07:16:43.312363Z","shell.execute_reply":"2022-05-04T07:17:00.154764Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nimport os\nimport pandas as pd\nimport numpy as np\nimport random \n\nimport albumentations as A\nimport cv2\nimport matplotlib.pyplot as plt\nfrom tqdm import tqdm\nfrom sklearn import preprocessing\nfrom sklearn.model_selection import StratifiedKFold\nimport timm\n\nimport torchvision\nfrom torch.utils.data import Dataset, DataLoader\nfrom torchvision import transforms\nimport torchvision.models as models\nimport torch.nn.functional as F\nfrom torch import nn\nimport torchmetrics ","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","papermill":{"duration":8.860223,"end_time":"2022-04-30T09:16:55.179518","exception":false,"start_time":"2022-04-30T09:16:46.319295","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-05-04T07:17:00.159127Z","iopub.execute_input":"2022-05-04T07:17:00.159348Z","iopub.status.idle":"2022-05-04T07:17:05.073314Z","shell.execute_reply.started":"2022-05-04T07:17:00.159324Z","shell.execute_reply":"2022-05-04T07:17:05.072478Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!nvidia-smi","metadata":{"papermill":{"duration":0.750411,"end_time":"2022-04-30T09:16:55.970277","exception":false,"start_time":"2022-04-30T09:16:55.219866","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-05-04T07:17:05.07502Z","iopub.execute_input":"2022-05-04T07:17:05.075268Z","iopub.status.idle":"2022-05-04T07:17:05.803126Z","shell.execute_reply.started":"2022-05-04T07:17:05.075232Z","shell.execute_reply":"2022-05-04T07:17:05.802223Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class GlobalConstantsConfigure():\n    def __init__(self):\n        self.continue_training = True\n        self.last_model = '../input/sorghum-100-cultivar-identification-8/tf_efficientnetv2_m_in21k_45_last.pt' \n        self.num_epochs_done = 55\n        self.seed = 107\n        self.num_classes = 100\n        self.biggest_loss = 999\n        self.training_size_rate = 0.8\n        self.training_dir = '../input/sorghum-id-fgvc-9/train_images'\n        self.model_name = 'tf_efficientnetv2_m_in21k'\n        self.model_path = './tf_efficientnetv2_m_in21k_sgd_50.pt'\n        self.image_size = 512\n        self.batch_size = 8\n        self.batch_size_testing = 32\n        self.lr = 3e-5 # 3e-5\n        self.num_epochs = 15\n        self.steps_per_decay = 5\n        self.device = torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu')\n        self.num_workers = 2  # if torch.cuda.is_available() else 4\ngcc = GlobalConstantsConfigure()","metadata":{"papermill":{"duration":0.104395,"end_time":"2022-04-30T09:16:56.11564","exception":false,"start_time":"2022-04-30T09:16:56.011245","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-05-04T07:17:05.805854Z","iopub.execute_input":"2022-05-04T07:17:05.806571Z","iopub.status.idle":"2022-05-04T07:17:05.880474Z","shell.execute_reply.started":"2022-05-04T07:17:05.806528Z","shell.execute_reply":"2022-05-04T07:17:05.879561Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def set_seed(seed) : \n    os.environ['PYTHONHASHSEED'] = str(seed)\n    random.seed(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = True\nset_seed(gcc.seed)","metadata":{"papermill":{"duration":0.054375,"end_time":"2022-04-30T09:16:56.210497","exception":false,"start_time":"2022-04-30T09:16:56.156122","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-05-04T07:17:05.884529Z","iopub.execute_input":"2022-05-04T07:17:05.884832Z","iopub.status.idle":"2022-05-04T07:17:05.894741Z","shell.execute_reply.started":"2022-05-04T07:17:05.884791Z","shell.execute_reply":"2022-05-04T07:17:05.894Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# df = pd.read_csv('../input/sorghum-id-fgvc-9/train_cultivar_mapping.csv')","metadata":{"papermill":{"duration":0.055968,"end_time":"2022-04-30T09:16:56.307004","exception":false,"start_time":"2022-04-30T09:16:56.251036","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-05-04T07:17:05.896869Z","iopub.execute_input":"2022-05-04T07:17:05.897414Z","iopub.status.idle":"2022-05-04T07:17:05.901595Z","shell.execute_reply.started":"2022-05-04T07:17:05.897324Z","shell.execute_reply":"2022-05-04T07:17:05.900764Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# df","metadata":{"papermill":{"duration":0.0464,"end_time":"2022-04-30T09:16:56.393419","exception":false,"start_time":"2022-04-30T09:16:56.347019","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-05-04T07:17:05.903048Z","iopub.execute_input":"2022-05-04T07:17:05.903348Z","iopub.status.idle":"2022-05-04T07:17:05.911835Z","shell.execute_reply.started":"2022-05-04T07:17:05.903312Z","shell.execute_reply":"2022-05-04T07:17:05.911106Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# for row in df['image']:\n#     if not os.path.isfile('../input/sorghum-id-fgvc-9/train_images/' + row):\n#         print(row)","metadata":{"papermill":{"duration":0.046728,"end_time":"2022-04-30T09:16:56.480282","exception":false,"start_time":"2022-04-30T09:16:56.433554","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-05-04T07:17:05.913239Z","iopub.execute_input":"2022-05-04T07:17:05.9135Z","iopub.status.idle":"2022-05-04T07:17:05.921107Z","shell.execute_reply.started":"2022-05-04T07:17:05.913465Z","shell.execute_reply":"2022-05-04T07:17:05.920251Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# df.drop(df[df['image'] == '.DS_Store'].index, inplace=True)","metadata":{"papermill":{"duration":0.04616,"end_time":"2022-04-30T09:16:56.566521","exception":false,"start_time":"2022-04-30T09:16:56.520361","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-05-04T07:17:05.922443Z","iopub.execute_input":"2022-05-04T07:17:05.922808Z","iopub.status.idle":"2022-05-04T07:17:05.929119Z","shell.execute_reply.started":"2022-05-04T07:17:05.922773Z","shell.execute_reply":"2022-05-04T07:17:05.928407Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# df.groupby(['cultivar']).count().describe()","metadata":{"papermill":{"duration":0.046225,"end_time":"2022-04-30T09:16:56.652915","exception":false,"start_time":"2022-04-30T09:16:56.60669","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-05-04T07:17:05.93304Z","iopub.execute_input":"2022-05-04T07:17:05.933234Z","iopub.status.idle":"2022-05-04T07:17:05.937774Z","shell.execute_reply.started":"2022-05-04T07:17:05.933211Z","shell.execute_reply":"2022-05-04T07:17:05.936967Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# df.describe()","metadata":{"papermill":{"duration":0.045761,"end_time":"2022-04-30T09:16:56.738317","exception":false,"start_time":"2022-04-30T09:16:56.692556","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-05-04T07:17:05.939304Z","iopub.execute_input":"2022-05-04T07:17:05.939801Z","iopub.status.idle":"2022-05-04T07:17:05.945921Z","shell.execute_reply.started":"2022-05-04T07:17:05.939767Z","shell.execute_reply":"2022-05-04T07:17:05.94516Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# label_encoder = preprocessing.LabelEncoder()\n# label_encoder.fit(df.cultivar)\n# labels = label_encoder.transform(df.cultivar)","metadata":{"papermill":{"duration":0.04568,"end_time":"2022-04-30T09:16:56.823805","exception":false,"start_time":"2022-04-30T09:16:56.778125","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-05-04T07:17:05.947817Z","iopub.execute_input":"2022-05-04T07:17:05.948286Z","iopub.status.idle":"2022-05-04T07:17:05.954294Z","shell.execute_reply.started":"2022-05-04T07:17:05.948253Z","shell.execute_reply":"2022-05-04T07:17:05.953483Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_all = pd.read_csv('../input/sorghum-id-fgvc-9/train_cultivar_mapping.csv')\nprint(len(df_all))\ndf_all.dropna(inplace=True)\nprint(len(df_all))\ndf_all.head()","metadata":{"papermill":{"duration":0.101104,"end_time":"2022-04-30T09:16:56.964882","exception":false,"start_time":"2022-04-30T09:16:56.863778","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-05-04T07:17:05.957169Z","iopub.execute_input":"2022-05-04T07:17:05.957785Z","iopub.status.idle":"2022-05-04T07:17:06.018929Z","shell.execute_reply.started":"2022-05-04T07:17:05.957753Z","shell.execute_reply":"2022-05-04T07:17:06.018209Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"unique_cultivars = list(df_all[\"cultivar\"].unique())","metadata":{"papermill":{"duration":0.053337,"end_time":"2022-04-30T09:16:57.059335","exception":false,"start_time":"2022-04-30T09:16:57.005998","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-05-04T07:17:06.020397Z","iopub.execute_input":"2022-05-04T07:17:06.020882Z","iopub.status.idle":"2022-05-04T07:17:06.032125Z","shell.execute_reply.started":"2022-05-04T07:17:06.020844Z","shell.execute_reply":"2022-05-04T07:17:06.031264Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_all[\"file_path\"] = df_all[\"image\"].apply(lambda image: '../input/sorghum-id-fgvc-9/train_images/' + image)\ndf_all[\"cultivar_index\"] = df_all[\"cultivar\"].map(lambda item: unique_cultivars.index(item))\ndf_all[\"is_exist\"] = df_all[\"file_path\"].apply(lambda file_path: os.path.exists(file_path))\ndf_all = df_all[df_all.is_exist==True]\ndf_all.head()","metadata":{"papermill":{"duration":18.92019,"end_time":"2022-04-30T09:17:16.020078","exception":false,"start_time":"2022-04-30T09:16:57.099888","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-05-04T07:17:06.033642Z","iopub.execute_input":"2022-05-04T07:17:06.033976Z","iopub.status.idle":"2022-05-04T07:17:19.630033Z","shell.execute_reply.started":"2022-05-04T07:17:06.033939Z","shell.execute_reply":"2022-05-04T07:17:19.629291Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"skf = StratifiedKFold(n_splits=4, shuffle=True, random_state=gcc.seed)\n\nfor train_idx, valid_idx in skf.split(df_all['image'], df_all[\"cultivar_index\"]):\n    df_train = df_all.iloc[train_idx]\n    df_valid = df_all.iloc[valid_idx]\n\nprint(f\"train size: {len(df_train)}\")\nprint(f\"valid size: {len(df_valid)}\")\n\nprint(df_train.cultivar.value_counts())\nprint(df_valid.cultivar.value_counts())","metadata":{"papermill":{"duration":0.076751,"end_time":"2022-04-30T09:17:16.138037","exception":false,"start_time":"2022-04-30T09:17:16.061286","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-05-04T07:17:19.63158Z","iopub.execute_input":"2022-05-04T07:17:19.632101Z","iopub.status.idle":"2022-05-04T07:17:19.6662Z","shell.execute_reply.started":"2022-05-04T07:17:19.63206Z","shell.execute_reply":"2022-05-04T07:17:19.665393Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# dirs = df['image'].map(lambda x: '../input/sorghum-id-fgvc-9/train_images/' + x)","metadata":{"papermill":{"duration":0.047498,"end_time":"2022-04-30T09:17:16.226746","exception":false,"start_time":"2022-04-30T09:17:16.179248","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-05-04T07:17:19.667426Z","iopub.execute_input":"2022-05-04T07:17:19.667706Z","iopub.status.idle":"2022-05-04T07:17:19.671546Z","shell.execute_reply.started":"2022-05-04T07:17:19.667667Z","shell.execute_reply":"2022-05-04T07:17:19.670829Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# dirs = np.array(dirs.tolist())","metadata":{"papermill":{"duration":0.076931,"end_time":"2022-04-30T09:17:16.347559","exception":false,"start_time":"2022-04-30T09:17:16.270628","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-05-04T07:17:19.67298Z","iopub.execute_input":"2022-05-04T07:17:19.673453Z","iopub.status.idle":"2022-05-04T07:17:19.681426Z","shell.execute_reply.started":"2022-05-04T07:17:19.673414Z","shell.execute_reply":"2022-05-04T07:17:19.680653Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# print(len(labels))\n# print(len(dirs))","metadata":{"papermill":{"duration":0.047246,"end_time":"2022-04-30T09:17:16.436645","exception":false,"start_time":"2022-04-30T09:17:16.389399","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-05-04T07:17:19.682742Z","iopub.execute_input":"2022-05-04T07:17:19.683222Z","iopub.status.idle":"2022-05-04T07:17:19.689672Z","shell.execute_reply.started":"2022-05-04T07:17:19.683187Z","shell.execute_reply":"2022-05-04T07:17:19.688919Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# def path_walks_split_set(dirs, labels):\n\n\n#     training_dirs = dirs[:int(gcc.training_size_rate * len(dirs))]\n#     training_labels = labels[:int(gcc.training_size_rate * len(labels))]\n#     validation_dirs = dirs[int(gcc.training_size_rate * len(dirs)):]\n#     validation_labels = labels[int(gcc.training_size_rate * len(labels)):] \n    \n\n    \n#     return training_dirs, training_labels, validation_dirs, validation_labels\n\n# training_dirs, training_labels, validation_dirs, validation_labels = path_walks_split_set(dirs, labels)","metadata":{"papermill":{"duration":0.04747,"end_time":"2022-04-30T09:17:16.525805","exception":false,"start_time":"2022-04-30T09:17:16.478335","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-05-04T07:17:19.691221Z","iopub.execute_input":"2022-05-04T07:17:19.691705Z","iopub.status.idle":"2022-05-04T07:17:19.697974Z","shell.execute_reply.started":"2022-05-04T07:17:19.691669Z","shell.execute_reply":"2022-05-04T07:17:19.697132Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# len(os.listdir('../input/sorghum-id-fgvc-9/train_images'))","metadata":{"papermill":{"duration":0.047195,"end_time":"2022-04-30T09:17:16.614614","exception":false,"start_time":"2022-04-30T09:17:16.567419","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-05-04T07:17:19.699065Z","iopub.execute_input":"2022-05-04T07:17:19.699434Z","iopub.status.idle":"2022-05-04T07:17:19.709296Z","shell.execute_reply.started":"2022-05-04T07:17:19.699398Z","shell.execute_reply":"2022-05-04T07:17:19.708502Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"papermill":{"duration":0.041709,"end_time":"2022-04-30T09:17:16.697265","exception":false,"start_time":"2022-04-30T09:17:16.655556","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train","metadata":{"papermill":{"duration":0.056903,"end_time":"2022-04-30T09:17:16.795361","exception":false,"start_time":"2022-04-30T09:17:16.738458","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-05-04T07:17:19.711408Z","iopub.execute_input":"2022-05-04T07:17:19.711937Z","iopub.status.idle":"2022-05-04T07:17:19.728101Z","shell.execute_reply.started":"2022-05-04T07:17:19.711879Z","shell.execute_reply":"2022-05-04T07:17:19.727403Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# avc","metadata":{"papermill":{"duration":0.04858,"end_time":"2022-04-30T09:17:16.885552","exception":false,"start_time":"2022-04-30T09:17:16.836972","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-05-04T07:17:19.729544Z","iopub.execute_input":"2022-05-04T07:17:19.730107Z","iopub.status.idle":"2022-05-04T07:17:19.733707Z","shell.execute_reply.started":"2022-05-04T07:17:19.73007Z","shell.execute_reply":"2022-05-04T07:17:19.732786Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class SorghumDataset(Dataset):\n    def __init__(self, dirs, labels, transformation=None):\n        super(SorghumDataset,self).__init__()\n        self.dirs = dirs\n        self.labels = labels\n        self.transformation = transformation\n    def __len__(self):\n        return len(self.dirs)\n\n    def __getitem__(self, index):\n        image = cv2.imread(self.dirs[index])\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n        label = self.labels[index] # need to one hot encoding here\n        \n        \n        image = np.array(image)\n\n        if self.transformation:\n            aug_image = self.transformation(image=image)\n            image = aug_image['image']\n            \n        image = image / 255.\n        image = image.transpose((2, 0, 1))\n        \n        image = torch.from_numpy(image).type(torch.float32)\n        image = transforms.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225))(image)\n        \n        labels = torch.from_numpy(np.array(self.labels[index])).type(torch.float32)\n        return image, labels","metadata":{"papermill":{"duration":0.053491,"end_time":"2022-04-30T09:17:16.980935","exception":false,"start_time":"2022-04-30T09:17:16.927444","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-05-04T07:17:19.735515Z","iopub.execute_input":"2022-05-04T07:17:19.736069Z","iopub.status.idle":"2022-05-04T07:17:19.74772Z","shell.execute_reply.started":"2022-05-04T07:17:19.736033Z","shell.execute_reply":"2022-05-04T07:17:19.747012Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"training_transformation = A.Compose([\n    A.Resize(width=gcc.image_size, height=gcc.image_size, p=1.0),\n    A.Flip(p=0.5),\n    A.RandomRotate90(p=0.5),\n    A.ShiftScaleRotate(p=0.5),\n#     A.HueSaturationValue(p=0.5),\n#     A.OneOf([\n#         A.RandomBrightnessContrast(p=0.5),\n#         A.RandomGamma(p=0.5),\n#     ], p=0.5),\n#     A.OneOf([\n#         A.Blur(p=0.1),\n#         A.GaussianBlur(p=0.1),\n#         A.MotionBlur(p=0.1),\n#     ], p=0.1),\n#     A.OneOf([\n#         A.GaussNoise(p=0.1),\n#         A.ISONoise(p=0.1),\n#         A.GridDropout(ratio=0.5, p=0.2),\n#         A.CoarseDropout(max_holes=16, min_holes=8, max_height=16, max_width=16, min_height=8, min_width=8, p=0.2)\n#     ], p=0.2),\n\n])\nvalidation_transformation = A.Compose([\n    A.Resize(width=gcc.image_size, height=gcc.image_size, p=1.0)\n])","metadata":{"papermill":{"duration":0.054185,"end_time":"2022-04-30T09:17:17.077134","exception":false,"start_time":"2022-04-30T09:17:17.022949","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-05-04T07:17:19.749415Z","iopub.execute_input":"2022-05-04T07:17:19.750038Z","iopub.status.idle":"2022-05-04T07:17:19.757262Z","shell.execute_reply.started":"2022-05-04T07:17:19.74995Z","shell.execute_reply":"2022-05-04T07:17:19.756437Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# training_set = SorghumDataset(training_dirs, training_labels, training_transformation)\n# validation_set = SorghumDataset(validation_dirs, validation_labels, validation_transformation)\n\ntraining_set = SorghumDataset(df_train.file_path.values, df_train.cultivar_index.values, training_transformation)\nvalidation_set = SorghumDataset(df_valid.file_path.values, df_valid.cultivar_index.values, validation_transformation)\n\n\ntraining_dataloader = DataLoader(\n    training_set,\n    batch_size = gcc.batch_size,\n    shuffle = True,\n    num_workers = gcc.num_workers,\n    pin_memory = True, \n    drop_last = True\n)\nvalidation_dataloader = DataLoader(\n    validation_set,\n    batch_size = gcc.batch_size,\n    # shuffle = True,\n    num_workers = gcc.num_workers,\n    pin_memory = True,\n    drop_last = True\n)","metadata":{"papermill":{"duration":0.051005,"end_time":"2022-04-30T09:17:17.1705","exception":false,"start_time":"2022-04-30T09:17:17.119495","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-05-04T07:17:19.758169Z","iopub.execute_input":"2022-05-04T07:17:19.760409Z","iopub.status.idle":"2022-05-04T07:17:19.769602Z","shell.execute_reply.started":"2022-05-04T07:17:19.760377Z","shell.execute_reply":"2022-05-04T07:17:19.768849Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CustomModel(torch.nn.Module): \n    def __init__(self, model_backbone):\n        super(CustomModel,self).__init__()\n        self.model = model_backbone\n        \n        self.model.classifier = nn.Sequential(\n            nn.BatchNorm1d(1280),\n            nn.Linear(1280, 512),\n            nn.Dropout(0.5),\n            nn.ReLU(inplace=True),\n            # nn.BatchNorm1d(512),\n            \n            nn.Linear(512, gcc.num_classes),\n            \n#             nn.BatchNorm1d(1280),\n#             nn.Linear(1280, 512),\n#             nn.Dropout(0.5),\n#             nn.SiLU(inplace=True),\n\n#             nn.Linear(512, 256),\n#             nn.Dropout(0.5),\n#             nn.SiLU(inplace=True),\n#             nn.Linear(256, gcc.num_classes)\n        )\n    def forward(self,x):\n        x = self.model(x)\n        return x\n","metadata":{"papermill":{"duration":0.050489,"end_time":"2022-04-30T09:17:17.262999","exception":false,"start_time":"2022-04-30T09:17:17.21251","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-05-04T07:17:19.771076Z","iopub.execute_input":"2022-05-04T07:17:19.771581Z","iopub.status.idle":"2022-05-04T07:17:19.77929Z","shell.execute_reply.started":"2022-05-04T07:17:19.771545Z","shell.execute_reply":"2022-05-04T07:17:19.778563Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# backbone = models.efficientnet_v2_s(pretrained=True) # models.efficientnet_b4(pretrained=True) # models.resnet50(pretrained=True) # models.resnet50(pretrained=True)\nbackbone = timm.create_model(gcc.model_name,pretrained=True)\n\n\n# print(index)\nmodel = CustomModel(backbone)\nloss_func = torch.nn.CrossEntropyLoss()\nmetrics_acc = torchmetrics.Accuracy(threshold=0.0, num_classes = gcc.num_classes)\nprint(model)\n\n\n# for index, child in enumerate(backbone.children()):\n#     print(index)\n#     if index <= 7:\n#         for param in child.parameters():\n#             param.requires_grad = False\n\n\ntrainable_parameters = [param for param in model.parameters() if param.requires_grad == True]\noptimizer = torch.optim.Adam(trainable_parameters, lr = gcc.lr)\n# optimizer = torch.optim.SGD(trainable_parameters, lr = gcc.lr, momentum = 0.9)\n# lr_scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, gcc.steps_per_decay)\nlr_scheduler = torch.optim.lr_scheduler.CosineAnnealingWarmRestarts(optimizer, T_0=5, T_mult=2)\n\nmodel.to(gcc.device)\n\nif gcc.continue_training == True:\n    # model.load_state_dict(torch.load(gcc.last_model))\n    checkpoint = torch.load(gcc.last_model)\n    \n    model.load_state_dict(checkpoint['model_state_dict'])\n    \n    \n    # optimizer = torch.optim.Adam(trainable_parameters, lr = gcc.lr)\n    optimizer.load_state_dict(checkpoint['optimizer_state_dict'])\n    \n    # lr_scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, gcc.steps_per_decay)\n    lr_scheduler.load_state_dict(checkpoint['scheduler_state_dict'])\n    \n    # print(lr_scheduler.state_dict())\n\nprint('load model done')","metadata":{"papermill":{"duration":10.660311,"end_time":"2022-04-30T09:17:27.966236","exception":false,"start_time":"2022-04-30T09:17:17.305925","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-05-04T07:17:19.784299Z","iopub.execute_input":"2022-05-04T07:17:19.784761Z","iopub.status.idle":"2022-05-04T07:18:05.403379Z","shell.execute_reply.started":"2022-05-04T07:17:19.784717Z","shell.execute_reply":"2022-05-04T07:18:05.401799Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# def calc_accuracy(pred, true):\n#     # print(pred, true)\n#     true = true.type(torch.int64) # label\n#     pred = F.softmax(pred, dim = 1)\n#     true = torch.zeros(pred.shape[0], pred.shape[1]).scatter_(1, true.unsqueeze(1), 1.)\n#     acc = (true.argmax(-1) == pred.argmax(-1)).float().detach().numpy()\n#     acc = float(acc.sum() / len(acc))\n#     return round(acc, 4)","metadata":{"papermill":{"duration":0.053649,"end_time":"2022-04-30T09:17:28.065129","exception":false,"start_time":"2022-04-30T09:17:28.01148","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-05-04T07:18:05.404694Z","iopub.execute_input":"2022-05-04T07:18:05.404964Z","iopub.status.idle":"2022-05-04T07:18:05.40958Z","shell.execute_reply.started":"2022-05-04T07:18:05.404929Z","shell.execute_reply":"2022-05-04T07:18:05.408577Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def training_progress(training_dataloader, loss_func, scheduler):\n    model.train()\n    training_loss = 0\n    training_acc = 0\n    cnt = 0 \n    print('Learning rate: ',scheduler.get_last_lr())\n    print(scheduler.state_dict())\n    training_loader = tqdm(training_dataloader, desc='Iterating through the training set')\n    for image, label in training_loader:\n        image = image.to(gcc.device)\n        label = label.to(gcc.device)\n        \n        output = model(image)\n        # output.to(gcc.device)\n\n        acc = metrics_acc(output.cpu().argmax(1), label.cpu().int())\n        loss = loss_func(output, label.long())\n        # calculate accuracy here\n\n        training_loss += loss.detach().item()\n        training_acc += acc\n        cnt +=1 \n        \n        optimizer.zero_grad()\n        loss.backward()\n        optimizer.step()\n        scheduler.step()\n    \n    mean_training_loss = training_loss / cnt\n    mean_training_acc = training_acc / cnt\n    \n    return mean_training_loss, mean_training_acc\n    ","metadata":{"papermill":{"duration":0.055976,"end_time":"2022-04-30T09:17:28.166093","exception":false,"start_time":"2022-04-30T09:17:28.110117","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-05-04T07:18:05.410813Z","iopub.execute_input":"2022-05-04T07:18:05.411667Z","iopub.status.idle":"2022-05-04T07:18:05.421437Z","shell.execute_reply.started":"2022-05-04T07:18:05.41163Z","shell.execute_reply":"2022-05-04T07:18:05.420694Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def validation_progress(validation_dataloader, loss_func):\n    model.eval()\n    validation_loss = 0\n    validation_acc = 0\n    cnt = 0 \n    validation_loader = tqdm(validation_dataloader, desc='Iterating through the validation set')\n    with torch.no_grad():\n        for image, label in validation_loader:\n            image = image.to(gcc.device)\n            label = label.to(gcc.device)\n\n            output = model(image)\n            loss = loss_func(output, label.long())\n            # acc = calc_accuracy(output.cpu(), label.cpu())\n            # output.to(gcc.device)\n            acc = metrics_acc(output.cpu().argmax(1), label.cpu().int())\n            # calculate accuracy here\n            validation_loss += loss.detach().item()\n            validation_acc += acc\n            \n            cnt += 1\n\n    mean_validation_loss = validation_loss / cnt\n    mean_validation_acc = validation_acc / cnt\n    return mean_validation_loss, mean_validation_acc","metadata":{"papermill":{"duration":0.053699,"end_time":"2022-04-30T09:17:28.263835","exception":false,"start_time":"2022-04-30T09:17:28.210136","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-05-04T07:18:05.422565Z","iopub.execute_input":"2022-05-04T07:18:05.423219Z","iopub.status.idle":"2022-05-04T07:18:05.433621Z","shell.execute_reply.started":"2022-05-04T07:18:05.423182Z","shell.execute_reply":"2022-05-04T07:18:05.432878Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def training_model(model, training_dataloader, validation_dataloader, loss_func, scheduler):\n    training_losses_history, validation_losses_history = [], []\n    training_acc_history, validation_acc_history = [], []\n    best_loss = gcc.biggest_loss\n    for epoch in range(gcc.num_epochs):\n        \n        training_loss, training_acc = training_progress(training_dataloader, loss_func, scheduler)\n        training_losses_history.append(training_loss)\n        training_acc_history.append(training_acc)\n        \n        validation_loss, validation_acc = validation_progress(validation_dataloader, loss_func)\n        validation_losses_history.append(validation_loss)\n        validation_acc_history.append(validation_acc)\n        \n        if validation_loss <= best_loss: # sussy baka\n            best_loss = validation_loss\n            torch.save({\n                'model_state_dict': model.state_dict(),\n                'optimizer_state_dict': optimizer.state_dict(),\n                'scheduler_state_dict': scheduler.state_dict()\n            }, gcc.model_name + '_best.pt')\n            # torch.save(model.state_dict(), gcc.model_name + '_best.pt')\n        \n        if epoch == gcc.num_epochs - 1: # i believe my timing capability\n            torch.save({\n                'model_state_dict': model.state_dict(),\n                'optimizer_state_dict': optimizer.state_dict(),\n                'scheduler_state_dict': scheduler.state_dict()\n            }, gcc.model_name + '_' + str(gcc.num_epochs_done + gcc.num_epochs) + '_last.pt')\n\n        print(f'Epoch {epoch + 1}/{gcc.num_epochs} | Training_loss : {training_loss:.3f} | Validation_loss : {validation_loss:.3f}' \n             + f' Training_acc : {training_acc:.3f} | Validation_acc : {validation_acc:.3f}'\n             )\n    return training_losses_history, validation_losses_history, training_acc_history, validation_acc_history\n","metadata":{"papermill":{"duration":0.05608,"end_time":"2022-04-30T09:17:28.364242","exception":false,"start_time":"2022-04-30T09:17:28.308162","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-05-04T07:18:05.435033Z","iopub.execute_input":"2022-05-04T07:18:05.435323Z","iopub.status.idle":"2022-05-04T07:18:05.446584Z","shell.execute_reply.started":"2022-05-04T07:18:05.435288Z","shell.execute_reply":"2022-05-04T07:18:05.445774Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"training_losses_history, validation_losses_history, training_acc_history, validation_acc_history = training_model(model, training_dataloader, validation_dataloader, loss_func, lr_scheduler)","metadata":{"papermill":{"duration":33802.202732,"end_time":"2022-04-30T18:40:50.611112","exception":false,"start_time":"2022-04-30T09:17:28.40838","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-05-04T07:18:05.447798Z","iopub.execute_input":"2022-05-04T07:18:05.448056Z","iopub.status.idle":"2022-05-04T08:28:06.504596Z","shell.execute_reply.started":"2022-05-04T07:18:05.448013Z","shell.execute_reply":"2022-05-04T08:28:06.50089Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# testing_progress(testing_dataloader, loss_func)","metadata":{"papermill":{"duration":23.140045,"end_time":"2022-04-30T18:41:37.791246","exception":false,"start_time":"2022-04-30T18:41:14.651201","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-05-04T08:28:06.507475Z","iopub.execute_input":"2022-05-04T08:28:06.507805Z","iopub.status.idle":"2022-05-04T08:28:06.513231Z","shell.execute_reply.started":"2022-05-04T08:28:06.507765Z","shell.execute_reply":"2022-05-04T08:28:06.511981Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_loss_history(model_name, train_loss_history, val_loss_history, num_epochs):\n    \n    x = np.arange(num_epochs)\n    fig = plt.figure(figsize=(10, 6))\n    plt.plot(x, train_loss_history, label='Train Loss', lw=3)\n    plt.plot(x, val_loss_history, label='Validation Loss', lw=3)\n\n    plt.title(f\"{model_name}\", fontsize=20)\n    plt.legend(fontsize=12)\n    plt.xlabel(\"Epoch\", fontsize=15)\n    plt.ylabel(\"Loss\", fontsize=15)\n\n    plt.show()\n    \nplot_loss_history(gcc.model_name, training_losses_history, validation_losses_history, gcc.num_epochs)","metadata":{"papermill":{"duration":23.105811,"end_time":"2022-04-30T18:42:23.479663","exception":false,"start_time":"2022-04-30T18:42:00.373852","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-05-04T08:28:06.515369Z","iopub.execute_input":"2022-05-04T08:28:06.51576Z","iopub.status.idle":"2022-05-04T08:28:06.821398Z","shell.execute_reply.started":"2022-05-04T08:28:06.515693Z","shell.execute_reply":"2022-05-04T08:28:06.820589Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_acc_history(model_name, train_acc_history, val_acc_history, num_epochs):\n    \n    x = np.arange(num_epochs)\n    fig = plt.figure(figsize=(10, 6))\n    plt.plot(x, train_acc_history, label='Training Accuracy', lw=3)\n    plt.plot(x, val_acc_history, label='Validation Accuracy', lw=3)\n\n    plt.title(f\"{model_name}\", fontsize=20)\n    plt.legend(fontsize=12)\n    plt.xlabel(\"Epoch\", fontsize=15)\n    plt.ylabel(\"Accuracy\", fontsize=15)\n\n    plt.show()\n    \nplot_acc_history(gcc.model_name, training_acc_history, validation_acc_history, gcc.num_epochs)","metadata":{"papermill":{"duration":23.289844,"end_time":"2022-04-30T18:43:10.278115","exception":false,"start_time":"2022-04-30T18:42:46.988271","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-05-04T08:28:06.823427Z","iopub.execute_input":"2022-05-04T08:28:06.825002Z","iopub.status.idle":"2022-05-04T08:28:07.091862Z","shell.execute_reply.started":"2022-05-04T08:28:06.824959Z","shell.execute_reply":"2022-05-04T08:28:07.087339Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"checkpoint = torch.load(gcc.model_name + '_best.pt')\n\nmodel.load_state_dict(checkpoint['model_state_dict'])","metadata":{"papermill":{"duration":23.282025,"end_time":"2022-04-30T18:43:56.435711","exception":false,"start_time":"2022-04-30T18:43:33.153686","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-05-04T08:28:07.093371Z","iopub.execute_input":"2022-05-04T08:28:07.093701Z","iopub.status.idle":"2022-05-04T08:28:08.201268Z","shell.execute_reply.started":"2022-05-04T08:28:07.093663Z","shell.execute_reply":"2022-05-04T08:28:08.200567Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub = pd.read_csv('../input/sorghum-id-fgvc-9/sample_submission.csv')\nsub.head()","metadata":{"papermill":{"duration":22.330454,"end_time":"2022-04-30T18:44:41.744865","exception":false,"start_time":"2022-04-30T18:44:19.414411","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-05-04T08:28:08.202745Z","iopub.execute_input":"2022-05-04T08:28:08.203689Z","iopub.status.idle":"2022-05-04T08:28:08.245843Z","shell.execute_reply.started":"2022-05-04T08:28:08.20365Z","shell.execute_reply":"2022-05-04T08:28:08.245044Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub[\"filename\"] = sub[\"filename\"].apply(lambda image: '../input/sorghum-id-fgvc-9/test/' + image)\nsub[\"cultivar\"] = 0\nsub.head()","metadata":{"papermill":{"duration":23.022476,"end_time":"2022-04-30T18:45:27.817636","exception":false,"start_time":"2022-04-30T18:45:04.79516","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-05-04T08:28:08.247541Z","iopub.execute_input":"2022-05-04T08:28:08.247815Z","iopub.status.idle":"2022-05-04T08:28:08.272556Z","shell.execute_reply.started":"2022-05-04T08:28:08.247779Z","shell.execute_reply":"2022-05-04T08:28:08.271799Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"testing_dataset = SorghumDataset(sub['filename'], sub['cultivar'], validation_transformation)\ntesting_dataloader = DataLoader(testing_dataset, \n                                batch_size=gcc.batch_size_testing, \n                                shuffle=False, \n                                num_workers=gcc.num_workers)","metadata":{"papermill":{"duration":22.302547,"end_time":"2022-04-30T18:46:13.164806","exception":false,"start_time":"2022-04-30T18:45:50.862259","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-05-04T08:28:08.27422Z","iopub.execute_input":"2022-05-04T08:28:08.274472Z","iopub.status.idle":"2022-05-04T08:28:08.28036Z","shell.execute_reply.started":"2022-05-04T08:28:08.274438Z","shell.execute_reply":"2022-05-04T08:28:08.279718Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# predictions = np.zeros(len(testing_dataloader))\npredictions = []\ncnt = 0 \nwith torch.no_grad():\n    for image, label in tqdm(testing_dataloader):\n        image = image.to(gcc.device)\n        outputs = model(image)\n        # print(outputs)\n        preds = outputs.detach().cpu()\n        predictions.append(preds.argmax(1)) # need optimize here\n        # print(predictions)","metadata":{"papermill":{"duration":1295.660485,"end_time":"2022-04-30T19:08:11.909737","exception":false,"start_time":"2022-04-30T18:46:36.249252","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-05-04T08:28:08.282459Z","iopub.execute_input":"2022-05-04T08:28:08.283414Z","iopub.status.idle":"2022-05-04T08:47:30.742349Z","shell.execute_reply.started":"2022-05-04T08:28:08.283375Z","shell.execute_reply":"2022-05-04T08:47:30.740954Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tmp = predictions[0]\nfor i in range(len(predictions) - 1):\n    tmp = torch.cat((tmp, predictions[i+1]))","metadata":{"papermill":{"duration":23.743387,"end_time":"2022-04-30T19:08:58.75323","exception":false,"start_time":"2022-04-30T19:08:35.009843","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-05-04T08:47:30.744836Z","iopub.execute_input":"2022-05-04T08:47:30.745195Z","iopub.status.idle":"2022-05-04T08:47:30.763203Z","shell.execute_reply.started":"2022-05-04T08:47:30.74515Z","shell.execute_reply":"2022-05-04T08:47:30.76242Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# predictions = label_encoder.inverse_transform(tmp)\npredictions = [unique_cultivars[pred] for pred in tmp]","metadata":{"papermill":{"duration":24.024431,"end_time":"2022-04-30T19:09:46.950135","exception":false,"start_time":"2022-04-30T19:09:22.925704","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-05-04T08:47:30.764539Z","iopub.execute_input":"2022-05-04T08:47:30.765864Z","iopub.status.idle":"2022-05-04T08:47:30.83182Z","shell.execute_reply.started":"2022-05-04T08:47:30.765825Z","shell.execute_reply":"2022-05-04T08:47:30.831087Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub = pd.read_csv('../input/sorghum-id-fgvc-9/sample_submission.csv')\nsub['cultivar'] = predictions\nsub.to_csv('submission.csv', index=False)\nsub.head()","metadata":{"papermill":{"duration":24.051225,"end_time":"2022-04-30T19:10:34.206169","exception":false,"start_time":"2022-04-30T19:10:10.154944","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-05-04T08:47:30.833131Z","iopub.execute_input":"2022-05-04T08:47:30.833934Z","iopub.status.idle":"2022-05-04T08:47:30.925653Z","shell.execute_reply.started":"2022-05-04T08:47:30.833894Z","shell.execute_reply":"2022-05-04T08:47:30.924744Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# !cd /kaggle/working","metadata":{"papermill":{"duration":23.228593,"end_time":"2022-04-30T19:11:21.288368","exception":false,"start_time":"2022-04-30T19:10:58.059775","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-05-04T08:47:30.92738Z","iopub.execute_input":"2022-05-04T08:47:30.92765Z","iopub.status.idle":"2022-05-04T08:47:30.931496Z","shell.execute_reply.started":"2022-05-04T08:47:30.927614Z","shell.execute_reply":"2022-05-04T08:47:30.930603Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# from IPython.display import FileLink\n# FileLink(r'./tf_efficientnetv2_m_in21k_50_last.pt')","metadata":{"papermill":{"duration":24.053985,"end_time":"2022-04-30T19:12:09.181314","exception":false,"start_time":"2022-04-30T19:11:45.127329","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-05-04T08:47:30.932854Z","iopub.execute_input":"2022-05-04T08:47:30.933103Z","iopub.status.idle":"2022-05-04T08:47:30.94013Z","shell.execute_reply.started":"2022-05-04T08:47:30.933064Z","shell.execute_reply":"2022-05-04T08:47:30.939311Z"},"trusted":true},"execution_count":null,"outputs":[]}]}