{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# About this notebook\n- PyTorch Resnet18 starter code\n- [train kernel](https://www.kaggle.com/yasufuminakama/herbarium-2020-pytorch-resnet18-train) -> inference kernel\n\nIf this notebook is helpful, feel free to upvote :)","metadata":{}},{"cell_type":"markdown","source":"# Library","metadata":{}},{"cell_type":"code","source":"import os\nimport numpy as np \nimport pandas as pd \nimport json","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","execution":{"iopub.status.busy":"2022-05-14T12:25:39.901327Z","iopub.execute_input":"2022-05-14T12:25:39.901718Z","iopub.status.idle":"2022-05-14T12:25:39.906712Z","shell.execute_reply.started":"2022-05-14T12:25:39.901661Z","shell.execute_reply":"2022-05-14T12:25:39.904768Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"os.listdir('../input/herbarium-2020-fgvc7')","metadata":{"execution":{"iopub.status.busy":"2022-05-14T12:25:39.972644Z","iopub.execute_input":"2022-05-14T12:25:39.973014Z","iopub.status.idle":"2022-05-14T12:25:39.981732Z","shell.execute_reply.started":"2022-05-14T12:25:39.972964Z","shell.execute_reply":"2022-05-14T12:25:39.980278Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data Loading","metadata":{}},{"cell_type":"code","source":"%%time\n\nwith open('../input/herbarium-2020-fgvc7/nybg2020/train/metadata.json', \"r\", encoding=\"ISO-8859-1\") as file:\n    train = json.load(file)\n\ntrain_img = pd.DataFrame(train['images'])\ntrain_ann = pd.DataFrame(train['annotations']).drop(columns='image_id')\ntrain_df = train_img.merge(train_ann, on='id')\ntrain_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-05-14T12:25:40.063626Z","iopub.execute_input":"2022-05-14T12:25:40.064011Z","iopub.status.idle":"2022-05-14T12:25:49.411396Z","shell.execute_reply.started":"2022-05-14T12:25:40.063960Z","shell.execute_reply":"2022-05-14T12:25:49.410639Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\nwith open('../input/herbarium-2020-fgvc7/nybg2020/test/metadata.json', \"r\", encoding=\"ISO-8859-1\") as file:\n    test = json.load(file)\n\ntest_df = pd.DataFrame(test['images'])\ntest_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-05-14T12:25:49.413190Z","iopub.execute_input":"2022-05-14T12:25:49.413502Z","iopub.status.idle":"2022-05-14T12:25:50.061743Z","shell.execute_reply.started":"2022-05-14T12:25:49.413456Z","shell.execute_reply":"2022-05-14T12:25:50.060788Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_submission = pd.read_csv('../input/herbarium-2020-fgvc7/sample_submission.csv')\nsample_submission.head()","metadata":{"execution":{"iopub.status.busy":"2022-05-14T12:25:50.063307Z","iopub.execute_input":"2022-05-14T12:25:50.063879Z","iopub.status.idle":"2022-05-14T12:25:50.099565Z","shell.execute_reply.started":"2022-05-14T12:25:50.063677Z","shell.execute_reply":"2022-05-14T12:25:50.098353Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# TARGET (\"category_id\")","metadata":{}},{"cell_type":"code","source":"train_df['category_id'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-05-14T12:25:50.101310Z","iopub.execute_input":"2022-05-14T12:25:50.101876Z","iopub.status.idle":"2022-05-14T12:25:50.125165Z","shell.execute_reply.started":"2022-05-14T12:25:50.101690Z","shell.execute_reply":"2022-05-14T12:25:50.124180Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"- imbalance\n- 32093 targets","metadata":{}},{"cell_type":"code","source":"from sklearn import preprocessing\n\nle = preprocessing.LabelEncoder()\nle.fit(train_df['category_id'])\ntrain_df['category_id_le'] = le.transform(train_df['category_id'])\nclass_map = dict(sorted(train_df[['category_id_le', 'category_id']].values.tolist()))","metadata":{"execution":{"iopub.status.busy":"2022-05-14T12:25:50.128700Z","iopub.execute_input":"2022-05-14T12:25:50.129074Z","iopub.status.idle":"2022-05-14T12:25:53.820994Z","shell.execute_reply.started":"2022-05-14T12:25:50.129022Z","shell.execute_reply":"2022-05-14T12:25:53.819982Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Library","metadata":{}},{"cell_type":"code","source":"# ====================================================\n# Library\n# ====================================================\n\nimport sys\n\nimport gc\nimport os\nimport random\nimport time\nfrom contextlib import contextmanager\nfrom pathlib import Path\n\nimport cv2\nfrom PIL import Image\nimport numpy as np\nimport pandas as pd\nimport scipy as sp\n\nimport sklearn.metrics\n\nfrom functools import partial\n\nimport torch\nimport torch.nn as nn\nfrom torch.optim import Adam, SGD\nfrom torch.optim.lr_scheduler import CosineAnnealingLR\nfrom torch.utils.data import DataLoader, Dataset\n\nfrom albumentations import Compose, Normalize, Resize\nfrom albumentations.pytorch import ToTensorV2\n\n #about torch...\nimport torch\nimport torch.nn as nn\nimport torch\nimport torch.optim as optim\nimport torch.nn.functional as F\nfrom PIL import Image\nfrom torchvision import datasets, models, transforms\nfrom torch.utils.data import DataLoader, Dataset\n\n#using numpy\nimport numpy as np\n\n#for data load or save\nimport pandas as pd\n\n#visualize some datasets\nimport matplotlib.pyplot as plt\n\n#check our work directory\nimport os\n\n#to unzip datasets\nimport zipfile\n\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\ndevice","metadata":{"execution":{"iopub.status.busy":"2022-05-14T12:25:53.822798Z","iopub.execute_input":"2022-05-14T12:25:53.823179Z","iopub.status.idle":"2022-05-14T12:25:53.838357Z","shell.execute_reply.started":"2022-05-14T12:25:53.823130Z","shell.execute_reply":"2022-05-14T12:25:53.837404Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Utils","metadata":{}},{"cell_type":"code","source":"# ====================================================\n# Utils\n# ====================================================\n\n@contextmanager\ndef timer(name):\n    t0 = time.time()\n    LOGGER.info(f'[{name}] start')\n    yield\n    LOGGER.info(f'[{name}] done in {time.time() - t0:.0f} s.')\n\n    \ndef init_logger(log_file='train.log'):\n    from logging import getLogger, DEBUG, FileHandler,  Formatter,  StreamHandler\n    \n    log_format = '%(asctime)s %(levelname)s %(message)s'\n    \n    stream_handler = StreamHandler()\n    stream_handler.setLevel(DEBUG)\n    stream_handler.setFormatter(Formatter(log_format))\n    \n    file_handler = FileHandler(log_file)\n    file_handler.setFormatter(Formatter(log_format))\n    \n    logger = getLogger('Herbarium')\n    logger.setLevel(DEBUG)\n    logger.addHandler(stream_handler)\n    logger.addHandler(file_handler)\n    \n    return logger\n\nLOG_FILE = 'train.log'\nLOGGER = init_logger(LOG_FILE)\n\n\ndef seed_torch(seed=777):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n\nSEED = 777\nseed_torch(SEED)","metadata":{"execution":{"iopub.status.busy":"2022-05-14T12:25:53.839985Z","iopub.execute_input":"2022-05-14T12:25:53.840637Z","iopub.status.idle":"2022-05-14T12:25:53.855879Z","shell.execute_reply.started":"2022-05-14T12:25:53.840568Z","shell.execute_reply":"2022-05-14T12:25:53.855043Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Dataset","metadata":{}},{"cell_type":"code","source":"N_CLASSES = 32093\n\nclass TrainDataset(Dataset):\n    def __init__(self, df, transform=None):\n        self.df = df\n        self.transform = transform\n        \n    def __len__(self):\n        return len(self.df)\n\n    def __getitem__(self, idx):\n        file_name = self.df['file_name'].values[idx]\n        file_path = f'../input/herbarium-2020-fgvc7/nybg2020/train/{file_name}'\n#         image = cv2.imread(file_path)\n#         image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n#         image = Image.fromarray(np.uint8(image))\n        image = Image.open(file_path)\n        \n        if self.transform is not None:\n            image = self.transform(image)\n            \n        label = self.df['category_id'].values[idx]\n            \n        \n        return image, label\n    \nclass TestDataset(Dataset):\n    def __init__(self, df, transform=None):\n        self.df = df\n        self.transform = transform\n        \n    def __len__(self):\n        return len(self.df)\n\n    def __getitem__(self, idx):\n        file_name = self.df['file_name'].values[idx]\n        file_path = f'../input/herbarium-2020-fgvc7/nybg2020/test/{file_name}'\n#         image = cv2.imread(file_path)\n#         image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n#         image = Image.fromarray(np.uint8(image))\n        image = Image.open(file_path)\n        \n        if self.transform is not None:\n            image = self.transform(image)\n            \n            \n        \n        return image","metadata":{"execution":{"iopub.status.busy":"2022-05-14T12:25:53.858919Z","iopub.execute_input":"2022-05-14T12:25:53.861648Z","iopub.status.idle":"2022-05-14T12:25:53.875831Z","shell.execute_reply.started":"2022-05-14T12:25:53.860479Z","shell.execute_reply":"2022-05-14T12:25:53.874939Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Transforms","metadata":{}},{"cell_type":"code","source":"HEIGHT = 128\nWIDTH = 128\n\ntrain_transforms =  transforms.Compose([\n        transforms.Resize([128,128]),\n        transforms.RandomRotation(10, center=(0, 0),expand=True),\n        transforms.RandomHorizontalFlip(),\n        transforms.Resize([128,128]),\n        transforms.ToTensor()\n    ])\n\ntest_transforms =  transforms.Compose([\n        transforms.Resize([128,128]),\n        transforms.ToTensor()\n])","metadata":{"execution":{"iopub.status.busy":"2022-05-14T12:25:53.877622Z","iopub.execute_input":"2022-05-14T12:25:53.878645Z","iopub.status.idle":"2022-05-14T12:25:53.889146Z","shell.execute_reply.started":"2022-05-14T12:25:53.878560Z","shell.execute_reply":"2022-05-14T12:25:53.888144Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\nindexs = [i for i in range(train_df.shape[0])]\ntrain_index, test_index = train_test_split(indexs, test_size=0.2)\ntrain_data = train_df.iloc[train_index]\ntest_data = train_df.iloc[test_index]","metadata":{"execution":{"iopub.status.busy":"2022-05-14T12:25:53.891041Z","iopub.execute_input":"2022-05-14T12:25:53.891771Z","iopub.status.idle":"2022-05-14T12:25:55.027054Z","shell.execute_reply.started":"2022-05-14T12:25:53.891713Z","shell.execute_reply":"2022-05-14T12:25:55.026187Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"batch_size = 512\n\n\ntrain_dataset = TrainDataset(train_data, transform=train_transforms)\ntrain_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True)\n\ntest_dataset = TrainDataset(test_data, transform=train_transforms)\ntest_loader = DataLoader(test_dataset, batch_size=batch_size, shuffle=True)","metadata":{"execution":{"iopub.status.busy":"2022-05-14T12:25:55.029926Z","iopub.execute_input":"2022-05-14T12:25:55.030388Z","iopub.status.idle":"2022-05-14T12:25:55.041962Z","shell.execute_reply.started":"2022-05-14T12:25:55.030323Z","shell.execute_reply":"2022-05-14T12:25:55.040993Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"val_dataset = TestDataset(test_df, transform=test_transforms)\nval_loader = DataLoader(val_dataset, batch_size=batch_size, shuffle=False)","metadata":{"execution":{"iopub.status.busy":"2022-05-14T12:25:55.043546Z","iopub.execute_input":"2022-05-14T12:25:55.044223Z","iopub.status.idle":"2022-05-14T12:25:55.051333Z","shell.execute_reply.started":"2022-05-14T12:25:55.044167Z","shell.execute_reply":"2022-05-14T12:25:55.050323Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"np.array(val_dataset[1]).shape","metadata":{"execution":{"iopub.status.busy":"2022-05-14T12:25:55.053084Z","iopub.execute_input":"2022-05-14T12:25:55.053845Z","iopub.status.idle":"2022-05-14T12:25:55.077043Z","shell.execute_reply.started":"2022-05-14T12:25:55.053788Z","shell.execute_reply":"2022-05-14T12:25:55.076331Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Modeling","metadata":{}},{"cell_type":"code","source":"# import torchvision.models as models\n\n# model = models.resnet18(pretrained=True)\n# model.avgpool = nn.AdaptiveAvgPool2d(1)\n# model.fc = nn.Linear(model.fc.in_features, N_CLASSES)\nclass MNIST_CNN(nn.Module):\n    def __init__(self):\n        super(MNIST_CNN,self).__init__()\n        self.layer1 = nn.Sequential(\n            nn.Conv2d(3,16,kernel_size=3, padding=0,stride=2),\n            nn.BatchNorm2d(16),\n            nn.ReLU(),\n            nn.MaxPool2d(2)\n        )\n        self.layer2 = nn.Sequential(\n            nn.Conv2d(16,32,kernel_size=3, padding=0,stride=2),\n            nn.BatchNorm2d(32),\n            nn.ReLU(),\n            nn.MaxPool2d(2)\n        )\n        self.layer3 = nn.Sequential(\n            nn.Conv2d(32,16,kernel_size=3, padding=0,stride=2),\n            nn.BatchNorm2d(16),\n            nn.ReLU(),\n#             nn.MaxPool2d(2)\n        )\n\n        self.L1 = nn.Sequential(         \n            nn.Linear(in_features=16*3*3,out_features=512),                              \n            nn.ReLU()\n        )\n#         self.L1_1 = nn.Sequential(         \n#             nn.Linear(in_features=512,out_features=512),                              \n#             nn.ReLU()\n#         )\n#         self.L1_2 = nn.Sequential(         \n#             nn.Linear(in_features=512,out_features=512),                              \n#             nn.ReLU()\n#         )\n        self.L2 = nn.Sequential(         \n            nn.Linear(in_features=512,out_features=256),     \n            nn.ReLU()                   \n        )\n        self.L3 = nn.Sequential(         \n            nn.Linear(in_features=256,out_features=128),    \n            nn.ReLU()                  \n        )\n        self.L4 = nn.Sequential(         \n            nn.Linear(in_features=128,out_features=64),    \n            nn.ReLU()                 \n        )\n        # fully connected layer, output 10 classes\n        self.L5 = nn.Sequential(         \n            nn.Linear(in_features=64,out_features=N_CLASSES),    \n            nn.ReLU()                     \n        )\n        self.lsfm = nn.Sigmoid()\n       \n    \n    def forward(self, x):\n#--------------------------------------------------------------------------\n        x = self.layer1(x)\n        x = self.layer2(x)\n        x = self.layer3(x)\n#         print(x.shape)             # torch.Size([36, 16, 3, 3])\n#--------------------------------------------------------------------------\n        x = x.view(x.size(0),-1)\n#         print(x.shape)             # torch.Size([36, 144])\n#--------------------------------------------------------------------------\n        x = self.L1(x)\n#         x = self.L1_1(x)\n#         x = self.L1_2(x)\n        x = self.L2(x)\n        x = self.L3(x)\n        x = self.L4(x)\n        x = self.L5(x) \n        output = self.lsfm(x)     \n        return output","metadata":{"execution":{"iopub.status.busy":"2022-05-14T12:25:55.079542Z","iopub.execute_input":"2022-05-14T12:25:55.080154Z","iopub.status.idle":"2022-05-14T12:25:55.156609Z","shell.execute_reply.started":"2022-05-14T12:25:55.080099Z","shell.execute_reply":"2022-05-14T12:25:55.155330Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training Data","metadata":{}},{"cell_type":"code","source":"device = 'cuda' if torch.cuda.is_available() else 'cpu'\nprint(device)","metadata":{"execution":{"iopub.status.busy":"2022-05-14T12:25:55.158371Z","iopub.execute_input":"2022-05-14T12:25:55.159045Z","iopub.status.idle":"2022-05-14T12:25:55.170648Z","shell.execute_reply.started":"2022-05-14T12:25:55.158987Z","shell.execute_reply":"2022-05-14T12:25:55.169547Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = MNIST_CNN().to(device)\nmodel.train()\noptimizer = optim.AdamW(model.parameters(), lr=0.0001)\ncriterion = nn.CrossEntropyLoss()","metadata":{"execution":{"iopub.status.busy":"2022-05-14T12:25:55.172361Z","iopub.execute_input":"2022-05-14T12:25:55.173081Z","iopub.status.idle":"2022-05-14T12:25:55.202100Z","shell.execute_reply.started":"2022-05-14T12:25:55.173024Z","shell.execute_reply":"2022-05-14T12:25:55.201238Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import time\nfrom tqdm import tqdm\nepochs=100\nhistory = []\nfor epoch in range(epochs):\n# ---------------------------------------training-------------------------\n    epoch_loss = 0\n    epoch_accuracy = 0\n    model.train()\n    for times, (data, label) in enumerate(train_loader):\n        data = data.to(device)\n        label = label.to(device)\n        \n        output = model(data)\n        loss = criterion(output, label)\n        \n        optimizer.zero_grad()\n        loss.backward()\n        optimizer.step()\n        \n        acc = ((output.argmax(dim=1) == label).float().mean())\n        \n        epoch_accuracy += acc/len(train_loader)\n        epoch_loss += loss/len(train_loader)\n        if times% 50 == 0:\n            print('Epoch:{} Complete!\\nEpoch Loss:{} \\nEpoch Accuracy:{}'.format(epoch+1,epoch_loss,epoch_accuracy))\n    # ----------------------evaluate--------------------------------\n#     model.eval()\n#     test_loss = 0\n#     test_acc = 0\n#     for data, label in (test_loader):\n#         data = data.to(device)\n#         label = label.to(device)\n        \n#         output = model(data)\n#         loss = criterion(output, label)\n#         acc = ((output.argmax(dim=1) == label).float().mean())\n        \n#         optimizer.zero_grad()\n#         loss.backward()\n#         optimizer.step()\n        \n#         test_acc += acc/len(test_loader)\n#         test_loss += loss/len(test_loader)\n        \n#     print('Val loss:{}, Val acc:{}'.format(test_loss, test_acc))\n#     print('-'*50)\n#     print('-'*50)\n    \n#     history.append([epoch+1,float(epoch_loss.cpu().detach().numpy()),float(epoch_accuracy.cpu().detach().numpy()),float(test_loss.cpu().detach().numpy()), float(test_acc.cpu().detach().numpy())])","metadata":{"execution":{"iopub.status.busy":"2022-05-14T12:43:55.667883Z","iopub.execute_input":"2022-05-14T12:43:55.668267Z","iopub.status.idle":"2022-05-14T12:44:07.553198Z","shell.execute_reply.started":"2022-05-14T12:43:55.668214Z","shell.execute_reply":"2022-05-14T12:44:07.550919Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"i = 0\nfor times, (data) in enumerate(val_loader):\n    print(times)\n    i+=1\n    if i == 5:\n        break","metadata":{"execution":{"iopub.status.busy":"2022-05-14T12:43:27.506863Z","iopub.execute_input":"2022-05-14T12:43:27.507228Z","iopub.status.idle":"2022-05-14T12:43:39.426998Z","shell.execute_reply.started":"2022-05-14T12:43:27.507178Z","shell.execute_reply":"2022-05-14T12:43:39.425928Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Submission","metadata":{}},{"cell_type":"code","source":"# test_df['preds'] = preds.astype(int)\n# submission = sample_submission.merge(test_df.rename(columns={'id': 'Id'})[['Id', 'preds']], on='Id').drop(columns='Predicted')\n# submission['Predicted'] = submission['preds'].map(class_map)\n# submission = submission.drop(columns='preds')\n# submission.to_csv('submission.csv', index=False)\n# submission.head()","metadata":{"execution":{"iopub.status.busy":"2022-05-14T12:36:59.483166Z","iopub.status.idle":"2022-05-14T12:36:59.484102Z"},"trusted":true},"execution_count":null,"outputs":[]}]}