{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"### Details\n\nThis notebook is to help people create a simple baseline and further improvise on it. \nThe notebook can be modified for any backbone accordingly.\n\n\n\nModel: **EfficientNet B3** \\\nTotal Epochs Trained :**40** \\\n[Checkpoint saved](https://www.kaggle.com/datasets/surajsharan/model-effnetb3) : **40th epoch** \n\n#### CV \nTest accuracy (public): **45.1%**","metadata":{}},{"cell_type":"code","source":"\n!pip install efficientnet-pytorch\n","metadata":{"execution":{"iopub.status.busy":"2022-03-28T19:44:51.468976Z","iopub.execute_input":"2022-03-28T19:44:51.469261Z","iopub.status.idle":"2022-03-28T19:45:05.486815Z","shell.execute_reply.started":"2022-03-28T19:44:51.469232Z","shell.execute_reply":"2022-03-28T19:45:05.485618Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# importing required modules\nimport os\nimport numpy as np\nimport pandas as pd\nimport albumentations as A\nimport cv2\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.optim as optim\nfrom efficientnet_pytorch import EfficientNet\nfrom tqdm.notebook import tqdm\nfrom torch.utils.data import Dataset, DataLoader\nfrom albumentations.pytorch import ToTensorV2\nfrom albumentations import Rotate \nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport torchvision\nfrom torchvision import transforms\nfrom tqdm import tqdm\n    \n\nimport warnings  \nwarnings.filterwarnings('ignore')\n\nfrom sklearn.model_selection import train_test_split\nimport gc","metadata":{"execution":{"iopub.status.busy":"2022-03-28T19:45:05.491016Z","iopub.execute_input":"2022-03-28T19:45:05.491251Z","iopub.status.idle":"2022-03-28T19:45:10.041456Z","shell.execute_reply.started":"2022-03-28T19:45:05.491223Z","shell.execute_reply":"2022-03-28T19:45:10.040415Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def seed_everything(seed):\n    \"\"\"\n    Seeds basic parameters for reproductibility of results\n    \n    Arguments:\n        seed {int} -- Number of the seed\n    \"\"\"\n    # random.seed(seed)\n    # os.environ[\"PYTHONHASHSEED\"] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = False\n","metadata":{"execution":{"iopub.status.busy":"2022-03-28T19:45:10.043157Z","iopub.execute_input":"2022-03-28T19:45:10.043455Z","iopub.status.idle":"2022-03-28T19:45:10.050502Z","shell.execute_reply.started":"2022-03-28T19:45:10.043401Z","shell.execute_reply":"2022-03-28T19:45:10.049568Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Defining hyperameters \n\nclass Config:\n    TRAIN_ROOT_DIR = '../input/ultra-mnist/train'\n    TEST_ROOT_DIR = '../input/ultra-mnist/test'\n    TRAIN_PATH = '../input/ultra-mnist/train.csv'\n    MODEL_NAME = 'efficientnet-b3'\n    MODEL = EfficientNet.from_pretrained(MODEL_NAME)\n    SEED = 42\n    EPOCH = 1\n    BATCH_SIZE = 32\n    IMG_SIZE = 300\n    NUM_WORKERS = 2\n    FP16 = True \n    FP16_OPT_LEVEL = \"O1\"\n    DEVICE = 'cuda' if torch.cuda.is_available() else 'cpu'\n    NUM_CLASSES = 28\n    \n    # Data transformations\n    TRANSFORMATION = transforms.Compose([transforms.ToPILImage(),\n                                           transforms.Resize((IMG_SIZE,IMG_SIZE)),\n                                           transforms.ToTensor(),\n                                           transforms.Normalize([0.5,0.5,0.5],\n                                                                [0.5,0.5,0.5])])\n    \n\n    \nseed_everything(Config.SEED)","metadata":{"execution":{"iopub.status.busy":"2022-03-28T19:45:10.053696Z","iopub.execute_input":"2022-03-28T19:45:10.054547Z","iopub.status.idle":"2022-03-28T19:45:14.231812Z","shell.execute_reply.started":"2022-03-28T19:45:10.054474Z","shell.execute_reply":"2022-03-28T19:45:14.230792Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#  Loading the train data\ntrain_df = pd.read_csv(Config.TRAIN_PATH)\ntrain_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-03-28T15:33:57.379629Z","iopub.execute_input":"2022-03-28T15:33:57.380032Z","iopub.status.idle":"2022-03-28T15:33:57.451757Z","shell.execute_reply.started":"2022-03-28T15:33:57.379994Z","shell.execute_reply":"2022-03-28T15:33:57.451129Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## Train Test Split\nX_train, X_test, y_train, y_test = train_test_split(\n    train_df['id'], train_df['digit_sum'], test_size=0.1, random_state=42)","metadata":{"execution":{"iopub.status.busy":"2022-03-28T15:33:57.455537Z","iopub.execute_input":"2022-03-28T15:33:57.457433Z","iopub.status.idle":"2022-03-28T15:33:57.481729Z","shell.execute_reply.started":"2022-03-28T15:33:57.457382Z","shell.execute_reply":"2022-03-28T15:33:57.480744Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f'Training :{X_train.shape[0]} , Test :{X_test.shape[0]}')\n","metadata":{"execution":{"iopub.status.busy":"2022-03-28T15:33:57.485224Z","iopub.execute_input":"2022-03-28T15:33:57.485523Z","iopub.status.idle":"2022-03-28T15:33:57.496907Z","shell.execute_reply.started":"2022-03-28T15:33:57.485476Z","shell.execute_reply":"2022-03-28T15:33:57.496091Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Defining Dataloader","metadata":{}},{"cell_type":"code","source":"# Training dataset\nclass UMNISTDataset(Dataset):\n    def __init__(self, root_dir, img_id,label=None, transform=None):\n        self.root_dir = root_dir\n        self.img_id = img_id\n        self.label=label\n        self.transform = transform\n    \n    def __len__(self):\n        return len(self.img_id)\n    \n    def __getitem__(self, item):\n        image = cv2.imread(f\"{self.root_dir}/{self.img_id.iloc[item]}.jpeg\", cv2.IMREAD_COLOR)\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n\n        if self.transform is not None:\n            image = self.transform(image)\n        \n        if self.label is None:\n            \n            return {\n                \"image\": torch.tensor(image)\n                    }\n            \n        else:\n            labels = self.label.iloc[item]\n            return {\n                \"image\": torch.tensor(image) , \n                \"label\" : torch.tensor(labels )\n                    }\n","metadata":{"execution":{"iopub.status.busy":"2022-03-28T19:45:14.233415Z","iopub.execute_input":"2022-03-28T19:45:14.234539Z","iopub.status.idle":"2022-03-28T19:45:14.245627Z","shell.execute_reply.started":"2022-03-28T19:45:14.234494Z","shell.execute_reply":"2022-03-28T19:45:14.244178Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_dataset = UMNISTDataset(Config.TRAIN_ROOT_DIR,X_train, y_train,Config.TRANSFORMATION)\ntrain_loader = DataLoader(train_dataset, batch_size=Config.BATCH_SIZE, shuffle=True, num_workers = Config.NUM_WORKERS)\n\nvalid_dataset = UMNISTDataset(Config.TRAIN_ROOT_DIR, X_test, y_test,Config.TRANSFORMATION)\nvalid_loader = DataLoader(valid_dataset,batch_size=Config.BATCH_SIZE, shuffle=True, num_workers = Config.NUM_WORKERS)","metadata":{"execution":{"iopub.status.busy":"2022-03-28T15:33:57.510764Z","iopub.execute_input":"2022-03-28T15:33:57.511134Z","iopub.status.idle":"2022-03-28T15:33:57.521469Z","shell.execute_reply.started":"2022-03-28T15:33:57.511097Z","shell.execute_reply":"2022-03-28T15:33:57.520722Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Model ","metadata":{}},{"cell_type":"code","source":"\n# Loading the pretrained model here and modifying it\n\nmodel = Config.MODEL\nin_features = model._fc.in_features\nmodel._fc = nn.Sequential(\n                            nn.Linear(in_features, 100),\n                            nn.ReLU(),\n                            nn.Linear(100, 28)\n                         )\n\n\n\nfor param in model.parameters():\n    param.requires_grad = False\n    \nfor params in model._fc.parameters():\n    params.requires_grad=True\n    \n\n    \n\nmodel.to(Config.DEVICE)\ncriterion = nn.CrossEntropyLoss()\noptimizer = torch.optim.Adam(model.parameters(), lr=0.001)\nscheduler = torch.optim.lr_scheduler.ExponentialLR(optimizer,gamma=0.94)\n","metadata":{"execution":{"iopub.status.busy":"2022-03-28T15:34:01.983862Z","iopub.execute_input":"2022-03-28T15:34:01.984299Z","iopub.status.idle":"2022-03-28T15:34:04.879628Z","shell.execute_reply.started":"2022-03-28T15:34:01.984264Z","shell.execute_reply":"2022-03-28T15:34:04.878898Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Training","metadata":{}},{"cell_type":"code","source":"# Train model\nbest_s = 0\nfor epoch in range(Config.EPOCH):\n    if epoch == 0:\n        for param in model.parameters():\n            param.requires_grad = True\n    print(f'Epoch: {epoch+1}/{Config.EPOCH}')\n\n    correct = 0\n    total = 0\n    losses = []\n    \n    for batch_idx, data in enumerate(tqdm(train_loader)):\n        images, targets = data['image'].to(Config.DEVICE) , data['label'].to(Config.DEVICE)\n        output = model(images)\n        loss = criterion(output, targets)\n        optimizer.zero_grad()\n        loss.backward()\n        optimizer.step()\n        \n        \n        _, pred = torch.max(output, 1)\n        correct += (pred == targets).sum().item()\n        total += pred.size(0)\n        losses.append(loss.item())        \n        loss.detach()\n        del images, targets, output, loss\n        gc.collect()\n        \n    train_loss = np.mean(losses)\n    train_acc = correct * 1.0 / total\n    del losses\n    total=0\n    correct=0\n    valid_acc=0\n    with torch.no_grad():\n        for batch_idx, data in enumerate(tqdm(valid_loader)):\n            images, targets = data['image'].to(Config.DEVICE) , data['label'].to(Config.DEVICE)\n            output = model(images)\n            _, pred = torch.max(output, 1)\n            correct += (pred == targets).sum().item()\n            \n            total += pred.size(0)\n\n        valid_acc = correct * 1.0 / total\n        # Saving State Dict\n        valid_acc+=epoch\n    if valid_acc > best_s:\n        checkpoint_name = 'checkpoint_' + str(epoch) + '.pth.tar'\n        torch.save({\n            'model_state_dict': model.state_dict(),\n            'accuracy': correct\n        }, checkpoint_name)\n        best_s = valid_acc\n        \n    print(f'Train Loss: {train_loss}\\tTrain Acc: {train_acc*100}\\tLR: {scheduler.get_lr()}\\tValid Accuracy: {correct/total * 100}')\n    scheduler.step()","metadata":{"execution":{"iopub.status.busy":"2022-03-28T11:49:00.734811Z","iopub.execute_input":"2022-03-28T11:49:00.735065Z","iopub.status.idle":"2022-03-28T11:52:26.841592Z","shell.execute_reply.started":"2022-03-28T11:49:00.735036Z","shell.execute_reply":"2022-03-28T11:52:26.840494Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"##    Inference ","metadata":{}},{"cell_type":"code","source":"## loading a pre-trained model till 40 epocs\npretrained_model = '../input/model-effnetb3/model_checkpoint.tar'","metadata":{"execution":{"iopub.status.busy":"2022-03-28T19:53:12.758262Z","iopub.execute_input":"2022-03-28T19:53:12.758614Z","iopub.status.idle":"2022-03-28T19:53:12.763999Z","shell.execute_reply.started":"2022-03-28T19:53:12.758581Z","shell.execute_reply":"2022-03-28T19:53:12.762925Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_csv_path = '../input/ultra-mnist/sample_submission.csv'\nsample_df = pd.read_csv(sample_csv_path)\n\nX_final = sample_df['id']\ny_final = sample_df[\"digit_sum\"]","metadata":{"execution":{"iopub.status.busy":"2022-03-28T19:53:13.421180Z","iopub.execute_input":"2022-03-28T19:53:13.421915Z","iopub.status.idle":"2022-03-28T19:53:13.452482Z","shell.execute_reply.started":"2022-03-28T19:53:13.421873Z","shell.execute_reply":"2022-03-28T19:53:13.451477Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_dataset = UMNISTDataset(Config.TEST_ROOT_DIR,X_final,transform = Config.TRANSFORMATION)\ntest_loader = DataLoader(test_dataset, batch_size=64, shuffle=False, num_workers = Config.NUM_WORKERS)","metadata":{"execution":{"iopub.status.busy":"2022-03-28T19:53:14.249542Z","iopub.execute_input":"2022-03-28T19:53:14.249844Z","iopub.status.idle":"2022-03-28T19:53:14.257493Z","shell.execute_reply.started":"2022-03-28T19:53:14.249812Z","shell.execute_reply":"2022-03-28T19:53:14.255821Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Loading the pretrained model here\nmodel = Config.MODEL\nmodel._fc = nn.Sequential(\n    nn.Linear(1536, 100),\n    nn.ReLU(),\n    nn.Linear(100, 28)\n)\n\n\n# loading the trained weights\nmodel.load_state_dict(torch.load(pretrained_model)['model_state_dict'])\nmodel.to(Config.DEVICE)\nprint(f'pretrained model:{pretrained_model} successfully loaded')\n","metadata":{"execution":{"iopub.status.busy":"2022-03-28T19:53:15.081501Z","iopub.execute_input":"2022-03-28T19:53:15.081825Z","iopub.status.idle":"2022-03-28T19:53:15.290223Z","shell.execute_reply.started":"2022-03-28T19:53:15.081777Z","shell.execute_reply":"2022-03-28T19:53:15.289131Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Test model\ndigit_sum = []\nmodel.eval()\nwith torch.no_grad():\n    for batch_idx, data in enumerate(tqdm(test_loader)):\n        images = data['image'].to(Config.DEVICE) \n        \n        output = model(images)\n        _, pred = torch.max(output, 1)\n        digit_sum.extend(pred.detach().cpu().numpy())\n        \n","metadata":{"execution":{"iopub.status.busy":"2022-03-28T19:58:29.023638Z","iopub.execute_input":"2022-03-28T19:58:29.023988Z","iopub.status.idle":"2022-03-28T19:58:29.028379Z","shell.execute_reply.started":"2022-03-28T19:58:29.023955Z","shell.execute_reply":"2022-03-28T19:58:29.027314Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Write Submission File ","metadata":{}},{"cell_type":"code","source":"sample_df['digit_sum'] = digit_sum\nsample_df['digit_sum'] = sample_df[\"digit_sum\"].astype(int)\nsample_df.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2022-03-28T19:57:00.668358Z","iopub.execute_input":"2022-03-28T19:57:00.668673Z","iopub.status.idle":"2022-03-28T19:57:00.675728Z","shell.execute_reply.started":"2022-03-28T19:57:00.668640Z","shell.execute_reply":"2022-03-28T19:57:00.672240Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"This is a lot of scope to improvise on this notebook .\n\n\n","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}