{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np \nimport pytorch_lightning as pl \nimport torch \nfrom torch.utils.data import DataLoader , Dataset ","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-03-08T07:34:09.416286Z","iopub.execute_input":"2022-03-08T07:34:09.417020Z","iopub.status.idle":"2022-03-08T07:34:18.492763Z","shell.execute_reply.started":"2022-03-08T07:34:09.416923Z","shell.execute_reply":"2022-03-08T07:34:18.491835Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# importing required stuff\n# !pip -q install --upgrade wandb\nimport pandas as pd\nimport albumentations\nimport torchvision \nfrom PIL import Image\nimport wandb\nimport numpy as np \nimport matplotlib.pyplot as plt \nimport seaborn as sns \nimport torch \nfrom kaggle_secrets import UserSecretsClient\nuser_secrets = UserSecretsClient()\nwandb.login(key = user_secrets.get_secret(\"wandb_key\"))\nimport cv2\nfrom sklearn.model_selection import train_test_split\nimport pytorch_lightning as pl \nimport torch \nimport numpy as np \nimport pandas as pd\nimport  os \nimport sys \nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.optim as optim\nfrom torch.utils.data import DataLoader, Dataset\nfrom torch.utils.data.sampler import SubsetRandomSampler\nsys.path.append(\"../input/timm-pytorch-image-models/pytorch-image-models-master\")\nimport timm\nimport math\nimport matplotlib.pyplot as plt\nimport torchvision\n\n# run = wandb.init(project = \"hyperwhaletpu\",name = \"pytorchlightning\")\n\ntrain = pd.read_csv(\"../input/happy-whale-and-dolphin/train.csv\")\nss = pd.read_csv(\"../input/happy-whale-and-dolphin/sample_submission.csv\")\ntrain['image'] = train[\"image\"].apply(lambda x: f\"../input/happy-whale-and-dolphin/train_images/{x}\")\nss['image'] = ss['image'].apply(lambda x : f\"../input/happy-whale-and-dolphin/test_images/{x}\")\n# train = train.sample(frac = 0.001)\nlandmark_id2idx = {landmark_id: idx for idx, landmark_id in enumerate(sorted(train['individual_id'].unique()))}\nidx2landmark_id = {idx: landmark_id for idx, landmark_id in enumerate(sorted(train['individual_id'].unique()))}\ntrain['individual_id'] = train['individual_id'].map(landmark_id2idx)\n# val['individual_id'] = val['individual_id'].map(landmark_id2idx)\n\ntrain,val = train_test_split(train)\n\n\n\n\n\ntransforms_224 = albumentations.Compose([\n    albumentations.Resize(224,224),\n    albumentations.Normalize()\n])\n\ntransforms_768 = albumentations.Compose([\n    albumentations.Resize(768, 768),\n    albumentations.Normalize()\n    \n    \n    \n    \n    \n    \n])\ntransforms_512 = albumentations.Compose([\n    albumentations.Resize(512, 512),\n    albumentations.Normalize()\n])\n\n\n# creating the pytroch model AlexModel \nclass MyDataset(torch.utils.data.Dataset):\n    def __init__(self, file_paths , labels , transforms):\n        # Not sure if I am gonnaa use species but let's see \n        self.file_paths = file_paths \n        self.labels = labels \n        self.transforms = transforms\n    \n    def __len__(self):\n        return len(self.labels)\n    \n    def __getitem__(self,idx):\n        lab = self.labels[idx]\n        image = cv2.imread(self.file_paths[idx])\n        image = image[:, :, ::-1]\n        res = self.transforms(image=image)\n        image = res['image'].astype(np.float32)\n        image = image.transpose(2, 0, 1)      \n        return {\"image\":image, \"label\":lab}\n    \nclass test_dataset(torch.utils.data.Dataset):\n    def __init__(self, file_paths , transforms):\n        # Not sure if I am gonnaa use species but let's see \n        self.file_paths = file_paths \n        self.transforms = transforms\n    \n    def __len__(self):\n        return len(self.file_paths)\n    \n    def __getitem__(self,idx):\n        image = cv2.imread(self.file_paths[idx])\n        image = image[:, :, ::-1]\n        res = self.transforms(image=image)\n        image = res['image'].astype(np.float32)\n        image = image.transpose(2, 0, 1)      \n        return {\"image\":image}\n\nclass ArcMarginProduct_subcenter(nn.Module):\n    def __init__(self, in_features, out_features, k=3):\n        super().__init__()\n        self.weight = nn.Parameter(torch.FloatTensor(out_features*k, in_features))\n        self.reset_parameters()\n        self.k = k\n        self.out_features = out_features\n        \n    def reset_parameters(self):\n        stdv = 1. / math.sqrt(self.weight.size(1))\n        self.weight.data.uniform_(-stdv, stdv)\n        \n    def forward(self, features):\n        cosine_all = F.linear(F.normalize(features), F.normalize(self.weight))\n        cosine_all = cosine_all.view(-1, self.out_features, self.k)\n        cosine, _ = torch.max(cosine_all, dim=2)\n        return cosine   \n\nclass DenseCrossEntropy(nn.Module):\n    def forward(self, x, target):\n        x = x.float()\n        target = target.float()\n        logprobs = torch.nn.functional.log_softmax(x, dim=-1)\n\n        loss = -logprobs * target\n        loss = loss.sum(-1)\n        return loss.mean()\nclass ArcFaceLossAdaptiveMargin(nn.modules.Module):\n    def __init__(self, margins, s=30.0):\n        super().__init__()\n        self.crit =DenseCrossEntropy()\n        self.s = s\n        self.margins = margins\n            \n    def forward(self, logits, labels, out_dim=15587):\n#         device = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n#         device = xm.xla_device()\n        ms = []\n        \n        ms = self.margins[labels.cpu().numpy()]\n        cos_m = torch.from_numpy(np.cos(ms)).float()\n        sin_m = torch.from_numpy(np.sin(ms)).float()\n        th = torch.from_numpy(np.cos(math.pi - ms)).float()\n        mm = torch.from_numpy(np.sin(math.pi - ms) * ms).float()\n        labels = F.one_hot(labels, out_dim).float()\n        logits = logits.float()\n        cosine = logits\n        sine = torch.sqrt(1.0 - torch.pow(cosine, 2))\n        phi = cosine * cos_m.view(-1,1) - sine * sin_m.view(-1,1)\n        phi = torch.where(cosine > th.view(-1,1), phi, cosine - mm.view(-1,1))\n        output = (labels * phi) + ((1.0 - labels) * cosine)\n        output *= self.s\n        loss = self.crit(output, labels)\n        return loss  \n\ntmp = np.sqrt(1 / np.sqrt(train.individual_id.value_counts().sort_index().values))\nmargins = (tmp - tmp.min()) / (tmp.max() - tmp.min()) * 0.45 + 0.05\n\ntransforms_224 = albumentations.Compose([\n    albumentations.Resize(224,224),\n    albumentations.Normalize()\n])\n\n# train_ds = MyDataset(file_paths = train.image, labels = train.individual_id, transforms = transforms_224)\n# train_dl = DataLoader(train_ds, batch_size=10, num_workers = 4 , pin_memory = True, drop_last = True)\n\n# val_ds = MyDataset(file_paths = val.image, labels = val.individual_id, transforms = transforms_224)\n# val_dl = DataLoader(val_ds,batch_size = 10, num_workers = 4 , pin_memory = True, drop_last = False)\n\nclass AlexModel(pl.LightningModule):\n    def __init__(self, batch_size):\n        super(AlexModel,self).__init__()\n        self.backbone = timm.create_model(\"efficientnet_b5\",pretrained = True) \n        self.linear = nn.Linear(1000, 512)\n        self.relu = nn.LeakyReLU()\n        self.out_layer = ArcMarginProduct_subcenter(512, 15587)\n        self.softmax = nn.Softmax(dim = 0)\n        self.save_hyperparameters()\n        self.batch_size = batch_size\n        \n    \n    def forward(self, input_image):\n        x = self.backbone(input_image)\n        x = self.relu(self.linear(x))\n        out_logits = self.softmax(self.out_layer(x) )\n        \n        return out_logits\n    \n    def training_step(self,batch,batch_idx):\n        data = batch[\"image\"]\n        target = batch[\"label\"]\n\n        logits_m = self(data)\n#         print(logits_m.shape, target.shape)\n        \n        loss = nn.CrossEntropyLoss()(logits_m,torch.tensor(target, dtype = torch.long))\n#         arc = ArcFaceLossAdaptiveMargin(margins=margins, s=80)\n#         loss = arc(logits_m, target, 15587)\n        return {\"loss\":loss}\n    \n    def validation_step(self,batch,batch_idx):\n        data = batch[\"image\"]\n        target = batch[\"label\"]\n        logits_m = self(data)\n#         print(logits_m.shape, target.shape)\n        loss = nn.CrossEntropyLoss()(logits_m, torch.tensor(target,dtype = torch.long))\n        self.log('val_loss', loss)\n        return {\"val_loss\":loss, \"pred\":logits_m}\n    \n    def validation_epoch_end(self, outputs):\n        avg_loss = torch.stack([x['val_loss'] for x in outputs]).mean()\n#         tensorboard_logs = {'val_loss': avg_loss}\n        return {'avg_val_loss': avg_loss}\n    \n    def configure_optimizers(self):\n        return torch.optim.Adam(self.parameters(), lr=0.02)\n\n    def train_dataloader(self):\n        dataset =  MyDataset(file_paths = train.image.to_numpy(), labels = train.individual_id.to_numpy(), transforms = transforms_224)\n        loader = DataLoader(dataset, batch_size=self.batch_size)\n        return loader\n\n    def val_dataloader(self):\n        dataset = MyDataset(file_paths = val.image.to_numpy(), labels = val.individual_id.to_numpy(), transforms = transforms_224)\n        loader = DataLoader(dataset, batch_size=self.batch_size)\n        return loader\n    \n    def predict_dataloader(self):\n        dataset = test_dataset(file_paths = ss.image.to_numpy(), transforms = transforms_224)\n        loader = DataLoader(dataset , batch_size = self.batch_size)\n        return loader \n\n    def predict_step(self, batch , batch_idx):\n        data = batch[\"image\"]\n        logits_m = self(data)\n        return logits_m\n","metadata":{"execution":{"iopub.status.busy":"2022-03-08T07:42:22.500953Z","iopub.execute_input":"2022-03-08T07:42:22.501207Z","iopub.status.idle":"2022-03-08T07:42:24.026517Z","shell.execute_reply.started":"2022-03-08T07:42:22.501179Z","shell.execute_reply":"2022-03-08T07:42:24.025716Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mod = AlexModel(10)","metadata":{"execution":{"iopub.status.busy":"2022-03-08T07:42:24.027934Z","iopub.execute_input":"2022-03-08T07:42:24.028168Z","iopub.status.idle":"2022-03-08T07:42:24.767969Z","shell.execute_reply.started":"2022-03-08T07:42:24.028135Z","shell.execute_reply":"2022-03-08T07:42:24.767224Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mod.load_state_dict(torch.load(\"../input/happywhale-pytorch-lightning-model-building/AlexModelHap.pt\"))","metadata":{"execution":{"iopub.status.busy":"2022-03-08T07:42:27.714777Z","iopub.execute_input":"2022-03-08T07:42:27.715035Z","iopub.status.idle":"2022-03-08T07:42:30.308759Z","shell.execute_reply.started":"2022-03-08T07:42:27.715006Z","shell.execute_reply":"2022-03-08T07:42:30.308027Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from pytorch_lightning.loggers import WandbLogger  # newline 1\nfrom pytorch_lightning import Trainer\nfrom pytorch_lightning.callbacks.early_stopping import EarlyStopping\nfrom pytorch_lightning.callbacks.model_checkpoint import ModelCheckpoint\nes = EarlyStopping(monitor=\"val_loss\")\nwandb_logger = WandbLogger() \ncheckpointing = ModelCheckpoint(monitor=\"val_loss\")\ntrainer = pl.Trainer(auto_scale_batch_size=True,logger=wandb_logger,callbacks=[es,checkpointing],progress_bar_refresh_rate=20,gpus = 0 , max_epochs = 7)\n# trainer.fit(mod)","metadata":{"execution":{"iopub.status.busy":"2022-03-08T07:42:30.310498Z","iopub.execute_input":"2022-03-08T07:42:30.310939Z","iopub.status.idle":"2022-03-08T07:42:30.377264Z","shell.execute_reply.started":"2022-03-08T07:42:30.310897Z","shell.execute_reply":"2022-03-08T07:42:30.375329Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# torch.save(mod.state_dict(),\"./AlexModelHap.pt\")","metadata":{"execution":{"iopub.status.busy":"2022-03-08T07:42:30.378699Z","iopub.execute_input":"2022-03-08T07:42:30.379125Z","iopub.status.idle":"2022-03-08T07:42:30.384353Z","shell.execute_reply.started":"2022-03-08T07:42:30.379081Z","shell.execute_reply":"2022-03-08T07:42:30.383456Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds = trainer.predict(mod)","metadata":{"execution":{"iopub.status.busy":"2022-03-08T07:42:30.534706Z","iopub.execute_input":"2022-03-08T07:42:30.535271Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"landmarks = []\nfor pred in preds:\n    pred_sorted_last_5 = pred.argsort(axis=1)[:,-4:]\n    for val in pred_sorted_last_5:\n        try:\n            ans = [idx2landmark_id[x.item()] for x in val]\n        except :\n            ans = val.numpy()\n#             ans = list(ans.numpy())\n        ans.append(\"new_individual\")\n        ans = ' '.join(ans)\n        landmarks.append(ans)\n\n# preds[0].argsort(axis = 1)[:,-5:]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(landmarks)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"landmarks","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ss = pd.read_csv('../input/happy-whale-and-dolphin/sample_submission.csv')\nss","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ss['predictions'] = landmarks","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ss","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ss.to_csv('submission.csv', index = False)","metadata":{"execution":{"iopub.status.busy":"2022-02-07T14:34:02.095083Z","iopub.execute_input":"2022-02-07T14:34:02.095364Z","iopub.status.idle":"2022-02-07T14:34:02.869827Z","shell.execute_reply.started":"2022-02-07T14:34:02.095335Z","shell.execute_reply":"2022-02-07T14:34:02.869119Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}