{"cells":[{"metadata":{},"cell_type":"markdown","source":"thanks to @abhishek brother and @xhlulu as i borrowed a lot of code from them,without kaggler like them i am nothing in kaggle. have been learning a lot from them since last 1+ year. "},{"metadata":{},"cell_type":"markdown","source":"# ChangeLog\n* v1. Training for 7 epochs because i have ~6 hours gpu quota left now (model resnet101 of torchvision)\n* v2. MSELoss - 10 epoch (forcefully stopped training because of version 3 attempt)\n* v3. Adding sigmoid during inference\n* v4. inferencing by solving inference issue of v3\n\n\n\n* Fork - changed to use pretrained model + more image augmentation"},{"metadata":{},"cell_type":"markdown","source":"# Import libraries and utility scripts"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"import os\n\n\nimport torch\n\nimport warnings\n\nimport pandas as pd\nimport numpy as np\nimport torch.nn as nn\n\nfrom sklearn.model_selection import train_test_split\n\nfrom sklearn import metrics\nfrom transformers import AdamW\nfrom transformers import get_linear_schedule_with_warmup\n\nimport time\nimport torchvision\nimport torch.nn as nn\nfrom tqdm import tqdm_notebook as tqdm\n\nfrom PIL import Image, ImageFile\nfrom torch.utils.data import Dataset\nimport torch\nimport torch.optim as optim\nfrom torchvision import transforms\nfrom torch.optim import lr_scheduler\nimport os\nwarnings.filterwarnings(\"ignore\")","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"BASE_PATH = \"/kaggle/input/alaska2-image-steganalysis\"\ntrain_imageids = pd.Series(os.listdir(BASE_PATH + '/Cover')).sort_values(ascending=True).reset_index(drop=True)\ntest_imageids = pd.Series(os.listdir(BASE_PATH + '/Test')).sort_values(ascending=True).reset_index(drop=True)\nsub = pd.read_csv('/kaggle/input/alaska2-image-steganalysis/sample_submission.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#https://www.kaggle.com/xhlulu/alaska2-efficientnet-on-tpus\ndef append_path(pre):\n    return np.vectorize(lambda file: os.path.join(BASE_PATH, pre, file))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_filenames = np.array(os.listdir(\"/kaggle/input/alaska2-image-steganalysis/Cover/\"))\nlen(train_filenames)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#https://www.kaggle.com/xhlulu/alaska2-efficientnet-on-tpus\nnp.random.seed(0)\npositives = train_filenames.copy()\nnegatives = train_filenames.copy()\nnp.random.shuffle(positives)\nnp.random.shuffle(negatives)\n\njmipod = append_path('JMiPOD')(positives[:10000])\njuniward = append_path('JUNIWARD')(positives[10000:20000])\nuerd = append_path('UERD')(positives[20000:30000])\n\npos_paths = np.concatenate([jmipod, juniward, uerd])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#https://www.kaggle.com/xhlulu/alaska2-efficientnet-on-tpus\ntest_paths = append_path('Test')(sub.Id.values)\nneg_paths = append_path('Cover')(negatives[:30000])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_paths = np.concatenate([pos_paths, neg_paths])\ntrain_labels = np.array([1] * len(pos_paths) + [0] * len(neg_paths))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#https://www.kaggle.com/xhlulu/alaska2-efficientnet-on-tpus\ntrain_paths, valid_paths, train_labels, valid_labels = train_test_split(\n    train_paths, train_labels, test_size=0.15, random_state=2020)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"len(valid_labels)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"l=np.array([train_paths,train_labels])\ntraindataset = pd.DataFrame({ 'images': list(train_paths), 'label': train_labels},columns=['images','label'])\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"val_l=np.array([valid_paths,valid_labels])\nvaliddataset=dataset = pd.DataFrame({ 'images': list(valid_paths), 'label': valid_labels},columns=['images','label'])\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**For now just using the train set only training**"},{"metadata":{"trusted":true},"cell_type":"code","source":"#traindataset = pd.concat([traindataset,validdataset])\nlen(traindataset)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"traindataset.head(2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"len(traindataset)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#i use this line of code for debugging\n#traindataset = traindataset.head(100)\nlen(traindataset)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"image = Image.open(train_paths[50] )\nimage","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# add image augmen tation\nclass train_images(Dataset):\n\n    def __init__(self, csv_file):\n\n        self.data = csv_file\n\n    def __len__(self):\n        return len(self.data)\n\n    def __getitem__(self, idx):\n        #print(idx)\n        img_name =  self.data.loc[idx][0]\n        image = Image.open(img_name)\n        image = image.resize((512, 512), resample=Image.BILINEAR)\n        label = self.data.loc[idx][1] #torch.tensor(self.data.loc[idx, 'label'])\n\n# ## https://pytorch.org/docs/stable/torchvision/transforms.html\n# transforms.Compose([\n# transforms.CenterCrop(10),\n# transforms.ToTensor(),\n# ])\n        \n#         return {'image': transforms.ToTensor()(image), # ORIG\n        return {'image': transforms.Compose([transforms.RandomVerticalFlip(),\n                                             transforms.RandomHorizontalFlip(),\n                                             transforms.ColorJitter(),\n                                             transforms.ToTensor()])(image),\n            'label': label\n            }","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_dataset = train_images(traindataset)\nvalid_dataset = train_images(validdataset)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = torchvision.models.resnet101(pretrained=True)\n#model.load_state_dict(torch.load(\"../input/pytorch-pretrained-models/resnet101-5d3b4d8f.pth\"))\n\nnum_features = model.fc.in_features\nmodel.fc = nn.Linear(2048, 1)\nmodel.load_state_dict(torch.load(\"../input/pytorch-transfer-learning-baseline/model.bin\"))\ndevice = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\") \nprint(device)\nmodel = model.to(device)\nmodel.eval()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data_loader = torch.utils.data.DataLoader(train_dataset, batch_size = 16, shuffle=True, num_workers=4)\n'''valid_loader = torch.utils.data.DataLoader(validdataset, batch_size=64, shuffle=True, num_workers=4)\nvalid_loader'''\n\n\nplist = [\n         {'params': model.layer4.parameters(), 'lr': 1e-4, 'weight': 0.001},\n         {'params': model.fc.parameters(), 'lr': 1e-3}\n         ]\n\noptimizer = optim.Adam(plist, lr=0.001)\nscheduler = lr_scheduler.StepLR(optimizer, step_size=10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#https://www.kaggle.com/abhishek/very-simple-pytorch-training-0-59/data\nsince = time.time()\ncriterion = torch.nn.MSELoss() # BCEWithLogitsLoss\n\nnum_epochs = 6 # train for longer for better results\n\nfor epoch in range(num_epochs):\n    print('Epoch {}/{}'.format(epoch, num_epochs - 1))\n    print('-' * 10)\n    scheduler.step()\n    model.train()\n    running_loss = 0.0\n    tk0 = tqdm(data_loader, total=int(len(data_loader)))\n    counter = 0\n    for bi, d in enumerate(tk0):\n        inputs = d[\"image\"]\n        labels = d[\"label\"].view(-1, 1)\n        inputs = inputs.to(device, dtype=torch.float)\n        labels = labels.to(device, dtype=torch.float)\n        optimizer.zero_grad()\n        with torch.set_grad_enabled(True):\n            outputs = model(inputs)\n            loss = criterion(outputs, labels)\n            #loss = criterion(outputs, torch.max(labels, 1)[1])\n            loss.backward()\n            optimizer.step()\n        running_loss += loss.item() * inputs.size(0)\n        #print(running_loss)\n        counter += 1\n        tk0.set_postfix(loss=(running_loss / (counter * data_loader.batch_size)))\n    epoch_loss = running_loss / len(data_loader)\n    print('Training Loss: {:.4f}'.format(epoch_loss))\n\ntime_elapsed = time.time() - since\nprint('Training complete in {:.0f}m {:.0f}s'.format(time_elapsed // 60, time_elapsed % 60))\ntorch.save(model.state_dict(), \"model.bin\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Inference"},{"metadata":{"trusted":true},"cell_type":"code","source":"class test_images(Dataset):\n\n    def __init__(self, csv_file):\n\n        self.data = csv_file\n\n    def __len__(self):\n        return len(self.data)\n\n    def __getitem__(self, idx):\n        img_name =  self.data.loc[idx][0]\n        image = Image.open(img_name)\n        image = image.resize((512, 512), resample=Image.BILINEAR)\n        #label = self.data.loc[idx][1] #torch.tensor(self.data.loc[idx, 'label'])\n        #image = self.transform(image)\n        return {'image': transforms.ToTensor()(image)}\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"testdataset = pd.DataFrame({ 'images': list(test_paths)},columns=['images'])\ntestdataset.head(2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"testdataset = test_images(testdataset)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub[\"Label\"] = pd.to_numeric(sub[\"Label\"].astype(float))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\ntest_loader = torch.utils.data.DataLoader(testdataset, batch_size=1, shuffle=False) # test_set contains only images directory\n\nfor param in model.parameters():\n    param.requires_grad = False\n\nprediction_list = []\ntk0 = tqdm(test_loader)\nfor i, x_batch in enumerate(tk0):\n    #print(i)\n    \n    x_batch = x_batch[\"image\"]\n    #print(x_batch)\n    pred =  model(x_batch.to(device))\n    #prediction_list.append(pred.cpu())\n    #print( type(pred.item()))\n    #print(\"\\n\")\n    sub.Label[i] = pred.item()\n    #print(sub.Label[i])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub.to_csv('submission.csv', index=False)\nsub.head(110)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# TakeAways : \n* [very simple pytorch training](https://www.kaggle.com/abhishek/very-simple-pytorch-training-0-59/data)\n\n* [Alaska2: EfficientNet on TPUs](https://www.kaggle.com/xhlulu/alaska2-efficientnet-on-tpus)\n\n\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}