{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import torch.nn as nn\nimport torch\nimport torch.optim as optim\nimport pickle\nimport numpy as np\nimport torchvision\nfrom torchvision import datasets, models, transforms\nfrom torch.utils.data import TensorDataset, DataLoader,Dataset\nimport matplotlib.pyplot as plt\nimport time\nimport os\nimport copy\nimport pandas as pd\nimport PIL.Image as Image\nfrom torch.utils.data.sampler import SubsetRandomSampler\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.preprocessing import OneHotEncoder\ntrain_on_gpu = True\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ad1140da72ee9c18ca842634a7dc150e6ea58995"},"cell_type":"code","source":"model_name = \"vgg\"\nnum_classes = 5005\nfeature_extract = False","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e5da8a99a90acad36983e5591ab6ca39c914633f"},"cell_type":"code","source":"print(os.listdir(\"../input\"))\n!ls \"../input/humpback-whale-identification\"\n!ls \"../input/train-val\"\npath_train = \"../input/train-val/train.csv\"\npath_val = \"../input/train-val/val.csv\"\n\ndf_train = pd.read_csv(\"../input/humpback-whale-identification/train.csv\")\n#df_train = pd.read_csv(\"../input/train-val/train.csv\")\ndf_val = pd.read_csv(\"../input/train-val/val.csv\")\nprint(len(df_train))\nprint(len(df_val))\nprint(df_train.head())\nprint(df_val.head())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7ba3c3ffd9609fb00c68b919391094d25d38838f"},"cell_type":"code","source":"def prepare_labels(y):\n    # From here: https://www.kaggle.com/pestipeti/keras-cnn-starter\n    values = np.array(y)\n    label_encoder = LabelEncoder()\n    integer_encoded = label_encoder.fit_transform(values)\n\n    onehot_encoder = OneHotEncoder(sparse=False)\n    integer_encoded = integer_encoded.reshape(len(integer_encoded), 1)\n    onehot_encoded = onehot_encoder.fit_transform(integer_encoded)\n\n    y = onehot_encoded\n    return y, label_encoder\n\n\ny_train, le_train = prepare_labels(df_train['Id'])\n#train:20286\n#val:5075\ny_train = y_train[range(20286)]\ny_val = y_train[range(5075)]\n#y_val, le_val = prepare_labels(df_val['Id'])\nprint((y_train).shape)\nprint(y_val.shape)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"# For train and val, datafolder = \"../input/humpback-whale-identification/train\"\n# For test, datafolder = \"../input/humpback-whale-identification/test\"\n\n# For train, datatype = \"train\"\n# For val, datatype = \"val\"\n# For test, datatype = \"test\"\n\n# For train, y = y_train\n# For val, y = y_val\n\n# For train, df = df_train\n# For val, df = df_val\n\n# Finished # IMPORTANT: Remember to split y into train and val before creating object of this class\nclass Whaledataset(Dataset):\n    def __init__(self, datafolder, datatype, transform, y, df):\n        self.datafolder = datafolder\n        self.datatype = datatype\n        self.transform = transform\n        self.y = y # y is same for both train and val\n        if datatype ==\"train\":\n            self.df = df.values\n        if datatype ==\"val\":\n            self.df = df.values\n        # self.df not there for test because no df for test        \n        self.image_list = [s for s in os.listdir(datafolder)]\n        \n    def __len__(self):\n        if self.datatype == \"train\":\n            return 20286\n        elif self.datatype == \"val\":\n            return 5075\n        elif self.datatype == \"test\":\n            return len(os.listdir(self.datafolder))\n        \n        #return len(os.listdir(self.image_list)) #Returns same value for train and val. Correct later\n    \n    def __getitem__(self, idx):\n        if self.datatype =='train' or self.datatype ==\"val\":\n            img_name = os.path.join(self.datafolder, self.df[idx][0])\n            label = self.y[idx]\n            \n        elif self.datatype == 'test':\n            img_name = os.path.join(self.datafolder, self.image_list[idx])\n            label = np.zeros((5005,))\n\n        image = Image.open(img_name).convert('RGB')\n        image = self.transform(image)\n        if self.datatype == 'train' or self.datatype == \"val\":\n            return image, label\n        elif self.datatype == 'test':\n            # so that the images will be in a correct order\n            return image, label, self.image_list[idx]\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1b9b9b9072b424a84b85d403aca27703a4e5732b"},"cell_type":"code","source":"def set_parameter_requires_grad(model, feature_extracting):\n    if feature_extracting:\n        for param in model.parameters():\n            param.requires_grad = False\n\n\ndef initialize_model(model_name, num_classes, feature_extract, use_pretrained=True):\n    # Initialize these variables which will be set in this if statement. Each of these\n    #   variables is model specific.\n    model_ft = None\n    input_size = 0\n\n    if model_name == \"resnet\":\n        \"\"\" Resnet34\n        \"\"\"\n        model_ft = models.resnet34(pretrained=use_pretrained)\n        set_parameter_requires_grad(model_ft, feature_extract)\n        num_ftrs = model_ft.fc.in_features\n        model_ft.fc = nn.Linear(num_ftrs, num_classes)\n        input_size = 224\n\n    elif model_name == \"alexnet\":\n        \"\"\" Alexnet\n        \"\"\"\n        model_ft = models.alexnet(pretrained=use_pretrained)\n        set_parameter_requires_grad(model_ft, feature_extract)\n        num_ftrs = model_ft.classifier[6].in_features\n        model_ft.classifier[6] = nn.Linear(num_ftrs,num_classes)\n        input_size = 224\n\n    elif model_name == \"vgg\":\n        \"\"\" VGG11_bn\n        \"\"\"\n        model_ft = models.vgg11_bn(pretrained=use_pretrained)\n        set_parameter_requires_grad(model_ft, feature_extract)\n        num_ftrs = model_ft.classifier[6].in_features\n        model_ft.classifier[6] = nn.Linear(num_ftrs,num_classes)\n        input_size = 224\n\n    elif model_name == \"squeezenet\":\n        \"\"\" Squeezenet\n        \"\"\"\n        model_ft = models.squeezenet1_0(pretrained=use_pretrained)\n        set_parameter_requires_grad(model_ft, feature_extract)\n        model_ft.classifier[1] = nn.Conv2d(512, num_classes, kernel_size=(1,1), stride=(1,1))\n        model_ft.num_classes = num_classes\n        input_size = 224\n\n    elif model_name == \"densenet\":\n        \"\"\" Densenet\n        \"\"\"\n        model_ft = models.densenet121(pretrained=use_pretrained)\n        set_parameter_requires_grad(model_ft, feature_extract)\n        num_ftrs = model_ft.classifier.in_features\n        model_ft.classifier = nn.Linear(num_ftrs, num_classes)\n        input_size = 224\n\n    elif model_name == \"inception\":\n        \"\"\" Inception v3\n        Be careful, expects (299,299) sized images and has auxiliary output\n        \"\"\"\n        model_ft = models.inception_v3(pretrained=use_pretrained)\n        set_parameter_requires_grad(model_ft, feature_extract)\n        # Handle the auxilary net\n        num_ftrs = model_ft.AuxLogits.fc.in_features\n        model_ft.AuxLogits.fc = nn.Linear(num_ftrs, num_classes)\n        # Handle the primary net\n        num_ftrs = model_ft.fc.in_features\n        model_ft.fc = nn.Linear(num_ftrs,num_classes)\n        input_size = 299\n\n    else:\n        print(\"Invalid model name, exiting...\")\n        exit()\n\n    return model_ft, input_size\n\n# Initialize the model for this run\nmodel_ft, input_size = initialize_model(model_name, num_classes, feature_extract, use_pretrained=True)\n\n# Print the model we just instantiated\nprint(model_ft)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"90a9c2dd2c99bcf7abd014c78e9a6f2c35efcda1"},"cell_type":"code","source":"train_transforms = transforms.Compose([\n                                      transforms.Resize((224,224)),\n                                      transforms.RandomHorizontalFlip(0.5),\n                                      transforms.RandomAffine(30),\n                                      transforms.ToTensor(),\n                                      transforms.Normalize(mean=[0.485, 0.456, 0.406],\n                                                             std=[0.229, 0.224, 0.225])\n    ])\nval_transforms = transforms.Compose([\n                                      transforms.Resize((224,224)),\n                                      transforms.RandomHorizontalFlip(0.5),\n                                      transforms.RandomAffine(30),\n                                      transforms.ToTensor(),\n                                      transforms.Normalize(mean=[0.485, 0.456, 0.406],\n                                                             std=[0.229, 0.224, 0.225])\n    ])\n\ntest_transforms = transforms.Compose([\n                                       transforms.Resize((224,224)),\n                                       transforms.ToTensor(),\n                                       transforms.Normalize(mean = [0.485, 0.456, 0.406],\n                                                              std = [0.229, 0.224, 0.225])\n])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b3c1d682c3de8b24a63f29c3a08b2ec5128fdee0"},"cell_type":"code","source":"\n# Write function def accuracy here\n\ndef accuracy(outputs,labels):\n    preds = outputs\n    #print(type(preds), type(outputs))\n    #preds_idx = np.argmax(preds, axis =1)\n    #labels_idx = np.argmax(labels, axis =1)\n    preds_idx = torch.argmax(preds, dim=1)\n    ##print(preds_idx)\n    labels_idx = torch.argmax(labels, dim=1)\n    ##print(labels_idx)\n    #preds_values, preds_idx = torch.max(preds, 1)\n    #outputs_values, outputs_idx = torch.max(outputs, 1)\n    #print(preds.shape)\n    #print(labels.shape)\n    return (preds_idx==labels_idx).float().sum()\n\ntrain_datafolder = \"../input/humpback-whale-identification/train\"\ntrain_datatype = \"train\"\ntrain_df = df_train\ntrain_transform = train_transforms\ntrain_y = y_train\nbatch_size = 32\nnum_workers = 0\n\nval_datafolder = \"../input/humpback-whale-identification/train\"\nval_datatype = \"val\"\nval_df = df_val\nval_transform = val_transforms\nval_y = y_val\nbatch_size = 32\nnum_workers = 0\n\n\ndef train(batch_size = 32, epochs = 10, transform = None):\n    tfmd_dataset = Whaledataset(datafolder = train_datafolder, datatype = train_datatype, df = train_df , \n                                transform = train_transform, y = train_y)\n    dataloader = torch.utils.data.DataLoader(tfmd_dataset, batch_size = batch_size, shuffle=True, \n                                             num_workers=num_workers)\n    \n    val_dataset = Whaledataset(datafolder = val_datafolder, datatype = val_datatype, df = val_df , \n                                transform = val_transform, y = val_y)\n    \n    val_dataloader = torch.utils.data.DataLoader(val_dataset, batch_size = batch_size, shuffle=True, \n                                             num_workers=num_workers)\n    \n    criterion = nn.BCEWithLogitsLoss()\n    optimizer = optim.Adam(model_ft.parameters(), lr=0.0001)\n    \n    if torch.cuda.is_available():\n        model_ft.cuda()\n        nn.DataParallel(model_ft)\n        print(\"GPU\")\n    epoch_loss_data = []\n    epoch_accuracy_data = []\n\n    device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n\n    print(device)\n    \n    for epoch in range(1,epochs+1):  # loop over the dataset multiple times\n        i = 0.0\n        running_loss, running_loss_total, epoch_accuracy = 0.0, 0.0, 0.0\n        val_running_loss, val_running_loss_total, val_epoch_accuracy = 0.0, 0.0, 0.0\n        for i, data in enumerate(dataloader, 0):\n            # get the inputs\n            inputs, labels = data\n            labels = labels.double()\n            inputs, labels = inputs.to(device), labels.to(device)\n        \n            # zero the parameter gradients\n            optimizer.zero_grad()\n            # forward + backward + optimize\n            outputs = model_ft(inputs).double()\n            #print(outputs.size())\n            #print(labels.size())\n            \n            loss = criterion(outputs, labels)\n            \n            loss.backward()\n            optimizer.step()\n\n            # print statistics\n            epoch_accuracy += accuracy(outputs,labels)\n            #print(epoch_accuracy)\n            #running_epoch_acc = float(epoch_accuracy)/float((labels.size()[0] * (i+1)))\n            running_loss += loss.item()\n            #running_loss_total += loss.item()\n            '''if i%30 == 29:\n                print('[%d, %5d] loss: %.3f, epoch_accuracy: %f' % (epoch, i + 1, running_loss, running_epoch_acc))\n                running_loss = 0.0'''\n            \n            '''if i%100 == 99:\n                torch.save(model_ft.state_dict(), 'checkpoint')\n                torch.save(optimizer.state_dict(), 'optimizer_checkpoint')\n                with open('Running_loss.p','wb') as f:\n                    pickle.dump(running_loss_total,f)\n                with open('Running_accuracy.p','wb') as f:\n                    pickle.dump(running_epoch_acc,f)\n        epoch_accuracy_data.append(running_epoch_acc)\n        epoch_loss_data.append(running_loss_total)'''\n            \n        epoch_accuracy = float(epoch_accuracy)/float(32 * (i+1))\n        print('[%d, %5d] loss: %.3f, epoch_accuracy: %f' % (epoch, i + 1, running_loss, epoch_accuracy))\n        running_loss = 0.0\n        epoch_accuracy = 0.0\n        \n        for i, data in enumerate(val_dataloader,0):\n            inputs, labels = data\n            labels = labels.double()\n            inputs, labels = inputs.to(device), labels.to(device)\n            outputs = model_ft(inputs).double()\n            loss = criterion(outputs, labels)\n            val_epoch_accuracy += accuracy(outputs,labels)\n            val_running_epoch_acc = float(val_epoch_accuracy)/float((labels.size()[0] * (i+1)))\n            val_running_loss += loss.item()\n            '''if i%30 == 29:\n                print('[%d, %5d] val loss: %.3f, val_epoch_accuracy: %f' % (epoch, i + 1, val_running_loss, val_running_epoch_acc))\n                val_running_loss = 0.0\n            '''\n        val_epoch_accuracy = float(val_epoch_accuracy)/float(32 * (i+1))    \n        print('[%d, %5d] val loss: %.3f, val_epoch_accuracy: %f' % (epoch, i + 1, val_running_loss, val_epoch_accuracy))\n        val_running_loss = 0.0\n        val_epoch_accuracy = 0.0\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e3eedff14c4eaba36ffdbcb0995887c4ba90ceed"},"cell_type":"code","source":"train(transform=train_transforms,epochs=25,batch_size=32)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4ac0fcc04b31179621e8213ed3635c764f1f93fd"},"cell_type":"code","source":"test_datafolder = \"../input/humpback-whale-identification/test\"\ntest_datatype = \"test\"\ntest_transform = test_transforms\nbatch_size = 32\nnum_workers = 0\nsub = pd.read_csv(\"../input/humpback-whale-identification/sample_submission.csv\")\ndef test(batch_size, transform, model):\n    test_set = Whaledataset(datafolder = test_datafolder, datatype = test_datatype, transform = test_transform, y = y_train, df = df_train)\n    test_loader = torch.utils.data.DataLoader(test_set, batch_size=32, num_workers=num_workers)\n    model.cuda()\n    model.eval()\n    for (data, target, name) in test_loader:\n        data = data.cuda()\n        output = model(data)\n        print(output)\n        output = output.cpu().detach().numpy()\n        for i, (e, n) in enumerate(list(zip(output, name))):\n            sub.loc[sub['Image'] == n, 'Id'] = ' '.join(le_train.inverse_transform(e.argsort()[-5:][::-1]))\n    sub.to_csv('basic_model.csv', index=False)   ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"460c1da562ffc483a29aeb6d551936aa694cbfb2"},"cell_type":"code","source":"test(batch_size, test_transform, model_ft)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ffe3ac96e8feaadf145d1ca7177839be266c3709"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}