{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-08-04T11:28:10.864054Z","iopub.execute_input":"2021-08-04T11:28:10.864489Z","iopub.status.idle":"2021-08-04T11:28:10.87603Z","shell.execute_reply.started":"2021-08-04T11:28:10.864399Z","shell.execute_reply":"2021-08-04T11:28:10.874634Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Import libraries**","metadata":{}},{"cell_type":"code","source":"import os\nimport time\nimport datetime\nimport random\nimport numpy as np\nfrom sklearn.metrics import accuracy_score\nimport torch.nn as nn\nfrom torch.autograd import Function\nimport torch.optim as optim\nimport torch.nn.functional as F\nimport tensorflow as tf\nimport torch\nfrom torchvision.transforms import ToTensor\nimport matplotlib.pyplot as plt\nfrom torch.utils.data import TensorDataset, DataLoader, RandomSampler, SequentialSampler, WeightedRandomSampler, Dataset\nimport logging\nfrom torchvision import datasets,transforms\nfrom tqdm import tqdm, trange","metadata":{"execution":{"iopub.status.busy":"2021-08-04T11:28:10.877629Z","iopub.execute_input":"2021-08-04T11:28:10.87795Z","iopub.status.idle":"2021-08-04T11:28:19.714375Z","shell.execute_reply.started":"2021-08-04T11:28:10.877919Z","shell.execute_reply":"2021-08-04T11:28:19.713077Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Setting devices**","metadata":{}},{"cell_type":"code","source":"#Get the GPU device name\ndevice_name = tf.test.gpu_device_name()\n\n#if device_name == '/device:GPU:0':\n#  print('GPU: {}'.format(device_name))\n#else:\n # raise SystemError('GPU not found')","metadata":{"execution":{"iopub.status.busy":"2021-08-04T11:28:19.716682Z","iopub.execute_input":"2021-08-04T11:28:19.717004Z","iopub.status.idle":"2021-08-04T11:28:19.733109Z","shell.execute_reply.started":"2021-08-04T11:28:19.716975Z","shell.execute_reply":"2021-08-04T11:28:19.731628Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# If there's a GPU\nif torch.cuda.is_available():\n  device = torch.device('cuda')\n  print('There are %d GPU(s).' % torch.cuda.device_count())\n  print('We will use the GPU:', torch.cuda.get_device_name())\nelse:\n  print('No GPU, using CPU.')\n  device = torch.device('cpu')","metadata":{"execution":{"iopub.status.busy":"2021-08-04T11:28:19.734821Z","iopub.execute_input":"2021-08-04T11:28:19.735192Z","iopub.status.idle":"2021-08-04T11:28:19.748948Z","shell.execute_reply.started":"2021-08-04T11:28:19.735156Z","shell.execute_reply":"2021-08-04T11:28:19.748005Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Setup Model**","metadata":{}},{"cell_type":"code","source":"class Net(nn.Module):\n  \"\"\"\n  Define the structure of model\n  \"\"\"\n  def __init__(self, args):\n    super(Net, self).__init__()\n\n #   self.conv1 = nn.Conv2d(1, 32, 3) # 28x28x32 -> 26x26x32 -> Pool(2,2) -> 13x13x32\n  #  self.conv2 = nn.Conv2d(32, 64, 3) # 13x13x64 -> 11x11x64 -> Pool(2,2) -> 5x5x64 \n   # self.pool = nn.MaxPool2d(2, 2) \n#    self.dropout = nn.Dropout(args.dropout_rate)\n#    self.fc1 = nn.Linear(5 * 5 * 64, 128)\n#    self.fc2 = nn.Linear(128, args.num_labels)\n#    self.num_labels = args.num_labels\n\n    self.conv1 = nn.Conv2d(3, 64, 3) # 224x224x3 -> 224x224x64 -> Pool(2,2) -> 112x112x64\n    self.conv2 = nn.Conv2d(64, 128, 3)\n    self.conv3 = nn.Conv2d(128, 256, 3) # 13x13x64 -> 11x11x64 -> Pool(2,2) -> 5x5x64 \n    self.conv4 = nn.Conv2d(256, 512, 3)\n    self.pool = nn.MaxPool2d(2, 2) \n    self.dropout = nn.Dropout(args.dropout_rate)\n    self.fc1 = nn.Linear(7 * 7 * 512, 4096)\n    self.fc2 = nn.Linear(4096, args.num_labels)\n    self.num_labels = args.num_labels\n\n  def forward(self, inputs, labels):\n    \"\"\"\n    The forward process of a model from input to output\n\n    :type inputs: Tensor[float]\n    :type labels: Tensor[float]\n    :rtype loss: float\n    :rtype preds: Tensor[int]\n    \"\"\"\n    \n    x = self.pool(F.relu(self.conv2(F.relu(self.conv1(inputs))))) #batchx13x13x32\n    x = self.pool(F.relu(self.conv3(x)))\n    x = self.pool(F.relu(self.conv4(x)))\n    x = self.pool(x);\n    x = self.dropout(x)\n    x = x.view(-1, 7 * 7 * 512) \n    x = F.relu(self.fc1(x))\n    x = self.dropout(x)\n    x = F.relu(self.fc2(x))\n\n    #Calculate losses\n    preds = nn.LogSoftmax(dim=1)(x)\n    loss_fct = nn.NLLLoss(reduction='mean')\n    loss = loss_fct(preds.view(-1, self.num_labels), labels.long().view(-1))\n\n    return loss, preds","metadata":{"execution":{"iopub.status.busy":"2021-08-04T11:28:19.750221Z","iopub.execute_input":"2021-08-04T11:28:19.750835Z","iopub.status.idle":"2021-08-04T11:28:19.764898Z","shell.execute_reply.started":"2021-08-04T11:28:19.750789Z","shell.execute_reply":"2021-08-04T11:28:19.764041Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Set up Trainer**","metadata":{}},{"cell_type":"code","source":"from sklearn.metrics import f1_score\n\ndef compute_metrics(preds, labels):\n    assert len(preds) == len(labels)\n    # print(preds)\n    # print(labels)\n    return acc_and_f1(preds, labels)\n\n\ndef simple_accuracy(preds, labels):\n    \"\"\"\n    Calculate average accuracy.\n    :type preds: numpy[int]\n    :type labels: numpy[int]\n    :rtype : float\n    \"\"\"\n    return (preds == labels).mean()\n\n\ndef acc_and_f1(preds, labels, average=\"macro\"):\n    \"\"\"\n    Calculate accuracy and f1-score. But in this problem, only accuracy is required.\n    :type preds: numpy[int]\n    :type labels: numpy[int]\n    :rtype : dict\n    \"\"\"\n\n    acc = simple_accuracy(preds, labels)\n\n    return {\n        \"acc\": acc\n    }\n\ndef init_logger():\n    logging.basicConfig(\n        format=\"%(asctime)s - %(levelname)s - %(name)s -   %(message)s\",\n        datefmt=\"%m/%d/%Y %H:%M:%S\",\n        level=logging.INFO,\n    )\n\ndef set_seed(args):\n    random.seed(args.seed)\n    np.random.seed(args.seed)\n    torch.manual_seed(args.seed)\n    if not args.no_cuda and torch.cuda.is_available():\n        torch.cuda.manual_seed_all(args.seed)\n","metadata":{"execution":{"iopub.status.busy":"2021-08-04T11:28:19.766116Z","iopub.execute_input":"2021-08-04T11:28:19.766747Z","iopub.status.idle":"2021-08-04T11:28:19.78467Z","shell.execute_reply.started":"2021-08-04T11:28:19.766703Z","shell.execute_reply":"2021-08-04T11:28:19.78371Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"logger = logging.getLogger(__name__)\n\nclass Trainer(object):\n    def __init__(self, args, train_dataset=None, dev_dataset=None):\n      \n        self.args = args\n        self.train_dataset = train_dataset\n        self.dev_dataset = dev_dataset\n  #      self.test_dataset = test_dataset\n        self.epochs_stop = args.early_stop\n        self.num_labels = args.num_labels\n\n        self.model = Net(self.args)\n\n        # GPU or CPU\n        self.device = \"cuda\" if torch.cuda.is_available() and not args.no_cuda else \"cpu\"\n        self.model.to(self.device)\n\n    def train(self):\n        train_dataloader = self.train_dataset\n\n        if self.args.max_steps > 0:\n            t_total = self.args.max_steps\n            self.args.num_train_epochs = (\n                self.args.max_steps // (len(train_dataloader) // self.args.gradient_accumulation_steps) + 1\n            )\n        else:\n            t_total = len(train_dataloader) // self.args.gradient_accumulation_steps * self.args.num_train_epochs\n\n        # Prepare optimizer. Using SGD\n        optimizer = optim.SGD(self.model.parameters(), lr=self.args.learning_rate)\n\n\n        # Train!\n        self.args.logging_steps = t_total // self.args.num_train_epochs\n        self.args.save_steps    = t_total // self.args.num_train_epochs\n        logger.info(\"***** Running training *****\")\n        logger.info(\"  Num examples = %d\", len(self.train_dataset))\n        logger.info(\"  Num Epochs = %d\", self.args.num_train_epochs)\n        logger.info(\"  Total train batch size = %d\", self.args.train_batch_size)\n        logger.info(\"  Gradient Accumulation steps = %d\", self.args.gradient_accumulation_steps)\n        logger.info(\"  Total optimization steps = %d\", t_total)\n        logger.info(\"  Logging steps = %d\", self.args.logging_steps)\n        logger.info(\"  Save steps = %d\", self.args.save_steps)\n\n        global_step = 0\n        tr_loss = 0.0\n        max_val_score = 0\n        early_stop = False\n        epochs_no_improve = 0 \n        self.model.zero_grad()\n\n        train_iterator = trange(int(self.args.num_train_epochs), desc=\"Epoch\") \n        for _ in train_iterator:\n            epoch_iterator = tqdm(train_dataloader, desc=\"Iteration\")\n            for step, batch in enumerate(epoch_iterator):\n                self.model.train()\n                batch = tuple(t.to(self.device) for t in batch)  # GPU or CPU\n                inputs = {\n                    \"inputs\": batch[0], #batch x W x H x C\n                    \"labels\": batch[1], #batch x 1\n                }\n                outputs = self.model(**inputs)\n                loss = outputs[0]\n\n                if self.args.gradient_accumulation_steps > 1:\n                    loss = loss / self.args.gradient_accumulation_steps\n\n                loss.backward()\n\n                tr_loss += loss.item()\n                if (step + 1) % self.args.gradient_accumulation_steps == 0:\n\n                    optimizer.step()\n                    self.model.zero_grad()\n                    global_step += 1\n\n                    if self.args.logging_steps > 0 and global_step % self.args.logging_steps == 0:\n                        res = self.evaluate(\"dev\")\n                        logger.info(\"Training total loss = %.4f\", outputs[0])  \n                        if res['acc']  > max_val_score:\n                          max_val_score = res['acc']\n                          final = res\n                          epochs_no_improve = 0\n                          self.save_model()\n                        else:\n                          epochs_no_improve += 1\n                        \n                        if epochs_no_improve == self.epochs_stop:\n                          early_stop = True\n                          logger.info(\" Early Stopping!!!!!!\")\n                          logger.info(\"***** Final results *****\")\n                          for key in sorted(final.keys()):\n                              logger.info(\"  {} = {:.4f}\".format(key, final[key]))\n                          break\n\n\n                if 0 < self.args.max_steps < global_step or early_stop==True:\n                    epoch_iterator.close()\n                    break\n            \n            if 0 < self.args.max_steps < global_step or early_stop==True:\n                train_iterator.close()\n                break\n\n\n        return global_step, tr_loss / global_step\n\n    def evaluate(self, mode, out_pred=False):\n        if mode == \"test\":\n            eval_dataloader = self.test_dataset\n        elif mode == \"dev\":\n            eval_dataloader = self.dev_dataset\n        else:\n            raise Exception(\"No dev and test dataset available!\")\n\n        # Eval!\n        logger.info(\"***** Running evaluation on %s dataset *****\", mode)\n        logger.info(\"  Num examples = %d\", len(eval_dataloader))\n        logger.info(\"  Batch size = %d\", self.args.eval_batch_size)\n        eval_loss = 0.0\n        nb_eval_steps = 0\n        preds = None\n        out_label_ids = None\n\n        self.model.eval()\n\n        for batch in tqdm(eval_dataloader, desc=\"Evaluating\"):\n            batch = tuple(t.to(self.device) for t in batch)\n            with torch.no_grad():\n                inputs = {\n                    \"inputs\": batch[0],\n                    \"labels\": batch[1],\n                }\n                outputs = self.model(**inputs)\n                logits, tmp_eval_loss = outputs[1], outputs[0]\n\n                eval_loss += tmp_eval_loss.mean().item()\n            nb_eval_steps += 1\n\n            if preds is None:\n                preds = logits.detach().cpu().numpy()   #Have to change device to cpu to be able converting to numpy\n                out_label_ids = inputs[\"labels\"].detach().cpu().numpy()\n                #att_weights = outputs[-1].detach().cpu().numpy()\n            else:\n                preds = np.append(preds, logits.detach().cpu().numpy(), axis=0)\n                out_label_ids = np.append(out_label_ids, inputs[\"labels\"].detach().cpu().numpy(), axis=0)\n                #att_weights = np.append(att_weights, outputs[-1].detach().cpu().numpy(), axis=0)\n\n        eval_loss = eval_loss / nb_eval_steps\n        results = {\"loss\": eval_loss}\n        # print(preds) \n        preds = np.argmax(preds, axis=1) #eval_len x num_label -> eval_len x 1\n        # write_prediction(self.args, os.path.join(self.args.eval_dir, \"proposed_answers.txt\"), preds)\n\n        result = compute_metrics(preds, out_label_ids)\n        results.update(result)\n\n        logger.info(\"***** Eval results *****\")\n        for key in sorted(results.keys()):\n            logger.info(\"  {} = {:.4f}\".format(key, results[key]))\n\n        if out_pred == True:\n          return preds\n        return results\n\n    def save_model(self):\n        model_to_save = self.model\n        torch.save(model_to_save, os.path.join(self.args.model_dir, \"model.pt\"))\n\n        # Save training arguments together with the trained model\n        torch.save(self.args, os.path.join(self.args.model_dir, \"training_args.bin\"))\n        logger.info(\"Saving model checkpoint to %s\", self.args.model_dir)\n\n    def load_model(self):\n        # Check whether model exists\n        if not os.path.exists(self.args.model_dir):\n            raise Exception(\"Model doesn't exists! Train first!\")\n\n        self.args = torch.load(os.path.join(self.args.model_dir, \"training_args.bin\"))\n        self.model = torch.load(os.path.join(self.args.model_dir, \"model.pt\"))\n        self.model.to(self.device)\n        logger.info(\"***** Model Loaded *****\")\n","metadata":{"execution":{"iopub.status.busy":"2021-08-04T11:28:19.786178Z","iopub.execute_input":"2021-08-04T11:28:19.786473Z","iopub.status.idle":"2021-08-04T11:28:19.823249Z","shell.execute_reply.started":"2021-08-04T11:28:19.786445Z","shell.execute_reply":"2021-08-04T11:28:19.822415Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Set up Params**","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nfrom torchvision.io import read_image\n\nclass data_file(Dataset):\n    def __init__(self, annotations_file, img_dir, transform=None, target_transform=None):\n        self.img_labels = pd.read_csv(annotations_file)\n        self.img_dir = img_dir\n        self.transform = transform\n        self.target_transform = target_transform\n\n    def __len__(self):\n        return len(self.img_labels)\n\n    def __getitem__(self, idx):\n        img_path = os.path.join(self.img_dir, self.img_labels.iloc[idx, 0] + \".PNG\")\n        image = read_image(img_path).float()\n        label = self.img_labels.iloc[idx, 1]\n        if self.transform:\n            image = self.transform(image)\n        if self.target_transform:\n            label = self.target_transform(label)\n        return image, label","metadata":{"execution":{"iopub.status.busy":"2021-08-04T11:28:19.824685Z","iopub.execute_input":"2021-08-04T11:28:19.825003Z","iopub.status.idle":"2021-08-04T11:28:19.841821Z","shell.execute_reply.started":"2021-08-04T11:28:19.824974Z","shell.execute_reply":"2021-08-04T11:28:19.840741Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def main(args):\n  \n    init_logger()\n    set_seed(args)\n    a = ['../input/hsgshackathon2021/train_data/Train_labels/video_10.csv', '../input/hsgshackathon2021/train_data/Train_labels/video_11.csv', \n         '../input/hsgshackathon2021/train_data/Train_labels/video_110.csv', '../input/hsgshackathon2021/train_data/Train_labels/video_120.csv',\n         '../input/hsgshackathon2021/train_data/Train_labels/video_130.csv','../input/hsgshackathon2021/train_data/Train_labels/video_20.csv',  \n         '../input/hsgshackathon2021/train_data/Train_labels/video_40.csv',  '../input/hsgshackathon2021/train_data/Train_labels/video_41.csv',\n         '../input/hsgshackathon2021/train_data/Train_labels/video_60.csv']\n    b = ['../input/hsgshackathon2021/train_data/Train/video_10', '../input/hsgshackathon2021/train_data/Train/video_11', \n         '../input/hsgshackathon2021/train_data/Train/video_110', '../input/hsgshackathon2021/train_data/Train/video_120',\n         '../input/hsgshackathon2021/train_data/Train/video_130', '../input/hsgshackathon2021/train_data/Train/video_20', \n         '../input/hsgshackathon2021/train_data/Train/video_40',  '../input/hsgshackathon2021/train_data/Train/video_41', \n         '../input/hsgshackathon2021/train_data/Train/video_60']\n    \n    for i, j in zip(a, b):\n        training_data = data_file(i, j)\n\n        train_dataloader = DataLoader(training_data, batch_size=64, shuffle=True)\n    #    test_dataloader = DataLoader(test_data, batch_size=64, shuffle=True)\n\n        train_features, train_labels = next(iter(train_dataloader))\n        print(f\"Feature batch shape: {train_features.size()}\")\n        print(f\"Labels batch shape: {train_labels.size()}\")\n        img = train_features[0].squeeze()\n        label = train_labels[0]\n        plt.imshow(img.transpose(0, 2))\n        plt.show()\n        print(f\"Label: {label}\")\n\n      #  data_loader = torch.utils.data.DataLoader(imagenet_data,batch_size=4,shuffle=True,num_workers=args.nThreads)\n\n      #  mnist_trainval = DataLoader(training_data, batch_size = 64, shuffle = True)\n      #  mnist_testset = DataLoader(Test_data, batch_size = 64, shuffle = True)\n    \n        n_samples = len(training_data) # n_samples is 60000\n        train_size = int(len(training_data) * 0.8) # train_size is 48000\n        val_size = n_samples - train_size # val_size is 48000\n\n    # split train/val\n        train_dataset, val_dataset = torch.utils.data.random_split(training_data, [train_size, val_size])\n        train_loader = torch.utils.data.DataLoader(train_dataset, batch_size=args.train_batch_size, shuffle=True)\n        val_loader = torch.utils.data.DataLoader(val_dataset, batch_size=args.train_batch_size, shuffle=True)\n     #   test_loader = torch.utils.data.DataLoader(mnist_testset, batch_size=args.eval_batch_size, shuffle=True)\n\n\n        trainer = Trainer(args, train_dataset=train_loader, dev_dataset=val_loader)\n        #test_dataset=test_loader)\n        if args.do_train:\n            trainer.train()\n    \n        if args.do_eval:\n            trainer.load_model()\n            trainer.evaluate(\"dev\")\n            trainer.evaluate(\"test\")","metadata":{"execution":{"iopub.status.busy":"2021-08-04T11:28:19.845123Z","iopub.execute_input":"2021-08-04T11:28:19.845929Z","iopub.status.idle":"2021-08-04T11:28:19.860751Z","shell.execute_reply.started":"2021-08-04T11:28:19.84588Z","shell.execute_reply":"2021-08-04T11:28:19.859167Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class arg():\n  def __init__(self):\n    \"\"\"Defining hyper-parameters\"\"\"\n    self.model_dir = './'\n    self.do_train = True\n    self.do_eval = True\n    self.no_cuda = False\n    self.dropout_rate = 0.2\n    self.max_steps = -1                      #total number of training steps to perform\n    self.gradient_accumulation_steps = 1     #Number of updates steps to accumulate before performing a backward/update pass\n    self.num_train_epochs = 1\n    self.learning_rate = 0.03\n    self.early_stop = 10\n    self.train_batch_size = 128\n    self.eval_batch_size = 128\n    self.seed = 2020\n    self.num_labels = 10","metadata":{"execution":{"iopub.status.busy":"2021-08-04T11:28:19.862972Z","iopub.execute_input":"2021-08-04T11:28:19.863471Z","iopub.status.idle":"2021-08-04T11:28:19.877957Z","shell.execute_reply.started":"2021-08-04T11:28:19.86339Z","shell.execute_reply":"2021-08-04T11:28:19.877122Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Train**","metadata":{}},{"cell_type":"code","source":"args = arg()","metadata":{"execution":{"iopub.status.busy":"2021-08-04T11:28:19.879109Z","iopub.execute_input":"2021-08-04T11:28:19.879543Z","iopub.status.idle":"2021-08-04T11:28:19.893973Z","shell.execute_reply.started":"2021-08-04T11:28:19.87949Z","shell.execute_reply":"2021-08-04T11:28:19.892973Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"main(args)","metadata":{"execution":{"iopub.status.busy":"2021-08-04T11:28:19.89532Z","iopub.execute_input":"2021-08-04T11:28:19.895797Z"},"trusted":true},"execution_count":null,"outputs":[]}]}