{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Description","metadata":{}},{"cell_type":"markdown","source":"So in this notebook I will try to create some model, which takes both factors into account: image and seqaunce analysis. That's why I came up with the Idea of the **CNN** Model for frames, which afterwards is fed into the **LSTM**. To achive this I split the data and feed every second (so 25 frames) into the mode. I assign the closest (by time) event to every second. Each input in the end is of the shape (25,3,128,128), with an output of 5 classes (**challenge, throwin, play, start, end**). In the end, i will use only **challenge, throwin and play** classes for the prediction. <span style=\"color:red\">The Inference of this model can be found </span>[here](https://www.kaggle.com/code/kostiantynlavronenko/dfl-bundesliga-simple-cnn-lstm-inference). I use really cool resized [dataset](https://www.kaggle.com/code/yokuyama/fast-video-resizing-with-ffmpeg-384x216/notebook) of [yokuyama](https://www.kaggle.com/yokuyama) in this notebook.","metadata":{}},{"cell_type":"markdown","source":"# Import libraries","metadata":{}},{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport os\nimport cv2\nfrom tqdm import tqdm\nfrom torch.utils.data import Dataset, DataLoader\nimport seaborn as sns\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torchvision.transforms as transforms\nfrom prettytable import PrettyTable\nfrom sklearn.model_selection import train_test_split\nfor dirname, _, filenames in os.walk('/kaggle/input/dfl-resize-384x216'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-08-14T09:03:41.624120Z","iopub.execute_input":"2022-08-14T09:03:41.624789Z","iopub.status.idle":"2022-08-14T09:03:45.069159Z","shell.execute_reply.started":"2022-08-14T09:03:41.624702Z","shell.execute_reply":"2022-08-14T09:03:45.068011Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\nprint(device)","metadata":{"execution":{"iopub.status.busy":"2022-08-14T09:03:45.074614Z","iopub.execute_input":"2022-08-14T09:03:45.077695Z","iopub.status.idle":"2022-08-14T09:03:45.153322Z","shell.execute_reply.started":"2022-08-14T09:03:45.077650Z","shell.execute_reply":"2022-08-14T09:03:45.151217Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def count_parameters(model):\n    table = PrettyTable([\"Modules\", \"Parameters\"])\n    total_params = 0\n    for name, parameter in model.named_parameters():\n        if not parameter.requires_grad: continue\n        params = parameter.numel()\n        table.add_row([name, params])\n        total_params+=params\n    print(table)\n    print(f\"Total Trainable Params: {total_params}\")\n    return total_params","metadata":{"execution":{"iopub.status.busy":"2022-08-14T09:03:45.156395Z","iopub.execute_input":"2022-08-14T09:03:45.157234Z","iopub.status.idle":"2022-08-14T09:03:45.164204Z","shell.execute_reply.started":"2022-08-14T09:03:45.157191Z","shell.execute_reply":"2022-08-14T09:03:45.163165Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Load train dataset csv","metadata":{}},{"cell_type":"code","source":"df_train = pd.read_csv(\"/kaggle/input/dfl-bundesliga-data-shootout/train.csv\")\ndf_train[\"file_path\"] = \"../input/dfl-resize-384x216/train_resize/\" + df_train[\"video_id\"] + \".mp4\"\ndf_train.head(20)","metadata":{"execution":{"iopub.status.busy":"2022-08-14T09:03:45.167043Z","iopub.execute_input":"2022-08-14T09:03:45.168027Z","iopub.status.idle":"2022-08-14T09:03:45.234553Z","shell.execute_reply.started":"2022-08-14T09:03:45.167981Z","shell.execute_reply":"2022-08-14T09:03:45.233329Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Here I create the dataframe where every video is split in a second by second patch","metadata":{}},{"cell_type":"code","source":"video_names = df_train.video_id.unique()\nvideo_names","metadata":{"execution":{"iopub.status.busy":"2022-08-14T09:07:09.787193Z","iopub.execute_input":"2022-08-14T09:07:09.787854Z","iopub.status.idle":"2022-08-14T09:07:09.795674Z","shell.execute_reply.started":"2022-08-14T09:07:09.787818Z","shell.execute_reply":"2022-08-14T09:07:09.794617Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labels_time = []\nvideo_ids = []\nfor name in video_names:    \n    time_to_end = df_train[df_train[\"video_id\"]==name].iloc[-1].time\n    TIME_SPLIT = 1 # 25 frames\n    times_of_one_video = np.arange(0,time_to_end,TIME_SPLIT)\n    labels_time.extend(times_of_one_video)\n    video_ids.extend([name]*len(times_of_one_video))\nD = {\"video_id\":video_ids, \"time\":labels_time}\ndf_labels = pd.DataFrame(D)\ndisplay(df_labels)","metadata":{"execution":{"iopub.status.busy":"2022-08-14T09:12:03.793333Z","iopub.execute_input":"2022-08-14T09:12:03.793792Z","iopub.status.idle":"2022-08-14T09:12:03.862277Z","shell.execute_reply.started":"2022-08-14T09:12:03.793752Z","shell.execute_reply":"2022-08-14T09:12:03.861065Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"This may be written not in the most optimal way, I am always open to your suggestions🤗","metadata":{}},{"cell_type":"code","source":"labels = []\npaths = []\nfor i, row in tqdm(df_labels.iterrows()):\n    df_train_only_specific_video = df_train[df_train.video_id == row.video_id]\n    video_clip = df_train_only_specific_video.iloc[(df_train_only_specific_video['time'] - row.time).abs().argsort()[:1]]\n    video_clip = video_clip.iloc[0]\n    labels.append(video_clip.event)\n    paths.append(video_clip.file_path)\ndf_labels[\"event\"] = labels\ndf_labels[\"file_path\"] = paths\ndf_labels","metadata":{"execution":{"iopub.status.busy":"2022-08-14T09:13:47.834073Z","iopub.execute_input":"2022-08-14T09:13:47.835069Z","iopub.status.idle":"2022-08-14T09:15:16.394421Z","shell.execute_reply.started":"2022-08-14T09:13:47.835024Z","shell.execute_reply":"2022-08-14T09:15:16.393307Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"num_start = df_labels[df_labels.event == \"start\"].event.count()\nnum_end = df_labels[df_labels.event == \"end\"].event.count()\nnum_throwin = df_labels[df_labels.event == \"throwin\"].event.count()\nnum_challenge = df_labels[df_labels.event == \"challenge\"].event.count()\nnum_play = df_labels[df_labels.event == \"play\"].event.count()\nsns.histplot(df_labels, x = \"event\")","metadata":{"execution":{"iopub.status.busy":"2022-08-14T09:25:53.233208Z","iopub.execute_input":"2022-08-14T09:25:53.234066Z","iopub.status.idle":"2022-08-14T09:25:53.469137Z","shell.execute_reply.started":"2022-08-14T09:25:53.234034Z","shell.execute_reply":"2022-08-14T09:25:53.468182Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Well, this is very unbalanced🤔 we'll have to take care of it.","metadata":{}},{"cell_type":"markdown","source":"# Dataset","metadata":{}},{"cell_type":"code","source":"class VideoFrameDataset(Dataset):\n    def __init__(self, df):\n        self.df = df\n        self.frames_split = 25 * TIME_SPLIT # 25 frames per second\n        \n    def __len__(self):\n        return self.df.shape[0]\n    \n    def __getitem__(self, index):\n        event = self.df.iloc[index].event\n        file_path = self.df.iloc[index].file_path\n        time = self.df.iloc[index].time\n        D = {\"challenge\":0, \"throwin\":1, \"play\":2, \"start\":3, \"end\":4}\n        label = D[event]\n        cap = cv2.VideoCapture(file_path)\n        video_clip = np.zeros((self.frames_split, 128, 128, 3))\n        #skip all unused frames\n        cap.set(cv2.CAP_PROP_POS_FRAMES,time*self.frames_split)\n        for i in range(self.frames_split):\n            _, frame = cap.read()\n            frame = cv2.resize(frame, (128,128))\n            video_clip[i] = frame\n        video_clip = video_clip.transpose(0, 3, 1, 2)\n        return torch.from_numpy(video_clip), label\n","metadata":{"execution":{"iopub.status.busy":"2022-08-14T09:25:55.925139Z","iopub.execute_input":"2022-08-14T09:25:55.925598Z","iopub.status.idle":"2022-08-14T09:25:55.941705Z","shell.execute_reply.started":"2022-08-14T09:25:55.925559Z","shell.execute_reply":"2022-08-14T09:25:55.940662Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model","metadata":{}},{"cell_type":"code","source":"class CNN(nn.Module):\n    def __init__(self):\n        super(CNN, self).__init__()\n        \n        self.conv1 = nn.Conv2d(3, 10, 5)\n        self.conv2 = nn.Conv2d(10, 20, 5)\n        self.conv3 = nn.Conv2d(20, 30, 5)\n        \n    def forward(self, i):\n        x = i.view(-1, i.shape[2], i.shape[3], i.shape[4])\n        x = F.relu(self.conv1(x))\n        x = F.relu(self.conv2(x))\n        x = F.relu(self.conv3(x))\n        x = nn.AvgPool2d(4)(x)\n        x = x.view(i.shape[0], i.shape[1], -1)\n        return x\n    \nclass LSTM(nn.Module):\n    def __init__(self):\n        super(LSTM, self).__init__()\n        self.lstm = nn.LSTM(25230, 100)\n        self.fc = nn.Linear(2500, 5)\n        \n    def forward(self, x):\n        x, _ = self.lstm(x)\n        x = x.view(x.shape[0], -1)\n        x = self.fc(x)\n        return x    \n\nclass LSTM_and_CNN(nn.Module):\n    def __init__(self):\n        super(LSTM_and_CNN, self).__init__()\n        self.model_cnn = CNN().to(device)\n        self.model_lstm = LSTM().to(device)\n    \n    def forward(self, x):\n        features = self.model_cnn(x)\n        out = self.model_lstm(features)\n        return out","metadata":{"execution":{"iopub.status.busy":"2022-08-14T09:25:57.776089Z","iopub.execute_input":"2022-08-14T09:25:57.776762Z","iopub.status.idle":"2022-08-14T09:25:57.788873Z","shell.execute_reply.started":"2022-08-14T09:25:57.776727Z","shell.execute_reply":"2022-08-14T09:25:57.787587Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = LSTM_and_CNN().to(device)\ncount_parameters(model)","metadata":{"execution":{"iopub.status.busy":"2022-08-14T09:25:59.730836Z","iopub.execute_input":"2022-08-14T09:25:59.731291Z","iopub.status.idle":"2022-08-14T09:25:59.855250Z","shell.execute_reply.started":"2022-08-14T09:25:59.731229Z","shell.execute_reply":"2022-08-14T09:25:59.854158Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Prepare before training","metadata":{}},{"cell_type":"markdown","source":"Here I load the dataset into the DataLoader and take into account, that the data is unbalanced, so I create weights for every output event, according to their occurances in the dataset.","metadata":{}},{"cell_type":"code","source":"train, val = train_test_split(df_labels, test_size=0.1, random_state=42, stratify = df_labels.event)\nbatch_size = 1\ntrain_loader = DataLoader(\n    VideoFrameDataset(train), \n    batch_size=batch_size, \n    shuffle=False, \n    num_workers=1\n)\nval_loader = DataLoader(\n    VideoFrameDataset(val), \n    batch_size=batch_size, \n    shuffle=False, \n    num_workers=1\n)\ndataloaders_dict = {\"train\": train_loader, \"val\": val_loader}\nweights = torch.zeros(5).cuda()\nweights[0] = 1/num_challenge\nweights[1] = 1/num_throwin\nweights[2] = 1/num_play\nweights[3] = 1/num_start\nweights[4] = 1/num_end\ncriterion = nn.CrossEntropyLoss(weight = weights, reduction = 'mean')","metadata":{"execution":{"iopub.status.busy":"2022-08-14T09:26:01.641752Z","iopub.execute_input":"2022-08-14T09:26:01.642110Z","iopub.status.idle":"2022-08-14T09:26:01.685148Z","shell.execute_reply.started":"2022-08-14T09:26:01.642077Z","shell.execute_reply":"2022-08-14T09:26:01.683987Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train","metadata":{}},{"cell_type":"code","source":"def train_model(model, dataloaders_dict, criterion, optimizer, num_epochs, train_loss_list = [], val_loss_list = [], train_acc_list = [], val_acc_list = []):\n    best_acc = 0.0\n    \n    for epoch in range(num_epochs):\n        model.cuda()\n        \n        for phase in ['train', 'val']:\n            if phase == 'train':\n                model.train()\n            else:\n                model.eval()\n                \n            epoch_loss = 0.0\n            epoch_acc = 0\n            \n            dataloader = dataloaders_dict[phase]\n            for item in tqdm(dataloader, leave=False):\n                frames = item[0].cuda().float()\n                classes = item[1].cuda().long()\n\n                optimizer.zero_grad()\n                \n                with torch.set_grad_enabled(phase == 'train'):\n                    output = model(frames)\n                    loss = criterion(output, classes)\n                    _, preds = torch.max(output, 1)\n\n                    if phase == 'train':\n                        loss.backward()\n                        optimizer.step()\n\n                    epoch_loss += loss.item() * len(output)\n                    epoch_acc += torch.sum(preds == classes.data)\n                    \n\n            data_size = len(dataloader.dataset)\n            epoch_loss = epoch_loss / data_size\n            epoch_acc = epoch_acc.double() / data_size\n            if phase == \"train\":\n                train_loss_list.append(epoch_loss)\n                train_acc_list.append(epoch_acc.cpu())\n            else:\n                val_loss_list.append(epoch_loss)\n                val_acc_list.append(epoch_acc.cpu())\n            print(f'Epoch {epoch + 1}/{num_epochs} | {phase:^5} | Loss: {epoch_loss:.4f} | Acc: {epoch_acc:.4f}')\n        \n        if epoch_acc > best_acc:\n            #traced = torch.jit.trace(model.cpu(), torch.rand(1, 25, 3, 128, 128))\n            #traced.save('lstm_cnn.pth')\n            torch.save(model.state_dict(), 'lstm_cnn.pth')\n            best_acc = epoch_acc\n    return train_loss_list, val_loss_list, train_acc_list, val_acc_list","metadata":{"execution":{"iopub.status.busy":"2022-08-14T09:26:04.182907Z","iopub.execute_input":"2022-08-14T09:26:04.183484Z","iopub.status.idle":"2022-08-14T09:26:04.196141Z","shell.execute_reply.started":"2022-08-14T09:26:04.183441Z","shell.execute_reply":"2022-08-14T09:26:04.194920Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"optimizer = torch.optim.AdamW(model.parameters(), lr=1e-4)\ntrain_loss_list, val_loss_list, train_acc_list, val_acc_list = train_model(model, dataloaders_dict, criterion, optimizer, 1)\n#optimizer = torch.optim.AdamW(model.parameters(), lr=1e-5)\n#train_loss_list, val_loss_list, train_acc_list, val_acc_list = train_model(model, dataloaders_dict, criterion, optimizer, 1, train_loss_list, val_loss_list, train_acc_list, val_acc_list)","metadata":{"execution":{"iopub.status.busy":"2022-08-14T09:26:07.204032Z","iopub.execute_input":"2022-08-14T09:26:07.204410Z"},"trusted":true},"execution_count":null,"outputs":[]}]}