{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Introduction\n\nHi everyone! I am a newbird to AI and data science. \n\nI'd love to learning the construction of AI structures by this competition.\n\nIn my project, I want to implement a LSTM network to fitting the data for each period of time so that the network can detect neutrino events and their azimuth and zenith angles in real-time. \n\nActually, I don't know whether it is realizable or efficient. My goal is to really learn something, and provide some ideas if possible.\n\n# Idea\n\nI considered the batch of data as whole time series, rather than groups of events, in order to make it \"real-time\".\n\nThe LSTM neural network takes time, x, y, z, charge as inputs, and take the auxiliary, azimuth and zenith angless as outputs, so that it can classify whether the datapoints are auxiliary or not. And fitting the direction information.\n\n# Implementation\n\nI am currently using a single LSTM with pytorch and then connecting it to a linear net. However, the code I wrote may have problems (especially I have not understand the LSTM output structures), so the training performance is weird. The loss fluctuate from 1 to 5.","metadata":{}},{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","jupyter":{"source_hidden":true,"outputs_hidden":true},"execution":{"iopub.status.busy":"2023-01-31T09:53:53.419726Z","iopub.execute_input":"2023-01-31T09:53:53.420737Z","iopub.status.idle":"2023-01-31T09:53:53.568942Z","shell.execute_reply.started":"2023-01-31T09:53:53.420624Z","shell.execute_reply":"2023-01-31T09:53:53.567700Z"},"collapsed":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport random\nimport math\nfrom pathlib import Path\nfrom collections import Counter\nimport torch\n\nfrom tqdm import tqdm\nimport pandas as pd\nimport numpy as np\nimport pyarrow.parquet as pq\n\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport plotly as py\nimport plotly.express as px\nimport plotly.graph_objects as go\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.optim as optim\nfrom torch.utils.data import DataLoader, TensorDataset\n\nsns.set_style(\"darkgrid\")\ndata_path = Path(\"/kaggle/input/icecube-neutrinos-in-deep-ice/\")\npy.offline.init_notebook_mode(connected = True)\n\ndef read_batch(path,index):\n    return pd.read_parquet(path/\"batch_{}.parquet\".format(index))\n\n## read the required data from files\nsensor_geometry = pd.read_csv(data_path/\"sensor_geometry.csv\")\ntrain_meta = pd.read_parquet(data_path/\"train_meta.parquet\")\ntest_meta = pd.read_parquet(data_path/\"test_meta.parquet\")","metadata":{"execution":{"iopub.status.busy":"2023-01-31T09:53:53.571329Z","iopub.execute_input":"2023-01-31T09:53:53.572113Z","iopub.status.idle":"2023-01-31T09:54:41.710443Z","shell.execute_reply.started":"2023-01-31T09:53:53.572064Z","shell.execute_reply":"2023-01-31T09:54:41.708862Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_dataset(path,index):\n    batch = read_batch(path, index)\n    batch = batch.reset_index().merge(sensor_geometry, how='left', on='sensor_id', left_index=False).set_index('event_id')\n    batch = batch.reset_index().merge(train_meta, how='left', on='event_id', left_index=False).set_index('event_id')\n    X = batch[[\"time\", \"charge\", \"x\", \"y\", \"z\"]]\n    y = batch[[\"auxiliary\",\"azimuth\",\"zenith\"]]\n    y[\"auxiliary\"] *= 1 # y.auxiliary.replace({True: 1, False: 0})\n    X = torch.from_numpy(X.values)\n    y = torch.from_numpy(y.values)\n    return X, y","metadata":{"execution":{"iopub.status.busy":"2023-01-31T09:54:41.712561Z","iopub.execute_input":"2023-01-31T09:54:41.713004Z","iopub.status.idle":"2023-01-31T09:54:41.721452Z","shell.execute_reply.started":"2023-01-31T09:54:41.712950Z","shell.execute_reply":"2023-01-31T09:54:41.720335Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class NeutrinoLSTM(nn.Module):\n    def __init__(self, input_dim, hidden_dim, layer_dim, output_dim):\n        super(NeutrinoLSTM, self).__init__()\n        self.input_dim = input_dim\n        self.hidden_dim = hidden_dim\n        self.layer_dim = layer_dim\n        self.output_dim = output_dim\n        self.lstm = nn.LSTM(input_dim, hidden_dim, layer_dim)\n        self.out = nn.Linear(hidden_dim, output_dim)\n#         self.out1 = nn.Linear(input_dim, 100)\n#         self.relu1 = nn.ReLU()\n#         self.out2 = nn.Linear(100, 100)\n#         self.relu2 = nn.ReLU()\n#         self.out3 = nn.Linear(100, output_dim)\n        \n    def forward(self, x):\n        lstm_out, _ = self.lstm(x.view(len(x), 1, -1))\n        value = self.out(lstm_out.view(len(x), -1))\n#         value = self.out1(x)\n#         value = self.relu1(value)\n#         value = self.out2(value)\n#         value = self.relu2(value)\n#         value = self.out3(value)\n        return value","metadata":{"execution":{"iopub.status.busy":"2023-01-31T09:54:41.723984Z","iopub.execute_input":"2023-01-31T09:54:41.724370Z","iopub.status.idle":"2023-01-31T09:54:41.746844Z","shell.execute_reply.started":"2023-01-31T09:54:41.724321Z","shell.execute_reply":"2023-01-31T09:54:41.744901Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## NN and training parameters\ninput_dim = 5\nhidden_dim = 32\nlayer_dim = 4\noutput_dim = 3\nlr = 0.001\n\n## training\nmodel = NeutrinoLSTM(input_dim, hidden_dim, layer_dim, output_dim)\nmodel = model.double()\nloss = nn.CrossEntropyLoss()\noptimizer = torch.optim.SGD(model.parameters(), lr=lr) \n\nloss_list = []\ntest_loss_list = []\niteration_list = []\naccuracy_list = []\nnum_epoch = 0","metadata":{"execution":{"iopub.status.busy":"2023-01-31T09:54:41.748817Z","iopub.execute_input":"2023-01-31T09:54:41.749511Z","iopub.status.idle":"2023-01-31T09:54:41.770024Z","shell.execute_reply.started":"2023-01-31T09:54:41.749467Z","shell.execute_reply":"2023-01-31T09:54:41.768731Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_test, y_test = get_dataset(data_path/\"test\", 661)\ntest = TensorDataset(X_test,y_test)\ntest_loader = DataLoader(test, batch_size = 200, shuffle = False)","metadata":{"execution":{"iopub.status.busy":"2023-01-31T09:54:41.771712Z","iopub.execute_input":"2023-01-31T09:54:41.772205Z","iopub.status.idle":"2023-01-31T09:57:06.056075Z","shell.execute_reply.started":"2023-01-31T09:54:41.772166Z","shell.execute_reply":"2023-01-31T09:57:06.053654Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"count = 0\nprint(\"Start training\")\nfor epoch in range(1, num_epoch+1):\n    print(\"epoch {} is starting\".format(epoch))\n    X_train, y_train = get_dataset(data_path/\"train\", epoch)\n#     X_train, y_train = get_dataset(data_path/\"test\", 661)\n    train = TensorDataset(X_train, y_train)\n    train_loader = DataLoader(train, batch_size = 1000, shuffle = False)\n    print(\"Dataset construction finished\")\n    for i, (X, y) in enumerate(train_loader):\n        optimizer.zero_grad()\n        pred = model(X)\n        error = loss(pred, y)\n        error.backward()\n        optimizer.step()\n        count += 1\n        \n        if count % 100 == 0:\n            # Calculate Accuracy         \n            correct = 0\n            total = 0\n            test_loss = 0\n            with torch.no_grad():\n                for X, y in test_loader:\n                    pred = model(X)\n                    test_loss += loss(pred, y).item()\n                    total += y.size(0)\n                    correct += (pred[0].round() == y[0].round()).type(torch.float).sum().item()\n            accuracy = 100 * correct / total\n            test_loss /= total\n            loss_list.append(error)\n            test_loss_list.append(test_loss)\n            iteration_list.append(count)\n            accuracy_list.append(accuracy)\n            print('Iteration: {}. Loss: {}. Accuracy: {}. TestLoss: {}.'.format(count, error, accuracy, test_loss))\n    ","metadata":{"execution":{"iopub.status.busy":"2023-01-31T09:57:06.058666Z","iopub.execute_input":"2023-01-31T09:57:06.060068Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ssub = pd.read_parquet(os.path.join(\"/kaggle/input/icecube-neutrinos-in-deep-ice\", \"sample_submission.parquet\"))\nssub.set_index(\"event_id\", inplace=True)\ndisplay(ssub.head())\nssub.fillna(0).to_csv('submission.csv', index=True)\n!head submission.csv","metadata":{},"execution_count":null,"outputs":[]}]}