{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np \nimport pandas as pd\nimport os\nimport matplotlib.pyplot as plt\nimport math\n\nimport pyarrow.parquet\nimport pyarrow.csv\nimport pyarrow as pa \nfrom pyarrow.parquet import ParquetFile\n\n\nimport plotly.express as px\nimport plotly.graph_objects as go\nimport gc\nfrom mpl_toolkits.mplot3d import Axes3D\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import r2_score\n\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.optim as optim\nfrom torch.utils.data import DataLoader, TensorDataset","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-03-20T01:49:03.276111Z","iopub.execute_input":"2023-03-20T01:49:03.277103Z","iopub.status.idle":"2023-03-20T01:49:09.988571Z","shell.execute_reply.started":"2023-03-20T01:49:03.277049Z","shell.execute_reply":"2023-03-20T01:49:09.987444Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# load train_data\ndef load_data(path):\n    df = pd.read_parquet(path, engine=\"pyarrow\", use_threads=True)\n    df.insert(0, df.index.name, df.index, True)\n    df = df.reset_index(drop=True)\n    return df\n\n#load train_meta\ndef load_train_meta(path, num_rows, batch_id):\n    pf = ParquetFile(path) \n    selected_rows = next(pf.iter_batches(batch_size = num_rows)) \n    df= pa.Table.from_batches([selected_rows]).to_pandas() \n    filt = (df['batch_id'].values == batch_id)\n    df = df[filt]\n    return df","metadata":{"execution":{"iopub.status.busy":"2023-03-20T01:49:28.351562Z","iopub.execute_input":"2023-03-20T01:49:28.352466Z","iopub.status.idle":"2023-03-20T01:49:28.360970Z","shell.execute_reply.started":"2023-03-20T01:49:28.352419Z","shell.execute_reply":"2023-03-20T01:49:28.359580Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"batch_id=1\nmeta_row_lens=2e5\n\nhome_dir        = \"/kaggle/input/icecube-neutrinos-in-deep-ice/\"\ntrain_data_file = f'train/batch_{batch_id}.parquet'\ntest_data_file  = \"test/batch_661.parquet\"\ntrain_meta_file = \"train_meta.parquet\"\ntest_meta_file  = \"test_meta.parquet\"\nsensor_file     = \"sensor_geometry.csv\"\nsubmission_file = \"sample_submission.parquet\"\n\n\n#6つのファイルを読み込む\ntrain_data_path = os.path.join(home_dir, train_data_file)\ndf_train_data   = load_data(train_data_path)\n\ntrain_meta_path = os.path.join(home_dir,train_meta_file)\ndf_train_meta   = load_train_meta(train_meta_path,meta_row_lens,batch_id)\n\nsensor_path     = os.path.join(home_dir,sensor_file)\ndf_sensor       = pd.read_csv (sensor_path)\n\ntest_data_path  = os.path.join(home_dir, test_data_file)\ndf_test_data    = load_data(test_data_path)\n\ntest_meta_path  = os.path.join(home_dir,test_meta_file)\ndf_test_meta    = pd.read_parquet(test_meta_path, engine=\"pyarrow\", use_threads=True)\n\nsubmission_path = os.path.join(home_dir,submission_file)\ndf_submission   = pd.read_parquet(submission_path,engine=\"pyarrow\", use_threads=True)","metadata":{"execution":{"iopub.status.busy":"2023-03-20T01:49:46.676500Z","iopub.execute_input":"2023-03-20T01:49:46.676981Z","iopub.status.idle":"2023-03-20T01:50:08.019034Z","shell.execute_reply.started":"2023-03-20T01:49:46.676943Z","shell.execute_reply":"2023-03-20T01:50:08.017377Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def angle_adjustment(df):\n    #角度が2πを超えた時,2π減らす\n    df[\"azimuth\"] = df[\"azimuth\"].apply(lambda x:x-2*np.pi if x>2*np.pi else x)\n    df[\"zenith\"]  = df[\"zenith\"].apply(lambda x:x-2*np.pi if x>2*np.pi else x)\n    \n    #角度が負の時,2π増やす\n    df[\"azimuth\"] = df[\"azimuth\"].apply(lambda x:x+2*np.pi if x<0 else x)\n    df[\"zenith\"]  = df[\"zenith\"].apply(lambda x:x+2*np.pi if x<0 else x)\n    \n    return df    ","metadata":{"execution":{"iopub.status.busy":"2023-03-20T01:52:58.993123Z","iopub.execute_input":"2023-03-20T01:52:58.993685Z","iopub.status.idle":"2023-03-20T01:52:59.003079Z","shell.execute_reply.started":"2023-03-20T01:52:58.993641Z","shell.execute_reply":"2023-03-20T01:52:59.001400Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#trainデータに対する処理\ndf_train_sensor = pd.merge(df_train_data  , df_sensor,on='sensor_id') \ndf_train        = pd.merge(df_train_sensor, df_train_meta,on=\"event_id\")\ndf_train        = df_train.rename(columns={'x': 'sensor_x', 'y': 'sensor_y','z':'sensor_z'})\ndf_train        = df_train.groupby(by=\"event_id\",as_index=False).median()\ndf_train        = angle_adjustment(df_train)\n\n#testデータに対する処理\ndf_test_sensor  = pd.merge(df_test_data  , df_sensor,on='sensor_id') \ndf_test         = pd.merge(df_test_sensor, df_test_meta,on=\"event_id\")\ndf_test         = df_test.rename(columns={'x': 'sensor_x', 'y': 'sensor_y','z':'sensor_z'})\ndf_test         = df_test.groupby(by=\"event_id\",as_index=False).median()\n\ndf_train","metadata":{"execution":{"iopub.status.busy":"2023-03-20T01:53:03.795134Z","iopub.execute_input":"2023-03-20T01:53:03.795651Z","iopub.status.idle":"2023-03-20T01:53:45.694140Z","shell.execute_reply.started":"2023-03-20T01:53:03.795606Z","shell.execute_reply":"2023-03-20T01:53:45.692348Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#モデルに使用するデータフレームを抽出\ntrain_col_list = [\"sensor_x\",\"sensor_y\",\"sensor_z\",\"charge\",\"zenith\",\"azimuth\"]\ntest_col_list  = [\"sensor_x\",\"sensor_y\",\"sensor_z\",\"charge\"]\n\ntrain_dataset  = df_train[train_col_list ]\ntest_dataset   = df_test[test_col_list]","metadata":{"execution":{"iopub.status.busy":"2023-03-20T01:53:53.613428Z","iopub.execute_input":"2023-03-20T01:53:53.613920Z","iopub.status.idle":"2023-03-20T01:53:53.629100Z","shell.execute_reply.started":"2023-03-20T01:53:53.613876Z","shell.execute_reply":"2023-03-20T01:53:53.627435Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train_dataset = train_dataset[[\"sensor_x\",\"sensor_y\",\"sensor_z\",\"charge\"]].values\ny_train_dataset = train_dataset[[\"azimuth\",\"zenith\"]].values\nX_test_dataset  = test_dataset[[\"sensor_x\",\"sensor_y\",\"sensor_z\",\"charge\"]].values\n\ndef df_normalize(df):\n    return (df - df.min()) / (df.max() - df.min())\n\n#正規化\nX_train_dataset = df_normalize(X_train_dataset)\nX_test_dataset  = df_normalize(X_test_dataset)\n\n#train:valid=8:2\ntrain_X, valid_X, train_y, valid_y = train_test_split(X_train_dataset,y_train_dataset,test_size=0.2, random_state=0)","metadata":{"execution":{"iopub.status.busy":"2023-03-20T01:53:59.224804Z","iopub.execute_input":"2023-03-20T01:53:59.225465Z","iopub.status.idle":"2023-03-20T01:53:59.285660Z","shell.execute_reply.started":"2023-03-20T01:53:59.225417Z","shell.execute_reply":"2023-03-20T01:53:59.284034Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_X = torch.FloatTensor(train_X)\ntrain_y = torch.FloatTensor(train_y)\n\nvalid_X = torch.FloatTensor(valid_X)\nvalid_y = torch.FloatTensor(valid_y)\n\ntest_X  = torch.FloatTensor(X_test_dataset)\n\ntrain_dataset = TensorDataset(train_X, train_y)\nvalid_dataset = TensorDataset(valid_X, valid_y)\n\ntrain_loader  = DataLoader(train_dataset, batch_size=8, shuffle=True)\nvalid_loader  = DataLoader(valid_dataset, batch_size=8, shuffle=True)","metadata":{"execution":{"iopub.status.busy":"2023-03-20T01:54:01.093688Z","iopub.execute_input":"2023-03-20T01:54:01.094197Z","iopub.status.idle":"2023-03-20T01:54:01.152722Z","shell.execute_reply.started":"2023-03-20T01:54:01.094137Z","shell.execute_reply":"2023-03-20T01:54:01.151543Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''モデルの定義'''\nclass Net(nn.Module):\n   def __init__(self):\n       super(Net, self).__init__()\n       self.fc1 = nn.Linear(4, 128)\n       self.fc2 = nn.Linear(128, 64)\n       self.fc3 = nn.Linear(64, 2)\n    \n   def forward(self, x):\n       x = self.fc1(x)\n       x = self.fc2(x)\n       x = self.fc3(x)\n       return x","metadata":{"execution":{"iopub.status.busy":"2023-03-20T01:54:02.912849Z","iopub.execute_input":"2023-03-20T01:54:02.913284Z","iopub.status.idle":"2023-03-20T01:54:02.922736Z","shell.execute_reply.started":"2023-03-20T01:54:02.913244Z","shell.execute_reply":"2023-03-20T01:54:02.921186Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model     = Net()\ncriterion = nn.MSELoss()\noptimizer = torch.optim.Adam(model.parameters(), lr=1e-6)","metadata":{"execution":{"iopub.status.busy":"2023-03-20T01:54:05.258354Z","iopub.execute_input":"2023-03-20T01:54:05.258864Z","iopub.status.idle":"2023-03-20T01:54:05.289342Z","shell.execute_reply.started":"2023-03-20T01:54:05.258821Z","shell.execute_reply":"2023-03-20T01:54:05.287755Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for epoch in range(10):\n   train_loss = 0\n   model.train()\n   for train_x, train_y in train_loader:\n       optimizer.zero_grad()\n       output      = model(train_x)\n       loss        = criterion(output, train_y )\n       loss.backward()\n       optimizer.step()\n       train_loss += loss.data\n   \n   valid_loss = 0\n   model.eval()\n   for valid_x, valid_y in valid_loader:\n       output      = model(valid_x)\n       loss        = criterion(output, valid_y)\n       valid_loss += loss.data\n    \n   print(epoch+1,':',train_loss,valid_loss)","metadata":{"execution":{"iopub.status.busy":"2023-03-20T01:54:06.771047Z","iopub.execute_input":"2023-03-20T01:54:06.771558Z","iopub.status.idle":"2023-03-20T01:57:26.031081Z","shell.execute_reply.started":"2023-03-20T01:54:06.771516Z","shell.execute_reply":"2023-03-20T01:57:26.029239Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_net    = model(test_X).detach()\ntest_azimuth = model_net[:,0].to('cpu').detach().numpy().copy()\ntest_zenith  = model_net[:,1].to('cpu').detach().numpy().copy()","metadata":{"execution":{"iopub.status.busy":"2023-03-18T07:04:22.799484Z","iopub.execute_input":"2023-03-18T07:04:22.799973Z","iopub.status.idle":"2023-03-18T07:04:22.806200Z","shell.execute_reply.started":"2023-03-18T07:04:22.799944Z","shell.execute_reply":"2023-03-18T07:04:22.805133Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_submission[\"azimuth\"] = test_azimuth\ndf_submission[\"zenith\"]  = test_zenith\ndf_submission            = angle_adjustment(df_submission)\ndf_submission","metadata":{"execution":{"iopub.status.busy":"2023-03-18T07:04:22.808277Z","iopub.execute_input":"2023-03-18T07:04:22.808691Z","iopub.status.idle":"2023-03-18T07:04:22.827104Z","shell.execute_reply.started":"2023-03-18T07:04:22.808650Z","shell.execute_reply":"2023-03-18T07:04:22.826033Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_submission.to_csv(\"submission.csv\", index=False)\ndf_submission.head()","metadata":{"execution":{"iopub.status.busy":"2023-03-18T07:04:49.930676Z","iopub.execute_input":"2023-03-18T07:04:49.931429Z","iopub.status.idle":"2023-03-18T07:04:49.944397Z","shell.execute_reply.started":"2023-03-18T07:04:49.931392Z","shell.execute_reply":"2023-03-18T07:04:49.943277Z"},"trusted":true},"execution_count":null,"outputs":[]}]}