{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"from pyarrow.parquet import ParquetFile\nimport pyarrow as pa \nfrom tensorflow.keras import layers\nfrom tensorflow import keras\nimport tensorflow as tf\nimport pandas as pd\nimport numpy as np\nimport gc\nimport os\nimport memory_profiler\nimport time","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-02-26T22:18:22.680542Z","iopub.execute_input":"2023-02-26T22:18:22.681608Z","iopub.status.idle":"2023-02-26T22:18:28.560024Z","shell.execute_reply.started":"2023-02-26T22:18:22.681480Z","shell.execute_reply":"2023-02-26T22:18:28.558993Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_train_data(path):\n    # load train_data\n    df = pd.read_parquet(path, engine=\"pyarrow\", use_threads=True)\n    df.insert(0, df.index.name, df.index, True)\n    df = df.reset_index(drop=True)\n    return df\n\n \ndef load_train_meta(path, num_rows, batch_id):\n    #load train_meta\n    pf = ParquetFile(path) \n    selected_rows = next(pf.iter_batches(batch_size = num_rows)) \n    df= pa.Table.from_batches([selected_rows]).to_pandas() \n    filt = (df['batch_id'].values == batch_id)\n    df = df[filt]\n    return df\n\n\ndef load_sensor_geometry(path):\n    #load sensor geometry \n    df = pd.read_csv(path)\n    df = df.rename(columns={\"x\": \"x-dimension [m]\",\n                            \"y\": \"y-dimension [m]\",\n                            \"z\": \"z-dimension [m]\"})\n    return df\n\nbatch_id = 1\nhome_dir = \"/kaggle/input/icecube-neutrinos-in-deep-ice/\"\n\n# load train_data\ntrain_file = f'train/batch_{batch_id}.parquet'\npath = os.path.join(home_dir, train_file)\ndf_train_data = load_train_data(path)\n\n#load train_meta\ntrain_meta = 'train_meta.parquet'\npath = os.path.join(home_dir, train_meta)\ndf_train_meta = load_train_meta(path, num_rows = 50e6, batch_id=batch_id)\n\n#load sensor geometry \ncsv_path = \"sensor_geometry.csv\"\npath = os.path.join(home_dir, csv_path)\ndf_sen_geom = load_sensor_geometry(path)\n","metadata":{"execution":{"iopub.status.busy":"2023-02-26T22:18:28.562286Z","iopub.execute_input":"2023-02-26T22:18:28.562897Z","iopub.status.idle":"2023-02-26T22:18:48.941671Z","shell.execute_reply.started":"2023-02-26T22:18:28.562865Z","shell.execute_reply":"2023-02-26T22:18:48.940542Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('train_meta:'), display(df_train_meta.head())\nprint('train_data:'), display(df_train_data.head())\nprint('sensor geometry:'), display(df_sen_geom.head())","metadata":{"execution":{"iopub.status.busy":"2023-02-26T22:20:30.478175Z","iopub.execute_input":"2023-02-26T22:20:30.478527Z","iopub.status.idle":"2023-02-26T22:20:30.507107Z","shell.execute_reply.started":"2023-02-26T22:20:30.478498Z","shell.execute_reply":"2023-02-26T22:20:30.506133Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Preprocessing","metadata":{}},{"cell_type":"code","source":"def filter_meta_data(meta_data,batch_id=batch_id):\n    filt = (meta_data['batch_id']== batch_id)\n    meta_data = meta_data.loc[filt]\n    return meta_data\n\ndef dataframe_for_each_eventid(dataset, meta_data, event_id):\n    # creates dataframe for each event_id\n    filt = (meta_data['event_id']== event_id)\n    start_index = meta_data.loc[filt, 'first_pulse_index'].values[0]\n    end_index = meta_data.loc[filt, 'last_pulse_index'].values[0]\n    df = dataset.iloc[start_index:end_index]\n    return df\n\ndef target_for_each_eventid(meta_data, event_id):\n    filt = (meta_data['event_id']== event_id)\n    target = list(meta_data.loc[filt, ['azimuth','zenith']].values[0])\n    return target\n\ndef deal_with_multi_pulse(df):\n    #if there is multipulse just keep the first one\n    filt = df.duplicated(subset=['sensor_id'])\n    df = df[~filt]\n    return df \n\ndef fliter_number_of_sensors(df,number_of_sensors):\n    # keep just \"number_of_sensors\" with largest charge\n    df = df.sort_values(by=['charge'],ascending=False)\n    if df.shape[0]< number_of_sensors:\n        df = pd.DataFrame()\n    elif df.shape[0]>= number_of_sensors:\n        df= df.iloc[:number_of_sensors]\n        df = df.sort_values(by=['time'])\n    return df\n\ndef add_sensors_location(df,df_sen_geom):\n    df.insert(2, \"x\",df_sen_geom.loc[df['sensor_id'],'x-dimension [m]'].values )\n    df.insert(3, \"y\", df_sen_geom.loc[df['sensor_id'],'y-dimension [m]'].values)\n    df.insert(4, \"z\", df_sen_geom.loc[df['sensor_id'],'z-dimension [m]'].values)\n    return df\n\ndef normalize_dataset(df):\n    df[\"auxiliary\"] = df[\"auxiliary\"].astype(int)\n    df.iloc[:,2:-1] = df.iloc[:,2:-1].apply(lambda x: (x-x.mean())/ x.std(), axis=0)\n    return df\n\ndef convert_df_to_list(df):\n    train_data = df.iloc[:,2:].values.tolist()\n    return train_data\n    \n\ntry:\n    del df,df_temp\n    gc.collect()\nexcept NameError:\n    pass\n\n# event_id = 24\nnum_samples = 1\nstart = 1000\nfor event_id in df_train_meta['event_id'][start:start+ num_samples]:\n#reduce size of meta_data for easier serach by filtering by batch_id\n    df_train_meta = filter_meta_data(df_train_meta,batch_id)\n    df = dataframe_for_each_eventid(df_train_data, df_train_meta, event_id)\n    df = deal_with_multi_pulse(df)\n    df = fliter_number_of_sensors(df,number_of_sensors=10)\n    if df.empty:\n        print('dataset is empty \\n')\n    else:\n        df = add_sensors_location(df,df_sen_geom)\n        df = normalize_dataset(df)\n        train_data = convert_df_to_list(df)\n        target = target_for_each_eventid(df_train_meta, event_id)\n\n    # display(df_train_meta)\n    display(df)\n    # # display(df)\n    train_data\n    target","metadata":{"execution":{"iopub.status.busy":"2023-02-26T22:25:56.814528Z","iopub.execute_input":"2023-02-26T22:25:56.814889Z","iopub.status.idle":"2023-02-26T22:25:56.858854Z","shell.execute_reply.started":"2023-02-26T22:25:56.814860Z","shell.execute_reply":"2023-02-26T22:25:56.857989Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train_dataset(dataset,meta_data,num_samples,number_of_sensors,batch_id):\n    meta_data = filter_meta_data(meta_data,batch_id)\n    train_data = []\n    targets =[]\n    count = 0\n    for event_id in meta_data['event_id'][:num_samples]:\n        target = target_for_each_eventid(meta_data, event_id)\n        targets.append(list(target)) # create the target data\n        \n        df = dataframe_for_each_eventid(dataset, meta_data, event_id) # df contaions all data related to the specefic 'event_id'\n        df = deal_with_multi_pulse(df)\n        df = fliter_number_of_sensors(df,number_of_sensors)\n        if df.empty:\n            pass\n        else:\n            df = add_sensors_location(df,df_sen_geom)\n            df = normalize_dataset(df)\n            data = convert_df_to_list(df)\n            train_data.append(data) \n            count +=1\n            if count % 1000 == 0:\n              print(f'Number of samples : {len(train_data)}')\n              \n    return train_data, targets\ntry:\n    del targets\n    gc.collect()\nexcept NameError:\n    pass\n\nnum_samples = 20000  # Total number of samples\nnumber_of_sensors = 10 # number of sensors at each sample\ntrain_data, targets = train_dataset(df_train_data, \n                                    df_train_meta, \n                                    num_samples = num_samples,\n                                    number_of_sensors= number_of_sensors,\n                                    batch_id=batch_id)","metadata":{"execution":{"iopub.status.busy":"2023-02-26T22:28:14.406505Z","iopub.execute_input":"2023-02-26T22:28:14.406858Z","iopub.status.idle":"2023-02-26T22:28:34.831746Z","shell.execute_reply.started":"2023-02-26T22:28:14.406829Z","shell.execute_reply":"2023-02-26T22:28:34.830396Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# separate train and validation and test datas\nsamples = np.asarray(train_data)\ntargets = np.asarray(targets)\n\nnum_train_samples = int(0.7 * num_samples)\nnum_val_samples = int(0.25 * num_samples)\nnum_test_samples = num_samples - num_train_samples - num_val_samples\n\nprint(\"num of train samples:\", num_train_samples)\nprint(\"num of val samples:\", num_val_samples)\nprint(\"num of test samples:\", num_test_samples)\n\ntrain_samples = samples[:num_train_samples]\ntrain_targets = targets[:num_train_samples]\n\nval_samples = samples[num_train_samples: num_train_samples + num_val_samples]\nval_targets = targets[num_train_samples: num_train_samples + num_val_samples]\n\ntest_samples = samples[num_train_samples + num_val_samples:]\ntest_targets = targets[num_train_samples + num_val_samples:]\n\ntrain_samples= train_samples.astype(\"float32\")\nval_samples = val_samples.astype(\"float32\")\ntest_samples = test_samples.astype(\"float32\")\n\ntrain_targets = np.asarray(train_targets).astype(\"float32\")\nval_targets = np.asarray(val_targets).astype(\"float32\")\ntest_targets = np.asarray(test_targets).astype(\"float32\")\n\n# del df_train_data, df_train_meta,train_data, targets\n# gc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-02-26T22:28:45.891813Z","iopub.execute_input":"2023-02-26T22:28:45.892184Z","iopub.status.idle":"2023-02-26T22:28:45.926892Z","shell.execute_reply.started":"2023-02-26T22:28:45.892147Z","shell.execute_reply":"2023-02-26T22:28:45.925758Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Build and Train the Model","metadata":{}},{"cell_type":"code","source":"tf.keras.backend.clear_session()\ndef model_build_and_train(train_samples,train_targets,\n                          val_samples,val_targets,\n                          number_of_sensors):\n    \n    inputs = keras.Input(shape=(number_of_sensors,6))\n    x = layers.Flatten()(inputs)\n    # x = layers.Dense(256, activation=\"relu\")(x)\n    # x = layers.Dropout(0.5)(x)\n    # x = layers.Dense(128, activation=\"relu\")(x)\n    # x = layers.Dropout(0.5)(x)\n#     x = layers.Dense(64, activation=\"relu\")(x)\n#     x = layers.Dropout(0.5)(x)\n    x = layers.Dense(16, activation=\"relu\")(x)\n    x = layers.Dropout(0.5)(x)\n    outputs = layers.Dense(2)(x)\n    model= keras.Model(inputs=inputs, outputs=outputs)\n    \n    \n    callbacks_list = [keras.callbacks.ModelCheckpoint(\n                 filepath=\"checkpoint_path.keras\",\n                 monitor=\"val_loss\",\n                save_best_only=True,\n                )\n            ]\n    model.compile(optimizer=keras.optimizers.RMSprop(1e-6)\n                  , loss=\"mse\", metrics=[\"mae\"])\n\n    history = model.fit(train_samples, train_targets,\n                        validation_data=(val_samples, val_targets),\n                        callbacks=callbacks_list,\n                        epochs=20,\n                        batch_size=64, \n                        verbose=0)\n    return model, history\n\nmodel, history = model_build_and_train(train_samples,train_targets,\n                                       val_samples,val_targets,\n                                       number_of_sensors= number_of_sensors)\n\n# keras.utils.plot_model(model, \"ticket_classifier.png\")\n# keras.utils.plot_model(\n#     model, \"ticket_classifier_with_shape_info.png\", show_shapes=True)\n# model.summary()","metadata":{"execution":{"iopub.status.busy":"2023-02-26T22:28:55.529635Z","iopub.execute_input":"2023-02-26T22:28:55.529994Z","iopub.status.idle":"2023-02-26T22:28:58.765242Z","shell.execute_reply.started":"2023-02-26T22:28:55.529963Z","shell.execute_reply":"2023-02-26T22:28:58.764220Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Evalulate the Model on Validation and Test samples","metadata":{}},{"cell_type":"code","source":"model = keras.models.load_model(\"checkpoint_path.keras\") \nprint('Model performance on Validation samples: ')\nmodel.evaluate(val_samples,val_targets)\nprint('Model performance on Test samples: ')\nmodel.evaluate(test_samples,test_targets)\nprint('Prediction vs True value')\nprint(f'prediction:{model.predict(test_samples)[0]}, True value: {test_targets[0]}')\nmodel.predict(test_samples)[0:10]","metadata":{"execution":{"iopub.status.busy":"2023-02-26T22:29:09.530191Z","iopub.execute_input":"2023-02-26T22:29:09.530581Z","iopub.status.idle":"2023-02-26T22:29:09.927953Z","shell.execute_reply.started":"2023-02-26T22:29:09.530552Z","shell.execute_reply":"2023-02-26T22:29:09.926695Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nplt.figure(figsize=(25,6))\nplt.suptitle(\"Network Performance\", fontsize=30)\nplt.subplots_adjust(wspace=0.3, hspace=0.4)\nhistory_dict = history.history\nkeys = ['loss', 'mae', 'val_loss', 'val_mae']\nlabel=[\"Training loss\", \"Validation loss\",\"Training mae\", \"Validation mae\"]\n\nfor i in range(2):\n  plt.subplot(1,2,i+1)\n  plt.ylabel(keys[i])\n  plt.xlabel(\"Epochs\")\n  plt.plot(range(1, len(history_dict[keys[i]]) + 1), history_dict[keys[i]], \"bo\", label=\"Training loss\") \n  plt.plot(range(1, len(history_dict[keys[i+2]]) + 1), history_dict[keys[i+2]], \"b\", label=\"Validation loss\") \n  plt.legend()","metadata":{"execution":{"iopub.status.busy":"2023-02-26T22:29:12.338312Z","iopub.execute_input":"2023-02-26T22:29:12.338668Z","iopub.status.idle":"2023-02-26T22:29:12.656752Z","shell.execute_reply.started":"2023-02-26T22:29:12.338639Z","shell.execute_reply":"2023-02-26T22:29:12.656028Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Read test metadata","metadata":{}},{"cell_type":"code","source":"# load test data: batch_661.parquet\ndef load_test_data(path):\n    df = pd.read_parquet(path, engine=\"pyarrow\", use_threads=True)\n    df.insert(0, df.index.name, df.index, True)\n    df = df.reset_index(drop=True)\n    return df\n\n#load test_meta \ndef load_test_meta(path):\n    df = pd.read_parquet(path, engine=\"pyarrow\", use_threads=True)\n    return df\n\ntest_meta = 'test_meta.parquet'\npath = os.path.join(home_dir , test_meta)\ndf_test_meta = load_test_meta(path)\ndisplay(df_test_meta.head())\n\ntest_file = f'test/batch_661.parquet'\npath = os.path.join(home_dir , test_file)\ndf_test_data = load_test_data(path)\ndisplay(df_test_data.head())","metadata":{"execution":{"iopub.status.busy":"2023-02-26T22:29:58.097403Z","iopub.execute_input":"2023-02-26T22:29:58.097773Z","iopub.status.idle":"2023-02-26T22:29:58.139510Z","shell.execute_reply.started":"2023-02-26T22:29:58.097743Z","shell.execute_reply":"2023-02-26T22:29:58.138447Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Read test data and predict batch-by-batch","metadata":{}},{"cell_type":"code","source":"def preprocess_test_data(dataset, meta_data,number_of_sensors):\n    event_ids = df_test_meta['event_id']\n    test_data = []\n    count = 0\n    for event_id in meta_data['event_id']:\n        df = dataframe_for_each_eventid(dataset, meta_data, event_id) # df contaions all data related to the specefic 'event_id'\n        df = deal_with_multi_pulse(df)\n        df = fliter_number_of_sensors(df,number_of_sensors)\n        if df.empty:\n            pass\n        else:\n            df = add_sensors_location(df,df_sen_geom)\n            df = normalize_dataset(df)\n            data = convert_df_to_list(df)\n            test_data.append(data) \n    return test_data,event_ids\n\nnumber_of_sensors = 10 # number of sensors at each sample\ntest_data,test_event_ids = preprocess_test_data(df_test_data,\n                                                df_test_meta,\n                                                number_of_sensors=number_of_sensors)\n\n# del df_test_data, df_test_meta, test_data\n# gc.collect()\n\ntest_ptredict = model.predict(test_data)\ntest_ptredict\n","metadata":{"execution":{"iopub.status.busy":"2023-02-26T22:32:24.743836Z","iopub.execute_input":"2023-02-26T22:32:24.744221Z","iopub.status.idle":"2023-02-26T22:32:24.849169Z","shell.execute_reply.started":"2023-02-26T22:32:24.744190Z","shell.execute_reply":"2023-02-26T22:32:24.848193Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Submit","metadata":{}},{"cell_type":"code","source":"test_event_id = list(test_event_ids)\ntest_azimuth = list(test_ptredict[:,0])\ntest_zenith = list(test_ptredict[:,1])\n\ntest_result_dict = {\n    \"event_id\": test_event_id,\n    \"azimuth\": test_azimuth,\n    \"zenith\": test_zenith,\n}\n\ntest_result_df = pd.DataFrame(test_result_dict)\ntest_result_df = test_result_df.sort_values(by=['event_id'])\n\ntest_result_df.to_csv(\"submission.csv\", index=False)\ntest_result_df.head()","metadata":{"execution":{"iopub.status.busy":"2023-02-26T22:33:13.207010Z","iopub.execute_input":"2023-02-26T22:33:13.207362Z","iopub.status.idle":"2023-02-26T22:33:13.222867Z","shell.execute_reply.started":"2023-02-26T22:33:13.207332Z","shell.execute_reply":"2023-02-26T22:33:13.221523Z"},"trusted":true},"execution_count":null,"outputs":[]}]}