{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# G2Net\nSolution for the https://www.kaggle.com/c/g2net-gravitational-wave-detection challenge.\n\n### Inspiration\nThere are a number of other interesting solutions for this task:\n- Exploratory Analysis: https://www.kaggle.com/ihelon/g2net-eda-and-modeling\n- Baseline: https://www.kaggle.com/yasufuminakama/g2net-efficientnet-b7-baseline-inference\n- Training: https://www.kaggle.com/yasufuminakama/g2net-efficientnet-b7-baseline-training\n\n### Library Imports\n","metadata":{}},{"cell_type":"code","source":"# Python Library Imports\n%matplotlib inline\nimport os\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport random\nfrom tqdm import tnrange, tqdm_notebook\n\n# might also need:\nimport torch\nimport json\nimport collections\nimport cv2\n\n# Notes on Kaggle Environment:\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-07-19T04:36:23.501836Z","iopub.execute_input":"2021-07-19T04:36:23.502413Z","iopub.status.idle":"2021-07-19T04:36:25.700247Z","shell.execute_reply.started":"2021-07-19T04:36:23.502329Z","shell.execute_reply":"2021-07-19T04:36:25.697817Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# input folders\npath_input = '/kaggle/input/g2net-gravitational-wave-detection/'\nfolder_train = 'train'\nfolder_test = 'test'\n\n# reading submission and training label files. \npath_output = '/kaggle/working'\nsubmission_df = pd.read_csv(path_input + 'sample_submission.csv')\ntrain_df = pd.read_csv(path_input + 'training_labels.csv')\ntrain_df","metadata":{"execution":{"iopub.status.busy":"2021-07-19T04:37:02.649831Z","iopub.execute_input":"2021-07-19T04:37:02.650200Z","iopub.status.idle":"2021-07-19T04:37:03.064749Z","shell.execute_reply.started":"2021-07-19T04:37:02.650176Z","shell.execute_reply":"2021-07-19T04:37:03.063546Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Data Exploration \nHere, we will explore the form and shape of the training data provided, with a focus on how we can best train a model on it. \n\nIn the G2Net Competition, we are provided with a time series data set, containing simulated measurements of gravitational waves. These measurements are simulated at three independent graviational wave interferometers: LIGO Hanford, LIGO Livingston, and Virgo. \n\nEach time series contains either detector noise, or detector noise with a simulated gravitational wave signal applied on top. Our model must identify when a signal is present in the data (target = 1). ","metadata":{}},{"cell_type":"code","source":"# visualisation code referenced from: \n# https://www.kaggle.com/ihelon/g2net-eda-and-modeling\n\ndef convert_id_to_path(_id):\n    return f\"{path_input}{folder_train}/{_id[0]}/{_id[1]}/{_id[2]}/{_id}.npy\"\n\ndef visualize_sample( _id, target, \n    colors=(\"black\", \"red\", \"green\"), \n    signal_names=(\"LIGO Hanford\", \"LIGO Livingston\", \"Virgo\")):\n    path = convert_id_to_path(_id)\n    x = np.load(path)\n    print(x.shape)\n    plt.figure(figsize=(16, 7))\n    for i in range(3):\n        plt.subplot(4, 1, i + 1)\n        plt.plot(x[i], color=colors[i])\n        plt.legend([signal_names[i]], fontsize=12, loc=\"lower right\")\n        \n        plt.subplot(4, 1, 4)\n        plt.plot(x[i], color=colors[i])\n    \n    plt.subplot(4, 1, 4)\n    plt.legend(signal_names, fontsize=12, loc=\"lower right\")\n\n    plt.suptitle(f\"id: {_id} target: {target}\", fontsize=16)\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2021-07-19T04:37:06.059011Z","iopub.execute_input":"2021-07-19T04:37:06.059461Z","iopub.status.idle":"2021-07-19T04:37:06.070247Z","shell.execute_reply.started":"2021-07-19T04:37:06.059432Z","shell.execute_reply":"2021-07-19T04:37:06.069283Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in random.sample(train_df.index.tolist(), 2):\n    _id = train_df.iloc[i][\"id\"]\n    target = train_df.iloc[i][\"target\"]\n\n    visualize_sample(_id, target)","metadata":{"execution":{"iopub.status.busy":"2021-07-19T04:37:10.512663Z","iopub.execute_input":"2021-07-19T04:37:10.513113Z","iopub.status.idle":"2021-07-19T04:37:11.709378Z","shell.execute_reply.started":"2021-07-19T04:37:10.513083Z","shell.execute_reply":"2021-07-19T04:37:11.708434Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Correlation Between features\nVisualise the correlation between the three inputs:","metadata":{}},{"cell_type":"code","source":"# correlation\n\nfor i in random.sample(train_df.index.tolist(), 2):\n    _id = train_df.iloc[i][\"id\"]\n    target = train_df.iloc[i][\"target\"]\n    print(f\"target = {target} !!\")\n    path = convert_id_to_path(_id)\n    x = np.load(path)\n    xt = x.transpose()\n    df = pd.DataFrame(data=xt[:100, :], columns=[\"LIGO Hanford\", \"LIGO Livingston\", \"Virgo\"])\n    #df = pd.DataFrame(data=numpy_data, index=[\"row1\", \"row2\"], columns=[\"column1\", \"column2\"])\n    #df = pd.DataFrame(data=x[:, :5], index=[\"LIGO Hanford\", \"LIGO Livingston\", \"Virgo\"])\n    \n    sns.pairplot(df)\n    plt.show()\n","metadata":{"execution":{"iopub.status.busy":"2021-07-19T04:38:34.832929Z","iopub.execute_input":"2021-07-19T04:38:34.833302Z","iopub.status.idle":"2021-07-19T04:38:38.498308Z","shell.execute_reply.started":"2021-07-19T04:38:34.833274Z","shell.execute_reply":"2021-07-19T04:38:38.497188Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from time import sleep\nfrom tqdm.notebook import tqdm\n\nfor i in tqdm(random.sample(train_df.index.tolist(), 2)):\n    _id = train_df.iloc[i][\"id\"]\n    target = train_df.iloc[i][\"target\"]\n    print(f\"target = {target} !!\")\n    path = convert_id_to_path(_id)\n    x = np.load(path)\n    xt = x.transpose()\n    df = pd.DataFrame(data=xt[:, :], columns=[\"LIGO Hanford\", \"LIGO Livingston\", \"Virgo\"])\n    #df = pd.DataFrame(data=numpy_data, index=[\"row1\", \"row2\"], columns=[\"column1\", \"column2\"])\n    #df = pd.DataFrame(data=x[:, :5], index=[\"LIGO Hanford\", \"LIGO Livingston\", \"Virgo\"])\n\n    sns.pairplot(df)\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2021-07-19T04:38:38.499714Z","iopub.execute_input":"2021-07-19T04:38:38.499980Z","iopub.status.idle":"2021-07-19T04:38:42.868743Z","shell.execute_reply.started":"2021-07-19T04:38:38.499953Z","shell.execute_reply":"2021-07-19T04:38:42.867451Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Observations:**\n- The LIGO Detectors have a higher correlation with eachother than they do with the Virgo Detector. This could be related to their geographical proximity. Since the LIGO detectors are closer, and the surface of the Earth is curved, then the LIGO detectors will be measuring the same polarity of spacetime. This means correlations MIGHT be less likely to be noise and more likely to be related to gravitational waves.\n    - Perhaps the model might increase (or decrease) the LIGO weightings if they are similar?\n- can consider using evaluations from physics/geography to quantify the data from another perspective. This can help with evaluating the raw data. ","metadata":{}},{"cell_type":"markdown","source":"### Data Feature Extraction?\nCould human-select and extract features from the data, like the \"frequency\", \"frequency deviation\", \"amplitude\", \"amplitude deviation\".","metadata":{}},{"cell_type":"markdown","source":"## Baseline: Logistic Regression \nHere, we will create a baseline linear regression model on a randomly selected portion of the training set. ","metadata":{}},{"cell_type":"markdown","source":"### Extracting Training Sets\nHere, we begin by extracting 50 randomly selected samples to a 3D array of shape (50, 3, 4096). ","metadata":{}},{"cell_type":"code","source":"# Returns randomly selected data samples from Training set. \n# \n# \ndef sample_data(training_size=50):\n    y_data = np.full(training_size, -1)\n    X_data = np.zeros((training_size, 3, 4096))\n    \n    count = 0 # increments. \n    for i in random.sample(train_df.index.tolist(), training_size):\n        img_id = train_df.iloc[i][\"id\"]\n        y_data[count] = train_df.iloc[i][\"target\"]\n        path = convert_id_to_path(img_id)\n        X_data[count] = np.load(path)\n        \n        count += 1\n        # stack X. \n    return (y_data, X_data)\n        \n\ny_train, X_train = sample_data(100)\n\nprint(X_train.shape)\nprint(y_train.shape)\nprint(X_train)","metadata":{"execution":{"iopub.status.busy":"2021-07-19T04:38:47.820967Z","iopub.execute_input":"2021-07-19T04:38:47.821415Z","iopub.status.idle":"2021-07-19T04:38:48.894979Z","shell.execute_reply.started":"2021-07-19T04:38:47.821390Z","shell.execute_reply":"2021-07-19T04:38:48.893871Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Normalising Data\nWe must normalise the data so it is in the range (0..1). ","metadata":{}},{"cell_type":"code","source":"# Normalise the Data in X_data (100, 3, 4096). \nfrom sklearn import preprocessing\n\n# one line scale: https://stackoverflow.com/questions/50125844/how-to-standard-scale-a-3d-matrix\nscaler = preprocessing.StandardScaler()\nX_scaled = scaler.fit_transform(X_train.reshape(-1, X_train.shape[-1])).reshape(X_train.shape)\nX_scaled\n# this scaling preserves the differences between channels in volumes.\n# these differences may be useful?\n\n# but we might want to individually scale channels. an alternate scaling \n# is being developed below: TODO\n\nscalers = {}\nX_scaled_2 = np.zeros_like(X_train)\nfor i in range(X_train.shape[1]):\n    scalers[i] = preprocessing.StandardScaler()\n    X_scaled_2[:, i, :] = scalers[i].fit_transform(X_train[:, i, :]) \n\nprint(X_scaled_2)","metadata":{"execution":{"iopub.status.busy":"2021-07-19T04:38:52.107655Z","iopub.execute_input":"2021-07-19T04:38:52.108002Z","iopub.status.idle":"2021-07-19T04:38:52.304718Z","shell.execute_reply.started":"2021-07-19T04:38:52.107973Z","shell.execute_reply":"2021-07-19T04:38:52.303441Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Visualising Scaled Data\nWork out which scaling is better: unscaled, scaled_across, scaled_individually.\n","metadata":{}},{"cell_type":"code","source":"def visualize_scaled_sample( x, target, colors=(\"black\", \"red\", \"green\"), signal_names=(\"LIGO Hanford\", \"LIGO Livingston\", \"Virgo\")):    \n    plt.figure(figsize=(16, 7))\n    for i in range(3):\n        plt.subplot(4, 1, i + 1)\n        plt.plot(x[i], color=colors[i])\n        plt.legend([signal_names[i]], fontsize=12, loc=\"lower right\")\n        \n        plt.subplot(4, 1, 4)\n        plt.plot(x[i], color=colors[i])\n    \n    plt.subplot(4, 1, 4)\n    plt.legend(signal_names, fontsize=12, loc=\"lower right\")\n\n    plt.suptitle(f\"id: {_id} target: {target}\", fontsize=16)\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2021-07-19T04:38:56.528553Z","iopub.execute_input":"2021-07-19T04:38:56.529000Z","iopub.status.idle":"2021-07-19T04:38:56.537163Z","shell.execute_reply.started":"2021-07-19T04:38:56.528975Z","shell.execute_reply":"2021-07-19T04:38:56.535931Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"visualize_scaled_sample(X_train[3], y_train[3])\nvisualize_scaled_sample(X_scaled[3], y_train[3])\nvisualize_scaled_sample(X_scaled_2[3], y_train[3])\n\nvisualize_scaled_sample(X_train[12], y_train[12])\nvisualize_scaled_sample(X_scaled[12], y_train[12])\nvisualize_scaled_sample(X_scaled_2[12], y_train[12])","metadata":{"execution":{"iopub.status.busy":"2021-07-19T04:39:00.773947Z","iopub.execute_input":"2021-07-19T04:39:00.774388Z","iopub.status.idle":"2021-07-19T04:39:03.218756Z","shell.execute_reply.started":"2021-07-19T04:39:00.774358Z","shell.execute_reply":"2021-07-19T04:39:03.217693Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}