{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\n# import numpy as np # linear algebra\n# import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\n# import os\n# for dirname, _, filenames in os.walk('/kaggle/input'):\n#     for filename in filenames:\n#         print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-08-17T10:31:13.466398Z","iopub.execute_input":"2021-08-17T10:31:13.466782Z","iopub.status.idle":"2021-08-17T10:31:13.470910Z","shell.execute_reply.started":"2021-08-17T10:31:13.466681Z","shell.execute_reply":"2021-08-17T10:31:13.470078Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"first trial \n","metadata":{}},{"cell_type":"code","source":"\n# # Python library to interact with the file system.\n# import os\n\n# #Visualization\n# import plotly.express as px\n\n# # Software library written for data manipulation and analysis.\n# import pandas as pd\n# from tqdm import tqdm as tq\n# tq.pandas()\n\n# # fastai library for computer vision tasks\n# # from fastai.vision.all import *\n# # from fastai.callback.wandb import *\n\n# # Developing and training neural network based deep learning models.\n# # import torch\n# # from torch import nn\n\n# # Python library for image augmentation\n# #import albumentations as A\n\n# # import wandb\n# # wandb.login()\n# #add Codeadd Markdown\n# # ls ../input/g2net-gravitational-wave-detection/\n# # add Codeadd Markdown\n# # path = Path('../kaggle/input/g2net-gravitational-wave-detection/')\n# # add Codeadd Markdown\n# # train_df = pd.read_csv(path/'training_labels.csv')\n# # train_df\n# # add Codeadd Markdown\n# # train_df['path'] = train_df.id.\n# # (lambda x: str(path)+'/train/'+'/'.join([char for char in x[:3]])+'/'+x+'.npy')","metadata":{"execution":{"iopub.status.busy":"2021-08-17T10:31:13.481600Z","iopub.execute_input":"2021-08-17T10:31:13.482018Z","iopub.status.idle":"2021-08-17T10:31:13.485477Z","shell.execute_reply.started":"2021-08-17T10:31:13.481991Z","shell.execute_reply":"2021-08-17T10:31:13.484857Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# cd /kaggle/input/g2net-gravitational-wave-detection/","metadata":{"execution":{"iopub.status.busy":"2021-08-17T10:31:13.489036Z","iopub.execute_input":"2021-08-17T10:31:13.489518Z","iopub.status.idle":"2021-08-17T10:31:13.495025Z","shell.execute_reply.started":"2021-08-17T10:31:13.489492Z","shell.execute_reply":"2021-08-17T10:31:13.494031Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ls /kaggle/input/g2net-gravitational-wave-detection/","metadata":{"execution":{"iopub.status.busy":"2021-08-17T10:31:13.496271Z","iopub.execute_input":"2021-08-17T10:31:13.496602Z","iopub.status.idle":"2021-08-17T10:31:13.506227Z","shell.execute_reply.started":"2021-08-17T10:31:13.496579Z","shell.execute_reply":"2021-08-17T10:31:13.505303Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train_labels = pd.read_csv(\"training_labels.csv\")\n# train_labels.head()","metadata":{"execution":{"iopub.status.busy":"2021-08-17T10:31:13.507806Z","iopub.execute_input":"2021-08-17T10:31:13.508365Z","iopub.status.idle":"2021-08-17T10:31:13.519309Z","shell.execute_reply.started":"2021-08-17T10:31:13.508310Z","shell.execute_reply":"2021-08-17T10:31:13.518557Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# cd /kaggle/input/g2net-gravitational-wave-detection/train/0/0/0/","metadata":{"execution":{"iopub.status.busy":"2021-08-17T10:31:13.520694Z","iopub.execute_input":"2021-08-17T10:31:13.520994Z","iopub.status.idle":"2021-08-17T10:31:13.530422Z","shell.execute_reply.started":"2021-08-17T10:31:13.520970Z","shell.execute_reply":"2021-08-17T10:31:13.529518Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n# import os\n\n# path, dirs, files = next(os.walk(\"/kaggle/input/g2net-gravitational-wave-detection/train/0/0/0/\"))\n# file_count = len(files)\n# files","metadata":{"execution":{"iopub.status.busy":"2021-08-17T10:31:13.531857Z","iopub.execute_input":"2021-08-17T10:31:13.532133Z","iopub.status.idle":"2021-08-17T10:31:13.541443Z","shell.execute_reply.started":"2021-08-17T10:31:13.532090Z","shell.execute_reply":"2021-08-17T10:31:13.540462Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# x=np.load(\"0002cc5125.npy\")\n# x","metadata":{"execution":{"iopub.status.busy":"2021-08-17T10:31:13.542811Z","iopub.execute_input":"2021-08-17T10:31:13.543173Z","iopub.status.idle":"2021-08-17T10:31:13.553088Z","shell.execute_reply.started":"2021-08-17T10:31:13.543092Z","shell.execute_reply":"2021-08-17T10:31:13.552084Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import numpy as np\n# #create a n\n# df = pd.DataFrame(columns=['wave name','value'])\n# #x=np.empty([3, 4096])\n# #x=np.load(str(files[0]))\n# #df[\"value\"].loc[0,1]=x\n# # df.loc[0, 'value']=x\n# # df.loc[0, 'wave name']=str(files[0]).split(\".\")[0]\n# #df[\"wave name\"].loc[0,0]=str(files[file])\n# for file in range(file_count):\n#     #x=np.append(x,np.load(str(file)),axis=0)\n#     x=np.load(str(files[file]))\n#     df.loc[file, 'value']=x\n#     df.loc[file, 'wave name']=str(files[file]).split(\".\")[0]\n\n# x.shape","metadata":{"execution":{"iopub.status.busy":"2021-08-17T10:31:13.554558Z","iopub.execute_input":"2021-08-17T10:31:13.555021Z","iopub.status.idle":"2021-08-17T10:31:13.563739Z","shell.execute_reply.started":"2021-08-17T10:31:13.554985Z","shell.execute_reply":"2021-08-17T10:31:13.562777Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# df.head()\n#cd /kaggle/input/g2net-gravitational-wave-detection/\n","metadata":{"execution":{"iopub.status.busy":"2021-08-17T10:31:13.565905Z","iopub.execute_input":"2021-08-17T10:31:13.566309Z","iopub.status.idle":"2021-08-17T10:31:13.574729Z","shell.execute_reply.started":"2021-08-17T10:31:13.566268Z","shell.execute_reply":"2021-08-17T10:31:13.573984Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# cd /kaggle/input/g2net-gravitational-wave-detection/","metadata":{"execution":{"iopub.status.busy":"2021-08-17T10:31:13.575986Z","iopub.execute_input":"2021-08-17T10:31:13.576264Z","iopub.status.idle":"2021-08-17T10:31:13.585132Z","shell.execute_reply.started":"2021-08-17T10:31:13.576241Z","shell.execute_reply":"2021-08-17T10:31:13.584173Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#I was trying to merge the 2 pds together\n# training_target=pd.read_csv(\"training_labels.csv\")\n\n# df[\"label\"]=training_target[\"target\"][df[\"wave name\"]==training_target[\"id\"]]\n# df.head()","metadata":{"execution":{"iopub.status.busy":"2021-08-17T10:31:13.586262Z","iopub.execute_input":"2021-08-17T10:31:13.586611Z","iopub.status.idle":"2021-08-17T10:31:13.596496Z","shell.execute_reply.started":"2021-08-17T10:31:13.586577Z","shell.execute_reply":"2021-08-17T10:31:13.595784Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# df_final=pd.merge(df,training_target.rename(columns={'id':'wave name'}))\n# df_final       ","metadata":{"execution":{"iopub.status.busy":"2021-08-17T10:31:13.598009Z","iopub.execute_input":"2021-08-17T10:31:13.598313Z","iopub.status.idle":"2021-08-17T10:31:13.606400Z","shell.execute_reply.started":"2021-08-17T10:31:13.598289Z","shell.execute_reply":"2021-08-17T10:31:13.605584Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# df_final[\"value\"][df_final[\"wave name\"]==\"0003d34cd0\"]\n#df_final[\"value\"][df_final[\"wave name\"]==\"0003d34cd0\"].reset_index()","metadata":{"execution":{"iopub.status.busy":"2021-08-17T10:31:13.607920Z","iopub.execute_input":"2021-08-17T10:31:13.608238Z","iopub.status.idle":"2021-08-17T10:31:13.615925Z","shell.execute_reply.started":"2021-08-17T10:31:13.608204Z","shell.execute_reply":"2021-08-17T10:31:13.615160Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n# import matplotlib.pyplot as plt\n# plt.figure(figsize=(20,5))\n# plt.subplot(2,1,1)\n# plt.plot(df_final[\"value\"][df_final[\"wave name\"]==\"0002cc5125\"][0][0], c=\"firebrick\", label=\"detector 1\")\n# plt.plot(df_final[\"value\"][df_final[\"wave name\"]==\"0002cc5125\"][0][1], c=\"blue\", label=\"detector 1\")\n# plt.plot(df_final[\"value\"][df_final[\"wave name\"]==\"0002cc5125\"][0][2], c=\"green\", label=\"detector 1\")\n# plt.subplot(2,1,2)\n# plt.plot(df_final[\"value\"][df_final[\"wave name\"]==\"0003d34cd0\"][1][0], c=\"firebrick\", label=\"detector 1\")\n# plt.plot(df_final[\"value\"][df_final[\"wave name\"]==\"0003d34cd0\"][1][1], c=\"blue\", label=\"detector 1\")\n# plt.plot(df_final[\"value\"][df_final[\"wave name\"]==\"0003d34cd0\"][1][2], c=\"green\", label=\"detector 1\")","metadata":{"execution":{"iopub.status.busy":"2021-08-17T10:31:13.617147Z","iopub.execute_input":"2021-08-17T10:31:13.617403Z","iopub.status.idle":"2021-08-17T10:31:13.624874Z","shell.execute_reply.started":"2021-08-17T10:31:13.617377Z","shell.execute_reply":"2021-08-17T10:31:13.624184Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# from scipy.fft import fft, fftfreq\n# # Number of sample points\n# N = 600\n# # sample spacing\n# T = 1.0 / 800.0\n# x = np.linspace(0.0, N*T, N, endpoint=False)\n# y = np.sin(50.0 * 2.0*np.pi*x) + 0.5*np.sin(80.0 * 2.0*np.pi*x)\n# yf = fft(y)\n# xf = fftfreq(N, T)[:N//2]\n# import matplotlib.pyplot as plt\n# plt.plot(xf, 2.0/N * np.abs(yf[0:N//2]))\n# plt.grid()\n# plt.show()","metadata":{"execution":{"iopub.status.busy":"2021-08-17T10:31:13.625965Z","iopub.execute_input":"2021-08-17T10:31:13.626295Z","iopub.status.idle":"2021-08-17T10:31:13.637877Z","shell.execute_reply.started":"2021-08-17T10:31:13.626270Z","shell.execute_reply":"2021-08-17T10:31:13.637013Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"second trial","metadata":{}},{"cell_type":"code","source":"from glob import glob\nimport pandas as pd\nfrom random import sample\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport seaborn as sns","metadata":{"execution":{"iopub.status.busy":"2021-08-17T10:31:13.639212Z","iopub.execute_input":"2021-08-17T10:31:13.639472Z","iopub.status.idle":"2021-08-17T10:31:14.524523Z","shell.execute_reply.started":"2021-08-17T10:31:13.639448Z","shell.execute_reply":"2021-08-17T10:31:14.523549Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\npaths = glob(\"../input/g2net-gravitational-wave-detection/train/*/*/*/*\")\n#paths","metadata":{"_kg_hide-output":false,"execution":{"iopub.status.busy":"2021-08-17T10:31:14.526237Z","iopub.execute_input":"2021-08-17T10:31:14.526501Z","iopub.status.idle":"2021-08-17T10:32:54.907014Z","shell.execute_reply.started":"2021-08-17T10:31:14.526475Z","shell.execute_reply":"2021-08-17T10:32:54.906167Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"paths[0].split(\"/\")[-1].split(\".\")[0] ","metadata":{"execution":{"iopub.status.busy":"2021-08-17T10:32:54.908485Z","iopub.execute_input":"2021-08-17T10:32:54.908760Z","iopub.status.idle":"2021-08-17T10:32:54.916432Z","shell.execute_reply.started":"2021-08-17T10:32:54.908734Z","shell.execute_reply":"2021-08-17T10:32:54.915452Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# list of ids of .npy files \nids = [path.split(\"/\")[-1].split(\".\")[0] for path in paths]\n\n# data frame containing paths and ids of .npy files \npath_df = pd.DataFrame({\"path\":paths,\"id\":ids})\npath_df\nlabels = pd.read_csv(\"../input/g2net-gravitational-wave-detection/training_labels.csv\")\n","metadata":{"execution":{"iopub.status.busy":"2021-08-17T10:32:54.917906Z","iopub.execute_input":"2021-08-17T10:32:54.918298Z","iopub.status.idle":"2021-08-17T10:32:55.735760Z","shell.execute_reply.started":"2021-08-17T10:32:54.918259Z","shell.execute_reply":"2021-08-17T10:32:55.734936Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = pd.merge(left=labels,right=path_df,on=\"id\")\ntrain_df.shape","metadata":{"execution":{"iopub.status.busy":"2021-08-17T10:32:55.736989Z","iopub.execute_input":"2021-08-17T10:32:55.737259Z","iopub.status.idle":"2021-08-17T10:32:56.345041Z","shell.execute_reply.started":"2021-08-17T10:32:55.737234Z","shell.execute_reply":"2021-08-17T10:32:56.344175Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display(train_df.head())","metadata":{"execution":{"iopub.status.busy":"2021-08-17T10:32:56.346211Z","iopub.execute_input":"2021-08-17T10:32:56.346474Z","iopub.status.idle":"2021-08-17T10:32:56.385415Z","shell.execute_reply.started":"2021-08-17T10:32:56.346448Z","shell.execute_reply":"2021-08-17T10:32:56.384402Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-08-17T10:32:56.389217Z","iopub.execute_input":"2021-08-17T10:32:56.389483Z","iopub.status.idle":"2021-08-17T10:32:56.398143Z","shell.execute_reply.started":"2021-08-17T10:32:56.389458Z","shell.execute_reply":"2021-08-17T10:32:56.397490Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"target_1 = train_df[train_df.target==1]\ntarget_0 = train_df[train_df.target==0]\ntarget_waves = target_1.sample(50).path.values\n#target_waves","metadata":{"execution":{"iopub.status.busy":"2021-08-17T10:32:56.399369Z","iopub.execute_input":"2021-08-17T10:32:56.399702Z","iopub.status.idle":"2021-08-17T10:32:56.530529Z","shell.execute_reply.started":"2021-08-17T10:32:56.399677Z","shell.execute_reply":"2021-08-17T10:32:56.529760Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nplt.figure(figsize=(20,15))\n#VISUALIZE 10 random samples form detectors 1 and they are gravitational wave \nfor i in range(1,len(target_waves)+1):\n#     pos =np.load(target_waves[i-1])[1]\n#for obtain the detector 1 only values\n    pos=np.load(target_waves[i-1])\n    plt.subplot(10,1,i)\n    plt.plot(pos[0], c=\"firebrick\")\n    plt.plot(pos[1], c=\"blue\")\n    plt.plot(pos[2], c=\"green\")\n    \npos.shape","metadata":{"execution":{"iopub.status.busy":"2021-08-17T10:32:56.531608Z","iopub.execute_input":"2021-08-17T10:32:56.531870Z","iopub.status.idle":"2021-08-17T10:32:58.017293Z","shell.execute_reply.started":"2021-08-17T10:32:56.531846Z","shell.execute_reply":"2021-08-17T10:32:58.015581Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"non_target_waves=target_0.sample(50).path.values\n#non_target_waves\n","metadata":{"execution":{"iopub.status.busy":"2021-08-17T10:32:58.018222Z","iopub.status.idle":"2021-08-17T10:32:58.018724Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(20,15))\n#VISUALIZE 10 random samples form detectors 1 and they are gravitational wave \nfor i in range(1,len(target_waves)+1):\n#     pos =np.load(target_waves[i-1])[1]\n#for obtain the detector 1 only values\n    neg=np.load(non_target_waves[i-1])\n    plt.subplot(10,1,i)\n    plt.plot(neg[0], c=\"firebrick\")\n    plt.plot(neg[1], c=\"blue\")\n    plt.plot(neg[2], c=\"green\")","metadata":{"execution":{"iopub.status.busy":"2021-08-17T10:32:58.019495Z","iopub.status.idle":"2021-08-17T10:32:58.019931Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(20,15))\n#VISUALIZE 10 random samples form detectors 1 and they are gravitational wave \nfor i in range(1,len(target_waves)+1):\n#     pos =np.load(target_waves[i-1])[1]\n#for obtain the detector 1 only values\n    neg=np.load(non_target_waves[i-1])\n    plt.subplot(10,1,i)\n    sns.kdeplot(neg[0], color=\"firebrick\")\n    sns.kdeplot(neg[1], color=\"blue\")\n    sns.kdeplot(neg[2], color=\"green\")","metadata":{"execution":{"iopub.status.busy":"2021-08-17T10:32:58.020768Z","iopub.status.idle":"2021-08-17T10:32:58.021177Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nplt.figure(figsize=(20,15))\n#VISUALIZE 10 random samples form detectors 1 and they are gravitational wave \nfor i in range(1,len(target_waves)+1):\n#     pos =np.load(target_waves[i-1])[1]\n#for obtain the detector 1 only values\n    pos=np.load(target_waves[i-1])\n    plt.subplot(10,1,i)\n    sns.kdeplot(pos[0], color=\"firebrick\")\n    sns.kdeplot(pos[1], color=\"blue\")\n    sns.kdeplot(pos[2], color=\"green\")","metadata":{"execution":{"iopub.status.busy":"2021-08-17T10:32:58.022151Z","iopub.status.idle":"2021-08-17T10:32:58.022533Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**When the target is 1 it can be seen in the above plot that the peak of the histogram is much more spread than the peak in target 0 which means that when the target is 0 the gw waves have values mostly zero but when the target is 1 then there are waves with values other than 0 which contribute to the peak of the data**","metadata":{}},{"cell_type":"code","source":"# plt.figure(figsize=(20,15))\n# #VISUALIZE 10 random samples form detectors 1 and they are gravitational wave \n# for i in range(1,len(target_waves)+1):\n# #     pos =np.load(target_waves[i-1])[1]\n# #for obtain the detector 1 only values\n#     pos=np.load(target_waves[i-1])\n#     plt.subplot(10,1,i)\n#     sns.scatterplot(data=pos[0], color=\"firebrick\")\n#     sns.scatterplot(data=pos[1], color=\"blue\")\n#     sns.scatterplot(data=pos[2], color=\"green\")","metadata":{"execution":{"iopub.status.busy":"2021-08-17T10:32:58.023572Z","iopub.status.idle":"2021-08-17T10:32:58.023953Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# plt.figure(figsize=(20,15))\n# #VISUALIZE 10 random samples form detectors 1 and they are gravitational wave \n# for i in range(1,len(target_waves)+1):\n# #     pos =np.load(target_waves[i-1])[1]\n# #for obtain the detector 1 only values\n#     neg=np.load(non_target_waves[i-1])\n#     plt.subplot(10,1,i)\n#     sns.scatterplot(data=neg[0], color=\"firebrick\")\n#     sns.scatterplot(data=neg[1], color=\"blue\")\n#     sns.scatterplot(data=neg[2], color=\"green\")","metadata":{"execution":{"iopub.status.busy":"2021-08-17T10:32:58.024926Z","iopub.status.idle":"2021-08-17T10:32:58.025329Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(20,15))\n#VISUALIZE 10 random samples form detectors 1 and they are gravitational wave \nfor i in range(1,len(target_waves)+1):\n#     pos =np.load(target_waves[i-1])[1]\n#for obtain the detector 1 only values\n    neg=np.load(non_target_waves[i-1])\n    plt.subplot(50,1,i)\n    sns.boxplot(x=neg[0], color=\"firebrick\")\n    #sns.boxplot(x=neg[1], color=\"blue\")\n    #sns.boxplot(x=neg[2], color=\"green\")","metadata":{"execution":{"iopub.status.busy":"2021-08-17T10:32:58.026268Z","iopub.status.idle":"2021-08-17T10:32:58.026650Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(20,15))\n#VISUALIZE 10 random samples form detectors 1 and they are gravitational wave \nfor i in range(1,len(target_waves)+1):\n#     pos =np.load(target_waves[i-1])[1]\n#for obtain the detector 1 only values\n    pos=np.load(target_waves[i-1])\n    plt.subplot(50,1,i)\n    sns.boxplot(x=pos[0], color=\"firebrick\")\n    #sns.boxplot(x=pos[1], color=\"blue\")\n    #sns.boxplot(x=pos[2], color=\"green\")","metadata":{"execution":{"iopub.status.busy":"2021-08-17T10:32:58.027407Z","iopub.status.idle":"2021-08-17T10:32:58.027792Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**what is discovered here from the 2 50s plots that the targeted wave\"gravitational wave\" has more outliers than the other waves specialy in detector one**","metadata":{}},{"cell_type":"markdown","source":"## Visulaizing the Q Transform","metadata":{}},{"cell_type":"markdown","source":"In mathematics and signal processing, the constant-Q transform, simply known as CQT transforms a data series to the frequency domain. It is related to the Fourier transform and very closely related to the complex Morlet wavelet transform","metadata":{}},{"cell_type":"markdown","source":"As the black holes spiral closer and closer in together, the frequency of the gravitational waves increases. Scientists call these sounds \"chirps,\" because some events that generate gravitation waves would sound like a bird's chirp\n\n[Gravitational wave sound from caltech's website](http://https://www.ligo.caltech.edu/video/ligo20160211v2)","metadata":{}},{"cell_type":"markdown","source":"## What is the difference between FFT and Constant Q transfrom ? ","metadata":{}},{"cell_type":"markdown","source":"A standard DFT uses a constant buffer size throughout all frequencies. This typically leads to a pretty consistent, fully continuous transform. However, the constant bin size for all frequencies leads to some problems when you map frequency on a logarithmic scale. Specifically, peaks on the lower end are incredibly wide (sometimes up to half an octave), lacking any sort of detail.\n\nThis is an issue for emulating human perception because humans perceive frequency on a logarithmic scale.\n\nA Constant Q transform seeks to solve this problem by increasing the buffer size for lower frequencies, and alleviate some of the computational strain caused by this by reducing the buffer size used for high frequencies. It's pretty effective at this, but has a few drawbacks.[answer is here ](http://https://dsp.stackexchange.com/questions/43811/what-is-the-difference-between-constant-q-transform-and-wavelet-transform-and-wh)","metadata":{}},{"cell_type":"code","source":"!pip install -q nnAudio -qq\nimport torch\nfrom nnAudio.Spectrogram import CQT1992v2","metadata":{"execution":{"iopub.status.busy":"2021-08-17T10:33:15.081209Z","iopub.execute_input":"2021-08-17T10:33:15.081573Z","iopub.status.idle":"2021-08-17T10:33:24.160443Z","shell.execute_reply.started":"2021-08-17T10:33:15.081541Z","shell.execute_reply":"2021-08-17T10:33:24.159223Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nQ_TRANSFORM = CQT1992v2(sr=2048, fmin=20, fmax=1024, hop_length=32)\n\ndef visualize_sample_qtransform(\n    id, \n    target,\n    signal_names=(\"LIGO Hanford\", \"LIGO Livingston\", \"Virgo\"),\n    sr=2048,\n):\n    x = np.load(id)\n    plt.figure(figsize=(16, 5))\n    for i in range(3):\n        waves = x[i] / np.max(x[i])\n        waves = torch.from_numpy(waves).float()\n        image = Q_TRANSFORM(waves)\n        #print(image.squeeze().shape)\n        plt.subplot(1, 3, i + 1)\n        plt.imshow(image.squeeze())\n        #plt.plot(image.squeeze())\n        #sns.heatmap(image.squeeze())\n        #np.vstack(()) to make the 3 graphs stacked overeach other\n        #image.squeeze() remove the third dimension as the original tensor dimension is [1,69,129]\n        plt.title(signal_names[i], fontsize=14)\n        \n    \n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2021-08-17T10:33:24.162543Z","iopub.execute_input":"2021-08-17T10:33:24.163042Z","iopub.status.idle":"2021-08-17T10:33:24.214030Z","shell.execute_reply.started":"2021-08-17T10:33:24.162994Z","shell.execute_reply":"2021-08-17T10:33:24.213289Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"non_target_waves[1]\ntrain_df.target[train_df[\"path\"]==non_target_waves[1]]","metadata":{"execution":{"iopub.status.busy":"2021-08-17T10:33:24.215789Z","iopub.execute_input":"2021-08-17T10:33:24.216114Z","iopub.status.idle":"2021-08-17T10:33:24.261621Z","shell.execute_reply.started":"2021-08-17T10:33:24.216046Z","shell.execute_reply":"2021-08-17T10:33:24.260311Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"the output of the plot will be time on x axis vs \\frequencies on y axis and the z axis in the amplitude of the signal at the moment with this frequency","metadata":{}},{"cell_type":"code","source":"# for i in range(1,len(target_waves)+1):\n#     #plt.subplot(plt.subplot(50,1,i))\n#     visualize_sample_qtransform(non_target_waves[i-1], train_df.target[train_df[\"path\"]==non_target_waves[i-1]])","metadata":{"execution":{"iopub.status.busy":"2021-08-17T10:32:58.032584Z","iopub.status.idle":"2021-08-17T10:32:58.032980Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # for i in range(1,len(target_waves)+1):\n# #     #plt.subplot(plt.subplot(50,1,i))\n# #     visualize_sample_qtransform(target_waves[i-1], train_df.target[train_df[\"path\"]==target_waves[i-1]])\nvisualize_sample_qtransform(target_waves[1], train_df.target[train_df[\"path\"]==target_waves[1]])","metadata":{"execution":{"iopub.status.busy":"2021-08-17T10:33:25.224604Z","iopub.execute_input":"2021-08-17T10:33:25.224947Z","iopub.status.idle":"2021-08-17T10:33:25.772524Z","shell.execute_reply.started":"2021-08-17T10:33:25.224920Z","shell.execute_reply":"2021-08-17T10:33:25.771360Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**We can hear gravitational waves, in the same sense that sound waves travel through water, or seismic waves move through the earth. The difference is that sound waves vibrate through a medium, like water or soil. For gravitational waves, spacetime is the medium. It just takes the right instrument to hear them.**\n\n**Detecting gravitational waves on Earth was a challenge that took roughly a century to complete, since the ones that wash through the planet are incredibly tiny.**[national geographic's article about gravitational waves](http://https://www.nationalgeographic.com/science/article/gravitational-waves#:~:text=We%20can%20hear%20gravitational%20waves,right%20instrument%20to%20hear%20them.)","metadata":{}},{"cell_type":"markdown","source":"[forbes's article about gravitational wwaves](http://https://www.forbes.com/sites/startswithabang/2020/08/21/ask-ethan-why-dont-light-and-gravitational-waves-arrive-simultaneously/#:~:text=Whenever%20masses%20accelerate%20through%20curved,%2C%20rather%20than%20electromagnetic%2C%20radiation.)","metadata":{}},{"cell_type":"markdown","source":"### Mel-frequency cepstrum","metadata":{}},{"cell_type":"code","source":"import librosa\nimport librosa.display","metadata":{"execution":{"iopub.status.busy":"2021-08-17T10:33:30.972157Z","iopub.execute_input":"2021-08-17T10:33:30.972497Z","iopub.status.idle":"2021-08-17T10:33:32.376093Z","shell.execute_reply.started":"2021-08-17T10:33:30.972469Z","shell.execute_reply":"2021-08-17T10:33:32.374942Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def visualize_sample_mfcc(\n    _id, \n    target,\n    signal_names=(\"LIGO Hanford\", \"LIGO Livingston\", \"Virgo\"),\n    sr=2048,\n):\n    x = np.load(_id)\n    plt.figure(figsize=(16, 5))\n    for i in range(3):\n        mfccs = librosa.feature.mfcc(x[i] / x[i].max(), sr=sr)\n        print(max(mfccs[1]))\n        plt.subplot(1, 3, i + 1)\n        librosa.display.specshow(mfccs, sr=sr, x_axis=\"time\", vmin=-200, vmax=60, cmap=\"coolwarm\")\n        plt.title(signal_names[i], fontsize=14)\n        plt.colorbar()\n\n \n\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2021-08-17T10:33:32.420001Z","iopub.execute_input":"2021-08-17T10:33:32.420367Z","iopub.status.idle":"2021-08-17T10:33:32.427636Z","shell.execute_reply.started":"2021-08-17T10:33:32.420333Z","shell.execute_reply":"2021-08-17T10:33:32.426539Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.decomposition import PCA\ndef mfcc_transformation(_id,sr=2048):\n    mfcc_coef=[]\n    x = np.load(_id)\n    for i in range(3):\n        mfccs = librosa.feature.mfcc(x[i] / x[i].max(), sr=sr)\n        pca = PCA(n_components=3)\n        principalComponents = pca.fit_transform(mfccs)\n        mfcc_coef.append(principalComponents)\n    return mfcc_coef","metadata":{"execution":{"iopub.status.busy":"2021-08-17T10:33:33.552744Z","iopub.execute_input":"2021-08-17T10:33:33.553073Z","iopub.status.idle":"2021-08-17T10:33:33.561582Z","shell.execute_reply.started":"2021-08-17T10:33:33.553045Z","shell.execute_reply":"2021-08-17T10:33:33.560448Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df\nlen(train_df)","metadata":{"execution":{"iopub.status.busy":"2021-08-17T10:33:35.054087Z","iopub.execute_input":"2021-08-17T10:33:35.054497Z","iopub.status.idle":"2021-08-17T10:33:35.063161Z","shell.execute_reply.started":"2021-08-17T10:33:35.054461Z","shell.execute_reply":"2021-08-17T10:33:35.062169Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nfrom tqdm import tqdm\n\ntarget_waves1 = train_df[0:50000].path.values\nfeature_dataset1=pd.DataFrame(columns = [\"feature_pca\"])\nfor i in tqdm(range(len(target_waves1))):\n        feature_dataset1.loc[i, 'feature_pca']=mfcc_transformation(target_waves1[i])\n    \n","metadata":{"execution":{"iopub.status.busy":"2021-08-17T10:40:22.506134Z","iopub.execute_input":"2021-08-17T10:40:22.506469Z","iopub.status.idle":"2021-08-17T11:10:42.040664Z","shell.execute_reply.started":"2021-08-17T10:40:22.506440Z","shell.execute_reply":"2021-08-17T11:10:42.038306Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"feature_dataset1","metadata":{"execution":{"iopub.status.busy":"2021-08-17T11:10:54.401571Z","iopub.execute_input":"2021-08-17T11:10:54.401923Z","iopub.status.idle":"2021-08-17T11:10:54.572831Z","shell.execute_reply.started":"2021-08-17T11:10:54.401890Z","shell.execute_reply":"2021-08-17T11:10:54.571773Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"(mfcc_transformation(target_waves[2])[1]).tolist()","metadata":{"execution":{"iopub.status.busy":"2021-08-17T10:32:58.042235Z","iopub.status.idle":"2021-08-17T10:32:58.042601Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mfcc_transformation(target_waves[2])\n#plt.plot(mfcc_transformation(target_waves[2])[1])\nsns.scatterplot(data=(mfcc_transformation(target_waves[2])[1]).tolist()[3], color=\"firebrick\")","metadata":{"execution":{"iopub.status.busy":"2021-08-17T10:32:58.043523Z","iopub.status.idle":"2021-08-17T10:32:58.043901Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"visualize_sample_mfcc(target_waves[2], train_df.target[train_df[\"path\"]==target_waves[2]])","metadata":{"execution":{"iopub.status.busy":"2021-08-17T10:32:58.044660Z","iopub.status.idle":"2021-08-17T10:32:58.045042Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}