{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Good Note book for reference\n#    1.    https://www.kaggle.com/mistag/data-preprocessing-with-gwpy \n# Special thanks to Darek, he made data conversion from numpy to Qtransformed png file \"https://www.kaggle.com/thedrcat/g2net-train-images-with-gpwy-sample/\"\n# I thought I could levrage that for now, to create base line ","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nfrom scipy import fft\nfrom scipy.signal import spectrogram\nfrom scipy import signal\nfrom os import path","metadata":{"execution":{"iopub.status.busy":"2021-08-28T15:34:25.521867Z","iopub.execute_input":"2021-08-28T15:34:25.522377Z","iopub.status.idle":"2021-08-28T15:34:26.606827Z","shell.execute_reply.started":"2021-08-28T15:34:25.522275Z","shell.execute_reply":"2021-08-28T15:34:26.605252Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"try:\n    from gwpy.timeseries import TimeSeries\n    from gwpy.plot import Plot\n    \nexcept:\n    !python -m pip install gwpy\n    !pip install astropy==4.2.1\n    from gwpy.timeseries import TimeSeries\n    from gwpy.plot import Plot","metadata":{"execution":{"iopub.status.busy":"2021-08-28T15:34:26.611089Z","iopub.execute_input":"2021-08-28T15:34:26.611776Z","iopub.status.idle":"2021-08-28T15:34:58.715164Z","shell.execute_reply.started":"2021-08-28T15:34:26.611738Z","shell.execute_reply":"2021-08-28T15:34:58.713907Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from pandarallel import pandarallel\npandarallel.initialize()","metadata":{"execution":{"iopub.status.busy":"2021-08-28T15:34:58.717922Z","iopub.execute_input":"2021-08-28T15:34:58.718458Z","iopub.status.idle":"2021-08-28T15:34:58.764673Z","shell.execute_reply.started":"2021-08-28T15:34:58.718406Z","shell.execute_reply":"2021-08-28T15:34:58.763322Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import seaborn as sns\nimport matplotlib.pyplot as plt\nimport plotly \nimport plotly.graph_objects as go\nimport plotly.express as px\nfrom plotly.subplots import make_subplots","metadata":{"execution":{"iopub.status.busy":"2021-08-28T15:34:58.766300Z","iopub.execute_input":"2021-08-28T15:34:58.766642Z","iopub.status.idle":"2021-08-28T15:34:59.724601Z","shell.execute_reply.started":"2021-08-28T15:34:58.766588Z","shell.execute_reply":"2021-08-28T15:34:59.723457Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.preprocessing import MinMaxScaler\nfrom PIL import Image","metadata":{"execution":{"iopub.status.busy":"2021-08-28T15:34:59.726054Z","iopub.execute_input":"2021-08-28T15:34:59.726370Z","iopub.status.idle":"2021-08-28T15:34:59.858064Z","shell.execute_reply.started":"2021-08-28T15:34:59.726341Z","shell.execute_reply":"2021-08-28T15:34:59.857214Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"try:\n    import efficientnet.keras as efn\nexcept:\n    \n    !pip install -U efficientnet\n    import efficientnet.keras as efn\n\n\nimport tensorflow as tf \nimport glob\nfrom sklearn.model_selection import train_test_split","metadata":{"execution":{"iopub.status.busy":"2021-08-28T15:34:59.859176Z","iopub.execute_input":"2021-08-28T15:34:59.859607Z","iopub.status.idle":"2021-08-28T15:35:14.660576Z","shell.execute_reply.started":"2021-08-28T15:34:59.859577Z","shell.execute_reply":"2021-08-28T15:35:14.659386Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.set_theme(\"talk\")\nsns.set_style(\"white\")","metadata":{"execution":{"iopub.status.busy":"2021-08-28T15:35:14.662396Z","iopub.execute_input":"2021-08-28T15:35:14.662913Z","iopub.status.idle":"2021-08-28T15:35:14.670009Z","shell.execute_reply.started":"2021-08-28T15:35:14.662862Z","shell.execute_reply":"2021-08-28T15:35:14.668818Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_file =\"../input/g2net-gravitational-wave-detection/training_labels.csv\"\nsubmission_file =\"../input/g2net-gravitational-wave-detection/sample_submission.csv\"","metadata":{"execution":{"iopub.status.busy":"2021-08-28T15:35:14.674298Z","iopub.execute_input":"2021-08-28T15:35:14.674827Z","iopub.status.idle":"2021-08-28T15:35:14.685257Z","shell.execute_reply.started":"2021-08-28T15:35:14.674773Z","shell.execute_reply":"2021-08-28T15:35:14.683992Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"master_df = pd.read_csv( train_file)\nmaster_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-08-28T15:35:14.687429Z","iopub.execute_input":"2021-08-28T15:35:14.687769Z","iopub.status.idle":"2021-08-28T15:35:15.187721Z","shell.execute_reply.started":"2021-08-28T15:35:14.687730Z","shell.execute_reply":"2021-08-28T15:35:15.186304Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Constructing complete path, it will be very helpfull for further analysis","metadata":{}},{"cell_type":"code","source":"master_df[\"path\"]= master_df[\"id\"].parallel_apply( lambda x : \"../input/g2net-gravitational-wave-detection/train/\" +x[0] +\"/\"+x[1] +\"/\"+x[2] +\"/\" + x+\".npy\")","metadata":{"execution":{"iopub.status.busy":"2021-08-28T15:35:15.190163Z","iopub.execute_input":"2021-08-28T15:35:15.190649Z","iopub.status.idle":"2021-08-28T15:35:16.143263Z","shell.execute_reply.started":"2021-08-28T15:35:15.190601Z","shell.execute_reply":"2021-08-28T15:35:16.142083Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"master_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-08-28T15:35:16.145025Z","iopub.execute_input":"2021-08-28T15:35:16.145348Z","iopub.status.idle":"2021-08-28T15:35:16.158270Z","shell.execute_reply.started":"2021-08-28T15:35:16.145316Z","shell.execute_reply":"2021-08-28T15:35:16.157192Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Check number of Singals present in dataframe","metadata":{}},{"cell_type":"code","source":"print (\" Number of Signal samples present ={}\".format( master_df.shape[0] ) )","metadata":{"execution":{"iopub.status.busy":"2021-08-28T15:35:16.159663Z","iopub.execute_input":"2021-08-28T15:35:16.160033Z","iopub.status.idle":"2021-08-28T15:35:16.174361Z","shell.execute_reply.started":"2021-08-28T15:35:16.160000Z","shell.execute_reply":"2021-08-28T15:35:16.172919Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Check path constructed correct or not ","metadata":{}},{"cell_type":"code","source":"file_exist = False\n\nfor i in range( 0, 20 ):\n    file = master_df[\"path\"].iloc[ np.random.randint( master_df.shape[0] ) ]\n    if not path.exists( file ): \n        print ( \"File does not exist, constructed path is wrong for file = {}\".format( file ) )\n        file_exist = True\nif not file_exist : print (\"All 20 file are present \")","metadata":{"execution":{"iopub.status.busy":"2021-08-28T15:35:16.176013Z","iopub.execute_input":"2021-08-28T15:35:16.176443Z","iopub.status.idle":"2021-08-28T15:35:16.270539Z","shell.execute_reply.started":"2021-08-28T15:35:16.176402Z","shell.execute_reply":"2021-08-28T15:35:16.269439Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"markdown","source":"# Visualize distribution of traget","metadata":{}},{"cell_type":"code","source":"fig = plt.figure( figsize = (5,5), dpi = 90 )\nsns.countplot( master_df[\"target\"] )\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-08-28T15:35:16.271816Z","iopub.execute_input":"2021-08-28T15:35:16.272244Z","iopub.status.idle":"2021-08-28T15:35:16.751827Z","shell.execute_reply.started":"2021-08-28T15:35:16.272209Z","shell.execute_reply":"2021-08-28T15:35:16.750505Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Good part is data is not imbalanced","metadata":{}},{"cell_type":"markdown","source":"# Let us see how datais stored\n1. Format\n2. Number of channels","metadata":{}},{"cell_type":"code","source":"sample_file = master_df[\"path\"].iloc[0]\ntarget= master_df[\"target\"].iloc[0]\ndata = np.load( sample_file  )\ndata.shape","metadata":{"execution":{"iopub.status.busy":"2021-08-28T15:35:16.753427Z","iopub.execute_input":"2021-08-28T15:35:16.753911Z","iopub.status.idle":"2021-08-28T15:35:16.772976Z","shell.execute_reply.started":"2021-08-28T15:35:16.753863Z","shell.execute_reply":"2021-08-28T15:35:16.771547Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## Data is stored in 2 dimension\nfor i in range( 0, 3 ):\n    file,target  = master_df[[\"path\",\"target\"]].iloc[np.random.randint(master_df.shape[0] ) ].values\n    data = np.load( file )\n    fig = go.Figure()\n    fig.add_trace( go.Line( y = data[0], name =\"LIGO Hanford\" ) )\n    fig.add_trace( go.Line( y = data[1], name =\"LIGO Livingston\" ) )\n    fig.add_trace( go.Line( y = data[2], name =\"LIGO Virgo\" ) )\n    fig.update_layout(title =\"Gravitation wave in Time domain from 3 Detectors and target={}\".format( target ) )\n    fig.show()","metadata":{"execution":{"iopub.status.busy":"2021-08-28T15:35:16.774775Z","iopub.execute_input":"2021-08-28T15:35:16.775154Z","iopub.status.idle":"2021-08-28T15:35:17.161112Z","shell.execute_reply.started":"2021-08-28T15:35:16.775120Z","shell.execute_reply":"2021-08-28T15:35:17.159814Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Measured amplitude is very small range lets visualize distrbution of values using histogram ","metadata":{}},{"cell_type":"code","source":"file_name, target= master_df[[\"path\",\"target\"]].iloc[np.random.randint( master_df.shape[0])].values\ndata = np.load( file_name)\n    \nfig = go.Figure()\nfig.add_trace( go.Histogram( x = data[0], autobinx= True, name = \"LIGO Hanford target={}\".format( target ) ))\nfig.add_trace( go.Histogram( x = data[0], autobinx= True,name = \"LIGO Livingston target={}\".format( target ) ))\nfig.add_trace( go.Histogram( x = data[0], autobinx= True,name = \"LIGO Virgo target={}\".format( target ) ))\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2021-08-28T15:35:17.162512Z","iopub.execute_input":"2021-08-28T15:35:17.162886Z","iopub.status.idle":"2021-08-28T15:35:17.240109Z","shell.execute_reply.started":"2021-08-28T15:35:17.162794Z","shell.execute_reply":"2021-08-28T15:35:17.239019Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# let usscale values by 10^-20 and visualize the distribution ","metadata":{}},{"cell_type":"code","source":"file_name, target= master_df[[\"path\",\"target\"]].iloc[np.random.randint( master_df.shape[0])].values\ndata = np.load( file_name)\ndata = np.multiply( data, 10**20 )    \nfig = go.Figure()\nfig.add_trace( go.Histogram( x = data[0], autobinx= True, name = \"LIGO Hanford target={}\".format( target ) ))\nfig.add_trace( go.Histogram( x = data[0], autobinx= True,name = \"LIGO Livingston target={}\".format( target ) ))\nfig.add_trace( go.Histogram( x = data[0], autobinx= True,name = \"LIGO Virgo target={}\".format( target ) ))\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2021-08-28T15:35:17.241529Z","iopub.execute_input":"2021-08-28T15:35:17.241886Z","iopub.status.idle":"2021-08-28T15:35:17.391254Z","shell.execute_reply.started":"2021-08-28T15:35:17.241852Z","shell.execute_reply":"2021-08-28T15:35:17.390203Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Time domain will provide only little info.\n## Frequency domain should give more detils on singal strenght and other frequency componenets.\n## Convert amplitude to db using formula 20log(data)","metadata":{}},{"cell_type":"code","source":"fig = make_subplots( rows = 1, cols = 2 )\nfor i in range( 0, 2):\n    file_name, target= master_df[[\"path\",\"target\"]].iloc[np.random.randint( master_df.shape[0])].values\n    data = np.load( file_name)\n    data = np.multiply( data, 10**20 )    \n    fig.add_trace( go.Line( x = fft.rfftfreq( 2* 2048, (1/ 4096)) , y = 20*np.log(np.abs( fft.rfft( data[0]) )),name =\"LIGO Hanford target={}\".format( target )), row = 1, col = i+1 )\n    fig.add_trace( go.Line( x = fft.rfftfreq( 2* 2048, (1/ 4096)) , y = 20*np.log(np.abs( fft.rfft( data[1]) )),name =\"LIGO Livingston target={}\".format( target )), row = 1, col = i+1 )\n    fig.add_trace( go.Line( x = fft.rfftfreq( 2* 2048, (1/ 4096)) , y = 20*np.log(np.abs( fft.rfft( data[2]) )),name =\"LIGO Virgo target={}\".format( target ) ) , row = 1, col = i+1)\n\nfig.update_layout( title =\"Gravitational wave inFreq Domain \")\nfig.update_xaxes( title =\"Freq in Hz\")\nfig.update_yaxes( title =\"Signal Power in dB\")\nfig.show()\n","metadata":{"execution":{"iopub.status.busy":"2021-08-28T15:35:17.392515Z","iopub.execute_input":"2021-08-28T15:35:17.393040Z","iopub.status.idle":"2021-08-28T15:35:17.760498Z","shell.execute_reply.started":"2021-08-28T15:35:17.392994Z","shell.execute_reply":"2021-08-28T15:35:17.759729Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## we can't analyze time domain singal or Frequency domain signal using Machine learning model.\n## Let us convert singal to SFT ( Short fourier Transform ) and then to Constant Q Transform","metadata":{}},{"cell_type":"code","source":"file_name, target= master_df[[\"path\",\"target\"]].iloc[np.random.randint( master_df.shape[0])].values\ndata = np.load( file_name)","metadata":{"execution":{"iopub.status.busy":"2021-08-28T15:35:17.761621Z","iopub.execute_input":"2021-08-28T15:35:17.762090Z","iopub.status.idle":"2021-08-28T15:35:17.796387Z","shell.execute_reply.started":"2021-08-28T15:35:17.762047Z","shell.execute_reply":"2021-08-28T15:35:17.795301Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"f,t,mag = spectrogram( np.multiply( data[0], 10**20) , fs = 2048, window = \"hamming\", noverlap= 100, mode =\"magnitude\")\nfig = go.Figure()\nfig.add_trace( go.Heatmap( x= t, y = f, z = np.abs(mag) ))\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2021-08-28T15:35:17.797908Z","iopub.execute_input":"2021-08-28T15:35:17.798251Z","iopub.status.idle":"2021-08-28T15:35:17.856527Z","shell.execute_reply.started":"2021-08-28T15:35:17.798219Z","shell.execute_reply":"2021-08-28T15:35:17.855434Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# I came across one note book very informative, \"https://www.kaggle.com/mistag/data-preprocessing-with-gwpy\"\n# Talks about converting giventime series data to Gwavitational wave using gwpy library.\n# Taken implementation and code from same note book.\n","metadata":{}},{"cell_type":"code","source":"\n## Fucntion converts data to GW wave format\ndef read_file(fname):\n    data = np.load(fname)\n    d1 = TimeSeries(data[0,:], sample_rate=2048)\n    d2 = TimeSeries(data[1,:], sample_rate=2048)\n    d3 = TimeSeries(data[2,:], sample_rate=2048)\n    return d1, d2, d3\n\ndef plot_time_data(d1, d2, d3):\n    plot = Plot(d1, d2, d3, separate=True, sharex=True, figsize=[12, 8])\n    ax = plot.gca()\n    #ax.set_title([\"1\",\"2\",\"3\"])\n    ax.set_xlim(0,2)\n    ax.set_xlabel('Time [s]')\n    plt.tight_layout()\n    plot.show( )","metadata":{"execution":{"iopub.status.busy":"2021-08-28T15:35:17.857958Z","iopub.execute_input":"2021-08-28T15:35:17.858260Z","iopub.status.idle":"2021-08-28T15:35:17.866012Z","shell.execute_reply.started":"2021-08-28T15:35:17.858231Z","shell.execute_reply":"2021-08-28T15:35:17.864952Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"d1, d2, d3 = read_file('../input/g2net-gravitational-wave-detection/train/0/0/0/000a5b6e5c.npy')\nplot_time_data(d1, d2, d3)\n","metadata":{"execution":{"iopub.status.busy":"2021-08-28T15:35:17.870914Z","iopub.execute_input":"2021-08-28T15:35:17.871283Z","iopub.status.idle":"2021-08-28T15:35:34.153348Z","shell.execute_reply.started":"2021-08-28T15:35:17.871252Z","shell.execute_reply":"2021-08-28T15:35:34.152101Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#applying window function, i.e creating Band pass filter \n\nwindow = signal.tukey( 4096 )\nplt.plot( window)\n\n","metadata":{"execution":{"iopub.status.busy":"2021-08-28T15:35:34.156244Z","iopub.execute_input":"2021-08-28T15:35:34.156752Z","iopub.status.idle":"2021-08-28T15:35:34.491420Z","shell.execute_reply.started":"2021-08-28T15:35:34.156701Z","shell.execute_reply":"2021-08-28T15:35:34.490211Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data after applying Band pass filter","metadata":{}},{"cell_type":"code","source":"d1, d2, d3 = d1*window, d2*window, d3*window\nplot_time_data(d1, d2, d3)","metadata":{"execution":{"iopub.status.busy":"2021-08-28T15:35:34.492775Z","iopub.execute_input":"2021-08-28T15:35:34.493266Z","iopub.status.idle":"2021-08-28T15:35:49.946784Z","shell.execute_reply.started":"2021-08-28T15:35:34.493226Z","shell.execute_reply":"2021-08-28T15:35:49.945703Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Frequency domaain data after passing through High Pass filter\n","metadata":{}},{"cell_type":"code","source":"fig2b = d1.highpass(15).asd(fftlength=2).plot(figsize=[12, 6])\nplt.xlim(10,1024)\nplt.ylim(1e-25, 1e-20);\n","metadata":{"execution":{"iopub.status.busy":"2021-08-28T15:35:49.948195Z","iopub.execute_input":"2021-08-28T15:35:49.948504Z","iopub.status.idle":"2021-08-28T15:35:51.147113Z","shell.execute_reply.started":"2021-08-28T15:35:49.948476Z","shell.execute_reply":"2021-08-28T15:35:51.145945Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Spectral whitening and bandpass filtering\n\n# This is super simple with GWpy:\n","metadata":{}},{"cell_type":"code","source":"white_data = d1.whiten(window=(\"tukey\",0.2)) # whiten-function has a built-in window function\nbp_data = white_data.bandpass(35, 350) # frequency range 35-350Hz\nfig3 = bp_data.plot(figsize=[12, 6])\nplt.xlim(0, 2)\nax = plt.gca()\nax.set_title('Whitened and bandpassed')\nax.set_xlabel('Time [s]')\nax.set_ylabel( \"amplitude \")\n","metadata":{"execution":{"iopub.status.busy":"2021-08-28T15:35:51.149264Z","iopub.execute_input":"2021-08-28T15:35:51.149762Z","iopub.status.idle":"2021-08-28T15:35:55.465447Z","shell.execute_reply.started":"2021-08-28T15:35:51.149712Z","shell.execute_reply":"2021-08-28T15:35:55.464449Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#  Now, we have a preprocessed data that is ready for further analysis. First, let's define a function that combines all the steps above and outputs preprocessed data:\n","metadata":{}},{"cell_type":"code","source":"\n#lf & hf are cut of freq for Band pass filter\ndef preprocess(d1, d2, d3, bandpass=False, lf=20, hf=350):\n    white_d1 = d1.whiten(window=(\"tukey\",0.2))\n    white_d2 = d2.whiten(window=(\"tukey\",0.2))\n    white_d3 = d3.whiten(window=(\"tukey\",0.2))\n    if bandpass: # bandpass filter\n        bp_d1 = white_d1.bandpass(lf, hf) \n        bp_d2 = white_d2.bandpass(lf, hf)\n        bp_d3 = white_d3.bandpass(lf, hf)\n        return bp_d1, bp_d2, bp_d3\n    else: # only whiten\n        return white_d1, white_d2, white_d3","metadata":{"execution":{"iopub.status.busy":"2021-08-28T15:35:55.466935Z","iopub.execute_input":"2021-08-28T15:35:55.467264Z","iopub.status.idle":"2021-08-28T15:35:55.474577Z","shell.execute_reply.started":"2021-08-28T15:35:55.467233Z","shell.execute_reply":"2021-08-28T15:35:55.473247Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\n# Q-Transform\n\n# The Q-Transform is related to the Fourier transform, and very closely related to a wavelet transform. The spectrogram is a possible candidate as input for a CNN model.\n","metadata":{}},{"cell_type":"code","source":"r1, r2, r3 = read_file('../input/g2net-gravitational-wave-detection/train/0/0/0/000a5b6e5c.npy') # this signal has target=1\np1, p2, p3 = preprocess(r1, r2, r3)\nhq = p2.q_transform(qrange=(16,32), frange=(30,400), logf=True, whiten=False)\nfig4 = hq.plot(figsize=[12, 10])\nax = fig4.gca()\nfig4.colorbar(label=\"Normalised energy\")\nax.grid(False)\nax.set_yscale('log')\nax.set_xlabel('Time [s]');","metadata":{"execution":{"iopub.status.busy":"2021-08-28T15:35:55.476313Z","iopub.execute_input":"2021-08-28T15:35:55.477031Z","iopub.status.idle":"2021-08-28T15:36:00.871240Z","shell.execute_reply.started":"2021-08-28T15:35:55.476753Z","shell.execute_reply":"2021-08-28T15:36:00.870351Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print ( \"Shape of Transoformationof single detector is ={}\".format(hq.shape) )","metadata":{"execution":{"iopub.status.busy":"2021-08-28T15:36:00.872486Z","iopub.execute_input":"2021-08-28T15:36:00.873031Z","iopub.status.idle":"2021-08-28T15:36:00.878572Z","shell.execute_reply.started":"2021-08-28T15:36:00.872991Z","shell.execute_reply":"2021-08-28T15:36:00.877515Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Combine three channels into one RGB image\n# Since we have 3 detectors, we can combine the Q-Transforms as RGB channels into one color image. Let's make a function for that:","metadata":{}},{"cell_type":"code","source":"Q_RANGE = (16,32)\nF_RANGE = (30,400)\n\ndef create_rgb(fname):\n    r1, r2, r3 = read_file(fname)\n    p1, p2, p3 = preprocess(r1, r2, r3)\n    hq1 = p1.q_transform(qrange=Q_RANGE, frange=F_RANGE, logf=True, whiten=False)\n    hq2 = p2.q_transform(qrange=Q_RANGE, frange=F_RANGE, logf=True, whiten=False)\n    hq3 = p3.q_transform(qrange=Q_RANGE, frange=F_RANGE, logf=True, whiten=False)\n    img = np.zeros([hq1.shape[0], hq1.shape[1], 3], dtype=np.uint8)\n    scaler = MinMaxScaler()\n    img[:,:,0] = 255*scaler.fit_transform(hq1)\n    img[:,:,1] = 255*scaler.fit_transform(hq2)\n    img[:,:,2] = 255*scaler.fit_transform(hq3)\n    return Image.fromarray(img).rotate(90, expand=1).resize((760,760))\n","metadata":{"execution":{"iopub.status.busy":"2021-08-28T15:36:00.880133Z","iopub.execute_input":"2021-08-28T15:36:00.880561Z","iopub.status.idle":"2021-08-28T15:36:00.899878Z","shell.execute_reply.started":"2021-08-28T15:36:00.880519Z","shell.execute_reply":"2021-08-28T15:36:00.898149Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"create_rgb('../input/g2net-gravitational-wave-detection/train/0/0/0/000a5b6e5c.npy')","metadata":{"execution":{"iopub.status.busy":"2021-08-28T15:36:00.901718Z","iopub.execute_input":"2021-08-28T15:36:00.902103Z","iopub.status.idle":"2021-08-28T15:36:01.491364Z","shell.execute_reply.started":"2021-08-28T15:36:00.902069Z","shell.execute_reply":"2021-08-28T15:36:01.489818Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train data might not give good View, let us vissualize test data","metadata":{}},{"cell_type":"code","source":"create_rgb( \"../input/g2net-gravitational-wave-detection/test/1/0/b/10b041376b.npy\")","metadata":{"execution":{"iopub.status.busy":"2021-08-28T15:36:01.493370Z","iopub.execute_input":"2021-08-28T15:36:01.493925Z","iopub.status.idle":"2021-08-28T15:36:02.071328Z","shell.execute_reply.started":"2021-08-28T15:36:01.493854Z","shell.execute_reply":"2021-08-28T15:36:02.070063Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Did not good, but decent amount of EDA lets start with building model and data pipeline","metadata":{}},{"cell_type":"code","source":"\n\n## Fucntion converts data to GW wave format\ndef read_file(fname):\n    data = np.load(fname)\n    d1 = TimeSeries(data[0,:], sample_rate=2048)\n    d2 = TimeSeries(data[1,:], sample_rate=2048)\n    d3 = TimeSeries(data[2,:], sample_rate=2048)\n    return d1, d2, d3\n\n\n\ndef convert_data_3channel_qtran( file_name ,Q_range_low = 16, q_range_high = 32, f_range_ll = 10, f_range_high = 400  ):\n    r1, r2, r3 = read_file(file_name)\n    p1, p2, p3 = preprocess(r1, r2, r3)\n    hq1 = p1.q_transform(qrange=(Q_range_low, q_range_high), frange=( f_range_ll, f_range_high), logf=True, whiten=False)\n    hq2 = p2.q_transform(qrange=(Q_range_low, q_range_high), frange=( f_range_ll, f_range_high), logf=True, whiten=False)\n    hq3 = p3.q_transform(qrange=(Q_range_low, q_range_high), frange=( f_range_ll, f_range_high), logf=True, whiten=False)\n    #img = np.zeros([hq1.shape[0], hq1.shape[1], 3], dtype=np.float32)\n    scaler = MinMaxScaler()\n    hq1 = 255*scaler.fit_transform(hq1)\n    hq2 = 255*scaler.fit_transform(hq2)\n    hq3=  255*scaler.fit_transform(hq3)\n    img = np.dstack( ( hq1, hq2, hq3 ) )\n    return img #Image.fromarray(img).rotate(90, expand=1).resize((256,256))\n\nif False:\n    \n    for each_file in glob.glob( \"../input/g2net-gravitational-wave-detection/train/*/*/*/*.npy\")[:10]:\n\n        convert_data_3channel_qtran(each_file).save(\"./\"+each_file.split(\"/\")[-1].replace(\".npy\",\".png\"))\n\n        print( each_file)\n    \n    ","metadata":{"execution":{"iopub.status.busy":"2021-08-28T15:36:02.072923Z","iopub.execute_input":"2021-08-28T15:36:02.073262Z","iopub.status.idle":"2021-08-28T15:36:02.085781Z","shell.execute_reply.started":"2021-08-28T15:36:02.073230Z","shell.execute_reply":"2021-08-28T15:36:02.084722Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"CFG= {\"IMG_LENGTH\": 300,\n         \"IMG_WIDTH\": 300,\n         \"CHANNELS\": 3,\n         \"RANDOM_STATE\": 100,\n      \"BATCH_SIZE\": 50\n     }","metadata":{"execution":{"iopub.status.busy":"2021-08-28T15:36:02.087149Z","iopub.execute_input":"2021-08-28T15:36:02.087482Z","iopub.status.idle":"2021-08-28T15:36:02.105219Z","shell.execute_reply.started":"2021-08-28T15:36:02.087442Z","shell.execute_reply":"2021-08-28T15:36:02.104032Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Q_range_low = 16\nq_range_high = 32\nf_range_ll = 10\nf_range_high = 400\nr1, r2, r3 = read_file(\"../input/g2net-gravitational-wave-detection/train/0/0/0/00000e74ad.npy\")\np1, p2, p3 = preprocess(r1, r2, r3)\nhq1 = p1.q_transform(qrange=(Q_range_low, q_range_high), frange=( f_range_ll, f_range_high), logf=True, whiten=False)\nhq2 = p2.q_transform(qrange=(Q_range_low, q_range_high), frange=( f_range_ll, f_range_high), logf=True, whiten=False)\nhq3 = p3.q_transform(qrange=(Q_range_low, q_range_high), frange=( f_range_ll, f_range_high), logf=True, whiten=False)\n#img = np.zeros([hq1.shape[0], hq1.shape[1], 3], dtype=np.float32)\nscaler = MinMaxScaler()\nhq1 = scaler.fit_transform(hq1)\nhq2 = scaler.fit_transform(hq2)\nhq3=  scaler.fit_transform(hq3)\nimg = np.dstack( ( hq1, hq2, hq3 ) )","metadata":{"execution":{"iopub.status.busy":"2021-08-28T15:36:02.107238Z","iopub.execute_input":"2021-08-28T15:36:02.107755Z","iopub.status.idle":"2021-08-28T15:36:02.465396Z","shell.execute_reply.started":"2021-08-28T15:36:02.107706Z","shell.execute_reply":"2021-08-28T15:36:02.464317Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\ndef decode_numpy_file ( ):\n    def convert_data_3channel_qtran( file_name, target   ):\n        \n        return  tf.io.decode_jpeg( tf.io.read_file( file_name)),target \n    \n    return convert_data_3channel_qtran\n\n\ndef data_augment ():\n    \n    def perform_aument(img, target ):\n        img = tf.image.ran\n        \n    \n    return perform_aument\n\n\ndef datagenerator(df, test = False  ):\n    \n    read_reshape_gw = decode_numpy_file()\n    #augment =  # augmentation Function still in development \n    datagen = tf.data.Dataset.from_tensor_slices( ( df[\"path_new\"].values, df[\"target\"].values ) )\n    datagen = datagen.map( read_reshape_gw, num_parallel_calls= tf.data.AUTOTUNE )\n    #datagen = datagen.map( read_reshape_gw, num_parallel_calls= tf.data.AUTOTUNE )\n    datagen = datagen.repeat() if not test else datagen\n    datagen = datagen.shuffle(1024) if not test else datagen\n    datagen = datagen.batch(CFG[\"BATCH_SIZE\"])\n    datagen = datagen.prefetch(tf.data.AUTOTUNE )\n    \n    return datagen","metadata":{"execution":{"iopub.status.busy":"2021-08-28T15:36:02.466748Z","iopub.execute_input":"2021-08-28T15:36:02.467052Z","iopub.status.idle":"2021-08-28T15:36:02.475929Z","shell.execute_reply.started":"2021-08-28T15:36:02.467025Z","shell.execute_reply":"2021-08-28T15:36:02.474668Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"master_df[\"path_new\"]= master_df[\"id\"].parallel_apply( lambda x : \"../input/g2net-train-images-with-gpwy-sample/kaggle/tmp/train/\" + x+\".png\")","metadata":{"execution":{"iopub.status.busy":"2021-08-28T15:36:02.477719Z","iopub.execute_input":"2021-08-28T15:36:02.478313Z","iopub.status.idle":"2021-08-28T15:36:03.318184Z","shell.execute_reply.started":"2021-08-28T15:36:02.478260Z","shell.execute_reply":"2021-08-28T15:36:03.316979Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"png_file_list = [ x.split(\"/\")[-1].replace(\".png\",\"\") for x in glob.glob(\"../input/g2net-train-images-with-gpwy-sample/kaggle/tmp/train/*.png\")]\nmaster_df_2 = pd.merge( master_df, pd.DataFrame({\"png_file\": png_file_list,\"status\":[1]* len( png_file_list) } ), left_on= \"id\",right_on=\"png_file\" )","metadata":{"execution":{"iopub.status.busy":"2021-08-28T15:36:03.319706Z","iopub.execute_input":"2021-08-28T15:36:03.320263Z","iopub.status.idle":"2021-08-28T15:36:08.732660Z","shell.execute_reply.started":"2021-08-28T15:36:03.320209Z","shell.execute_reply":"2021-08-28T15:36:08.731565Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"master_df_2[\"path_new\"] = master_df_2[\"id\"].parallel_apply( lambda x : \"../input/g2net-train-images-with-gpwy-sample/kaggle/tmp/train/\" + x+\".png\")\ntrain_df, val_df = train_test_split( master_df_2, test_size =0.1, random_state = CFG[\"RANDOM_STATE\"],shuffle = True,stratify = master_df_2[\"target\"])","metadata":{"execution":{"iopub.status.busy":"2021-08-28T15:36:08.734782Z","iopub.execute_input":"2021-08-28T15:36:08.735312Z","iopub.status.idle":"2021-08-28T15:36:09.773672Z","shell.execute_reply.started":"2021-08-28T15:36:08.735266Z","shell.execute_reply":"2021-08-28T15:36:09.772425Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig,axis = plt.subplots( nrows = 1, ncols = 2,figsize =( 15,5), sharey = True ) \nsns.countplot(  val_df[\"target\"], ax = axis[0])\nsns.countplot( train_df[\"target\"], ax = axis[1])\naxis[0].set_title(\"Number of Validation Sample with distribution \")\naxis[1].set_title(\"Number of Test Sample with distribution \")\nplt.tight_layout( )\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-08-28T15:36:09.775628Z","iopub.execute_input":"2021-08-28T15:36:09.776107Z","iopub.status.idle":"2021-08-28T15:36:10.302367Z","shell.execute_reply.started":"2021-08-28T15:36:09.776053Z","shell.execute_reply":"2021-08-28T15:36:10.301297Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def short_effnet_model():\n    #with strategy.scope():\n        \n    model_input = tf.keras.layers.Input( shape=  ( CFG[\"IMG_LENGTH\"],CFG[\"IMG_WIDTH\"], 3 ) , name= \"encoder_input_layer\" )\n\n\n    efff_net =efn.EfficientNetB0(include_top = False, \n                                   weights =\"noisy-student\" , \n                                   input_shape = ( CFG[\"IMG_LENGTH\"], CFG[\"IMG_WIDTH\"], CFG[\"CHANNELS\"]) ,\n                                   input_tensor = model_input ,\n                                   classes=2,\n                                   pooling = True,\n                                   #classifier_activation='softmax',\n                                   drop_connect_rate= 0.7\n                                  ) \n\n    for layer in  efff_net.layers  : layer.trainable = True\n    \n    gaussian_noise = tf.keras.layers.GaussianNoise( stddev = 0.3 ) ( model_input )\n    random_crop = tf.keras.layers.experimental.preprocessing.RandomCrop( height = 30, width = 30  ) (gaussian_noise)\n    random_flip =tf.keras.layers.experimental.preprocessing.RandomFlip( mode=\"horizontal_and_vertical\") ( random_crop )\n    zoom_layer = tf.keras.layers.experimental.preprocessing.RandomZoom(  height_factor =(-0.3, -0.2)  , width_factor=(-0.3, -0.2), fill_mode='reflect', interpolation='bilinear', fill_value=0.0 ) ( random_flip)\n    random_contrast = tf.keras.layers.experimental.preprocessing.RandomContrast( factor =[0.2, 0.8 ]  ) ( zoom_layer )\n    \n    efff_net.layers[0] ( random_contrast )\n    layer_00 = efff_net.layers[-1].output\n    layer_01 = tf.keras.layers.Flatten()( layer_00 )\n    layer_02 = tf.keras.layers.Dense( 1, activation =\"sigmoid\") ( layer_01)\n    model_short = tf.keras.Model( inputs = model_input, outputs = layer_02 )\n\n    optimizer = tf.keras.optimizers.Adam(learning_rate= 0.00126000004/4 ) \n    model_short.compile( optimizer= optimizer,loss=tf.keras.losses.BinaryCrossentropy(from_logits=True), \n                 metrics=[tf.keras.metrics.AUC() ])#AUC(curve='ROC')\n    \n    return model_short","metadata":{"execution":{"iopub.status.busy":"2021-08-28T15:36:10.303816Z","iopub.execute_input":"2021-08-28T15:36:10.304357Z","iopub.status.idle":"2021-08-28T15:36:10.318880Z","shell.execute_reply.started":"2021-08-28T15:36:10.304261Z","shell.execute_reply":"2021-08-28T15:36:10.317497Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_1 = short_effnet_model()\nmodel_1.summary()","metadata":{"execution":{"iopub.status.busy":"2021-08-28T15:36:10.320790Z","iopub.execute_input":"2021-08-28T15:36:10.321337Z","iopub.status.idle":"2021-08-28T15:36:13.004343Z","shell.execute_reply.started":"2021-08-28T15:36:10.321275Z","shell.execute_reply":"2021-08-28T15:36:13.002907Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.shape[0]/10000","metadata":{"execution":{"iopub.status.busy":"2021-08-28T15:36:13.006549Z","iopub.execute_input":"2021-08-28T15:36:13.007095Z","iopub.status.idle":"2021-08-28T15:36:13.015542Z","shell.execute_reply.started":"2021-08-28T15:36:13.007040Z","shell.execute_reply":"2021-08-28T15:36:13.014338Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if False:\n    \n    CFG[\"BATCH_SIZE\"]  = 50\n    train_df_2 = train_df.head(50000)\n    val_df_2 = val_df.head( 50000 )\n    train_data_gen = datagenerator( train_df_2 )\n    val_data_gen = datagenerator( val_df_2 )\n\n    model_effnet = short_effnet_model()\n    lr_reducer = tf.keras.callbacks.ReduceLROnPlateau(  patience=2,\n                                                        min_lr= 0.000001,\n                                                        monitor='val_loss', \n                                                        factor=0.45, \n                                                        verbose=1,\n                                                        min_delta = 0.2,\n                                                        cooldown=2,\n                                                        mode='auto', \n                                                       )\n\n\n\n\n\n\n\n    CFG[\"TRAIN_STEPS\"] = int ( train_df_2.shape[0] /CFG[\"BATCH_SIZE\"] ) + (1 if train_df_2.shape[0] % CFG[\"BATCH_SIZE\"] != 0 else 0)\n    CFG[\"VAL_STEPS\"] = int ( val_df_2.shape[0]/CFG[\"BATCH_SIZE\"] ) + (1 if val_df_2.shape[0]% CFG[\"BATCH_SIZE\"] != 0 else 0)\n\n\n\n    #model_effnet.load_weights(\"../input/seti-gpu-rev-01-model/Efficient_Net_Model_Rev_01.h5\")\n    checkpoint = tf.keras.callbacks.ModelCheckpoint( f'model{1}.h5', save_best_only=True, monitor='val_loss', mode='min')\n\n\n    model_history = model_effnet.fit( train_data_gen ,\n                            #class_weight= class_weight ,\n                             steps_per_epoch= CFG[\"TRAIN_STEPS\"], \n                             epochs =11, \n                             validation_data= val_data_gen,\n                             validation_steps = CFG[\"VAL_STEPS\"],\n                             callbacks=[ checkpoint,lr_reducer ]\n                           )\n","metadata":{"execution":{"iopub.status.busy":"2021-08-28T15:36:13.084545Z","iopub.status.idle":"2021-08-28T15:36:13.085036Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Code for TF record ","metadata":{}},{"cell_type":"code","source":"CREATE_TF_RECORD = True\n\ndef _bytes_feature(value):\n    \"\"\"Returns a bytes_list from a string / byte.\"\"\"\n    #if isinstance(value, type(tf.constant(0))):\n        #value = value.numpy() # BytesList won't unpack a string from an EagerTensor.\n    return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value]))\n\ndef _float_feature(value):\n    \"\"\"Returns a float_list from a float / double.\"\"\"\n    return tf.train.Feature(float_list=tf.train.FloatList(value=[value]))\n\ndef _int64_feature(value):\n    \"\"\"Returns an int64_list from a bool / enum / int / uint.\"\"\"\n    return tf.train.Feature(int64_list=tf.train.Int64List(value=[value]))\n\ndef train_serialize_example(image, img_id, target ):\n    feature = {\n      'image'         : _bytes_feature(image),\n      'image_id'      : _bytes_feature(img_id),   \n      'target'        : _int64_feature(target),\n      }\n    example_proto = tf.train.Example(features=tf.train.Features(feature=feature))\n    return example_proto.SerializeToString()\n\n\nif CREATE_TF_RECORD :\n    \n    each_tfrec_size = 1000000\n    \n    if (master_df_2.shape[0] % each_tfrec_size) != 0 :\n        end  = 1 + (master_df_2.shape[0] % each_tfrec_size)\n    else:\n        end = master_df_2.shape[0] % each_tfrec_size\n        \n        \n    for i in range( 0, end):\n        \n        \n        if ((i +1)* each_tfrec_size) > master_df_2.shape[0]:\n            \n            img_id = master_df_2[\"id\"].iloc[ i*each_tfrec_size : ].values\n            file_name = master_df_2[\"path_new\"].iloc[ i*each_tfrec_size : ].values\n            target = master_df_2[\"target\"].iloc[ i*each_tfrec_size : ].values\n            \n        else:\n            \n            img_id= master_df_2[\"id\"].iloc[ i*each_tfrec_size : each_tfrec_size *(i +1)].values\n            file_name = master_df_2[\"path_new\"].iloc[ i*each_tfrec_size : each_tfrec_size *(i +1)].values\n            target = master_df_2[\"target\"].iloc[ i*each_tfrec_size : each_tfrec_size *(i +1)].values\n        \n        \n    \n        with tf.io.TFRecordWriter( \"./300x300_tf_record/300x300_tfrecord_\" +str( i) + \".tfrec\" ) as writer:\n            \n                    for each_id, each_file, each_target  in zip( img_id,file_name, target ):\n                        \n                        writer.write(train_serialize_example( plt.imread( each_file).astype( np.float32).tobytes() , str.encode(each_id ), each_target ))\n                        \n                    writer.close()\n            \n        print ( \"completed slice {}/{}\".format( i,end))\n    ","metadata":{"execution":{"iopub.status.busy":"2021-08-28T15:36:26.716152Z","iopub.execute_input":"2021-08-28T15:36:26.716524Z","iopub.status.idle":"2021-08-28T15:36:26.768781Z","shell.execute_reply.started":"2021-08-28T15:36:26.716495Z","shell.execute_reply":"2021-08-28T15:36:26.766989Z"},"trusted":true},"execution_count":null,"outputs":[]}]}