{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# G2Net Gravitational Wave Detection: Creation of CWT transform of the gravitational signal\n\nThe very first question that comes to mind is that what are gravitational waves?. The precise and crisp definition of gravitational waves is given in Wikipedia [1](https://en.wikipedia.org/wiki/Gravitational_wave), and it is as follows, \n\"***Gravitational waves are disturbances in the curvature of spacetime, generated by accelerated masses, that propagate as waves outward from their source at the speed of light.*** \"\n","metadata":{}},{"cell_type":"code","source":"# Import basic libraries\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport os\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom glob import glob\nfrom PIL import Image\n# Importing all the training_labels.csv \ndf_train_label=pd.read_csv('../input/g2net-gravitational-wave-detection/training_labels.csv')\ndf_train_label.head()","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-09-17T09:22:59.279736Z","iopub.execute_input":"2021-09-17T09:22:59.280102Z","iopub.status.idle":"2021-09-17T09:23:00.782908Z","shell.execute_reply.started":"2021-09-17T09:22:59.280006Z","shell.execute_reply":"2021-09-17T09:23:00.781865Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Importing all the address to the files .npy present inside the train folder","metadata":{}},{"cell_type":"code","source":"# path of the files\npaths_files = glob(\"../input/g2net-gravitational-wave-detection/train/*/*/*/*\")","metadata":{"execution":{"iopub.status.busy":"2021-09-17T09:23:00.785128Z","iopub.execute_input":"2021-09-17T09:23:00.785789Z","iopub.status.idle":"2021-09-17T09:25:58.181247Z","shell.execute_reply.started":"2021-09-17T09:23:00.785705Z","shell.execute_reply":"2021-09-17T09:25:58.180291Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Creating a dataframe path_df, where one column stores the ids and another column holds path to those .npy files","metadata":{}},{"cell_type":"code","source":"# the ids will store the ids of each of the files\nids=[]\nfor filext in paths_files:\n    ids.append(filext[filext.rindex('/')+1:\\\n                              len(filext)].replace('.npy',''))\n    \n# data frame containing paths and ids of .npy files \npath_df = pd.DataFrame({\"id\":ids,\"path\":paths_files})\npath_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-09-17T09:25:58.182516Z","iopub.execute_input":"2021-09-17T09:25:58.182787Z","iopub.status.idle":"2021-09-17T09:25:58.877164Z","shell.execute_reply.started":"2021-09-17T09:25:58.182758Z","shell.execute_reply":"2021-09-17T09:25:58.876053Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Merging the df_train_label and the path_df dataframes into the df dataframe","metadata":{}},{"cell_type":"code","source":"df=pd.merge(path_df,df_train_label,on='id')\ndel path_df, df_train_label;\ndf.head()","metadata":{"execution":{"iopub.status.busy":"2021-09-17T09:25:58.879838Z","iopub.execute_input":"2021-09-17T09:25:58.880213Z","iopub.status.idle":"2021-09-17T09:25:59.614101Z","shell.execute_reply.started":"2021-09-17T09:25:58.880167Z","shell.execute_reply":"2021-09-17T09:25:59.613490Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.shape","metadata":{"execution":{"iopub.status.busy":"2021-09-17T09:25:59.615143Z","iopub.execute_input":"2021-09-17T09:25:59.615483Z","iopub.status.idle":"2021-09-17T09:25:59.620708Z","shell.execute_reply.started":"2021-09-17T09:25:59.615456Z","shell.execute_reply":"2021-09-17T09:25:59.620005Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### For the purpose of training our model we consider smaller sample of the larger training dataset, therefore we randomly choose 60000 target=1 .npy files and 60000 target=0 .npy files.","metadata":{}},{"cell_type":"code","source":"df=pd.concat([df[df['target']==1][['id','path','target']].sample(n=60000, random_state=0),\\\n          df[df['target']==0][['id','path','target']].sample(n=60000, random_state=0)]).sample(frac=1,random_state=0)\ndf.head()","metadata":{"execution":{"iopub.status.busy":"2021-09-17T09:25:59.621688Z","iopub.execute_input":"2021-09-17T09:25:59.622363Z","iopub.status.idle":"2021-09-17T09:25:59.822276Z","shell.execute_reply.started":"2021-09-17T09:25:59.622331Z","shell.execute_reply":"2021-09-17T09:25:59.821446Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.shape","metadata":{"execution":{"iopub.status.busy":"2021-09-17T09:25:59.823340Z","iopub.execute_input":"2021-09-17T09:25:59.823552Z","iopub.status.idle":"2021-09-17T09:25:59.828822Z","shell.execute_reply.started":"2021-09-17T09:25:59.823525Z","shell.execute_reply":"2021-09-17T09:25:59.827823Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Continuous Wavelet Transform of smaller training set ","metadata":{}},{"cell_type":"code","source":"import tensorflow as tf\nfrom tensorflow.keras.utils import Sequence\nfrom tensorflow.keras.models import Sequential\nfrom tensorflow.keras.layers import *\nfrom tensorflow.keras.optimizers import RMSprop,Adam\nfrom tensorflow.keras.callbacks import ReduceLROnPlateau,ModelCheckpoint","metadata":{"execution":{"iopub.status.busy":"2021-09-17T09:25:59.829781Z","iopub.execute_input":"2021-09-17T09:25:59.830491Z","iopub.status.idle":"2021-09-17T09:26:05.582208Z","shell.execute_reply.started":"2021-09-17T09:25:59.830455Z","shell.execute_reply":"2021-09-17T09:26:05.581065Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The following piece of code was originally written by Alexander Neergaard Olesen's CWT Python implementation, and reference is given [here](https://github.com/neergaard/CWT). The Keras implementation of the above code was done by Geir Drange. The reference and the functionaluty of the code is given [here](https://www.kaggle.com/mistag/wavelet1d-custom-keras-wavelet-transform-layer). \n\nThere maybe two question that may have struck your mind,\n1. Why not perform Fourier Transform (FT)?.\n\nAns. Fourier Transform is more suited for stationary waves. It gives complete resolution in frequency but complete uncertainity in time.\n\n2. What is the upper hand of using CWT over FT?.\n\nAns. CWT is more suited for non stationary waves. The CWT is a multiresolution analysis where we have good time resolution and poor frequency resolution at high frequencies\nand good frequency resolution and poor time resolution at low frequencies. \n\nMore detail analysis of FT and CWT has been elegentaly explained in the, \"***POLIKAR, ROBI. \"The Wavelet Tutorial Second Edition Part I.***\" [2](https://web.iitd.ac.in/~sumeet/WaveletTutorial.pdf). The mathematical details of the following code is explained briefly in the following article, \"***Torrence, Christopher, and Gilbert P. Compo. \"A practical guide to wavelet analysis.\" Bulletin of the American Meteorological society 79.1 (1998): 61-78.***\"","metadata":{}},{"cell_type":"code","source":"%%writefile tf_cwt.py\n# Based on Alexander Neergaard Olesen's CWT Python implementation. https://github.com/neergaard/CWT\n# Adapted to Keras by Geir Drange\n# MIT License\n\nimport tensorflow as tf\nfrom tensorflow import keras\nimport tensorflow.experimental.numpy as tnp\nimport numpy as np\nimport math\n\n# calculate CWT of input signal\nclass Wavelet1D(keras.layers.Layer):\n    def __init__(self, nv=12, sr=1., flow=0., fhigh=0.5, batch_size=None, trainable=False):\n        super(Wavelet1D, self).__init__()\n        assert fhigh > flow, 'fhigh parameters must be > flow!'\n        assert batch_size != None, 'batch size must be set!'\n        \n        self.batch_size = batch_size \n        self.nv = nv # number of voices\n        self.sr = sr # sample rate (Hz)\n        self.flow = flow # lowest frequency of interest (Hz)\n        self.fhigh = fhigh # highest frequency of interest (Hz)\n        self.trainable = trainable # True to train the wavelet filter bank\n\n    def build(self, input_shape):\n        assert len(input_shape) == 2,\\\n        'Input dimension must be 2! Dimension is {}'.format(len(input_shape))\n        \n        max_scale = input_shape[-1] // (np.sqrt(2) * 2)\n        if max_scale <= 1:\n            max_scale = input_shape[-1] // 2\n        max_scale = np.floor(self.nv * np.log2(max_scale))\n        scales = 2 * (2**(1/self.nv)) ** np.arange(0, max_scale + 1)\n        frequencies = self.sr * (6 / (2 * np.pi)) / scales\n        frequencies = frequencies[frequencies >= self.flow] # remove low frequencies\n        scales = scales[0:len(frequencies)]\n        frequencies = frequencies[frequencies <= self.fhigh] # remove high frequencies\n        scales = scales[len(scales)-len(frequencies):len(scales)]\n        # wavft\n        padvalue = input_shape[-1] // 2\n        n = padvalue*2+input_shape[-1]\n        omega = np.arange(1, math.floor(n / 2) + 1, dtype=np.float64)\n        omega = omega*(2 * np.pi) / n\n        omega = np.concatenate((np.array([0]), omega, -omega[\\\n                                                             np.arange(math.floor((n - 1) / 2),\\\n                                                                       0, -1, dtype=int) - 1]))\n        _wft = np.zeros([scales.size, omega.size])\n        for jj, scale in enumerate(scales):\n            expnt = -(scale * omega - 6) ** 2 / 2 * (omega > 0)\n            _wft[jj, ] = 2 * np.exp(expnt) * (omega > 0)\n        # parameters we want to use during call():\n        self.wft = tf.Variable(_wft, trainable=self.trainable) # yes, the wavelets can be trainable if desired\n        self.padvalue = padvalue\n        self.num_scales = scales.shape[-1]\n    \n    # uses a loop - better to vectorize (TODO)\n    def call(self, inputs):\n        max_loop = tf.shape(inputs)[0]\n        \n        def sum_cwt(i, pre_data):\n            next_data = tf.nn.embedding_lookup(inputs, i) \n            x = tf.concat([tf.reverse(next_data[0:self.padvalue], axis=[0]),\\\n                           next_data, tf.reverse(next_data[-self.padvalue:], axis=[0])], 0)\n            f = tf.signal.fft(tf.cast(x, tf.complex64))\n            cwtcfs = tf.signal.ifft(tnp.kron(tf.ones([self.num_scales, 1],\\\n                                                     dtype=tf.complex64)\\\n                                             , f) * tf.cast(self.wft, tf.complex64))\n            logcwt = tf.math.log(tf.math.abs(cwtcfs[:,\\\n                                                    self.padvalue:self.padvalue + next_data.shape[-1]]))\n            pre_data = tf.tensor_scatter_nd_add(pre_data, indices=[[i]], updates=[logcwt])\n            i_next = i + 1\n            return i_next, pre_data\n                                 \n        _, cwt = tf.while_loop(cond = lambda i, result: tf.less(i, max_loop),\n                              body = sum_cwt,\n                              loop_vars = (tf.constant(0, dtype=tf.int32),\\\n                                           tf.zeros([self.batch_size, self.num_scales,\\\n                                                     inputs.shape[-1]], dtype = tf.float32)))\n        return cwt\n    \n# scale input to range 0.0 - upper\nclass Scaler(keras.layers.Layer):\n    def __init__(self, upper=1.0):\n        super(Scaler, self).__init__()\n        self.upper = tf.cast(upper, dtype=tf.float32) # upper value (typically 1.0 or 255.0 for image CNNs)\n    \n    def call(self, inputs):\n        min_val = tf.math.reduce_min(inputs)\n        max_val = tf.math.reduce_max(tf.math.subtract(inputs, min_val))\n        return tf.math.multiply(tf.math.subtract(inputs, min_val), self.upper/max_val)\n    \n# Stack three channels into RGB image\nclass RGBStack(keras.layers.Layer):\n    def __init__(self):\n        super(RGBStack, self).__init__()\n        \n    def call(self, inputs):\n        return tf.stack(inputs, axis = 3)","metadata":{"execution":{"iopub.status.busy":"2021-09-17T09:26:05.583700Z","iopub.execute_input":"2021-09-17T09:26:05.583978Z","iopub.status.idle":"2021-09-17T09:26:05.594136Z","shell.execute_reply.started":"2021-09-17T09:26:05.583947Z","shell.execute_reply":"2021-09-17T09:26:05.592856Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Importing the customized Keras layers","metadata":{}},{"cell_type":"code","source":"from tf_cwt import Wavelet1D, Scaler, RGBStack","metadata":{"execution":{"iopub.status.busy":"2021-09-17T09:26:05.597328Z","iopub.execute_input":"2021-09-17T09:26:05.597617Z","iopub.status.idle":"2021-09-17T09:26:05.616066Z","shell.execute_reply.started":"2021-09-17T09:26:05.597587Z","shell.execute_reply":"2021-09-17T09:26:05.615014Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Now we going to load one of the strong gravitational wave signal","metadata":{}},{"cell_type":"code","source":"# Loading the first .npy data\ndata=np.load(df[df['target']==1]['path'].iloc[3])\ndata","metadata":{"execution":{"iopub.status.busy":"2021-09-17T09:26:05.617890Z","iopub.execute_input":"2021-09-17T09:26:05.618196Z","iopub.status.idle":"2021-09-17T09:26:05.658401Z","shell.execute_reply.started":"2021-09-17T09:26:05.618155Z","shell.execute_reply":"2021-09-17T09:26:05.657337Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Following part of the code gives us the visualization how the Number of voices per octave (nv), Low frequency (flow), and High frequency (fhigh) affect the CWT of an input signal. ","metadata":{}},{"cell_type":"code","source":"# The following piece of code is taken from  Geir Drange's Notebook and hyperlink is given below,\n# https://www.kaggle.com/mistag/wavelet1d-custom-keras-wavelet-transform-layer\nfig = plt.figure(figsize=(16,22))\nidx = 0\nfor nv in [8,16]:\n    for flow in [8,32]:\n        for fhigh in [500, 1000]:\n            y = Wavelet1D(nv=nv, sr=2048., flow=flow, fhigh=fhigh, batch_size=1)\\\n            (tf.expand_dims(data[0,:], axis=0))\n            y = Scaler(upper=1)(y)\n            ax = plt.subplot(4, 2, 1+idx)\n            plt.xlabel('Sample')\n            plt.ylabel('Scale')\n            plt.title('Voices: {}, Flow: {}Hz, Fhigh: {}Hz'.format(nv, flow, fhigh))\n            plt.imshow(np.squeeze(y.numpy()), cmap='magma', aspect='auto') \n            idx += 1","metadata":{"execution":{"iopub.status.busy":"2021-09-17T09:26:05.660199Z","iopub.execute_input":"2021-09-17T09:26:05.660547Z","iopub.status.idle":"2021-09-17T09:26:09.788927Z","shell.execute_reply.started":"2021-09-17T09:26:05.660501Z","shell.execute_reply":"2021-09-17T09:26:09.787489Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### From the above graphs it is quite clear that by setting suitable value for lower frequecy (flow) we can remove the low frequency noise. ","metadata":{}},{"cell_type":"markdown","source":"### Downsampling\nThe scaleogram has the same width as the input signal. This needs to be reduced to our target image size (here we choose 224). Rather than doing an image resize type operation, we can use several types of layers:\n\n1. Max pooling\n2. Average pooling\n3. 2D convolution (this can also be trained!)\nThe trick is to set the pool/filter size to (1,n), where n is the reduction factor, and stride to the same size. Let's try it out:","metadata":{}},{"cell_type":"code","source":"# The following piece of code is taken from  Geir Drange's Notebook and hyperlink is given below,\n# https://www.kaggle.com/mistag/wavelet1d-custom-keras-wavelet-transform-layer\nTARGET_IMG_SIZE = 224\nSTRIDE = int(np.ceil(4096/TARGET_IMG_SIZE))\n\ny1 = Wavelet1D(nv=16, sr=2048., flow=20, fhigh=750, batch_size=1)(tf.expand_dims(data[0,:], axis=0))\ny1 = Scaler(upper=1)(y1)\ny1 = tf.reshape(y1, [y1.shape[0],y1.shape[1],y1.shape[2],1])\nfig = plt.figure(figsize=(10,12))\nax = plt.subplot(3, 1, 1)\ny2a = MaxPool2D(pool_size=(1,STRIDE), strides=(1,STRIDE))(y1)\ny2a = Scaler(upper=1)(y2a)\nplt.title('Max Pooling')\nplt.imshow(tf.reshape(y2a, [y2a.shape[1],y2a.shape[2]]).numpy(), cmap='magma', aspect='auto')\nax = plt.subplot(3, 1, 2)\ny2b = AvgPool2D(pool_size=(1,STRIDE), strides=(1,STRIDE))(y1)\ny2b = Scaler(upper=1)(y2b)\nplt.title('Average Pooling')\nplt.imshow(tf.reshape(y2b, [y2b.shape[1],y2b.shape[2]]).numpy(), cmap='magma', aspect='auto') \nax = plt.subplot(3, 1, 3)\ny2c = Conv2D(filters=1, kernel_size=(1,STRIDE), strides=(1,STRIDE))(y1)\ny2c = Scaler(upper=1)(y2c)\nplt.title('2D Convolution (untrained)')\nplt.imshow(tf.reshape(y2c, [y2c.shape[1],y2c.shape[2]]).numpy(), cmap='magma', aspect='auto');","metadata":{"execution":{"iopub.status.busy":"2021-09-17T09:26:09.792210Z","iopub.execute_input":"2021-09-17T09:26:09.792562Z","iopub.status.idle":"2021-09-17T09:26:10.952368Z","shell.execute_reply.started":"2021-09-17T09:26:09.792520Z","shell.execute_reply":"2021-09-17T09:26:10.950897Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"r = Wavelet1D(nv=16, sr=2048., flow=20, fhigh=10000, batch_size=1)(tf.expand_dims(data[0,:], axis=0))\nr = AvgPool2D(pool_size=(1,STRIDE), strides=(1,STRIDE))(tf.reshape(r, [r.shape[0],r.shape[1],r.shape[2],1]))\nr = Scaler(upper=1)(r)\nr = tf.reshape(r, [r.shape[1],r.shape[2]])\ng = Wavelet1D(nv=16, sr=2048., flow=20, fhigh=10000, batch_size=1)(tf.expand_dims(data[1,:], axis=0))\ng = AvgPool2D(pool_size=(1,STRIDE), strides=(1,STRIDE))(tf.reshape(g, [g.shape[0],g.shape[1],g.shape[2],1]))\ng = Scaler(upper=1)(g)\ng = tf.reshape(g, [g.shape[1],g.shape[2]])\nb = Wavelet1D(nv=16, sr=2048., flow=20, fhigh=10000, batch_size=1)(tf.expand_dims(data[2,:], axis=0))\nb = AvgPool2D(pool_size=(1,STRIDE), strides=(1,STRIDE))(tf.reshape(b, [b.shape[0],b.shape[1],b.shape[2],1]))\nb = Scaler(upper=1)(b)\nb = tf.reshape(b, [b.shape[1],b.shape[2]])\nrgb = tf.stack([r, g, b], axis = 2)\nfig = plt.figure(figsize=(10,5))\nplt.title('RGB image constructed from the 3 detectors')\nplt.imshow(rgb.numpy(), cmap='magma', aspect='auto');","metadata":{"execution":{"iopub.status.busy":"2021-09-17T09:26:10.953952Z","iopub.execute_input":"2021-09-17T09:26:10.954342Z","iopub.status.idle":"2021-09-17T09:26:11.888238Z","shell.execute_reply.started":"2021-09-17T09:26:10.954300Z","shell.execute_reply":"2021-09-17T09:26:11.887257Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Combing the output of the 3 detectors into a RBG image as shown below,","metadata":{}},{"cell_type":"markdown","source":"**Now we construct a cwt model which perform all the above operation to finally obtain the RBG image.**","metadata":{}},{"cell_type":"code","source":"# The following piece of code is taken from  Geir Drange's Notebook and hyperlink is given below,\n# https://www.kaggle.com/mistag/wavelet1d-custom-keras-wavelet-transform-layer\nTARGET_IMG_SIZE = 224 # image size expected by the CNN model\n# skip start to make img_size integer multiple of 4096\nOFFSET = 4096-int(np.floor(4096/224))*TARGET_IMG_SIZE\nSTRIDE = (4096-OFFSET)//TARGET_IMG_SIZE\nBATCH_SIZE=20\n\n# CTW model\ndef build_cwt_model(nv=16, flow=20, fhigh=10000, batch_size=64):\n    inputs = Input(shape=(3, 4096))\n    # channels\n    r = Wavelet1D(nv=nv, sr=2048., flow=flow, fhigh=fhigh, batch_size=batch_size)(inputs[:,0,OFFSET:])  \n    g = Wavelet1D(nv=nv, sr=2048., flow=flow, fhigh=fhigh, batch_size=batch_size)(inputs[:,1,OFFSET:]) \n    b = Wavelet1D(nv=nv, sr=2048., flow=flow, fhigh=fhigh, batch_size=batch_size)(inputs[:,2, OFFSET:]) \n    # combine into rgb\n    rgb = RGBStack()([r, g, b])\n    # downsample\n    rgb = AvgPool2D(pool_size=(1,STRIDE), strides=(1,STRIDE))(rgb) \n    rgb = Scaler(upper=255.)(rgb) # adjust 'upper' according to CNN \n    \n    return tf.keras.Model(inputs, rgb, name=\"Wavlet\")\n\ncwt_model = build_cwt_model(batch_size=BATCH_SIZE)\ncwt_model.summary()","metadata":{"execution":{"iopub.status.busy":"2021-09-17T09:26:11.889650Z","iopub.execute_input":"2021-09-17T09:26:11.889949Z","iopub.status.idle":"2021-09-17T09:26:12.561733Z","shell.execute_reply.started":"2021-09-17T09:26:11.889918Z","shell.execute_reply":"2021-09-17T09:26:12.560910Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Now the question raises how to convert the smaller subset of these .npy files in to the required RGB image of the CWT. In the following section we address the preceding question.","metadata":{}},{"cell_type":"markdown","source":"Directly training our model on the .npy files takes a lot of time because loading the data takes a lot of time than performing the ML computations. Mainly the ML computation are done on a GPU, and loading the data task is done by CPU. Former data pipelines made the GPU wait for the CPU to load the data, leading to performance issues [ref.(2)](https://cs230.stanford.edu/blog/datapipeline/). Therefore, the tf.data API enables you to build complex input pipelines from simple, reusable pieces [ref.(4)](https://www.tensorflow.org/guide/data). But this (tf.data) API lack the feature of reading .npy files which doesn't fit in the memory. However, I found a solution in the stackover flow, and link of the solution is given [here](https://stackoverflow.com/questions/48889482/feeding-npy-numpy-files-into-tensorflow-data-pipeline).","metadata":{}},{"cell_type":"code","source":"def npy_header_offset(npy_path):\n    with open(str(npy_path), 'rb') as f:\n        if f.read(6) != b'\\x93NUMPY':\n            raise ValueError('Invalid NPY file.')\n        version_major, version_minor = f.read(2)\n        if version_major == 1:\n            header_len_size = 2\n        elif version_major == 2:\n            header_len_size = 4\n        else:\n            raise ValueError('Unknown NPY file version {}.{}.'.format(version_major, version_minor))\n        header_len = sum(b << (8 * i) for i, b in enumerate(f.read(header_len_size)))\n        header = f.read(header_len)\n        if not header.endswith(b'\\n'):\n            raise ValueError('Invalid NPY file.')\n        return f.tell()","metadata":{"execution":{"iopub.status.busy":"2021-09-17T09:26:12.562888Z","iopub.execute_input":"2021-09-17T09:26:12.563129Z","iopub.status.idle":"2021-09-17T09:26:12.571596Z","shell.execute_reply.started":"2021-09-17T09:26:12.563101Z","shell.execute_reply":"2021-09-17T09:26:12.570363Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"header_size = npy_header_offset(df['path'].iloc[0])\nheader_size","metadata":{"execution":{"iopub.status.busy":"2021-09-17T09:26:12.573585Z","iopub.execute_input":"2021-09-17T09:26:12.573978Z","iopub.status.idle":"2021-09-17T09:26:12.599831Z","shell.execute_reply.started":"2021-09-17T09:26:12.573933Z","shell.execute_reply":"2021-09-17T09:26:12.598760Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Creating two separate directories to store the RBG images for the .npy files with target=1 in the target1 folder, and for the .npy files with target=0 in the target0 folder ","metadata":{}},{"cell_type":"code","source":"os.makedirs('./target0',exist_ok=True)\nos.makedirs('./target1',exist_ok=True)","metadata":{"execution":{"iopub.status.busy":"2021-09-17T09:26:12.601470Z","iopub.execute_input":"2021-09-17T09:26:12.601970Z","iopub.status.idle":"2021-09-17T09:26:12.607404Z","shell.execute_reply.started":"2021-09-17T09:26:12.601932Z","shell.execute_reply":"2021-09-17T09:26:12.606343Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df[df['target']==1]['path'].iloc[59000:60000].tail()","metadata":{"execution":{"iopub.status.busy":"2021-09-17T09:26:12.608917Z","iopub.execute_input":"2021-09-17T09:26:12.609202Z","iopub.status.idle":"2021-09-17T09:26:12.630651Z","shell.execute_reply.started":"2021-09-17T09:26:12.609172Z","shell.execute_reply":"2021-09-17T09:26:12.629849Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#import time\ncount=0\nfor i in range(0,int(len(df)/2),1000):    \n    #start=time.perf_counter()\n    # Importing those .npy files which has target=1\n    tf_target1=tf.data.FixedLengthRecordDataset( df[df['target']==1]['path'].iloc[i:i+1000],\\\n                                                   3*4096*tf.float64.size,\\\n                                             header_bytes=header_size,\\\n                                                num_parallel_reads=BATCH_SIZE)\n    tf_target1 = tf_target1.map(lambda s: tf.reshape(\\\n                                                           tf.io.decode_raw(s, tf.float64),\\\n                                                           (3,4096)))\n    # Importing those .npy files which has target=0\n    tf_target0=tf.data.FixedLengthRecordDataset( df[df['target']==0]['path'].iloc[i:i+1000],\\\n                                                   3*4096*tf.float64.size,\\\n                                             header_bytes=header_size,\\\n                                                num_parallel_reads=BATCH_SIZE)\n    tf_target0 = tf_target0.map(lambda s: tf.reshape(\\\n                                                           tf.io.decode_raw(s, tf.float64),\\\n                                                           (3,4096)))\n    target1 = tf_target1.batch(BATCH_SIZE).prefetch(tf.data.AUTOTUNE)\n    target0 = tf_target0.batch(BATCH_SIZE).prefetch(tf.data.AUTOTUNE)\n    # It will store the RBG images for target=1 images\n    data=cwt_model.predict(target1)\n    for ii in range(0,data.shape[0]):\n    #    start=time.perf_counter()\n        # Saving the images\n        img = Image.fromarray(data[ii,:,:,:].reshape(90,224,3).astype(np.uint8))\n        img.save(f'./target1/target1_pic{(ii+1+count)}.png')\n    #    finish=time.perf_counter()\n    #    print(\"Creating each file time = \",finish-start)\n    # It will store the RBG images for target=0 images\n    data=cwt_model.predict(target0)\n    for ii in range(0,data.shape[0]):\n        #start=time.perf_counter()\n        # Saving the images\n        img = Image.fromarray(data[ii,:,:,:].reshape(90,224,3).astype(np.uint8))\n        img.save(f'./target0/target0_pic{(ii+1+count)}.png')\n        #finish=time.perf_counter()\n    #print(\"Predict time = \",finish-start)\n    count=count+1000","metadata":{"execution":{"iopub.status.busy":"2021-08-28T16:24:19.633211Z","iopub.execute_input":"2021-08-28T16:24:19.633569Z","iopub.status.idle":"2021-08-28T16:26:12.776619Z","shell.execute_reply.started":"2021-08-28T16:24:19.633536Z","shell.execute_reply":"2021-08-28T16:26:12.775136Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### The above code runs above 4-5 hrs in GPU settings.","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}