{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"This kernel creates Constant Q Transformed (CQT) image dataset. Thanks to [AlexNitz](https://www.kaggle.com/alexnitz) for sharing his [PyCBC: Making Images](https://www.kaggle.com/alexnitz/pycbc-making-images) kernel that helped me to create this dataset. \n\nThe dataset is created by stacking the CQT vertically (`np.vstack`).\n\n### [Find the Dataset here $\\rightarrow$](http://wandb.me/g2net-cqt)\n\n### [Visualize data interatively using W&B Tables $\\rightarrow$](http://wandb.me/cqt-wandb-table-viz)\n\nLearn more about W&B Tables [here](http://wandb.me/better-tables).","metadata":{}},{"cell_type":"code","source":"# Install W&B for experiment tracking and Visualizations\n!pip install --upgrade -q wandb\n\n# Install pycbc to do easy preprocessing of the data\n# Thanks AlexNitz for intruducing this library\n!pip -q install pycbc","metadata":{"execution":{"iopub.status.busy":"2021-07-14T12:20:30.260907Z","iopub.execute_input":"2021-07-14T12:20:30.261514Z","iopub.status.idle":"2021-07-14T12:21:09.028856Z","shell.execute_reply.started":"2021-07-14T12:20:30.261478Z","shell.execute_reply":"2021-07-14T12:21:09.027422Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# General imports\nimport os\nimport gc \nimport glob\nimport numpy as np\nimport pandas as pd\nfrom PIL import Image\nimport seaborn as sns\nfrom tqdm import tqdm\nimport matplotlib.pyplot as plt\n%matplotlib inline\n\n# Deeplearning import\nimport tensorflow as tf\nprint(f'TensorFlow version: {tf.__version__}')\n\n# PyCBC imports\nimport pylab\nimport pycbc.types\n\n# Multiprocessing \nfrom multiprocessing import Pool\nfrom multiprocessing import cpu_count\n\n# W&B imports\nimport wandb\nfrom wandb.keras import WandbCallback","metadata":{"execution":{"iopub.status.busy":"2021-07-14T12:21:26.367913Z","iopub.execute_input":"2021-07-14T12:21:26.368922Z","iopub.status.idle":"2021-07-14T12:21:27.971622Z","shell.execute_reply.started":"2021-07-14T12:21:26.368853Z","shell.execute_reply":"2021-07-14T12:21:27.970464Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"TRAIN_PATH = '../input/g2net-gravitational-wave-detection/train/'\ntrain_files = glob.glob(TRAIN_PATH+'*/*/*/*')","metadata":{"execution":{"iopub.status.busy":"2021-07-14T12:21:32.308027Z","iopub.execute_input":"2021-07-14T12:21:32.308528Z","iopub.status.idle":"2021-07-14T12:23:16.645416Z","shell.execute_reply.started":"2021-07-14T12:21:32.308484Z","shell.execute_reply":"2021-07-14T12:23:16.644336Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(train_files)","metadata":{"execution":{"iopub.status.busy":"2021-07-14T12:23:41.946573Z","iopub.execute_input":"2021-07-14T12:23:41.947222Z","iopub.status.idle":"2021-07-14T12:23:41.958889Z","shell.execute_reply.started":"2021-07-14T12:23:41.947165Z","shell.execute_reply":"2021-07-14T12:23:41.957466Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_files[0]","metadata":{"execution":{"iopub.status.busy":"2021-07-14T12:27:16.019653Z","iopub.execute_input":"2021-07-14T12:27:16.020109Z","iopub.status.idle":"2021-07-14T12:27:16.029704Z","shell.execute_reply.started":"2021-07-14T12:27:16.020068Z","shell.execute_reply":"2021-07-14T12:27:16.028075Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = pd.read_csv('../input/g2net-gravitational-wave-detection/training_labels.csv')\nprint(f'Size of training_labels.csv: {len(train_df)}')\ntrain_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-07-14T12:23:49.150386Z","iopub.execute_input":"2021-07-14T12:23:49.150885Z","iopub.status.idle":"2021-07-14T12:23:49.795805Z","shell.execute_reply.started":"2021-07-14T12:23:49.150846Z","shell.execute_reply":"2021-07-14T12:23:49.794715Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"os.makedirs('train_cqt', exist_ok=True)","metadata":{"execution":{"iopub.status.busy":"2021-07-14T12:30:25.348266Z","iopub.execute_input":"2021-07-14T12:30:25.348695Z","iopub.status.idle":"2021-07-14T12:30:25.353803Z","shell.execute_reply.started":"2021-07-14T12:30:25.348651Z","shell.execute_reply":"2021-07-14T12:30:25.352660Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_constant_q_transform(file_names):\n    esp=1e-6\n    normalize=True\n    \n    for file_name in file_names:\n        example_id = file_name.split('/')[-1].split('.')[0]\n\n        # load the specific 2s sample\n        data = np.load(file_name)\n\n        channels = []\n        for i in range(3):\n            # convert the data to a TimeSeries instance\n            ts = pycbc.types.TimeSeries(data[i, :], epoch=0, delta_t=1.0/2048) \n\n            # whiten the data (i.e. normalize the noise power at different frequencies)\n            ts = ts.whiten(0.125, 0.125)\n\n            # calculate the qtransform\n            time, freq, power = ts.qtransform(.002, logfsteps=100, qrange=(10, 10), frange=(20, 512))\n\n            # normalize and scale to 0-255\n            if normalize:\n                mean = power.mean()\n                std = power.std()\n\n                power = (power - mean) / (std + esp)\n                _min, _max = power.min(), power.max()\n\n                power[power < _min] = _min\n                power[power > _max] = _max\n                power = 255 * (power - _min) / (_max - _min)\n                power = power.astype(np.uint8)\n\n            channels.append(np.flip(power, 0))\n\n        vstacked = np.vstack(channels)\n\n        im = Image.fromarray(vstacked)\n        im.save(f\"train_cqt/{example_id}.png\")","metadata":{"execution":{"iopub.status.busy":"2021-07-14T12:30:28.126600Z","iopub.execute_input":"2021-07-14T12:30:28.127324Z","iopub.status.idle":"2021-07-14T12:30:28.140372Z","shell.execute_reply.started":"2021-07-14T12:30:28.127280Z","shell.execute_reply":"2021-07-14T12:30:28.138917Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def chunk(l, n):\n    # loop over the list in n-sized chunks\n    for i in range(0, len(l), n):\n        # yield the current n-sized chunk to the calling function\n        yield l[i: i + n]","metadata":{"execution":{"iopub.status.busy":"2021-07-14T12:30:28.727376Z","iopub.execute_input":"2021-07-14T12:30:28.728092Z","iopub.status.idle":"2021-07-14T12:30:28.733026Z","shell.execute_reply.started":"2021-07-14T12:30:28.728050Z","shell.execute_reply":"2021-07-14T12:30:28.732078Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"procs = cpu_count()\nprint(procs)\nprocIDs = list(range(0, procs))\n# grab the paths to the input images, then determine the number\n# of images each process will handle\nnumImagesPerProc = len(train_files) / float(procs)\nnumImagesPerProc = int(np.ceil(numImagesPerProc))\n# chunk the image paths into N (approximately) equal sets, one\n# set of image paths for each individual process\nchunkedPaths = list(chunk(train_files, numImagesPerProc))","metadata":{"execution":{"iopub.status.busy":"2021-07-14T12:30:29.453075Z","iopub.execute_input":"2021-07-14T12:30:29.453781Z","iopub.status.idle":"2021-07-14T12:30:29.477197Z","shell.execute_reply.started":"2021-07-14T12:30:29.453738Z","shell.execute_reply":"2021-07-14T12:30:29.474546Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"[INFO] launching pool using {} processes...\".format(procs))\npool = Pool(processes=procs)\npool.map(get_constant_q_transform, chunkedPaths)","metadata":{"execution":{"iopub.status.busy":"2021-07-14T12:30:30.606597Z","iopub.execute_input":"2021-07-14T12:30:30.607279Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# close the pool and wait for all processes to finish\nprint(\"[INFO] waiting for processes to finish...\")\npool.close()\npool.join()\nprint(\"[INFO] multiprocessing complete\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}