{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## <center>G2Net competiton</center>\n\n#### <center>Data transformation and storage</center>\n\nData is provided in time series stored in numpy files. Readings have been simulated by 3 gravitational wave interferometers (LIGO Hanford, LIGO Livingston, and Virgo). \n\nIn this notebook i try to explore 3 different data transformations for time series data - \n1. Recurrence plot\n2. Gramian Angualar field plots - summation \n3. Gramian Angualar field plots - difference\n\nMore complex pre-trained models can be used on these transformations.\n\nReferences - \n* https://www.kaggle.com/ihelon/g2net-eda-and-modeling/output\n\nRead more at -\n[Imaging time series](https://pyts.readthedocs.io/en/stable/modules/image.html)\n\n#### If you find something useful or gain some insights. Please upvote.\n\n","metadata":{}},{"cell_type":"code","source":"pip install pyts","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2021-07-02T07:59:20.187231Z","iopub.execute_input":"2021-07-02T07:59:20.187573Z","iopub.status.idle":"2021-07-02T07:59:26.592197Z","shell.execute_reply.started":"2021-07-02T07:59:20.187531Z","shell.execute_reply":"2021-07-02T07:59:26.59136Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Dependencies","metadata":{}},{"cell_type":"code","source":"import os\nimport json\nimport random\nimport collections\n\nimport numpy as np\nimport pandas as pd\nimport cv2\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport cv2\nsns.set_theme(style=\"darkgrid\")\n\nfrom scipy.spatial.distance import pdist, squareform #scipy spatial distance\nimport sklearn as sk\nimport sklearn.metrics.pairwise\nfrom skimage.transform import resize\n\nfrom pyts.image import RecurrencePlot, GramianAngularField, MarkovTransitionField","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-07-02T07:59:26.593995Z","iopub.execute_input":"2021-07-02T07:59:26.59428Z","iopub.status.idle":"2021-07-02T07:59:26.601854Z","shell.execute_reply.started":"2021-07-02T07:59:26.594253Z","shell.execute_reply":"2021-07-02T07:59:26.60117Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Config:\n    data_dir = '../input/g2net-gravitational-wave-detection/{}/{}/{}/{}/{}.npy'\n    train_file = '../input/g2net-gravitational-wave-detection/training_labels.csv'\n    store_dir = '{}-plots/{}/{}/{}/{}/{}.npy'\n    test_file = ''\n    submission_file = ''\n    signal_names = (\"LIGO Hanford\", \"LIGO Livingston\", \"Virgo\")\n    colors = (\"black\", \"red\", \"green\")\n    ","metadata":{"execution":{"iopub.status.busy":"2021-07-02T07:59:26.603456Z","iopub.execute_input":"2021-07-02T07:59:26.604003Z","iopub.status.idle":"2021-07-02T07:59:26.616881Z","shell.execute_reply.started":"2021-07-02T07:59:26.603974Z","shell.execute_reply":"2021-07-02T07:59:26.616149Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Utilites","metadata":{}},{"cell_type":"code","source":"def convert_image_id_2_path(image_id: str, is_train: bool = True) -> str:\n    folder = \"train\" if is_train else \"test\"\n    return Config.data_dir.format(\n        folder, image_id[0], image_id[1], image_id[2], image_id \n    )","metadata":{"execution":{"iopub.status.busy":"2021-07-02T07:59:26.618311Z","iopub.execute_input":"2021-07-02T07:59:26.618838Z","iopub.status.idle":"2021-07-02T07:59:26.630442Z","shell.execute_reply.started":"2021-07-02T07:59:26.618807Z","shell.execute_reply":"2021-07-02T07:59:26.629648Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def convert_image_id_2_store_path(image_id: str, is_train: bool = True, plot_type = 'rec') -> str:\n    folder = \"train\" if is_train else \"test\"\n    store_path = Config.store_dir.format(\n        plot_type, folder, image_id[0], image_id[1], image_id[2], image_id \n    )\n    os.makedirs(os.path.dirname(store_path), exist_ok=True)\n    return store_path","metadata":{"execution":{"iopub.status.busy":"2021-07-02T07:59:26.631644Z","iopub.execute_input":"2021-07-02T07:59:26.632105Z","iopub.status.idle":"2021-07-02T07:59:26.645783Z","shell.execute_reply.started":"2021-07-02T07:59:26.632075Z","shell.execute_reply":"2021-07-02T07:59:26.644816Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_random_sample_for_both_targets():\n    i0 = random.choice(train_df.index[train_df['target']==0].tolist())\n    i1 = random.choice(train_df.index[train_df['target']==1].tolist())\n\n    id0 = train_df.iloc[i0][\"id\"]\n    id1 = train_df.iloc[i1][\"id\"]\n    \n    return id0, id1","metadata":{"execution":{"iopub.status.busy":"2021-07-02T07:59:26.646969Z","iopub.execute_input":"2021-07-02T07:59:26.647252Z","iopub.status.idle":"2021-07-02T07:59:26.66436Z","shell.execute_reply.started":"2021-07-02T07:59:26.647224Z","shell.execute_reply":"2021-07-02T07:59:26.663332Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = pd.read_csv(Config.train_file)\ntrain_df","metadata":{"execution":{"iopub.status.busy":"2021-07-02T07:59:26.665814Z","iopub.execute_input":"2021-07-02T07:59:26.66628Z","iopub.status.idle":"2021-07-02T07:59:26.979788Z","shell.execute_reply.started":"2021-07-02T07:59:26.66625Z","shell.execute_reply":"2021-07-02T07:59:26.978903Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Helper functions for visualisation EDA for all different transformations","metadata":{}},{"cell_type":"code","source":"def visualize_line_plot(\n    id0,\n    id1,\n):\n    path0 = convert_image_id_2_path(id0)\n    data0 = np.load(path0)\n    \n    path1 = convert_image_id_2_path(id1)\n    data1 = np.load(path1)\n    \n    fig, axes = plt.subplots(nrows=3, ncols=2, figsize=(20, 12))\n    \n    for i, ax in enumerate(axes):        \n        x0 = range(len(data0[i]))\n        y0 = data0[i]\n        \n        x1 = range(len(data1[i]))\n        y1 = data1[i]\n        if i==0:\n            ax[0].set_title(id0 + ' Target = 0', fontsize=20)\n            ax[1].set_title(id1 + ' Target = 1', fontsize=20)\n            \n        ax[0].plot(x0, y0, color=Config.colors[i])\n        ax[0].set_xlabel(Config.signal_names[i], fontsize=14)\n        \n        ax[1].plot(x1, y1, color=Config.colors[i])\n        ax[1].set_xlabel(Config.signal_names[i], fontsize=14)\n        \n    # set the spacing between subplots\n    plt.subplots_adjust(left=0.1,\n                    bottom=0.1, \n                    right=0.9, \n                    top=0.9, \n                    wspace=0.2, \n                    hspace=0.5)","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-07-02T07:59:26.982439Z","iopub.execute_input":"2021-07-02T07:59:26.982739Z","iopub.status.idle":"2021-07-02T07:59:26.993192Z","shell.execute_reply.started":"2021-07-02T07:59:26.982711Z","shell.execute_reply":"2021-07-02T07:59:26.992106Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def visualize_recurrence_plot(\n    id0,\n    target,\n):\n    path = convert_image_id_2_path(id0)\n    data = np.load(path)\n    \n    rp = RecurrencePlot(threshold='distance', percentage=20)\n    X_rp = rp.fit_transform(data)\n    \n    fig, axes = plt.subplots(nrows=1, ncols=3, figsize=(20, 7))\n    \n    for i, ax in enumerate(axes):        \n        ax.imshow(X_rp[i], cmap='binary', origin='lower')\n        ax.set_xlabel(Config.signal_names[i], fontsize=16)\n            \n    plt.suptitle(f\"id: {id0} target: {target}\", fontsize=16)\n    plt.show()\n","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-07-02T07:59:26.995044Z","iopub.execute_input":"2021-07-02T07:59:26.995409Z","iopub.status.idle":"2021-07-02T07:59:27.007268Z","shell.execute_reply.started":"2021-07-02T07:59:26.995356Z","shell.execute_reply":"2021-07-02T07:59:27.006068Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def visualize_gramian_angular_fields_plot(\n    id0,\n    target,\n    method = 'summation'\n):\n    path = convert_image_id_2_path(id0)\n    data = np.load(path)\n    \n    gaf = GramianAngularField(image_size=24, method=method)\n    X_gaf = gaf.fit_transform(data)\n    \n    fig, axes = plt.subplots(nrows=1, ncols=3, figsize=(20, 7))\n    \n    for i, ax in enumerate(axes):\n        \n        ax.imshow(X_gaf[i], cmap='rainbow', origin='lower')\n        ax.set_xlabel(Config.signal_names[i], fontsize=16)\n            \n    plt.suptitle(f\"id: {id0} target: {target}\", fontsize=16)\n    plt.show()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-07-02T07:59:27.008843Z","iopub.execute_input":"2021-07-02T07:59:27.009805Z","iopub.status.idle":"2021-07-02T07:59:27.026302Z","shell.execute_reply.started":"2021-07-02T07:59:27.009749Z","shell.execute_reply":"2021-07-02T07:59:27.025316Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def visualize_mft_plot(\n    id0,\n    target,\n):\n    path = convert_image_id_2_path(id0)\n    data = np.load(path)\n    \n    mft = MarkovTransitionField(image_size = 0.3, n_bins=3)\n    X_mft = mft.fit_transform(data)\n    \n    fig, axes = plt.subplots(nrows=1, ncols=3, figsize=(20, 7))\n    \n    for i, ax in enumerate(axes):\n        ax.imshow(X_mft[i], cmap='inferno', origin='lower')\n        ax.set_xlabel(Config.signal_names[i], fontsize=16)\n            \n    plt.suptitle(f\"id: {id0} target: {target}\", fontsize=16)\n    plt.show()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-07-02T07:59:27.027749Z","iopub.execute_input":"2021-07-02T07:59:27.028075Z","iopub.status.idle":"2021-07-02T07:59:27.041327Z","shell.execute_reply.started":"2021-07-02T07:59:27.028037Z","shell.execute_reply":"2021-07-02T07:59:27.040377Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Different type of plots demonstrated - \n1. Line plot\n2. Recurrence plot\n3. Gramian Angualar field plots - summation and difference type\n4. Markov Transition Field\n\nBelow i have demonstrated the above plots for a sine wave","metadata":{}},{"cell_type":"code","source":"fig, axes = plt.subplots(nrows=2, ncols=2, figsize=(15, 12))\nx = np.linspace(-4*np.pi, 4*np.pi, 1000)\ndata = x, np.sin(x)\nrp = RecurrencePlot(threshold='point', percentage=20)\nX_rp = rp.fit_transform(data)\ngaf = GramianAngularField(image_size=24)\nX_gaf = gaf.fit_transform(data)\nmft = MarkovTransitionField(image_size = 0.3, n_bins=3)\nX_mft = mft.fit_transform(data)\n\naxes = axes.flatten()\n\naxes[0].plot(x, np.sin(x))\naxes[0].set_xlabel('Line Plot', fontsize=14)\naxes[1].imshow(X_rp[1], cmap='rainbow', origin='lower')\naxes[1].set_xlabel('Recurssion Plot', fontsize=14)\naxes[2].imshow(X_gaf[1], cmap='rainbow', origin='lower')\naxes[2].set_xlabel('Gramian Angular fields Plot', fontsize=14)\naxes[3].imshow(X_mft[1], cmap='inferno', origin='lower')\naxes[3].set_xlabel('Markov Transition Field Plot', fontsize=14)\nplt.suptitle(\"Sine Wave example\", fontsize=16)\nplt.show()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-07-02T07:59:27.042979Z","iopub.execute_input":"2021-07-02T07:59:27.043265Z","iopub.status.idle":"2021-07-02T07:59:27.899012Z","shell.execute_reply.started":"2021-07-02T07:59:27.043238Z","shell.execute_reply":"2021-07-02T07:59:27.898012Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Line plots comparison for a random sample of both the targets","metadata":{}},{"cell_type":"code","source":"id0, id1 = get_random_sample_for_both_targets()\nvisualize_line_plot(id0, id1)","metadata":{"execution":{"iopub.status.busy":"2021-07-02T07:59:27.900362Z","iopub.execute_input":"2021-07-02T07:59:27.900654Z","iopub.status.idle":"2021-07-02T07:59:28.901014Z","shell.execute_reply.started":"2021-07-02T07:59:27.900626Z","shell.execute_reply":"2021-07-02T07:59:28.90009Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Recurrence plots comparison for a random sample of both the targets","metadata":{}},{"cell_type":"code","source":"id0, id1 = get_random_sample_for_both_targets()\nvisualize_recurrence_plot(id0, 0)\nvisualize_recurrence_plot(id1, 1)","metadata":{"execution":{"iopub.status.busy":"2021-07-02T07:59:28.902346Z","iopub.execute_input":"2021-07-02T07:59:28.902652Z","iopub.status.idle":"2021-07-02T07:59:39.974617Z","shell.execute_reply.started":"2021-07-02T07:59:28.902623Z","shell.execute_reply":"2021-07-02T07:59:39.97367Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Gramian Angular Fields - summation plots comparison for a random sample of both the targets","metadata":{}},{"cell_type":"code","source":"id0, id1 = get_random_sample_for_both_targets()\nvisualize_gramian_angular_fields_plot(id0, 0)\nvisualize_gramian_angular_fields_plot(id1, 1)","metadata":{"execution":{"iopub.status.busy":"2021-07-02T07:59:39.976003Z","iopub.execute_input":"2021-07-02T07:59:39.976561Z","iopub.status.idle":"2021-07-02T07:59:41.256077Z","shell.execute_reply.started":"2021-07-02T07:59:39.976499Z","shell.execute_reply":"2021-07-02T07:59:41.254853Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Gramian Angular Fields - difference plots comparison for a random sample of both the targets","metadata":{}},{"cell_type":"code","source":"id0, id1 = get_random_sample_for_both_targets()\nvisualize_gramian_angular_fields_plot(id0, 0, method='difference')\nvisualize_gramian_angular_fields_plot(id1, 1, method='difference')","metadata":{"execution":{"iopub.status.busy":"2021-07-02T07:59:41.257549Z","iopub.execute_input":"2021-07-02T07:59:41.258211Z","iopub.status.idle":"2021-07-02T07:59:42.293646Z","shell.execute_reply.started":"2021-07-02T07:59:41.258167Z","shell.execute_reply":"2021-07-02T07:59:42.29257Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Markov Transition field plots comparison for a random sample of both the targets","metadata":{}},{"cell_type":"code","source":"id0, id1 = get_random_sample_for_both_targets()\nvisualize_mft_plot(id0, 0)\nvisualize_mft_plot(id1, 1)","metadata":{"execution":{"iopub.status.busy":"2021-07-02T07:59:42.295311Z","iopub.execute_input":"2021-07-02T07:59:42.295757Z","iopub.status.idle":"2021-07-02T07:59:45.35766Z","shell.execute_reply.started":"2021-07-02T07:59:42.295715Z","shell.execute_reply":"2021-07-02T07:59:45.356636Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Helper functions for transforming time-series to plots, storing and loading them","metadata":{}},{"cell_type":"code","source":"def store_plot(reading_id, is_train = True, plot_type = 'rec', transformer = RecurrencePlot(threshold='point', percentage=20)):\n    path = convert_image_id_2_path(reading_id)\n    data = np.load(path)\n    X = transformer.fit_transform(data)\n    if plot_type == 'rec':\n        X = X.reshape(4096, 4096, 3)\n        X = cv2.resize(X, dsize=(256, 256), interpolation=cv2.INTER_CUBIC)\n        X = X.reshape(3, 256, 256)\n    store_path = convert_image_id_2_store_path(reading_id, is_train, plot_type)\n    np.save(store_path, X)","metadata":{"execution":{"iopub.status.busy":"2021-07-02T07:59:58.474551Z","iopub.execute_input":"2021-07-02T07:59:58.474947Z","iopub.status.idle":"2021-07-02T07:59:58.482143Z","shell.execute_reply.started":"2021-07-02T07:59:58.474914Z","shell.execute_reply":"2021-07-02T07:59:58.481124Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_plot(reading_id, is_train = True, plot_type = 'rec'):\n    store_path = convert_image_id_2_store_path(reading_id, is_train, plot_type)\n    X_loaded = np.load(store_path)\n    return X_loaded","metadata":{"execution":{"iopub.status.busy":"2021-07-02T07:59:45.368976Z","iopub.execute_input":"2021-07-02T07:59:45.369381Z","iopub.status.idle":"2021-07-02T07:59:45.387864Z","shell.execute_reply.started":"2021-07-02T07:59:45.36934Z","shell.execute_reply":"2021-07-02T07:59:45.386657Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def visualize_stored_plot(reading_id, is_train = True, plot_type='rec', cmap='binary'):\n    X = load_plot(reading_id, is_train, plot_type)\n    fig, axes = plt.subplots(nrows=1, ncols=3, figsize=(20, 7))\n    \n    for i, ax in enumerate(axes):        \n        ax.imshow(X[i], cmap=cmap, origin='lower')\n        ax.set_xlabel(Config.signal_names[i], fontsize=16)\n            \n    plt.suptitle(f\"id: {reading_id}\", fontsize=16)\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2021-07-02T07:59:45.389419Z","iopub.execute_input":"2021-07-02T07:59:45.389885Z","iopub.status.idle":"2021-07-02T07:59:45.401454Z","shell.execute_reply.started":"2021-07-02T07:59:45.389842Z","shell.execute_reply":"2021-07-02T07:59:45.400465Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"rp = RecurrencePlot(threshold='distance', percentage=20)\nid_list = random.sample(train_df.index.tolist(), 2)\nfor i in id_list:\n    reading_id = train_df.iloc[i][\"id\"]\n    store_plot(reading_id, rp)\nfor i in id_list:\n    reading_id = train_df.iloc[i][\"id\"]\n    visualize_stored_plot(reading_id)","metadata":{"execution":{"iopub.status.busy":"2021-07-02T08:00:01.041434Z","iopub.execute_input":"2021-07-02T08:00:01.041826Z","iopub.status.idle":"2021-07-02T08:00:05.170384Z","shell.execute_reply.started":"2021-07-02T08:00:01.041789Z","shell.execute_reply":"2021-07-02T08:00:05.169671Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gaf = GramianAngularField(image_size=24, method='summation')\nid_list = random.sample(train_df.index.tolist(), 2)\nfor i in id_list:\n    reading_id = train_df.iloc[i][\"id\"]\n    store_plot(reading_id, transformer = gaf, plot_type = 'gaf')\n    \nfor i in id_list:\n    reading_id = train_df.iloc[i][\"id\"]\n    visualize_stored_plot(reading_id, plot_type = 'gaf', cmap = 'rainbow')","metadata":{"execution":{"iopub.status.busy":"2021-07-02T08:00:16.97044Z","iopub.execute_input":"2021-07-02T08:00:16.970817Z","iopub.status.idle":"2021-07-02T08:00:18.00638Z","shell.execute_reply.started":"2021-07-02T08:00:16.970786Z","shell.execute_reply":"2021-07-02T08:00:18.005264Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## <center>Work in progress </center>\n#### <center>Baseline models coming up</center>","metadata":{}}]}