{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"markdown","source":"In this notebook I will create images from signal data.  \nAn output of the notebook may be used to create a model based on CNN.\n\nThe approach is taken from the following gist:\n    - https://gist.github.com/oguiza/26020067f499d48dc52e5bcb8f5f1c57\n \nMore info on  Gramian Angular Fields, Markov Transition Fields and  Recurrence Plots may be found here:\n    - https://medium.com/analytics-vidhya/encoding-time-series-as-images-b043becbdbf3 (GAF)\n    - http://coral-lab.umbc.edu/wp-content/uploads/2015/05/10179-43348-1-SM1.pdf (MTF)\n    - https://en.wikipedia.org/wiki/Recurrence_plot (RP)"},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"! pip install pyts","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b0daa7a9484f07a656ed2dc532cf80516ef1996a"},"cell_type":"code","source":"import csv\nimport os\nimport re\nimport numpy as np\nimport pickle\nfrom pyts.approximation import PAA\nfrom pyts.image import GADF, MTF, RecurrencePlots\nfrom sklearn.preprocessing import MinMaxScaler\nfrom tqdm import tqdm_notebook\nfrom time import time\nfrom matplotlib import pyplot as plt\nfrom matplotlib import image as mpimg\n%matplotlib inline","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"26e94751c4e2b7e93d894f96124efd8d7d5ba92f"},"cell_type":"markdown","source":"# Signal -> images"},{"metadata":{"trusted":true,"_uuid":"e568f8e02f6f925742a88fe2b75ceee84e6ee058"},"cell_type":"code","source":"input_csv_path = '../input/train.csv'\nsample_size = 150_000\nn_rows = 629145480\nn_samples = n_rows // sample_size\nIMG_SIZE = 512  # bigger images won't get inside of 5.2GB of Disk allocated by Kaggle kernel","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":false,"_kg_hide-output":false,"trusted":true,"_uuid":"363c713ee9dc2cbfcfdfafc80c84a206b74b50b8"},"cell_type":"code","source":"def create_folder(folder):\n    if not os.path.exists(folder):\n        os.mkdir(folder)\n        \ndef _make_log10(num):\n    if num == 0:\n        return 0.\n    new_num = np.log10(abs(num)) + 0.05\n    if num < 0:\n        new_num = -new_num\n    return new_num\n\nmake_log10 = np.vectorize(_make_log10)\n\ndef create_image(array, name=None, folder=None, func=None):\n    if func:\n        array = func(array)\n    uvts = PAA(output_size=IMG_SIZE).fit_transform([array])\n    encoder1 = RecurrencePlots()\n    encoder2 = MTF(IMG_SIZE, n_bins=IMG_SIZE//20, quantiles='gaussian')\n    encoder3 = GADF(IMG_SIZE)\n        \n    r = np.squeeze(encoder1.fit_transform(uvts)) \n    g = np.squeeze(encoder2.fit_transform(uvts))\n    b = np.squeeze(encoder3.fit_transform([array]))\n    \n    scaler = MinMaxScaler(feature_range=(0, 1))\n    shape = r.shape\n    r = scaler.fit_transform(r.reshape(-1, 1)).reshape(shape)\n    g = scaler.fit_transform(g.reshape(-1, 1)).reshape(shape)\n    b = scaler.fit_transform(b.reshape(-1, 1)).reshape(shape)\n    rgbArray = np.zeros((IMG_SIZE, IMG_SIZE, 3), 'uint8')\n    rgbArray[..., 0] = r * 256\n    rgbArray[..., 1] = g * 256\n    rgbArray[..., 2] = b * 256\n    \n    if not (name and folder):\n        plt.imshow(rgbArray)\n    else:\n        filename = name + \".png\"\n        plt.imsave(os.path.join(folder, filename),\n                   rgbArray)\n\ndef create_dataset(path, n_samples, sample_size, folder):\n    create_folder(folder)\n    with open(path, \"r\") as read_f:\n        reader = csv.reader(read_f)\n        counter = 0\n        pbar = tqdm_notebook(total=n_samples)\n        row = np.zeros(sample_size)\n        y = np.zeros(n_samples)\n        next(reader)\n        for val, ttf in reader:\n            n = counter // sample_size\n            m = counter % sample_size\n            row[m] = int(val)\n            if m == sample_size - 1:\n                create_image(row.copy(), str(n), folder,\n                             make_log10)\n                y[n] = float(ttf)\n                row = np.zeros(sample_size)\n                pbar.update(1)\n            counter += 1\n        return y\n\ndef print_rand_images(folder, y=None):\n    fig=plt.figure(figsize=(10, 12))\n    columns, rows = 3, 3\n    ax = []\n    for i in range(columns*rows):\n        img_name = np.random.choice(os.listdir(folder))\n        path = os.path.join(folder, img_name)\n        img = mpimg.imread(path)\n        ax.append(fig.add_subplot(rows, columns, i+1))\n        if y is not None:\n            img_id = int(os.path.splitext(img_name)[0])\n            img_name = f\"ttf: {y[img_id]:.5f}\"\n        ax[-1].set_title(img_name)\n        plt.imshow(img)\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"21091a37a471d37df40ccd706d085df8eb94e071"},"cell_type":"markdown","source":"# Create images for training set"},{"metadata":{"trusted":true,"_uuid":"110c3384714dbfe4e62e0eaa2599780d3e99491a"},"cell_type":"code","source":"start = time()\ny = create_dataset(input_csv_path, n_samples, sample_size, \"train\")\nprint(\"Completed in {} seconds\".format(int(time()-start)))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a22893db4f22d9d26a6e81068395a9c8009ecad1"},"cell_type":"markdown","source":" ### Example of the resulting images:"},{"metadata":{"trusted":true,"_uuid":"12c4b5f14b3a4de1094f65387df268d793295c2c"},"cell_type":"code","source":"print_rand_images(\"train\", y)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"9db04fc141b94d27e188126b40b52987891052b3"},"cell_type":"markdown","source":"### Saving results"},{"metadata":{"trusted":true,"_uuid":"494f43148c00c20b451402870782f89d2f0379df"},"cell_type":"code","source":"with open(\"y_train.pkl\", \"wb\") as f:\n    f.write(pickle.dumps(y))","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-output":true,"trusted":true,"_uuid":"39ddbcdd02d45558bee6ecfc50a24848b4de36dc"},"cell_type":"code","source":"! tar -zcvf train.tar.gz train","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-output":true,"trusted":true,"_uuid":"ac29a84b0bdae4c4caa09bdf03c87f71b9a316d0"},"cell_type":"code","source":"! rm -rf train","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"4b2588bc2a2aff5db87aa1459d2f38880867855c"},"cell_type":"markdown","source":"# Create images for test set"},{"metadata":{"trusted":true,"_uuid":"80a87ec8ba584dc46572ea0afdbf56e9921e9aa9"},"cell_type":"code","source":"test_folder = \"test\"\ncreate_folder(test_folder)\ntest_input_dir = \"../input/test/\"\ntest_files = [os.path.join(test_input_dir, x) for x in os.listdir(test_input_dir)]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c6c0209a9b59b1b9422824851af91ff6f328d42a"},"cell_type":"code","source":"start = time()\nfor file in tqdm_notebook(test_files):\n    array = np.loadtxt(file, skiprows=1)\n    name = os.path.splitext(os.path.basename(file))[0]\n    create_image(array, name, test_folder, make_log10)\nprint(\"Completed in {} seconds\".format(int(time()-start)))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"53f455321590e41a06d6cea18b87491d7f2529bc"},"cell_type":"markdown","source":" ### Example of the resulting images:"},{"metadata":{"trusted":true,"scrolled":true,"_uuid":"cca4f2d92c4162ba201d049f9c4d5ecfa157b641"},"cell_type":"code","source":"print_rand_images(test_folder)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"65cd88900d6cdfec79f14319ea78f3b2a35ecb02"},"cell_type":"markdown","source":"### Saving results"},{"metadata":{"_kg_hide-output":true,"trusted":true,"_uuid":"65f09d3ef604b83b5826541d1c7ed499d7d1f169"},"cell_type":"code","source":"! tar -zcvf test.tar.gz test","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-output":true,"trusted":true,"_uuid":"632e497fa3cb9f1fa4ce29a7259fc481d274b330"},"cell_type":"code","source":"! rm -rf test","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}