{"cells":[{"metadata":{"_uuid":"efe99fc9f6a1fa26d7fb8d05974514278fd0ec0a"},"cell_type":"markdown","source":"### This is the first part of @afajohn kernel with the addition of multiprocessing on the test set\nCheck out his brilliant kernel here https://www.kaggle.com/afajohn/cnn-lstm-for-signal-classification-lb-0-513\n\nAlso, many thanks to following kernels:\n- For shortening the signals with a simple feature extraction thanks to: https://www.kaggle.com/ashishpatel26/transfer-learning-in-basic-nn\n- For signal denoising and fft: https://www.kaggle.com/theoviel/fast-fourier-transform-denoising"},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"import sys\nimport gc\nimport time\nimport os\nimport logging\nfrom multiprocessing import Pool, current_process\nfrom multiprocessing import log_to_stderr, get_logger\nfrom tqdm import tqdm\nfrom numba import jit\n\nimport pyarrow.parquet as pq\nimport pandas as pd\nimport numpy as np\n\nimport keras\nimport keras.backend as K\nfrom keras.layers import LSTM,Dropout,Dense,TimeDistributed,Conv1D,MaxPooling1D,Flatten\nfrom keras.models import Sequential\nimport tensorflow as tf\n\nfrom IPython.display import display, clear_output\n\nimport matplotlib.pyplot as plt\n%matplotlib inline\nimport seaborn as sns\n\nsns.set_style(\"whitegrid\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e06765b00810004c271944be1360e0b62fa64134"},"cell_type":"code","source":"import pyarrow.parquet as pq\nimport pandas as pd\nimport numpy as np","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"935c2ba4d25cae753e04645a168decb84b6ce964"},"cell_type":"code","source":"%%time \ntrain_set = pq.read_pandas('../input/train.parquet').to_pandas()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e217109aeec443c5cb64fdc716ef433f306c436c"},"cell_type":"code","source":"%%time\nmeta_train = pd.read_csv('../input/metadata_train.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"36f9662e947b286fa6505a4c8183daff1b8c1570"},"cell_type":"code","source":"@jit('float32(float32[:,:], int32)')\ndef feature_extractor(x, n_part=1000):\n    lenght = len(x)\n    pool = np.int32(np.ceil(lenght/n_part))\n    output = np.zeros((n_part,))\n    for j, i in enumerate(range(0,lenght, pool)):\n        if i+pool < lenght:\n            k = x[i:i+pool]\n        else:\n            k = x[i:]\n        output[j] = np.max(k, axis=0) - np.min(k, axis=0)\n    return output","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"83ce36dd1a85aaad8b446e452cbc7e19b882164b"},"cell_type":"code","source":"x_train = []\ny_train = []\nfor i in tqdm(meta_train.signal_id):\n    idx = meta_train.loc[meta_train.signal_id==i, 'signal_id'].values.tolist()\n    y_train.append(meta_train.loc[meta_train.signal_id==i, 'target'].values)\n    x_train.append(abs(feature_extractor(train_set.iloc[:, idx].values, n_part=400)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4cc6eef2ae63a2c79d08c5fe1ce8670449fb807a"},"cell_type":"code","source":"del train_set; gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cad66118494f25f70c9ace7b85bea500f1c680c8"},"cell_type":"code","source":"y_train = np.array(y_train).reshape(-1,)\nX_train = np.array(x_train).reshape(-1,x_train[0].shape[0])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5250bf32e062836cbf10394e5f9df4d6b695380e"},"cell_type":"code","source":"def keras_auc(y_true, y_pred):\n    auc = tf.metrics.auc(y_true, y_pred)[1]\n    K.get_session().run(tf.local_variables_initializer())\n    return auc","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"034ea6d8cf8bd8a09eb5943ebb3faa01cd991e9d"},"cell_type":"code","source":"n_signals = 1 #So far each instance is one signal. We will diversify them in next step\nn_outputs = 1 #Binary Classification","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1c05ecc6abef0b822e422950a5824ea8c55ba861"},"cell_type":"code","source":"#Build the model\nverbose, epochs, batch_size = True, 15, 16\nn_steps, n_length = 40, 10\nX_train = X_train.reshape((X_train.shape[0], n_steps, n_length, n_signals))\n# define model\nmodel = Sequential()\nmodel.add(TimeDistributed(Conv1D(filters=64, kernel_size=3, activation='relu'), input_shape=(None,n_length,n_signals)))\nmodel.add(TimeDistributed(Conv1D(filters=64, kernel_size=3, activation='relu')))\nmodel.add(TimeDistributed(Dropout(0.5)))\nmodel.add(TimeDistributed(MaxPooling1D(pool_size=2)))\nmodel.add(TimeDistributed(Flatten()))\nmodel.add(LSTM(100))\nmodel.add(Dropout(0.5))\nmodel.add(Dense(100, activation='relu'))\nmodel.add(Dense(n_outputs, activation='sigmoid'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e48f5b3c1e9cffb34ef0f395f668a205b9cd2508"},"cell_type":"code","source":"model.compile(loss='binary_crossentropy', optimizer='adam', metrics=[keras_auc])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7092f9b2feb4b98b47a72738b956a38f178e0075"},"cell_type":"code","source":"model.fit(X_train, y_train, epochs=epochs, batch_size=batch_size, verbose=verbose)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"470dee0dc3d9edaaad403eb14275308cb84fb9f8"},"cell_type":"code","source":"model.save_weights('model1.hdf5')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b515018688a91e01db25789607fba7e2be3b2216"},"cell_type":"code","source":"#%%time\nmeta_test = pd.read_csv('../input/metadata_test.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ea6ad7ea1e2489484c451c1fed4e0bd5f9f770a3"},"cell_type":"code","source":"def process_chunk(arg):\n    start_index = arg['start_index']\n    chunk_size = arg['chunk_size']\n    \n    # Test set indices start at 8712\n    test_set_start = 8712\n    offset_index = (test_set_start + start_index)\n    \n    # Column name must be a string\n    subset_test = pq.read_pandas('../input/test.parquet', columns=[str(offset_index + j) for j in range(chunk_size)]).to_pandas()    \n    x_test = []\n    for j in range(chunk_size):\n        subset_test_row = subset_test[str(offset_index + j)]\n        x_test.append(abs(feature_extractor(subset_test_row.values, n_part=400)))\n    return x_test","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"28cf5f6f80b5032b735384c4ddaf8b6b55dc046c"},"cell_type":"code","source":"# Define 21 chunks for processing the test set\n# on multiple cpus. I have choosen to process in chunks of 1000 (plus the remainder)\n# so as to keep within the kernels memory limit\nargs = []\nfor i in range(0, 20000, 1000):\n    args.append({\n        'start_index': i,\n        'chunk_size': 1000\n    })\n    \n# Add a chunk for the remainder\nargs.append({\n    'start_index': 20000,\n    'chunk_size': 337\n})\n\nn_cpu = processes=os.cpu_count()\nprint('n_cpu: ', n_cpu)\n\np = Pool(processes=n_cpu)\n\n# Map the chunk args to the the process_chunk function\nx_test_chunks = p.map(process_chunk, args)\nprint(f\"multi processing complete. len: {len(x_test_chunks)}\")\n\np.close()\np.join()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e367d00a29dabeb7e85d8239d5d252f112e6c819"},"cell_type":"code","source":"x_test = [item for sublist in x_test_chunks for item in sublist]\nx_test = np.array(x_test)\nprint('x_test.shape: ', x_test.shape)\nX_test = x_test.reshape((x_test.shape[0], n_steps, n_length, n_signals))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fe048f7596bb0309df92293e183586425794213c"},"cell_type":"code","source":"del x_test_chunks","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a849fe7c203cb3bf1e31bca15c85ea1e12d6a3fc"},"cell_type":"code","source":"preds = model.predict(X_test)\npreds.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"96ad54116ff05d45fee610b69fcabe96b16db00b"},"cell_type":"code","source":"threshpreds = (preds>0.5)*1\nsub = pd.read_csv('../input/sample_submission.csv')\nsub.target = threshpreds\n\n# Gave me an LB score of ~0.450\nsub.to_csv('submission.csv',index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b76b5959e2127dc660d8fd0d9bae548621580241"},"cell_type":"code","source":"check_sub = pd.read_csv('submission.csv')\ncheck_sub.head(20)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d9e155e960b61d529796df6f384ff968412ea622"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}