{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport scipy.stats as stats\nimport tensorflow as tf\nfrom tensorflow import keras\nfrom tensorflow.keras.layers import *\nimport tensorflow.keras.backend as K\nfrom sklearn.preprocessing import MinMaxScaler\nfrom sklearn.model_selection import StratifiedShuffleSplit\n\nimport pyarrow.parquet as pq\nfrom numba import jit, njit\nimport dask.dataframe as dd\nimport dask.array as da\nfrom tqdm import tqdm\nimport gc\n\n%matplotlib inline\nimport matplotlib.pyplot as plt\n\nplt.style.use('seaborn-whitegrid')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ccc41b95445821420f04adc3e7036a96e0f1daf1"},"cell_type":"code","source":"def mcc(y_true, y_pred):\n    '''Calculates the Matthews correlation coefficient measure for quality\n    of binary classification problems.\n    '''\n    y_pred_pos = K.round(K.clip(y_pred, 0, 1))\n    y_pred_neg = 1 - y_pred_pos\n\n    y_pos = K.round(K.clip(y_true, 0, 1))\n    y_neg = 1 - y_pos\n\n    tp = K.sum(y_pos * y_pred_pos)\n    tn = K.sum(y_neg * y_pred_neg)\n\n    fp = K.sum(y_neg * y_pred_pos)\n    fn = K.sum(y_pos * y_pred_neg)\n\n    numerator = (tp * tn - fp * fn)\n    denominator = K.sqrt((tp + fp) * (tp + fn) * (tn + fp) * (tn + fn))\n\n    return numerator / (denominator + K.epsilon())","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"%%time\ntrain_df = pq.read_pandas('../input/train.parquet').to_pandas()\nmeta_train = pd.read_csv('../input/metadata_train.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7c2e842062e4c671de5ab0f3f15a07b81ba0ec8d"},"cell_type":"code","source":"@jit('float32[:, :](float32[:,:], int32)')\ndef feature_extractor(x, n_part=1000,):\n    lenght = len(x)\n    n_feat = 7\n    pool = np.int32(np.ceil(lenght/n_part))\n    output = np.zeros((n_part, n_feat))\n    for j, i in enumerate(range(0,lenght, pool)):\n        if i+pool < lenght:\n            k = x[i:i+pool]\n        else:\n            k = x[i:]\n        output[j, 0] = np.mean(k, axis=0) #mean\n        output[j, 1] = np.min(k, axis=0) #min\n        output[j, 2] = np.max(k, axis=0) #max\n        output[j, 3] = np.std(k, axis=0) #std\n        output[j, 4] = np.median(k, axis=0) #median\n        output[j, 5] = stats.skew(k, axis=0) #skew\n        output[j, 6] = stats.kurtosis(k, axis=0) # kurtosis\n    return output","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"933425e7cef723eddd97121a3eeab122a2cf4869"},"cell_type":"code","source":"X = []\ny = []\nfor i in tqdm(meta_train.signal_id):\n    idx = meta_train.loc[meta_train.signal_id==i, 'signal_id'].values.tolist()\n    y.append(meta_train.loc[meta_train.signal_id==i, 'target'].values)\n    X.append(feature_extractor(train_df.iloc[:, idx].values, n_part=400))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b01f0ed35e0cce01014214184fa8eb39e5e5a625"},"cell_type":"code","source":"del train_df; gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bb358231a6581c139994a9ceaaa08b2b211275eb"},"cell_type":"code","source":"X = np.array(X).reshape(-1, X[0].shape[0], X[0].shape[1])\ny = np.array(y).reshape(-1,)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"822886a9a9513ed5d8aac30cfcf422d166726dfb"},"cell_type":"code","source":"sss = StratifiedShuffleSplit(n_splits=1, test_size=0.5, random_state=0)\n(train_idx, val_idx) = next(sss.split(X, y))\n\nX_train, X_val = X[train_idx], X[val_idx]\ny_train, y_val = y[train_idx], y[val_idx]\n#X_train, X_val, y_train, y_val = train_test_split(\n#    X, y, stratify=y, test_size=0.3, shuffle=True, random_state=0)\n\nscalers = {}\nfor i in range(X_train.shape[2]):\n    scalers[i] = MinMaxScaler(feature_range=(-1, 1))\n    X_train[:, i, :] = scalers[i].fit_transform(X_train[:, i, :]) \n\nfor i in range(X_val.shape[2]):\n    X_val[:, i, :] = scalers[i].transform(X_val[:, i, :]) ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2ea6544d5228f786e2636bbde5573216b9ba7229"},"cell_type":"code","source":"def keras_auc(y_true, y_pred):\n    auc = tf.metrics.auc(y_true, y_pred)[1]\n    K.get_session().run(tf.local_variables_initializer())\n    return auc","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"49850c88db04ef1ad1b22f023795a21e555ca9f4"},"cell_type":"code","source":"model = keras.Sequential([\n    Conv1D(filters=64, kernel_size=4, activation='relu', input_shape=(X.shape[1], X.shape[2])),\n    Conv1D(filters=64, kernel_size=4, activation='relu'),\n    MaxPooling1D(2),\n    Conv1D(filters=20, kernel_size=4, activation='relu'),\n    Conv1D(filters=20, kernel_size=4, activation='relu'),\n    GlobalAveragePooling1D(),\n    Dropout(0.2),\n    Flatten(),\n    Dense(32, activation='tanh'),\n    Dropout(0.5),\n    Dense(8, activation='tanh'),\n    Dropout(0.5),\n    Dense(1, activation='sigmoid'),\n])\n\nmodel.compile(optimizer='adam',\n              loss='binary_crossentropy',\n              metrics=[mcc, keras_auc])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2c5c23328248c9f76fc70d29c23fe0fe2faa699c"},"cell_type":"code","source":"from sklearn.utils import class_weight","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"29f35733ae8ad0fc85e5352c336f5fed7a97200a"},"cell_type":"code","source":"epochs = 30\nbatch_size = 16\n\n#class_weights = class_weight.compute_class_weight('balanced',\n#                                                 np.unique(y_train),\n#                                                 y_train)\n\nclass_weights = {\n    0: 1.,\n    1: 1.2,\n}\n\nmodel.fit(X_train, y_train, validation_data=(X_val, y_val), \n          epochs=epochs, batch_size=batch_size,\n          class_weight=class_weights)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"77238d903ab66d7ace34f28a00a1852b50cb8a7a"},"cell_type":"code","source":"del X, X_train, X_val; gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b6c8448467c40ef1648a98fdf29b8c0acdea87b0"},"cell_type":"code","source":"meta_test = pd.read_csv('../input/metadata_test.csv')\nstart_test = meta_test.signal_id.min()\nend_test = meta_test.signal_id.max()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"759906223e5e8f0b901e4a9989a04baed78d80a7"},"cell_type":"code","source":"X_test = []\ny_test = []\n\npool_test = 2000\n\nfor start_col in tqdm(range(start_test, end_test + 1, pool_test)):\n    end_col = min(start_col + pool_test, end_test + 1)\n    print('cols {}-{}'.format(start_col, end_col-1))\n    test = pq.read_pandas('../input/test.parquet',\n                          columns=[str(c) for c in range(start_col, end_col)]).to_pandas()\n    print(test.shape)\n    for i in tqdm(test.columns):\n        X_test.append(feature_extractor(test[i].values, n_part=400))\n        test.drop([i], axis=1, inplace=True); gc.collect()\n    del test; gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"efa88553653e3c4230a19dcb60d927a900fba0d4"},"cell_type":"code","source":"X_test = np.array(X_test).reshape(-1, X_test[0].shape[0], X_test[0].shape[1])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"446d9d93f4ab792a813641a8c1ac0b5998bd22b2"},"cell_type":"code","source":"for i in range(X_test.shape[2]):\n    X_test[:, i, :] = scalers[i].transform(X_test[:, i, :])\n    \ny_pred = model.predict_classes(X_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c87c2e8b7b8688e4293edf05129caf315d20345f"},"cell_type":"code","source":"submission = pd.read_csv('../input/sample_submission.csv')\nsubmission['signal_id'] = meta_test.signal_id.values\nsubmission['target'] = y_pred.astype(int)\nsubmission.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}