{"cells":[{"metadata":{"_uuid":"18f984163f6e5a579bc85ecd4c25deb202ce2ce5"},"cell_type":"markdown","source":"It's kernel investigate problems of the use CNN for line fault detection."},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import os\nimport gc\nimport pandas as pd\nimport numpy as np\nimport pyarrow.parquet as pq\nimport matplotlib.pyplot as plt\nfrom sklearn.metrics import matthews_corrcoef\nfrom scipy import signal\nimport seaborn as sns\nfrom tqdm import tqdm_notebook as tqdm\nfrom numba import jit, int32","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"18cfb52759669c246b2637b5ad5b0adc45cda076"},"cell_type":"code","source":"INIT_DIR = '../input'\nSIZE = 1024","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"17abd91b2462f9ff0811ff25376ae7cc54b41607"},"cell_type":"code","source":"os.listdir(INIT_DIR)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"765e4f94cd85deb4ab416c6a93cf15f5919f7c9b"},"cell_type":"markdown","source":"<center>  **Preprocessing**"},{"metadata":{"trusted":true,"_uuid":"997237499e7c5d0d577c248cc6ef3720c66e6b82"},"cell_type":"code","source":"meta = pd.read_csv(os.path.join(INIT_DIR, 'metadata_train.csv'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"21ded06cd44ca6fc032668ece397b6687e128305"},"cell_type":"code","source":"train = pq.read_pandas(os.path.join(INIT_DIR, 'train.parquet'), columns=[str(i) for i in range(1000)]).to_pandas()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2a235b125fd1c7c0362b4eede4076aa2452d3011"},"cell_type":"code","source":"meta.describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"33dcd3198419a410a7a6de8c8bd25faf5e1d4269"},"cell_type":"code","source":"meta.corr()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"288bbd8a081dace51d409beeed69466fe1f5a0d4"},"cell_type":"markdown","source":"Data contain 3 phase signal for each mesuarment. From table above we can see that target independant from phase and id_mesurment."},{"metadata":{"trusted":true,"_uuid":"db90a3967cc3899dd9fde9eb4e14e2ee628b0a63"},"cell_type":"code","source":"positive_mid = np.unique(meta.loc[meta.target == 1, 'id_measurement'].values)\nnegative_mid = np.unique(meta.loc[meta.target == 0, 'id_measurement'].values)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4159232468173c39c4b062adba05abe6b6247a6e"},"cell_type":"code","source":"pid = meta.loc[meta.id_measurement == positive_mid[0], 'signal_id']\nnid = meta.loc[meta.id_measurement == negative_mid[0], 'signal_id']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4684d29e42ac71e5c3bce32fe6a5dd4eebec53a9"},"cell_type":"code","source":"positive_sample = train.iloc[:, pid]\nnegative_sample = train.iloc[:, nid]","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e04cb00604df27f281265fbe9f64d1a619161dae"},"cell_type":"markdown","source":"Signal with phase, for my mind, will not be useful for CNN or RNN model.  For this case I'm apply filter like HPF for signal flatten. And thus I can more easier extract specific noise and anomaly feature.  "},{"metadata":{"_uuid":"91552079223e29ca5e0eeb3f2320a996d87c350a"},"cell_type":"markdown","source":"!!! Numba is very useful tool for situation like this !!!"},{"metadata":{"trusted":true,"_uuid":"e369bfb5c85f81af8ba921d94d2667b6317f2bf5"},"cell_type":"code","source":"@jit('float32(float32[:,:], int32, int32)')\ndef flatiron(x, alpha=50., beta=1):\n    new_x = np.zeros_like(x)\n    zero = x[0]\n    for i in range(1, len(x)):\n        zero = zero*(alpha-beta)/alpha + beta*x[i]/alpha\n        new_x[i] =  x[i] - zero\n    return new_x","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6718b4ccc496b945994cddaa46ba00c7acad02ca"},"cell_type":"code","source":"plt.figure(figsize=(24, 8))\nplt.plot(positive_sample, alpha=0.8);","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b97d9030db0d84de6f846bfb03364ad5aee02dd8"},"cell_type":"code","source":"x_filt = flatiron(positive_sample.values)\nplt.figure(figsize=(24, 8))\nplt.plot(x_filt, alpha=0.5);","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d885633618c55d41f12f4c0cbfa499fb3c9540a8"},"cell_type":"code","source":"plt.figure(figsize=(24, 8))\nplt.plot(negative_sample, alpha=0.7);","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"11d430768f0f5da9bdbba91fe386f98bd6aa9b91"},"cell_type":"code","source":"x_filt = flatiron(negative_sample.values)\nplt.figure(figsize=(24, 8))\nplt.plot(x_filt, alpha=0.5);","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"18c385af411668ff6a59a3c4a255fba6837355ea"},"cell_type":"code","source":"@jit('float32(float32[:,:], int32, int32)')\ndef feature_extractor(x, n_part=1000, n_dim=3):\n    lenght = len(x)\n    pool = np.int32(np.ceil(lenght/n_part))\n    output = np.zeros((n_part, n_dim))\n    for j, i in enumerate(range(0,lenght, pool)):\n        if i+pool < lenght:\n            k = x[i:i+pool]\n        else:\n            k = x[i:]\n        output[j] = np.max(k, axis=0) - np.min(k, axis=0)\n    return output\n\n@jit('float32(float32[:,:])')\ndef basic_feature_extractor(x):\n    return [np.max(x, axis=0), np.min(x, axis=0), np.std(x, axis=0), np.sum(x, axis=0)/len(x)]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"112196d0b994f9ebfe3a4b88679afc3d3463824b"},"cell_type":"code","source":"x_train = []\nbasic = []\ny_basic = []\ny_train = []\nmid = np.unique(meta.id_measurement.values)\nfor b in tqdm(range(4)):\n    start = b*len(meta)//12\n    if len(meta)//3 - start < len(meta)//12:\n        end = -1\n    else:\n        end = start + len(meta)//12\n    \n    columns = []\n    for i in mid[start:end]:\n        columns.extend(meta.loc[meta.id_measurement==i, 'signal_id'].values.tolist())\n    train = pq.read_pandas(os.path.join(INIT_DIR, 'train.parquet'), columns=[str(i) for i in columns]).to_pandas()\n    \n    for i in range(len(train.columns)):\n        train.iloc[:, i] = flatiron(train.iloc[:, i].values)\n        \n    for i in mid[start:end]:\n            idx = meta.loc[meta.id_measurement==i, 'signal_id'].values\n            x_train.append(abs(feature_extractor(train.loc[:, [str(kj) for kj in idx]].values, n_part=SIZE)))\n            basic.extend([basic_feature_extractor(train.loc[:, str(kj)].values) for kj in idx])\n            y_basic.extend([meta.loc[meta.signal_id==kj, 'target'].values for kj in idx])\n            y_train.append(meta.loc[meta.id_measurement==i, 'target'].values)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7974b59f34419f56fe74733f8befb7c4c16cc8af"},"cell_type":"code","source":"del train;gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"44826c0ee4f1c7499b1a715b7a6a8c58507b42ad"},"cell_type":"code","source":"np.shape(basic)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c832e928e6f65cd3e511b80cfceb4538507f9487"},"cell_type":"code","source":"x_base = np.array(basic)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cb1d07b093631057f6087c8fbeca890946e6e228"},"cell_type":"code","source":"from sklearn.decomposition import PCA","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5ce66482da187821b7f1fa8aa54070c3ad47f001"},"cell_type":"code","source":"pca = PCA(n_components=2)\nx_pca = pca.fit_transform(x_base)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d75bd8196c70eb3acf7c23c4c3108ccd1e5b8a4c"},"cell_type":"code","source":"y_basic = np.array(y_basic)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8f395f029412e131091a02e83706bc9a3e3d97a6"},"cell_type":"code","source":"x_pos, x_neg = [], []\nfor _x, _y in zip(x_pca, y_basic):\n    if _y == 1:\n        x_pos.append(_x)\n    else:\n        x_neg.append(_x)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"410d4b1deb86b307f61c5f18e3fe692bcb715ced"},"cell_type":"code","source":"x_pos, x_neg = np.array(x_pos), np.array(x_neg)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7d1b19d8e9113e5d4b7c4fc9b7b6d1223e481d3e"},"cell_type":"code","source":"x_pos.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8382468f57ebf0df5832800f0c68e5973850fb9b"},"cell_type":"code","source":"plt.figure(figsize=(10, 10))\nplt.scatter(x_neg[:, 0], x_neg[:, 1])\nplt.scatter(x_pos[:, 0], x_pos[:, 1])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ea6b997654036dc2dae51ce71f116378e5704a51"},"cell_type":"code","source":"np.unique(y_train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2d964d05876d085ccf355566cadc6817e5a58a1a"},"cell_type":"code","source":"x_train = np.array(x_train)\ny_train = np.array(y_train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6122e7b8519250e1c06bd907033dbaf358f4a47f"},"cell_type":"code","source":"print(np.shape(x_train), np.shape(y_train))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"999a6a23978015619ded38d054df2065f0d00732"},"cell_type":"markdown","source":"In cell below we can see, that for one measurement various number channels can be fault."},{"metadata":{"trusted":true,"_uuid":"dba3379cda05f9e1a7637e8189fb56192bdde2e9"},"cell_type":"code","source":"csum = np.sum(y_train, axis=-1)\nnp.unique(csum)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0ebf3d15942251f9835f9f1df34bee811f16d904"},"cell_type":"code","source":"pos_index = np.where(csum>0)[0]\nneg_index = np.where(csum==0)[0]","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"43c1c983dfa99917e541d8d0b0aa2c2794724fd1"},"cell_type":"markdown","source":"At the plots below we can see features(amplitude), which extracted from signal, for positive and negative case."},{"metadata":{"trusted":true,"_uuid":"f22bbbfab87076860cdc7827582116bafc2c253f"},"cell_type":"code","source":"figure, (ax1, ax2, ax3) = plt.subplots(1,3, figsize=(24,8))\nsns.heatmap(x_train[pos_index[0], :, :], ax=ax1)\nsns.heatmap(x_train[pos_index[10], :, :], ax=ax2)\nsns.heatmap(x_train[pos_index[20], :, :], ax=ax3)\nax1.set_axis_off()\nax2.set_axis_off()\nax3.set_axis_off();","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9332f7fd3c724e41438bb9d809f4d92f11dec253"},"cell_type":"code","source":"figure, (ax1, ax2, ax3) = plt.subplots(1,3, figsize=(24,8))\nsns.heatmap(x_train[neg_index[0], :, :], ax=ax1)\nsns.heatmap(x_train[neg_index[10], :, :], ax=ax2)\nsns.heatmap(x_train[neg_index[20], :, :], ax=ax3)\nax1.set_axis_off()\nax2.set_axis_off()\nax3.set_axis_off();","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"aeb85d0de2df22b0b6c975521dc5d7579be9f3ef"},"cell_type":"markdown","source":"<center> **CNN**"},{"metadata":{"trusted":true,"_uuid":"a032918a1f3df800021d254295dd8a4d9bf70103"},"cell_type":"code","source":"from keras.layers import *\nfrom keras import Model\nfrom keras.optimizers import Nadam\nfrom keras.utils import Sequence\nfrom keras.callbacks import ModelCheckpoint\nfrom keras.constraints import max_norm\nimport keras.backend as K","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3e5324dfe04313a8196fea7b14d3940ee0cfcaf5"},"cell_type":"code","source":"class DataGenerator(Sequence):\n    def __init__(self, x, y, batch_size=64):\n        self._x = x\n        self._y = y\n        self._batch_size = batch_size\n        \n    def __getitem__(self, index):\n        index = np.random.choice([i for i in range(len(self._y))], size=(self._batch_size))\n        \n        x_batch, y_batch, w = [], [], []\n        for _x, _y in zip(self._x[index], self._y[index]):\n            _x = self.cyclic_shift(_x)\n            #_x, _y = self.phase_permutation(_x, _y)\n            x_batch.append(_x)\n            y_batch.append(_y)\n            #w.append(np.sum(_y)*100+1)\n        return np.array(x_batch), np.array(y_batch)#, np.array(w)\n    \n    def __len__(self):\n        return len(self._y)//self._batch_size\n    \n    @staticmethod\n    def cyclic_shift(x, alpha=0.5):\n        s = np.random.uniform(0, alpha)\n        part = int(len(x)*s)\n        x_ = x[:part, :]\n        _x = x[-len(x)+part:, :]\n        return np.concatenate([_x, x_], axis=0)\n    \n    @staticmethod\n    def phase_permutation(x, y):\n        phase = np.random.permutation([0,1,2])\n        out_x, out_y = [], []\n        for indx in phase:\n            out_x.append(x[..., indx])\n            out_y.append(y[indx])\n        return np.stack(out_x, axis=-1), np.array(out_y)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1e4a851a7110c11b59a725a85886055f7d65b579"},"cell_type":"code","source":"def matthews_corr_coeff(y_true, y_pred):\n    y_pos_pred = K.round(K.clip(y_pred, 0, 1))\n    y_pos_true = K.round(K.clip(y_true, 0, 1))\n    \n    y_neg_pred = 1 - y_pos_pred\n    y_neg_true = 1 - y_pos_true\n\n    tp = K.sum(y_pos_true * y_pos_pred)\n    tn = K.sum(y_neg_true * y_neg_pred)\n    fp = K.sum(y_neg_true * y_pos_pred)\n    fn = K.sum(y_pos_true * y_neg_pred)\n    return (tp * tn - fp * fn) / (K.sqrt((tp + fp) * (tp + fn) * (tn + fp) * (tn + fn)) + K.epsilon())","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"f0c1aa6f938890dd4ccc7735eb0dade7c90e9bf0"},"cell_type":"markdown","source":"For time series very useful use dilation and  for CNN useful apply in architecture residual connection."},{"metadata":{"_uuid":"ee70d99703757e958bd3f63d14be7b14642999ee"},"cell_type":"markdown","source":"https://www.kaggle.com/ashishpatel26/transfer-learning-in-basic-nn"},{"metadata":{"trusted":true,"_uuid":"04b718bb275fb5f73866fb43d8d2b338372d9741"},"cell_type":"code","source":"def get_model(inp_shape=(SIZE, 3)):\n    inp = Input(inp_shape)\n    # 256\n    x = Conv1D(32, kernel_size=3, dilation_rate=3,use_bias=False,kernel_constraint=max_norm(2.))(inp)\n    x = BatchNormalization()(x)\n    x = Activation('selu')(x)\n    # 250\n    x = Conv1D(32, kernel_size=3, dilation_rate=2,use_bias=False,kernel_constraint=max_norm(2.))(x)\n    x = BatchNormalization()(x)\n    x = Activation('selu')(x)\n    # 245\n    x = MaxPooling1D(pool_size=3, strides=2)(x)\n    \n    #  122\n    x = Conv1D(64, kernel_size=3, dilation_rate=3,use_bias=False,kernel_constraint=max_norm(2.))(x)\n    x = BatchNormalization()(x)\n    x = Activation('selu')(x)\n    # 116\n    x = Conv1D(64, kernel_size=3, dilation_rate=2,use_bias=False,kernel_constraint=max_norm(2.))(x)\n    x = BatchNormalization()(x)\n    x = Activation('selu')(x)\n    # 112\n    x = Conv1D(64, kernel_size=3, dilation_rate=1,use_bias=False,kernel_constraint=max_norm(2.))(x)\n    x = BatchNormalization()(x)\n    x = Activation('selu')(x)\n    # 110\n    x = MaxPooling1D(pool_size=3, strides=2)(x)\n    \n    # 54\n    x = Conv1D(128, kernel_size=3, dilation_rate=3,use_bias=False,kernel_constraint=max_norm(2.))(x)\n    x = BatchNormalization()(x)\n    x = Activation('selu')(x)\n    # 48\n    x = Conv1D(128, kernel_size=3, dilation_rate=2,use_bias=False,kernel_constraint=max_norm(2.))(x)\n    x = BatchNormalization()(x)\n    x = Activation('selu')(x)\n    # 44\n    x = Conv1D(128, kernel_size=3, dilation_rate=1,use_bias=False,kernel_constraint=max_norm(2.))(x)\n    x = BatchNormalization()(x)\n    x = Activation('selu')(x)\n    # 42\n    x = MaxPooling1D(pool_size=3, strides=2)(x)\n    \n    #  20\n    x = Conv1D(256, kernel_size=3, dilation_rate=3,use_bias=False,kernel_constraint=max_norm(2.))(x)\n    x = BatchNormalization()(x)\n    x = Activation('selu')(x)\n    # 15\n    x = Conv1D(256, kernel_size=3, dilation_rate=2,use_bias=False,kernel_constraint=max_norm(2.))(x)\n    x = BatchNormalization()(x)\n    x = Activation('selu')(x)\n    # 10\n    x = Conv1D(256, kernel_size=3, dilation_rate=1,use_bias=False,kernel_constraint=max_norm(2.))(x)\n    x = BatchNormalization()(x)\n    x = Activation('selu')(x)\n    # 8\n    x = GlobalMaxPooling1D()(x)\n    \n    x = Dropout(0.75)(x)\n    \n    max_out = []\n    for _ in range(5):\n        max_out.append(Dense(128,use_bias=False,kernel_constraint=max_norm(2.))(x))\n    x = Maximum()(max_out)\n    x = BatchNormalization()(x)\n    \n    out = Dense(3, activation='sigmoid',kernel_constraint=max_norm(2.))(x)\n    return Model(inp, out)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ab38a0e5f95bef869770464206962615604182c1"},"cell_type":"code","source":"mcp = ModelCheckpoint('model.h5',monitor='val_matthews_corr_coeff', mode='max')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0549bf7df04eafab7066f3b942af9eba2b85d2da"},"cell_type":"code","source":"from sklearn.model_selection import train_test_split, KFold, StratifiedKFold","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8a87bd3dcfa3cdd2549b74887800d8c5af9c818d"},"cell_type":"code","source":"x_tr, x_val, y_tr, y_val = train_test_split(x_train, y_train, shuffle=True, train_size=0.75, random_state=28, stratify=csum)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e6728a23a3ab1ff7d8d25c4a8ab04a70200330d6"},"cell_type":"code","source":"tr_gen = DataGenerator(x_tr, y_tr)\nvl_gen = DataGenerator(x_val, y_val)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8eba96d9f45763f81c5a36db65e6898e21e4217c","scrolled":false},"cell_type":"code","source":"model = get_model()\nprint(model.summary())\nmodel.compile(optimizer=Nadam(4*1e-3, schedule_decay=1e-7),loss='binary_crossentropy', metrics=['accuracy', matthews_corr_coeff])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":false,"_uuid":"45810b5700c47d89fe4e6a7c9c1fa0b641726810"},"cell_type":"code","source":"model.fit_generator(tr_gen, steps_per_epoch=1000, epochs=10, callbacks=[mcp], validation_data=vl_gen, validation_steps=400)\nmodel.load_weights('model.h5')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d3062f8904372c4f61bc3bf80829ecbf5a84f7be"},"cell_type":"code","source":"best_thr = 0.01\nbest_metric = 0\ny_val = y_val.flatten()\nproba = model.predict(x_val)\nskf = StratifiedKFold(n_splits=5, shuffle=True, random_state=28)\nfor i in tqdm(np.linspace(0.01, 0.9999, 30)):\n    for j in np.linspace(0.01, 0.9999, 30):\n        for k in np.linspace(0.01, 0.9999, 30):\n            y_pred = np.int32(np.stack([proba[:, 0] > i, proba[:, 1] > j, proba[:, 2] > k], axis=-1)).flatten()\n            m = matthews_corrcoef(y_val, y_pred)\n            if m > best_metric:\n                best_thr = (i, j, k)\n                best_metric= m\nprint('Best threshold: ',best_thr, ' ; Best metric: ',best_metric)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ef94b200bd64d8dc472fa1cc64624f0be947d448"},"cell_type":"markdown","source":"<center> **Predict**"},{"metadata":{"trusted":true,"_uuid":"c337116ccf416df4fd45f0dabe7b7a8f9549384c"},"cell_type":"code","source":"meta = pd.read_csv(os.path.join(INIT_DIR, 'metadata_test.csv'))\nsubmission = pd.read_csv(os.path.join(INIT_DIR, 'sample_submission.csv'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e1fd36f9577fd64940db16247521f719c97ec6b8"},"cell_type":"code","source":"meta.corr()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e015fdba05bc69fbb45f7eac2c2445685f75cf19"},"cell_type":"code","source":"len(meta.id_measurement.unique())*3","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3c277826de54c587e2f3d369eee3d01e927293c8"},"cell_type":"code","source":"len(meta.signal_id)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true,"_uuid":"b71fe8cc926b0f64bdf85c132907e211b4740df6"},"cell_type":"code","source":"for b in tqdm(range(0, len(meta), 3000)):\n    idx = []\n    if b+3000 < len(meta):\n        idx = meta.signal_id[b:b+3000].values\n    else:\n        idx = meta.signal_id[b:].values\n    subset_test = pq.read_pandas(os.path.join(INIT_DIR, 'test.parquet'), columns=[str(j) for j in idx]).to_pandas()\n    x_batch = []\n    for i in range(0, len(idx)//3):\n        _x  = []\n        for j in range(0, 3):\n            _x.append(flatiron(subset_test.iloc[:, i*3+j].values))\n        _x = np.concatenate(_x, axis=-1)\n        x_batch.append(feature_extractor(_x, n_part=SIZE))\n    y_batch = model.predict(np.array(x_batch), verbose=0)\n    pred = []\n    for yj in y_batch:\n        for j, yi in enumerate(yj):\n            pred.append(np.int32(yi > best_thr[j]))\n    for jdx, iy in zip(idx, pred):\n        submission.loc[submission.signal_id == jdx, 'target'] = iy","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5f5b74efda85d490a79556ff61c973b756835a4a"},"cell_type":"code","source":"submission.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e277dade6cc0561a5c2cab026a0d35387384feb9"},"cell_type":"code","source":"submission.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"949d8ab2e68afbd8f7be1f08d82ad49275a186b6"},"cell_type":"markdown","source":"Unsolved problems:\n* apply stratifiedkfold"},{"metadata":{"trusted":true,"_uuid":"2f91d8280e1b68e7185ab5ef1c397ce2649686da"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d595a249859fa34502eaaf2e027e9d8acbb9a7c5"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}