{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install keras","metadata":{"_uuid":"e154a47bf09b8770980486e87786317a1b3038e1","execution":{"iopub.status.busy":"2022-05-20T15:24:18.445113Z","iopub.execute_input":"2022-05-20T15:24:18.445715Z","iopub.status.idle":"2022-05-20T15:24:25.322627Z","shell.execute_reply.started":"2022-05-20T15:24:18.445647Z","shell.execute_reply":"2022-05-20T15:24:25.321892Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install --upgrade pip","metadata":{"execution":{"iopub.status.busy":"2022-05-20T15:24:25.324331Z","iopub.execute_input":"2022-05-20T15:24:25.324612Z","iopub.status.idle":"2022-05-20T15:24:35.883560Z","shell.execute_reply.started":"2022-05-20T15:24:25.324555Z","shell.execute_reply":"2022-05-20T15:24:35.882676Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nimport pyarrow.parquet as pq # Used to read the data\nimport os \nimport numpy as np\nfrom keras.layers import * # Keras is the most friendly Neural Network library, this Kernel use a lot of layers classes\nfrom keras.models import Model\nfrom tqdm import tqdm # Processing time measurement\nfrom sklearn.model_selection import train_test_split \nfrom keras import backend as K # The backend give us access to tensorflow operations and allow us to create the Attention class\nfrom keras import optimizers # Allow us to access the Adam class to modify some parameters\nfrom sklearn.model_selection import GridSearchCV, StratifiedKFold # Used to use Kfold to train our model\nfrom keras.callbacks import * # This object helps the model to train in a smarter way, avoiding overfitting","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-05-20T15:24:35.886876Z","iopub.execute_input":"2022-05-20T15:24:35.887147Z","iopub.status.idle":"2022-05-20T15:24:39.955997Z","shell.execute_reply.started":"2022-05-20T15:24:35.887106Z","shell.execute_reply":"2022-05-20T15:24:39.955055Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"INIT_DIR = '../input'\npraq_train_data = pq.read_pandas(os.path.join(INIT_DIR, 'train.parquet'), columns=[str(i) for i in range(2001)]).to_pandas()\npraq_train_data.shape","metadata":{"execution":{"iopub.status.busy":"2022-05-20T15:24:39.961172Z","iopub.execute_input":"2022-05-20T15:24:39.963445Z","iopub.status.idle":"2022-05-20T15:25:00.604615Z","shell.execute_reply.started":"2022-05-20T15:24:39.963388Z","shell.execute_reply":"2022-05-20T15:25:00.603981Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"praq_train_data.head","metadata":{"execution":{"iopub.status.busy":"2022-05-20T15:25:00.605912Z","iopub.execute_input":"2022-05-20T15:25:00.606209Z","iopub.status.idle":"2022-05-20T15:25:00.786427Z","shell.execute_reply.started":"2022-05-20T15:25:00.606141Z","shell.execute_reply":"2022-05-20T15:25:00.785472Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"meta_train_df = pd.read_csv(INIT_DIR + '/metadata_train.csv')\nmeta_test_df = pd.read_csv(INIT_DIR + '/metadata_test.csv')","metadata":{"execution":{"iopub.status.busy":"2022-05-20T15:25:00.787747Z","iopub.execute_input":"2022-05-20T15:25:00.788266Z","iopub.status.idle":"2022-05-20T15:25:00.820969Z","shell.execute_reply.started":"2022-05-20T15:25:00.788207Z","shell.execute_reply":"2022-05-20T15:25:00.820016Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"meta_train_df.head\n","metadata":{"execution":{"iopub.status.busy":"2022-05-20T15:25:00.822284Z","iopub.execute_input":"2022-05-20T15:25:00.822582Z","iopub.status.idle":"2022-05-20T15:25:00.844986Z","shell.execute_reply.started":"2022-05-20T15:25:00.822532Z","shell.execute_reply":"2022-05-20T15:25:00.843665Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"meta_train_df.shape","metadata":{"execution":{"iopub.status.busy":"2022-05-20T15:25:00.846642Z","iopub.execute_input":"2022-05-20T15:25:00.847125Z","iopub.status.idle":"2022-05-20T15:25:00.853327Z","shell.execute_reply.started":"2022-05-20T15:25:00.846936Z","shell.execute_reply":"2022-05-20T15:25:00.852168Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"meta_test_df.info","metadata":{"execution":{"iopub.status.busy":"2022-05-20T15:25:00.854892Z","iopub.execute_input":"2022-05-20T15:25:00.855223Z","iopub.status.idle":"2022-05-20T15:25:00.878202Z","shell.execute_reply.started":"2022-05-20T15:25:00.855161Z","shell.execute_reply":"2022-05-20T15:25:00.877539Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"shape: \", meta_test_df.shape)\nprint(\"Count: \", meta_train_df.target.value_counts())\n","metadata":{"execution":{"iopub.status.busy":"2022-05-20T15:25:00.880681Z","iopub.execute_input":"2022-05-20T15:25:00.881187Z","iopub.status.idle":"2022-05-20T15:25:00.890345Z","shell.execute_reply.started":"2022-05-20T15:25:00.881129Z","shell.execute_reply":"2022-05-20T15:25:00.889433Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# select how many folds will be created\nN_SPLITS = 5\n# it is just a constant with the measurements data size\nsample_size = 800000","metadata":{"_uuid":"6e6379386e44afc69bee8895a52da22199e888fb","execution":{"iopub.status.busy":"2022-05-20T15:25:00.891799Z","iopub.execute_input":"2022-05-20T15:25:00.892419Z","iopub.status.idle":"2022-05-20T15:25:00.897191Z","shell.execute_reply.started":"2022-05-20T15:25:00.892220Z","shell.execute_reply":"2022-05-20T15:25:00.896022Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# It is the official metric used in this competition\n# below is the declaration of a function used inside the keras model, calculation with K (keras backend / thensorflow)\ndef matthews_correlation(y_true, y_pred):\n    '''Calculates the Matthews correlation coefficient measure for quality\n    of binary classification problems.\n    '''\n    y_pred_pos = K.round(K.clip(y_pred, 0, 1))\n    y_pred_neg = 1 - y_pred_pos\n\n    y_pos = K.round(K.clip(y_true, 0, 1))\n    y_neg = 1 - y_pos\n\n    tp = K.sum(y_pos * y_pred_pos)\n    tn = K.sum(y_neg * y_pred_neg)\n\n    fp = K.sum(y_neg * y_pred_pos)\n    fn = K.sum(y_pos * y_pred_neg)\n\n    numerator = (tp * tn - fp * fn)\n    denominator = K.sqrt((tp + fp) * (tp + fn) * (tn + fp) * (tn + fn))\n\n    return numerator / (denominator + K.epsilon())","metadata":{"_uuid":"c3340ee96becb5ca8f075d9c44b7df383ddba5ee","execution":{"iopub.status.busy":"2022-05-20T15:25:00.898637Z","iopub.execute_input":"2022-05-20T15:25:00.899169Z","iopub.status.idle":"2022-05-20T15:25:00.908384Z","shell.execute_reply.started":"2022-05-20T15:25:00.899101Z","shell.execute_reply":"2022-05-20T15:25:00.907209Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# https://www.kaggle.com/suicaokhoailang/lstm-attention-baseline-0-652-lb\n\nclass Attention(Layer):\n    def __init__(self, step_dim,\n                 W_regularizer=None, b_regularizer=None,\n                 W_constraint=None, b_constraint=None,\n                 bias=True, **kwargs):\n        self.supports_masking = True\n        self.init = initializers.get('glorot_uniform')\n\n        self.W_regularizer = regularizers.get(W_regularizer)\n        self.b_regularizer = regularizers.get(b_regularizer)\n\n        self.W_constraint = constraints.get(W_constraint)\n        self.b_constraint = constraints.get(b_constraint)\n\n        self.bias = bias\n        self.step_dim = step_dim\n        self.features_dim = 0\n        super(Attention, self).__init__(**kwargs)\n\n    def build(self, input_shape):\n        assert len(input_shape) == 3\n\n        self.W = self.add_weight((input_shape[-1],),\n                                 initializer=self.init,\n                                 name='{}_W'.format(self.name),\n                                 regularizer=self.W_regularizer,\n                                 constraint=self.W_constraint)\n        self.features_dim = input_shape[-1]\n\n        if self.bias:\n            self.b = self.add_weight((input_shape[1],),\n                                     initializer='zero',\n                                     name='{}_b'.format(self.name),\n                                     regularizer=self.b_regularizer,\n                                     constraint=self.b_constraint)\n        else:\n            self.b = None\n\n        self.built = True\n\n    def compute_mask(self, input, input_mask=None):\n        return None\n\n    def call(self, x, mask=None):\n        features_dim = self.features_dim\n        step_dim = self.step_dim\n\n        eij = K.reshape(K.dot(K.reshape(x, (-1, features_dim)),\n                        K.reshape(self.W, (features_dim, 1))), (-1, step_dim))\n\n        if self.bias:\n            eij += self.b\n\n        eij = K.tanh(eij)\n\n        a = K.exp(eij)\n\n        if mask is not None:\n            a *= K.cast(mask, K.floatx())\n\n        a /= K.cast(K.sum(a, axis=1, keepdims=True) + K.epsilon(), K.floatx())\n\n        a = K.expand_dims(a)\n        weighted_input = x * a\n        return K.sum(weighted_input, axis=1)\n\n    def compute_output_shape(self, input_shape):\n        return input_shape[0],  self.features_dim","metadata":{"_uuid":"eda7ea366117d1ce8e5fce69e5bba333821d8b48","execution":{"iopub.status.busy":"2022-05-20T15:25:00.910042Z","iopub.execute_input":"2022-05-20T15:25:00.910583Z","iopub.status.idle":"2022-05-20T15:25:00.928838Z","shell.execute_reply.started":"2022-05-20T15:25:00.910367Z","shell.execute_reply":"2022-05-20T15:25:00.928029Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# just load train data\ndf_train = pd.read_csv('../input/metadata_train.csv')\n# set index, it makes the data access much faster\ndf_train = df_train.set_index(['id_measurement', 'phase'])\ndf_train.head()","metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","execution":{"iopub.status.busy":"2022-05-20T15:25:00.930864Z","iopub.execute_input":"2022-05-20T15:25:00.931243Z","iopub.status.idle":"2022-05-20T15:25:00.962291Z","shell.execute_reply.started":"2022-05-20T15:25:00.931149Z","shell.execute_reply":"2022-05-20T15:25:00.961425Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# the min and max values from the train data, the measurements\nmax_num = 127\nmin_num = -128","metadata":{"_uuid":"26df6c7fbfecd537404866faec13d1238ae3ebc6","execution":{"iopub.status.busy":"2022-05-20T15:25:00.963565Z","iopub.execute_input":"2022-05-20T15:25:00.964050Z","iopub.status.idle":"2022-05-20T15:25:00.968788Z","shell.execute_reply.started":"2022-05-20T15:25:00.963996Z","shell.execute_reply":"2022-05-20T15:25:00.967583Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# This function standardize the data from (-128 to 127) to (-1 to 1)\n# Theoretically it helps in the NN Model training\ndef min_max_transf(ts, min_data, max_data, range_needed=(-1,1)):\n    if min_data < 0:\n        ts_std = (ts + abs(min_data)) / (max_data + abs(min_data))\n    else:\n        ts_std = (ts - min_data) / (max_data - min_data)\n    if range_needed[0] < 0:    \n        return ts_std * (range_needed[1] + abs(range_needed[0])) + range_needed[0]\n    else:\n        return ts_std * (range_needed[1] - range_needed[0]) + range_needed[0]","metadata":{"_uuid":"7b0717b14bcfcba1f48d33c8161ae51c778687af","execution":{"iopub.status.busy":"2022-05-20T15:25:00.970451Z","iopub.execute_input":"2022-05-20T15:25:00.971039Z","iopub.status.idle":"2022-05-20T15:25:00.979500Z","shell.execute_reply.started":"2022-05-20T15:25:00.970983Z","shell.execute_reply":"2022-05-20T15:25:00.978624Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# This is one of the most important peace of code of this Kernel\n# Any power line contain 3 phases of 800000 measurements, or 2.4 millions data \n# It would be praticaly impossible to build a NN with an input of that size\n# The ideia here is to reduce it each phase to a matrix of <n_dim> bins by n features\n# Each bean is a set of 5000 measurements (800000 / 160), so the features are extracted from this 5000 chunk data.\ndef transform_ts(ts, n_dim=160, min_max=(-1,1)):\n    # convert data into -1 to 1\n    ts_std = min_max_transf(ts, min_data=min_num, max_data=max_num)\n    # bucket or chunk size, 5000 in this case (800000 / 160)\n    bucket_size = int(sample_size / n_dim)\n    # new_ts will be the container of the new data\n    new_ts = []\n    # this for iteract any chunk/bucket until reach the whole sample_size (800000)\n    for i in range(0, sample_size, bucket_size):\n        # cut each bucket to ts_range\n        ts_range = ts_std[i:i + bucket_size]\n        # calculate each feature\n        mean = ts_range.mean()\n        std = ts_range.std() # standard deviation\n        std_top = mean + std # I have to test it more, but is is like a band\n        std_bot = mean - std\n        # I think that the percentiles are very important, it is like a distribuiton analysis from eath chunk\n        percentil_calc = np.percentile(ts_range, [0, 1, 25, 50, 75, 99, 100]) \n        max_range = percentil_calc[-1] - percentil_calc[0] # this is the amplitude of the chunk\n        relative_percentile = percentil_calc - mean # maybe it could heap to understand the asymmetry\n        # now, we just add all the features to new_ts and convert it to np.array\n        new_ts.append(np.concatenate([np.asarray([mean, std, std_top, std_bot, max_range]),percentil_calc, relative_percentile]))\n    return np.asarray(new_ts)","metadata":{"_uuid":"c6137bbbe75c3a1509a5f98e08805dbbd492aa37","execution":{"iopub.status.busy":"2022-05-20T15:25:00.981722Z","iopub.execute_input":"2022-05-20T15:25:00.982223Z","iopub.status.idle":"2022-05-20T15:25:00.991942Z","shell.execute_reply.started":"2022-05-20T15:25:00.982019Z","shell.execute_reply":"2022-05-20T15:25:00.991097Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# this function take a piece of data and convert using transform_ts(), but it does to each of the 3 phases\n# if we would try to do in one time, could exceed the RAM Memmory\ndef prep_data(start, end):\n    # load a piece of data from file\n    praq_train = pq.read_pandas('../input/train.parquet', columns=[str(i) for i in range(start, end)]).to_pandas()\n    X = []\n    y = []\n    # using tdqm to evaluate processing time\n    # takes each index from df_train and iteract it from start to end\n    # it is divided by 3 because for each id_measurement there are 3 id_signal, and the start/end parameters are id_signal\n    for id_measurement in tqdm(df_train.index.levels[0].unique()[int(start/3):int(end/3)]):\n        X_signal = []\n        # for each phase of the signal\n        for phase in [0,1,2]:\n            # extract from df_train both signal_id and target to compose the new data sets\n            signal_id, target = df_train.loc[id_measurement].loc[phase]\n            # but just append the target one time, to not triplicate it\n            if phase == 0:\n                y.append(target)\n            # extract and transform data into sets of features\n            X_signal.append(transform_ts(praq_train[str(signal_id)]))\n        # concatenate all the 3 phases in one matrix\n        X_signal = np.concatenate(X_signal, axis=1)\n        # add the data to X\n        X.append(X_signal)\n    X = np.asarray(X)\n    y = np.asarray(y)\n    return X, y","metadata":{"_uuid":"7460e718a605803f1d9e4fbec61750a0deb02a47","execution":{"iopub.status.busy":"2022-05-20T15:25:00.993802Z","iopub.execute_input":"2022-05-20T15:25:00.994160Z","iopub.status.idle":"2022-05-20T15:25:01.004866Z","shell.execute_reply.started":"2022-05-20T15:25:00.994091Z","shell.execute_reply":"2022-05-20T15:25:01.003969Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"praq_train_data.head","metadata":{"execution":{"iopub.status.busy":"2022-05-20T15:25:01.006314Z","iopub.execute_input":"2022-05-20T15:25:01.006854Z","iopub.status.idle":"2022-05-20T15:25:01.184665Z","shell.execute_reply.started":"2022-05-20T15:25:01.006798Z","shell.execute_reply":"2022-05-20T15:25:01.183647Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# this code is very simple, divide the total size of the df_train into two sets and process it\nX = []\ny = []\ndef load_all():\n    total_size = len(df_train)\n    for ini, end in [(0, int(total_size/2)), (int(total_size/2), total_size)]:\n        X_temp, y_temp = prep_data(ini, end)\n        X.append(X_temp)\n        y.append(y_temp)\nload_all()\nX = np.concatenate(X)\ny = np.concatenate(y)","metadata":{"_uuid":"52dc826ab9ee1dd56c9fb29bd5c1b2d26b5928bf","execution":{"iopub.status.busy":"2022-05-20T15:25:01.187609Z","iopub.execute_input":"2022-05-20T15:25:01.188109Z","iopub.status.idle":"2022-05-20T15:39:21.116565Z","shell.execute_reply.started":"2022-05-20T15:25:01.188050Z","shell.execute_reply":"2022-05-20T15:39:21.115820Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# The X shape here is very important. It is also important undertand a little how a LSTM works\n# X.shape[0] is the number of id_measuremts contained in train data\n# X.shape[1] is the number of chunks resultant of the transformation, each of this date enters in the LSTM serialized\n# This way the LSTM can understand the position of a data relative with other and activate a signal that needs\n# a serie of inputs in a specifc order.\n# X.shape[3] is the number of features multiplied by the number of phases (3)\nprint(X.shape, y.shape)\n# save data into file, a numpy specific format\nnp.save(\"X.npy\",X)\nnp.save(\"y.npy\",y)","metadata":{"_uuid":"51ad0e25b00536de6170168499923d82ae1d735f","execution":{"iopub.status.busy":"2022-05-20T15:39:21.117975Z","iopub.execute_input":"2022-05-20T15:39:21.118289Z","iopub.status.idle":"2022-05-20T15:39:21.295451Z","shell.execute_reply.started":"2022-05-20T15:39:21.118227Z","shell.execute_reply":"2022-05-20T15:39:21.294605Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# This is NN LSTM Model creation\ndef model_lstm(input_shape):\n    # The shape was explained above, must have this order\n    inp = Input(shape=(input_shape[1], input_shape[2],))\n    # This is the LSTM layer\n    # Bidirecional implies that the 160 chunks are calculated in both ways, 0 to 159 and 159 to zero\n    # although it appear that just 0 to 159 way matter, I have tested with and without, and tha later worked best\n    # 128 and 64 are the number of cells used, too many can overfit and too few can underfit\n    x = Bidirectional(CuDNNLSTM(128, return_sequences=True))(inp)\n    # The second LSTM can give more fire power to the model, but can overfit it too\n    x = Bidirectional(CuDNNLSTM(64, return_sequences=True))(x)\n    # Attention is a new tecnology that can be applyed to a Recurrent NN to give more meanings to a signal found in the middle\n    # of the data, it helps more in longs chains of data. A normal RNN give all the responsibility of detect the signal\n    # to the last cell. Google RNN Attention for more information :)\n    x = Attention(input_shape[1])(x)\n    # A intermediate full connected (Dense) can help to deal with nonlinears outputs\n    x = Dense(64, activation=\"relu\")(x)\n    # A binnary classification as this must finish with shape (1,)\n    x = Dense(1, activation=\"sigmoid\")(x)\n    model = Model(inputs=inp, outputs=x)\n    # Pay attention in the addition of matthews_correlation metric in the compilation, it is a success factor key\n    model.compile(loss='binary_crossentropy', optimizer='adam', metrics=[matthews_correlation])\n    \n    return model","metadata":{"_uuid":"289bc7d1ab8048a60025801b457f8df1d848acbc","execution":{"iopub.status.busy":"2022-05-20T15:39:21.296721Z","iopub.execute_input":"2022-05-20T15:39:21.297228Z","iopub.status.idle":"2022-05-20T15:39:21.305843Z","shell.execute_reply.started":"2022-05-20T15:39:21.297173Z","shell.execute_reply":"2022-05-20T15:39:21.304639Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Here is where the training happens\n\n# First, create a set of indexes of the 5 folds\nsplits = list(StratifiedKFold(n_splits=N_SPLITS, shuffle=True, random_state=2019).split(X, y))\npreds_val = []\ny_val = []\n# Then, iteract with each fold\n# If you dont know, enumerate(['a', 'b', 'c']) returns [(0, 'a'), (1, 'b'), (2, 'c')]\nfor idx, (train_idx, val_idx) in enumerate(splits):\n    K.clear_session() # I dont know what it do, but I imagine that it \"clear session\" :)\n    print(\"Beginning fold {}\".format(idx+1))\n    # use the indexes to extract the folds in the train and validation data\n    train_X, train_y, val_X, val_y = X[train_idx], y[train_idx], X[val_idx], y[val_idx]\n    # instantiate the model for this fold\n    model = model_lstm(train_X.shape)\n    # This checkpoint helps to avoid overfitting. It just save the weights of the model if it delivered an\n    # validation matthews_correlation greater than the last one.\n    ckpt = ModelCheckpoint('weights_{}.h5'.format(idx), save_best_only=True, save_weights_only=True, verbose=1, monitor='val_matthews_correlation', mode='max')\n    # Train, train, train\n    history=model.fit(train_X, train_y, batch_size=128, epochs=50, validation_data=[val_X, val_y], callbacks=[ckpt])\n    # loads the best weights saved by the checkpoint\n    model.load_weights('weights_{}.h5'.format(idx))\n    # Add the predictions of the validation to the list preds_val\n    preds_val.append(model.predict(val_X, batch_size=512))\n    # and the val true y\n    y_val.append(val_y)\n\n# concatenates all and prints the shape    \npreds_val = np.concatenate(preds_val)[...,0]\ny_val = np.concatenate(y_val)\npreds_val.shape, y_val.shape","metadata":{"_uuid":"8d6f4ca319c383b1b4f671a37c5a324136e7a466","execution":{"iopub.status.busy":"2022-05-20T15:39:21.307343Z","iopub.execute_input":"2022-05-20T15:39:21.307888Z","iopub.status.idle":"2022-05-20T15:44:53.139393Z","shell.execute_reply.started":"2022-05-20T15:39:21.307612Z","shell.execute_reply":"2022-05-20T15:44:53.138619Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ls -lrth","metadata":{"execution":{"iopub.status.busy":"2022-05-20T15:44:53.141018Z","iopub.execute_input":"2022-05-20T15:44:53.141334Z","iopub.status.idle":"2022-05-20T15:44:53.962789Z","shell.execute_reply.started":"2022-05-20T15:44:53.141268Z","shell.execute_reply":"2022-05-20T15:44:53.961935Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nplt.plot(history.history[\"matthews_correlation\"])\nplt.plot(history.history[\"val_matthews_correlation\"])\nplt.title(\"model accuracy\")\nplt.ylabel(\"accuracy\")\nplt.xlabel(\"epoch\")\nplt.legend([\"Train\",\"test\"], loc=\"upper left\")\nplt.show()\n\nplt.plot(history.history[\"loss\"])\nplt.plot(history.history[\"val_loss\"])\nplt.title(\"model loss\")\nplt.ylabel(\"loss\")\nplt.xlabel(\"epoch\")\nplt.legend([\"train\",\"test\"],loc=\"upper left\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-05-20T15:44:53.966088Z","iopub.execute_input":"2022-05-20T15:44:53.966457Z","iopub.status.idle":"2022-05-20T15:44:54.437872Z","shell.execute_reply.started":"2022-05-20T15:44:53.966400Z","shell.execute_reply":"2022-05-20T15:44:54.436890Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# The output of this kernel must be binary (0 or 1), but the output of the NN Model is float (0 to 1).\n# So, find the best threshold to convert float to binary is crucial to the result\n# this piece of code is a function that evaluates all the possible thresholds from 0 to 1 by 0.01\ndef threshold_search(y_true, y_proba):\n    best_threshold = 0\n    best_score = 0\n    for threshold in tqdm([i * 0.01 for i in range(100)]):\n        score = K.eval(matthews_correlation(y_true.astype(np.float64), (y_proba > threshold).astype(np.float64)))\n        if score > best_score:\n            best_threshold = threshold\n            best_score = score\n    search_result = {'threshold': best_threshold, 'matthews_correlation': best_score}\n    return search_result","metadata":{"_uuid":"d28151fd0be9fd9762f3f55e307d82f89bfbd291","execution":{"iopub.status.busy":"2022-05-20T15:44:54.439568Z","iopub.execute_input":"2022-05-20T15:44:54.445340Z","iopub.status.idle":"2022-05-20T15:44:54.455980Z","shell.execute_reply.started":"2022-05-20T15:44:54.445269Z","shell.execute_reply":"2022-05-20T15:44:54.455052Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"best_threshold = threshold_search(y_val, preds_val)['threshold']","metadata":{"_uuid":"6fee7f722ed08bc1453a822a4371ed2d48e08abc","execution":{"iopub.status.busy":"2022-05-20T15:44:54.459283Z","iopub.execute_input":"2022-05-20T15:44:54.460024Z","iopub.status.idle":"2022-05-20T15:45:20.491357Z","shell.execute_reply.started":"2022-05-20T15:44:54.459958Z","shell.execute_reply":"2022-05-20T15:45:20.489566Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ls ../input","metadata":{"execution":{"iopub.status.busy":"2022-05-20T15:45:20.492735Z","iopub.execute_input":"2022-05-20T15:45:20.493028Z","iopub.status.idle":"2022-05-20T15:45:21.296666Z","shell.execute_reply.started":"2022-05-20T15:45:20.492977Z","shell.execute_reply":"2022-05-20T15:45:21.295780Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n# Now load the test data\n# This first part is the meta data, not the main data, the measurements\nmeta_test = pd.read_csv('../input/metadata_test.csv')","metadata":{"_uuid":"ae9bd3fa9d8c0781c0708846bb7f2a9f9e6cbd3c","execution":{"iopub.status.busy":"2022-05-20T15:45:21.300878Z","iopub.execute_input":"2022-05-20T15:45:21.301148Z","iopub.status.idle":"2022-05-20T15:45:21.324926Z","shell.execute_reply.started":"2022-05-20T15:45:21.301099Z","shell.execute_reply":"2022-05-20T15:45:21.324186Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"meta_test = meta_test.set_index(['signal_id'])\nmeta_test.head()","metadata":{"_uuid":"3eb186d032f79c99ffba05dd1a7fabb77e13cec5","execution":{"iopub.status.busy":"2022-05-20T15:45:21.326646Z","iopub.execute_input":"2022-05-20T15:45:21.327210Z","iopub.status.idle":"2022-05-20T15:45:21.345448Z","shell.execute_reply.started":"2022-05-20T15:45:21.327150Z","shell.execute_reply":"2022-05-20T15:45:21.344775Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n# First we daclarete a series of parameters to initiate the loading of the main data\n# it is too large, it is impossible to load in one time, so we are doing it in dividing in 10 parts\nfirst_sig = meta_test.index[0]\nn_parts = 10\nmax_line = len(meta_test)\npart_size = int(max_line / n_parts)\nlast_part = max_line % n_parts\nprint(first_sig, n_parts, max_line, part_size, last_part, n_parts * part_size + last_part)\n# Here we create a list of lists with start index and end index for each of the 10 parts and one for the last partial part\nstart_end = [[x, x+part_size] for x in range(first_sig, max_line + first_sig, part_size)]\nstart_end = start_end[:-1] + [[start_end[-1][0], start_end[-1][0] + last_part]]\nprint(start_end)\nX_test = []\n# now, very like we did above with the train data, we convert the test data part by part\n# transforming the 3 phases 800000 measurement in matrix (160,57)\nfor start, end in start_end:\n    subset_test = pq.read_pandas('../input/test.parquet', columns=[str(i) for i in range(start, end)]).to_pandas()\n    for i in tqdm(subset_test.columns):\n        id_measurement, phase = meta_test.loc[int(i)]\n        subset_test_col = subset_test[i]\n        subset_trans = transform_ts(subset_test_col)\n        X_test.append([i, id_measurement, phase, subset_trans])","metadata":{"_uuid":"6f8e94387f625bff0a9a6289e1ee038908bc5856","execution":{"iopub.status.busy":"2022-05-20T15:45:21.348676Z","iopub.execute_input":"2022-05-20T15:45:21.348945Z","iopub.status.idle":"2022-05-20T16:19:42.960768Z","shell.execute_reply.started":"2022-05-20T15:45:21.348898Z","shell.execute_reply":"2022-05-20T16:19:42.959910Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from keras.utils import plot_model\nplot_model(model,to_file='model.png')","metadata":{"execution":{"iopub.status.busy":"2022-05-20T16:21:42.291445Z","iopub.execute_input":"2022-05-20T16:21:42.291823Z","iopub.status.idle":"2022-05-20T16:21:42.615693Z","shell.execute_reply.started":"2022-05-20T16:21:42.291755Z","shell.execute_reply":"2022-05-20T16:21:42.614694Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_test_input = np.asarray([np.concatenate([X_test[i][3],X_test[i+1][3], X_test[i+2][3]], axis=1) for i in range(0,len(X_test), 3)])\nnp.save(\"X_test.npy\",X_test_input)\nX_test_input.shape","metadata":{"_uuid":"af9aa6b2b8f8a2beda1a02ff998e3072fcad8d06","execution":{"iopub.status.busy":"2022-05-20T16:19:44.883600Z","iopub.execute_input":"2022-05-20T16:19:44.884013Z","iopub.status.idle":"2022-05-20T16:19:46.087832Z","shell.execute_reply.started":"2022-05-20T16:19:44.883923Z","shell.execute_reply":"2022-05-20T16:19:46.084686Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = pd.read_csv('../input/sample_submission.csv')\nprint(len(submission))\nsubmission.head()","metadata":{"_uuid":"cfd265d3e07c4cc1679d2c4d55fe7de631c813e7","execution":{"iopub.status.busy":"2022-05-20T16:19:46.089471Z","iopub.execute_input":"2022-05-20T16:19:46.089911Z","iopub.status.idle":"2022-05-20T16:19:46.155440Z","shell.execute_reply.started":"2022-05-20T16:19:46.089853Z","shell.execute_reply":"2022-05-20T16:19:46.154399Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds_test = []\nfor i in range(N_SPLITS):\n    model.load_weights('weights_{}.h5'.format(i))\n    pred = model.predict(X_test_input, batch_size=300, verbose=1)\n    pred_3 = []\n    for pred_scalar in pred:\n        for i in range(3):\n            pred_3.append(pred_scalar)\n    preds_test.append(pred_3)\n","metadata":{"_uuid":"2f7342296138f6bfd3e9cedd029e1035de3b98fc","execution":{"iopub.status.busy":"2022-05-20T16:26:29.906483Z","iopub.execute_input":"2022-05-20T16:26:29.906845Z","iopub.status.idle":"2022-05-20T16:26:34.537007Z","shell.execute_reply.started":"2022-05-20T16:26:29.906768Z","shell.execute_reply":"2022-05-20T16:26:34.536231Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds_test = (np.squeeze(np.mean(preds_test, axis=0)) > best_threshold).astype(np.int)\npreds_test.shape","metadata":{"_uuid":"9f76c471eaf983707d446c5081ab3d50c4e40ea5","execution":{"iopub.status.busy":"2022-05-20T16:26:54.763754Z","iopub.execute_input":"2022-05-20T16:26:54.764126Z","iopub.status.idle":"2022-05-20T16:26:54.837978Z","shell.execute_reply.started":"2022-05-20T16:26:54.764062Z","shell.execute_reply":"2022-05-20T16:26:54.836806Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(preds_test)","metadata":{"execution":{"iopub.status.busy":"2022-05-20T16:26:59.260338Z","iopub.execute_input":"2022-05-20T16:26:59.260641Z","iopub.status.idle":"2022-05-20T16:26:59.266072Z","shell.execute_reply.started":"2022-05-20T16:26:59.260588Z","shell.execute_reply":"2022-05-20T16:26:59.265063Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission['target'] = preds_test\nsubmission.to_csv('shital_submission.csv', index=False)\nsubmission.head(30)","metadata":{"_uuid":"b35723f85d494b4b6ec630dd7c79135a110a4062","execution":{"iopub.status.busy":"2022-05-20T16:27:06.711908Z","iopub.execute_input":"2022-05-20T16:27:06.712287Z","iopub.status.idle":"2022-05-20T16:27:06.822637Z","shell.execute_reply.started":"2022-05-20T16:27:06.712208Z","shell.execute_reply":"2022-05-20T16:27:06.821888Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ls","metadata":{"execution":{"iopub.status.busy":"2022-05-20T16:27:14.486245Z","iopub.execute_input":"2022-05-20T16:27:14.486555Z","iopub.status.idle":"2022-05-20T16:27:15.326000Z","shell.execute_reply.started":"2022-05-20T16:27:14.486497Z","shell.execute_reply":"2022-05-20T16:27:15.324884Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submit_df = pd.read_csv('shital_submission.csv')\nprint(\"shape: \", submit_df.shape)\nprint(\"Count: \", submit_df.target.value_counts())","metadata":{"execution":{"iopub.status.busy":"2022-05-20T16:20:50.670344Z","iopub.execute_input":"2022-05-20T16:20:50.670694Z","iopub.status.idle":"2022-05-20T16:20:50.721876Z","shell.execute_reply.started":"2022-05-20T16:20:50.670640Z","shell.execute_reply":"2022-05-20T16:20:50.720747Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"_uuid":"d7600d0093a9880003240ef9ce0a1f1303e4d982","trusted":true},"execution_count":null,"outputs":[]}]}