{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Load feature and target","metadata":{}},{"cell_type":"code","source":"%%time\nimport numpy as np \nimport pandas as pd \nimport gc\n\nfeature_path = '/kaggle/input/single-cell-features/'\n\ntrain_df = pd.read_feather(feature_path+'train_cite_inputs_id.feather')\ntest_df = pd.read_feather(feature_path+'test_cite_inputs_id.feather')\n\ntrain_cite_X = np.load(feature_path+'train_cite_X.npy')\ntest_cite_X = np.load(feature_path+'test_cite_X.npy')\n\ntrain_cite_y = np.load(feature_path+'train_cite_targets.npy')  ","metadata":{"execution":{"iopub.status.busy":"2022-11-16T06:39:37.770113Z","iopub.execute_input":"2022-11-16T06:39:37.770496Z","iopub.status.idle":"2022-11-16T06:39:46.935679Z","shell.execute_reply.started":"2022-11-16T06:39:37.770465Z","shell.execute_reply":"2022-11-16T06:39:46.934648Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.shape,test_df.shape,train_cite_X.shape,test_cite_X.shape,train_cite_y.shape","metadata":{"execution":{"iopub.status.busy":"2022-11-16T06:39:46.938529Z","iopub.execute_input":"2022-11-16T06:39:46.939444Z","iopub.status.idle":"2022-11-16T06:39:46.946798Z","shell.execute_reply.started":"2022-11-16T06:39:46.939413Z","shell.execute_reply":"2022-11-16T06:39:46.945837Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Utils","metadata":{}},{"cell_type":"code","source":"import tensorflow as tf\nimport tensorflow_addons as tfa\nfrom sklearn.model_selection import StratifiedKFold,KFold,GroupKFold\n\ndef correlation_score(y_true, y_pred):\n    \"\"\"Scores the predictions according to the competition rules. \n    \n    It is assumed that the predictions are not constant.\n    \n    Returns the average of each sample's Pearson correlation coefficient\"\"\"\n    if type(y_true) == pd.DataFrame: y_true = y_true.values\n    if type(y_pred) == pd.DataFrame: y_pred = y_pred.values\n    corrsum = 0\n    for i in range(len(y_true)):\n        corrsum += np.corrcoef(y_true[i], y_pred[i])[1, 0]\n    return corrsum / len(y_true)\n\ndef zscore(x):\n    x_zscore = []\n    for i in range(x.shape[0]):\n        x_row = x[i]\n        x_row = (x_row - np.mean(x_row)) / np.std(x_row)\n        x_zscore.append(x_row)\n    x_std = np.array(x_zscore)    \n    return x_std\n\ndef cosine_similarity_loss(y_true, y_pred):\n    x = y_true\n    y = y_pred\n    mx = tf.reduce_mean(x, axis=1, keepdims=True)\n    my = tf.reduce_mean(y, axis=1, keepdims=True)\n    xm, ym = x - mx, y - my\n    t1_norm = tf.math.l2_normalize(xm, axis = 1)\n    t2_norm = tf.math.l2_normalize(ym, axis = 1)\n    cosine = tf.keras.losses.CosineSimilarity(axis = 1)(t1_norm, t2_norm)\n    return cosine\n\nclass DataGenerator(tf.keras.utils.Sequence):\n    'Generates data for Keras'\n    def __init__(self, train_X, train_y, list_IDs, shuffle, batch_size, labels, ): \n        self.train_X = train_X\n        self.train_y = train_y\n        self.list_IDs = list_IDs        \n        self.shuffle = shuffle\n        self.batch_size = batch_size\n        self.labels = labels\n        self.on_epoch_end()\n        \n    def __len__(self):\n        'Denotes the number of batches per epoch'\n        ct = len(self.list_IDs) // self.batch_size\n        return ct\n    \n    def __getitem__(self, idx):\n        'Generate one batch of data'\n        indexes = self.list_IDs[idx*self.batch_size:(idx+1)*self.batch_size]\n    \n        # Find list of IDs\n        list_IDs_temp = [self.list_IDs[k] for k in indexes]\n\n        # Generate data\n        X, y = self.__data_generation(list_IDs_temp)\n\n        if self.labels: return X, y\n        else: return X\n \n    def on_epoch_end(self):\n        'Updates indexes after each epoch'\n        self.indexes = np.arange( len(self.list_IDs) )\n        if self.shuffle: \n            np.random.shuffle(self.indexes)\n            \n    def __data_generation(self, list_IDs_temp):\n        'Generates data containing batch_size samples'    \n        X = self.train_X[list_IDs_temp]\n        y = self.train_y[list_IDs_temp]        \n        return X, y\n    \ndef nn_kfold(train_df, train_cite_X, train_cite_y, test_df, test_cite_X, network, folds, model_name):\n    oof_preds = np.zeros((train_df.shape[0],140))\n    sub_preds = np.zeros((test_df.shape[0],140))\n    cv_corr = []\n    for n_fold, (train_idx, valid_idx) in enumerate(folds.split(train_df,)):          \n        print (n_fold)\n        train_x = train_cite_X[train_idx]\n        valid_x = train_cite_X[valid_idx]\n        train_y = train_cite_y[train_idx]\n        valid_y = train_cite_y[valid_idx]\n\n        train_x_index = train_df.iloc[train_idx].reset_index(drop=True).index\n        valid_x_index = train_df.iloc[valid_idx].reset_index(drop=True).index\n        \n        model = network(train_cite_X.shape[1])\n        filepath = model_name+'_'+str(n_fold)+'.h5'\n        es = tf.keras.callbacks.EarlyStopping(patience=10, mode='min', verbose=1) \n        checkpoint = tf.keras.callbacks.ModelCheckpoint(monitor='val_loss', filepath=filepath, save_best_only=True,save_weights_only=True,mode='min') \n        reduce_lr_loss = tf.keras.callbacks.ReduceLROnPlateau(monitor='val_loss', factor=LR_FACTOR, patience=6, verbose=1)\n    \n        train_dataset = DataGenerator(\n            train_x,\n            train_y,\n            list_IDs=train_x_index, \n            shuffle=True, \n            batch_size=BATCH_SIZE, \n            labels=True,\n        )\n        \n        valid_dataset = DataGenerator(\n            valid_x,\n            valid_y,\n            list_IDs=valid_x_index, \n            shuffle=False, \n            batch_size=BATCH_SIZE, \n            labels=True,\n        )\n    \n        hist = model.fit(train_dataset,\n                        validation_data=valid_dataset,  \n                        epochs=EPOCHS, \n                        callbacks=[checkpoint,es,reduce_lr_loss],\n                        workers=4,\n                        verbose=1)  \n    \n        model.load_weights(filepath)\n        \n        oof_preds[valid_idx] = model.predict(valid_x, \n                                batch_size=BATCH_SIZE,\n                                verbose=1)\n        \n        oof_corr = correlation_score(valid_y,  oof_preds[valid_idx])\n        cv_corr.append(oof_corr)\n        print (cv_corr)       \n        \n        sub_preds += model.predict(test_cite_X, \n                                batch_size=BATCH_SIZE,\n                                verbose=1) / folds.n_splits \n            \n        del model\n        gc.collect()\n        tf.keras.backend.clear_session()    \n    cv = correlation_score(train_cite_y,  oof_preds)\n    print ('Overall:',cv)           \n    return oof_preds,sub_preds    ","metadata":{"execution":{"iopub.status.busy":"2022-11-16T06:39:46.948475Z","iopub.execute_input":"2022-11-16T06:39:46.949221Z","iopub.status.idle":"2022-11-16T06:39:52.993444Z","shell.execute_reply.started":"2022-11-16T06:39:46.949183Z","shell.execute_reply":"2022-11-16T06:39:52.992373Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model1 - cosine similarity loss","metadata":{}},{"cell_type":"code","source":"%%time\n\ndef cite_cos_sim_model(len_num):\n    \n    #######################  svd  #######################   \n    input_num = tf.keras.Input(shape=(len_num))     \n    x = input_num\n    x0 =  tf.keras.layers.Reshape((1,x.shape[1]))(x)\n    x0 = tf.keras.layers.Bidirectional(tf.keras.layers.GRU(1800, activation='elu', kernel_initializer='Identity',return_sequences=False))(x0)\n    x1 = tf.keras.layers.GaussianDropout(0.2)(x0)         \n    x2 = tf.keras.layers.Dense(1800,activation ='elu',kernel_initializer='Identity',)(x1) \n    x3 = tf.keras.layers.GaussianDropout(0.2)(x2) \n    x4 = tf.keras.layers.Dense(1800,activation ='elu',kernel_initializer='Identity',)(x3) \n    x5 = tf.keras.layers.GaussianDropout(0.2)(x4)         \n    x = tf.keras.layers.Concatenate()([\n                       x1,x3,x5\n                      ])\n    output = tf.keras.layers.Dense(140, activation='linear')(x) \n    model = tf.keras.models.Model(input_num, output)\n    lr=0.001\n    adam = tf.keras.optimizers.Adam(learning_rate=lr, beta_1=0.9, beta_2=0.999, epsilon=None, )\n    model.compile(loss=cosine_similarity_loss, optimizer=adam,)\n    model.summary()\n    return model\n\n\nBATCH_SIZE = 620\nEPOCHS = 100\nLR_FACTOR = 0.05\nSEED = 666\nN_FOLD = 5\nfolds = KFold(n_splits= N_FOLD, shuffle=True, random_state=SEED)     \noof_preds_cos,sub_preds_cos = nn_kfold(train_df, train_cite_X, train_cite_y,test_df, test_cite_X, cite_cos_sim_model, folds, 'cite_cos_model')","metadata":{"execution":{"iopub.status.busy":"2022-11-16T06:39:52.995901Z","iopub.execute_input":"2022-11-16T06:39:52.997000Z","iopub.status.idle":"2022-11-16T06:41:35.974330Z","shell.execute_reply.started":"2022-11-16T06:39:52.996939Z","shell.execute_reply":"2022-11-16T06:41:35.973183Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model2 - mse loss","metadata":{}},{"cell_type":"code","source":"%%time\n\ndef cite_mse_model(len_num):\n    \n    #######################  svd  #######################   \n    input_num = tf.keras.Input(shape=(len_num))     \n\n    x = input_num\n    x = tf.keras.layers.Dense(1500,activation ='swish',)(x)    \n    x = tf.keras.layers.GaussianDropout(0.1)(x)   \n    x = tf.keras.layers.Dense(1500,activation ='swish',)(x) \n    x = tf.keras.layers.GaussianDropout(0.1)(x)   \n    x = tf.keras.layers.Dense(1500,activation ='swish',)(x) \n    x = tf.keras.layers.GaussianDropout(0.1)(x)    \n    x =  tf.keras.layers.Reshape((1,x.shape[1]))(x)\n    x = tf.keras.layers.Bidirectional(tf.keras.layers.GRU(700, activation='swish',return_sequences=False))(x)\n    x = tf.keras.layers.GaussianDropout(0.1)(x)  \n    \n    output = tf.keras.layers.Dense(140, activation='linear')(x) \n\n    model = tf.keras.models.Model(input_num, output)\n    \n    lr=0.0005\n    weight_decay = 0.0001\n    \n    opt = tfa.optimizers.AdamW(\n        learning_rate=lr, weight_decay=weight_decay\n    )    \n\n    model.compile(loss=tf.keras.losses.MeanSquaredError(), optimizer=opt,)\n    model.summary()\n    return model\n\nBATCH_SIZE = 600\nEPOCHS = 100\nLR_FACTOR = 0.1\nSEED = 666\nfolds = KFold(n_splits= 5, shuffle=True, random_state=SEED)    \n\n# zscore for target\ntrain_cite_y = zscore(train_cite_y)\n\noof_preds_mse,sub_preds_mse = nn_kfold(train_df, train_cite_X, train_cite_y, test_df, test_cite_X, cite_mse_model, folds, 'cite_mse_model')","metadata":{"execution":{"iopub.status.busy":"2022-11-16T06:41:35.975914Z","iopub.execute_input":"2022-11-16T06:41:35.976537Z","iopub.status.idle":"2022-11-16T06:42:33.018262Z","shell.execute_reply.started":"2022-11-16T06:41:35.976497Z","shell.execute_reply":"2022-11-16T06:42:33.017024Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Weighted Average","metadata":{}},{"cell_type":"code","source":"%%time\noof_preds_cos = zscore(oof_preds_cos)\noof_preds_mse = zscore(oof_preds_mse)\noof_preds = oof_preds_cos*0.55 + oof_preds_mse*0.45\ncv = correlation_score(train_cite_y,  oof_preds)\nprint ('Blend:',cv)     \n\nsub_preds_cos = zscore(sub_preds_cos)\nsub_preds_mse = zscore(sub_preds_mse)\nsub_preds = sub_preds_cos*0.55 + sub_preds_mse*0.45\n","metadata":{"execution":{"iopub.status.busy":"2022-11-16T06:44:02.813374Z","iopub.execute_input":"2022-11-16T06:44:02.813785Z","iopub.status.idle":"2022-11-16T06:44:20.724525Z","shell.execute_reply.started":"2022-11-16T06:44:02.813745Z","shell.execute_reply":"2022-11-16T06:44:20.723212Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del train_df,test_df,train_cite_X,test_cite_X,train_cite_y\ndel oof_preds_cos,oof_preds_mse,sub_preds_cos,sub_preds_mse\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-11-16T06:44:28.176028Z","iopub.execute_input":"2022-11-16T06:44:28.176750Z","iopub.status.idle":"2022-11-16T06:44:28.493706Z","shell.execute_reply.started":"2022-11-16T06:44:28.176711Z","shell.execute_reply":"2022-11-16T06:44:28.492020Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Merge Multi part submission","metadata":{}},{"cell_type":"code","source":"%%time\nsub_preds = sub_preds.reshape(-1)\nmulti_preds = np.load(feature_path+'senkin_multi_predictions.npy')\n\nsub = pd.read_csv('../input/open-problems-multimodal/sample_submission.csv')\nsub['target'] = np.concatenate([sub_preds,multi_preds])\nsub.to_csv('submission.csv',index=False)","metadata":{"execution":{"iopub.status.busy":"2022-11-16T06:47:13.506268Z","iopub.execute_input":"2022-11-16T06:47:13.506634Z","iopub.status.idle":"2022-11-16T06:47:13.514048Z","shell.execute_reply.started":"2022-11-16T06:47:13.506602Z","shell.execute_reply":"2022-11-16T06:47:13.512823Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub","metadata":{"execution":{"iopub.status.busy":"2022-11-16T06:52:08.477903Z","iopub.execute_input":"2022-11-16T06:52:08.478358Z","iopub.status.idle":"2022-11-16T06:52:08.491587Z","shell.execute_reply.started":"2022-11-16T06:52:08.478303Z","shell.execute_reply":"2022-11-16T06:52:08.490395Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}