{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Train 1DCNN on TPU\n\nWhen I look sharing code, many people use LightGBM.\nI want to try NN approach because I like it.\n\nTo save time and memory, I converted train.csv to a numpy array beforehand. ([dataset link](https://www.kaggle.com/takamichitoda/ump-npy-dataset))\n\nThis dataset made from [this notebook](https://www.kaggle.com/takamichitoda/ump-train-csv-to-npy).  \n\nThe model architecture was based on Mr. @sishihara's notebook.  \nhttps://www.kaggle.com/sishihara/1dcnn-for-tabular-from-moa-2nd-place\n\nThanks:)\n\n\n\n`update`\n- Version 4: baseline, CV=0.9105 / LB=0.135\n- Version 6: add dropout, CV=0.9101 / LB=0.132\n- Version 8: dropout ratio 0.2 -> 0.1, CV=0.9135 / LB=0.117\n- Version 9: dropout ratio 0.1 -> 0.4, CV=0.9142 / LB=0.125\n- Version 11: remove dropout & [add lag feature](https://www.kaggle.com/takamichitoda/ump-lag-freatures)\n- Version 15: remove lag feature & use small batch, StratifiedKFold, ReduceLROnPlateau\n- Version 16: batch=4096, use correlationLoss\n- Version 17: use MSE loss, small model(param 1/4)\n- Version 20: TimeSeriesSplit\n- Version 21: MC Dropout(0.75), large model\n- Version 22: MC Dropout(0.75), small model, correlationLoss\n- Version 22: skip connect model","metadata":{}},{"cell_type":"code","source":"import os\nimport gc\nimport pandas as pd\nimport numpy as np\nimport random\nfrom matplotlib import pyplot as plt\n\nfrom sklearn.metrics import mean_squared_error\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.model_selection import TimeSeriesSplit\n\nimport tensorflow as tf\nfrom tensorflow import keras\nfrom tensorflow.keras import layers\nfrom tensorflow.keras.callbacks import ReduceLROnPlateau\nfrom tensorflow.python.ops import math_ops\nfrom tensorflow.python.keras import backend as K\n\n\ntpu = tf.distribute.cluster_resolver.TPUClusterResolver()\ntf.config.experimental_connect_to_cluster(tpu)\ntf.tpu.experimental.initialize_tpu_system(tpu)\nstrategy = tf.distribute.experimental.TPUStrategy(tpu)\n\nprint('Running on TPU ', tpu.master())\nprint(\"REPLICAS: \", strategy.num_replicas_in_sync)","metadata":{"execution":{"iopub.status.busy":"2022-01-29T12:37:36.922805Z","iopub.execute_input":"2022-01-29T12:37:36.923560Z","iopub.status.idle":"2022-01-29T12:37:48.822691Z","shell.execute_reply.started":"2022-01-29T12:37:36.923472Z","shell.execute_reply":"2022-01-29T12:37:48.822073Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class GCF:\n    INPUT_ROOT = \"/kaggle/input/ump-npy-dataset/\"\n    LAG_FEATURES = \"/kaggle/input/ump-lag-freatures/target_shift_1.npy\"\n    #TIME_ID_LIMIT = 500\n    N_TRAIN = 1_500_000\n    N_FOLDS = 5\n    SEED = 0\n    \n    N_EPOCHS = 800\n    BATCH_SIZE = 4096\n    EARLY_STOPPING_PATIENCE = 10\n    EARLY_STOPPING_MIN_DELTA = 1e-3","metadata":{"execution":{"iopub.status.busy":"2022-01-29T12:37:48.830508Z","iopub.execute_input":"2022-01-29T12:37:48.830730Z","iopub.status.idle":"2022-01-29T12:37:48.842113Z","shell.execute_reply.started":"2022-01-29T12:37:48.830702Z","shell.execute_reply":"2022-01-29T12:37:48.841360Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def seed_everything(seed=GCF.SEED):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    tf.random.set_seed(seed)","metadata":{"execution":{"iopub.status.busy":"2022-01-29T12:37:50.260499Z","iopub.execute_input":"2022-01-29T12:37:50.261004Z","iopub.status.idle":"2022-01-29T12:37:50.266465Z","shell.execute_reply.started":"2022-01-29T12:37:50.260961Z","shell.execute_reply":"2022-01-29T12:37:50.265105Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\nX = np.load(f\"{GCF.INPUT_ROOT}/features_std_scaled.npy\")\ny = np.load(f\"{GCF.INPUT_ROOT}/targets.npy\")\ntime_id = np.load(f\"{GCF.INPUT_ROOT}/time_id.npy\")\n#investment_id = np.load(f\"{GCF.INPUT_ROOT}/investment_id.npy\")\n\n# Use only newer data to save memory.\n#X = X[time_id > GCF.TIME_ID_LIMIT, :]\n#y = y[time_id > GCF.TIME_ID_LIMIT]\n#investment_id = investment_id[time_id > GCF.TIME_ID_LIMIT]\n#time_id = time_id[time_id > GCF.TIME_ID_LIMIT]\n#gc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-01-29T12:37:51.426754Z","iopub.execute_input":"2022-01-29T12:37:51.427088Z","iopub.status.idle":"2022-01-29T12:38:24.978694Z","shell.execute_reply.started":"2022-01-29T12:37:51.427056Z","shell.execute_reply":"2022-01-29T12:38:24.977769Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# https://www.kaggle.com/c/ubiquant-market-prediction/discussion/302977\n\ndef correlationMetric(x, y, axis=-2):\n    \"\"\"Metric returning the Pearson correlation coefficient of two tensors over some axis, default -2.\"\"\"\n    x = tf.convert_to_tensor(x)\n    y = math_ops.cast(y, x.dtype)\n    n = tf.cast(tf.shape(x)[axis], x.dtype)\n    xsum = tf.reduce_sum(x, axis=axis)\n    ysum = tf.reduce_sum(y, axis=axis)\n    xmean = xsum / n\n    ymean = ysum / n\n    xvar = tf.reduce_sum( tf.math.squared_difference(x, xmean), axis=axis)\n    yvar = tf.reduce_sum( tf.math.squared_difference(y, ymean), axis=axis)\n    cov = tf.reduce_sum( (x - xmean) * (y - ymean), axis=axis)\n    corr = cov / tf.sqrt(xvar * yvar)\n    return tf.constant(1.0, dtype=x.dtype) - corr\n\ndef correlationLoss(x,y, axis=-2):\n    \"\"\"Loss function that maximizes the pearson correlation coefficient between the predicted values and the labels,\n    while trying to have the same mean and variance\"\"\"\n    x = tf.convert_to_tensor(x)\n    y = math_ops.cast(y, x.dtype)\n    n = tf.cast(tf.shape(x)[axis], x.dtype)\n    xsum = tf.reduce_sum(x, axis=axis)\n    ysum = tf.reduce_sum(y, axis=axis)\n    xmean = xsum / n\n    ymean = ysum / n\n    xsqsum = tf.reduce_sum( tf.math.squared_difference(x, xmean), axis=axis)\n    ysqsum = tf.reduce_sum( tf.math.squared_difference(y, ymean), axis=axis)\n    cov = tf.reduce_sum( (x - xmean) * (y - ymean), axis=axis)\n    corr = cov / tf.sqrt(xsqsum * ysqsum)\n    sqdif = tf.reduce_sum(tf.math.squared_difference(x, y), axis=axis) / n / tf.sqrt(ysqsum / n)\n    return tf.convert_to_tensor( K.mean(tf.constant(1.0, dtype=x.dtype) - corr + (0.01 * sqdif)) , dtype=tf.float32 )\n\n\n#　https://www.kaggle.com/c/ubiquant-market-prediction/discussion/301987\ndef pearson_coef(data):\n    return data.corr()['target']['preds']\n\ndef comp_metric(time_id, y, pred):\n    return np.mean(\n        pd.DataFrame(np.stack([time_id, y, pred]).T, columns=['time_id', 'target', 'preds']\n    ).groupby('time_id').apply(pearson_coef))","metadata":{"execution":{"iopub.status.busy":"2022-01-29T12:38:24.980629Z","iopub.execute_input":"2022-01-29T12:38:24.980851Z","iopub.status.idle":"2022-01-29T12:38:24.996972Z","shell.execute_reply.started":"2022-01-29T12:38:24.980825Z","shell.execute_reply":"2022-01-29T12:38:24.996219Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# https://www.kaggle.com/sishihara/1dcnn-for-tabular-from-moa-2nd-place\ndef create_model():\n     model = keras.Sequential([\n         layers.Dense(4096//4, activation='relu', input_shape=(300,)),\n         layers.Reshape((256//4, 16)),\n         layers.Dropout(0.75),\n         layers.Conv1D(filters=16, kernel_size=5, strides=1, activation='relu'),\n         layers.MaxPooling1D(pool_size=2),\n         layers.Flatten(),\n         layers.Dense(16, activation='relu'),\n         layers.Dense(1, activation='linear'),\n     ])\n    \nfeatures_inputs = keras.Input(shape=(300,))\n    \nx = layers.BatchNormalization()(features_inputs)\nx = layers.Dropout(0.1)(x)\nx = layers.Dense(1024, activation='swish')(x)\nx = layers.Reshape((64, 16))(x)\n    \nx = layers.BatchNormalization()(x)\nx = layers.Dropout(0.1)(x)\nx = layers.Conv1D(filters=64, kernel_size=5, strides=1, padding=\"same\", activation='swish')(x)\n\nx = layers.AveragePooling1D(pool_size=2)(x)\n    \nx = layers.BatchNormalization()(x)\nx = layers.Dropout(0.1)(x)\nx = layers.Conv1D(filters=64, kernel_size=5, strides=1, padding=\"same\", activation='swish')(x)\n    \nx2 = layers.BatchNormalization()(x)\nx2 = layers.Dropout(0.2)(x2)\nx2 = layers.Conv1D(filters=64, kernel_size=3, strides=1, padding=\"same\", activation='swish')(x2)\n    \nx2 = layers.BatchNormalization()(x2)\nx2 = layers.Dropout(0.3)(x2)\nx2 = layers.Conv1D(filters=64, kernel_size=3, strides=1, padding=\"same\", activation='swish')(x2)\n    \nx = layers.Multiply()([x, x2])\n    \nx = layers.MaxPooling1D(pool_size=4)(x)\n    \nx = layers.Flatten()(x)\n    \nx = layers.BatchNormalization()(x)\nx = layers.Dropout(0.2)(x)\nx = layers.Dense(16, activation='swish')(x)\n    \noutputs = layers.Dense(1, activation='linear')(x)\n\nmodel = keras.Model(inputs=features_inputs, outputs=outputs)\n    \nmodel.compile(\n    optimizer=tf.optimizers.Adam(1e-4),\n    loss='mse',\n    metrics=[keras.metrics.RootMeanSquaredError(), correlationMetric]\n    )\n    \nmodel","metadata":{"execution":{"iopub.status.busy":"2022-01-29T12:42:47.683395Z","iopub.execute_input":"2022-01-29T12:42:47.684252Z","iopub.status.idle":"2022-01-29T12:42:48.006908Z","shell.execute_reply.started":"2022-01-29T12:42:47.684197Z","shell.execute_reply":"2022-01-29T12:42:48.005972Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"seed_everything()\n\n#kf = StratifiedKFold(5, shuffle=True, random_state=GCF.SEED)\nkf = TimeSeriesSplit(n_splits=GCF.N_FOLDS, max_train_size=GCF.N_TRAIN)\n\nrmse_lst, score_lst = [], []\n#oof = np.zeros((len(y),))\n#for fold, (train_idx, valid_idx) in enumerate(kf.split(X, investment_id)):\nfor fold, (train_idx, valid_idx) in enumerate(kf.split(X)):\n    with strategy.scope():\n        model = create_model()\n\n    early_stopping = keras.callbacks.EarlyStopping(\n        patience=GCF.EARLY_STOPPING_PATIENCE,\n        min_delta=GCF.EARLY_STOPPING_MIN_DELTA,\n        restore_best_weights=True,\n    )\n    reduce_lr = ReduceLROnPlateau(\n                        monitor='val_loss',\n                        factor=0.5,\n                        patience=3,\n                        min_lr=1e-5,\n                        verbose=1\n    )\n\n    history = model.fit(\n        X[train_idx, :], y[train_idx],\n        validation_data=(X[valid_idx, :], y[valid_idx]),\n        batch_size=GCF.BATCH_SIZE,\n        epochs=GCF.N_EPOCHS,\n        callbacks=[early_stopping, reduce_lr],\n    )\n    \n    #oof[valid_idx] = model.predict(X[valid_idx, :]).reshape(1, -1)[0]\n    valid_pred = model.predict(X[valid_idx, :]).reshape(1, -1)[0]\n    \n    rmse = mean_squared_error(y[valid_idx], valid_pred, squared=False)\n    score = comp_metric(time_id[valid_idx], y[valid_idx], valid_pred)\n    print(f'Fold-{fold}: RMSR={rmse}, SCORE={score}')\n    \n    pd.DataFrame(history.history)[['loss', 'val_loss']].plot()\n    plt.title(\"loss\")\n    plt.show()\n    \n    pd.DataFrame(history.history)[['root_mean_squared_error', 'val_root_mean_squared_error']].plot()\n    plt.title(\"rmse\")\n    plt.show()\n    \n    pd.DataFrame(history.history)[['correlationMetric', 'val_correlationMetric']].plot()\n    plt.title(\"correlation\")\n    plt.show()\n    \n    model.save(f\"ump_1dcnn_f{fold}.h5\")\n    rmse_lst.append(rmse)\n    score_lst.append(score)","metadata":{"execution":{"iopub.status.busy":"2022-01-29T12:42:49.220369Z","iopub.execute_input":"2022-01-29T12:42:49.220633Z","iopub.status.idle":"2022-01-29T12:42:49.621669Z","shell.execute_reply.started":"2022-01-29T12:42:49.220607Z","shell.execute_reply":"2022-01-29T12:42:49.620372Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(rmse_lst, np.mean(rmse_lst))\nprint(score_lst, np.mean(score_lst))","metadata":{"execution":{"iopub.status.busy":"2022-01-28T03:00:42.99837Z","iopub.status.idle":"2022-01-28T03:00:42.998707Z","shell.execute_reply.started":"2022-01-28T03:00:42.99854Z","shell.execute_reply":"2022-01-28T03:00:42.998556Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!ls","metadata":{"execution":{"iopub.status.busy":"2022-01-28T03:00:43.000063Z","iopub.status.idle":"2022-01-28T03:00:43.000433Z","shell.execute_reply.started":"2022-01-28T03:00:43.000251Z","shell.execute_reply":"2022-01-28T03:00:43.000268Z"},"trusted":true},"execution_count":null,"outputs":[]}]}