{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"Credit : https://www.kaggle.com/code/chengskin/dnn-model-ensemble-of-ubiquant and many more notebooks\n\nThe majority of the content of this notebook comes from the original author. Only a little change is don ein the model selection and weight contribution.\n\nIf you find this notebook useful please DO UPVOTE.\n","metadata":{}},{"cell_type":"code","source":"import os\nimport pandas as pd\nimport numpy as np\nimport gc\nimport tensorflow as tf\nfrom tensorflow.keras import layers\nfrom tensorflow import keras\nfrom keras.models import Sequential, Model\nfrom keras.layers import Input, Embedding, Dense, Flatten, Concatenate, Dot, Reshape, Add, Subtract\nfrom keras import regularizers \nfrom tensorflow.keras.optimizers import Adam\nfrom keras.callbacks import EarlyStopping, ModelCheckpoint\nfrom keras.regularizers import l2\nimport matplotlib.pyplot as plt\nfrom scipy import stats\nfrom tensorflow.keras.losses import Loss\nfrom sklearn.model_selection import TimeSeriesSplit, StratifiedKFold, KFold, GroupKFold\nfrom tqdm import tqdm\nfrom tensorflow.python.ops import math_ops\nfrom tensorflow.keras import backend as K","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-07-08T07:06:17.293597Z","iopub.execute_input":"2022-07-08T07:06:17.293979Z","iopub.status.idle":"2022-07-08T07:06:23.454802Z","shell.execute_reply.started":"2022-07-08T07:06:17.293888Z","shell.execute_reply":"2022-07-08T07:06:23.454044Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nn_features = 300\nfeatures = [f'f_{i}' for i in range(n_features)]\nfeature_columns = ['investment_id', 'time_id'] + features\ntrain = pd.read_pickle('../input/ubiquant-market-prediction-half-precision-pickle/train.pkl')\ntrain.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-08T07:09:09.967251Z","iopub.execute_input":"2022-07-08T07:09:09.968117Z","iopub.status.idle":"2022-07-08T07:09:11.405027Z","shell.execute_reply.started":"2022-07-08T07:09:09.968061Z","shell.execute_reply":"2022-07-08T07:09:11.404321Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"investment_id = train.pop(\"investment_id\")\ninvestment_id.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-08T07:09:14.023942Z","iopub.execute_input":"2022-07-08T07:09:14.024612Z","iopub.status.idle":"2022-07-08T07:09:14.039226Z","shell.execute_reply.started":"2022-07-08T07:09:14.024574Z","shell.execute_reply":"2022-07-08T07:09:14.038501Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"_ = train.pop(\"time_id\")\ny = train.pop(\"target\")\ny.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-08T07:09:16.053319Z","iopub.execute_input":"2022-07-08T07:09:16.054035Z","iopub.status.idle":"2022-07-08T07:09:16.075141Z","shell.execute_reply.started":"2022-07-08T07:09:16.053976Z","shell.execute_reply":"2022-07-08T07:09:16.074154Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ninvestment_ids = list(investment_id.unique())\ninvestment_id_size = len(investment_ids) + 1\ninvestment_id_lookup_layer = layers.IntegerLookup(max_tokens=investment_id_size)\nwith tf.device(\"cpu\"):\n    investment_id_lookup_layer.adapt(investment_id)","metadata":{"execution":{"iopub.status.busy":"2022-07-08T07:09:19.430528Z","iopub.execute_input":"2022-07-08T07:09:19.430790Z","iopub.status.idle":"2022-07-08T07:10:14.461310Z","shell.execute_reply.started":"2022-07-08T07:09:19.430760Z","shell.execute_reply":"2022-07-08T07:10:14.460500Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def preprocess(X, y):\n    print(X)\n    print(y)\n    return X, y\ndef make_dataset(feature, investment_id, y, batch_size=1024, mode=\"train\"):\n    ds = tf.data.Dataset.from_tensor_slices(((investment_id, feature), y))\n    ds = ds.map(preprocess)\n    if mode == \"train\":\n        ds = ds.shuffle(256)\n    ds = ds.batch(batch_size).cache().prefetch(tf.data.experimental.AUTOTUNE)\n    return ds\n\ndef make_dataset_2(feature, y, batch_size=1024, mode=\"train\"):\n    ds = tf.data.Dataset.from_tensor_slices((feature, y))\n    ds = ds.map(preprocess)\n    if mode == \"train\":\n        ds = ds.shuffle(256)\n    ds = ds.batch(batch_size).cache().prefetch(tf.data.experimental.AUTOTUNE)\n    return ds","metadata":{"execution":{"iopub.status.busy":"2022-07-08T07:10:14.463350Z","iopub.execute_input":"2022-07-08T07:10:14.463886Z","iopub.status.idle":"2022-07-08T07:10:14.471948Z","shell.execute_reply.started":"2022-07-08T07:10:14.463843Z","shell.execute_reply":"2022-07-08T07:10:14.471208Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_model():\n    investment_id_inputs = tf.keras.Input((1, ), dtype=tf.uint16)\n    features_inputs = tf.keras.Input((300, ), dtype=tf.float16)\n    \n    investment_id_x = investment_id_lookup_layer(investment_id_inputs)\n    investment_id_x = layers.Embedding(investment_id_size, 32, input_length=1)(investment_id_x)\n    investment_id_x = layers.Reshape((-1, ))(investment_id_x)\n    investment_id_x = layers.Dense(64, activation='swish')(investment_id_x)\n    investment_id_x = layers.Dense(64, activation='swish')(investment_id_x)\n    investment_id_x = layers.Dense(64, activation='swish')(investment_id_x)\n    \n    feature_x = layers.Dense(256, activation='swish')(features_inputs)\n    feature_x = layers.Dense(256, activation='swish')(feature_x)\n    feature_x = layers.Dense(256, activation='swish')(feature_x)\n    \n    x = layers.Concatenate(axis=1)([investment_id_x, feature_x])\n    x = layers.Dense(512, activation='swish', kernel_regularizer=\"l2\")(x)\n    x = layers.Dense(128, activation='swish', kernel_regularizer=\"l2\")(x)\n    x = layers.Dense(32, activation='swish', kernel_regularizer=\"l2\")(x)\n    output = layers.Dense(1)(x)\n    rmse = keras.metrics.RootMeanSquaredError(name=\"rmse\")\n    model = tf.keras.Model(inputs=[investment_id_inputs, features_inputs], outputs=[output])\n    model.compile(optimizer=tf.optimizers.Adam(0.001), loss='mse', metrics=['mse', \"mae\", \"mape\", rmse])\n    return model\n\n\ndef get_model2():\n    investment_id_inputs = tf.keras.Input((1, ), dtype=tf.uint16)\n    features_inputs = tf.keras.Input((300, ), dtype=tf.float16)\n    \n    investment_id_x = investment_id_lookup_layer(investment_id_inputs)\n    investment_id_x = layers.Embedding(investment_id_size, 32, input_length=1)(investment_id_x)\n    investment_id_x = layers.Reshape((-1, ))(investment_id_x)\n    investment_id_x = layers.Dense(64, activation='swish')(investment_id_x)    \n    investment_id_x = layers.Dense(64, activation='swish')(investment_id_x)\n    investment_id_x = layers.Dense(64, activation='swish')(investment_id_x)\n    investment_id_x = layers.Dense(64, activation='swish')(investment_id_x)\n   # investment_id_x = layers.Dropout(0.65)(investment_id_x)\n   \n    \n    feature_x = layers.Dense(256, activation='swish')(features_inputs)\n    feature_x = layers.Dense(256, activation='swish')(feature_x)\n    feature_x = layers.Dense(256, activation='swish')(feature_x)\n    feature_x = layers.Dense(256, activation='swish')(feature_x)\n    feature_x = layers.Dropout(0.65)(feature_x)\n    \n    x = layers.Concatenate(axis=1)([investment_id_x, feature_x])\n    x = layers.Dense(512, activation='swish', kernel_regularizer=\"l2\")(x)\n   # x = layers.Dropout(0.2)(x)\n    x = layers.Dense(128, activation='swish', kernel_regularizer=\"l2\")(x)\n  #  x = layers.Dropout(0.4)(x)\n    x = layers.Dense(32, activation='swish', kernel_regularizer=\"l2\")(x)\n    x = layers.Dense(32, activation='swish', kernel_regularizer=\"l2\")(x)\n    x = layers.Dropout(0.75)(x)\n    output = layers.Dense(1)(x)\n    rmse = keras.metrics.RootMeanSquaredError(name=\"rmse\")\n    model = tf.keras.Model(inputs=[investment_id_inputs, features_inputs], outputs=[output])\n    model.compile(optimizer=tf.optimizers.Adam(0.001), loss='mse', metrics=['mse', \"mae\", \"mape\", rmse])\n    return model\n\n\ndef get_model3():\n    investment_id_inputs = tf.keras.Input((1, ), dtype=tf.uint16)\n    features_inputs = tf.keras.Input((300, ), dtype=tf.float32)\n    \n    investment_id_x = investment_id_lookup_layer(investment_id_inputs)\n    investment_id_x = layers.Embedding(investment_id_size, 32, input_length=1)(investment_id_x)\n    investment_id_x = layers.Reshape((-1, ))(investment_id_x)\n    investment_id_x = layers.Dense(64, activation='swish')(investment_id_x)\n    investment_id_x = layers.Dropout(0.5)(investment_id_x)\n    investment_id_x = layers.Dense(32, activation='swish')(investment_id_x)\n    investment_id_x = layers.Dropout(0.5)(investment_id_x)\n    #investment_id_x = layers.Dense(64, activation='swish')(investment_id_x)\n    \n    feature_x = layers.Dense(256, activation='swish')(features_inputs)\n    feature_x = layers.Dropout(0.5)(feature_x)\n    feature_x = layers.Dense(128, activation='swish')(feature_x)\n    feature_x = layers.Dropout(0.5)(feature_x)\n    feature_x = layers.Dense(64, activation='swish')(feature_x)\n    \n    x = layers.Concatenate(axis=1)([investment_id_x, feature_x])\n    x = layers.Dropout(0.5)(x)\n    x = layers.Dense(64, activation='swish', kernel_regularizer=\"l2\")(x)\n    x = layers.Dropout(0.5)(x)\n    x = layers.Dense(32, activation='swish', kernel_regularizer=\"l2\")(x)\n    x = layers.Dropout(0.5)(x)\n    x = layers.Dense(16, activation='swish', kernel_regularizer=\"l2\")(x)\n    x = layers.Dropout(0.5)(x)\n    output = layers.Dense(1)(x)\n    output = tf.keras.layers.BatchNormalization(axis=1)(output)\n    rmse = keras.metrics.RootMeanSquaredError(name=\"rmse\")\n    model = tf.keras.Model(inputs=[investment_id_inputs, features_inputs], outputs=[output])\n    model.compile(optimizer=tf.optimizers.Adam(0.001), loss='mse', metrics=['mse', \"mae\", \"mape\", rmse])\n    return model\n\ndef get_model5():\n    features_inputs = tf.keras.Input((300, ), dtype=tf.float16)\n    \n    ## feature ##\n    feature_x = layers.Dense(256, activation='swish')(features_inputs)\n    feature_x = layers.Dropout(0.1)(feature_x)\n    ## convolution 1 ##\n    feature_x = layers.Reshape((-1,1))(feature_x)\n    feature_x = layers.Conv1D(filters=16, kernel_size=4, strides=1, padding='same')(feature_x)\n    feature_x = layers.BatchNormalization()(feature_x)\n    feature_x = layers.LeakyReLU()(feature_x)\n    ## convolution 2 ##\n    feature_x = layers.Conv1D(filters=16, kernel_size=4, strides=4, padding='same')(feature_x)\n    feature_x = layers.BatchNormalization()(feature_x)\n    feature_x = layers.LeakyReLU()(feature_x)\n    ## convolution 3 ##\n    feature_x = layers.Conv1D(filters=64, kernel_size=4, strides=1, padding='same')(feature_x)\n    feature_x = layers.BatchNormalization()(feature_x)\n    feature_x = layers.LeakyReLU()(feature_x)\n    ## convolution 4 ##\n    feature_x = layers.Conv1D(filters=64, kernel_size=4, strides=4, padding='same')(feature_x)\n    feature_x = layers.BatchNormalization()(feature_x)\n    feature_x = layers.LeakyReLU()(feature_x)\n    ## convolution 5 ##\n    feature_x = layers.Conv1D(filters=64, kernel_size=4, strides=2, padding='same')(feature_x)\n    feature_x = layers.BatchNormalization()(feature_x)\n    feature_x = layers.LeakyReLU()(feature_x)\n    ## flatten ##\n    feature_x = layers.Flatten()(feature_x)\n    \n    x = layers.Dense(512, activation='swish', kernel_regularizer=\"l2\")(feature_x)\n    \n    x = layers.Dropout(0.1)(x)\n    x = layers.Dense(128, activation='swish', kernel_regularizer=\"l2\")(x)\n    x = layers.Dropout(0.1)(x)\n    x = layers.Dense(32, activation='swish', kernel_regularizer=\"l2\")(x)\n    x = layers.Dropout(0.1)(x)\n    output = layers.Dense(1)(x)\n    rmse = keras.metrics.RootMeanSquaredError(name=\"rmse\")\n    model = tf.keras.Model(inputs=[features_inputs], outputs=[output])\n    model.compile(optimizer=tf.optimizers.Adam(0.001), loss='mse', metrics=['mse', \"mae\", \"mape\", rmse])\n    return model\ndel train\ndel investment_id\ndel y\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-07-08T07:10:14.473383Z","iopub.execute_input":"2022-07-08T07:10:14.473625Z","iopub.status.idle":"2022-07-08T07:10:14.687420Z","shell.execute_reply.started":"2022-07-08T07:10:14.473594Z","shell.execute_reply":"2022-07-08T07:10:14.686750Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"models = []\n\nfor i in range(5):\n    model = get_model()\n    model.load_weights(f'../input/dnn-base/model_{i}')\n    models.append(model)\n\nfor i in range(10):\n    model = get_model2()\n    model.load_weights(f'../input/train-dnn-v2-10fold/model_{i}')\n    models.append(model)\n    \n    \nfor i in range(10):\n    model = get_model3()\n    model.load_weights(f'../input/dnnmodelnew/model_{i}')\n    models.append(model)\n    \n    \nmodels2 = []\n    \nfor i in range(5):\n    model = get_model5()\n    model.load_weights(f'../input/prediction-including-spatial-info-with-conv1d/model_{i}.tf')\n    models2.append(model)","metadata":{"execution":{"iopub.status.busy":"2022-07-08T07:10:14.689185Z","iopub.execute_input":"2022-07-08T07:10:14.689426Z","iopub.status.idle":"2022-07-08T07:10:25.116420Z","shell.execute_reply.started":"2022-07-08T07:10:14.689394Z","shell.execute_reply":"2022-07-08T07:10:25.115675Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mod = get_model5()\nmod.summary()","metadata":{"execution":{"iopub.status.busy":"2022-07-08T07:39:11.248942Z","iopub.execute_input":"2022-07-08T07:39:11.249546Z","iopub.status.idle":"2022-07-08T07:39:11.430309Z","shell.execute_reply.started":"2022-07-08T07:39:11.249510Z","shell.execute_reply":"2022-07-08T07:39:11.429620Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def preprocess_test(investment_id, feature):\n    return (investment_id, feature), 0\n\ndef preprocess_test_s(feature):\n    return (feature), 0\n\ndef make_test_dataset(feature, investment_id, batch_size=1024):\n    ds = tf.data.Dataset.from_tensor_slices(((investment_id, feature)))\n    ds = ds.map(preprocess_test)\n    ds = ds.batch(batch_size).cache().prefetch(tf.data.experimental.AUTOTUNE)\n    return ds\n\ndef make_test_dataset2(feature, batch_size=1024):\n    ds = tf.data.Dataset.from_tensor_slices(((feature)))\n    ds = ds.batch(batch_size).cache().prefetch(tf.data.AUTOTUNE)\n    return ds\n\ndef inference(models, ds):\n    y_preds = []\n    for model in models:\n        y_pred = model.predict(ds)\n        y_preds.append(y_pred)\n    return np.mean(y_preds, axis=0)\n\ndef make_test_dataset3(feature, batch_size=1024):\n    ds = tf.data.Dataset.from_tensor_slices((feature))\n    ds = ds.map(preprocess_test_s)\n    ds = ds.batch(batch_size).cache().prefetch(tf.data.experimental.AUTOTUNE)\n    return ds\n\ndef infer(models, ds):\n    y_preds = []\n    for model in models:\n        y_pred = model.predict(ds)\n        y_preds.append((y_pred-y_pred.mean())/y_pred.std())\n    return np.mean(y_preds, axis=0)","metadata":{"execution":{"iopub.status.busy":"2022-07-08T07:39:15.597938Z","iopub.execute_input":"2022-07-08T07:39:15.598663Z","iopub.status.idle":"2022-07-08T07:39:15.609911Z","shell.execute_reply.started":"2022-07-08T07:39:15.598623Z","shell.execute_reply":"2022-07-08T07:39:15.609125Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import ubiquant\nenv = ubiquant.make_env()\niter_test = env.iter_test() \nfor (test_df, sample_prediction_df) in iter_test:\n    ds = make_test_dataset(test_df[features], test_df[\"investment_id\"])\n    p1 = inference(models, ds)\n    ds2 = make_test_dataset2(test_df[features])\n    p2 = inference(models2, ds2)\n    #ds3 = make_test_dataset3(test_df[features])\n    #p3 = infer(models3, ds3)\n    #p4 = inference(models_best, ds)\n    sample_prediction_df['target'] = p1 * 0.50 + p2 * 0.50 #+ p4 * 0.05  \n    env.predict(sample_prediction_df) \n    display(sample_prediction_df)","metadata":{"execution":{"iopub.status.busy":"2022-07-08T07:39:18.921373Z","iopub.execute_input":"2022-07-08T07:39:18.921620Z","iopub.status.idle":"2022-07-08T07:39:30.252664Z","shell.execute_reply.started":"2022-07-08T07:39:18.921592Z","shell.execute_reply":"2022-07-08T07:39:30.252027Z"},"trusted":true},"execution_count":null,"outputs":[]}]}