{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":59093,"databundleVersionId":7469972,"sourceType":"competition"},{"sourceId":159601602,"sourceType":"kernelVersion"}],"dockerImageVersionId":30635,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# developing","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-01-17T14:01:08.099204Z","iopub.execute_input":"2024-01-17T14:01:08.099497Z","iopub.status.idle":"2024-01-17T14:01:20.618983Z","shell.execute_reply.started":"2024-01-17T14:01:08.099469Z","shell.execute_reply":"2024-01-17T14:01:20.618060Z"}}},{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport os\nimport time\nimport numpy as np\nimport tensorflow as tf\nfrom sklearn.model_selection import KFold\nimport random\nimport joblib\nimport gc","metadata":{"execution":{"iopub.status.busy":"2024-01-20T13:55:49.836189Z","iopub.execute_input":"2024-01-20T13:55:49.836808Z","iopub.status.idle":"2024-01-20T13:55:53.738870Z","shell.execute_reply.started":"2024-01-20T13:55:49.836771Z","shell.execute_reply":"2024-01-20T13:55:53.737972Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# variables","metadata":{}},{"cell_type":"code","source":"output_size = 500\nbatch_size = 64\nN_Folds = 3","metadata":{"execution":{"iopub.status.busy":"2024-01-20T13:55:53.740557Z","iopub.execute_input":"2024-01-20T13:55:53.741124Z","iopub.status.idle":"2024-01-20T13:55:53.745323Z","shell.execute_reply.started":"2024-01-20T13:55:53.741091Z","shell.execute_reply":"2024-01-20T13:55:53.744365Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# confirm train_data ","metadata":{}},{"cell_type":"code","source":"X = np.nan_to_num(joblib.load(\"/kaggle/input/hms-train-dataset-create-baseline/train_matrix.pkl\"))\nX[1,:,:]","metadata":{"execution":{"iopub.status.busy":"2024-01-20T13:55:53.746309Z","iopub.execute_input":"2024-01-20T13:55:53.746583Z","iopub.status.idle":"2024-01-20T13:57:00.894288Z","shell.execute_reply.started":"2024-01-20T13:55:53.746557Z","shell.execute_reply":"2024-01-20T13:57:00.893046Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## starderize....","metadata":{}},{"cell_type":"markdown","source":"from sklearn.preprocessing import StandardScaler\nscalers = {}\nfor i in range(X.shape[2]):\n    scalers[i] = StandardScaler()\n    X[:, :, i] = scalers[i].fit_transform(X[:, :, i]) ","metadata":{"execution":{"iopub.status.busy":"2024-01-20T13:55:00.249859Z","iopub.status.idle":"2024-01-20T13:55:00.250241Z","shell.execute_reply.started":"2024-01-20T13:55:00.250068Z","shell.execute_reply":"2024-01-20T13:55:00.250085Z"}}},{"cell_type":"markdown","source":"X[1,:,:]","metadata":{"execution":{"iopub.status.busy":"2024-01-20T13:55:00.251818Z","iopub.status.idle":"2024-01-20T13:55:00.252359Z","shell.execute_reply.started":"2024-01-20T13:55:00.252079Z","shell.execute_reply":"2024-01-20T13:55:00.252108Z"}}},{"cell_type":"markdown","source":"#X = X[:5000,:,:]\nX.shape","metadata":{"execution":{"iopub.status.busy":"2024-01-20T13:55:00.253844Z","iopub.status.idle":"2024-01-20T13:55:00.254261Z","shell.execute_reply.started":"2024-01-20T13:55:00.254063Z","shell.execute_reply":"2024-01-20T13:55:00.254082Z"}}},{"cell_type":"code","source":"Y = joblib.load(\"/kaggle/input/hms-train-dataset-create-baseline/target.pkl\")\nY","metadata":{"execution":{"iopub.status.busy":"2024-01-20T13:57:00.897235Z","iopub.execute_input":"2024-01-20T13:57:00.897901Z","iopub.status.idle":"2024-01-20T13:57:00.916410Z","shell.execute_reply.started":"2024-01-20T13:57:00.897868Z","shell.execute_reply":"2024-01-20T13:57:00.915567Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Y = Y[:5000,:]\nY.shape","metadata":{"execution":{"iopub.status.busy":"2024-01-20T13:57:00.917344Z","iopub.execute_input":"2024-01-20T13:57:00.917594Z","iopub.status.idle":"2024-01-20T13:57:00.923410Z","shell.execute_reply.started":"2024-01-20T13:57:00.917570Z","shell.execute_reply":"2024-01-20T13:57:00.922473Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# model","metadata":{}},{"cell_type":"code","source":"def positional_encoding(length, depth):\n  depth = depth/2\n\n  positions = np.arange(length)[:, np.newaxis]     # (seq, 1)\n  depths = np.arange(depth)[np.newaxis, :]/depth   # (1, depth)\n\n  angle_rates = 1 / (10000**depths)         # (1, depth)\n  angle_rads = positions * angle_rates      # (pos, depth)\n\n  pos_encoding = np.concatenate(\n      [np.sin(angle_rads), np.cos(angle_rads)],\n      axis=-1) \n\n  return tf.cast(pos_encoding, dtype=tf.float32)\n                 \nclass PositionalEmbedding(tf.keras.layers.Layer):\n  def __init__(self, d_model):\n    super().__init__()\n    self.d_model = d_model \n    self.pos_encoding = positional_encoding(length=2048, depth=d_model)\n\n  def call(self, x):\n    length = tf.shape(x)[1]\n    # This factor sets the relative scale of the embedding and positonal_encoding.\n    #x *= tf.math.sqrt(tf.cast(self.d_model, tf.float32))\n    x = x + self.pos_encoding[tf.newaxis, :length, :]\n    return x","metadata":{"execution":{"iopub.status.busy":"2024-01-20T13:57:00.924450Z","iopub.execute_input":"2024-01-20T13:57:00.924735Z","iopub.status.idle":"2024-01-20T13:57:00.962167Z","shell.execute_reply.started":"2024-01-20T13:57:00.924703Z","shell.execute_reply":"2024-01-20T13:57:00.961314Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class BaseAttention(tf.keras.layers.Layer):\n  def __init__(self, **kwargs):\n    super().__init__()\n    self.mha = tf.keras.layers.MultiHeadAttention(**kwargs)\n    self.layernorm = tf.keras.layers.LayerNormalization()\n    self.add = tf.keras.layers.Add()","metadata":{"execution":{"iopub.status.busy":"2024-01-20T13:57:00.963462Z","iopub.execute_input":"2024-01-20T13:57:00.963756Z","iopub.status.idle":"2024-01-20T13:57:00.968981Z","shell.execute_reply.started":"2024-01-20T13:57:00.963730Z","shell.execute_reply":"2024-01-20T13:57:00.967920Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class GlobalSelfAttention(BaseAttention):\n  def call(self, x):\n    attn_output = self.mha(\n        query=x,\n        value=x,\n        key=x)\n    x = self.add([x, attn_output])\n    x = self.layernorm(x)\n    return x","metadata":{"execution":{"iopub.status.busy":"2024-01-20T13:57:00.970189Z","iopub.execute_input":"2024-01-20T13:57:00.970618Z","iopub.status.idle":"2024-01-20T13:57:00.980171Z","shell.execute_reply.started":"2024-01-20T13:57:00.970584Z","shell.execute_reply":"2024-01-20T13:57:00.979318Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class FeedForward(tf.keras.layers.Layer):\n  def __init__(self, d_model, dff, dropout_rate=0.1):\n    super().__init__()\n    self.seq = tf.keras.Sequential([\n      tf.keras.layers.Dense(dff, activation='relu'),\n      tf.keras.layers.Dense(d_model),\n      tf.keras.layers.Dropout(dropout_rate)\n    ])\n    self.add = tf.keras.layers.Add()\n    self.layer_norm = tf.keras.layers.LayerNormalization()\n\n  def call(self, x):\n    x = self.add([x, self.seq(x)])\n    x = self.layer_norm(x) \n    return x","metadata":{"execution":{"iopub.status.busy":"2024-01-20T13:57:00.981167Z","iopub.execute_input":"2024-01-20T13:57:00.981397Z","iopub.status.idle":"2024-01-20T13:57:00.990130Z","shell.execute_reply.started":"2024-01-20T13:57:00.981376Z","shell.execute_reply":"2024-01-20T13:57:00.989286Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class EncoderLayer(tf.keras.layers.Layer):\n  def __init__(self,*, d_model, num_heads, dff, dropout_rate=0.1):\n    super().__init__()\n\n    self.self_attention = GlobalSelfAttention(\n        num_heads=num_heads,\n        key_dim=d_model,\n        dropout=dropout_rate)\n\n    self.ffn = FeedForward(d_model, dff)\n\n  def call(self, x):\n    x = self.self_attention(x)\n    x = self.ffn(x)\n    return x","metadata":{"execution":{"iopub.status.busy":"2024-01-20T13:57:00.992694Z","iopub.execute_input":"2024-01-20T13:57:00.992983Z","iopub.status.idle":"2024-01-20T13:57:01.004365Z","shell.execute_reply.started":"2024-01-20T13:57:00.992958Z","shell.execute_reply":"2024-01-20T13:57:01.003495Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class RnnModel(tf.keras.Model):\n  def __init__(self, *, num_layers, d_model, num_heads,\n               dff, dropout_rate=0.1):\n    super().__init__()\n\n    self.d_model = d_model\n    self.num_layers = num_layers\n    self.pos_embedding = PositionalEmbedding(d_model=d_model)\n    \n    self.enc_layers = [\n        EncoderLayer(d_model=d_model,\n                     num_heads=num_heads,\n                     dff=dff,\n                     dropout_rate=dropout_rate)\n        for _ in range(num_layers)]\n    self.dropout = tf.keras.layers.Dropout(dropout_rate)\n    self.layer1_100 = tf.keras.layers.Dense(108, activation='relu')\n    self.layer1_6 = tf.keras.layers.Dense(6,activation='relu')\n    self.add = tf.keras.layers.Add()\n  def call(self, x):\n    #print(\"1:\",x.shape)\n    # `x` is token-IDs shape: (batch, seq_len)\n    x = self.pos_embedding(x)  # Shape `(batch_size, seq_len, d_model)`.\n    #print(\"2:\",x.shape)\n    # Add dropout.\n    x = self.dropout(x)\n    for i in range(self.num_layers):\n        x = self.enc_layers[i](x)\n    #print(\"3:\",x.shape)\n    x = tf.keras.layers.GlobalAveragePooling1D()(x)\n    #print(\"4:\",x.shape)\n    x = self.layer1_100(x)   \n    x = self.layer1_6(x) \n    x = tf.keras.layers.Softmax(-1)(x)\n    #print(\"5:\",x.shape)\n    return x","metadata":{"execution":{"iopub.status.busy":"2024-01-20T13:57:01.005293Z","iopub.execute_input":"2024-01-20T13:57:01.005603Z","iopub.status.idle":"2024-01-20T13:57:01.015981Z","shell.execute_reply.started":"2024-01-20T13:57:01.005577Z","shell.execute_reply":"2024-01-20T13:57:01.015123Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# train model","metadata":{}},{"cell_type":"code","source":"N_EPOCHS = 71\nsteps_per_epoch = X.shape[0]//batch_size\n#val_steps_per_epoch = 64","metadata":{"execution":{"iopub.status.busy":"2024-01-20T13:57:01.016936Z","iopub.execute_input":"2024-01-20T13:57:01.017243Z","iopub.status.idle":"2024-01-20T13:57:01.029760Z","shell.execute_reply.started":"2024-01-20T13:57:01.017218Z","shell.execute_reply":"2024-01-20T13:57:01.028745Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def loss_fn(labels, targets):\n    loss = tf.math.abs(labels - targets)\n    #loss = tf.math.reduce_mean(loss,1)\n    #loss = tf.math.reduce_mean(loss,0)\n    loss = tf.math.reduce_mean(loss)\n    return loss\nsave_folder = '/kaggle/working'\ndef create_save_callback(fold):\n    try:\n        os.mkdir(f'{save_folder}/{fold}_weights/')\n    except:\n        pass\n    class save_model_callback(tf.keras.callbacks.Callback):\n        def __init__(self,fold):\n            super().__init__()\n            self.fold = fold\n        def on_epoch_end(self, epoch: int, logs=None):\n            if epoch == 5 or (epoch)%10 == 0:\n                self.model.save_weights(f\"{save_folder}/{self.fold}_weights/model_epoch_{epoch}.h5\")\n    return save_model_callback(fold)","metadata":{"execution":{"iopub.status.busy":"2024-01-20T13:57:01.030858Z","iopub.execute_input":"2024-01-20T13:57:01.031142Z","iopub.status.idle":"2024-01-20T13:57:01.041420Z","shell.execute_reply.started":"2024-01-20T13:57:01.031117Z","shell.execute_reply":"2024-01-20T13:57:01.040367Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"try:\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver.connect(tpu=\"local\") # \"local\" for 1VM TPU\n    print('Running on TPU ')#, tpu.cluster_spec().as_dict()['worker'])\nexcept:\n    tpu = None\nif tpu:\n    strategy = tf.distribute.TPUStrategy(tpu)\n    print(\"on TPU\")\n    print(\"REPLICAS: \", strategy.num_replicas_in_sync)\nelse:\n    print(\"on GPU\")\n    strategy = tf.distribute.get_strategy()\nwith strategy.scope():\n    kf = KFold(n_splits=N_Folds, shuffle=True, random_state=100)\n    for fold, (train_idx, valid_idx) in enumerate(kf.split(X, Y)):\n        print(fold,\"/\",N_Folds , \"###################\")\n        X_train, X_valid = X[train_idx,:], X[valid_idx,:]\n        y_train, y_valid = Y[train_idx], Y[valid_idx]\n        \n        learning_rate = 1e-4\n        epsilon = 1e-15\n        loss = loss_fn #tf.keras.losses.MeanSquaredError()\n\n        optimizer = tf.keras.optimizers.Adam(learning_rate=learning_rate, epsilon=epsilon)\n        model = RnnModel(num_layers=3,d_model=120,num_heads=1,dff=200,dropout_rate=0.5)\n        model.compile(optimizer=optimizer, loss=loss)\n        callback1 = tf.keras.callbacks.EarlyStopping(monitor='loss', patience=5)\n        model.fit(  x=X_train, y=y_train,\n                    validation_data=(X_valid, y_valid),\n                    epochs=N_EPOCHS,\n                    steps_per_epoch = steps_per_epoch,\n                    #validation_steps=val_steps_per_epoch,\n                    verbose = 2,\n                    callbacks=[callback1,create_save_callback(fold)]\n                 )\n        del model\n        gc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-01-20T13:57:01.042592Z","iopub.execute_input":"2024-01-20T13:57:01.042912Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"joblib.dump(scalers,\"transformer.pkl\")","metadata":{}}]}