{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":59093,"databundleVersionId":7469972,"sourceType":"competition"},{"sourceId":162586676,"sourceType":"kernelVersion"}],"dockerImageVersionId":30636,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# developing","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-01-17T14:01:08.099204Z","iopub.execute_input":"2024-01-17T14:01:08.099497Z","iopub.status.idle":"2024-01-17T14:01:20.618983Z","shell.execute_reply.started":"2024-01-17T14:01:08.099469Z","shell.execute_reply":"2024-01-17T14:01:20.61806Z"}}},{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport os\nimport time\nimport numpy as np\nimport tensorflow as tf\nfrom sklearn.model_selection import KFold\nimport random\nimport joblib\nimport gc\nfrom tqdm import tqdm","metadata":{"execution":{"iopub.status.busy":"2024-02-12T11:56:23.892164Z","iopub.execute_input":"2024-02-12T11:56:23.892763Z","iopub.status.idle":"2024-02-12T11:56:36.419981Z","shell.execute_reply.started":"2024-02-12T11:56:23.892730Z","shell.execute_reply":"2024-02-12T11:56:36.419136Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# variables","metadata":{}},{"cell_type":"code","source":"target_cols =['seizure_vote', 'lpd_vote','gpd_vote', 'lrda_vote', 'grda_vote', 'other_vote']\ntest_path = \"/kaggle/input/hms-harmful-brain-activity-classification/test.csv\"\ntest_eegs = \"/kaggle/input/hms-harmful-brain-activity-classification/test_eegs/\"\ntest_spectrograms = \"/kaggle/input/hms-harmful-brain-activity-classification/test_spectrograms/\"","metadata":{"execution":{"iopub.status.busy":"2024-02-12T11:56:36.421922Z","iopub.execute_input":"2024-02-12T11:56:36.422921Z","iopub.status.idle":"2024-02-12T11:56:36.427875Z","shell.execute_reply.started":"2024-02-12T11:56:36.422877Z","shell.execute_reply":"2024-02-12T11:56:36.427060Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# confirm train_data ","metadata":{}},{"cell_type":"code","source":"ds_test = pd.read_csv(test_path)\nds_test.head(100)","metadata":{"execution":{"iopub.status.busy":"2024-02-12T11:56:36.428903Z","iopub.execute_input":"2024-02-12T11:56:36.429167Z","iopub.status.idle":"2024-02-12T11:56:36.477588Z","shell.execute_reply.started":"2024-02-12T11:56:36.429143Z","shell.execute_reply":"2024-02-12T11:56:36.476703Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# model","metadata":{}},{"cell_type":"code","source":"def positional_encoding(length, depth):\n  depth = depth/2\n\n  positions = np.arange(length)[:, np.newaxis]     # (seq, 1)\n  depths = np.arange(depth)[np.newaxis, :]/depth   # (1, depth)\n\n  angle_rates = 1 / (10000**depths)         # (1, depth)\n  angle_rads = positions * angle_rates      # (pos, depth)\n\n  pos_encoding = np.concatenate(\n      [np.sin(angle_rads), np.cos(angle_rads)],\n      axis=-1) \n\n  return tf.cast(pos_encoding, dtype=tf.float32)\n                 \nclass PositionalEmbedding(tf.keras.layers.Layer):\n  def __init__(self, d_model):\n    super().__init__()\n    self.d_model = d_model \n    self.pos_encoding = positional_encoding(length=2048, depth=d_model)\n\n  def call(self, x):\n    length = tf.shape(x)[1]\n    # This factor sets the relative scale of the embedding and positonal_encoding.\n    #x *= tf.math.sqrt(tf.cast(self.d_model, tf.float32))\n    x = x + self.pos_encoding[tf.newaxis, :length, :]\n    return x","metadata":{"execution":{"iopub.status.busy":"2024-02-12T11:56:36.479550Z","iopub.execute_input":"2024-02-12T11:56:36.479854Z","iopub.status.idle":"2024-02-12T11:56:36.583840Z","shell.execute_reply.started":"2024-02-12T11:56:36.479827Z","shell.execute_reply":"2024-02-12T11:56:36.583008Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class BaseAttention(tf.keras.layers.Layer):\n  def __init__(self, **kwargs):\n    super().__init__()\n    self.mha = tf.keras.layers.MultiHeadAttention(**kwargs)\n    self.layernorm = tf.keras.layers.LayerNormalization()\n    self.add = tf.keras.layers.Add()","metadata":{"execution":{"iopub.status.busy":"2024-02-12T11:56:36.584885Z","iopub.execute_input":"2024-02-12T11:56:36.585146Z","iopub.status.idle":"2024-02-12T11:56:36.590728Z","shell.execute_reply.started":"2024-02-12T11:56:36.585123Z","shell.execute_reply":"2024-02-12T11:56:36.589702Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class GlobalSelfAttention(BaseAttention):\n  def call(self, x):\n    attn_output = self.mha(\n        query=x,\n        value=x,\n        key=x)\n    x = self.add([x, attn_output])\n    x = self.layernorm(x)\n    return x","metadata":{"execution":{"iopub.status.busy":"2024-02-12T11:56:36.592093Z","iopub.execute_input":"2024-02-12T11:56:36.592429Z","iopub.status.idle":"2024-02-12T11:56:36.604557Z","shell.execute_reply.started":"2024-02-12T11:56:36.592404Z","shell.execute_reply":"2024-02-12T11:56:36.603684Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class FeedForward(tf.keras.layers.Layer):\n  def __init__(self, d_model, dff, dropout_rate=0.1):\n    super().__init__()\n    self.seq = tf.keras.Sequential([\n      tf.keras.layers.Dense(dff, activation='relu'),\n      tf.keras.layers.Dense(d_model),\n      tf.keras.layers.Dropout(dropout_rate)\n    ])\n    self.add = tf.keras.layers.Add()\n    self.layer_norm = tf.keras.layers.LayerNormalization()\n\n  def call(self, x):\n    x = self.add([x, self.seq(x)])\n    x = self.layer_norm(x) \n    return x","metadata":{"execution":{"iopub.status.busy":"2024-02-12T11:56:36.605647Z","iopub.execute_input":"2024-02-12T11:56:36.606180Z","iopub.status.idle":"2024-02-12T11:56:36.617284Z","shell.execute_reply.started":"2024-02-12T11:56:36.606155Z","shell.execute_reply":"2024-02-12T11:56:36.616546Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class EncoderLayer(tf.keras.layers.Layer):\n  def __init__(self,*, d_model, num_heads, dff, dropout_rate=0.1):\n    super().__init__()\n\n    self.self_attention = GlobalSelfAttention(\n        num_heads=num_heads,\n        key_dim=d_model,\n        dropout=dropout_rate)\n\n    self.ffn = FeedForward(d_model, dff)\n\n  def call(self, x):\n    x = self.self_attention(x)\n    x = self.ffn(x)\n    return x","metadata":{"execution":{"iopub.status.busy":"2024-02-12T11:56:36.618240Z","iopub.execute_input":"2024-02-12T11:56:36.618505Z","iopub.status.idle":"2024-02-12T11:56:36.628170Z","shell.execute_reply.started":"2024-02-12T11:56:36.618481Z","shell.execute_reply":"2024-02-12T11:56:36.627314Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class RnnModel(tf.keras.Model):\n  def __init__(self, *, num_layers, d_model, num_heads,\n               dff, dropout_rate=0.1):\n    super().__init__()\n\n    self.d_model = d_model\n    self.num_layers = num_layers\n    #self.cov11 = tf.keras.layers.Conv1D(20, 1, activation='relu')\n    #self.cov21 = tf.keras.layers.Conv1D(200, 1, activation='relu')\n    #self.cov22 = tf.keras.layers.Conv1D(100, 1, activation='relu')\n    \n    self.pos_embedding = PositionalEmbedding(d_model=d_model)\n    \n    self.enc_layers = [\n        EncoderLayer(d_model=d_model,\n                     num_heads=num_heads,\n                     dff=dff,\n                     dropout_rate=dropout_rate)\n        for _ in range(num_layers)]\n    self.dropout = tf.keras.layers.Dropout(dropout_rate)\n    self.layer1_100 = tf.keras.layers.Dense(108, activation='relu')\n    self.layer1_6 = tf.keras.layers.Dense(6,activation='relu')\n    self.add = tf.keras.layers.Add()\n  def call(self, X):\n    x1 = X['x1']\n    x2 = X['x2']\n    #print(\"x1\",x1.shape)\n    #print(\"x2\",x2.shape)\n    #x1 = self.cov11(x1)\n    #x2 = self.cov21(x2)\n    #x2 = self.cov22(x2)\n    #print(\"x1\",x1.shape)\n    #print(\"x2\",x2.shape)\n    \n    x = tf.concat([x1,x2],2)\n    x = self.pos_embedding(x)  # Shape `(batch_size, seq_len, d_model)`.\n    # Add dropout.\n    x = self.dropout(x)\n    for i in range(self.num_layers):\n        x = self.enc_layers[i](x)\n    x = tf.keras.layers.GlobalAveragePooling1D()(x)\n    x = self.layer1_100(x)   \n    x = self.layer1_6(x) \n    x = tf.keras.layers.Softmax(-1)(x)\n    return x","metadata":{"execution":{"iopub.status.busy":"2024-02-12T11:56:36.629404Z","iopub.execute_input":"2024-02-12T11:56:36.629751Z","iopub.status.idle":"2024-02-12T11:56:36.815675Z","shell.execute_reply.started":"2024-02-12T11:56:36.629716Z","shell.execute_reply":"2024-02-12T11:56:36.814702Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# prediction","metadata":{}},{"cell_type":"code","source":"def loss_fn(labels, targets):\n    loss = tf.math.abs(labels - targets)\n    #loss = tf.math.reduce_mean(loss,1)\n    #loss = tf.math.reduce_mean(loss,0)\n    loss = tf.math.reduce_mean(loss)\n    return loss","metadata":{"execution":{"iopub.status.busy":"2024-02-12T11:56:36.819448Z","iopub.execute_input":"2024-02-12T11:56:36.819775Z","iopub.status.idle":"2024-02-12T11:56:36.830304Z","shell.execute_reply.started":"2024-02-12T11:56:36.819749Z","shell.execute_reply":"2024-02-12T11:56:36.829368Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nmodels_infor =[[0,100],[0,90],[0,80],[0,70],[0,60],[0,50],\n               #[1,78],\n               #[2,78],\n               #[3,78],\n               #[4,78],[4,74],\n               #[5,78],[5,72],\n               #[6,78]\n              ]","metadata":{"execution":{"iopub.status.busy":"2024-02-12T11:56:36.831488Z","iopub.execute_input":"2024-02-12T11:56:36.832142Z","iopub.status.idle":"2024-02-12T11:56:36.841463Z","shell.execute_reply.started":"2024-02-12T11:56:36.832107Z","shell.execute_reply":"2024-02-12T11:56:36.840559Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"try:\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver.connect(tpu=\"local\") # \"local\" for 1VM TPU\n    print('Running on TPU ')#, tpu.cluster_spec().as_dict()['worker'])\nexcept:\n    tpu = None\nif tpu:\n    strategy = tf.distribute.TPUStrategy(tpu)\n    print(\"on TPU\")\n    print(\"REPLICAS: \", strategy.num_replicas_in_sync)\nelse:\n    print(\"on GPU\")\n    strategy = tf.distribute.get_strategy()\nmodel_list = []\nwith strategy.scope():\n    for m_i in models_infor:\n        model = RnnModel(num_layers=3,d_model=420,num_heads=2,dff=400,dropout_rate=0.5)\n        model.build(input_shape = {\"x1\":[1,300,20],\"x2\":[1,300,400]})\n        model.load_weights(f'/kaggle/input/hms-train-model-tf-transformer/{m_i[0]}_weights/model_epoch_{m_i[1]}.weights.h5')\n        model_list.append(model)","metadata":{"execution":{"iopub.status.busy":"2024-02-12T11:56:36.842713Z","iopub.execute_input":"2024-02-12T11:56:36.843050Z","iopub.status.idle":"2024-02-12T11:56:39.031925Z","shell.execute_reply.started":"2024-02-12T11:56:36.843018Z","shell.execute_reply":"2024-02-12T11:56:39.030952Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def prediction_re(ds):\n    predictins = 0\n    for model_p in model_list:\n        predictins += model_p.predict(ds)\n    predictins = predictins/len(model_list)\n    return predictins","metadata":{"execution":{"iopub.status.busy":"2024-02-12T11:56:39.033394Z","iopub.execute_input":"2024-02-12T11:56:39.033748Z","iopub.status.idle":"2024-02-12T11:56:39.038664Z","shell.execute_reply.started":"2024-02-12T11:56:39.033713Z","shell.execute_reply":"2024-02-12T11:56:39.037799Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"span = 1999\neps = 1e-6\nn = (ds_test.shape[0]//span)+1\nfor i in range(n):\n    temp_ds = ds_test.iloc[i*span:(i+1)*span,:].reset_index()\n    print(temp_ds)\n    list_x1 = []\n    list_x2 = []\n    for ii in range(temp_ds.shape[0]):\n        eeg_id = temp_ds.loc[ii,\"eeg_id\"]\n        min = 0\n        values_eeg = pd.read_parquet(test_eegs+str(eeg_id)+\".parquet\").fillna(0).values[min:min+300,:]\n        spectrogram_id = temp_ds.loc[ii,\"spectrogram_id\"]\n        values_spc = pd.read_parquet(test_spectrograms+str(spectrogram_id)+\".parquet\").fillna(-1).values[min:min+300,1:]\n\n        values_eeg=np.clip(values_eeg,np.exp(-6),np.exp(10))#最大值为89209464.0\n        x1= np.log(values_eeg)#对数变换\n        # Normalize the data\n        data_mean = x1.mean(axis=(0, 1))\n        data_std = x1.std(axis=(0, 1))\n        x1 = (x1 - data_mean) / (data_std + eps)\n\n        values_spc=np.clip(values_spc,np.exp(-6),np.exp(10))#最大值为89209464.0\n        x2= np.log(values_spc)#对数变换\n        # Normalize the data\n        data_mean = x2.mean(axis=(0, 1))\n        data_std = x2.std(axis=(0, 1))\n        x2 = (x2 - data_mean) / (data_std + eps)\n        list_x1.append(x1)\n        list_x2.append(x2)\n    pre_data = {'x1':np.array(list_x1),'x2':np.array(list_x2)}\n    if i== 0:\n        predictins = prediction_re(pre_data)\n    else:\n        predictins = np.concatenate([predictins,prediction_re(pre_data)])","metadata":{"execution":{"iopub.status.busy":"2024-02-12T11:56:39.039758Z","iopub.execute_input":"2024-02-12T11:56:39.040631Z","iopub.status.idle":"2024-02-12T11:56:40.491447Z","shell.execute_reply.started":"2024-02-12T11:56:39.040597Z","shell.execute_reply":"2024-02-12T11:56:40.490546Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predic_ds = ds_test[[\"eeg_id\"]]\nfor i in range(len(target_cols)):\n    print(target_cols[i])\n    predic_ds[target_cols[i]] = list(predictins[:,i])","metadata":{"execution":{"iopub.status.busy":"2024-02-12T11:56:40.492790Z","iopub.execute_input":"2024-02-12T11:56:40.493143Z","iopub.status.idle":"2024-02-12T11:56:40.508942Z","shell.execute_reply.started":"2024-02-12T11:56:40.493108Z","shell.execute_reply":"2024-02-12T11:56:40.507864Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n#sample_submission.csv\npredic_ds.to_csv(\"submission.csv\",index = None)","metadata":{"execution":{"iopub.status.busy":"2024-02-12T11:56:40.510024Z","iopub.execute_input":"2024-02-12T11:56:40.510297Z","iopub.status.idle":"2024-02-12T11:56:40.519849Z","shell.execute_reply.started":"2024-02-12T11:56:40.510272Z","shell.execute_reply":"2024-02-12T11:56:40.518978Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.read_csv(\"submission.csv\")","metadata":{"execution":{"iopub.status.busy":"2024-02-12T11:56:40.523124Z","iopub.execute_input":"2024-02-12T11:56:40.523367Z","iopub.status.idle":"2024-02-12T11:56:40.536780Z","shell.execute_reply.started":"2024-02-12T11:56:40.523345Z","shell.execute_reply":"2024-02-12T11:56:40.535957Z"},"trusted":true},"execution_count":null,"outputs":[]}]}