{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# 0. Target","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport torch\nimport torch.nn as nn\n\ntarget = pd.read_csv('/kaggle/input/predict-student-performance-from-game-play/train_labels.csv')\ntarget","metadata":{"execution":{"iopub.status.busy":"2023-06-15T13:30:27.204669Z","iopub.execute_input":"2023-06-15T13:30:27.205123Z","iopub.status.idle":"2023-06-15T13:30:27.500594Z","shell.execute_reply.started":"2023-06-15T13:30:27.205081Z","shell.execute_reply":"2023-06-15T13:30:27.499451Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"target.session_id.astype(str).str.split('_').str[1].value_counts()","metadata":{"execution":{"iopub.status.busy":"2023-06-15T13:30:58.414076Z","iopub.execute_input":"2023-06-15T13:30:58.415665Z","iopub.status.idle":"2023-06-15T13:30:59.961106Z","shell.execute_reply.started":"2023-06-15T13:30:58.415620Z","shell.execute_reply":"2023-06-15T13:30:59.959902Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"I don't know exactly what the session ID means, but the following q1 to q18 are solved by the session ID.\n\nand the `correct` column shows whether the problem is solved or not.\n\nIn this case, I thought it would be good to see this information as a sequence and process it, \n\nas students had to solve problems 1 through 18 continuously.","metadata":{}},{"cell_type":"markdown","source":"# 1. Read the DataFrame","metadata":{}},{"cell_type":"code","source":"# Import required libraries\nimport pandas as pd\nimport numpy as np\nimport torch\nimport torch.nn as nn\n\ndf = pd.read_csv('/kaggle/input/predict-student-performance-from-game-play/train.csv', \n                 usecols=['session_id', 'elapsed_time', 'index'])\n\n# Print the first 5 rows\ndf.head()","metadata":{"execution":{"iopub.status.busy":"2023-06-11T09:18:23.77675Z","iopub.execute_input":"2023-06-11T09:18:23.777704Z","iopub.status.idle":"2023-06-11T09:19:39.65518Z","shell.execute_reply.started":"2023-06-11T09:18:23.777649Z","shell.execute_reply":"2023-06-11T09:19:39.65413Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 2. Intuition of using LSTM\n\nIn specific, taking a `session_id`...","metadata":{}},{"cell_type":"code","source":"df.set_index(['session_id', 'index'], inplace=True)\ndf.head()","metadata":{"execution":{"iopub.status.busy":"2023-06-11T09:19:51.219939Z","iopub.execute_input":"2023-06-11T09:19:51.220346Z","iopub.status.idle":"2023-06-11T09:19:52.396838Z","shell.execute_reply.started":"2023-06-11T09:19:51.220311Z","shell.execute_reply":"2023-06-11T09:19:52.395623Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for col in df.columns:\n    # Scaling the coordinates and durations\n    df[col] = (df[col] - df[col].min()) / (df[col].max() - df[col].min())\n    df[col] = df[col].fillna(0)","metadata":{"execution":{"iopub.status.busy":"2023-06-11T09:19:52.399151Z","iopub.execute_input":"2023-06-11T09:19:52.399559Z","iopub.status.idle":"2023-06-11T09:19:52.818976Z","shell.execute_reply.started":"2023-06-11T09:19:52.399521Z","shell.execute_reply":"2023-06-11T09:19:52.817733Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 3. One-Hot Encoding and Aggregation","metadata":{}},{"cell_type":"code","source":"import sklearn\n\n\nclass GetDummies(sklearn.base.TransformerMixin):\n    def __init__(self, dtypes=None):\n        self.input_columns = None\n        self.final_columns = None\n        if dtypes is None:\n            dtypes = [object, 'category']\n        self.dtypes = dtypes\n\n    def fit(self, X, y=None, **kwargs):\n        self.input_columns = list(X.select_dtypes(self.dtypes).columns)\n        X = pd.get_dummies(X, columns=self.input_columns)\n        self.final_columns = X.columns\n        return self\n        \n    def transform(self, X, y=None, **kwargs):\n        X = pd.get_dummies(X, columns=self.input_columns)\n        X_columns = X.columns\n        missing = set(self.final_columns) - set(X_columns)\n        for c in missing:\n            X[c] = 0\n        return X[self.final_columns]\n    \n    def get_feature_names(self):\n        return tuple(self.final_columns)","metadata":{"execution":{"iopub.status.busy":"2023-06-11T09:19:53.863354Z","iopub.execute_input":"2023-06-11T09:19:53.863746Z","iopub.status.idle":"2023-06-11T09:19:53.875323Z","shell.execute_reply.started":"2023-06-11T09:19:53.863712Z","shell.execute_reply":"2023-06-11T09:19:53.874089Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"get_dummies = GetDummies()\ndf = get_dummies.fit_transform(df)\ndf.shape","metadata":{"execution":{"iopub.status.busy":"2023-06-11T09:19:54.851748Z","iopub.execute_input":"2023-06-11T09:19:54.853002Z","iopub.status.idle":"2023-06-11T09:19:56.937707Z","shell.execute_reply.started":"2023-06-11T09:19:54.852949Z","shell.execute_reply":"2023-06-11T09:19:56.936442Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Aggregate data in each session into a 2D numpy array\ngrouped_data = df.groupby('session_id').apply(lambda x: np.array(x))\ngrouped_data","metadata":{"execution":{"iopub.status.busy":"2023-06-11T09:19:56.940099Z","iopub.execute_input":"2023-06-11T09:19:56.940519Z","iopub.status.idle":"2023-06-11T09:19:59.674392Z","shell.execute_reply.started":"2023-06-11T09:19:56.940479Z","shell.execute_reply":"2023-06-11T09:19:59.673189Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 4. Convert to PyTorch Dataloader","metadata":{}},{"cell_type":"code","source":"from torch.utils.data import Dataset, DataLoader\n\nclass MyDataset(Dataset):\n    def __init__(self, data):\n        self.data = data\n        \n    def __len__(self):\n        return len(self.data)\n    \n    def __getitem__(self, idx):\n        # Get the numpy array at the given index\n        return torch.from_numpy(self.data[idx])","metadata":{"execution":{"iopub.status.busy":"2023-06-11T09:19:59.676505Z","iopub.execute_input":"2023-06-11T09:19:59.676977Z","iopub.status.idle":"2023-06-11T09:19:59.683855Z","shell.execute_reply.started":"2023-06-11T09:19:59.676926Z","shell.execute_reply":"2023-06-11T09:19:59.682666Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Since the length of each sequence are different, batch transformation is required. ","metadata":{}},{"cell_type":"code","source":"def padd(batch):\n    \"\"\"\n    Padds batch of variable length\n\n    Note: it converts things ToTensor manually here since the ToTensor transform\n    assume it takes in images rather than arbitrary tensors.\n    \"\"\"\n    ## Get sequence lengths\n    lengths = [t.shape[0] for t in batch]\n    try:\n        n_features = batch[0].shape[1]\n    except:\n        n_features = 1\n    max_length = max(lengths)\n    if max_length == 0:\n        max_length += 1\n    batch_size = len(lengths)\n\n    padded_tensor = torch.zeros(batch_size, max_length, n_features)\n    for i, val in enumerate(batch):\n        l = lengths[i]\n        if n_features == 1:\n            padded_tensor[i, :l] = val.reshape(-1, 1)\n        else:\n            padded_tensor[i, :l] = val\n    \n    return padded_tensor","metadata":{"execution":{"iopub.status.busy":"2023-06-11T09:19:59.685531Z","iopub.execute_input":"2023-06-11T09:19:59.68622Z","iopub.status.idle":"2023-06-11T09:19:59.69595Z","shell.execute_reply.started":"2023-06-11T09:19:59.68615Z","shell.execute_reply":"2023-06-11T09:19:59.694856Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Create an instance of the custom dataset\ndataset = MyDataset(grouped_data.values)\n\n# Create a PyTorch DataLoader\ndataloader = DataLoader(dataset, batch_size=32, shuffle=True, collate_fn=padd)","metadata":{"execution":{"iopub.status.busy":"2023-06-11T09:19:59.698474Z","iopub.execute_input":"2023-06-11T09:19:59.698971Z","iopub.status.idle":"2023-06-11T09:19:59.707457Z","shell.execute_reply.started":"2023-06-11T09:19:59.698908Z","shell.execute_reply":"2023-06-11T09:19:59.706416Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 5. Processing the labels\n\nNow we collect and process the labels...","metadata":{}},{"cell_type":"code","source":"# Collect and process the label\n\nlabel_df = pd.read_csv('/kaggle/input/predict-student-performance-from-game-play/train_labels.csv')\nlabel_df['session'] = label_df.session_id.apply(lambda x: int(x.split('_')[0]) )\nlabel_df['question_idx'] = label_df.session_id.apply(lambda x: int(x.split('_')[-1][1:]) )\nlabel_df.drop(\"session_id\", axis=1, inplace=True)\npivoted_questions = label_df.pivot(columns='question_idx', values='correct', index='session')\n\n\n# We have a total_score column here just for analysis if needed\npivoted_questions['total_score'] = pivoted_questions.iloc[:, 0:18].sum(axis=1)\n\n\n# Rename the columns\npivoted_questions.columns = [f'q_{i}' for i in range(1, 19)] + ['total_score']\npivoted_questions","metadata":{"execution":{"iopub.status.busy":"2023-06-11T09:20:11.299355Z","iopub.execute_input":"2023-06-11T09:20:11.299742Z","iopub.status.idle":"2023-06-11T09:20:12.728208Z","shell.execute_reply.started":"2023-06-11T09:20:11.299709Z","shell.execute_reply":"2023-06-11T09:20:12.727003Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tqdm import tqdm\nbatch_size = 10\nfor i, sample in tqdm(enumerate(dataloader)):\n\n    # Get label\n    y = torch.from_numpy(pivoted_questions.iloc[i*batch_size:(i+1)*batch_size, :18].values)\n    X = sample\n    y = y\nX = X.numpy()\ny = y.numpy()","metadata":{"execution":{"iopub.status.busy":"2023-06-11T09:20:19.681919Z","iopub.execute_input":"2023-06-11T09:20:19.682332Z","iopub.status.idle":"2023-06-11T09:20:20.474782Z","shell.execute_reply.started":"2023-06-11T09:20:19.682295Z","shell.execute_reply":"2023-06-11T09:20:20.473642Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 6. Transformer model and Traning","metadata":{}},{"cell_type":"code","source":"import random, os, sys\nimport numpy as np\nfrom keras.models import *\nfrom keras.layers import *\nfrom keras.callbacks import *\nfrom keras.initializers import *\nimport tensorflow as tf\nfrom tensorflow.keras.layers import Layer\n\ntry:\n    from dataloader import TokenList, pad_to_longest\n    # for transformer\nexcept: pass\n\nclass LayerNormalization(Layer):\n    def __init__(self, eps=1e-6, **kwargs):\n        self.eps = eps\n        super(LayerNormalization, self).__init__(**kwargs)\n    def build(self, input_shape):\n        self.gamma = self.add_weight(name='gamma', shape=input_shape[-1:],\n                                     initializer=Ones(), trainable=True)\n        self.beta = self.add_weight(name='beta', shape=input_shape[-1:],\n                                    initializer=Zeros(), trainable=True)\n        super(LayerNormalization, self).build(input_shape)\n    def call(self, x):\n        mean = K.mean(x, axis=-1, keepdims=True)\n        std = K.std(x, axis=-1, keepdims=True)\n        return self.gamma * (x - mean) / (std + self.eps) + self.beta\n    def compute_output_shape(self, input_shape):\n        return input_shape\n\nclass ScaledDotProductAttention():\n    def __init__(self, d_model, attn_dropout=0.1):\n        self.temper = np.sqrt(d_model)\n        self.dropout = Dropout(attn_dropout)\n    def __call__(self, q, k, v, mask):\n        attn = Lambda(lambda x:K.batch_dot(x[0],x[1],axes=[2,2])/self.temper)([q, k])\n        if mask is not None:\n            mmask = Lambda(lambda x:(-1e+10)*(1-x))(mask)\n            attn = Add()([attn, mmask])\n        attn = Activation('softmax')(attn)\n        attn = self.dropout(attn)\n        output = Lambda(lambda x:K.batch_dot(x[0], x[1]))([attn, v])\n        return output, attn\n\nclass MultiHeadAttention():\n    # mode 0 - big martixes, faster; mode 1 - more clear implementation\n    def __init__(self, n_head, d_model, d_k, d_v, dropout, mode=0, use_norm=True):\n        self.mode = mode\n        self.n_head = n_head\n        self.d_k = d_k\n        self.d_v = d_v\n        self.dropout = dropout\n        if mode == 0:\n            self.qs_layer = Dense(n_head*d_k, use_bias=False)\n            self.ks_layer = Dense(n_head*d_k, use_bias=False)\n            self.vs_layer = Dense(n_head*d_v, use_bias=False)\n        elif mode == 1:\n            self.qs_layers = []\n            self.ks_layers = []\n            self.vs_layers = []\n            for _ in range(n_head):\n                self.qs_layers.append(TimeDistributed(Dense(d_k, use_bias=False)))\n                self.ks_layers.append(TimeDistributed(Dense(d_k, use_bias=False)))\n                self.vs_layers.append(TimeDistributed(Dense(d_v, use_bias=False)))\n        self.attention = ScaledDotProductAttention(d_model)\n        self.layer_norm = LayerNormalization() if use_norm else None\n        self.w_o = TimeDistributed(Dense(d_model))\n\n    def __call__(self, q, k, v, mask=None):\n        d_k, d_v = self.d_k, self.d_v\n        n_head = self.n_head\n\n        if self.mode == 0:\n            qs = self.qs_layer(q)  # [batch_size, len_q, n_head*d_k]\n            ks = self.ks_layer(k)\n            vs = self.vs_layer(v)\n\n            def reshape1(x):\n                s = tf.shape(x)   # [batch_size, len_q, n_head * d_k]\n                x = tf.reshape(x, [s[0], s[1], n_head, d_k])\n                x = tf.transpose(x, [2, 0, 1, 3])  \n                x = tf.reshape(x, [-1, s[1], d_k])  # [n_head * batch_size, len_q, d_k]\n                return x\n            qs = Lambda(reshape1)(qs)\n            ks = Lambda(reshape1)(ks)\n            vs = Lambda(reshape1)(vs)\n\n            if mask is not None:\n                mask = Lambda(lambda x:K.repeat_elements(x, n_head, 0))(mask)\n            head, attn = self.attention(qs, ks, vs, mask=mask)  \n                \n            def reshape2(x):\n                s = tf.shape(x)   # [n_head * batch_size, len_v, d_v]\n                x = tf.reshape(x, [n_head, -1, s[1], s[2]]) \n                x = tf.transpose(x, [1, 2, 0, 3])\n                x = tf.reshape(x, [-1, s[1], n_head*d_v])  # [batch_size, len_v, n_head * d_v]\n                return x\n            head = Lambda(reshape2)(head)\n        elif self.mode == 1:\n            heads = []; attns = []\n            for i in range(n_head):\n                qs = self.qs_layers[i](q)   \n                ks = self.ks_layers[i](k) \n                vs = self.vs_layers[i](v) \n                head, attn = self.attention(qs, ks, vs, mask)\n                heads.append(head); attns.append(attn)\n            head = Concatenate()(heads) if n_head > 1 else heads[0]\n            attn = Concatenate()(attns) if n_head > 1 else attns[0]\n\n        outputs = self.w_o(head)\n        outputs = Dropout(self.dropout)(outputs)\n        if not self.layer_norm: return outputs, attn\n        outputs = Add()([outputs, q])\n        return self.layer_norm(outputs), attn\n\nclass PositionwiseFeedForward():\n    def __init__(self, d_hid, d_inner_hid, dropout=0.1):\n        self.w_1 = Conv1D(d_inner_hid, 1, activation='relu')\n        self.w_2 = Conv1D(d_hid, 1)\n        self.layer_norm = LayerNormalization()\n        self.dropout = Dropout(dropout)\n    def __call__(self, x):\n        output = self.w_1(x) \n        output = self.w_2(output)\n        output = self.dropout(output)\n        output = Add()([output, x])\n        return self.layer_norm(output)\n\nclass EncoderLayer():\n    def __init__(self, d_model, d_inner_hid, n_head, d_k, d_v, dropout=0.1):\n        self.self_att_layer = MultiHeadAttention(n_head, d_model, d_k, d_v, dropout=dropout)\n        self.pos_ffn_layer  = PositionwiseFeedForward(d_model, d_inner_hid, dropout=dropout)\n    def __call__(self, enc_input, mask=None):\n        output, slf_attn = self.self_att_layer(enc_input, enc_input, enc_input, mask=mask)\n        output = self.pos_ffn_layer(output)\n        return output, slf_attn\n\ndef GetPosEncodingMatrix(max_len, d_emb):\n    pos_enc = np.array([\n        [pos / np.power(10000, 2 * (j // 2) / d_emb) for j in range(d_emb)] \n        if pos != 0 else np.zeros(d_emb) \n            for pos in range(max_len)\n            ])\n    pos_enc[1:, 0::2] = np.sin(pos_enc[1:, 0::2]) # dim 2i\n    pos_enc[1:, 1::2] = np.cos(pos_enc[1:, 1::2]) # dim 2i+1\n    return pos_enc\n\ndef GetPadMask(q, k):\n    ones = K.expand_dims(K.ones_like(q, 'float32'), -1)\n    mask = K.cast(K.expand_dims(K.not_equal(k, 0), 1), 'float32')\n    mask = K.batch_dot(ones, mask, axes=[2,1])\n    return mask\n\ndef GetSubMask(s):\n    len_s = tf.shape(s)[1]\n    bs = tf.shape(s)[:1]\n    mask = K.cumsum(tf.eye(len_s, batch_shape=bs), 1)\n    return mask\n\nclass Encoder():\n    def __init__(self, d_model, d_inner_hid, n_head, d_k, d_v, \\\n                layers=6, dropout=0.1, pos_emb=None):\n        self.pos_layer = pos_emb\n        self.dropout = Dropout(dropout)\n        self.layers = [EncoderLayer(d_model, d_inner_hid, n_head, d_k, d_v, dropout) for _ in range(layers)]\n        \n    def __call__(self, src_seq, src_pos, return_att=False, active_layers=999):\n        x = src_seq\n        if src_pos is not None:\n            pos = self.pos_layer(src_pos)\n            x = Add()([x, pos])\n        x = self.dropout(x)\n        if return_att: atts = []\n        # mask = Lambda(lambda x:GetPadMask(x, x))(src_seq)\n        mask = None\n        for enc_layer in self.layers[:active_layers]:\n            x, att = enc_layer(x, mask)\n            if return_att: atts.append(att)\n        return (x, atts) if return_att else x\n\n\nclass Transformer():\n    def __init__(self, len_limit, d_model=X.shape[-1], \\\n              d_inner_hid=8, n_head=8, d_k=8, d_v=8, layers=2, dropout=0.1, \\\n              share_word_emb=False, **kwargs):\n        self.name = 'Transformer'\n        self.len_limit = len_limit\n        self.src_loc_info = True\n        self.d_model = d_model\n        d_emb = d_model\n\n        pos_emb = Embedding(len_limit, d_emb, trainable=False, \\\n                            weights=[GetPosEncodingMatrix(len_limit, d_emb)])\n\n        self.encoder = Encoder(d_model, d_inner_hid, n_head, d_k, d_v, layers, dropout, \\\n                            pos_emb=pos_emb)\n\n        \n    def get_pos_seq(self, x):\n        pos = K.cumsum(K.ones_like(x[...,0], 'int32'), 1)\n        return pos\n\n    def compile(self, active_layers=16):\n        src_seq = Input(shape=(X.shape[1],X.shape[2]))\n        src_pos = Lambda(self.get_pos_seq)(src_seq)\n        if not self.src_loc_info: src_pos = None\n\n        x = self.encoder(src_seq, src_pos, active_layers=active_layers)\n        x = GlobalAveragePooling1D()(x)\n        outp = Dense(18)(x)\n\n        self.model = Model(inputs=src_seq, outputs=outp)\n        self.model.compile(loss='binary_crossentropy', optimizer='adam')\n        \n        \ndevice = 'cuda' if torch.cuda.is_available() else 'cpu'","metadata":{"execution":{"iopub.status.busy":"2023-06-11T09:20:27.77435Z","iopub.execute_input":"2023-06-11T09:20:27.774881Z","iopub.status.idle":"2023-06-11T09:20:35.23902Z","shell.execute_reply.started":"2023-06-11T09:20:27.774844Z","shell.execute_reply":"2023-06-11T09:20:35.238108Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X.shape","metadata":{"execution":{"iopub.status.busy":"2023-06-11T09:20:35.241547Z","iopub.execute_input":"2023-06-11T09:20:35.242759Z","iopub.status.idle":"2023-06-11T09:20:35.250304Z","shell.execute_reply.started":"2023-06-11T09:20:35.242715Z","shell.execute_reply":"2023-06-11T09:20:35.249059Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y.shape","metadata":{"execution":{"iopub.status.busy":"2023-06-11T09:20:35.25198Z","iopub.execute_input":"2023-06-11T09:20:35.252753Z","iopub.status.idle":"2023-06-11T09:20:35.260702Z","shell.execute_reply.started":"2023-06-11T09:20:35.252713Z","shell.execute_reply":"2023-06-11T09:20:35.259252Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import GridSearchCV, KFold\nfrom keras import backend as K\n\nsplits = list(KFold(n_splits=5, shuffle=True, random_state=1234).split(X, y))\npreds_val = []\ny_val = []\nfor idx, (train_idx, val_idx) in enumerate(splits):\n\n    print(\"Beginning fold {}\".format(idx+1))\n    train_X, train_y, val_X, val_y = X[train_idx], y[train_idx], X[val_idx], y[val_idx]\n\n    model = Transformer(train_X.shape[1], layers=3,d_inner_hid=3)\n    model.compile()\n    model = model.model\n    model.fit(train_X, train_y, batch_size=32, epochs=10, validation_data=[val_X, val_y])#\n    #model.load_weights('weights_{}.h5'.format(idx))\n    preds_val.append(model.predict(val_X, batch_size=16))\n    y_val.append(val_y)\n\n# concatenates all and prints the shape    \npreds_val = np.concatenate(preds_val)[...,0]\ny_val = np.concatenate(y_val)\npreds_val.shape, y_val.shape","metadata":{"execution":{"iopub.status.busy":"2023-06-11T09:21:23.696388Z","iopub.execute_input":"2023-06-11T09:21:23.696761Z","iopub.status.idle":"2023-06-11T09:23:11.025952Z","shell.execute_reply.started":"2023-06-11T09:21:23.696727Z","shell.execute_reply":"2023-06-11T09:23:11.024702Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds_val","metadata":{"execution":{"iopub.status.busy":"2023-06-11T09:27:21.119989Z","iopub.execute_input":"2023-06-11T09:27:21.120533Z","iopub.status.idle":"2023-06-11T09:27:21.130474Z","shell.execute_reply.started":"2023-06-11T09:27:21.120489Z","shell.execute_reply":"2023-06-11T09:27:21.129023Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_val","metadata":{"execution":{"iopub.status.busy":"2023-06-11T09:27:30.66995Z","iopub.execute_input":"2023-06-11T09:27:30.670433Z","iopub.status.idle":"2023-06-11T09:27:30.682171Z","shell.execute_reply.started":"2023-06-11T09:27:30.67039Z","shell.execute_reply":"2023-06-11T09:27:30.680877Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 7. Submission","metadata":{}},{"cell_type":"code","source":"import jo_wilder\nenv = jo_wilder.make_env()\niter_test = env.iter_test()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!head submission.csv","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}