{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# *KAGGLE CHALLENGE: LANL Earthquake Prediction*\n\nUn projet de Matthieu Dagommer, Paul Boulgakoff, Godefroy Bichon, Germain L'Hostis\n\nVersions utilisées:\n\nPython: 3.10.4\nTorch: 1.11","metadata":{"id":"JqdSSxEJLtds"}},{"cell_type":"code","source":"import numpy as np\nimport math\nimport matplotlib\nimport matplotlib.pyplot as plt \nimport time\n\nimport torch as th\nth.cuda.empty_cache()\n\nimport torch.autograd as autograd\nfrom torch.autograd import Variable\nimport torch.nn.functional as F\nimport torch.nn as nn\n\n#from torchsummary import summary \n\nimport gzip\nimport pickle\n\nimport pandas as pd\nimport scipy\nimport scipy.stats as stats\nfrom scipy.stats import kurtosis, skew\nimport csv\nimport os\nimport pickle\nimport gc\n\nfrom sklearn.preprocessing import StandardScaler, MinMaxScaler","metadata":{"execution":{"iopub.status.busy":"2022-05-29T22:53:50.294689Z","iopub.execute_input":"2022-05-29T22:53:50.29523Z","iopub.status.idle":"2022-05-29T22:53:52.943196Z","shell.execute_reply.started":"2022-05-29T22:53:50.295037Z","shell.execute_reply":"2022-05-29T22:53:52.94239Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"### Ensuring CPU is connected\n\nprint(th.cuda.is_available())\nprint(th.cuda.get_device_name())\ndevice = th.device('cuda' if th.cuda.is_available() else 'cpu')","metadata":{"execution":{"iopub.status.busy":"2022-05-29T22:53:52.944935Z","iopub.execute_input":"2022-05-29T22:53:52.945494Z","iopub.status.idle":"2022-05-29T22:53:53.020418Z","shell.execute_reply.started":"2022-05-29T22:53:52.945457Z","shell.execute_reply":"2022-05-29T22:53:53.019086Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"os.mkdir(\"./Models\")","metadata":{"execution":{"iopub.status.busy":"2022-05-29T22:53:53.021972Z","iopub.execute_input":"2022-05-29T22:53:53.022936Z","iopub.status.idle":"2022-05-29T22:53:53.030076Z","shell.execute_reply.started":"2022-05-29T22:53:53.022841Z","shell.execute_reply":"2022-05-29T22:53:53.029206Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# *General Hyperparameters*","metadata":{}},{"cell_type":"code","source":"### Setting General Hyperparameters \n\nbatch_size = 100 # number of patches per batch\nvalid_rate = 0.1 # fraction of data dedicated to validation\noverlap_rate = 0.2 # overlap\n\n# Parameters\n\nnrows = 50_000_000\nseed = 1\npatch_size = 150_000","metadata":{"execution":{"iopub.status.busy":"2022-05-29T22:53:53.032056Z","iopub.execute_input":"2022-05-29T22:53:53.033098Z","iopub.status.idle":"2022-05-29T22:53:53.039179Z","shell.execute_reply.started":"2022-05-29T22:53:53.033056Z","shell.execute_reply":"2022-05-29T22:53:53.0384Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"th.manual_seed(seed)\nnp.random.seed(seed)","metadata":{"execution":{"iopub.status.busy":"2022-05-29T22:53:53.041143Z","iopub.execute_input":"2022-05-29T22:53:53.041678Z","iopub.status.idle":"2022-05-29T22:53:53.046684Z","shell.execute_reply.started":"2022-05-29T22:53:53.041641Z","shell.execute_reply":"2022-05-29T22:53:53.045829Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# *Data Preparation*\n\n","metadata":{}},{"cell_type":"code","source":"### Loading Data\n\n#rootpath = os.getcwd() + \"/\"\nrootpath = \"../input/LANL-Earthquake-Prediction/\"\n\ntrain_data = pd.read_csv(rootpath + \"train.csv\", usecols = ['acoustic_data', 'time_to_failure'], \\\n                         dtype = {'acoustic_data': np.int16, 'time_to_failure': np.float64}, nrows = nrows)","metadata":{"execution":{"iopub.status.busy":"2022-05-29T22:53:53.048098Z","iopub.execute_input":"2022-05-29T22:53:53.048569Z","iopub.status.idle":"2022-05-29T22:54:08.603778Z","shell.execute_reply.started":"2022-05-29T22:53:53.048534Z","shell.execute_reply":"2022-05-29T22:54:08.602949Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X = th.squeeze(th.tensor(train_data['acoustic_data'].values, dtype = th.int16))\nY = th.squeeze(th.tensor(train_data['time_to_failure'].values, dtype = th.float64))\n\ndel train_data\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-05-29T22:54:08.605129Z","iopub.execute_input":"2022-05-29T22:54:08.605493Z","iopub.status.idle":"2022-05-29T22:54:08.999651Z","shell.execute_reply.started":"2022-05-29T22:54:08.605456Z","shell.execute_reply":"2022-05-29T22:54:08.998866Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"### Function to create patched sequences with some overlap out of the training data\n\n# Overlap is expressed as a fraction of patch size\ndef patching(patch_size, X, Y, overlap_rate):\n    \n    overlap = int(patch_size*overlap_rate)\n    L = X.shape[0] # total length of the training acoustic signal\n    n_patch = int(np.floor((L-patch_size)/(patch_size-overlap)+1))\n    ids_no_seism = []\n\n    X_patch = th.zeros(n_patch,patch_size)\n    Y_patch = th.zeros(n_patch)\n    \n    for i in range (n_patch):\n        X_patch[i,:] = X[i*(patch_size-overlap):(i+1)*patch_size-i*overlap] \n        Y_patch[i] = Y[(i+1)*patch_size - i*overlap] \n    \n        # Removing patches with no seism\n        if th.min(Y[i*(patch_size-overlap):(i+1)*patch_size - i*overlap]) > 0.001:\n            ids_no_seism.append(i)\n    \n    X_patch = X_patch[ids_no_seism]\n    Y_patch = Y_patch[ids_no_seism]\n    \n    return(n_patch, X_patch, Y_patch)","metadata":{"id":"nn_5lrssRSpJ","outputId":"e0eb114e-3c57-458a-bec2-53709bbd85d8","execution":{"iopub.status.busy":"2022-05-29T22:54:09.000856Z","iopub.execute_input":"2022-05-29T22:54:09.001431Z","iopub.status.idle":"2022-05-29T22:54:09.009738Z","shell.execute_reply.started":"2022-05-29T22:54:09.001393Z","shell.execute_reply":"2022-05-29T22:54:09.008727Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"### Normalizing Data\n\nss = StandardScaler()\nmm = MinMaxScaler()\n    \nX = th.squeeze(th.from_numpy(ss.fit_transform(np.array(X).reshape(-1, 1))))\nY = th.squeeze(th.from_numpy(mm.fit_transform(np.array(Y).reshape(-1, 1))))","metadata":{"execution":{"iopub.status.busy":"2022-05-29T22:54:09.012637Z","iopub.execute_input":"2022-05-29T22:54:09.013068Z","iopub.status.idle":"2022-05-29T22:54:10.320055Z","shell.execute_reply.started":"2022-05-29T22:54:09.013017Z","shell.execute_reply":"2022-05-29T22:54:10.319245Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"### Distribution of data in patches\n\n_, X_patch, Y_patch = patching(patch_size, X, Y, overlap_rate = overlap_rate)\n\ndel X; del Y\ngc.collect()\n\nprint(X_patch.shape)\nprint(\"There are \", X_patch.shape[0], \"time series available for training and validation after patching with overlap. \\n\")","metadata":{"execution":{"iopub.status.busy":"2022-05-29T22:54:10.321209Z","iopub.execute_input":"2022-05-29T22:54:10.321712Z","iopub.status.idle":"2022-05-29T22:54:11.025928Z","shell.execute_reply.started":"2022-05-29T22:54:10.321676Z","shell.execute_reply":"2022-05-29T22:54:11.024666Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"### Initializating Data Sets\n\n\n# Shuffling data\nidx = np.arange(X_patch.shape[0])\nshuffled_idx = np.random.shuffle(idx)\nX_patch = th.squeeze(X_patch[shuffled_idx,:])\nY_patch = th.squeeze(Y_patch[shuffled_idx])\n\nN_samples = X_patch.shape[0]\nN_valid = int(valid_rate*N_samples) # number of validation patches\n\nX_valid = X_patch[:N_valid,:].cpu()\nY_valid = Y_patch[:N_valid].cpu()\n\n\n# Inputs of nn.Conv1d must have the following shape: (N, C_in, *),\n#where N: number of samples (batch size), C_in: number of input channels, *: can be any dimension (150_000 for our time series)\n\n# Add channel dimension\nX_valid = th.unsqueeze(X_valid, 1)\nX_train = X_patch[N_valid:,:].cpu()\nY_train = Y_patch[N_valid:].cpu()\nN_train = X_train.shape[0]\n\nn_batch = int(X_train.shape[0] / batch_size) # Number of batch per epoch\n\nX_train = X_train.reshape([N_train, 1, patch_size])","metadata":{"execution":{"iopub.status.busy":"2022-05-29T22:54:11.030748Z","iopub.execute_input":"2022-05-29T22:54:11.032358Z","iopub.status.idle":"2022-05-29T22:54:11.045081Z","shell.execute_reply.started":"2022-05-29T22:54:11.032316Z","shell.execute_reply":"2022-05-29T22:54:11.044211Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# *Function for Results plotting*","metadata":{}},{"cell_type":"code","source":"### Plotting Graphs\n\ndef plot_and_save_results(train_losses, valid_losses, best_mvd, best_mtd, min_tl, min_vl, mm, model_name):\n\n    best_epoch = np.fromiter(valid_losses, dtype=float).argmin()\n    \n    fig, ax = plt.subplots(1, 2, figsize = (20,10))\n    plt.rcParams['font.size'] = '20'\n    ax[0].set(title = \"Losses: Training and Validation\") \n    ax[0].set_xlabel(\"epochs\", fontsize = 20)\n    ax[0].set_ylabel(\"MSE\", fontsize = 20)\n    ax[0].plot(train_losses,\"r\", label = \"Training\", linewidth = 3)\n    ax[0].plot(valid_losses, \"b\", label = \"Validation\", linewidth = 3)\n    ax[0].legend(loc = \"upper right\", fontsize = 18)\n    ax[0].axvline(x=int(best_epoch), color = 'black', linestyle =\"--\", linewidth = 3)\n    ax[0].annotate(\"Best epoch: {}\\nMSE_train: {:.3f}\\nMSE_valid: {:.3f}\".format(int(best_epoch), min_tl, min_vl), \\\n                   xy = (0.5,0.5), xycoords = 'axes fraction')\n    \n    best_mvd_plot = mm.inverse_transform(best_mvd.reshape(-1, 1))\n    best_mtd_plot = mm.inverse_transform(best_mtd.reshape(-1, 1))\n\n    N_valid = best_mvd_plot.shape[0]\n    \n    mean = float(np.mean(best_mvd_plot))\n    std_dev = float(np.std(best_mvd_plot))\n    kurt = float(kurtosis(best_mvd_plot))\n    skewn = float(skew(best_mvd_plot))\n    q1 = float(np.quantile(best_mvd_plot, 0.25))\n    median = float(np.quantile(best_mvd_plot, 0.5))\n    q3 = float(np.quantile(best_mvd_plot, 0.75))\n    mae = np.absolute(best_mvd_plot).sum() / N_valid\n    mse = np.square(best_mvd_plot).sum() / N_valid\n    \n    text = \"mean: {:.3f}\\nstd: {:.3f}\\nkurt: {:.3f}\\nskew: {:.3f}\\nq1: {:.3f}\\nmed: {:.3f}\\nq3: {:.3f}\\niqr: {:.3f}\\nmae: {:.3f}\\nmse: {:.3f}\\n\".format(mean, std_dev, kurt, skewn, q1, median, q3, q3-q1, mae, mse)\n    \n    ax[1].hist(best_mvd_plot, alpha = 0.3, label = \"validation set\", bins = 100, density = True, range = (-16, 16))\n    ax[1].set(title = \"TTF error distributions at best epoch\")\n    ax[1].set_xlabel(\"Error (seconds)\", fontsize = 20)\n    ax[1].set_ylabel(\"Density\", fontsize = 20)\n    ax[1].hist(best_mtd_plot, alpha = 0.3, label = \"training set\", bins = 100, density = True, range = (-16, 16))\n    ax[1].annotate(text, xy =(-15, 0.1))\n    ax[1].legend()\n\n\n    plt.gcf()\n    plt.savefig('Models/' + model_name + '/' + model_name + \"_plot.jpg\")\n    plt.show()\n\n    model_features = {\"N_samples\": N_samples, \"N_train\": N_train, \"N_valid\": N_valid, \\\n                      \"overlap_rate\": overlap_rate, \"learning_rate\": learning_rate, \"num_epochs\": num_epochs, \\\n                     \"seed\": seed, \"batch_size\": batch_size, \"train_losses\": train_losses, \"valid_losses\": valid_losses, \\\n                     \"valid_differences\": valid_differences, \"best_mtd\": best_mtd, \"best_mvd\": best_mvd, \"min_tl\": min_tl, \\\n                      \"min_vl\": min_vl}\n\n    pickle.dump(model_features, open('Models/' + model_name + '/' + model_name + \".p\", \"wb\" ))\n    \n    model_features_display = {\"N_samples\": N_samples, \"N_train\": N_train, \"N_valid\": N_valid, \\\n                  \"overlap_rate\": overlap_rate, \"learning_rate\": learning_rate, \"num_epochs\": num_epochs, \\\n                 \"seed\": seed, \"batch_size\": batch_size}\n    \n    pickle.dump(model_features_display, open('Models/' + model_name + '/' + model_name + \"display.p\", \"wb\" ))","metadata":{"execution":{"iopub.status.busy":"2022-05-29T22:54:11.04669Z","iopub.execute_input":"2022-05-29T22:54:11.047467Z","iopub.status.idle":"2022-05-29T22:54:11.084379Z","shell.execute_reply.started":"2022-05-29T22:54:11.047424Z","shell.execute_reply":"2022-05-29T22:54:11.083556Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# *LSTM*\n\nLSTM is a type of recurrent neural network and is a relevant architecture to treat temporal sequences. \n\nPytorch Notes:\n\n\"Before getting to the example, note a few things. Pytorch’s LSTM expects all of its inputs to be 3D tensors. The semantics of the axes of these tensors is important. The first axis is the sequence itself, the second indexes instances in the mini-batch, and the third indexes elements of the input.\"\n\nhttps://pytorch.org/tutorials/beginner/nlp/sequence_models_tutorial.html\n","metadata":{"id":"eqi1BoEdwan0"}},{"cell_type":"code","source":"### Hyperparameters specific to LSTM\n\nnum_epochs = 1000 #1000 epochs\nlearning_rate = 0.001 #0.001 lr\nhidden_size = 1 #number of features in hidden state\nnum_layers = 1 #number of stacked lstm layers => should stay at 1 unless you want to combine two LSTMs together\nN_sub_patches = 250\nN_features = 10\ndevice = \"cuda\"\nn_batch = 1 # One batch with all training data","metadata":{"execution":{"iopub.status.busy":"2022-05-29T22:54:11.086491Z","iopub.execute_input":"2022-05-29T22:54:11.086809Z","iopub.status.idle":"2022-05-29T22:54:11.09591Z","shell.execute_reply.started":"2022-05-29T22:54:11.086778Z","shell.execute_reply":"2022-05-29T22:54:11.094876Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def feature_expansion(X, N_sub_patches, L_sub_patch):\n    \n    #print(type(X))\n    N_samples = X.shape[0]\n    x = X.reshape(N_samples, N_sub_patches, L_sub_patch)\n    x_mean = np.mean(x, axis = 2).reshape(N_samples, N_sub_patches, 1)\n    x_std = np.std(x, axis = 2).reshape(N_samples, N_sub_patches, 1)\n    x_skew = np.array(scipy.stats.skew(x, axis = 2), dtype = np.double).reshape(N_samples, N_sub_patches, 1)\n    x_kurt = np.array(scipy.stats.kurtosis(x, axis = 2), dtype = np.double).reshape(N_samples, N_sub_patches, 1)\n    x_min = np.min(x, axis = 2).reshape(N_samples, N_sub_patches, 1)\n    x_max = np.max(x, axis = 2).reshape(N_samples, N_sub_patches, 1)\n    x_q1 = np.quantile(x, 0.25, axis = 2).reshape(N_samples, N_sub_patches, 1)\n    x_med = np.quantile(x, 0.5, axis = 2).reshape(N_samples, N_sub_patches, 1)\n    x_q3 = np.quantile(x, 0.75, axis = 2).reshape(N_samples, N_sub_patches, 1)\n    x_iqr = x_q3 - x_q1\n    \n    X_rearranged = np.concatenate((x_mean, x_std, x_skew, x_kurt, x_min, x_max, x_q1, x_med, x_q3, x_iqr), axis = 2)\n    return X_rearranged\n    \n\ndef lstm_feature_engineering(X_train, Y_train, X_valid, Y_valid, patch_size, N_sub_patches):\n\n    X_train = np.array(th.squeeze(X_train)); Y_train = np.array(Y_train)\n    X_valid = np.array(th.squeeze(X_valid)); Y_valid = np.array(Y_valid)\n\n    L_seq = X_train.shape[1]\n    L_sub_patch = int(L_seq / N_sub_patches)\n    N_features = 10 # mean, std, skew, kurt, min, max, quantiles 0.25, 0.5, 0.75, inter-quartile range\n    \n    X_train_rearranged = feature_expansion(X_train, N_sub_patches, L_sub_patch)\n    X_valid_rearranged = feature_expansion(X_valid, N_sub_patches, L_sub_patch)\n\n    X_train = th.from_numpy(X_train_rearranged)\n    X_valid = th.from_numpy(X_valid_rearranged)\n\n    Y_train = th.from_numpy(Y_train); Y_valid = th.from_numpy(Y_valid)\n    #Y_train = th.unsqueeze(Y_train, -1); Y_valid = th.unsqueeze(Y_valid, -1)\n\n    return X_train, Y_train, X_valid, Y_valid","metadata":{"id":"_4ULTZ0TYn8A","execution":{"iopub.status.busy":"2022-05-29T22:54:11.0996Z","iopub.execute_input":"2022-05-29T22:54:11.099938Z","iopub.status.idle":"2022-05-29T22:54:11.115377Z","shell.execute_reply.started":"2022-05-29T22:54:11.099902Z","shell.execute_reply":"2022-05-29T22:54:11.114359Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train_, Y_train_, X_valid_, Y_valid_ = lstm_feature_engineering(X_train, Y_train, X_valid, Y_valid, patch_size, N_sub_patches = N_sub_patches)\nN_features = X_train_.shape[-1]","metadata":{"id":"CFL8PvDnjZvs","execution":{"iopub.status.busy":"2022-05-29T22:54:11.118438Z","iopub.execute_input":"2022-05-29T22:54:11.119797Z","iopub.status.idle":"2022-05-29T22:54:16.364213Z","shell.execute_reply.started":"2022-05-29T22:54:11.119762Z","shell.execute_reply":"2022-05-29T22:54:16.363389Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Inspired from https://cnvrg.io/pytorch-lstm/\n\nclass Lstm(nn.Module):\n    \n    def __init__(self, N_features, hidden_size, num_layers, seq_length):\n        super(Lstm, self).__init__()\n\n        self.num_layers = num_layers # number of layers\n        self.N_features = N_features # number of features\n        self.hidden_size = hidden_size # hidden state\n        self.seq_length = seq_length # sequence length\n\n        self.lstm = nn.LSTM(input_size=N_features, hidden_size=hidden_size,\n                          num_layers=num_layers, batch_first=True) #lstm => Input Shape : (N_batch, L_seq, N_feature)\n    \n    def forward(self,x):\n        \n        output, (_,_) = self.lstm(x.float())\n        out = output[:,-1,0] # Retrieving predicted time at the end of the training\n        \n        return out","metadata":{"id":"VgH9isMrwZkp","execution":{"iopub.status.busy":"2022-05-29T22:54:16.365375Z","iopub.execute_input":"2022-05-29T22:54:16.365735Z","iopub.status.idle":"2022-05-29T22:54:16.374431Z","shell.execute_reply.started":"2022-05-29T22:54:16.365701Z","shell.execute_reply":"2022-05-29T22:54:16.373687Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"L_seq = N_sub_patches","metadata":{"id":"gc9D9u4T1C33","execution":{"iopub.status.busy":"2022-05-29T22:54:16.375569Z","iopub.execute_input":"2022-05-29T22:54:16.376397Z","iopub.status.idle":"2022-05-29T22:54:16.382113Z","shell.execute_reply.started":"2022-05-29T22:54:16.376359Z","shell.execute_reply":"2022-05-29T22:54:16.381239Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = Lstm(N_features, hidden_size, num_layers, L_seq) #our lstm class\nmodel.cuda()","metadata":{"id":"g7DE7GjCGeRR","execution":{"iopub.status.busy":"2022-05-29T22:54:16.385248Z","iopub.execute_input":"2022-05-29T22:54:16.385605Z","iopub.status.idle":"2022-05-29T22:54:19.898871Z","shell.execute_reply.started":"2022-05-29T22:54:16.385567Z","shell.execute_reply":"2022-05-29T22:54:19.897978Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"loss = th.nn.MSELoss()    # mean-squared error for regression\noptimizer = th.optim.Adam(model.parameters(), lr=learning_rate) ","metadata":{"id":"P9m0tjrtEzO2","execution":{"iopub.status.busy":"2022-05-29T22:54:19.900057Z","iopub.execute_input":"2022-05-29T22:54:19.900486Z","iopub.status.idle":"2022-05-29T22:54:19.905859Z","shell.execute_reply.started":"2022-05-29T22:54:19.900449Z","shell.execute_reply":"2022-05-29T22:54:19.904896Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Training LSTM\n\nth.cuda.empty_cache()\n\ntrain_losses, valid_losses = [], []\nbest_mvd, best_mtd = [], []\n\nmin_vl = 1000\nmin_tl = 1000\n\nstart = time.perf_counter()\n\nfor epoch in range(num_epochs):\n    \n    outputs = model.forward(X_train_.cuda()) #forward pass\n    optimizer.zero_grad() #calculate the gradient, manually setting to 0\n    # obtain the loss function\n    _loss = loss(outputs, Y_train_.cuda())\n    _loss.backward() #calculates the loss of the loss function\n    running_loss = _loss.item()\n    optimizer.step() #improve from loss, i.e backprop\n\n\n    model.eval()\n    with th.no_grad():\n        \n        Y_valid_pred = th.squeeze(model(X_valid_.cuda())).cpu()\n        valid_loss = loss(Y_valid_pred, Y_valid_)\n        valid_differences = Y_valid_pred[:] - Y_valid_[:]\n        valid_differences = valid_differences.numpy()\n        \n        valid_losses.append(valid_loss.item())\n        train_losses.append(running_loss / n_batch)\n        \n        if valid_loss < min_vl:\n\n            min_vl = valid_loss\n            min_tl = running_loss\n\n            best_mvd = valid_differences\n\n            Y_final = th.squeeze(model(X_train_.cuda())).cpu()\n\n            best_mtd = Y_train_ - Y_final\n            best_mtd = best_mtd.cpu().detach().numpy()\n            \n    model.train() \n    \n    if epoch%10 == 0:\n        print(\"Epoch: {}\\t\".format(epoch),\n                \"train Loss: {:.5f}.. \".format(train_losses[-1]),\n                \"valid Loss: {:.5f}.. \".format(valid_losses[-1])) \n        \nprint(\"---------- Best : {:.3f}\".format(min(valid_losses)), \" at epoch \" \n    , np.fromiter(valid_losses, dtype=float).argmin(), \" / \", epoch + 1)\n    \nend = time.perf_counter()\nprint(\"\\ntime elapsed: {:.3f}\".format(end - start))","metadata":{"id":"1k1geqipE79l","outputId":"67aa1abe-5aee-49a5-84f3-7d627ed8c6cc","scrolled":true,"execution":{"iopub.status.busy":"2022-05-29T22:54:19.907306Z","iopub.execute_input":"2022-05-29T22:54:19.907832Z","iopub.status.idle":"2022-05-29T22:54:35.694439Z","shell.execute_reply.started":"2022-05-29T22:54:19.907798Z","shell.execute_reply":"2022-05-29T22:54:35.693599Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#model_name = input(\"Choose a name for the model: \")\nmodel_name = \"Lstm\"\nos.mkdir('Models/' + model_name)","metadata":{"execution":{"iopub.status.busy":"2022-05-29T22:54:35.695735Z","iopub.execute_input":"2022-05-29T22:54:35.69611Z","iopub.status.idle":"2022-05-29T22:54:35.700458Z","shell.execute_reply.started":"2022-05-29T22:54:35.696073Z","shell.execute_reply":"2022-05-29T22:54:35.699359Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_and_save_results(train_losses, valid_losses, best_mvd, best_mtd, min_tl, min_vl, mm, model_name)\nth.save(model.state_dict(), 'Models/' + model_name + '/' + model_name)","metadata":{"execution":{"iopub.status.busy":"2022-05-29T22:54:35.701723Z","iopub.execute_input":"2022-05-29T22:54:35.702208Z","iopub.status.idle":"2022-05-29T22:54:36.797608Z","shell.execute_reply.started":"2022-05-29T22:54:35.70217Z","shell.execute_reply":"2022-05-29T22:54:36.796895Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del X_train; gc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-05-29T22:55:53.507534Z","iopub.execute_input":"2022-05-29T22:55:53.507892Z","iopub.status.idle":"2022-05-29T22:55:53.631416Z","shell.execute_reply.started":"2022-05-29T22:55:53.507863Z","shell.execute_reply":"2022-05-29T22:55:53.630636Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del X_valid; gc.collect()\ndel Y_valid; gc.collect()\ndel Y_train; gc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-05-29T22:56:13.314975Z","iopub.execute_input":"2022-05-29T22:56:13.315804Z","iopub.status.idle":"2022-05-29T22:56:13.63375Z","shell.execute_reply.started":"2022-05-29T22:56:13.315768Z","shell.execute_reply":"2022-05-29T22:56:13.63304Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# *Retrieve Test Data*","metadata":{}},{"cell_type":"code","source":"### Retrieve Test Data\n\n#rootpath = os.getcwd() + \"/\"\nrootpath = \"../input/LANL-Earthquake-Prediction/\"\n\nX_test_ = []\n\nfor filename in os.listdir(rootpath + \"test\"):\n    temp_df = pd.read_csv(rootpath + \"test/\" + filename)\n    X_test_.append(temp_df)\n\npatch_size = X_test_[0].shape[0]\nsample_submission = pd.read_csv(rootpath + \"sample_submission.csv\")\n\nX_test = th.zeros((len(X_test_), patch_size))\nfor i in range(len(X_test_)):\n    X_test[i,:] = th.tensor(X_test_[i][\"acoustic_data\"], dtype = th.float32)\n\ndel X_test_; gc.collect()\n\nY_test = th.tensor(sample_submission[\"time_to_failure\"])\n\nN_test = X_test.shape[0]\nX_test = X_test.reshape([N_test, 1, patch_size])","metadata":{"execution":{"iopub.status.busy":"2022-05-29T22:57:00.580478Z","iopub.execute_input":"2022-05-29T22:57:00.580829Z","iopub.status.idle":"2022-05-29T22:59:03.277431Z","shell.execute_reply.started":"2022-05-29T22:57:00.580799Z","shell.execute_reply":"2022-05-29T22:59:03.276593Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# *Submission*","metadata":{}},{"cell_type":"code","source":"### Load Model\n\nmodel = Lstm(N_features, hidden_size, num_layers, L_seq).cuda()\nmodel.load_state_dict(th.load('Models/Lstm/Lstm'))","metadata":{"execution":{"iopub.status.busy":"2022-05-29T22:59:58.846901Z","iopub.execute_input":"2022-05-29T22:59:58.847795Z","iopub.status.idle":"2022-05-29T22:59:58.857757Z","shell.execute_reply.started":"2022-05-29T22:59:58.847759Z","shell.execute_reply":"2022-05-29T22:59:58.85682Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"L_sub_patch = int(patch_size / N_sub_patches)\nX_test = np.array(th.squeeze(X_test))\n#X_test.to('cuda')\nX_test_expanded = feature_expansion(X_test, N_sub_patches, L_sub_patch)\nX_test_expanded = th.from_numpy(X_test_expanded).cuda()\n\nmodel.eval()\nwith th.no_grad():\n    Y_test_predicted = model(X_test_expanded)","metadata":{"execution":{"iopub.status.busy":"2022-05-29T23:05:47.11394Z","iopub.execute_input":"2022-05-29T23:05:47.1145Z","iopub.status.idle":"2022-05-29T23:06:33.156447Z","shell.execute_reply.started":"2022-05-29T23:05:47.114462Z","shell.execute_reply":"2022-05-29T23:06:33.155616Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Y_test_predicted.cpu()\nY_test_predicted = np.array(Y_test_predicted.cpu()).squeeze()","metadata":{"execution":{"iopub.status.busy":"2022-05-29T23:09:09.056124Z","iopub.execute_input":"2022-05-29T23:09:09.056482Z","iopub.status.idle":"2022-05-29T23:09:09.060976Z","shell.execute_reply.started":"2022-05-29T23:09:09.056454Z","shell.execute_reply":"2022-05-29T23:09:09.059958Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_submission['time_to_failure'] = Y_test_predicted","metadata":{"execution":{"iopub.status.busy":"2022-05-29T23:09:18.724228Z","iopub.execute_input":"2022-05-29T23:09:18.724577Z","iopub.status.idle":"2022-05-29T23:09:18.732502Z","shell.execute_reply.started":"2022-05-29T23:09:18.724548Z","shell.execute_reply":"2022-05-29T23:09:18.731499Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_submission.to_csv('submission.csv', index = False)","metadata":{"execution":{"iopub.status.busy":"2022-05-29T23:09:20.249Z","iopub.execute_input":"2022-05-29T23:09:20.249667Z","iopub.status.idle":"2022-05-29T23:09:20.267444Z","shell.execute_reply.started":"2022-05-29T23:09:20.249624Z","shell.execute_reply":"2022-05-29T23:09:20.266728Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}