{"cells":[{"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.utils.data import Dataset, DataLoader\nimport torch.optim as optim\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\n# Any results you write to the current directory are saved as output.\n\nfrom tqdm import tqdm\n\nimport matplotlib.pyplot as plt\n%matplotlib inline\n\nfrom fastai.basics import *\nfrom fastai.callbacks import * ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9b41ca4194332d01def3c8ff798a13d2f5552c84"},"cell_type":"code","source":"PATH='../input/'\n!ls '../input/'","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"5491170ed0f1f7792b2168a62b6e3d1a0144d136","trusted":true},"cell_type":"code","source":"#float_data = pd.read_csv(\"../input/train.csv\",  nrows=10000000, dtype={'acoustic_data': np.int16, 'time_to_failure': np.float64})\nfloat_data = pd.read_csv(PATH+'train.csv', dtype={'acoustic_data': np.int16, 'time_to_failure': np.float32}).values","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f6558103cf2a5c746d9be523afaf395217a1758","trusted":true},"cell_type":"code","source":"# Data is a 629M long time series. \n# The data is recorded in bins of 4096 samples. Withing those bins seismic data is recorded at 4MHz, \n# but there is a 12 microseconds (??? or is it 1.2 millisecond) gap between each bin, an artifact of the recording device.\n# Test data consists of 150,000 long time series. We don't know where the gaps are in the test data.\n\n# Our dataset will create items that are each based on 150,000 long chunks of data\n\nclass EQdataset (Dataset):\n    def __init__(self, data, min_index =0, max_index = None, bin_density = 0.1, is_test = False):\n        \n        self.LENGTH_SAMPLE =150000\n        self.BIN_SIZE = 4096   #4096\n        self.BIN_CHUNK_SIZE = 64    #needs to be a divisor of 4096 (bin size)  #16\n        self.BIN_CHUNK_COUNT = int(self.BIN_SIZE/self.BIN_CHUNK_SIZE) \n        self.min_index = min_index\n        self.max_index = max_index\n        if max_index is None:\n            self.max_index = len(data) - 1\n        \n        \n        self.data = data[self.min_index:self.max_index+1]\n        \n        self.is_test = is_test\n        \n        if is_test:\n            self.dataset_size = 1\n            self.ending_positions = [150000]\n        else:\n            # Pick indices of ending positions\n            # dataset_size = number of bins * bin_density\n            self.dataset_size = int(np.round(len(self.data)/ self.BIN_SIZE * bin_density) )\n            # remember self.data index starts at 0 and not at min_index\n            self.ending_positions = np.random.randint(self.LENGTH_SAMPLE, self.max_index - self.min_index , size=self.dataset_size)\n\n        \n        \n            \n    \n    def __len__(self):\n        return self.dataset_size\n                 \n    def __getitem__(self, idx):\n        \n        ending_position = self.ending_positions[idx]\n        starting_position = ending_position - self.LENGTH_SAMPLE\n        idx_data = self.data[starting_position: ending_position,:]\n        \n        \n        # for each data point in the last bin, we form a 35 data point time series using data points separated by 4096 rows\n        # This way we have a constant time gap between each data point in the time series\n        rnn_length = int(np.floor (self.LENGTH_SAMPLE/self.BIN_SIZE)) -1   # =36\n        rnn_data_indexes = np.array([self.LENGTH_SAMPLE -1 - (rnn_length -1 - n) * self.BIN_SIZE for  n in range(rnn_length)])\n        \n        # 256 rnn inputs of length 36 and depth 16\n        \n        #X = np.array([[idx_data[index-self.BIN_CHUNK_SIZE+1 -bin_chunk_num*self.BIN_CHUNK_SIZE : index+1-bin_chunk_num*self.BIN_CHUNK_SIZE][:,0] for index in rnn_data_indexes] \n                     #for bin_chunk_num in range(self.BIN_CHUNK_COUNT)])\n        \n        X = idx_data[self.LENGTH_SAMPLE-self.BIN_CHUNK_COUNT*rnn_length*self.BIN_CHUNK_SIZE:,0].reshape((\n            rnn_length,self.BIN_CHUNK_COUNT,self.BIN_CHUNK_SIZE)).transpose(1,0,2)\n        # 256 rnn output of length 36\n        #y = np.array([idx_data[rnn_data_indexes - bin_chunk_num*self.BIN_CHUNK_SIZE][:,1] for bin_chunk_num in range(self.BIN_CHUNK_COUNT)])\n        \n        #y = idx_data[150000-self.BIN_CHUNK_COUNT*rnn_length*self.BIN_CHUNK_SIZE:,1].reshape((rnn_length,self.BIN_CHUNK_COUNT,self.BIN_CHUNK_SIZE)).transpose(1,0,2)[:,:,15] \n        \n        if self.is_test:\n            y =0\n            \n        else: \n            y = idx_data[self.LENGTH_SAMPLE -1,1]\n            \n        # 'meta' data\n        #inspired by https://www.kaggle.com/artgor/earthquakes-fe-more-features-and-samples\n        meta = []\n        idx_data_series = pd.Series(idx_data[:,0])\n        for windows in [10, 100, 1000]:\n            \n            x_roll_std = idx_data_series.rolling(windows).std().dropna().values\n            meta.append(np.quantile(x_roll_std, 0.05))    \n        \n        \n        return [torch.from_numpy(X), torch.from_numpy(np.array(meta,dtype='float32'))] , y\n        \n        ","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"68b2731123a139dee75463873b12784c94afc64d","trusted":true},"cell_type":"code","source":"\n\nclass EQRNN(nn.Module):\n    def __init__(self, input_size = 64,meta_size=0, fc1_size = 40, hidden_size=80,num_layers=1,bidirectional=True, dropout=0.5):\n        \n        super().__init__()\n        self.input_size = input_size\n        \n        self.meta_size = meta_size\n        \n        self.fc1_size = fc1_size\n        self.hidden_size = hidden_size\n        \n        self.fc1 = nn.Linear(input_size, fc1_size)\n        \n        self.bidirectional,self.num_layers = bidirectional,num_layers\n        if bidirectional: self.num_directions = 2\n        else: self.num_directions = 1\n            \n        self.dropout = dropout\n            \n        self.rnn = nn.GRU(fc1_size, hidden_size,bidirectional=self.bidirectional,batch_first=True)\n        \n       \n        self.layers2 = nn.Sequential(\n            \n            nn.Linear(self.num_directions * hidden_size +meta_size,128),    #32\n            nn.ReLU(),\n            nn.Dropout(self.dropout),   ##added\n            nn.Linear(128,128),\n            nn.ReLU(),\n            nn.Dropout(self.dropout),\n            nn.Linear(128,1),\n            \n            \n        )\n        \n        \n        #self.final_layers = nn.Sequential(\n            \n           # nn.Linear(256,128),\n           # nn.ReLU(),\n            #nn.Dropout(self.dropout),   ##added\n           # nn.Linear(128,128),\n           # nn.ReLU(),\n           # nn.Dropout(self.dropout),\n           # nn.Linear(128,1),\n            \n            \n        #)\n        self.final_layers = nn.Sequential(\n            \n            nn.Linear(4,16),\n            nn.ReLU(),\n            nn.Dropout(self.dropout),   ##added\n            nn.Linear(16,16),\n            nn.ReLU(),\n            nn.Dropout(self.dropout),\n            nn.Linear(16,1)\n        )\n            \n        \n        \n    def forward(self,input_seq, meta):\n    \n        \n        #cont_input_seq, cat_input_seq, meta_input_seq = input_seqs[0]\n        # Note: dataloader provides vector in batch first form, whereas recurent layers need to have batch_first=True specified\n        # hidden layers still have batch size in second position even with batch_first=True\n        batch_size = input_seq.size(0)\n        rnn_count = input_seq.size(1)\n        seq_len = input_seq.size(2)\n        \n        \n        \n        \n        input_seq = F.relu(self.fc1(input_seq))\n        \n        \n        # combine first two dimensions (batchsize and rnncount) so that our input is 3D\n        \n        input_seq = input_seq.view(batch_size*rnn_count,seq_len, -1)\n        \n        \n        #output of shape ( batch_size*rnn_count, seq_len, num_directions * hidden_size)\n        #h_n (not needed)\n        output, h_n = self.rnn(input_seq)#,h_0)\n        \n        \n        \n        \n        #SpatialDropout1D in TensorFlow\n        #https://discuss.pytorch.org/t/spatial-dropout-in-pytorch/21400/2\n        \n        # drop whole channels across all timesteps\n        \n        output = output.permute(0, 2, 1)    # convert to [batch,rnn_count, channels, time]\n        output = F.dropout2d(output, self.dropout, training=self.training)\n        output = output.permute(0,2, 1)   # back to original\n        \n        \n        #same as GlobalMaxPool1D in TensorFlow?\n        # take max over seq_len dimension\n        # could also try concatenation of Average and Max Pooling \n        \n        output = F.adaptive_max_pool1d(output.permute(0, 2, 1),output_size=1).permute(0, 2, 1)\n        output = F.dropout(output,self.dropout,training=self.training)\n        \n        #output2 = F.adaptive_avg_pool1d(output.permute(0,2,1),output_size=1).permute(0, 2, 1).view(batch_size,-1)\n        #output2 = F.dropout(output2,self.dropout,training=self.training)\n        #output = torch.cat((output1,output2),1)\n        \n        # now shape (bs, rnn_count, num_directions * hidden_size)\n        \n        # reseparate the first two dims\n        \n        output = output.view(batch_size,rnn_count, -1)\n        \n        \n        \n        #output = torch.cat((output,meta.view(batch_size,1,self.meta_size).repeat(1,rnn_count,1)), 2)\n        \n        #add meta data to output\n        #output = torch.cat((meta_input_seq.view(batch_size,-1),output),1)\n        \n        #shared layers for all rnn_count outputs\n        output = self.layers2(output)\n        \n        # now shape (bs, rnn_count, num_directions * hidden_size)\n        \n        output = output.view(batch_size,-1)\n        #output = self.final_layers(output)\n        \n        output = torch.mean(output,dim =1)\n        \n        \n        #output = torch.cat((output.view(batch_size,-1),meta), 1)\n        \n        #output = self.final_layers(output)\n        \n        return output\n        \n        ","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"59948f62a21802340fa878b9c07705718bc766be","trusted":true},"cell_type":"code","source":"val_max_index = int(len(float_data) * 0.2 )    #last index for validation\n\n\nval_ds = EQdataset(float_data,min_index =0, max_index = val_max_index)\ntrn_ds = EQdataset(float_data,min_index =val_max_index +1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"52d5495f5597b66517a137c5e110f440f1a19cac"},"cell_type":"code","source":"trn_ds[0][0][0].shape","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a4033c1ff86ec318338e3160cd95c36fd289ddf3","trusted":true},"cell_type":"code","source":"bs = 4096\nbs2 = 4096\nnum_workers = 4\n\n#train_dl = DataLoader(trn_ds, batch_size=bs,shuffle=True, num_workers=num_workers)\n#val_dl = DataLoader(val_ds, batch_size=bs2,shuffle=False, num_workers=num_workers)\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"93b75b3a36ded3373d822590e92b22445dd6f3a3","trusted":true},"cell_type":"code","source":"net = EQRNN(dropout=0.5)\nnet","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c3df90c85b8b32ee1e1f71426d0e76472ae329eb","trusted":true},"cell_type":"code","source":"\n# CUDA for PyTorch\nuse_cuda = torch.cuda.is_available()\ndevice = torch.device(\"cuda:0\" if use_cuda else \"cpu\")\n\n#criterion = nn.NLLLoss(weight=torch.from_numpy(weights).to(device))\n\ncriterion =  nn.L1Loss()\n#criterion = mywloss\n\ndatabunch = DataBunch.create(trn_ds,val_ds, device=device, bs =32)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"db30474c03e0ece274dcdb1c734966a723674b47","trusted":true},"cell_type":"code","source":"learn = Learner(databunch,net,callback_fns=[ShowGraph], loss_func = criterion)\nlearn.opt_func=AdamW\n# save the best model\nlearn.callbacks = [SaveModelCallback(learn,monitor='val_loss',mode='min')]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"83251a411a9d76becbc66accd7282efcab77467b"},"cell_type":"code","source":"#lr_find(learn)\n#learn.recorder.plot()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"63513e8362d515d946bb32a7d915c49337ba3adb","trusted":true},"cell_type":"code","source":"lr=1e-2\nwd = 1e-3\nlearn.fit(2,lr,wd=wd)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"14c7a2cf80ca2a2bc85e4a26252242920268d999"},"cell_type":"code","source":"trn_ds = EQdataset(float_data,min_index =val_max_index +1)\ndatabunch = DataBunch.create(trn_ds,val_ds, device=device, bs=32)\nlearn.data = databunch","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"40882752f516fefb838966d6fc4454b1a3457029"},"cell_type":"code","source":"learn.fit(2,lr,wd=wd)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a3acf33f341b0c2bbf9a7dd398a25ed8a5183541"},"cell_type":"code","source":"lr=1e-3\nwd = 1e-3\nlearn.fit(2,lr,wd=wd)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"d478595ebe65c9f5b8247536036f09bd95d537e4"},"cell_type":"code","source":"learn.fit(4,lr,wd=wd)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"e2cf439becbbd7fa0bfa6db39a78d12119d72b2b"},"cell_type":"code","source":"learn.save('EQ205')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e96a000492474776545feefbf5720a109feb33f7","trusted":false},"cell_type":"code","source":"#preds, target = learn.get_preds(ds_type=DatasetType.Valid)\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"f4e797751eae05c7d38b08f5ddc3829006ad218f"},"cell_type":"markdown","source":"## Submission"},{"metadata":{"trusted":false,"_uuid":"5d0ec81a39c682c882c29edf1ea58580cd9e59d7"},"cell_type":"code","source":"# Load submission file\nsubmission = pd.read_csv(PATH+'sample_submission.csv', index_col='seg_id', dtype={\"time_to_failure\": np.float32})\n\n# Load each test data\n\ntest= []\nfor i, seg_id in enumerate(tqdm(submission.index)):\n    \n    seg = pd.read_csv(PATH+ seg_id + '.csv',dtype={'acoustic_data': np.float32})\n    test_ds  = EQdataset(np.array(seg),is_test = True)\n    learn.data = DataBunch.create(trn_ds,val_ds,test_ds, device=device)  \n    preds, _ = learn.get_preds(ds_type=DatasetType.Test)\n    submission['time_to_failure'][i] =preds\n   ","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"1e421513b59b994c9e4ae7558de7f80a47c5ae63"},"cell_type":"code","source":"test_ds  = EQdataset(np.array(seg),is_test = True)\nlearn.data = DataBunch.create(trn_ds,val_ds,test_ds, device=device)  \npreds, _ = learn.get_preds(ds_type=DatasetType.Test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"bbceef951fffbb1f1bbc8428dbdfd30112310476"},"cell_type":"code","source":"submission['time_to_failure'].min()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"201fffc0575011324e6be27cad40e60ccff4fa2c","trusted":false},"cell_type":"code","source":"submission.to_csv('EQsubmission2.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"07c5ad83d59adb4a9f0c1de461aedbff4ebbb485"},"cell_type":"code","source":"float_data[:,1].min()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"276f8b056f212d9ad0340b79bbfd17bb27553828"},"cell_type":"code","source":"float_data[150000][1] - float_data[0][1]","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"fc1e83c388a432d10bdf98aa154a89be06e9548b"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.6.7"}},"nbformat":4,"nbformat_minor":1}