{"cells":[{"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","trusted":true},"cell_type":"code","source":"%matplotlib inline\nimport warnings\nwarnings.filterwarnings('ignore')\nimport os\nimport gc\nimport sys\nimport glob\nimport time\nimport tqdm\nimport pickle\nimport numpy as np\nimport pandas as pd\nimport seaborn as sns\nimport matplotlib.pyplot as plt","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"DATA_DIR = f'../input/'\nCACHE_DIR = f'./'\nif not os.path.isdir(CACHE_DIR):\n    os.makedirs(CACHE_DIR)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"bb773a25117ef0c50c245fe09fa32fe760ad8f6c","trusted":true},"cell_type":"code","source":"train = pd.read_csv(DATA_DIR+'training_set.csv')\n# test = pd.read_csv(DATA_DIR+'test_set.csv')\ntrain_meta = pd.read_csv(DATA_DIR+'training_set_metadata.csv')\ntest_meta = pd.read_csv(DATA_DIR+'test_set_metadata.csv')\n\ntrain_meta.shape, test_meta.shape","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"0ee3482779ac68635e537827076465238377057d","trusted":true},"cell_type":"code","source":"target = train_meta['target'].values.copy()\nlabels2weight = {x:1 for x in np.unique(target)}\ntrain_mask = train_meta['distmod'].isnull().values #galactic\ntest_mask  = test_meta['distmod'].isnull().values\n\nlabels2weight[15] = 2\nlabels2weight[64] = 2\nlabels2weight[99] = 2\n\nimport collections\ntarget2y = dict(map(reversed, enumerate(np.unique(target))))\ny2target = dict(enumerate(np.unique(target)))\ny = np.array(list(map(target2y.get, target)))\nclass_weight = np.array(list(map(lambda x: labels2weight[y2target[x]], sorted(np.unique(y)))))\ny_cntr = collections.Counter(y)\nwtable = np.array([y_cntr[i] for i in sorted(np.unique(y))]) / len(y)\n\nprint(sorted(np.unique(y)))\nprint(wtable)\nprint(class_weight)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d8d4594a319e4036d16b6f5f92472763356ec20c","trusted":true},"cell_type":"code","source":"from sklearn.model_selection import StratifiedKFold as KFold\nnfolds = 5\nkf = KFold(n_splits=nfolds, shuffle=True, random_state=42)\ncv_folds = np.arange(len(target))\nfor i,_ in enumerate(kf.split(train_meta, target)):\n    cv_folds[_[1]] = i\nevals = pd.DataFrame()\nevals['object_id'] = train_meta['object_id']\nevals['target'] = target\nevals['cv_folds'] = cv_folds\nevals['is_gal'] = train_mask.astype('int')\nevals['is_ddf'] = train_meta['ddf'].values\n# evals.to_csv('evals.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"05373387db4736c58cfe6963c88fc72106de599a","trusted":true},"cell_type":"code","source":"remove_cols = ['hostgal_specz', 'target']\nfor c in remove_cols:\n    if c in train_meta.columns:\n        del train_meta[c]\n    if c in test_meta.columns:\n        del test_meta[c]","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"bdf61fa0200fbd457545daf61ecb338d8189af8b","trusted":true},"cell_type":"code","source":"train_meta['distmod'].fillna(0, inplace=True)\ntest_meta['distmod'].fillna(0, inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d1c36d0f90e0d332b07ac2344581c08737718f19","trusted":true},"cell_type":"code","source":"# for c in ['flux', 'flux_err']: \n#     train[c] = train[c].apply(lambda x: np.sign(x) * np.log(np.abs(x)))\nc = 'mjd'\ntrain[c] = (train[c] - train[c].mean()) / train[c].std()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"5deed441525d49b5bcd9b8091fc656d8b4f48f3c","trusted":true},"cell_type":"code","source":"train = train[['object_id', 'passband', 'mjd', 'flux', 'flux_err', 'detected']]","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"5e119474382fc6b05d1c113a9b3167993ed32d94","trusted":true},"cell_type":"code","source":"%%time\ntrain = train.groupby(['object_id']).apply(\n    lambda x: x.set_index(['object_id']).to_dict(orient='list')\n)\n# train.to_pickle('train_ts.pkl')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"b2cc553e768e86801971c3e1965fd3856e69ec67","trusted":true},"cell_type":"code","source":"print(train.loc[615].keys())\ntrain.to_frame().head(12)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"503b8de48dd277a2075a2eea90b22f7a74418c88","trusted":true},"cell_type":"code","source":"train_ids = train_meta['object_id'].values\ntrain_meta = train_meta.set_index('object_id')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"4b18d37fa261fe6eb0ccfff23b49ddb47ed6e7fb","trusted":true},"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.autograd import Variable\nfrom torch.utils.data import Dataset, DataLoader\n\ndevice = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c8ed39220d133e7a121c72f4a986aef293a37bb8","trusted":true},"cell_type":"code","source":"from torch.nn.utils.rnn import pack_padded_sequence, pad_packed_sequence, pad_sequence","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"3a2552de353d5ffc1826156044890670b8388003","trusted":true},"cell_type":"code","source":"meta_cols = []\n#meta_cols+= ['ra', 'decl', 'gal_l', 'gal_b']\nmeta_cols+= ['ddf', 'hostgal_photoz', 'hostgal_photoz_err', 'distmod', 'mwebv']\n\ndef get_xs_by_idx(idx, data):\n    xs = pd.DataFrame(data[idx]).values\n    return xs\n\ndef get_meta_by_idx(idx, metadata):\n    return metadata.loc[idx, meta_cols].values\n\ndef get_ts_mt_by_ids(ids, tsdata, metadata):\n    ts = []\n    mt = []\n    for _id in ids:\n        ts.append(get_xs_by_idx(_id, tsdata))\n        mt.append(get_meta_by_idx(_id, metadata))\n    return ts, mt","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"006059ecaf44d3a70edd4459f2409674b0107c27","trusted":true},"cell_type":"code","source":"num_class = int(y.max()+1)\nnum_embed_dim = 16\nnum_rnn_unit = 32\nnum_rnn_layer = 2\ndropout_rnn = 0.5\nnum_linear = 64\nRNN = nn.GRU\n\nlr = 0.0009\nweight_decay = 1e-5\nlr_decay_ratio = 0.5\nlr_decay_interval = 30\n\nepochs = 150\nbatch_size = 128","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ffa9e85fd9353b5a6bbc08e80a12b635e1f0bba1","trusted":true},"cell_type":"code","source":"def softmax(x):\n    \"\"\"Compute softmax values for each sets of scores in x.\"\"\"\n    e_x = np.exp(x - np.max(x, 1).reshape(-1, 1))\n    return e_x / e_x.sum(axis=1).reshape(-1, 1)\n\ndef loss_fn(preds, target, num_class=num_class, class_weight=class_weight, wtable=wtable):\n    class_weight = torch.from_numpy(class_weight).type(preds.type())\n    wtable = torch.from_numpy(wtable).type(preds.type())\n    y_ohe = torch.zeros(\n        target.size(0), num_class, requires_grad=False\n    ).type(preds.type()).scatter(1, target.reshape(-1, 1), 1)\n    preds = F.softmax(preds, dim=1)\n    preds = torch.clamp(preds, 1e-15, 1-1e-15)\n    prod = torch.sum(torch.log(preds) * y_ohe, dim=0)\n    prod = prod * class_weight / wtable / target.size(0)\n    loss = -torch.sum(prod) / torch.sum(class_weight)\n    return loss\n\nclass EncoderRNN(nn.Module):\n    \n    def __init__(self, RNN=nn.GRU, use_cuda=torch.cuda.is_available()):\n        super(EncoderRNN, self).__init__()\n        self.use_cuda = use_cuda\n        self.embed = nn.Embedding(6, num_embed_dim)\n        self.rnn = RNN(\n            4+num_embed_dim, num_rnn_unit, num_rnn_layer, \n            batch_first=True, bidirectional=True, dropout=dropout_rnn\n        )\n        \n    def forward(self, li):\n        lens = [_.shape[0] for _ in li]\n        indices = np.argsort(lens)[::-1].tolist()\n        rev_ind = [indices.index(i) for i in range(len(indices))]\n        x = [torch.from_numpy(li[i]).float() for i in indices]\n        x = pad_sequence(x, batch_first=True)\n        x = Variable(x)\n        if self.use_cuda:\n            x = x.to(device)\n        emb = self.embed(x[:, :, 0].long())\n        #print(x[:, :, 1:].size(), emb.size())\n        x = torch.cat([x[:, :, 1:].contiguous(), emb], -1)\n        input_lengths = [lens[i] for i in indices]\n        packed = pack_padded_sequence(x, input_lengths, batch_first=True)\n        ro,_ = self.rnn(packed)\n        ro,_ = pad_packed_sequence(ro, batch_first=True)\n        ro = torch.transpose(ro, 1, 2)\n        res = F.max_pool1d(ro, ro.size(2)).squeeze()\n        return res[rev_ind, :].contiguous()\n\nclass Net(nn.Module):\n    \n    def __init__(self, \n                 use_cuda=torch.cuda.is_available(), \n                 num_class=num_class):\n        super(Net, self).__init__()\n        self.use_cuda = use_cuda\n        #for i in range(6):\n        #    self.add_module(f't{i}', EncoderRNN(nn.LSTM))\n        self.add_module(f'ts', EncoderRNN(RNN, use_cuda=use_cuda))\n        self.clf_in = num_rnn_unit * 2 + len(meta_cols)\n        self.clf_ts = nn.Sequential(\n            nn.BatchNorm1d(self.clf_in),\n            nn.Linear(self.clf_in, num_linear),\n            nn.BatchNorm1d(num_linear),\n            nn.ReLU(inplace=True),\n            nn.Linear(num_linear, num_linear),\n            nn.BatchNorm1d(num_linear),\n            nn.ReLU(inplace=True),\n            nn.Linear(num_linear, num_class)\n        )\n                \n    def forward(self, ts, m):\n        m = torch.from_numpy(np.array(m)).float()\n        m = Variable(m)\n        if self.use_cuda:\n            m = m.to(device)\n        #x = torch.cat([getattr(self, f't{i}')(ts[i]) for i in range(len(ts))] + [m], 1)\n        x = torch.cat([getattr(self, f'ts')(ts), m], 1)\n        logit = self.clf_ts(x)\n        return logit","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"edd635f8239007acaadfd83f4691e0248effa024","trusted":true},"cell_type":"code","source":"torch.cuda.empty_cache()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"9155867322678eea0bcbfbd3e4fc8bd57d6bffe9","trusted":true},"cell_type":"code","source":"print('Checking...')\nindices = train_ids[:batch_size]\nbx, bm = get_ts_mt_by_ids(indices, train, train_meta)\nby = [y[idx] for idx in range(batch_size)]\n\nby = torch.LongTensor([y[idx] for idx in range(batch_size)])\nby = Variable(by)#.to(device)\nprint('by.type', by.type(), 'by.size', by.size(), 'bx length:', len(bx))\n\nmodel = Net(use_cuda=False)\n#model = model.to(device)\npred = model(bx, bm)\nprint('pred.size', pred.size())\n\nloss = loss_fn(pred, by)\nloss.backward()\nprint('loss', loss)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"22c23e3bdf93c6916c6929041b205a7c1dd74592","trusted":true},"cell_type":"code","source":"'''\nhttps://scikit-learn.org/stable/auto_examples/model_selection/plot_confusion_matrix.html\n'''\nfrom sklearn.metrics import confusion_matrix\nimport itertools\ndef plot_confusion_matrix(cm, classes,\n                          normalize=True,\n                          title='Confusion matrix',\n                          cmap=plt.cm.Blues):\n    \"\"\"\n    This function prints and plots the confusion matrix.\n    Normalization can be applied by setting `normalize=True`.\n    \"\"\"\n    if normalize:\n        cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]\n        print(\"Normalized confusion matrix\")\n    else:\n        print('Confusion matrix, without normalization')\n    #print(cm)\n    plt.figure(figsize=[10, 8], dpi=90)\n    plt.imshow(cm, interpolation='nearest', cmap=cmap)\n    plt.title(title)\n    plt.colorbar()\n    tick_marks = np.arange(len(classes))\n    plt.xticks(tick_marks, classes, rotation=45)\n    plt.yticks(tick_marks, classes)\n\n    fmt = '.2f' if normalize else 'd'\n    thresh = cm.max() / 2.\n    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):\n        plt.text(j, i, format(cm[i, j], fmt),\n                 horizontalalignment=\"center\",\n                 color=\"white\" if cm[i, j] > thresh else \"black\")\n\n    plt.ylabel('True label')\n    plt.xlabel('Predicted label')\n    plt.tight_layout()\n    plt.show();","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ef3a6ee6ffc863f53660de12e1b9dc530e2a17c4","trusted":true},"cell_type":"code","source":"class Dset(Dataset):\n    \n    def __init__(self, data_ids, labels):\n        super(Dset, self).__init__()\n        self.data_ids = data_ids\n        self.labels = labels\n        self._len = len(labels)\n    \n    def __len__(self):\n        return self._len\n    \n    def __getitem__(self, index):\n        idx = self.data_ids[index]\n        y_i = self.labels[index]\n        return idx, y_i\n    \ndef collate_fn(batch, tsdata=train, metadata=train_meta):\n    indices = []\n    labels = []\n    for _ in batch:\n        indices.append(_[0])\n        labels.append(_[1])\n    bx, bm = get_ts_mt_by_ids(indices, tsdata, metadata)\n    by = torch.from_numpy(np.array(labels)).long()\n    return bx, bm, by","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"1eb9281e6adfc55e3fc96adc5a177dee97e26350","trusted":true},"cell_type":"code","source":"def train_by_fold(valid_fold):\n    model_path = f'{CACHE_DIR}model_{valid_fold}.pth'\n\n    trn_ids = train_ids[cv_folds!=valid_fold]\n    trn_lbl = y[cv_folds!=valid_fold]\n    val_ids = train_ids[cv_folds==valid_fold]\n    val_lbl = y[cv_folds==valid_fold]\n\n    train_steps = int(np.ceil(len(trn_ids) / batch_size))\n    valid_steps = int(np.ceil(len(val_ids) / batch_size))\n\n    train_set = Dset(trn_ids, trn_lbl)\n    valid_set = Dset(val_ids, val_lbl)\n    train_loader = DataLoader(train_set, batch_size, shuffle=True, collate_fn=collate_fn)\n    valid_loader = DataLoader(valid_set, batch_size, shuffle=False, collate_fn=collate_fn)\n\n    #print('valid_fold', valid_fold+1)\n    #print('batch_size', batch_size, 'epochs', epochs)\n    #print('train_steps', train_steps, 'valid_steps', valid_steps)\n\n    torch.cuda.empty_cache()\n    model = Net()\n    model = model.to(device)\n\n    optimizer = torch.optim.Adam(model.parameters(), lr=lr, weight_decay=weight_decay)\n    \n    failed_count = 0\n    restart = 0\n    loss_li = []\n    val_loss_li = []\n    val_loss = None\n    pred_val = None\n    verbose = False\n    \n    t00 = time.time()\n    for epoch_i in range(epochs):\n\n        t0 = time.time()\n        gen = train_loader if not verbose else tqdm.tqdm_notebook(train_loader, total=train_steps)\n        losses = 0\n\n        for bx,bm,by in gen:\n            model.train()\n            by = Variable(by).to(device)\n            pred = model(bx, bm)\n            optimizer.zero_grad()\n            loss = loss_fn(pred, by)\n            loss.backward()\n            optimizer.step()\n            losses += float(loss) * int(by.size(0))\n        losses = losses / len(train_loader.dataset.labels)\n        loss_li.append(losses)\n\n        y_true = []\n        y_pred = []\n        losses = 0\n        for bx,bm,by in valid_loader:\n            model.eval()\n            y_true.extend(by.numpy())\n            by = Variable(by).to(device)\n            pred = model(bx, bm)\n            loss = loss_fn(pred, by)\n            y_pred.extend(pred.cpu().data.numpy())\n            losses += float(loss) * int(by.size(0))\n\n        losses = losses / len(valid_loader.dataset.labels)\n        y_true = np.stack(y_true)\n        y_pred = np.stack(y_pred)\n\n        star = ' '\n        if val_loss is None or losses < val_loss:\n            star = '*'\n            pred_val = y_pred.copy()\n            val_loss = losses\n            torch.save(model.state_dict(), model_path)\n            failed_count = 0\n        else:\n            failed_count += 1\n        if failed_count==lr_decay_interval:\n            star = 'H'\n            restart += 1\n            optimizer = torch.optim.Adam(\n                model.parameters(), lr=lr*(lr_decay_ratio**restart), weight_decay=weight_decay\n            )\n            failed_count = 0\n\n        val_loss_li.append(losses)\n        toc = time.time() - t0\n        print(f'Epoch {epoch_i+1:>2} | valid loss {val_loss_li[-1]:.4f}{star} in {toc:.2f} sec')\n\n    #np.save(f'pred_val_{valid_fold}_{val_loss:.4f}.npy', pred_val)\n    print(f'valid_fold {valid_fold+1} loss: {val_loss:.4f} in {time.time() - t00:.2f} sec')\n    \n    plt.figure(figsize=[6, 4], dpi=90)\n    plt.plot(loss_li)\n    plt.plot(val_loss_li)\n    plt.grid()\n    plt.legend(['train', 'valid']);\n    plt.show()\n\n    #pred_val_lbl = np.argmax(softmax(pred_val), axis=1)\n    #classes = list(target2y.keys())\n    #cm = confusion_matrix(y_true, pred_val_lbl)\n    #plot_confusion_matrix(cm, classes)\n    \n    return pred_val, val_loss","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"49b584f6d7e45ad727838f5f53aebbc6fb38dbbb","trusted":true},"cell_type":"code","source":"preds = np.zeros((len(train_ids), num_class))","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-output":true,"_uuid":"b3f64298fff671d216f18b55220817045d5a72b3","scrolled":false,"trusted":true},"cell_type":"code","source":"for valid_fold in range(nfolds):\n    pred_val, val_loss = train_by_fold(valid_fold)\n    preds[cv_folds==valid_fold, :] = pred_val","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"31d15ffac24d24194de58abf94bebde4b9b97ad4","trusted":true},"cell_type":"code","source":"oof_score = float(loss_fn(torch.from_numpy(preds), torch.from_numpy(y)))\nprint(f'oof_score {oof_score:.6f}')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"7cbd634586f60d2eb2e42de6479825b174a738ac","trusted":true},"cell_type":"code","source":"np.save(f'{CACHE_DIR}valid_pred_{oof_score:.4f}.npy', preds)\nprint(f'{CACHE_DIR}valid_pred_{oof_score:.4f}.npy saved')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"70570195262a9e9ef5be2c149a9996976aeb9418","trusted":true},"cell_type":"code","source":"pred_val_lbl = np.argmax(softmax(preds), axis=1)\nclasses = list(target2y.keys())\ncm = confusion_matrix(y, pred_val_lbl)\nplot_confusion_matrix(cm, classes)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"dc0fc13c58a7b614082bb65a4b5e7e1630928026"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.6.6"}},"nbformat":4,"nbformat_minor":1}