{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"%matplotlib inline\nimport warnings\nwarnings.filterwarnings('ignore')\nimport os\nimport gc\nimport sys\nimport glob\nimport time\nimport tqdm\nimport pickle\nimport numpy as np\nimport pandas as pd\nimport seaborn as sns\nimport matplotlib.pyplot as plt","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"DATA_DIR = '../input/'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bb773a25117ef0c50c245fe09fa32fe760ad8f6c"},"cell_type":"code","source":"train = pd.read_csv(DATA_DIR+'training_set.csv')\n# test = pd.read_csv(DATA_DIR+'test_set.csv')\ntrain_meta = pd.read_csv(DATA_DIR+'training_set_metadata.csv')\ntest_meta = pd.read_csv(DATA_DIR+'test_set_metadata.csv')\n\ntrain_meta.shape, test_meta.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0ee3482779ac68635e537827076465238377057d"},"cell_type":"code","source":"target = train_meta['target'].values.copy()\nlabels2weight = {x:1 for x in np.unique(target)}\ntrain_mask = train_meta['distmod'].isnull().values #galactic\ntest_mask  = test_meta['distmod'].isnull().values\n\nlabels2weight[15] = 2\nlabels2weight[64] = 2\nlabels2weight[99] = 2\n\nimport collections\ntarget2y = dict(map(reversed, enumerate(np.unique(target))))\ny2target = dict(enumerate(np.unique(target)))\ny = np.array(list(map(target2y.get, target)))\nclass_weight = np.array(list(map(lambda x: labels2weight[y2target[x]], sorted(np.unique(y)))))\ny_cntr = collections.Counter(y)\nwtable = np.array([y_cntr[i] for i in sorted(np.unique(y))]) / len(y)\n\nprint(sorted(np.unique(y)))\nprint(wtable)\nprint(class_weight)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d8d4594a319e4036d16b6f5f92472763356ec20c"},"cell_type":"code","source":"from sklearn.model_selection import StratifiedKFold as KFold\nnfolds = 5\nkf = KFold(n_splits=nfolds, shuffle=True, random_state=42)\ncv_folds = np.arange(len(target))\nfor i,_ in enumerate(kf.split(train_meta, target)):\n    cv_folds[_[1]] = i\nevals = pd.DataFrame()\nevals['object_id'] = train_meta['object_id']\nevals['target'] = target\nevals['cv_folds'] = cv_folds\nevals['is_gal'] = train_mask.astype('int')\nevals['is_ddf'] = train_meta['ddf'].values\nevals.to_csv('evals.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"05373387db4736c58cfe6963c88fc72106de599a"},"cell_type":"code","source":"remove_cols = ['hostgal_specz', 'target']\nfor c in remove_cols:\n    if c in train_meta.columns:\n        del train_meta[c]\n    if c in test_meta.columns:\n        del test_meta[c]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bdf61fa0200fbd457545daf61ecb338d8189af8b"},"cell_type":"code","source":"train_meta['distmod'].fillna(0, inplace=True)\ntest_meta['distmod'].fillna(0, inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f9f0c48da0add450503dbb4cbcc11ad09eb6fd60"},"cell_type":"code","source":"plt.figure(figsize=[12, 4], dpi=90)\nplt.subplot(1, 2, 1)\nsns.distplot(train['flux'].sample(frac=0.1))\nplt.title('flux raw distribution')\nplt.grid()\nplt.subplot(1, 2, 2)\nsns.distplot(train['flux_err'].sample(frac=0.1))\nplt.title('flux_err raw distribution')\nplt.grid();\n\nplt.figure(figsize=[12, 4], dpi=90)\nplt.subplot(1, 2, 1)\nsns.distplot(train['flux'].sample(frac=0.1).apply(lambda x: np.sign(x) * np.log(np.abs(x))))\nplt.title('flux symlog distribution')\nplt.grid()\nplt.subplot(1, 2, 2)\nsns.distplot(train['flux_err'].sample(frac=0.1).apply(lambda x: np.sign(x) * np.log(np.abs(x))))\nplt.title('flux_err symlog distribution')\nplt.grid();","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d1c36d0f90e0d332b07ac2344581c08737718f19"},"cell_type":"code","source":"for c in ['flux', 'flux_err']: \n    train[c] = train[c].apply(lambda x: np.sign(x) * np.log(np.abs(x)))\nc = 'mjd'\ntrain[c] = (train[c] - train[c].mean()) / train[c].std()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5e119474382fc6b05d1c113a9b3167993ed32d94"},"cell_type":"code","source":"%%time\ntrain = train.groupby(['object_id', 'passband']).apply(\n    lambda x: x.set_index(['object_id', 'passband']).to_dict(orient='list')\n)\ntrain.to_pickle('train_ts.pkl')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b2cc553e768e86801971c3e1965fd3856e69ec67"},"cell_type":"code","source":"print(train.loc[615, 0].keys())\ntrain.to_frame().head(12)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"503b8de48dd277a2075a2eea90b22f7a74418c88"},"cell_type":"code","source":"train_ids = train_meta['object_id'].values\ntrain_meta = train_meta.set_index('object_id')\ntrain_meta.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4b18d37fa261fe6eb0ccfff23b49ddb47ed6e7fb"},"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.autograd import Variable\nfrom torch.utils.data import Dataset, DataLoader\n\ndevice = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c8ed39220d133e7a121c72f4a986aef293a37bb8"},"cell_type":"code","source":"from torch.nn.utils.rnn import pack_padded_sequence, pad_packed_sequence, pad_sequence","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3a2552de353d5ffc1826156044890670b8388003"},"cell_type":"code","source":"meta_cols = []\nmeta_cols+= ['ra', 'decl', 'gal_l', 'gal_b']\nmeta_cols+= ['ddf', 'hostgal_photoz', 'hostgal_photoz_err', 'distmod', 'mwebv']\n\ndef get_xs_by_idx(idx, data):\n    xs = [pd.DataFrame(data[idx, pb]).values for pb in range(6)]\n    return xs\n\ndef get_meta_by_idx(idx, metadata):\n    return train_meta.loc[idx, meta_cols].values\n\ndef get_ts_mt_by_ids(ids, tsdata, metadata):\n    ts = [[] for pb in range(6)]\n    mt = []\n    for _id in ids:\n        xs = get_xs_by_idx(_id, tsdata)\n        for i,x in enumerate(xs):\n            ts[i].append(x)\n        mt.append(get_meta_by_idx(_id, metadata))\n    return ts, mt","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"006059ecaf44d3a70edd4459f2409674b0107c27"},"cell_type":"code","source":"valid_fold = 0\n\nnum_class = int(y.max()+1)\nnum_rnn_unit = 32\nnum_rnn_layer = 2\ndropout_rnn = 0.25\nnum_linear = 64\n\nlr = 0.0009\nweight_decay = 0\n\nepochs = 50\nbatch_size = 128","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ffa9e85fd9353b5a6bbc08e80a12b635e1f0bba1"},"cell_type":"code","source":"def softmax(x):\n    \"\"\"Compute softmax values for each sets of scores in x.\"\"\"\n    e_x = np.exp(x - np.max(x, 1).reshape(-1, 1))\n    return e_x / e_x.sum(axis=1).reshape(-1, 1)\n\ndef loss_fn(preds, target, num_class=num_class, class_weight=class_weight, wtable=wtable):\n    class_weight = torch.from_numpy(class_weight).type(preds.type())\n    wtable = torch.from_numpy(wtable).type(preds.type())\n    y_ohe = torch.zeros(\n        target.size(0), num_class, requires_grad=False\n    ).type(preds.type()).scatter(1, target.reshape(-1, 1), 1)\n    preds = F.softmax(preds, dim=1)\n    preds = torch.clamp(preds, 1e-15, 1-1e-15)\n    prod = torch.sum(torch.log(preds) * y_ohe, dim=0)\n    prod = prod * class_weight / wtable / target.size(0)\n    loss = -torch.sum(prod) / torch.sum(class_weight)\n    return loss\n\nclass EncoderRNN(nn.Module):\n    \n    def __init__(self, RNN=nn.GRU, use_cuda=torch.cuda.is_available()):\n        super(EncoderRNN, self).__init__()\n        self.use_cuda = use_cuda\n        self.rnn = RNN(\n            4, num_rnn_unit, num_rnn_layer, \n            batch_first=True, bidirectional=True, dropout=dropout_rnn\n        )\n        \n    def forward(self, li):\n        lens = [_.shape[0] for _ in li]\n        indices = np.argsort(lens)[::-1].tolist()\n        rev_ind = [indices.index(i) for i in range(len(indices))]\n        x = [torch.from_numpy(li[i]).float() for i in indices]\n        x = pad_sequence(x, batch_first=True)\n        x = Variable(x)\n        if self.use_cuda:\n            x = x.to(device)\n        input_lengths = [lens[i] for i in indices]\n        packed = pack_padded_sequence(x, input_lengths, batch_first=True)\n        ro,_ = self.rnn(packed)\n        ro,_ = pad_packed_sequence(ro, batch_first=True)\n        ro = torch.transpose(ro, 1, 2)\n        res = F.max_pool1d(ro, ro.size(2)).squeeze()\n        return res[rev_ind, :].contiguous()\n\nclass Net(nn.Module):\n    \n    def __init__(self, \n                 use_cuda=torch.cuda.is_available(), \n                 num_class=num_class):\n        super(Net, self).__init__()\n        self.use_cuda = use_cuda\n        for i in range(6):\n            self.add_module(f't{i}', EncoderRNN(nn.GRU))\n        self.clf_in = num_rnn_unit * 2 * 6 + len(meta_cols)\n        self.clf_ts = nn.Sequential(\n            nn.BatchNorm1d(self.clf_in),\n            nn.Linear(self.clf_in, num_linear),\n            nn.BatchNorm1d(num_linear),\n            nn.ReLU(inplace=True),\n            nn.Linear(num_linear, num_linear),\n            nn.BatchNorm1d(num_linear),\n            nn.ReLU(inplace=True),\n            nn.Linear(num_linear, num_class)\n        )\n                \n    def forward(self, ts, m):\n        m = torch.from_numpy(np.array(m)).float()\n        m = Variable(m)\n        if self.use_cuda:\n            m = m.to(device)\n        x = torch.cat([getattr(self, f't{i}')(ts[i]) for i in range(len(ts))] + [m], 1)\n        logit = self.clf_ts(x)\n        return logit","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9155867322678eea0bcbfbd3e4fc8bd57d6bffe9"},"cell_type":"code","source":"print('Checking...')\nindices = train_ids[:batch_size]\nbx, bm = get_ts_mt_by_ids(indices, train, train_meta)\nby = [y[idx] for idx in range(batch_size)]\n\nby = torch.LongTensor([y[idx] for idx in range(batch_size)])\nby = Variable(by).to(device)\nprint('by.type', by.type(), 'by.size', by.size(), 'bx length:', len(bx))\n\nmodel = Net()\nmodel = model.to(device)\npred = model(bx, bm)\nprint('pred.size', pred.size())\n\nloss = loss_fn(pred, by)\nloss.backward()\nprint('loss', loss)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ec752e96319d505962d7a32ab2cc5db84aa6d759"},"cell_type":"code","source":"trn_ids = train_ids[cv_folds!=valid_fold]\ntrn_lbl = y[cv_folds!=valid_fold]\nval_ids = train_ids[cv_folds==valid_fold]\nval_lbl = y[cv_folds==valid_fold]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1eb9281e6adfc55e3fc96adc5a177dee97e26350"},"cell_type":"code","source":"class Dset(Dataset):\n    \n    def __init__(self, data_ids, labels):\n        super(Dset, self).__init__()\n        self.data_ids = data_ids\n        self.labels = labels\n        self._len = len(labels)\n    \n    def __len__(self):\n        return self._len\n    \n    def __getitem__(self, index):\n        idx = self.data_ids[index]\n        y_i = self.labels[index]\n        return idx, y_i\n    \ndef collate_fn(batch, tsdata=train, metadata=train_meta):\n    indices = []\n    labels = []\n    for _ in batch:\n        indices.append(_[0])\n        labels.append(_[1])\n    bx, bm = get_ts_mt_by_ids(indices, tsdata, metadata)\n    by = torch.from_numpy(np.array(labels)).long()\n    return bx, bm, by\n\ntrain_steps = int(np.ceil(len(trn_ids) / batch_size))\nvalid_steps = int(np.ceil(len(val_ids) / batch_size))\n\ntrain_set = Dset(trn_ids, trn_lbl)\nvalid_set = Dset(val_ids, val_lbl)\ntrain_loader = DataLoader(train_set, batch_size, shuffle=True, collate_fn=collate_fn)\nvalid_loader = DataLoader(valid_set, batch_size, shuffle=False, collate_fn=collate_fn)\n\nprint('batch_size', batch_size, 'epochs', epochs)\nprint('train_steps', train_steps, 'valid_steps', valid_steps)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"15332d8189e965c400c98d5173b6b7de29f86c00"},"cell_type":"code","source":"torch.cuda.empty_cache()\nmodel = Net()\nmodel = model.to(device)\n\noptimizer = torch.optim.Adam(model.parameters(), lr=lr, weight_decay=weight_decay)\n\nloss_li = []\nval_loss_li = []\n\nval_loss = None\npred_val = None","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d63c72970fae68add9278ec673162b9163e775fc"},"cell_type":"code","source":"verbose = False","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e9ffff204e5054e4e2f7a06353731ab197ed90c6"},"cell_type":"code","source":"for epoch_i in range(epochs):\n    #print(f'Training epoch {epoch_i+1}')\n    \n    t0 = time.time()\n    gen = train_loader if not verbose else tqdm.tqdm_notebook(train_loader, total=train_steps)\n    losses = 0\n    \n    for bx,bm,by in gen:\n        model.train()\n        by = Variable(by).to(device)\n        pred = model(bx, bm)\n        optimizer.zero_grad()\n        loss = loss_fn(pred, by)\n        loss.backward()\n        optimizer.step()\n        losses += float(loss) * int(by.size(0))\n    losses = losses / len(train_loader.dataset.labels)\n    loss_li.append(losses)\n    \n    y_true = []\n    y_pred = []\n    losses = 0\n    for bx,bm,by in valid_loader:\n        model.eval()\n        y_true.extend(by.numpy())\n        by = Variable(by).to(device)\n        pred = model(bx, bm)\n        loss = loss_fn(pred, by)\n        y_pred.extend(pred.cpu().data.numpy())\n        losses += float(loss) * int(by.size(0))\n    \n    losses = losses / len(valid_loader.dataset.labels)\n    y_true = np.stack(y_true)\n    y_pred = np.stack(y_pred)\n    \n    star = ' '\n    if val_loss is None:\n        pred_val = y_pred.copy()\n        val_loss = losses\n        star = '*'\n    elif losses < val_loss:\n        pred_val = y_pred.copy()\n        val_loss = losses\n        star = '*'\n    else:\n        pass\n    val_loss_li.append(losses)\n    toc = time.time() - t0\n    print(f'Epoch {epoch_i+1:>2} | valid loss {val_loss_li[-1]:.4f}{star} in {toc:.2f} sec')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"64d84053da8a5047829a884f6134fd0dc78d79a0"},"cell_type":"code","source":"plt.figure(figsize=[6, 4], dpi=90)\nplt.plot(loss_li)\nplt.plot(val_loss_li)\nplt.grid()\nplt.legend(['train', 'valid']);","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b38f7a507352587bba5493cf7f3be0992651ffe9"},"cell_type":"code","source":"'''\nhttps://scikit-learn.org/stable/auto_examples/model_selection/plot_confusion_matrix.html\n'''\nfrom sklearn.metrics import confusion_matrix\nimport itertools\ndef plot_confusion_matrix(cm, classes,\n                          normalize=True,\n                          title='Confusion matrix',\n                          cmap=plt.cm.Blues):\n    \"\"\"\n    This function prints and plots the confusion matrix.\n    Normalization can be applied by setting `normalize=True`.\n    \"\"\"\n    if normalize:\n        cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]\n        print(\"Normalized confusion matrix\")\n    else:\n        print('Confusion matrix, without normalization')\n    #print(cm)\n    plt.figure(figsize=[10, 8], dpi=90)\n    plt.imshow(cm, interpolation='nearest', cmap=cmap)\n    plt.title(title)\n    plt.colorbar()\n    tick_marks = np.arange(len(classes))\n    plt.xticks(tick_marks, classes, rotation=45)\n    plt.yticks(tick_marks, classes)\n\n    fmt = '.2f' if normalize else 'd'\n    thresh = cm.max() / 2.\n    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):\n        plt.text(j, i, format(cm[i, j], fmt),\n                 horizontalalignment=\"center\",\n                 color=\"white\" if cm[i, j] > thresh else \"black\")\n\n    plt.ylabel('True label')\n    plt.xlabel('Predicted label')\n    plt.tight_layout();\n    \npred_val_lbl = np.argmax(softmax(pred_val), axis=1)\nclasses = list(target2y.keys())\ncm = confusion_matrix(y_true, pred_val_lbl)\nplot_confusion_matrix(cm, classes)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b3f64298fff671d216f18b55220817045d5a72b3"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}