{"cells":[{"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","trusted":true,"collapsed":true},"cell_type":"code","source":"%matplotlib inline\nimport os\nimport gc\nimport numpy as np\nimport pandas as pd\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nfrom tqdm import tqdm\n\nDATA_DIR = '../input/avito-demand-prediction/'\ntextdata_path = '../input/adp-prepare-kfold-text/textdata.csv'\n#EMB_PATH = '../input/fasttest-common-crawl-russian/cc.ru.300.vec'\nEMB_PATH = '../input/fasttext-russian-2m/wiki.ru.vec'\ntarget_col = 'deal_probability'\nos.listdir(DATA_DIR)","execution_count":1,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"3325edcb8dcb35e9a374c5baf647be6631fcf21f"},"cell_type":"code","source":"usecols = ['parent_category_name', 'category_name', 'param_1', 'param_2', 'param_3', 'title']\ntext = pd.concat([\n    pd.read_csv(DATA_DIR+'train.csv', usecols=usecols),\n    pd.read_csv(DATA_DIR+'test.csv', usecols=usecols)\n], ignore_index=True)\ntext.fillna('unknown', inplace=True)","execution_count":2,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d4c100372eb7dddb786247b114746f29d59b59e2","collapsed":true},"cell_type":"code","source":"text['context'] = ''\nfor c in ['parent_category_name', 'category_name', 'param_1', 'param_2', 'param_3']:\n    text['context'] += text[c].str.lower() + ' '\ntext['text'] = text['title'].str.lower()","execution_count":3,"outputs":[]},{"metadata":{"_cell_guid":"1f06f14b-2ae8-40d6-a169-5ee5de48d2c8","_uuid":"4e84f6df0eda8673c33028dcc870341acea62d43","trusted":true,"collapsed":true},"cell_type":"code","source":"dtype = {\n    'context': 'object',    \n    'text': 'object',\n    'eval_set': 'int8',\n    'label': 'float64'\n}\ndf = pd.read_csv(textdata_path, usecols=['eval_set', 'label'], dtype=dtype)\ndf = pd.concat([text[['context', 'text']], df], axis=1)\ndel text; gc.collect();\ndf.head()","execution_count":4,"outputs":[]},{"metadata":{"_cell_guid":"05112d53-43f3-4e30-9cc3-9e4f8322ed02","_uuid":"39893b1597375642e59df2c8cdc815e606ac1125","collapsed":true,"trusted":true},"cell_type":"code","source":"max_features = 30000\nmaxlen = 30#120\nembed_size = 300","execution_count":5,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true,"collapsed":true},"cell_type":"code","source":"%%time\nimport keras\nfrom keras.preprocessing import text, sequence\nprint('tokenizing...')\ntokenizer = text.Tokenizer(num_words=max_features)\ntokenizer.fit_on_texts(df['context'].values.tolist()+df['text'].values.tolist())","execution_count":6,"outputs":[]},{"metadata":{"_cell_guid":"9aa7335e-aa1c-473d-bfb3-81a026d1f85a","_uuid":"ac577cda60e65df38f2a9da87be4dd3b613324bd","trusted":true,"collapsed":true},"cell_type":"code","source":"%%time\ndef get_coefs(word, *arr, tokenizer=None):\n    if tokenizer is None:\n        return word, np.asarray(arr, dtype='float32')\n    else:\n        if word not in tokenizer.word_index:\n            return None\n        else:\n            return word, np.asarray(arr, dtype='float32')\nmax_features = min(max_features, len(tokenizer.word_index))\nembedding_matrix = np.zeros((max_features, embed_size))\nfor o in tqdm(open(EMB_PATH), desc='getting embeddings'):\n    res = get_coefs(*o.rstrip().rsplit(' '), tokenizer=tokenizer)\n    if res is not None:\n        idx = tokenizer.word_index[res[0]]\n        if idx < max_features:\n            embedding_matrix[idx] = res[1]\ngc.collect()","execution_count":7,"outputs":[]},{"metadata":{"_cell_guid":"52aa9c63-1009-4d87-921d-43e6f262ea16","_uuid":"b56d1c89e6807c0cc410a30b8c54ebfb0e7aa9d8","trusted":true,"collapsed":true},"cell_type":"code","source":"%%time\ndef fill_rand_norm(embedding_matrix):\n    mask = embedding_matrix.sum(axis=1)==0\n    zero_ratio = (mask).sum() / embedding_matrix.shape[0]\n    print('zero ratio:', zero_ratio)\n    emb_zero_shape = ((mask).sum(), embedding_matrix.shape[1])\n    emb_non_zero_mean = embedding_matrix[~mask].mean()\n    emb_non_zero_std = embedding_matrix[~mask].std()\n    embedding_matrix[mask] = np.random.normal(emb_non_zero_mean, \n                                              emb_non_zero_std, \n                                              emb_zero_shape)\n    return embedding_matrix\nembedding_matrix = fill_rand_norm(embedding_matrix)","execution_count":8,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a329fec614d3b5fdd7cc124ce002aa807c5977f8","collapsed":true},"cell_type":"code","source":"np.unique(df['context'].apply(lambda x:len(x.split())).values, return_counts=True)","execution_count":9,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"66459f8517be5e071b843518b05b85d76c4f731a","collapsed":true},"cell_type":"code","source":"np.unique(df['text'].apply(lambda x:len(x.split())).values, return_counts=True)","execution_count":10,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"df0f7619c85b5f6fa7db5ff174598af84a3bebfd"},"cell_type":"code","source":"max_context_len = 15\nmax_text_len = 12","execution_count":11,"outputs":[]},{"metadata":{"_cell_guid":"4af0e186-9123-409e-971e-22cf39cff6e6","_uuid":"8e0767f90ab0df6c326a99dffb67ea43ca131cb5","trusted":true,"collapsed":true},"cell_type":"code","source":"%%time\ncontext = df['context'].values\ntext = df['text'].values\neval_sets = df['eval_set'].values\nlabels = df['label'].values\ntrain_num = (df['label']<2).sum()\ndel df; gc.collect()\ncontext = tokenizer.texts_to_sequences(context)\ncontext = sequence.pad_sequences(context, maxlen=max_context_len)\ntext = tokenizer.texts_to_sequences(text)\ntext = sequence.pad_sequences(text, maxlen=max_text_len)\ndel tokenizer; gc.collect();","execution_count":12,"outputs":[]},{"metadata":{"_cell_guid":"c34fe2cf-d178-4956-976d-e9f03e29f0a5","_uuid":"a58901e3caf16571a3fc4e3e9c2a8578e2023683","trusted":true,"collapsed":true},"cell_type":"code","source":"#print((text!=0).sum(axis=0)/text.shape[0])\nf = plt.subplot(1, 2, 1)\nplt.plot((context!=0).sum(axis=0)/context.shape[0])\nplt.title('Context Non-zero Ratio')\nplt.axis([0., max_context_len-1, 0., 1.])\nf = plt.subplot(1, 2, 2)\nplt.plot((text!=0).sum(axis=0)/text.shape[0])\nplt.title('Text Non-zero Ratio')\nplt.axis([0., max_text_len-1, 0., 1.])\nplt.show()","execution_count":13,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"93228ac29654f033884080a55e51ddc2be81b9af","collapsed":true},"cell_type":"code","source":"text = np.hstack((context, text))\ndel context; gc.collect()","execution_count":14,"outputs":[]},{"metadata":{"_cell_guid":"89becb24-3ed8-4ef6-88e7-3e24749d1c2c","_uuid":"9ae556b31e2cfbccd9894093fc5a3ca20613b970","collapsed":true,"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"from time import time\nimport pickle\nimport torch\nfrom torch.autograd import Variable\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.backends.cudnn\nfrom torch.utils.data.dataset import Dataset\nfrom torch.utils.data import DataLoader\n\nclass DataBuilder(Dataset):\n    def __init__(self, X, y=None):\n        self.X = X\n        self.y = y\n        self.size = len(X)\n    def __getitem__(self, index):\n        X_i = torch.LongTensor(self.X[index].tolist())\n        if self.y is not None:\n            y_i = torch.FloatTensor([float(self.y[index])])\n        else:\n            y_i = torch.FloatTensor([-1])\n        return X_i, y_i\n    def __len__(self):\n        return self.size\n    \nclass BaseModel(nn.Module):\n    def __init__(self):\n        super(BaseModel, self).__init__()\n        \n    def init_args(self):\n        raise NotImplementedError('subclass must implement this')\n    \n    def check_args(self):\n        default_values = {\n            'optimizer_type': 'adam',\n            'save_sub_dir': '.', \n            #'valid_scores_topk': [-1, -1, -1],\n            'log_interval': 50,\n            'valid_interval': 200,\n            'patience': 3,\n            'valid_interval_re_decay': 0.7,\n            'valid_interval_min': 50,\n            'lr_re_decay': 0.5,\n            'batch_size': 32,\n            'lr': 0.001,\n            'weight_decay': 0.0,\n            'n_epochs': 2,\n        }\n        args = self.args\n        if 'greater_is_better' not in args.__dict__ or \\\n            args.greater_is_better not in [True, False]:\n            raise NotImplementedError('args.greater_is_better must be in [True, False]')\n        if args.greater_is_better:\n            default_values['valid_scores_topk'] = [-999]*default_values['patience']\n        else:\n            default_values['valid_scores_topk'] = [999]*default_values['patience']\n        for k, v in default_values.items():\n            if k not in args.__dict__:\n                args.__dict__[k] = v\n                print('Fill in arg %s with default value'%(k), v)\n    \n    def forward(self, x):\n        raise NotImplementedError('subclass must implement this')\n    \n    def get_optimizer_caller(self, optimizer_type):\n        choice_d = {'sgd' : torch.optim.SGD,\n                    'adam': torch.optim.Adam,\n                    'rmsp': torch.optim.RMSprop,\n                    'adag': torch.optim.Adagrad}\n        assert optimizer_type in choice_d\n        return choice_d[optimizer_type]\n    \n    def logit2label(self, logits, onehot=False):\n        logits_ = np.array(logits)\n        if onehot:\n            return (logits_/logits_.max()).astype(np.int8).astype(np.float32)\n        else:\n            return np.argmax(logits_, axis=len(logits_.shape)-1)\n            \n    def save(self, path):\n        torch.save(self.state_dict(), path)\n        print('model saved at', path)\n        \n    def load(self, path):\n        self.load_state_dict(torch.load(path))\n        print('model loaded from', path)\n    \n    def save_args_dict(self, args, path):\n        with open(path, 'wb') as f:\n            pickle.dump(args.__dict__, f)\n        print('args_dict saved at', path)\n                    \n    def load_args_dict(self, path):\n        with open(path, 'rb') as f:\n            args_dict = pickle.load(f)\n        print('args_dict loaded from', path)\n        print('returned')\n        return args_dict\n\n    def save_finished_args(self, args):\n        args.finished = True\n        args_fin_path = os.path.join(args.save_sub_dir, 'args.pkl')\n        self.save_args_dict(args, args_fin_path)\n        print('Finished! Topk:', args.valid_scores_topk)\n        return args  \n\n    def fit_batch(self, \n                  X_batch, y_batch, weight=None):\n        model = self.train()\n        x = Variable(X_batch)\n        y = Variable(y_batch).view(-1)\n        if self.use_cuda:\n            x, y = x.cuda(), y.cuda()\n        self.optimizer.zero_grad()\n        outputs = model(x)\n        pred = outputs\n        if self.use_cuda:\n            pred = pred.cpu()\n        loss = self.criterion(outputs, y, weight=weight)\n        loss.backward()\n        self.optimizer.step()\n        return loss.data[0], pred.data.numpy()\n    \n    def valid_batch(self, \n                    X_batch, y_batch, weight=None):\n        model = self.eval()\n        x = Variable(X_batch, volatile=True)\n        y = Variable(y_batch).view(-1)\n        if self.use_cuda:\n            x, y = x.cuda(), y.cuda()\n        outputs = model(x)\n        pred = outputs\n        if self.use_cuda:\n            pred = pred.cpu()\n        loss = self.criterion(outputs, y, weight=weight)\n        return loss.data[0], pred.data.numpy()\n\n    def predict_batch(self,\n                      X_batch):\n        model = self.eval()\n        x = Variable(X_batch, volatile=True)\n        if self.use_cuda:\n            x = x.cuda()\n        outputs = model(x)\n        pred = outputs\n        if self.use_cuda:\n            pred = pred.cpu()\n        return pred.data.numpy()\n\n    def predict(self, \n                X_test=None, use_topk=-1, reduce=True, reduce_mode='weighted'):\n        assert X_test is not None or self.test_generator is not None, \\\n        \"Either 'X_test' or 'self.test_generator' need to be provided\"\n        if X_test is not None:\n            test_dataset = DataBuilder(X_test)\n            self.test_generator = DataLoader(dataset=test_dataset,\n                                             batch_size=self.batch_size)\n            print(\"'self.test_generator' is updated by 'X_test'\")\n        return self.predict_generator(self.test_generator, \n                                      use_topk=use_topk, \n                                      reduce=reduce, \n                                      reduce_mode=reduce_mode)\n    \n    def predict_generator(self, \n                          test_generator, \n                          use_topk=-1,\n                          reduce=True,\n                          reduce_mode='weighted'):\n        args = self.args\n        model = self.eval()\n        print('predict with checkpoint at', args.save_sub_dir)\n        pred_all = []\n        cnt = 0\n        n_pred = len(args.valid_scores_topk)\n        if use_topk==-1:\n            use_topk = n_pred\n        for top_idx in range(use_topk):\n            cnt += 1\n            pred = []\n            model_path = os.path.join(args.save_sub_dir, str(top_idx)+'.pth')\n            if os.path.exists(model_path):\n                model.load(model_path)\n            else:\n                continue\n            model.eval()\n            for bx, _ in test_generator:\n                p = model.predict_batch(bx)\n                pred.extend(p)\n            pred_all.append(pred)\n            if cnt==use_topk:\n                break\n        if not reduce:\n            pred_res = pred_all\n        elif reduce and reduce_mode=='mean':\n            pred_res = np.mean(pred_all, axis=0)\n        elif reduce and reduce_mode=='weighted':\n            weights = np.array(args.valid_scores_topk[:len(pred_all)])\n            weights = np.exp(-weights)/np.exp(-weights).sum()\n            pred_res = np.sum([np.array(pred_all[i])*weights[i] for i in range(len(pred_all))], axis=0)\n        print('prediction done!')\n        return pred_res\n\n    def fit(self, \n            X_train=None, y_train=None, \n            X_valid=None, y_valid=None):\n        TRAIN_NULL_FLAG = (X_train is None) or (y_train is None)\n        VALID_NULL_FLAG = (X_valid is None) or (y_valid is None)\n        assert TRAIN_NULL_FLAG==False or self.train_generator is not None, \\\n        \"Either 'X/y_train' or 'self.train_generator' need to be provided\"\n        \n        args = self.args\n\n        if args.save_sub_dir and not os.path.exists(args.save_sub_dir):\n            print(\"Save path is not existed!\")\n            print('Creating dir at', args.save_sub_dir)\n            os.makedirs(args.save_sub_dir, exist_ok=True)\n        \n        if not TRAIN_NULL_FLAG:\n            train_dataset = DataBuilder(X_train, y_train)\n            self.train_generator = DataLoader(dataset=train_dataset,\n                                              batch_size=self.batch_size)\n            print(\"'self.train_generator' is updated by 'X/y_train'\")\n            print('Train with {} samples'.format(len(y_train)))\n            \n        if not VALID_NULL_FLAG:\n            valid_dataset = DataBuilder(X_valid, y_valid)\n            self.valid_generator = DataLoader(dataset=valid_dataset,\n                                              batch_size=self.batch_size)\n            print(\"'self.valid_generator' is updated by 'X/y_valid'\")\n            print('Validate with {} samples'.format(len(y_valid)))\n        \n        args = self.fit_generator(args, \n                                  self.train_generator,\n                                  self.valid_generator)\n    \n    def fit_generator(self, \n                      args, train_generator, valid_generator=None):\n        args.n_iter = 0\n        args.restarted = 0\n        args.finished = False\n        args.valid_scores = []\n        args.train_begin_time = time()\n        \n        self.optimizer = self.optimizer_caller(self.parameters(), \n                                               lr=args.lr,\n                                               weight_decay=args.weight_decay)\n        \n        total_loss = 0.0\n        for epoch in range(args.n_epochs):\n            if args.finished:\n                break\n            batch_begin_time = time()\n            ma_loss = 0.0\n            running_pred_train = []\n            running_y_train = []\n            for batch_idx, (bx, by) in enumerate(train_generator):\n                if args.finished:\n                    break\n                args.n_iter += 1\n                loss_tr, pred_tr = self.fit_batch(bx, by)\n                total_loss += loss_tr\n                ma_loss += loss_tr\n                running_pred_train.extend(pred_tr)\n                running_y_train.extend(by)\n                if args.n_iter % args.log_interval == 0:\n                    score = self.eval_metric(running_y_train, \n                                             running_pred_train)\n                    print('[%d, %5d] loss: %.6f metric: %.6f time: %.1f s' % \\\n                          (epoch + 1, \n                           args.n_iter, \n                           ma_loss/args.log_interval, \n                           score, \n                           time()-batch_begin_time))\n                    ma_loss = 0.0\n                    running_pred_train = []\n                    running_y_train = []\n                    batch_begin_time = time()\n                if valid_generator is not None and \\\n                args.n_iter % args.valid_interval == 0:\n                    args = self.evaluate_generator(args, valid_generator)\n                    args = self.check_early_stopping(args)\n                    print('valid time: %.1f s' % (time()-batch_begin_time))\n                    batch_begin_time = time()\n        return args\n    \n    def valid(self, \n              X_valid=None, y_valid=None):\n        VALID_NULL_FLAG = (X_valid is None) or (y_valid is None)\n        assert VALID_NULL_FLAG==False or self.valid_generator is not None, \\\n        \"Either 'X/y_valid' or 'self.valid_generator' need to be provided\"\n        \n        args = self.args\n        \n        if not VALID_NULL_FLAG:\n            valid_dataset = DataBuilder(X_valid, y_valid)\n            self.valid_generator = DataLoader(dataset=valid_dataset,\n                                              batch_size=self.batch_size)\n            print(\"'self.valid_generator' is updated by 'X/y_valid'\")\n            print('Validate with {} samples'.format(len(y_valid)))\n        begin_time = time()\n        args = self.evaluate_generator(args, self.valid_generator)\n        print('valid time: %.1f s' % (time()-begin_time))\n    \n    def evaluate_generator(self, \n                           args, valid_generator):\n        running_pred_valid = []\n        running_y_valid = []\n        val_total_loss = 0.0\n        for _bx,_y in valid_generator:\n            loss_val, pred_val = self.valid_batch(_bx,_y)\n            running_pred_valid.extend(pred_val)\n            running_y_valid.extend(_y)\n            val_total_loss += loss_val*len(_y)\n        _score = self.eval_metric(running_y_valid, running_pred_valid)\n        args.valid_scores.append(_score)\n        print('*'*50)\n        print('valid loss: %.6f metric: %.6f total time: %.1f s' %\n              (val_total_loss/len(running_y_valid), \n               _score, \n               time()-args.train_begin_time))\n        print('*'*50)\n        return args\n    \n    def check_early_stopping(self, args):\n        _score = args.valid_scores[-1]\n        early_stopping_flag = True\n        for top_idx, top_scr in enumerate(args.valid_scores_topk):\n            if (_score - top_scr > 0) == args.greater_is_better:\n                args.valid_scores_topk[top_idx] = _score\n                print('Best %d-th valid score:' % top_idx, _score)\n                save_topkth_path = os.path.join(args.save_sub_dir, \n                                                str(top_idx)+'.pth')\n                self.save(save_topkth_path)\n                early_stopping_flag = False\n                break\n        if early_stopping_flag:\n            if args.restarted < args.patience:\n                save_top0th_path = os.path.join(args.save_sub_dir, str(0)+'.pth')\n                print()\n                print('\\t\\tEarly stopped, restarting from', save_top0th_path)\n                print()\n                self.load(save_top0th_path)\n                args.restarted += 1\n                args.valid_interval = max(int(args.valid_interval * \\\n                                              args.valid_interval_re_decay), \n                    args.valid_interval_min)\n                args.lr = args.lr * args.lr_re_decay\n                self.optimizer = self.optimizer_caller(self.parameters(), \n                                                       lr=args.lr, \n                                                       weight_decay=args.weight_decay)\n            else:\n                args = self.save_finished_args(args)\n        return args","execution_count":15,"outputs":[]},{"metadata":{"_cell_guid":"2d4dc416-dcea-4754-86e8-b99581f7788b","_uuid":"3cc2951483340b462cb3383855b72447bb99e369","collapsed":true,"trusted":true},"cell_type":"code","source":"from sklearn.metrics import mean_squared_error\n\ndef kmax_pooling(x, dim, k):\n    index = x.topk(k, dim = dim)[1].sort(dim = dim)[0]\n    return x.gather(dim, index)\n\ndef emb_cos_distance(x_context, x_text, seq_dim=1):\n    len_c = x_context.size(1)\n    len_t = x_text.size(1)\n    outputs = []\n    avg = x_context.mean(1)\n    for j in range(len_t):\n        output = F.cosine_similarity(avg, x_text[:, j, :])\n        output = output.view(-1, 1)\n        outputs.append(output)\n    return torch.cat(outputs, dim=1)\n\n## https://github.com/zachAlbus/pyTorch-text-classification/blob/master/Yoon/model.py\nclass TextCNN(BaseModel):\n    def __init__(self):\n        super(TextCNN, self).__init__()\n    def _eval_metric(self, labels, preds):\n        return np.sqrt(mean_squared_error(labels, preds))\n    def _criterion(self, input, target, weight=None):\n        if weight is None:\n            return torch.sqrt(F.mse_loss(input, target, size_average=True))\n        else:\n            return torch.sum(weight * (input - target) ** 2)\n    \n    def init_args(self, args, n_output,\n                  max_context_len, max_text_len, max_sim_k,\n                  max_features, embed_size, embedding_init, max_pooling_k,\n                  in_channel, out_channel, kernel_sizes, dilations,\n                  dropout, n_final_state):\n        if not torch.cuda.is_available():\n            args.use_cuda = False\n            print(\"Cuda is not available, automatically changed into cpu model\")\n        else:\n            args.use_cuda = True\n        \n        torch.manual_seed(233)\n        \n        self.use_cuda = args.use_cuda\n        self.optimizer_caller = self.get_optimizer_caller(args.optimizer_type)\n        \n        self.criterion = self._criterion\n        self.eval_metric = self._eval_metric\n        self.batch_size = args.batch_size\n        self.args = args\n        self.check_args()\n        print('args initialized')\n\n        self.n_output = n_output\n        self.in_channel = in_channel\n        self.out_channel = out_channel\n        self.max_context_len = max_context_len\n        self.max_text_len = max_text_len\n        self.max_features = max_features\n        self.embed_size = embed_size\n        self.max_pooling_k = max_pooling_k\n        self.max_sim_k = max_sim_k\n        self.kernel_sizes = kernel_sizes\n        self.dilations = dilations\n        self.n_final_state = n_final_state\n        self.dropout = dropout\n        \n        self.return_final_state = False\n        \n        self.embed = nn.Embedding(max_features, embed_size)\n        if embedding_init is not None:\n            self.embed.weight.data.copy_(torch.from_numpy(embedding_init))\n        self.conv = nn.ModuleList([\n            nn.Conv2d(in_channel, out_channel, \n                      kernel_size=(K, embed_size), dilation=(D, 1)) \\\n            for K, D in zip(kernel_sizes, dilations)\n        ])\n        final_input_dim = len(kernel_sizes) * out_channel * max_pooling_k + max_sim_k + embed_size\n        self.final_state = nn.Linear(final_input_dim, \n                                     n_final_state)\n        self.dropout = nn.Dropout(dropout)\n        self.bn_1 = nn.BatchNorm1d(final_input_dim)\n        self.bn_2 = nn.BatchNorm1d(n_final_state)\n        self.fc = nn.Linear(n_final_state, n_output)\n        \n        if self.use_cuda:\n            return self.cuda()\n        else:\n            return self\n    def forward(self, x):\n        batch_size = x.size(0)\n        # Embedding\n        emb = self.embed(x)  # dim: (batch_size, max_len, embed_size)\n        # Split context, text -> # dim: (batch_size, max_len, embedding_size)\n        x_context = emb[:, :self.max_context_len, :]\n        x_text = emb[:, self.max_context_len:, :]\n        # Calculate Distance\n        dist = emb_cos_distance(x_context, x_text) # dim: (batch_size, max_context_len*max_text_len)\n        # Conv & max pool\n        x = emb.unsqueeze(1)\n        # turns to be a list of ele with dim: ([batch, num_kernels, max_seq_len-i+1])\n        x = [F.relu(conv(x)).squeeze(3) for conv in self.conv] # dim: [(batch_size, num_kernels), ...]*len(kernel_sizes)\n        #x = [F.max_pool1d(conv_out, conv_out.size(2)).squeeze(2) for conv_out in x]\n        x = [kmax_pooling(conv_out, 2, self.max_pooling_k).view(batch_size, -1) for conv_out in x]\n        #x = x + [kmax_pooling(dist, 1, self.max_sim_k)] #[dist]\n        x = x + [kmax_pooling(dist, 1, self.max_sim_k)] + [emb.mean(1).view(batch_size, -1)]\n        x = torch.cat(x, 1)\n        # Dropout & output\n        #x = self.dropout(x)  #dim: (batch_size, len(kernel_sizes)*num_kernels)\n        x = self.bn_1(x)\n        x = self.final_state(x)\n        ##\n        x = self.bn_2(x)\n        ##\n        if self.return_final_state:\n            return x\n        else:\n            x = F.relu(x)\n            x = self.fc(x)\n            return torch.clamp(x, 0, 1)","execution_count":24,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"16860c0aecd86c03eeb33fd646d9fa8fade4f971"},"cell_type":"code","source":"# for bx,_ in model.valid_generator: break\n#     x = model.embed(Variable(bx).cuda())","execution_count":25,"outputs":[]},{"metadata":{"_cell_guid":"367a3924-1912-45bf-b4f6-4a804e116210","_uuid":"2c82a49fe04bf977af4a41b838dd9554b1ded257","trusted":true,"collapsed":true},"cell_type":"code","source":"batch_size = 256\nn_epochs = 5\n\nfrom argparse import Namespace\nargs = Namespace()\n\nargs.use_cuda = True\nargs.optimizer_type = 'adam'\nargs.save_sub_dir = '.'\nargs.patience = 5\nargs.valid_scores_topk = [999] * args.patience\nargs.greater_is_better = False\nargs.log_interval = 100\nargs.valid_interval = 500\nargs.valid_interval_re_decay = 0.7\nargs.valid_interval_min = 50\nargs.lr_re_decay = 0.5\n\nargs.batch_size = batch_size\nargs.lr=0.0001\nargs.weight_decay=0.0\nargs.n_epochs=n_epochs\nargs.model_name='TextCNN'\n\nargs.model_params = dict(n_output=1, \n                         max_context_len=max_context_len,  \n                         max_text_len=max_text_len,\n                         max_sim_k=1, max_pooling_k=1,\n                         max_features=max_features, embed_size=embed_size, \n                         embedding_init=embedding_matrix, \n                         in_channel=1, out_channel=32, \n                         kernel_sizes=[1, 2, 3, 4, 5], \n                         dilations=[1, 1, 1, 1, 1], \n                         dropout=0.5, \n                         n_final_state=16)","execution_count":26,"outputs":[]},{"metadata":{"_cell_guid":"12d84814-0bec-4e2e-9984-81788d43d183","_uuid":"216e8e836a3043888e7c69d3b0cef7aca0e52a04","collapsed":true,"trusted":true},"cell_type":"code","source":"def get_split_masks(eval_sets, valid_fold, test_fold):\n    mask_val = eval_sets==valid_fold\n    mask_te = eval_sets==test_fold\n    mask_tr = ~mask_val & ~mask_te\n    return mask_tr, mask_val, mask_te\nvalid_fold = 0\nmask_tr, mask_val, mask_te = get_split_masks(eval_sets, valid_fold, 10)","execution_count":27,"outputs":[]},{"metadata":{"_cell_guid":"9e72a74f-9499-4489-83ce-6afec730647b","_uuid":"df4efd90b52adda5ecf82a511bd42e00d5dd30d9","trusted":true,"collapsed":true},"cell_type":"code","source":"model = TextCNN()\nmodel = model.init_args(args, **args.model_params)","execution_count":28,"outputs":[]},{"metadata":{"trusted":true,"scrolled":false,"_uuid":"e5c073473680cfa2efd2a1beacc37edc358ce9c0","_kg_hide-output":true,"collapsed":true},"cell_type":"code","source":"model.fit(text[mask_tr], labels[mask_tr], text[mask_val], labels[mask_val])","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"bfccdba9-6446-4916-8082-f57151a704d7","_uuid":"c4c05df4402a07ad9e8e9d1f7d77f16ea216e71c","trusted":true,"collapsed":true},"cell_type":"code","source":"pred_val_all = model.predict(text[mask_val], use_topk=-1, reduce=False)","execution_count":15,"outputs":[]},{"metadata":{"_cell_guid":"6b28b51a-ad6f-4912-8b11-fed824ee1c20","_uuid":"3494c4261a65a5707e68ec99dae6282c08aabc37","trusted":true,"collapsed":true},"cell_type":"code","source":"for pred_val in pred_val_all:\n    print(np.sqrt(mean_squared_error(labels[mask_val], pred_val)))","execution_count":16,"outputs":[]},{"metadata":{"_cell_guid":"454dc295-a3ce-4c4c-bdcd-5e199b04a1cc","_uuid":"3c42c7ce59a148f671b1e7047b0f7a118ab78b35","trusted":true,"collapsed":true},"cell_type":"code","source":"topk_avg_scores = []\nfor idx, pred_val in enumerate(pred_val_all):\n    topk_avg_scores.append(np.sqrt(mean_squared_error(labels[mask_val], np.mean(pred_val_all[:idx+1], 0))))\n    print('top %d'%(idx+1), topk_avg_scores[-1])\ntopk_wavg_scores = []\nfor idx, pred_val in enumerate(pred_val_all):\n    weights = np.array(args.valid_scores_topk[:idx+1])\n    weights = np.exp(-weights)/np.exp(-weights).sum()\n    topk_wavg_scores.append(np.sqrt(mean_squared_error(labels[mask_val], \n                                                       np.dot(np.hstack(pred_val_all[:idx+1]), weights.reshape(-1, 1)))))\n    print('top %d weighted'%(idx+1), topk_wavg_scores[-1])","execution_count":17,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c657cdb198db853110ce69dcd431f9c5057d4b4c","collapsed":true},"cell_type":"code","source":"if min(topk_avg_scores)<=min(topk_wavg_scores):\n    best_topk = np.argmin(topk_avg_scores)+1\n    best_reduce_mode = 'mean'\nelse:\n    best_topk = np.argmin(topk_wavg_scores)+1\n    best_reduce_mode = 'weighted'\nbest_valid_score = min(min(topk_avg_scores), min(topk_wavg_scores))\nprint('best top:', best_topk, 'mode:', best_reduce_mode)","execution_count":18,"outputs":[]},{"metadata":{"_cell_guid":"c45fbf5a-6710-467d-957f-04316c21f5b9","_uuid":"57212a113338566a6836931156552cd8db95df4b","trusted":true,"collapsed":true},"cell_type":"code","source":"pred_val = model.predict(text[mask_val], use_topk=best_topk, reduce_mode=best_reduce_mode)\nnp.save('valid_%d_pred.npy'%valid_fold, pred_val)\npred_test = model.predict(text[mask_te], use_topk=best_topk, reduce_mode=best_reduce_mode)\nnp.save('test_pred.npy', pred_test)","execution_count":19,"outputs":[]},{"metadata":{"_cell_guid":"56bd47c3-4157-49ba-9cd5-d049414e01e8","_uuid":"b7c5a35ffe2bad568bf69100d2a34dbb04ab6f9b","trusted":true,"collapsed":true},"cell_type":"code","source":"sns.distplot(pred_test)\nsns.distplot(labels[mask_val])","execution_count":20,"outputs":[]},{"metadata":{"_cell_guid":"a38f66ac-907b-4f1e-9192-23afbbbe2075","_uuid":"282f2859d2f3372891f84d5a5b8170cc56cac5e8","trusted":true,"collapsed":true},"cell_type":"code","source":"sub = pd.read_csv(DATA_DIR+'sample_submission.csv')\nsub[target_col] = pred_test\nsub.to_csv('textcnn_%.6f.csv'%best_valid_score, index=False)\nprint('save to', 'textcnn_%.6f.csv'%best_valid_score)\nsub.head()","execution_count":21,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a916d9376fd3d660c248b9ce477a971dd745a511","collapsed":true},"cell_type":"code","source":"del model; gc.collect()\nmodel = TextCNN()\nmodel = model.init_args(args, **args.model_params)","execution_count":22,"outputs":[]},{"metadata":{"_cell_guid":"210641ad-a3db-47ed-8909-270b49464058","_uuid":"f4b7df562b0f6b9bb660842fbf18d8911e71ee80","trusted":true,"scrolled":true,"collapsed":true},"cell_type":"code","source":"model.return_final_state = True\ntest_state = model.predict(text[mask_te], use_topk=best_topk, reduce_mode=best_reduce_mode)","execution_count":23,"outputs":[]},{"metadata":{"_cell_guid":"150ea238-82df-4c57-b30d-015cf804175e","_uuid":"3fb3f39627c0d10fc46a7fd0bc775b4ef754c062","trusted":true,"collapsed":true},"cell_type":"code","source":"test_state.shape","execution_count":24,"outputs":[]},{"metadata":{"_cell_guid":"6bfbb849-c377-4c01-983b-7f340e46293d","_uuid":"23858cf5dbf073595ec43df78d45f55e20ad4010","trusted":true,"collapsed":true},"cell_type":"code","source":"plt.plot(test_state.mean(0))","execution_count":25,"outputs":[]},{"metadata":{"_cell_guid":"703f53dd-f869-4351-936e-aa0885ea585c","_uuid":"8a7be35862702530372c786a5fcc1c4dd77db83a","trusted":true,"collapsed":true},"cell_type":"code","source":"valid_state = model.predict(text[mask_val], use_topk=best_topk, reduce_mode=best_reduce_mode)","execution_count":26,"outputs":[]},{"metadata":{"_cell_guid":"45d0096d-509f-4951-acee-d879f91b84e4","_uuid":"8ae9e4c47939fec987f57c7623e3d46458632b69","trusted":true,"collapsed":true},"cell_type":"code","source":"from scipy import sparse\nvalid_state = sparse.csr_matrix(valid_state)\ntest_state = sparse.csr_matrix(test_state)\nsparse.save_npz('valid_%d_state.npz'%valid_fold, valid_state, compressed=True)\nsparse.save_npz('test_state.npz', test_state, compressed=True)\n!ls .","execution_count":27,"outputs":[]},{"metadata":{"collapsed":true,"trusted":true,"_uuid":"07eea935293ec4510fb1082f8a280fc7beb834f5"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}