{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport os\nfrom multiprocessing import Pool","execution_count":1,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = pd.read_csv(\"../input/train.csv\")\ntest_df = pd.read_csv(\"../input/test.csv\")\nprint(\"Train shape : \",train_df.shape)\nprint(\"Test shape : \",test_df.shape)","execution_count":2,"outputs":[{"output_type":"stream","text":"Train shape :  (1306122, 3)\nTest shape :  (375806, 2)\n","name":"stdout"}]},{"metadata":{"trusted":true},"cell_type":"code","source":"def parallelize_apply(df,func,colname,num_process,newcolnames):\n    # takes as input a df and a function for one of the columns in df\n    pool =Pool(processes=num_process)\n    arraydata = pool.map(func,tqdm(df[colname].values))\n    pool.close()\n    \n    newdf = pd.DataFrame(arraydata,columns = newcolnames)\n    df = pd.concat([df,newdf],axis=1)\n    return df\n\ndef parallelize_dataframe(df, func):\n    df_split = np.array_split(df, 4)\n    pool = Pool(4)\n    df = pd.concat(pool.map(func, df_split))\n    pool.close()\n    pool.join()\n    return df\n\n# some fetures \n# ['','','india/n','','quora','','sex','','','country/countries','china','','','chinese','','']\ndef add_features(df):\n    df['question_text'] = df['question_text'].apply(lambda x:str(x))\n    # df = parallelize_apply(df,sentiment,'question_text',4,['sentiment','subjectivity']) \n    # df['sentiment'] = df['question_text'].progress_apply(lambda x:sentiment(x))\n    df['total_length'] = df['question_text'].apply(len)\n    df['capitals'] = df['question_text'].apply(lambda comment: sum(1 for c in comment if c.isupper()))\n    df['caps_vs_length'] = df.apply(lambda row: float(row['capitals'])/float(row['total_length']),\n                                axis=1)\n    df['num_words'] = df.question_text.str.count('\\S+')\n    df['num_unique_words'] = df['question_text'].apply(lambda comment: len(set(w for w in comment.split())))\n    df['words_vs_unique'] = df['num_unique_words'] / df['num_words'] \n    return df\n\n","execution_count":3,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = parallelize_dataframe(train_df, add_features)\ntest_df = parallelize_dataframe(test_df, add_features)","execution_count":4,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras.preprocessing.text import Tokenizer\nfrom keras.preprocessing.sequence import pad_sequences\n\nmax_features = 100000\n\ntokenizer = Tokenizer(num_words=max_features, lower=True)\nfull_text = list(train_df['question_text'].values)\ntokenizer.fit_on_texts(full_text)\n\ntrain_tokenized = tokenizer.texts_to_sequences(train_df['question_text'].fillna('missing'))\ntest_tokenized = tokenizer.texts_to_sequences(test_df['question_text'].fillna('missing'))","execution_count":5,"outputs":[{"output_type":"stream","text":"Using TensorFlow backend.\n","name":"stderr"}]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_meta = np.array(train_df.iloc[:,3:])\ntest_meta = np.array(test_df.iloc[:,2:])\nprint(train_meta.shape)\nprint(test_meta.shape)","execution_count":6,"outputs":[{"output_type":"stream","text":"(1306122, 6)\n(375806, 6)\n","name":"stdout"}]},{"metadata":{"trusted":true},"cell_type":"code","source":"max_len = 72\ntrain_X = np.column_stack((pad_sequences(train_tokenized, maxlen=max_len), train_meta))\ntest_X = np.column_stack((pad_sequences(test_tokenized, maxlen=max_len), test_meta))\ntrain_y = train_df.target.values","execution_count":7,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from gensim.models import KeyedVectors\nfrom gensim.test.utils import datapath, get_tmpfile\nfrom gensim.scripts.glove2word2vec import glove2word2vec\n\nnews_path = '../input/embeddings/GoogleNews-vectors-negative300/GoogleNews-vectors-negative300.bin'\nembeddings = KeyedVectors.load_word2vec_format(news_path, binary=True)","execution_count":8,"outputs":[{"output_type":"stream","text":"/opt/conda/lib/python3.6/site-packages/smart_open/ssh.py:34: UserWarning: paramiko missing, opening SSH/SCP/SFTP paths will be disabled.  `pip install paramiko` to suppress\n  warnings.warn('paramiko missing, opening SSH/SCP/SFTP paths will be disabled.  `pip install paramiko` to suppress')\n","name":"stderr"}]},{"metadata":{"trusted":true},"cell_type":"code","source":"mean = np.mean(embeddings.vectors)\nstd = np.std(embeddings.vectors)\n_, size = embeddings.vectors.shape\nnum_words = min(len(tokenizer.word_index), max_features)\nembedding_matrix = np.random.normal(mean, std, (num_words, size))","execution_count":9,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for word, index in tokenizer.word_index.items():\n    if index >= max_features: continue\n    try:\n        word_vector = embeddings.get_vector(word)\n        embedding_matrix[index] = word_vector\n    except:\n        pass\nprint(embedding_matrix.shape)","execution_count":10,"outputs":[{"output_type":"stream","text":"(100000, 300)\n","name":"stdout"}]},{"metadata":{"trusted":true},"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.optim as optim\nimport random\nfrom tqdm import  tqdm\ntqdm.pandas()\n\ndef seed_torch(seed=0):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    \nseed = 0\nseed_torch(seed)\n\ndef sigmoid(x):\n    return 1 / (1 + np.exp(-x))\n\nclass BiLSTM(nn.Module):\n    def __init__(self, embedding_matrix = embedding_matrix, embed_size = 300, hidden_size = 128):\n        super().__init__()\n        self.embedding = nn.Embedding(max_features, size)\n        if embedding_matrix is not None:\n            self.embedding.weight = nn.Parameter(torch.tensor(embedding_matrix, dtype=torch.float32))\n            self.embedding.weight.requires_grad = False\n        \n        self.lstm = nn.LSTM(embed_size, hidden_size, bidirectional = True, batch_first = True, num_layers = 2, dropout = 0.1)\n        self.fc = nn.Linear(hidden_size * 2 + 6, 1)\n        \n    def forward(self, x):\n        embedded = self.embedding(x[:,:72])\n        f = torch.tensor(x[:,72:], dtype=torch.float32).cuda()\n        h_lstm, (h,c) = self.lstm(embedded)\n        h_fc = torch.cat((h[-1],h[-2],f), dim = 1)\n        \n        return self.fc(h_fc)","execution_count":18,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.model_selection import StratifiedKFold\nfrom torch.utils.data import DataLoader, TensorDataset\nimport time\nimport re\nimport random\nfrom sklearn.metrics import f1_score\n\nsplits = list(StratifiedKFold(n_splits=4, shuffle=True, random_state=0).split(train_X, train_y))\ntrain_epochs = 5\nbatch_size = 64\n\nprint(type(train_X), train_X.shape)\nprint(type(train_y), train_y.shape)\ntrain_X = torch.Tensor(train_X)\ntrain_y = torch.Tensor(train_y)\ntrain_meta = torch.Tensor(train_meta)\ntest_meta = torch.Tensor(test_meta)\n\n\ntrain_preds = np.zeros((len(train_X)))\ntest_preds = np.zeros((len(test_X)))\n\nx_test_cuda = torch.tensor(test_X, dtype=torch.long).cuda()\ntest = torch.utils.data.TensorDataset(x_test_cuda)\ntest_loader = torch.utils.data.DataLoader(test, batch_size=batch_size, shuffle=False)\n\nfor i, (train_idx, valid_idx) in enumerate(splits):\n    print('Fold:{0}'.format(i))\n    x_train_fold = torch.tensor(train_X[train_idx], dtype=torch.long).cuda()\n    y_train_fold = torch.tensor(train_y[train_idx, np.newaxis], dtype=torch.float32).cuda()\n    x_val_fold = torch.tensor(train_X[valid_idx], dtype=torch.long).cuda()\n    y_val_fold = torch.tensor(train_y[valid_idx, np.newaxis], dtype=torch.float32).cuda()\n    \n    train_data = torch.utils.data.TensorDataset(x_train_fold, y_train_fold)\n    valid_data = torch.utils.data.TensorDataset(x_val_fold, y_val_fold)\n    \n    train_loader = torch.utils.data.DataLoader(train_data, batch_size=batch_size, shuffle=True)\n    valid_loader = torch.utils.data.DataLoader(valid_data, batch_size=batch_size, shuffle=False)\n    \n    model = BiLSTM()\n    model.cuda()\n    loss_function = nn.BCEWithLogitsLoss()\n    optimiser = optim.Adam(model.parameters())\n    \n    for epoch in range(train_epochs):\n        start_time = time.time()\n        model.train()\n        avg_loss = 0.0\n        # train  model\n        for x_batch, y_batch in tqdm(train_loader, disable=True):\n            y_predicted = model(x_batch)\n            loss = loss_function(y_predicted, y_batch)\n            optimiser.zero_grad()\n            loss.backward()\n            optimiser.step()\n            avg_loss += loss.item()/len(train_loader)\n\n        # evaluate model\n        model.eval()\n        val_predicted_fold = np.zeros(x_val_fold.size(0))\n        test_predicted_fold = np.zeros(len(test_X))\n        avg_val_loss =0.0\n\n        for i, (x_batch, y_batch) in enumerate(valid_loader):\n            y_predicted = model(x_batch).detach()\n            avg_val_loss += loss_function(y_predicted, y_batch).item()/len(valid_loader)\n            val_predicted_fold[i*batch_size:(i+1)*batch_size] = sigmoid(y_predicted.cpu().numpy())[:,0]   \n\n        elapsed_time = time.time() - start_time\n        print('Epoch {}/{} \\t loss={:.4f} \\t val_loss={:.4f} \\t time={:.2f}s'.format(epoch + 1,\n                                                        train_epochs, avg_loss, avg_val_loss, elapsed_time))\n    # predict testing dataset\n    test_predicted_fold = np.zeros(len(test_X))\n    for i, (x_batch,) in enumerate(test_loader):\n        y_pred = model(x_batch).detach()\n        test_predicted_fold[i * batch_size:(i+1) * batch_size] = sigmoid(y_pred.cpu().numpy())[:, 0]\n    \n    train_preds[valid_idx] = val_predicted_fold\n    test_preds += test_predicted_fold / len(splits) # take average","execution_count":null,"outputs":[{"output_type":"stream","text":"<class 'torch.Tensor'> torch.Size([1306122, 78])\n<class 'torch.Tensor'> torch.Size([1306122])\nFold:0\n","name":"stdout"},{"output_type":"stream","text":"/opt/conda/lib/python3.6/site-packages/ipykernel_launcher.py:29: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n/opt/conda/lib/python3.6/site-packages/ipykernel_launcher.py:30: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n/opt/conda/lib/python3.6/site-packages/ipykernel_launcher.py:31: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n/opt/conda/lib/python3.6/site-packages/ipykernel_launcher.py:32: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n/opt/conda/lib/python3.6/site-packages/ipykernel_launcher.py:36: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n","name":"stderr"},{"output_type":"stream","text":"Epoch 1/5 \t loss=0.1240 \t val_loss=0.1092 \t time=336.50s\n","name":"stdout"}]},{"metadata":{"trusted":true},"cell_type":"code","source":"def threshold_search(y_true, y_proba):\n    best_threshold = 0\n    best_score = 0\n    for threshold in tqdm([i * 0.01 for i in range(100)]):\n        score = f1_score(y_true=y_true, y_pred=y_proba > threshold)\n        if score > best_score:\n            best_threshold = threshold\n            best_score = score\n    search_result = {'threshold': best_threshold, 'f1': best_score}\n    return search_result","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"search_result = threshold_search(train_y, train_preds)\nsearch_result","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub = pd.read_csv('../input/sample_submission.csv')\nsub.prediction = test_preds > search_result['threshold']\nsub.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}