{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import os, gc\nfrom fastai.text import *\nfrom tqdm import tqdm_notebook as tqdm\nprint(os.listdir(\"../input\"))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c48c17827846b5be89c471f5cd0de9d092899e54"},"cell_type":"code","source":"# make training deterministic/reproducible\ndef seed_everything(seed=42):\n    random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed_all(seed)\n    np.random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    torch.backends.cudnn.deterministic = True\nseed_everything()\n\ndef f1_score(y_pred, targets):\n    epsilon = 1e-07\n\n    y_pred = y_pred.argmax(dim=1)\n#     targets = targets.argmax(dim=1)\n\n    tp = (y_pred*targets).float().sum(dim=0)\n    tn = ((1-targets)*(1-y_pred)).float().sum(dim=0)\n    fp = ((1-targets)*y_pred).float().sum(dim=0)\n    fn = (targets*(1-y_pred)).sum(dim=0)\n\n    p = tp / (tp + fp + epsilon)\n    r = tp / (tp + fn + epsilon)\n\n    f1 = 2*p*r / (p+r+epsilon)\n    f1 = torch.where(f1!=f1, torch.zeros_like(f1), f1)\n    return f1.mean()","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"EMBED_SIZE = 100\nMAX_FEATURES = 150000\nMAX_LENGTH = 100\nEMBEDDING_FILE = '../input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"07589f4184b3547306afecb73e2177c04f0f4850"},"cell_type":"code","source":"# train_df = pd.read_csv('../input/train.csv')\ntest_df = pd.read_csv('../input/test.csv')\n# train_df.head()\ntrain_df = pd.read_csv('../input/train.csv')\n\ninsincere_df = train_df[train_df.target==1]\nsincere_df = train_df[train_df.target==0]\n\nsincere_df = sincere_df.iloc[np.random.permutation(len(sincere_df))]\nsincere_df = sincere_df[:int(len(insincere_df)*2)]\n\ndel train_df\n\ntrain_df = pd.concat([insincere_df, sincere_df])\ntrain_df = train_df.iloc[np.random.permutation(len(train_df))]\n\ndel insincere_df\ndel sincere_df\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f0dc10225ae9614d15d5cf92d50f511cf47cc5aa"},"cell_type":"code","source":"%%time\nmispell_dict = {\"ain't\": \"is not\", \"aren't\": \"are not\",\"can't\": \"cannot\", \"'cause\": \"because\", \n                \"could've\": \"could have\", \"couldn't\": \"could not\", \"didn't\": \"did not\", \n                \"doesn't\": \"does not\", \"don't\": \"do not\", \"hadn't\": \"had not\", \"hasn't\": \"has not\", \n                \"haven't\": \"have not\", \"he'd\": \"he would\",\"he'll\": \"he will\", \"he's\": \"he is\", \n                \"how'd\": \"how did\", \"how'd'y\": \"how do you\", \"how'll\": \"how will\", \"how's\": \"how is\",  \n                \"I'd\": \"I would\", \"I'd've\": \"I would have\", \"I'll\": \"I will\", \"I'll've\": \"I will have\",\n                \"I'm\": \"I am\", \"I've\": \"I have\", \"i'd\": \"i would\", \"i'd've\": \"i would have\", \n                \"i'll\": \"i will\",  \"i'll've\": \"i will have\",\"i'm\": \"i am\", \"i've\": \"i have\", \n                \"isn't\": \"is not\", \"it'd\": \"it would\", \"it'd've\": \"it would have\", \"it'll\": \"it will\", \n                \"it'll've\": \"it will have\",\"it's\": \"it is\", \"let's\": \"let us\", \"ma'am\": \"madam\", \n                \"mayn't\": \"may not\", \"might've\": \"might have\",\"mightn't\": \"might not\",\n                \"mightn't've\": \"might not have\", \"must've\": \"must have\", \"mustn't\": \"must not\", \n                \"mustn't've\": \"must not have\", \"needn't\": \"need not\", \"needn't've\": \"need not have\",\n                \"o'clock\": \"of the clock\", \"oughtn't\": \"ought not\", \"oughtn't've\": \"ought not have\", \n                \"shan't\": \"shall not\", \"sha'n't\": \"shall not\", \"shan't've\": \"shall not have\", \n                \"she'd\": \"she would\", \"she'd've\": \"she would have\", \"she'll\": \"she will\", \n                \"she'll've\": \"she will have\", \"she's\": \"she is\", \"should've\": \"should have\", \n                \"shouldn't\": \"should not\", \"shouldn't've\": \"should not have\", \"so've\": \"so have\",\n                \"so's\": \"so as\", \"this's\": \"this is\",\"that'd\": \"that would\", \"that'd've\": \"that would have\", \n                \"that's\": \"that is\", \"there'd\": \"there would\", \"there'd've\": \"there would have\", \n                \"there's\": \"there is\", \"here's\": \"here is\",\"they'd\": \"they would\", \n                \"they'd've\": \"they would have\", \"they'll\": \"they will\", \"they'll've\": \"they will have\", \n                \"they're\": \"they are\", \"they've\": \"they have\", \"to've\": \"to have\", \"wasn't\": \"was not\", \n                \"we'd\": \"we would\", \"we'd've\": \"we would have\", \"we'll\": \"we will\", \"we'll've\": \"we will have\", \n                \"we're\": \"we are\", \"we've\": \"we have\", \"weren't\": \"were not\", \"what'll\": \"what will\", \n                \"what'll've\": \"what will have\", \"what're\": \"what are\",  \"what's\": \"what is\", \n                \"what've\": \"what have\", \"when's\": \"when is\", \"when've\": \"when have\", \n                \"where'd\": \"where did\", \"where's\": \"where is\", \"where've\": \"where have\", \n                \"who'll\": \"who will\", \"who'll've\": \"who will have\", \"who's\": \"who is\", \n                \"who've\": \"who have\", \"why's\": \"why is\", \"why've\": \"why have\", \n                \"will've\": \"will have\", \"won't\": \"will not\", \"won't've\": \"will not have\", \n                \"would've\": \"would have\", \"wouldn't\": \"would not\", \"wouldn't've\": \"would not have\", \n                \"y'all\": \"you all\", \"y'all'd\": \"you all would\",\"y'all'd've\": \"you all would have\",\n                \"y'all're\": \"you all are\",\"y'all've\": \"you all have\",\"you'd\": \"you would\", \n                \"you'd've\": \"you would have\", \"you'll\": \"you will\", \"you'll've\": \"you will have\", \n                \"you're\": \"you are\", \"you've\": \"you have\", 'colour': 'color', 'centre': 'center', \n                'favourite': 'favorite', 'travelling': 'traveling', 'counselling': 'counseling', \n                'theatre': 'theater', 'cancelled': 'canceled', 'labour': 'labor', \n                'organisation': 'organization', 'wwii': 'world war 2', 'citicise': 'criticize', \n                'youtu ': 'youtube ', 'Qoura': 'Quora', 'sallary': 'salary', 'Whta': 'What', \n                'narcisist': 'narcissist', 'howdo': 'how do', 'whatare': 'what are', 'howcan': 'how can', \n                'howmuch': 'how much', 'howmany': 'how many', 'whydo': 'why do', 'doI': 'do I', \n                'theBest': 'the best', 'howdoes': 'how does', 'mastrubation': 'masturbation', \n                'mastrubate': 'masturbate', \"mastrubating\": 'masturbating', 'pennis': 'penis', \n                'Etherium': 'Ethereum', 'narcissit': 'narcissist', 'bigdata': 'big data', '2k17': '2017', \n                '2k18': '2018', 'qouta': 'quota', 'exboyfriend': 'ex boyfriend', \n                'airhostess': 'air hostess', \"whst\": 'what', 'watsapp': 'whatsapp', \n                'demonitisation': 'demonetization', 'demonitization': 'demonetization', \n                'demonetisation': 'demonetization', \"n’t\": \"not\", \"n't\": \"not\", \"’ve\": \"have\",\n                \"’re\": \"are\", \"’ll\": \"will\", \"howmuch\": \"how much\", \"i`m\": \"I am\", \"can`t\": \"can not\",\n                \"dosen't\": \"does not\", \"what's​\": \"what is\", \"did't\": \"did not\", \"doesn`t\": \"dose not\",\n                \"ya'll\": \"you alll\", \"it`s\": \"it is \", \"does'nt\": \"does not\", \"what`s\": \"what is\",\n                \"dosn't\": \"does not\", \"is'nt\": \"is not\", \"don'y\": \"do not you\", \"wan't\": \"will not\",\n                \"that`s\": \"that is\", \"didn`t\": \"dod not\", \"hold'em\": \"holdaem\", \"din't\": \"did not\",\n                \"isn't\": \"is not\"}\n\ndef _get_mispell(mispell_dict):\n    mispell_re = re.compile('(%s)' % '|'.join(mispell_dict.keys()))\n    return mispell_dict, mispell_re\n\nmispellings, mispellings_re = _get_mispell(mispell_dict)\nrx_xxxheight = re.compile('\\d+[\\'\"]\\d+(.\\d+)?')\nrx_xxxtime_or_score = re.compile('\\d+:\\d+')\n\ndef replace(match):\n        return mispellings[match.group(0)]\n\ndef replace_typical_misspell(text):\n    text = rx_xxxheight.sub('xxxheight', text)\n    text = rx_xxxtime_or_score.sub('xxxtime_or_score', text)\n    return mispellings_re.sub(replace, text)\n\n# Clean speelings\ntrain_df[\"question_text\"] = train_df[\"question_text\"].apply(lambda x: replace_typical_misspell(x))\ntest_df[\"question_text\"] = test_df[\"question_text\"].apply(lambda x: replace_typical_misspell(x))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bf0f1a27fa3fc28cef98d3f8417811d91fef31e6"},"cell_type":"code","source":"%%time\n# Load weights\ndef get_coefs(w, *x):\n    return w,np.asarray(x, dtype='float32')\n\n\nkeywords = ['xxunk','xxpad','xxbos','xxfld','xxmaj','xxup','xxrep','xxwrep','xxxheight', 'xxxtime_or_score']\n\nemb_mean = -0.0033469964\nemb_std = 0.109855406\nnb_r_words = len(keywords)\n\nwith open(EMBEDDING_FILE) as f:\n    total, size = f.readline().split(' ')\n    total = int(total)\n    \n    embeddings = torch.zeros((total+nb_r_words, EMBED_SIZE), dtype=torch.float32)\n    embeddings.normal_(emb_mean, emb_std)\n#     embeddings = np.random.normal(emb_mean, emb_std, (total+nb_r_words, EMBED_SIZE)).astype(np.float32)\n    \n    for i, line in enumerate(tqdm(f, total=total)):\n        word, weight = get_coefs(*line.split(' '))\n        embeddings[i+nb_r_words] = torch.from_numpy(weight[:EMBED_SIZE])\n        keywords.append(word)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f00123ce4a98b4a8169fbbdd4c57ed60cd96bb1e"},"cell_type":"code","source":"# emb_mean,emb_std = embeddings.mean(), embeddings.std()\n# emb_mean,emb_std # (-0.00334699644103647, 0.10985540269880754)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"40590730fe79f369a4a334a4d81e13476db638a4"},"cell_type":"code","source":"vocab = Vocab(itos=keywords)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d456198e89e5d40fd5f707dcad8eb5d5b313a6ae"},"cell_type":"code","source":"train_df = train_df.iloc[np.random.permutation(len(train_df))]\ncut = int(0.1 * len(train_df)) + 1\ntrain_df, valid_df = train_df[cut:], train_df[:cut]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f348dc0717c08af292bd4f139e75dedb90585fd7"},"cell_type":"code","source":"%%time\ndata = TextDataBunch.from_df(path='.',\n                             train_df=train_df, \n                             valid_df=valid_df,\n                             test_df=test_df,\n                             text_cols='question_text', \n                             label_cols='target',\n                             max_vocab=MAX_FEATURES,\n                            vocab=vocab)\nprint(len(data.vocab.itos))\ndata.save()\ndel train_df\ndel valid_df \ndel test_df \ndel data\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"03e97e47f6ccfcde3c92a7a619fba0fdd6c5d53e"},"cell_type":"markdown","source":"### Classifier"},{"metadata":{"trusted":true,"_uuid":"3f86fd068919a9e1c2f56720a0f64009b230b232"},"cell_type":"code","source":"%%time\ndata = TextClasDataBunch.load(path='.', bs=64)\ndata.show_batch()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"40be7db82c623acfbe48fb12f37bf87e911548ed"},"cell_type":"code","source":"# # Load weights\n# def get_coefs(w, *x):\n#     return w,np.asarray(x, dtype='float16')\n\n# embeddings_index = {}\n# with open(EMBEDDING_FILE) as f:\n#     total, size = f.readline().split(' ')\n#     total = int(total)\n    \n#     for line in tqdm(f, total=total):\n#         word, weight = get_coefs(*line.split(' '))\n#         embeddings_index[word] = weight[:EMBED_SIZE]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5163ece6fa9bae934117c4e54d8be51e078133dd"},"cell_type":"code","source":"# %%time\n# # mean, std\n# all_embs = np.stack(list(embeddings_index.values()))\n# emb_mean,emb_std = all_embs.mean(), all_embs.std(dtype=np.float32)\n# del all_embs\n# gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d764d545b651d236181b1a34f84eee2a723ca918"},"cell_type":"code","source":"# # random weights\n# vocab_size = len(data.vocab.itos)\n# embedding_matrix = np.random.normal(emb_mean, emb_std, (vocab_size, EMBED_SIZE)).astype(np.float32)\n# embedding_matrix.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7fd17c526975b3665be13537bcbdbc67cf267670"},"cell_type":"code","source":"# words_without_vec = 0\n# # map pre-trained weights with our data\n# for i, word in enumerate(tqdm(data.vocab.itos)):\n#     if i >= vocab_size: continue\n#     embedding_vector = embeddings_index.get(word)\n#     if embedding_vector is not None: \n#         embedding_matrix[i] = embedding_vector\n#     else:\n#         words_without_vec = words_without_vec+1\n#         print(i, word)\n\n# embedding_matrix = torch.from_numpy(embedding_matrix) \n# del embeddings_index\n# gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"aa9e8c981a029c0a1d84b749e4365f1c40e86f3d"},"cell_type":"code","source":"# class QuoraInsincere(nn.Module):\n#     def __init__(self, embedding_wights):\n#         super(QuoraInsincere, self).__init__()\n#         self.embeddings = nn.Embedding.from_pretrained(embedding_wights, freeze=False)\n#         self.linear1 = nn.Linear(EMBED_SIZE*MAX_LENGTH, 1)\n        \n#     def forward(self, inputs):\n#         x = self.embeddings(inputs)\n#         x = x.view(x.size(0), -1)\n#         x = self.linear1(x)\n#         x = torch.sigmoid(x)        \n#         return x\n# model = QuoraInsincere(embeddings)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5ef2c1622900ba0469e424df962f2504440a9655"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f00067076e0a544d9c6f9789253f4c478d2def18"},"cell_type":"code","source":"learner = text_classifier_learner(data, drop_mult=0.5, emb_sz=EMBED_SIZE, nl=1, nh=10, max_len=MAX_LENGTH)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"373bb557055bfb3291d928907e8357472c55af9c"},"cell_type":"code","source":"learner.model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d71a4b3361c8240598205e4a7b53d3a5158a3ec8"},"cell_type":"code","source":"# load our new weights to extsing model\nencoderModel =  next(learner.model.children())\n# encoder = nn.Embedding.from_pretrained(embedding_matrix, freeze=False)\nencoderModel.encoder.load_state_dict({'weight': embeddings})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"92a025a5e3bcf2d15c3620fcfd8b7be45b96d080"},"cell_type":"code","source":"learner.unfreeze()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"df101a4b5402deb3f77e1985ab9c2ca09ebfe363"},"cell_type":"code","source":"learner.lr_find()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d328d937d0097efe0baa8fbe3fbadfe355b76ed7"},"cell_type":"code","source":"learner.recorder.plot(skip_end=5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"300ade2679d46d8f0006adc52bb0ae77aa2474d3"},"cell_type":"code","source":"learner.metrics.append(f1_score)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"87b9d18abd2f80b43652003e4647eede229da9db"},"cell_type":"code","source":"learner.fit_one_cycle(1, 7e-2, moms=(0.8,0.7))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ccbb3daaf3a310543a9ae3cab8c267a49e113de7"},"cell_type":"code","source":"learner.save('first')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d0cd335623f18df054b3e2a21641f6cb4961606b"},"cell_type":"code","source":"learner.load('first');","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"401c2b10611310e1773e4f6d2c702f99877cdc5e"},"cell_type":"code","source":"learner.freeze_to(-2)\nlearner.fit_one_cycle(2, slice(1e-3,7e-2), moms=(0.8,0.7))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"462261773845b6112dc1494efe7ed3b86ade2be1"},"cell_type":"code","source":"learner.save('second')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0cf46c8efac9b98e2add6178e2040c9bd7eb8d0f"},"cell_type":"code","source":"learner.load('second');","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"aca8e0696051c3dbf93fb4a4996ea041e88247bd"},"cell_type":"code","source":"learner.unfreeze()\nlearner.fit_one_cycle(4, slice(1e-3/(2.6**4),1e-3), moms=(0.8,0.7))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ff27707f84a911c254c878de0fb307707d69954c"},"cell_type":"code","source":"# learner.fit_one_cycle(3, slice(1e-3/(2.6**4),1e-3), moms=(0.8,0.7))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e98558c4401ce04b1169f678fd8a947ab902f7b9"},"cell_type":"markdown","source":"### Test set"},{"metadata":{"trusted":true,"_uuid":"b07b812012da492c5aacac22cdd9ac6cafdb8c7b"},"cell_type":"code","source":"%time learner.predict(\"How much does a tutor earn in Bangalore?\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0a7274f8dadb0d6e6e4ce57bcf8342189d6287d5"},"cell_type":"code","source":"preds = learner.get_preds(ds_type=DatasetType.Test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b77950f8046e7f312fb9add079c9757526ad88d6"},"cell_type":"code","source":"preds = preds[0].argmax(dim=1)\npreds.sum()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b9af1c632c62469e3927364cccec50d7e2a26150"},"cell_type":"code","source":"test_df = pd.read_csv('../input/test.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e57f3df7d3391cfe43b8058670241dcd23bb9ee3"},"cell_type":"code","source":"test_df.drop(['question_text'], axis=1, inplace=True)\ntest_df['prediction'] = preds.numpy()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e800e07519b839cd148e1240b6f2ca90dc7f94a4"},"cell_type":"code","source":"test_df.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d9a0ecf83cdbb0e180b216c5c285ad33867103e6"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}