{"cells":[{"metadata":{"_uuid":"97b92845b85f289ba795c8c8f7117526abe073d0"},"cell_type":"markdown","source":"## IMPORTS "},{"metadata":{"_uuid":"abb7e3c30b8a412a50c6b451c49939e3cf4bc11b","scrolled":true,"trusted":true},"cell_type":"code","source":"import random\nimport copy\nimport time\nimport pandas as pd\nimport numpy as np\nimport gc\nimport re\nimport torch\nfrom torchtext import data\n#import spacy\nfrom tqdm import tqdm_notebook, tnrange\nfrom tqdm.auto import tqdm\n\ntqdm.pandas(desc='Progress')\nfrom collections import Counter\nfrom textblob import TextBlob\nfrom nltk import word_tokenize\n\nimport torch.nn as nn\nimport torch.optim as optim\nimport torch.nn.functional as F\nfrom torch.utils.data import Dataset, DataLoader\nfrom torch.nn.utils.rnn import pack_padded_sequence, pad_packed_sequence\nfrom torch.autograd import Variable\nfrom torchtext.data import Example\nfrom sklearn.metrics import f1_score\nimport torchtext\nimport os \n\nfrom keras.preprocessing.text import Tokenizer\nfrom keras.preprocessing.sequence import pad_sequences\n\n# cross validation and metrics\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.metrics import f1_score\nfrom torch.optim.optimizer import Optimizer\nfrom unidecode import unidecode\n\nfrom sklearn.preprocessing import StandardScaler\nfrom textblob import TextBlob\nfrom multiprocessing import  Pool\nfrom functools import partial\nimport numpy as np\nfrom sklearn.decomposition import PCA\nimport torch as t\nimport torch.nn as nn\nimport torch.nn.functional as F\n\nfrom fastai.text import * ","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"9a4ff5590a6f152dc1bec5aeca79aef10218f7de"},"cell_type":"markdown","source":"### Basic Parameters"},{"metadata":{"_uuid":"deee49df5ca1c4413f71677939e26aa1ff784e44","scrolled":true,"trusted":true},"cell_type":"code","source":"n_epochs = 5 # how many times to iterate over all samples\nn_splits = 5 # Number of K-fold Splits\nSEED = 10\ndebug = 0","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"63cb21525251b060aeb309e7be4b48772f8720f5","scrolled":true,"trusted":true},"cell_type":"code","source":"def load_and_prec():\n    if debug:\n        train_df = pd.read_csv(\"../input/train.csv\")[:200]\n        test_df = pd.read_csv(\"../input/test.csv\")[:200]\n    else:\n        train_df = pd.read_csv(\"../input/train.csv\")\n        test_df = pd.read_csv(\"../input/test.csv\")\n    print(\"Train shape : \",train_df.shape)\n    print(\"Test shape : \",test_df.shape)\n\n    return train_df,test_df","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"3c72fcddb4f680879e231c3dbfc0c71e27fc424c","scrolled":true,"trusted":true},"cell_type":"code","source":"start = time.time()\ntrain_df,test_df = load_and_prec() \nprint(time.time()-start)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"1921d753ccc29fb2ac1da6fa16c0a4307eb6b07f"},"cell_type":"markdown","source":"# Load data for use in FastAI\n\n## 1. Load Language model data, and Finetune Language model"},{"metadata":{"trusted":true,"_uuid":"aa6a41607b804d76a2ddc530c912b5673bcd2423"},"cell_type":"code","source":"# Language model data : We use test_df as validation for language model\ndata_lm = TextLMDataBunch.from_df(path = \"\",train_df= train_df ,valid_df = test_df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0afac9a7e0484be80f4f4dd0b736101e2b0eaaae"},"cell_type":"code","source":"learn = language_model_learner(data_lm, AWD_LSTM, drop_mult=0.5)\nlearn.fit_one_cycle(1, 1e-2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c2edaf21fd112767361f7688743bd2320be8fa93"},"cell_type":"code","source":"learn.unfreeze()\nlearn.fit_one_cycle(1, 1e-3)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a9831b0fdfc2d323668476897c9b119fcecad697"},"cell_type":"code","source":"# check how the language model performs\nlearn.predict(\"What should\", n_words=10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a79adc5d66a3443bfe46afddb68295a189269d89"},"cell_type":"code","source":"# save the language model\nlearn.save_encoder('ft_enc')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"806a874a48b0e36faec0cd22caba6cecddfdd7b9"},"cell_type":"markdown","source":"\n## 2. Load Classifier data, and learn classifier weights"},{"metadata":{"trusted":true,"_uuid":"6c15fcaf2135dbd081732c2a9e99880311a9bf1f"},"cell_type":"code","source":"def ulmfit_model_run_cv(train_df,test_df,target):\n    # matrix for the out-of-fold predictions\n    train_preds = np.zeros((len(train_df)))\n    # matrix for the predictions on the test set\n    test_preds = np.zeros((len(test_df)))\n    splits = list(StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=SEED).split(train_df,target))\n    for i, (train_idx, valid_idx) in enumerate(splits):\n        print(\"Running Fold:\"+str(i+1))\n        train = train_df.iloc[train_idx]\n        valid = train_df.iloc[valid_idx]\n        # Creating Classification Data\n        print(\"Creating Classification Data\")\n        data_clas = TextClasDataBunch.from_df(path =\"\", train_df=train, valid_df =valid,  test_df=test_df, vocab=data_lm.train_ds.vocab, bs=32,label_cols = 'target')\n        \n        print(\"Creating Classifier Object\")\n        learn = text_classifier_learner(data_clas, AWD_LSTM, drop_mult=0.5)\n        learn.load_encoder('ft_enc')\n        print(\"Fitting Classifier Object\")\n        learn.fit_one_cycle(1, 1e-2)\n        print(\"Fitting Classifier Object after freezing all but last 2 layers\")\n        learn.freeze_to(-2)\n        learn.fit_one_cycle(1, slice(5e-3/2., 5e-3))\n        print(\"Fitting Classifier Object - discriminative learning\")\n        learn.unfreeze()\n        learn.fit_one_cycle(1, slice(2e-3/100, 2e-3))\n        \n        valid_preds_fold = np.array(learn.get_preds(DatasetType.Valid, ordered=True)[0])[:,1]\n        test_preds_fold = np.array(learn.get_preds(DatasetType.Test, ordered=True)[0])[:,1]\n        train_preds[valid_idx] = valid_preds_fold\n        test_preds += test_preds_fold / len(splits)\n        \n    return train_preds, test_preds","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-output":false,"trusted":true,"_uuid":"233b084f64b40584abc46eae40b726a554742474"},"cell_type":"code","source":"train_oof_preds, test_preds = ulmfit_model_run_cv(train_df,test_df,train_df.target)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"12285e49150b847bd381582696f7db5a4f012be7"},"cell_type":"code","source":"def bestThresshold(y_train,train_preds):\n    tmp = [0,0,0] # idx, cur, max\n    delta = 0\n    for tmp[0] in tqdm(np.arange(0.1, 0.501, 0.01)):\n        tmp[1] = f1_score(y_train, np.array(train_preds)>tmp[0])\n        if tmp[1] > tmp[2]:\n            delta = tmp[0]\n            tmp[2] = tmp[1]\n    print('best threshold is {:.4f} with F1 score: {:.4f}'.format(delta, tmp[2]))\n    return delta , tmp[2]\n\ndelta, _ = bestThresshold(train_df.target,train_oof_preds)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"9d1b3fc0c8bc59f91a203adcf3dbd9d89759c8df","trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"fc3e72f43d1ddba1a9d82be0cf9b642f6d5795d6","trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"10c39195707e4710e11d31bafd4bde1e21ff2d6f","trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"162772df446ca6ad6dda6a003d8ea2e58106192f","trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"fdca46457c79c6b56409e6cfbc76d90de4d19cbe","trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"717f6f2fbaa37d01db018baafcf8216fee98ce6a","trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"7a713658e272a3d7ec00267faa5e3d02362269ed","trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"7ce9e3159ce70cdaa0d706249cf80ea3157b39d0","trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"0fb26d857e36473fe7a458cf61692ceff5cadceb","trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8dee2aefa02222d4dd5e67640708cad59285a796","trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8682fc20ab59210e2aba1381232e68ce4e9519d2","trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"2681fc633782d5a7005a0db95de14c6b05b88dac","trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"39e764384d4073282b6a51bcd1ecf382134a33a6","trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"3703e549d2d8d5bb0948c2771b1e6636f0fccef2","trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ee777c692947883aaa2a04349e32c6f45d5702b5"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6e8cd0bbd6cfcd4dd507f4353c29b56cd7594fe2"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f6abad47bd7c3544fc8a3ba49438afa0831131dc"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b9f763bf089050aaa711fc1e371bf83fdca515e8"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"330cc55c14571e7a88586d88676c13ec2677efed"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"eca5b5595de60fd388a4ecb4f602c89eae83d1ec"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"af8a0a6e78675868655470b4b22a8352c15933bb"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"211b2441b513ff07c13c0a5f8920728235db9507"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bc490533a242766921044c4e672e0ae76808850b"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a7c69bba55bf136310a3b890404d03e1628d985b"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"428f9cad34c6eadaf54cf316d107c9d686441fbf"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"16c92267bbbc1d611b3f7f9a1dc1eea24e009b16"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}