{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport os\nfrom fastai.nlp import *\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nfrom sklearn.model_selection import *\nfrom sklearn.metrics import *\nfrom sklearn.linear_model import *\nprint(os.listdir(\"../input\"))","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"train = pd.read_csv('../input/train.csv')\ntest  = pd.read_csv('../input/test.csv')\nprint(\"Training Data Size: \" + str(train.shape))\nprint(\"Test Data Size: \" + str(test.shape))\ntrain.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"56a8809dceda8440571c5575645855b18a2489db"},"cell_type":"code","source":"train = train.sample(frac = 0.2, random_state = 42)\ntrn=train.iloc[:,1]\ny = train.iloc[:,2]\nx,x_test,y,y_test = train_test_split(trn, y,test_size = 0.2, random_state = 42)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1cccd220e9d54a659f08031baeaf16545bd9d3c4"},"cell_type":"markdown","source":"Competitoin is evaluated based on F1 Score, the formulation of wihich is \n![test](https://wikimedia.org/api/rest_v1/media/math/render/svg/057ffc6b4fa80dc1c0e1f2f1f6b598c38cdd7c23)\nWhere:\n![Precision](https://wikimedia.org/api/rest_v1/media/math/render/svg/26106935459abe7c266f7b1ebfa2a824b334c807)\n![Recall](https://wikimedia.org/api/rest_v1/media/math/render/svg/4c233366865312bc99c832d1475e152c5074891b)\nSince we're dealing with a binary classificaiton problem the F1 formula is relatively straight foreward and can be defined ourselves\n\nAlternatively can use sklearn.metrics.f1_score"},{"metadata":{"trusted":true,"_uuid":"25a697548a7778d4e459ce6c13189683e7079ca1"},"cell_type":"code","source":"def print_scores(y_test, pred):\n    print(\"Accuracy: \"+ str(accuracy_score(y_test, pred)))\n    print(\"F1 Score: \" + str(f1(y_test, pred)))\n\ndef f1(y_true, y_pred):\n    tp = np.logical_and(y_pred == 1, y_true == 1).sum() #true positives\n    fn = np.logical_and(y_pred == 0, y_true == 1).sum() #False Negatives\n    fp = np.logical_and(y_pred == 1, y_true == 0).sum() #False Positives\n    p = tp / (tp+fp)\n    r = tp / (tp+fn)\n    return 2*((p*r)/(p+r))\n\n#lets try it out\ny_pred = np.array([0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,1])\ny_true = np.array([0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,1,1])\nprint_scores(y_true, y_pred)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8e4c12d8f82f233ef5587fe631a055478a3d5e89"},"cell_type":"raw","source":"Note that the F1 score focuses on the positive predcitions. A good example is disease detection. In a rare disease we can get a really high accuracy by simply predicting it will never happen. However that's not very useful. The F1 score is a way to take this into account by really rewarding true positives at relatively little cost to false negatives. In orther words, the score is more about getting the predicting disease where there is disease than predicting disease where there is not disesase (For instance, famously, if you test positive for breast cancer you have still have a better chance of not actually having the disease)"},{"metadata":{"_uuid":"b296448079dfb9244b695b7e144f61d233660ae9"},"cell_type":"markdown","source":"# Set Baseline with Linear Regression"},{"metadata":{"_uuid":"be23121d6438fc7cb50f29c84e7096d770666079","trusted":true},"cell_type":"code","source":"vec = CountVectorizer(tokenizer=tokenize) #Using the FastAI tokenizer\ntrain_tdm = vec.fit_transform(x).sign()\ntest_tdm = vec.transform(x_test).sign()\ntrain_tdm","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9b3e68559fe7746f947a90d9cb7585ee63eb1f81"},"cell_type":"code","source":"m_logreg = LogisticRegression(C= 1e10, dual = True, max_iter = 1000) #C=1e-1 95% accuracy \nm_logreg.fit(train_tdm, y)\npred = m_logreg.predict(test_tdm)\nprint_scores(y_test, pred)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"040299a3015a4d6a9266c566db5c71ed48e3be0b"},"cell_type":"markdown","source":"Not a greate F1 score of 0.45, but a good starting point to compare some other models, recall that the top models in the competition are currently at ~0.7"},{"metadata":{"trusted":true,"_uuid":"06f1d051ae83f4e8c846a9b3da5303290b52c644"},"cell_type":"markdown","source":"## Try with N-Gram Features"},{"metadata":{"trusted":true,"_uuid":"ccccaca667cd6a138eebe560cec49efaa90e7d96"},"cell_type":"code","source":"vec = CountVectorizer(ngram_range = (1,3),tokenizer=tokenize, max_features = 1000000)\ntrain_tdm_ngram = vec.fit_transform(x).sign()\ntest_tdm_ngram = vec.transform(x_test).sign()\ntrain_tdm","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"87c8b51d8269b26fbcdaa4b39c9d3398844bb784"},"cell_type":"code","source":"%%time\nm_logreg = LogisticRegression(C= 1e10, dual = True, max_iter=1000) \nm_logreg.fit(train_tdm_ngram, y)\npred = m_logreg.predict(test_tdm_ngram)\nprint_scores(y_test, pred)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"0d983badacecd950d208d753a85b02c2127c09e7"},"cell_type":"markdown","source":"Using N-Gram gets us another 0.05 in F1 Score"},{"metadata":{"_uuid":"aa35344a37c1e95fec2715aca71c79acd70a9898"},"cell_type":"markdown","source":"## Train a Logistic Regression model with Naive Bayes as a starting point\nFollowing along the FastAI NLP coursework\n\n$$r = \\log \\frac{\\text{ratio of feature $f$ in positive documents}}{\\text{ratio of feature $f$ in negative documents}}$$"},{"metadata":{"trusted":true,"_uuid":"732d17c8def7406d50db499aba476ac85a698cc4"},"cell_type":"code","source":"train_tdm_ngram","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9b02de3ffeed48cffd345747ff00ac25d44a3e63"},"cell_type":"code","source":"def pr(x,y,y_i):\n    p= x[(y==y_i).values].sum(0)\n    return (p+1)/((y==y_i).sum()+1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1a15a01dad6bef15a554a1ec024a4032c7530d23"},"cell_type":"code","source":"r = np.log(pr(train_tdm_ngram,y,1)/pr(train_tdm_ngram,y,0))\nb = np.log((y==1).mean()/ (y==0).mean())\npre_pred = test_tdm_ngram @ r.T + b\npreds = pre_pred>0\nprint_scores(y_test, pd.DataFrame(preds).iloc[:,0])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"82d4aa0269512eadeac59cb2d6445213018aea3d"},"cell_type":"code","source":"x_nb = train_tdm_ngram.multiply(r)\nm= LogisticRegression(C= 1e10, dual = True, max_iter=1000) \nm.fit(x_nb, y)\nx_test_nb = test_tdm_ngram.multiply(r)\npred = m.predict(x_test_nb)\nprint_scores(y_test, pred)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"22b3bb67c98bf6056b7d3d99ae40d7b13f3e53df"},"cell_type":"markdown","source":"# FastAI"},{"metadata":{"trusted":true,"_uuid":"9ce8d14e76d5085d2aa75b48b24cae0fe669e4f8"},"cell_type":"code","source":"sl = 2000\nmd = TextClassifierData.from_bow(train_tdm_ngram, y, test_tdm_ngram, y_test, sl)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"508acfa175c1ea91a1c2aefe0c4fc5b7fe0fc16f"},"cell_type":"code","source":"learner = md.dotprod_nb_learner()\nlearner.fit(0.02, 2, wds=1e-6, cycle_len=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"31ff31ba5fea8e61e2e2a06b6f87f369b21441d4"},"cell_type":"code","source":"preds=learner.predict()\npreds=pd.DataFrame(preds)[1]>0\nprint_scores(y_test, preds)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"5d7cd2588214e390e10b1f55d8955db8570dbf3e"},"cell_type":"markdown","source":"# Build simple Keras NN"},{"metadata":{"trusted":true,"_uuid":"17c9521d245d35777431b0f17e515f62701eb347"},"cell_type":"code","source":"from keras.models import *\nfrom keras.layers import *\nimport tensorflow as tf\nimport keras.backend as K","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"dc007c0c2172e7c86c25ec559e044430d15dd612"},"cell_type":"code","source":"tf.Session()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4d3f29a303938853b2294b6b2d63ba4184e5e5e6"},"cell_type":"code","source":"model = Sequential()\nmodel.add(Dense(500, activation='relu', input_dim=len(vec.get_feature_names())))\nmodel.add(Dense(100, activation='relu'))\nmodel.add(Dense(1, activation='sigmoid'))\nmodel.compile(optimizer='adam',\n             loss = 'binary_crossentropy',\n             metrics=['binary_accuracy'])\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e4ac0218fd4fd3e8a84aba4104c91d641e879922"},"cell_type":"code","source":"%%time\nmodel.fit(train_tdm_ngram, y, epochs=3, batch_size = 64)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"62fa5eabc863040049505627906dd447cfd4b36e"},"cell_type":"code","source":"preds=model.predict(test_tdm_ngram)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b84df2189676e04a4b358eaca0ea7d1e7ef587db"},"cell_type":"code","source":"pred=preds[:,0]>0.19\nprint_scores(y_test, pred)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"dd37e6c11d79b95a0808bc0fe7fca672862cf386"},"cell_type":"markdown","source":"# Final Model Submit"},{"metadata":{"trusted":true,"_uuid":"bc5ac7903cd3c4c25e1d38e2b790b716ba2cb2ef"},"cell_type":"code","source":"test_text = test['question_text']\ntest_tdm = vec.transform(test_text).sign() #maintain same VEC structure \nfinal_pred=model.predict(test_tdm)[:,0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"acd4d7181baf7a10378afb31c4db73872451fe15"},"cell_type":"code","source":"my_submission = pd.DataFrame({'qid': test.qid, 'prediction': final_pred.astype(int)})\nmy_submission.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"efcfd8e8fc27e942f39112044ed536f706099f94"},"cell_type":"code","source":"my_submission.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"988ea7697e9bc7eb6ed89e46f7b9e5293293c2a9"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a226a7dc4c6c8ea7f0fc71f9d546ac1166cbe58e"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}