{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Insincerity Classification using AWD-LSTM\nImplementing AWD-LSTM [1] with fastai."},{"metadata":{},"cell_type":"markdown","source":"## Intializing libraries and dataset"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import os\nimport numpy as np\nimport pandas as pd\nfrom fastai.text import * \nfrom fastai.callbacks import CSVLogger\nfrom shutil import copyfile","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"#Setting path for learner\npath = Path(os.path.abspath(os.curdir))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Create directory\ndirName = 'models'\n \ntry:\n    # Create target Directory\n    os.mkdir(dirName)\n    print(\"Directory \" , dirName ,  \" Created \") \nexcept FileExistsError:\n    print(\"Directory \" , dirName ,  \" already exists\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#copying files into working path\nmodelpath = Path('../input/awd-lstm-1')\n\ncopyfile(modelpath/\"models/final.pth\", path/\"models/final.pth\")\ncopyfile(modelpath/\"models/ft_enc1.pth\", path/\"models/ft_enc1.pth\")\ncopyfile(modelpath/\"data_clas_export.pkl\", path/\"data_clas_export.pkl\")\ncopyfile(modelpath/\"data_lm_export.pkl\", path/\"data_lm_export.pkl\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\"\"\"\n#reading into pandas and renaming columns for easier api access\nfilepath = Path('../input/quora-insincere-questions-classification')\ntrn = pd.read_csv(filepath/'train.csv')\ntst = pd.read_csv(filepath/'test.csv')\n\n#For training language model, using both train and test data for more data to learn from\ndf = pd.concat([trn,tst], sort=False)\ndf.rename(columns={'target':'label', 'question_text':'text'},inplace=True)\ndf = df[['label','text']]\ndf.head(2)\n\n#Simple 90-10 split into train/validation set\ntrain = df[:int(len(df)*.9)]\nvalid = df[int(len(df)*.9):]\n\n\"\"\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Language model data\n#data_lm = TextLMDataBunch.from_df(path, train, valid)\n#data_lm.save('data_lm_export.pkl')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data_lm = load_data(path, 'data_lm_export.pkl')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Training a language model, i.e. to predict the next few words\n#learn = language_model_learner(data_lm, AWD_LSTM, drop_mult=0.3, callback_fns=[partial(CSVLogger, append=True)])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#learn.lr_find()\n#learn.recorder.plot()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#learn.fit_one_cycle(4, 1e-2)\n#learn.save('fit_head'); learn.load('fit_head')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#learn.unfreeze()\n#learn.lr_find(); learn.recorder.plot()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#learn.fit_one_cycle(4, 1e-3)\n#learn.save_encoder('ft_enc1')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#learn.predict(\"Why are people\", n_words=10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\"\"\"\ntrn.rename(columns={'target':'label', 'question_text':'text'},inplace=True)\ndf = trn[['label','text']]\n\ntrain = df[:int(len(df)*.80)]\nvalid = df[int(len(df)*.80):]\n\"\"\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data_clas = load_data(path, 'data_clas_export.pkl', bs=16)\n# Classifier model data\n#data_clas = TextClasDataBunch.from_df(path, train, valid, vocab=data_lm.train_ds.vocab, bs=16)\n#data_clas.save('data_clas_export.pkl') ; data_clas = load_data(path, 'data_clas_export.pkl', bs=16)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"learn = text_classifier_learner(data_clas, AWD_LSTM, drop_mult=.3, metrics=[accuracy, FBeta(beta=1, average='binary')],\n                               callback_fns=[partial(CSVLogger, append=True)])\nlearn.load_encoder('ft_enc1') #encoder from first training has 42% accuracy in predicting next word","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"learn.fit_one_cycle(4, 1e-2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"learn.freeze_to(-2)\nlearn.fit_one_cycle(4, slice(1e-3/(2.6**4), 1e-3))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"learn.freeze_to(-3)\nlearn.fit_one_cycle(4, slice(1e-4/(2.6**4), 1e-4))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"learn.unfreeze()\nlearn.fit_one_cycle(4, slice(1e-5/(2.6**4),1e-5))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"learn.predict(\"Why are foreigners so lazy?\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"learn.predict(\"When was SMU founded and why?\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"learn.save('clas-1')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## References\n[1] Stephen Merity, Nitish Shirish Keskar, and Richard Socher. 2017. Regularizing and optimizing lstm language models."}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}