{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Import libraries"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\n\nfrom fastai.text import *\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Import data"},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"train_pure_data = pd.read_csv(\"/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-toxic-comment-train.csv\")\nvalidation_pure_data = pd.read_csv(\"/kaggle/input/jigsaw-multilingual-toxic-test-translated/jigsaw_miltilingual_valid_translated.csv\")\ntest_pure_data = pd.read_csv(\"/kaggle/input/jigsaw-multilingual-toxic-test-translated/jigsaw_miltilingual_test_translated.csv\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Let's take a look at the imported data"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_pure_data.head(2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"validation_pure_data.head(2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_pure_data.head(2)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Translate test text into english using another dataset"},{"metadata":{"trusted":true},"cell_type":"code","source":"test_pure_data['content'] = test_pure_data['translated']\ntest_pure_data.drop(['translated', 'id'], axis=1, inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_pure_data.head(2)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Translate validation text into english using another dataset"},{"metadata":{"trusted":true},"cell_type":"code","source":"validation_pure_data['comment_text'] = validation_pure_data['translated']\nvalidation_pure_data.drop(['translated', 'id'], axis=1, inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"validation_pure_data.head(2)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Drop 'lang' from validation and test data"},{"metadata":{"trusted":true},"cell_type":"code","source":"validation_pure_data.drop(['lang'], inplace=True, axis=1)\ntest_pure_data.drop(['lang'], inplace=True, axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"validation_pure_data.head(2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_pure_data.head(2)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Rebuild our train data so it contain only one label - toxic if there is any other label"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_pure_data[['toxic', 'severe_toxic', 'obscene', 'threat', 'insult', 'identity_hate']].sum(axis = 0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_pure_data['toxic'] = train_pure_data[['toxic', 'severe_toxic', 'obscene', 'threat', 'insult', 'identity_hate']].sum(axis = 1) > 0","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_pure_data.drop(['severe_toxic', 'obscene', 'threat', 'insult', 'identity_hate', 'id'], inplace=True, axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_pure_data['toxic'] = train_pure_data['toxic'].astype(int)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_pure_data.head(2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_positive_samples = train_pure_data[train_pure_data['toxic'] == 1]\ntrain_negative_samples = train_pure_data[train_pure_data['toxic'] == 0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#final_train = pd.concat([train_positive_samples, train_negative_samples.sample(24000, random_state=3543)])\nfinal_train = train_pure_data.sample(frac=1, random_state=3543)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"final_train.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Modelling"},{"metadata":{},"cell_type":"markdown","source":"Load data into model"},{"metadata":{"trusted":true},"cell_type":"code","source":"data_lm = (TextList.from_df(final_train)\n                   .split_by_rand_pct()\n                   .label_from_df(cols='toxic')\n                   .databunch())\n\ndata_lm.save()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Take a look at one batch"},{"metadata":{"trusted":true},"cell_type":"code","source":"data_lm.show_batch()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Create learner"},{"metadata":{"trusted":true},"cell_type":"code","source":"learn = text_classifier_learner(data_lm, AWD_LSTM)\nlearn.unfreeze()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"learn.fit_one_cycle(10, slice(1e-7, 1e-1))\n#learn.save('mini_train_clas')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"preds,y,losses = learn.get_preds(with_loss=True)\ninterp = ClassificationInterpretation(learn, preds, y, losses)\ninterp.plot_confusion_matrix()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"learn.data.add_test(test_pure_data)\npreds,y = learn.get_preds(ds_type=DatasetType.Test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission = pd.read_csv(\"/kaggle/input/jigsaw-multilingual-toxic-comment-classification/sample_submission.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission['toxic'] = [x[1].item() for x in preds]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}