{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\n\n%matplotlib notebook\n\nnp.random.seed(19)","metadata":{"_uuid":"4198b79f00882af3c1de3470a67f3ae8e2585200","_cell_guid":"95dd1390-8d11-4852-bfec-cbf060b4214c","execution":{"iopub.status.busy":"2022-08-08T01:21:05.769998Z","iopub.execute_input":"2022-08-08T01:21:05.770609Z","iopub.status.idle":"2022-08-08T01:21:05.792361Z","shell.execute_reply.started":"2022-08-08T01:21:05.770555Z","shell.execute_reply":"2022-08-08T01:21:05.791679Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_csv('../input/sampless/train.csv')\ntest = pd.read_csv('../input/sampless/test.csv')\nprint(\"Training samples = \", train.shape[0])\nprint(\"Test samples = \", test.shape[0])","metadata":{"_uuid":"5cb822a542916ee1dce17c0e86657f725b3f13f4","_cell_guid":"a24acbe5-c1eb-460a-86f2-36dce29c9147","execution":{"iopub.status.busy":"2022-08-08T01:21:05.817972Z","iopub.execute_input":"2022-08-08T01:21:05.818417Z","iopub.status.idle":"2022-08-08T01:21:08.681244Z","shell.execute_reply.started":"2022-08-08T01:21:05.818371Z","shell.execute_reply":"2022-08-08T01:21:08.679508Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_rows =train.shape[0]\nlabels = train[['toxic', 'severe_toxic', 'obscene', 'threat', 'insult', 'identity_hate']] \ntrain.drop(['id', 'toxic', 'severe_toxic', 'obscene', 'threat', 'insult', 'identity_hate'], axis=1, inplace=True)\ntest_id = test.pop('id')\ntrain.head()","metadata":{"_uuid":"ebe3b7dc8b4bdce8353b474d36886ca7b78ee3d2","_cell_guid":"6138ebc3-e824-47c9-8297-92c679aeda55","execution":{"iopub.status.busy":"2022-08-08T01:21:08.682304Z","iopub.execute_input":"2022-08-08T01:21:08.682559Z","iopub.status.idle":"2022-08-08T01:21:08.729478Z","shell.execute_reply.started":"2022-08-08T01:21:08.682522Z","shell.execute_reply":"2022-08-08T01:21:08.728615Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Null values in training data\", train.isnull().sum(), sep=\"\\n\")\nprint(\"Null values in testing data\", test.isnull().sum(), sep=\"\\n\")","metadata":{"_uuid":"12ede281d2613003381ee8753622c6dfd04194be","_cell_guid":"82f4da1b-8461-49f8-ab03-ff42719c4031","execution":{"iopub.status.busy":"2022-08-08T01:21:08.730478Z","iopub.execute_input":"2022-08-08T01:21:08.730708Z","iopub.status.idle":"2022-08-08T01:21:08.787222Z","shell.execute_reply.started":"2022-08-08T01:21:08.730667Z","shell.execute_reply":"2022-08-08T01:21:08.786588Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data = pd.concat([train, test])\ndel train\ndel test\ndata.shape","metadata":{"_uuid":"d8937f9ab46c99cfdbfa318c0b987866be11515d","_cell_guid":"1d29fc81-0dc0-485b-aac1-81a942e8b6f0","execution":{"iopub.status.busy":"2022-08-08T01:21:08.788294Z","iopub.execute_input":"2022-08-08T01:21:08.788562Z","iopub.status.idle":"2022-08-08T01:21:08.811760Z","shell.execute_reply.started":"2022-08-08T01:21:08.788521Z","shell.execute_reply":"2022-08-08T01:21:08.810980Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Preprocessing","metadata":{}},{"cell_type":"code","source":"import re\nimport nltk\n\nstop_words = set(nltk.corpus.stopwords.words('english'))\n\ndef preprocess_input(comment):\n    comment = comment.strip()\n    comment = comment.lower()\n    comment = re.sub('[\\s0-9]',' ', comment)\n    words = nltk.word_tokenize(comment)\n    words = [word for word in words if len(word) > 2]\n    comment = ' '.join(words)\n    return comment","metadata":{"_uuid":"4bc35fbb93b6e4f788a78c120a987210f92b7693","_cell_guid":"da65e5aa-fe23-4ac8-a623-0822b190f576","execution":{"iopub.status.busy":"2022-08-08T01:21:08.812940Z","iopub.execute_input":"2022-08-08T01:21:08.813212Z","iopub.status.idle":"2022-08-08T01:21:10.288685Z","shell.execute_reply.started":"2022-08-08T01:21:08.813155Z","shell.execute_reply":"2022-08-08T01:21:10.287970Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"SAMPLE PREPROCESSING\")\nprint(\"\\nOriginal comment: \", data.comment_text.iloc[0], sep='\\n')\nprint(\"\\nProcessed comment: \", preprocess_input(data.comment_text.iloc[0]), sep='\\n')","metadata":{"_uuid":"b366343f8c12fd20673624df3c8a494befe39dae","_cell_guid":"54ac822d-ebf3-4f30-8b7a-a217a542a0c3","execution":{"iopub.status.busy":"2022-08-08T01:21:10.289833Z","iopub.execute_input":"2022-08-08T01:21:10.290256Z","iopub.status.idle":"2022-08-08T01:21:10.323670Z","shell.execute_reply.started":"2022-08-08T01:21:10.290212Z","shell.execute_reply":"2022-08-08T01:21:10.322555Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data.comment_text = data.comment_text.apply(lambda row: preprocess_input(row))","metadata":{"_uuid":"f6c9976c4ce1ef161d87c2abbd56fc5484b32fa3","_cell_guid":"32de7e8f-72fb-4abf-8a5a-754526595580","execution":{"iopub.status.busy":"2022-08-08T01:21:10.325215Z","iopub.execute_input":"2022-08-08T01:21:10.325590Z","iopub.status.idle":"2022-08-08T01:25:16.349879Z","shell.execute_reply.started":"2022-08-08T01:21:10.325524Z","shell.execute_reply":"2022-08-08T01:25:16.349021Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data.head()","metadata":{"_uuid":"688548c442868a0c928d45d2688f3cfbd6b98208","_cell_guid":"b1012591-d16b-43e3-b87e-454da453d5bb","execution":{"iopub.status.busy":"2022-08-08T01:25:16.351129Z","iopub.execute_input":"2022-08-08T01:25:16.351484Z","iopub.status.idle":"2022-08-08T01:25:16.361153Z","shell.execute_reply.started":"2022-08-08T01:25:16.351393Z","shell.execute_reply":"2022-08-08T01:25:16.360303Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.feature_extraction.text import TfidfVectorizer\n\nvect = TfidfVectorizer(min_df=0.1, max_df=0.7, \n                       analyzer='char',\n                       ngram_range=(1, 3),\n                       strip_accents='unicode',\n                       sublinear_tf=True,\n                       max_features=30000\n                      )","metadata":{"_uuid":"5d39d86c0a79f879134e100ced3ba98f0061750f","_cell_guid":"0c2f39c5-3bbf-4b65-ab55-56d27799a081","execution":{"iopub.status.busy":"2022-08-08T01:25:16.362538Z","iopub.execute_input":"2022-08-08T01:25:16.362875Z","iopub.status.idle":"2022-08-08T01:25:16.372393Z","shell.execute_reply.started":"2022-08-08T01:25:16.362809Z","shell.execute_reply":"2022-08-08T01:25:16.371648Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test = data[train_rows:]\ntrain = data[:train_rows]\ndel data","metadata":{"_uuid":"9db54d5cc0b08a7dd037fc518f196ae793d9c7e2","execution":{"iopub.status.busy":"2022-08-08T01:25:16.373541Z","iopub.execute_input":"2022-08-08T01:25:16.373888Z","iopub.status.idle":"2022-08-08T01:25:16.383547Z","shell.execute_reply.started":"2022-08-08T01:25:16.373832Z","shell.execute_reply":"2022-08-08T01:25:16.382893Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"vect = vect.fit(train.comment_text)\ntrain = vect.transform(train.comment_text)\ntest = vect.transform(test.comment_text)","metadata":{"_uuid":"a640aeebfba836465f9a1a54248e58de6b820aeb","_cell_guid":"cfc6fe76-1e39-4bd8-8e3c-e4c4aee013f9","execution":{"iopub.status.busy":"2022-08-08T01:25:16.384841Z","iopub.execute_input":"2022-08-08T01:25:16.385380Z","iopub.status.idle":"2022-08-08T01:29:20.683835Z","shell.execute_reply.started":"2022-08-08T01:25:16.385333Z","shell.execute_reply":"2022-08-08T01:29:20.682918Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('Training feature set = ', train.shape)\nprint('Testing feature set = ', test.shape)","metadata":{"_uuid":"8a0d938f78d1612990f15203ed20a592df62fc46","_cell_guid":"df9ae18a-064a-41d4-8556-e6685174c1ac","execution":{"iopub.status.busy":"2022-08-08T01:29:20.685518Z","iopub.execute_input":"2022-08-08T01:29:20.685871Z","iopub.status.idle":"2022-08-08T01:29:20.692064Z","shell.execute_reply.started":"2022-08-08T01:29:20.685808Z","shell.execute_reply":"2022-08-08T01:29:20.691251Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Testing the probability for each of the words\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.model_selection import cross_val_score\ncols = ['toxic', 'severe_toxic', 'obscene', 'threat', 'insult', 'identity_hate']\ny_pred = pd.read_csv('../input/sampless/sample_submission.csv')\n\nfor c in cols:\n    clf = LogisticRegression(C=4, solver='sag')\n    clf.fit(train, labels[c])\n    y_pred[c] = clf.predict_proba(test)[:,1]\n    score = np.mean(cross_val_score(clf, train, labels[c], scoring='roc_auc', cv=5))\n    print(\"ROC_AUC score for\", c, \"=\",  score)","metadata":{"_uuid":"c5989564052abd8e84b091edad0a024127728398","_cell_guid":"3c832f33-abbf-408e-8a48-55e396dc81e3","execution":{"iopub.status.busy":"2022-08-08T01:30:27.336371Z","iopub.execute_input":"2022-08-08T01:30:27.336960Z","iopub.status.idle":"2022-08-08T01:38:58.159641Z","shell.execute_reply.started":"2022-08-08T01:30:27.336911Z","shell.execute_reply":"2022-08-08T01:38:58.158966Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_pred.head()","metadata":{"_uuid":"6625be4463dbe7367426768275aad45a32369276","_cell_guid":"6f47d4b6-f2c3-4687-ad37-c76330148ebc","execution":{"iopub.status.busy":"2022-08-08T01:39:28.628166Z","iopub.execute_input":"2022-08-08T01:39:28.628477Z","iopub.status.idle":"2022-08-08T01:39:28.643721Z","shell.execute_reply.started":"2022-08-08T01:39:28.628407Z","shell.execute_reply":"2022-08-08T01:39:28.643005Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Implementing it to a csv file","metadata":{}},{"cell_type":"code","source":"y_pred.to_csv('submission.csv', index=False)","metadata":{"_uuid":"8a543002d5b54866c8f7344be9cf60f46709d6d9","_cell_guid":"ef203c28-e8f5-4855-b5a1-148670ca966a","execution":{"iopub.status.busy":"2022-08-08T01:39:31.253679Z","iopub.execute_input":"2022-08-08T01:39:31.254258Z","iopub.status.idle":"2022-08-08T01:39:33.217973Z","shell.execute_reply.started":"2022-08-08T01:39:31.254179Z","shell.execute_reply":"2022-08-08T01:39:33.217210Z"},"trusted":true},"execution_count":null,"outputs":[]}]}