{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c07c4a173069b9a8ef020debf51495e99069da66"},"cell_type":"markdown","source":"## Import necessary packages"},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd \nimport string\nimport re\nfrom string import digits\nfrom nltk.corpus import stopwords\nfrom nltk.stem import WordNetLemmatizer\nfrom sklearn.feature_extraction.text import TfidfVectorizer\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.pipeline import Pipeline\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import accuracy_score, confusion_matrix\nimport seaborn as sns\nfrom matplotlib import pyplot as plt","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e79674f2fdb6081c817bd93ca0bf48a32b4fd409"},"cell_type":"markdown","source":"## Read the data into a DataFrame"},{"metadata":{"trusted":true,"_uuid":"e375ac05138175be21c8630f96b477aaf53006ae","scrolled":true},"cell_type":"code","source":"train = pd.read_csv('../input/train.csv')\ntest = pd.read_csv('../input/test.csv')\nprint(\"\\nTrain data: \\n\",train.head())\nprint(\"\\nTest data: \\n\",test.head())","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"9328170e03e9e69e89e47a286f5a38e3b2afbe6d"},"cell_type":"markdown","source":"## Drop the null values"},{"metadata":{"trusted":true,"_uuid":"0a93f4308d0e7998009890eee99bd74e570f6906"},"cell_type":"code","source":"train_data=train.drop(train.columns[0], axis=1) \ntest_data=test\nprint(train_data.head())\nprint(test_data.head())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"48fe16f043a161c4a8f05d661c1007ae421980f8","scrolled":true},"cell_type":"code","source":"train_comments=train_data.iloc[:,0]\ntest_comments=test_data.iloc[:,1]\n\n#saving index to separate them later\ntrain_comments_index=train_comments.index\ntest_comments_index=test_comments.index\n\nframes = [train_comments, test_comments]\ncomments = pd.concat(frames, ignore_index=True)\n\n\nlabels=train_data.iloc[:,1:]\n\nprint(\"Train Comments Shape: \",train_comments.shape)\nprint(\"Test Comments Shape: \",test_comments.shape)\nprint(\"Comments Shape after Merge: \",comments.shape)\nprint(\"Comments are: \\n\",comments.head())\nprint(\"\\nLabels are: \\n\", labels.head())","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d7e1a1154d60a05f6159b6350b2005d5f927a728"},"cell_type":"markdown","source":"## Remove Punctuation"},{"metadata":{"trusted":true,"_uuid":"b04105533477ca5b06a55d2bada055149a80e0df"},"cell_type":"code","source":"c=comments.str.translate(str.maketrans(' ', ' ', string.punctuation))\nc.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e6c3d50164d2022742dfdb4c7ec0d5354b016742"},"cell_type":"markdown","source":"## Removing '\\n' and digits"},{"metadata":{"trusted":true,"_uuid":"9e796a626bd2242a229e4170e8aebab03ced144d"},"cell_type":"code","source":"c=c.str.translate(str.maketrans(' ', ' ', '\\n'))\nc=c.str.translate(str.maketrans(' ', ' ', digits))\nc.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8ef2262010377a24362efc3962c164d541d45c2d"},"cell_type":"markdown","source":"## Split combined words \nExample - Convert 'Whoareyou' to 'Who are you'."},{"metadata":{"trusted":true,"_uuid":"f5b02bef1352d5e5ae459043d91274cca69c336c"},"cell_type":"code","source":"c=c.apply(lambda tweet: re.sub(r'([a-z])([A-Z])',r'\\1 \\2',tweet))\nc.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"5d1d21bf817c536cf52b6a0d57f34ddfbe8f26d8"},"cell_type":"markdown","source":"## Convert to lowercase"},{"metadata":{"trusted":true,"_uuid":"16f0b199d076f8e78adec11cbdba189e609a0435"},"cell_type":"code","source":"c=c.str.lower()\nc.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e91f9332070f0b92becd1abf60a73bc6f44c97a6"},"cell_type":"markdown","source":"## Split each sentence using delimiter"},{"metadata":{"trusted":true,"_uuid":"ec6fbf84cc9965983d0a7c936a47daffeb87e4ab"},"cell_type":"code","source":"c=c.str.split()\nc.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"925780ba9180f0764300dd95693d05eae210c625"},"cell_type":"markdown","source":"## Remove Stop Words"},{"metadata":{"trusted":true,"_uuid":"701e601d06c5b26c2242773754e69c980a013f98"},"cell_type":"code","source":"stop = set(stopwords.words('english'))\nc=c.apply(lambda x: [item for item in x if item not in stop])\nc.head()    ","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"15ac3b4b09387dcc9e28270fe64d44d08ca631cb"},"cell_type":"markdown","source":"## Convert Word to Base Form or Lematize"},{"metadata":{"trusted":true,"_uuid":"af8411652f9576854308d606fc37cb6e3fe4b0e9"},"cell_type":"code","source":"from tqdm import tqdm\nlemmatizer = WordNetLemmatizer()\ncom=[]\nfor y in tqdm(c):\n    new=[]\n    for x in y:\n        z=lemmatizer.lemmatize(x)\n        z=lemmatizer.lemmatize(z,'v')\n        new.append(z)\n    y=new\n    com.append(y)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e26bf4808586f4afce1f7c3c5325d57bb8414673"},"cell_type":"markdown","source":"## Lemmatized form is in array form. Convert it to DataFrame using stored index."},{"metadata":{"trusted":true,"_uuid":"92fdf601d8b42628192cab447e2fc548033059c8"},"cell_type":"code","source":"clean_data=pd.DataFrame(np.array(com), index=comments.index,columns={'comment_text'})\nclean_data['comment_text']=clean_data['comment_text'].str.join(\" \")\nprint(clean_data.head())\ntrain_clean_data=clean_data.loc[train_comments_index]\ntest_clean_data=clean_data.drop(train_comments_index,axis=0).reset_index(drop=True)\nprint(\"PreProcessed Train Data : \",train_clean_data.head(5))\nprint(\"PreProcessed Test Data : \",test_clean_data.head(5))\nframes=[train_clean_data,labels]\ntrain_result = pd.concat(frames,axis=1)\nframes=[test.iloc[:,0],test_clean_data]\ntest_result = pd.concat(frames,axis=1)\nprint(train_result.head())\nprint(test_result.head())","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c6d169b2116c7d6a8441b2527445db90eeb0b290"},"cell_type":"markdown","source":"## Are the labels inter-related?"},{"metadata":{"trusted":true,"_uuid":"5b6d87b615597e2fbe46f82eed50274afe1edd45"},"cell_type":"code","source":"temp_df=train_result.iloc[:,2:-1]\ncorr=temp_df.corr()\nplt.figure(figsize=(10,8))\nsns.heatmap(corr,\n            xticklabels=corr.columns.values,\n            yticklabels=corr.columns.values, annot=True)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"6dc23c5565aefcb1238e1855672b798520dc8071"},"cell_type":"markdown","source":"## Convert a collection of raw documents to a matrix of TF-IDF features"},{"metadata":{"trusted":true,"_uuid":"c4ed9dbd7d634185945ead30eab36aa779c5df77"},"cell_type":"code","source":"tf_idf = TfidfVectorizer(max_features=50000, min_df=2)\ntfidf_train = tf_idf.fit_transform(train_result['comment_text'])\ntfidf_test = tf_idf.transform(test_result['comment_text'])\n# import pickle\n# pickle.dump(tf_idf.vocabulary_,open(\"feature.pkl\",\"wb\"))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"b329337ea39d661622e0aefc347a91b967ea851a"},"cell_type":"markdown","source":"## Neural network implementation : Building the model"},{"metadata":{"trusted":true,"_uuid":"49a2661024f5c9cf24942e2a8bb6bd0e3dd60d6d","scrolled":true},"cell_type":"code","source":"from keras.layers import Dense\nfrom keras.models import Sequential\nfrom keras.utils import to_categorical\nmodel = Sequential()\nmodel.add(Dense(100,activation='relu',input_shape=(50000,)))\nmodel.add(Dense(100,activation='relu',input_shape=(50000,)))\nmodel.add(Dense(6,activation='sigmoid'))\nmodel.compile(optimizer='adam',loss='mean_squared_error',metrics=['accuracy'])\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"366c1b957f35ee32335b1166f759e8efcdfc66c9"},"cell_type":"markdown","source":"## Fit the training data using the vectorized matrix"},{"metadata":{"trusted":true,"_uuid":"b4621742f4aad6a820b117ad2a86c4dedac63eb9","scrolled":true},"cell_type":"code","source":"model.fit(tfidf_train, train_result[['toxic','severe_toxic','obscene','threat','insult','identity_hate']].values)\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"7a548e790fc8977b6d64454be053b24d35fc80e8"},"cell_type":"markdown","source":"## Predic the probability of each label in the test dataset"},{"metadata":{"trusted":true,"_uuid":"0132a79667a3a93e5c07ef747739195bbdda6838"},"cell_type":"code","source":"y_pred = model.predict(tfidf_test)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"016a86f5941ee2a13565fecc8bee47046bd279e5"},"cell_type":"markdown","source":"## Save the output as csv file"},{"metadata":{"trusted":true,"_uuid":"d724142b2ee061209a831af0307962b7ea985a1f"},"cell_type":"code","source":"dict = {\n    'id': test_result.id.values,\n    'toxic' : y_pred[:,0],\n    'severe_toxic' : y_pred[:,1],\n    'obscene':y_pred[:,2],\n    'threat':y_pred[:,3],\n    'insult':y_pred[:,4],\n    'identity_hate':y_pred[:,5]\n}\nans = pd.DataFrame(dict)\nans\nans.to_csv('Submit1.csv',index=False)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"1ccb2a36063c1d01eece08224a49b8f47fea492e"},"cell_type":"markdown","source":"## Try and classify your comment"},{"metadata":{"trusted":true,"_uuid":"8c8e8370aac3eb2945b0b87dad1d0b0c914bdfec"},"cell_type":"code","source":"s = input()\nc = s.translate(str.maketrans(' ', ' ', string.punctuation))\nc = c.translate(str.maketrans(' ', ' ', '\\n'))\nc = c.translate(str.maketrans(' ', ' ', digits))\nc = re.sub(r'([a-z])([A-Z])', r'\\1 \\2', c)\nc = c.lower()\nc = c.split()\nstop = set(stopwords.words('english'))\nc = [item for item in c if item not in stop]\nfrom tqdm import tqdm\nlemmatizer = WordNetLemmatizer()\ncom = []\nfor y in tqdm(c):\n    new = []\n    for x in y:\n        z = lemmatizer.lemmatize(x)\n        z = lemmatizer.lemmatize(z, 'v')\n        new.append(z)\n    y = new\n    com.append(y)\nclean = \"\"\nfor i in com:\n    t = ''\n    clean += t.join(i) + \" \"\ntest = tf_idf.transform(np.array([clean]))\ny_pred = model.predict(test)\npred = pd.DataFrame(\n{\n    'label':labels.columns,\n    'probability':y_pred[0]\n})\n# print(train.columns)\nprint(pred)\n","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}