{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport seaborn as sns # visualization tools\n\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":40,"outputs":[{"output_type":"stream","text":"['test.csv', 'train.csv', 'sample_submission.csv', 'embeddings']\n","name":"stdout"}]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","collapsed":true,"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":false},"cell_type":"markdown","source":"# Exploratory Data Analysis"},{"metadata":{"trusted":true},"cell_type":"code","source":"#Loading in dataset\ndata = pd.read_csv('../input/train.csv')\n\n# Distribution of labels\ndata['target_encode'] = data['target'].replace([0,1],['sincere','insincere'])\ntarget_plot = sns.countplot(x='target_encode', data=data)\ntarget_plot\nprint('Number of sincere questions:',len(data[data['target_encode']=='sincere']))\nprint('Number of sincere questions:',len(data[data['target_encode']=='insincere']))","execution_count":41,"outputs":[{"output_type":"stream","text":"Number of sincere questions: 1225312\nNumber of sincere questions: 80810\n","name":"stdout"},{"output_type":"display_data","data":{"text/plain":"<Figure size 432x288 with 1 Axes>","image/png":"iVBORw0KGgoAAAANSUhEUgAAAaEAAAELCAYAAABwLzlKAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAGBFJREFUeJzt3Xu0XnV95/H3x0TQarmfMprghKmZzgTqBTKIMnUsOBic1lCLiqOSIsvUEe/TVpjOmliVDoxaCl6YMhAFxxERL0SLxizw0osBEkWuImfwQhiUmCDeliD0O388v6MPh3NOTg45+QXO+7XWs569v/u39+/34IMf9n5+Z+9UFZIk9fCo3gOQJM1dhpAkqRtDSJLUjSEkSerGEJIkdWMISZK6MYQkSd0YQpKkbgwhSVI383sPYFe333771aJFi3oPQ5IeVjZu3PiDqhrZVjtDaBsWLVrEhg0beg9Dkh5WknxnOu28HCdJ6sYQkiR1YwhJkroxhCRJ3RhCkqRuDCFJUjeGkCSpG0NIktSNISRJ6mbW7piQZDXwe8CdVXVwq70T+H3gXuD/AidW1Q/btlOBk4D7gddX1dpWXwacBcwDzquq01v9QOAiYF9gI/CKqro3ye7AhcChwBbgJVX17an6mG2H/umFO6MbPcxsfOcJvYcgdTebZ0IfBJaNq60DDq6qpwDfBE4FSLIEOB44qO3z/iTzkswD3gccAywBXtraApwBnFlVTwbuYhAutPe7Wv3M1m7SPnb0h5YkTd+shVBVfRnYOq72+aq6r62uBxa25eXARVV1T1V9CxgFDmuv0aq6taruZXDmszxJgCOBS9r+FwDHDh3rgrZ8CXBUaz9ZH5KkTnr+JvRK4LNteQFw29C2Ta02WX1f4IdDgTZWf8Cx2va7W/vJjiVJ6qRLCCX5c+A+4MM9+t+WJCuTbEiyYfPmzb2HI0mPWDs9hJL8EYMJCy+rqmrl24EDhpotbLXJ6luAvZLMH1d/wLHa9j1b+8mO9SBVdW5VLa2qpSMj23wchiRphnZqCLWZbn8GvKCqfja0aQ1wfJLd26y3xcBVwNXA4iQHJtmNwcSCNS28vgAc1/ZfAVw6dKwVbfk44IrWfrI+JEmdzOYU7Y8AzwH2S7IJWMVgNtzuwLrBXAHWV9Wrq+qGJBcDNzK4THdyVd3fjvNaYC2DKdqrq+qG1sVbgIuSvAP4GnB+q58PfCjJKIOJEccDTNWHJKmP/OqKmCaydOnSeqhPVvXvhDQR/05Ij2RJNlbV0m21844JkqRuDCFJUjeGkCSpG0NIktSNISRJ6sYQkiR1YwhJkroxhCRJ3RhCkqRuDCFJUjeGkCSpG0NIktSNISRJ6sYQkiR1YwhJkroxhCRJ3RhCkqRuDCFJUjeGkCSpG0NIktSNISRJ6sYQkiR1YwhJkroxhCRJ3RhCkqRuZi2EkqxOcmeS64dq+yRZl+SW9r53qyfJ2UlGk1yb5JChfVa09rckWTFUPzTJdW2fs5Nkpn1IkvqYzTOhDwLLxtVOAS6vqsXA5W0d4BhgcXutBM6BQaAAq4BnAIcBq8ZCpbV51dB+y2bShySpn1kLoar6MrB1XHk5cEFbvgA4dqh+YQ2sB/ZK8gTgecC6qtpaVXcB64BlbdseVbW+qgq4cNyxtqcPSVInO/s3of2r6o62/D1g/7a8ALhtqN2mVpuqvmmC+kz6kCR10m1iQjuDqV2xjyQrk2xIsmHz5s2zMDJJEuz8EPr+2CWw9n5nq98OHDDUbmGrTVVfOEF9Jn08SFWdW1VLq2rpyMjIdn1ASdL07ewQWgOMzXBbAVw6VD+hzWA7HLi7XVJbCxydZO82IeFoYG3b9qMkh7dZcSeMO9b29CFJ6mT+bB04yUeA5wD7JdnEYJbb6cDFSU4CvgO8uDW/DHg+MAr8DDgRoKq2Jnk7cHVr97aqGpvs8BoGM/AeC3y2vdjePiRJ/cxaCFXVSyfZdNQEbQs4eZLjrAZWT1DfABw8QX3L9vYhSerDOyZIkroxhCRJ3RhCkqRuDCFJUjeGkCSpG0NIktSNISRJ6sYQkiR1YwhJkroxhCRJ3RhCkqRuDCFJUjeGkCSpG0NIktSNISRJ6sYQkiR1YwhJkroxhCRJ3RhCkqRuDCFJUjeGkCSpG0NIktSNISRJ6sYQkiR1YwhJkrrpEkJJ3pTkhiTXJ/lIksckOTDJlUlGk3w0yW6t7e5tfbRtXzR0nFNb/eYkzxuqL2u10SSnDNUn7EOS1MdOD6EkC4DXA0ur6mBgHnA8cAZwZlU9GbgLOKntchJwV6uf2dqRZEnb7yBgGfD+JPOSzAPeBxwDLAFe2toyRR+SpA56XY6bDzw2yXzg14A7gCOBS9r2C4Bj2/Lytk7bflSStPpFVXVPVX0LGAUOa6/Rqrq1qu4FLgKWt30m60OS1MFOD6Gquh14F/BdBuFzN7AR+GFV3deabQIWtOUFwG1t3/ta+32H6+P2may+7xR9SJI66HE5bm8GZzEHAk8EHsfgctouI8nKJBuSbNi8eXPv4UjSI1aPy3HPBb5VVZur6hfAJ4AjgL3a5TmAhcDtbfl24ACAtn1PYMtwfdw+k9W3TNHHA1TVuVW1tKqWjoyMPJTPKkmaQo8Q+i5weJJfa7/THAXcCHwBOK61WQFc2pbXtHXa9iuqqlr9+DZ77kBgMXAVcDWwuM2E243B5IU1bZ/J+pAkddDjN6ErGUwO+CpwXRvDucBbgDcnGWXw+835bZfzgX1b/c3AKe04NwAXMwiwzwEnV9X97Tef1wJrgZuAi1tbpuhDktTB/G032fGqahWwalz5VgYz28a3/TnwokmOcxpw2gT1y4DLJqhP2IckqQ/vmCBJ6sYQkiR1YwhJkroxhCRJ3RhCkqRuDCFJUjeGkCSpG0NIktSNISRJ6mZaIZTk8unUJEnaHlPetifJYxg8dG6/9giGtE174LN4JEkP0bbuHffHwBsZPPdnI78KoR8B753FcUmS5oApQ6iqzgLOSvK6qnrPThqTJGmOmNZdtKvqPUmeBSwa3qeqLpylcUmS5oBphVCSDwG/CVwD3N/KBRhCkqQZm+7zhJYCS9rTSSVJ2iGm+3dC1wP/bDYHIkmae6Z7JrQfcGOSq4B7xopV9YJZGZUkaU6Ybgi9dTYHIUmam6Y7O+5Lsz0QSdLcM93ZcT9mMBsOYDfg0cBPq2qP2RqYJOmRb7pnQr8+tpwkwHLg8NkalCRpbtjuu2jXwKeA583CeCRJc8h0L8e9cGj1UQz+bujnszIiSdKcMd3Zcb8/tHwf8G0Gl+QkSZqxaV2Oq6oTh16vqqrTqurOmXaaZK8klyT5RpKbkjwzyT5J1iW5pb3v3domydlJRpNcm+SQoeOsaO1vSbJiqH5okuvaPme337GYrA9JUh/TfajdwiSfTHJne308ycKH0O9ZwOeq6l8BTwVuAk4BLq+qxcDlbR3gGGBxe60Ezmlj2gdYBTwDOAxYNRQq5wCvGtpvWatP1ockqYPpTkz4ALCGwXOFngh8utW2W5I9gWcD5wNU1b1V9UMGl/cuaM0uAI5ty8uBC9uEiPXAXkmewGBixLqq2lpVdwHrgGVt2x5Vtb7d6+7CcceaqA9JUgfTDaGRqvpAVd3XXh8ERmbY54HAZuADSb6W5LwkjwP2r6o7WpvvAfu35QXAbUP7b2q1qeqbJqgzRR8PkGRlkg1JNmzevHkmn1GSNA3TDaEtSV6eZF57vRzYMsM+5wOHAOdU1dOBnzLuslg7g5nVO3ZP1UdVnVtVS6tq6cjITLNWkrQt0w2hVwIvZnD2cAdwHPBHM+xzE7Cpqq5s65cwCKXvt0tptPexiQ+3AwcM7b+w1aaqL5ygzhR9SJI6mG4IvQ1YUVUjVfUbDELpL2bSYVV9D7gtyW+10lHAjQx+cxqb4bYCuLQtrwFOaLPkDgfubpfU1gJHJ9m7TUg4Gljbtv0oyeFtVtwJ4441UR+SpA6m+3dCT2k//gNQVVuTPP0h9Ps64MNJdgNuBU5kEIgXJzkJ+A6DMy+Ay4DnA6PAz1rbsTG8Hbi6tXtbVW1ty68BPgg8FvhsewGcPkkfkqQOphtCj0qy91gQtenR0933QarqGgZ3XRjvqAnaFnDyJMdZDayeoL4BOHiC+paJ+pAk9THdIHk38JUkH2vrLwJOm50hSZLmiuneRfvCJBuAI1vphVV14+wNS5I0F0z7kloLHYNHkrTDbPejHCRJ2lEMIUlSN4aQJKkbQ0iS1I0hJEnqxhCSJHVjCEmSujGEJEndGEKSpG4MIUlSN4aQJKkbQ0iS1I0hJEnqxhCSJHVjCEmSujGEJEndGEKSpG4MIUlSN4aQJKkbQ0iS1I0hJEnqxhCSJHXTLYSSzEvytSSfaesHJrkyyWiSjybZrdV3b+ujbfuioWOc2uo3J3neUH1Zq40mOWWoPmEfkqQ+ep4JvQG4aWj9DODMqnoycBdwUqufBNzV6me2diRZAhwPHAQsA97fgm0e8D7gGGAJ8NLWdqo+JEkddAmhJAuB/wCc19YDHAlc0ppcABzblpe3ddr2o1r75cBFVXVPVX0LGAUOa6/Rqrq1qu4FLgKWb6MPSVIHvc6E/hr4M+Cf2vq+wA+r6r62vglY0JYXALcBtO13t/a/rI/bZ7L6VH08QJKVSTYk2bB58+aZfkZJ0jbs9BBK8nvAnVW1cWf3PV1VdW5VLa2qpSMjI72HI0mPWPM79HkE8IIkzwceA+wBnAXslWR+O1NZCNze2t8OHABsSjIf2BPYMlQfM7zPRPUtU/QhSepgp58JVdWpVbWwqhYxmFhwRVW9DPgCcFxrtgK4tC2vaeu07VdUVbX68W323IHAYuAq4GpgcZsJt1vrY03bZ7I+JEkd7Ep/J/QW4M1JRhn8fnN+q58P7NvqbwZOAaiqG4CLgRuBzwEnV9X97SzntcBaBrPvLm5tp+pDktRBj8txv1RVXwS+2JZvZTCzbXybnwMvmmT/04DTJqhfBlw2QX3CPiRJfexKZ0KSpDnGEJIkdWMISZK6MYQkSd0YQpKkbgwhSVI3hpAkqRtDSJLUjSEkSerGEJIkdWMISZK6MYQkSd0YQpKkbgwhSVI3hpAkqRtDSJLUjSEkSerGEJIkdWMISZK6MYQkSd0YQpKkbgwhSVI3hpAkqRtDSJLUzU4PoSQHJPlCkhuT3JDkDa2+T5J1SW5p73u3epKcnWQ0ybVJDhk61orW/pYkK4bqhya5ru1zdpJM1YckqY8eZ0L3Af+5qpYAhwMnJ1kCnAJcXlWLgcvbOsAxwOL2WgmcA4NAAVYBzwAOA1YNhco5wKuG9lvW6pP1IUnqYKeHUFXdUVVfbcs/Bm4CFgDLgQtaswuAY9vycuDCGlgP7JXkCcDzgHVVtbWq7gLWAcvatj2qan1VFXDhuGNN1IckqYOuvwklWQQ8HbgS2L+q7mibvgfs35YXALcN7bap1aaqb5qgzhR9SJI66BZCSR4PfBx4Y1X9aHhbO4Op2ex/qj6SrEyyIcmGzZs3z+YwJGlO6xJCSR7NIIA+XFWfaOXvt0tptPc7W/124ICh3Re22lT1hRPUp+rjAarq3KpaWlVLR0ZGZvYhJUnb1GN2XIDzgZuq6q+GNq0Bxma4rQAuHaqf0GbJHQ7c3S6prQWOTrJ3m5BwNLC2bftRksNbXyeMO9ZEfUiSOpjfoc8jgFcA1yW5ptX+C3A6cHGSk4DvAC9u2y4Dng+MAj8DTgSoqq1J3g5c3dq9raq2tuXXAB8EHgt8tr2Yog9JUgc7PYSq6u+BTLL5qAnaF3DyJMdaDayeoL4BOHiC+paJ+pAk9eEdEyRJ3RhCkqRuDCFJUjeGkCSpG0NIktSNISRJ6sYQkiR1YwhJkroxhCRJ3RhCkqRuDCFJUjeGkCSpG0NIktSNISRJ6sYQkiR10+OhdpJ2Ed9922/3HoJ2QU/6b9fttL48E5IkdWMISZK6MYQkSd0YQpKkbgwhSVI3hpAkqRtDSJLUjSEkSerGEJIkdTMnQyjJsiQ3JxlNckrv8UjSXDXnQijJPOB9wDHAEuClSZb0HZUkzU1zLoSAw4DRqrq1qu4FLgKWdx6TJM1JczGEFgC3Da1vajVJ0k7mXbQnkGQlsLKt/iTJzT3H8wizH/CD3oPYFeRdK3oPQQ/kd3PMquyIo/zz6TSaiyF0O3DA0PrCVvulqjoXOHdnDmquSLKhqpb2Hoc0nt/NPubi5birgcVJDkyyG3A8sKbzmCRpTppzZ0JVdV+S1wJrgXnA6qq6ofOwJGlOmnMhBFBVlwGX9R7HHOVlTu2q/G52kKrqPQZJ0hw1F38TkiTtIgwhzViS87zbhHa2JP84w/1eneSEHT0ePTRejtMuK8m8qrq/9zik6Ugyv6ru6z2OhxvPhDQtSR6X5G+TfD3J9UlekuSLSZa27T9Jclrbvj7J/q2+f5JPtvrXkzyr1V+e5Kok1yT5m3ZPv7HjvDvJ14FnJjk0yZeSbEyyNskTuv1D0C4hyU/a+3Pad/CSJN9I8uEkadtOT3JjkmuTvKvV3prkT9ryF5Oc0b6D30zyO60+L8m72nf82iSva/UJv4ftOH+dZAPwhiQjST6e5Or2OqLDP6KHFUNI07UM+H9V9dSqOhj43LjtjwPWV9VTgS8Dr2r1s4EvtfohwA1J/jXwEuCIqnoacD/wsqHjXNnaXwm8Bziuqg4FVgOnzdon1MPR04E3MrgZ8b8AjkiyL/AHwEFV9RTgHZPsO7+qDmv7r2q1lcAi4Glt3w8neTRTfw93q6qlVfVu4CzgzKr6N8AfAuftuI/6yDQnp2hrRq4D3p3kDOAzVfV37T86x9wLfKYtbwT+fVs+EjgBoF1auzvJK4BDgavbMR4L3Nna3w98vC3/FnAwsK61mwfcscM/mR7OrqqqTQBJrmEQIOuBnwPnJ/kMv/pejveJ9r6x7QfwXOB/jl1Wq6qtSQ5m6u/hR4eWnwssGfp3Y48kj6+qn8z0Az7SGUKalqr6ZpJDgOcD70hy+bgmv6hf/cB4P1N/twJcUFWnTrDt50O/AwW4oaqe+VDGrke0e4aW72dwdnNfksOAo4DjgNcy+I+hyfadzvd1qu/hT4eWHwUcXlU/n87g5eU4TVOSJwI/q6r/DbyTwaW16bgc+E/tGPOS7NlqxyX5jVbfJ8lENzu8GRhJ8szW7tFJDnqIH0WPcEkeD+zZ/ij9TcBTt2P3dcAfJ5nfjrUP2/c9/DzwuqGxPG0GH2FOMYQ0Xb8NXNUueaxi8uvs470B+N0k1zG47LGkqm4E/ivw+STXMvgX/0ETDtrzno4DzmgTFa4BnvWQP4ke6X4d+Ez7bv098Obt2Pc84LvAte079x+383v4emBpm9RwI/DqmX6IucIp2pKkbjwTkiR1YwhJkroxhCRJ3RhCkqRuDCFJUjeGkCSpG0NI2g5J9krymp3Qz3PGbva6qxi7cai0IxlC0vbZC5h2CGVgJv+ePQf/MFdzgCEkbZ/Tgd9sj6A4M8nlSb6a5LokywGSLEpyc5ILgeuBA5Kc1B4ZcFWS/5Xkva3tg279n2QRg7+0f1Pr53cmGshkjw1ojyxY3R4zcGuS1w/tc0L7a/6vJ/nQ0HivaPXLkzyp1Q9M8pX22d4xru8/bX1em+QvdvA/Y80lVeXLl69pvhjcbfn6tjwf2KMt7weMMrjZ5SLgnxjcyBLgicC3gX2ARwN/B7y3bfs/wL9ty08CbmrLbwX+ZBtjmWrffwR2b+Pa0vo9CPgmsF9rt097/zSwoi2/EvhUW14DnNCWTwZ+0paPBs5tn/VRDO5S/eze/9v4eni+vIu2NHMB/jLJsxmEzgJg/7btO1W1vi0fxuCZSlsBknwM+Jdt24S3/p9m/1Pt+7dVdQ9wT5I727iOBD5WVT+AwWMKWttnAi9syx8C/kdbPoLBM3HG6me05aPb62tt/fHAYgbPkZK2iyEkzdzLgBHg0Kr6RZJvA49p23466V4PNOGt/8c9q2km+z7oEQfTHM94E91cMsB/r6q/meExpV/yNyFp+/yYwV2aAfYE7mwB9LvARI+jALga+HdJ9m6PCPjDoW2T3fp/uJ/JbO9jA64AXtSePDr2mAIYXLo7vi2/jMHlQoB/GFcfsxZ45dhZV5IFY4/lkLaXISRth6raAvxDkuuBpzG4bf91DJ4e+41J9rkd+EvgKgb/x/5t4O62ebJb/38a+IOpJiZMse9kY7+BwWOpv9QeSfBXbdPrgBPbow9eweDxG7T3k9vnWzB0nM8z+D3qK23bJWw7MKUJ+SgHaScYe8RzOxP6JLC6qj7Ze1xSb54JSTvHW9sDAa8HvgV8qvN4pF2CZ0LSLi7JnwMvGlf+WFWd1mM80o5kCEmSuvFynCSpG0NIktSNISRJ6sYQkiR1YwhJkrr5/+MRBYEvmqVwAAAAAElFTkSuQmCC\n"},"metadata":{}}]},{"metadata":{},"cell_type":"markdown","source":"As expected beforehand, the the general distribtuion of the dataset is highly skewed, meaning the amount of insincere comments is much less than the amount of sincere comments in the training dataset. It will definitely later affect the choice of evaluation metrics for the models. "},{"metadata":{},"cell_type":"markdown","source":"# Text Pre-processing / Feature Engineering"},{"metadata":{"trusted":true},"cell_type":"code","source":"import string\nfrom nltk.corpus import stopwords\n\n#Tokenization function\ndef text_process(mess):\n    \"\"\"\n    Takes in a string of text, then performs the following:\n    1. Remove all punctuation\n    2. Remove all stopwords\n    3. Returns a list of the cleaned text\n    \"\"\"\n    # Check characters to see if they are in punctuation\n    nopunc = [char for char in mess if char not in string.punctuation]\n\n    # Join the characters again to form the string.\n    nopunc = ''.join(nopunc)\n    \n    # Now just remove any stopwords\n    return [word for word in nopunc.split() if word.lower() not in stopwords.words('english')]\n\n# Vectorization & TF-IDF Transformation tools\nfrom sklearn.feature_extraction.text import CountVectorizer, TfidfTransformer","execution_count":42,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Preidcition Model & Data Pipeline \nFor the prediction models, Naives Bayes Classifier turns out to be a simple, yet pretty powerful model for many text classification problem for various reasons. We will incorporate the model into our data pipeline with all other preivous preprocessing techniques"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Naives Bayes model\nfrom sklearn.naive_bayes import MultinomialNB\n\n# Create a data pipeline for training and testing our models\nfrom sklearn.pipeline import Pipeline\n\npipeline = Pipeline([\n    ('bow', CountVectorizer(analyzer=text_process)),  # strings to token integer counts\n    ('tfidf', TfidfTransformer()),  # integer counts to weighted TF-IDF scores\n    ('classifier', MultinomialNB()),  # train on TF-IDF vectors w/ Naive Bayes classifier\n])","execution_count":43,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Train-Test Split our the public dataset"},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.model_selection import train_test_split\n\nX_train, X_test, Y_train, Y_test = \\\ntrain_test_split(data['question_text'], data['target'], test_size=0.2)\n\nprint(\"Number of Training Instances:\",len(X_train))\nprint(\"Number of Testing Instances:\",len(X_test))","execution_count":44,"outputs":[{"output_type":"stream","text":"Number of Training Instances: 1044897\nNumber of Testing Instances: 261225\n","name":"stdout"}]},{"metadata":{"trusted":true},"cell_type":"code","source":"training_data = pd.DataFrame({'questions_text':X_train,'target':Y_train})","execution_count":45,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Under/Down Sampling the imblance dataset"},{"metadata":{},"cell_type":"markdown","source":"The resampling technique should be just applied to the training dataset. There are two advantages of downsampling our dataset:\n* **Training time is faster**\n* **The classification algorithm (in this case Naives Bayes) might not be overwhelmed with the number of instances from one particular class. **"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Downsampling the dataset so the ratio of sincere:insincere apprximmately 1:1\ndata_sincere = training_data[training_data['target']==0]\ndata_insincere = training_data[training_data['target']==1]\ndata_sincere_downsample = data_sincere.sample(n=98000)\n\ndata_downsample = pd.concat([data_insincere, data_sincere_downsample], axis=0)\n\nprint(data_downsample.target.value_counts())\n","execution_count":46,"outputs":[{"output_type":"stream","text":"0    98000\n1    64457\nName: target, dtype: int64\n","name":"stdout"}]},{"metadata":{},"cell_type":"markdown","source":"# Train, Test, and Evaluate the Model\nSince our training example is pretty balance now, it should be expected that the precision socre for each class should be high (as long as the we have the right parameter and algorithm). However, all other metrics should also be taken into consideration."},{"metadata":{"trusted":true},"cell_type":"code","source":"# Train the model\npipeline.fit(data_downsample['questions_text'],data_downsample['target'])","execution_count":47,"outputs":[{"output_type":"execute_result","execution_count":47,"data":{"text/plain":"Pipeline(memory=None,\n     steps=[('bow', CountVectorizer(analyzer=<function text_process at 0x7ffaddb68620>,\n        binary=False, decode_error='strict', dtype=<class 'numpy.int64'>,\n        encoding='utf-8', input='content', lowercase=True, max_df=1.0,\n        max_features=None, min_df=1, ngram_range=(1, 1), preprocessor=No...f=False, use_idf=True)), ('classifier', MultinomialNB(alpha=1.0, class_prior=None, fit_prior=True))])"},"metadata":{}}]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Predict on the testing set and evaluate the model\npredictions = pipeline.predict(X_test)\n\n# Classication report\nfrom sklearn.metrics import classification_report\nprint(classification_report(predictions,Y_test))","execution_count":48,"outputs":[{"output_type":"stream","text":"              precision    recall  f1-score   support\n\n           0       0.87      0.99      0.93    216663\n           1       0.83      0.31      0.45     44562\n\n   micro avg       0.87      0.87      0.87    261225\n   macro avg       0.85      0.65      0.69    261225\nweighted avg       0.87      0.87      0.84    261225\n\n","name":"stdout"}]},{"metadata":{},"cell_type":"markdown","source":"# Predict and Submit the Test file"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Read in test data and check the dataframes\ntest_data = pd.read_csv('../input/test.csv')\n\n# Make a prediction and output to csv file\ntest_data_predictions = pipeline.predict(test_data['question_text'])\n\ntest_data['prediction'] = test_data_predictions\ntest_data.to_csv('submission.csv', columns=['qid','prediction'], index=False)","execution_count":49,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}