{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-01-07T07:39:22.229606Z","iopub.execute_input":"2022-01-07T07:39:22.229874Z","iopub.status.idle":"2022-01-07T07:39:22.240170Z","shell.execute_reply.started":"2022-01-07T07:39:22.229842Z","shell.execute_reply":"2022-01-07T07:39:22.239220Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!apt-get install zip","metadata":{"execution":{"iopub.status.busy":"2022-01-07T07:12:56.117492Z","iopub.execute_input":"2022-01-07T07:12:56.117967Z","iopub.status.idle":"2022-01-07T07:12:59.287461Z","shell.execute_reply.started":"2022-01-07T07:12:56.117926Z","shell.execute_reply":"2022-01-07T07:12:59.286066Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Preparation","metadata":{}},{"cell_type":"code","source":"train_csv_data = '../input/quora-insincere-questions-classification/train.csv'\ntest_csv_data = '../input/quora-insincere-questions-classification/test.csv'","metadata":{"execution":{"iopub.status.busy":"2022-01-07T07:39:25.702006Z","iopub.execute_input":"2022-01-07T07:39:25.702252Z","iopub.status.idle":"2022-01-07T07:39:25.706032Z","shell.execute_reply.started":"2022-01-07T07:39:25.702226Z","shell.execute_reply":"2022-01-07T07:39:25.705131Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#import the libraries\nimport matplotlib.pyplot as plt\nimport string\nimport nltk\nimport re\nimport seaborn as sns\nfrom unidecode import unidecode\nfrom nltk.tokenize import word_tokenize\nfrom nltk.corpus import stopwords\nfrom nltk.stem import PorterStemmer, WordNetLemmatizer\nfrom sklearn.utils import resample","metadata":{"execution":{"iopub.status.busy":"2022-01-07T07:39:27.587663Z","iopub.execute_input":"2022-01-07T07:39:27.587915Z","iopub.status.idle":"2022-01-07T07:39:27.592804Z","shell.execute_reply.started":"2022-01-07T07:39:27.587889Z","shell.execute_reply":"2022-01-07T07:39:27.591989Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nprint(os.getcwd())","metadata":{"execution":{"iopub.status.busy":"2022-01-07T07:12:59.315094Z","iopub.execute_input":"2022-01-07T07:12:59.315950Z","iopub.status.idle":"2022-01-07T07:12:59.326839Z","shell.execute_reply.started":"2022-01-07T07:12:59.315856Z","shell.execute_reply":"2022-01-07T07:12:59.325656Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#use pandas to get data from csv\ndf_train = pd.read_csv(train_csv_data)\n\ndf_train.head(5)\n","metadata":{"execution":{"iopub.status.busy":"2022-01-07T07:39:29.499084Z","iopub.execute_input":"2022-01-07T07:39:29.499866Z","iopub.status.idle":"2022-01-07T07:39:32.322084Z","shell.execute_reply.started":"2022-01-07T07:39:29.499821Z","shell.execute_reply":"2022-01-07T07:39:32.321133Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test = pd.read_csv(test_csv_data)\ndf_test.head(5)","metadata":{"execution":{"iopub.status.busy":"2022-01-07T07:39:33.889296Z","iopub.execute_input":"2022-01-07T07:39:33.889593Z","iopub.status.idle":"2022-01-07T07:39:34.739343Z","shell.execute_reply.started":"2022-01-07T07:39:33.889562Z","shell.execute_reply":"2022-01-07T07:39:34.738717Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#drop NA data\ndf_train.dropna(inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-01-07T07:39:36.939066Z","iopub.execute_input":"2022-01-07T07:39:36.939559Z","iopub.status.idle":"2022-01-07T07:39:37.143138Z","shell.execute_reply.started":"2022-01-07T07:39:36.939520Z","shell.execute_reply":"2022-01-07T07:39:37.142386Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#get number of labels'values\ndf_train['target'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-01-07T07:39:38.965712Z","iopub.execute_input":"2022-01-07T07:39:38.965985Z","iopub.status.idle":"2022-01-07T07:39:38.980382Z","shell.execute_reply.started":"2022-01-07T07:39:38.965956Z","shell.execute_reply":"2022-01-07T07:39:38.979572Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#insincere which target is 1 and sincere is 0\ninsincere_data = df_train[df_train['target'] == 1]\nsincere_data = df_train[df_train['target'] == 0]\n","metadata":{"execution":{"iopub.status.busy":"2022-01-07T07:39:40.939872Z","iopub.execute_input":"2022-01-07T07:39:40.940348Z","iopub.status.idle":"2022-01-07T07:39:41.037571Z","shell.execute_reply.started":"2022-01-07T07:39:40.940317Z","shell.execute_reply":"2022-01-07T07:39:41.036916Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#get the percentage of sincere and insincere\n\ny = df_train['target']\ny.value_counts().plot(kind='bar', rot=0)","metadata":{"execution":{"iopub.status.busy":"2022-01-07T07:46:34.479117Z","iopub.execute_input":"2022-01-07T07:46:34.479406Z","iopub.status.idle":"2022-01-07T07:46:34.618643Z","shell.execute_reply.started":"2022-01-07T07:46:34.479372Z","shell.execute_reply":"2022-01-07T07:46:34.615593Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.utils import resample\n\n#under sampling the data\nsincere = df_train[df_train.target == 0]\ninsincere = df_train[df_train.target == 1]\ndf_train_sampled = pd.concat([resample(sincere, replace = True, n_samples = len(insincere)*4), insincere])\ndf_train_sampled","metadata":{"execution":{"iopub.status.busy":"2022-01-07T07:39:45.552812Z","iopub.execute_input":"2022-01-07T07:39:45.553643Z","iopub.status.idle":"2022-01-07T07:39:45.862453Z","shell.execute_reply.started":"2022-01-07T07:39:45.553604Z","shell.execute_reply":"2022-01-07T07:39:45.861639Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#data after under sampling\ny = df_train_sampled['target']\ny.value_counts().plot(kind='bar', rot=0)","metadata":{"execution":{"iopub.status.busy":"2022-01-07T07:44:48.661716Z","iopub.execute_input":"2022-01-07T07:44:48.662005Z","iopub.status.idle":"2022-01-07T07:44:48.801005Z","shell.execute_reply.started":"2022-01-07T07:44:48.661975Z","shell.execute_reply":"2022-01-07T07:44:48.800439Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Preprocess data","metadata":{}},{"cell_type":"code","source":"# get stopwords, punkt, wordnet of english from nltk\nnltk.download('stopwords')\nnltk.download('punkt')\nnltk.download('wordnet')\nstop_words = set(stopwords.words('english'))","metadata":{"execution":{"iopub.status.busy":"2022-01-07T07:40:06.132923Z","iopub.execute_input":"2022-01-07T07:40:06.133208Z","iopub.status.idle":"2022-01-07T07:40:06.140228Z","shell.execute_reply.started":"2022-01-07T07:40:06.133176Z","shell.execute_reply":"2022-01-07T07:40:06.139180Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#define badwords\nbad_words = \"2 girls 1 cup, 2g1c, 4r5e, 5h1t, 5hit, a$$\"\nbad_words = [x.strip() for x in bad_words.split(\",\")]","metadata":{"execution":{"iopub.status.busy":"2022-01-07T07:40:08.345977Z","iopub.execute_input":"2022-01-07T07:40:08.346493Z","iopub.status.idle":"2022-01-07T07:40:08.350998Z","shell.execute_reply.started":"2022-01-07T07:40:08.346460Z","shell.execute_reply":"2022-01-07T07:40:08.350296Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#define acronyms\nACRONYMS = {\n    \"aren't\" : \"are not\", \"can't\" : \"cannot\", \"cant\": \"cannot\", \"couldn't\" : \"could not\", \"didn't\" : \"did not\", \"doesn't\" : \"does not\", \"don't\" : \"do not\", \"hadn't\" : \"had not\", \"hasn't\" : \"has not\", \"haven't\" : \"have not\", \n    \"he'd\" : \"he would\", \"he'll\" : \"he will\", \"he's\" : \"he is\", \n    \"i'd\" : \"I would\", \"i'd\" : \"I had\", \"i'll\" : \"I will\", \"i'm\" : \"I am\", \"isn't\" : \"is not\", \n    \"it's\" : \"it is\", \"it'll\":\"it will\", \"i've\" : \"I have\", \"let's\" : \"let us\", \n    \"mightn't\" : \"might not\", \"mustn't\" : \"must not\", \"shan't\" : \"shall not\", \n    \"she'd\" : \"she would\", \"she'll\" : \"she will\", \"she's\" : \"she is\", \n    \"shouldn't\" : \"should not\", \"that's\" : \"that is\", \"there's\" : \"there is\", \n    \"they'd\" : \"they would\", \"they'll\" : \"they will\", \"they're\" : \"they are\", \"they've\" : \"they have\", \"we'd\" : \"we would\", \"we're\" : \"we are\", \"weren't\" : \"were not\", \"we've\" : \"we have\", \n    \"what'll\" : \"what will\", \"what're\" : \"what are\", \"what's\" : \"what is\", \"what've\" : \"what have\",\n    \"where's\" : \"where is\", \"who'd\" : \"who would\", \"who'll\" : \"who will\", \"who're\" : \"who are\", \"who's\" : \"who is\", \"who've\" : \"who have\",\n    \"won't\" : \"will not\", \"wouldn't\" : \"would not\", \n    \"you'd\" : \"you would\", \"you'll\" : \"you will\", \"you're\" : \"you are\", \"you've\" : \"you have\", \"y'all'd\": \"you all would\",\"y'all'd've\": \"you all would have\",\"y'all're\": \"you all are\", \"y'all've\": \"you all have\", \"you'd\": \"you would\", \"you'd've\": \"you would have\", \"you'll've\": \"you will have\",\n    \"'re\": \" are\", \"wasn't\": \"was not\", \"we'll\":\" will\", \"didn't\": \"did not\", \"tryin'\":\"trying\"\n}","metadata":{"execution":{"iopub.status.busy":"2022-01-07T07:40:10.114194Z","iopub.execute_input":"2022-01-07T07:40:10.114468Z","iopub.status.idle":"2022-01-07T07:40:10.122717Z","shell.execute_reply.started":"2022-01-07T07:40:10.114436Z","shell.execute_reply":"2022-01-07T07:40:10.121783Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''\nTo correct acronym or mispell word on text\n'''\ndef correct_acronym(text):\n    tokens = word_tokenize(text)\n    tokens = [ACRONYMS.get(token) if (ACRONYMS.get(token) != None) else token for token in tokens]\n    text = \" \".join(tokens)\n\n    \n'''\nRemove stopwords which appear from nltk stopwords\n'''\ndef remove_stopword(text):\n    tokens = word_tokenize(text)\n\n    tokens_without_sw = [word for word in tokens if not word in stop_words]\n    text = (' ').join(tokens_without_sw)","metadata":{"execution":{"iopub.status.busy":"2022-01-07T07:40:13.193734Z","iopub.execute_input":"2022-01-07T07:40:13.194505Z","iopub.status.idle":"2022-01-07T07:40:13.200265Z","shell.execute_reply.started":"2022-01-07T07:40:13.194467Z","shell.execute_reply":"2022-01-07T07:40:13.199610Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''\npreprocess data with nomalize text, remove url, puntk, emal, sign, number\n'''\ndef preprocess(text):\n    text = unidecode(text).encode(\"ascii\")\n    text = str(text, \"ascii\")\n    \n    #remove bad word\n    for word in bad_words:\n        text = text.replace(word, \"BAD WORDS\")\n        \n    text = text.lower() #normalize\n    text = re.sub('https?://\\S+|www\\.\\S+', ' ', text) #remove url\n    text = re.sub('<.*?>+', '', text) #remove special character\n    text = re.sub('\\S+@\\S+', ' ', text) #remove email\n    text = re.sub('[%s]' % re.escape(string.punctuation), '', text) #remove sign\n    text = re.sub('(.)\\1+', '\\1', text)\n    text = re.sub('\\d+', ' ', text) #remove number\n    \n    tokens = word_tokenize(text)\n    tokens = [ACRONYMS.get(token) if (ACRONYMS.get(token) != None) else token for token in tokens]\n    text = \" \".join(tokens)\n    \n\n    tokens_without_sw = [word for word in tokens if not word in stop_words]\n    text = (' ').join(tokens_without_sw)\n    return text","metadata":{"execution":{"iopub.status.busy":"2022-01-07T07:40:15.981463Z","iopub.execute_input":"2022-01-07T07:40:15.981987Z","iopub.status.idle":"2022-01-07T07:40:15.989477Z","shell.execute_reply.started":"2022-01-07T07:40:15.981953Z","shell.execute_reply":"2022-01-07T07:40:15.988787Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#create new column of question_text column after preprocess\ndf_train_sampled['question_text_preprocess'] = df_train_sampled['question_text'].apply(preprocess)\ndf_train_sampled.head(5)","metadata":{"execution":{"iopub.status.busy":"2022-01-07T07:40:26.991325Z","iopub.execute_input":"2022-01-07T07:40:26.991766Z","iopub.status.idle":"2022-01-07T07:41:23.634262Z","shell.execute_reply.started":"2022-01-07T07:40:26.991723Z","shell.execute_reply":"2022-01-07T07:41:23.633452Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from wordcloud import WordCloud","metadata":{"execution":{"iopub.status.busy":"2022-01-07T07:42:30.953312Z","iopub.execute_input":"2022-01-07T07:42:30.953704Z","iopub.status.idle":"2022-01-07T07:42:30.958197Z","shell.execute_reply.started":"2022-01-07T07:42:30.953651Z","shell.execute_reply":"2022-01-07T07:42:30.957290Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#using wordcloud to know the words which have most prequence or importance of sincere\nsincere = sincere_wordcloud = WordCloud(width=800, height=600, background_color='white', min_font_size=10).generate(str(df_train_sampled[df_train_sampled[\"target\"] == 0][\"question_text_preprocess\"]))\nplt.figure(figsize=(8, 8))\nplt.imshow(sincere_wordcloud)\nplt.axis(\"off\")\nplt.tight_layout(pad=0)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-01-07T07:47:59.437759Z","iopub.execute_input":"2022-01-07T07:47:59.438031Z","iopub.status.idle":"2022-01-07T07:48:00.397661Z","shell.execute_reply.started":"2022-01-07T07:47:59.438004Z","shell.execute_reply":"2022-01-07T07:48:00.396617Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#using wordcloud to know the words which have most prequence or importance of insincere\nsincere = sincere_wordcloud = WordCloud(width=800, height=600, background_color='white', min_font_size=10).generate(str(df_train_sampled[df_train_sampled[\"target\"] == 1][\"question_text_preprocess\"]))\nplt.figure(figsize=(8, 8))\nplt.imshow(sincere_wordcloud)\nplt.axis(\"off\")\nplt.tight_layout(pad=0)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-01-07T07:48:09.321868Z","iopub.execute_input":"2022-01-07T07:48:09.322183Z","iopub.status.idle":"2022-01-07T07:48:10.172822Z","shell.execute_reply.started":"2022-01-07T07:48:09.322097Z","shell.execute_reply":"2022-01-07T07:48:10.172016Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.linear_model import LogisticRegression\nfrom sklearn.feature_extraction.text import TfidfVectorizer\nfrom sklearn.feature_extraction.text import CountVectorizer\nfrom time import time\n\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.metrics import precision_score, recall_score, f1_score\n","metadata":{"execution":{"iopub.status.busy":"2022-01-07T07:48:38.635452Z","iopub.execute_input":"2022-01-07T07:48:38.635729Z","iopub.status.idle":"2022-01-07T07:48:38.640437Z","shell.execute_reply.started":"2022-01-07T07:48:38.635701Z","shell.execute_reply":"2022-01-07T07:48:38.639591Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Vectorize data","metadata":{}},{"cell_type":"code","source":"#caculate the weight of words\ncvec = CountVectorizer()\ncvec.fit(df_train_sampled.question_text)\n\n#transform train data to weight matrix\nsin_doc_matrix = cvec.transform(df_train_sampled[df_train_sampled.target == 0].question_text_preprocess)\ninsin_doc_matrix = cvec.transform(df_train_sampled[df_train_sampled.target == 1].question_text_preprocess)\n\nsin_tf = np.sum(sin_doc_matrix,axis=0)\ninsin_tf = np.sum(insin_doc_matrix,axis=0)\n\nsin = np.squeeze(np.asarray(sin_tf))\ninsin = np.squeeze(np.asarray(insin_tf))\n\nterm_freq_df = pd.DataFrame([sin, insin],\n                            columns=cvec.get_feature_names()).transpose()","metadata":{"execution":{"iopub.status.busy":"2022-01-07T07:48:55.092083Z","iopub.execute_input":"2022-01-07T07:48:55.092611Z","iopub.status.idle":"2022-01-07T07:49:07.827988Z","shell.execute_reply.started":"2022-01-07T07:48:55.092576Z","shell.execute_reply":"2022-01-07T07:49:07.826973Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"term_freq_df.columns = ['sincere', 'insincere']\nterm_freq_df['total'] = term_freq_df['sincere'] + term_freq_df['insincere']\n\n#the most words appear in both label\nterm_freq_df.sort_values(by='total', ascending=False).iloc[:10]","metadata":{"execution":{"iopub.status.busy":"2022-01-07T07:49:07.829743Z","iopub.execute_input":"2022-01-07T07:49:07.830072Z","iopub.status.idle":"2022-01-07T07:49:07.859288Z","shell.execute_reply.started":"2022-01-07T07:49:07.830030Z","shell.execute_reply":"2022-01-07T07:49:07.858446Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#show the top 10 insincere\ny_pos = np.arange(10)\nplt.figure(figsize=(20,10))\nplt.bar(y_pos, term_freq_df.sort_values(by='insincere', ascending=False)['insincere'][:10], align='center', alpha=0.5)\nplt.xticks(y_pos, term_freq_df.sort_values(by='sincere', ascending=False)['sincere'][:10].index,rotation='vertical')\nplt.ylabel('Frequency')\nplt.xlabel('Top 10 insincere tokens')\nplt.title('Top 10 tokens in insincere tweets')","metadata":{"execution":{"iopub.status.busy":"2022-01-07T07:49:15.515891Z","iopub.execute_input":"2022-01-07T07:49:15.516376Z","iopub.status.idle":"2022-01-07T07:49:15.791433Z","shell.execute_reply.started":"2022-01-07T07:49:15.516344Z","shell.execute_reply":"2022-01-07T07:49:15.790502Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#show the top 10 sincere\ny_pos = np.arange(10)\nplt.figure(figsize=(20,10))\nplt.bar(y_pos, term_freq_df.sort_values(by='sincere', ascending=False)['insincere'][:10], align='center', alpha=0.5)\nplt.xticks(y_pos, term_freq_df.sort_values(by='sincere', ascending=False)['insincere'][:10].index,rotation='vertical')\nplt.ylabel('Frequency')\nplt.xlabel('Top 10 sincere tokens')\nplt.title('Top 10 tokens in sincere tweets')","metadata":{"execution":{"iopub.status.busy":"2022-01-07T07:49:31.399057Z","iopub.execute_input":"2022-01-07T07:49:31.399676Z","iopub.status.idle":"2022-01-07T07:49:31.711475Z","shell.execute_reply.started":"2022-01-07T07:49:31.399635Z","shell.execute_reply":"2022-01-07T07:49:31.710686Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#the prequent appearance of both insincere and sincere\nplt.figure(figsize=(8,6))\nax = sns.regplot(x=\"sincere\", y=\"insincere\",fit_reg=False, scatter_kws={'alpha':0.5},data=term_freq_df)\nplt.ylabel('Insincere Frequency')\nplt.xlabel('Sincere Frequency')\nplt.title('Sincere Frequency vs Insincere Frequency')","metadata":{"execution":{"iopub.status.busy":"2022-01-07T07:49:46.204267Z","iopub.execute_input":"2022-01-07T07:49:46.204598Z","iopub.status.idle":"2022-01-07T07:49:46.989737Z","shell.execute_reply.started":"2022-01-07T07:49:46.204562Z","shell.execute_reply":"2022-01-07T07:49:46.988755Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.preprocessing import LabelBinarizer\nlabel_target = LabelBinarizer(sparse_output=True)\ntrain_target= label_target.fit_transform(df_train_sampled['target'])\n\ntrain_target.shape","metadata":{"execution":{"iopub.status.busy":"2022-01-07T07:50:14.630647Z","iopub.execute_input":"2022-01-07T07:50:14.630934Z","iopub.status.idle":"2022-01-07T07:50:14.677252Z","shell.execute_reply.started":"2022-01-07T07:50:14.630901Z","shell.execute_reply":"2022-01-07T07:50:14.676457Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#devide train data to 3 part: train, validation and test\n\nx = df_train_sampled.question_text_preprocess\ny = df_train_sampled.target\nfrom sklearn.model_selection import train_test_split\nSEED = 2000\nx_train, x_validation_and_test, y_train, y_validation_and_test = train_test_split(x, y, test_size=.02, random_state=SEED)\nx_validation, x_test, y_validation, y_test = train_test_split(x_validation_and_test, y_validation_and_test, test_size=.5, random_state=SEED)\nprint(\"Train set has total {0} entries with \\n {1:.2f}% sincere, {2:.2f}% insincere\".format(\n    len(x_train), \n    (len(x_train[y_train == 0]) / (len(x_train)*1.))*100, \n    (len(x_train[y_train == 1]) / (len(x_train)*1.))*100))\nprint(\"Validation set has total {0} entries with \\n {1:.2f}% sincere, {2:.2f}% insincere\".format(\n    len(x_validation), \n    (len(x_validation[y_validation == 0]) / (len(x_validation)*1.))*100, \n    (len(x_validation[y_validation == 1]) / (len(x_validation)*1.))*100))\nprint(\"Test set has total {0} entries with \\n {1:.2f}% sincere, {2:.2f}% insincere\".format(\n    len(x_test),\n    (len(x_test[y_test == 0]) / (len(x_test)*1.))*100,\n    (len(x_test[y_test == 1]) / (len(x_test)*1.))*100))","metadata":{"execution":{"iopub.status.busy":"2022-01-07T07:50:23.572350Z","iopub.execute_input":"2022-01-07T07:50:23.573145Z","iopub.status.idle":"2022-01-07T07:50:23.782944Z","shell.execute_reply.started":"2022-01-07T07:50:23.573097Z","shell.execute_reply":"2022-01-07T07:50:23.781904Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#but in here I just train with train and test with validation and test\ncvec.fit(x_train)\n\nvt_x_train = cvec.transform(x_train)\nvt_x_test = cvec.transform(x_validation_and_test)\n\n","metadata":{"execution":{"iopub.status.busy":"2022-01-07T07:50:29.965726Z","iopub.execute_input":"2022-01-07T07:50:29.966155Z","iopub.status.idle":"2022-01-07T07:50:37.460085Z","shell.execute_reply.started":"2022-01-07T07:50:29.966123Z","shell.execute_reply":"2022-01-07T07:50:37.459369Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train model","metadata":{}},{"cell_type":"code","source":"#train model\ncount_vectorizer = LogisticRegression(n_jobs=10, solver='saga', C=0.1, verbose=1)\n\ncount_vectorizer.fit(vt_x_train, y_train)\n\ny_prediction_count_vectorizer = count_vectorizer.predict(vt_x_test)","metadata":{"execution":{"iopub.status.busy":"2022-01-07T07:50:45.512286Z","iopub.execute_input":"2022-01-07T07:50:45.512910Z","iopub.status.idle":"2022-01-07T07:51:12.429119Z","shell.execute_reply.started":"2022-01-07T07:50:45.512860Z","shell.execute_reply":"2022-01-07T07:51:12.428187Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import classification_report","metadata":{"execution":{"iopub.status.busy":"2022-01-07T07:51:14.966693Z","iopub.execute_input":"2022-01-07T07:51:14.966961Z","iopub.status.idle":"2022-01-07T07:51:14.971125Z","shell.execute_reply.started":"2022-01-07T07:51:14.966934Z","shell.execute_reply":"2022-01-07T07:51:14.969871Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#the results\nprint(\"results\\n\")\nprint(classification_report(y_validation_and_test, y_prediction_count_vectorizer))","metadata":{"execution":{"iopub.status.busy":"2022-01-07T07:51:17.915934Z","iopub.execute_input":"2022-01-07T07:51:17.916195Z","iopub.status.idle":"2022-01-07T07:51:17.937123Z","shell.execute_reply.started":"2022-01-07T07:51:17.916164Z","shell.execute_reply":"2022-01-07T07:51:17.936439Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Prediction","metadata":{}},{"cell_type":"code","source":"#prediction with test data\ndf_test['clean_questions'] = df_test['question_text'].apply(preprocess)\ntest_vt_x = cvec.transform(df_test['clean_questions'])\npredictions_test_data = count_vectorizer.predict(test_vt_x)","metadata":{"execution":{"iopub.status.busy":"2022-01-07T07:51:27.426418Z","iopub.execute_input":"2022-01-07T07:51:27.427007Z","iopub.status.idle":"2022-01-07T07:52:21.720424Z","shell.execute_reply.started":"2022-01-07T07:51:27.426971Z","shell.execute_reply":"2022-01-07T07:52:21.719565Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Results and submission","metadata":{}},{"cell_type":"code","source":"#submission\ndf_test['prediction'] = predictions_test_data\nsubmissions = df_test[['qid', 'prediction']]\nsubmissions","metadata":{"execution":{"iopub.status.busy":"2022-01-07T07:52:36.594263Z","iopub.execute_input":"2022-01-07T07:52:36.595170Z","iopub.status.idle":"2022-01-07T07:52:36.652625Z","shell.execute_reply.started":"2022-01-07T07:52:36.595119Z","shell.execute_reply":"2022-01-07T07:52:36.652064Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submissions.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2022-01-07T07:52:40.450825Z","iopub.execute_input":"2022-01-07T07:52:40.451248Z","iopub.status.idle":"2022-01-07T07:52:40.979948Z","shell.execute_reply.started":"2022-01-07T07:52:40.451217Z","shell.execute_reply":"2022-01-07T07:52:40.979068Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}