{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Text Classification with Bag of Words - Natural Language Processing\n\n![](https://i.imgur.com/hlEQ5X8.png)\n\n> _\"Natural language processing (NLP) is a subfield of linguistics, computer science, and artificial intelligence concerned with the interactions between computers and human language, in particular how to program computers to process and analyze large amounts of natural language data.\"_ - Wikipedia\n\n> _**Bag of Words**: The bag-of-words (BOW) model is a representation that turns arbitrary text into fixed-length vectors by counting how many times each word appears._\n \nOutline:\n\n1. Download and explore a real-world dataset\n2. Apply text preprocessing techniques\n3. Implement the bag of words model\n4. Train ML models for text classification\n5. Make predictions and submit to Kaggle\n\n","metadata":{"id":"tBxC0flTQDd3"}},{"cell_type":"markdown","source":"## Download and Explore the Data\n\nOutline:\n\n1. Download the dataset from Kaggle to Colab\n2. Explore the data using Pandas\n3. Create a small working sample","metadata":{"id":"ZOcWwlvxQoSJ"}},{"cell_type":"markdown","source":"### Explore the Data using Pandas","metadata":{"id":"Ql6Gv4shTSBJ"}},{"cell_type":"code","source":"import pandas as pd","metadata":{"executionInfo":{"elapsed":358,"status":"ok","timestamp":1667912043319,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"pf65NRzdQ-hU","execution":{"iopub.status.busy":"2022-12-01T11:20:40.701979Z","iopub.execute_input":"2022-12-01T11:20:40.702492Z","iopub.status.idle":"2022-12-01T11:20:40.734711Z","shell.execute_reply.started":"2022-12-01T11:20:40.702387Z","shell.execute_reply":"2022-12-01T11:20:40.733153Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"raw_df = pd.read_csv('/kaggle/input/quora-insincere-questions-classification/train.csv')","metadata":{"executionInfo":{"elapsed":4754,"status":"ok","timestamp":1667912065510,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"H1FjN5gDTfUY","execution":{"iopub.status.busy":"2022-12-01T11:20:40.737116Z","iopub.execute_input":"2022-12-01T11:20:40.737649Z","iopub.status.idle":"2022-12-01T11:20:46.099755Z","shell.execute_reply.started":"2022-12-01T11:20:40.737602Z","shell.execute_reply":"2022-12-01T11:20:46.098432Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"raw_df","metadata":{"executionInfo":{"elapsed":474,"status":"ok","timestamp":1667912069885,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"CHMgt_5iTfRt","outputId":"7ad71a7a-b478-4a42-bce0-ee467479480f","execution":{"iopub.status.busy":"2022-12-01T11:20:46.101205Z","iopub.execute_input":"2022-12-01T11:20:46.101651Z","iopub.status.idle":"2022-12-01T11:20:46.134887Z","shell.execute_reply.started":"2022-12-01T11:20:46.101613Z","shell.execute_reply":"2022-12-01T11:20:46.133602Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"raw_df.shape","metadata":{"execution":{"iopub.status.busy":"2022-12-01T11:22:47.965704Z","iopub.execute_input":"2022-12-01T11:22:47.967480Z","iopub.status.idle":"2022-12-01T11:22:47.976734Z","shell.execute_reply.started":"2022-12-01T11:22:47.967414Z","shell.execute_reply":"2022-12-01T11:22:47.975112Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sincere_df = raw_df[raw_df.target == 0]","metadata":{"executionInfo":{"elapsed":382,"status":"ok","timestamp":1667912255712,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"wV2xGZ3STfPx","execution":{"iopub.status.busy":"2022-12-01T11:20:46.137802Z","iopub.execute_input":"2022-12-01T11:20:46.138186Z","iopub.status.idle":"2022-12-01T11:20:46.250703Z","shell.execute_reply.started":"2022-12-01T11:20:46.138152Z","shell.execute_reply":"2022-12-01T11:20:46.249269Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sincere_df.question_text.values[:10]","metadata":{"executionInfo":{"elapsed":375,"status":"ok","timestamp":1667912261902,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"me4srOFdUMCQ","outputId":"6f5e9340-bdb2-4c22-b75e-57177ccf6ea9","execution":{"iopub.status.busy":"2022-12-01T11:20:46.252532Z","iopub.execute_input":"2022-12-01T11:20:46.253501Z","iopub.status.idle":"2022-12-01T11:20:46.263589Z","shell.execute_reply.started":"2022-12-01T11:20:46.253451Z","shell.execute_reply":"2022-12-01T11:20:46.262085Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"insincere_df = raw_df[raw_df.target == 1]","metadata":{"executionInfo":{"elapsed":572,"status":"ok","timestamp":1667912293701,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"cTApBtxIUMAJ","execution":{"iopub.status.busy":"2022-12-01T11:20:46.264924Z","iopub.execute_input":"2022-12-01T11:20:46.265328Z","iopub.status.idle":"2022-12-01T11:20:46.296314Z","shell.execute_reply.started":"2022-12-01T11:20:46.265277Z","shell.execute_reply":"2022-12-01T11:20:46.294587Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"insincere_df.question_text.values[:10]","metadata":{"executionInfo":{"elapsed":652,"status":"ok","timestamp":1667912301181,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"-nFN5bZVUL91","outputId":"ce551e6c-3255-408d-9182-4408269d67bf","execution":{"iopub.status.busy":"2022-12-01T11:20:46.298561Z","iopub.execute_input":"2022-12-01T11:20:46.299736Z","iopub.status.idle":"2022-12-01T11:20:46.310438Z","shell.execute_reply.started":"2022-12-01T11:20:46.299408Z","shell.execute_reply":"2022-12-01T11:20:46.308823Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"raw_df.target.value_counts(normalize=True)","metadata":{"executionInfo":{"elapsed":431,"status":"ok","timestamp":1667912372012,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"n2VQSfKmUn7Z","outputId":"abe27eed-82e0-472d-f0bb-3c28a5a77b06","execution":{"iopub.status.busy":"2022-12-01T11:20:46.312062Z","iopub.execute_input":"2022-12-01T11:20:46.312586Z","iopub.status.idle":"2022-12-01T11:20:46.341956Z","shell.execute_reply.started":"2022-12-01T11:20:46.312536Z","shell.execute_reply":"2022-12-01T11:20:46.340600Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"raw_df.target.value_counts(normalize=True).plot(kind='bar')","metadata":{"executionInfo":{"elapsed":395,"status":"ok","timestamp":1667912455099,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"ZSzHylWuU0MF","outputId":"557af698-754b-49e6-a8f7-a504aaa74737","execution":{"iopub.status.busy":"2022-12-01T11:20:46.346747Z","iopub.execute_input":"2022-12-01T11:20:46.347175Z","iopub.status.idle":"2022-12-01T11:20:46.619737Z","shell.execute_reply.started":"2022-12-01T11:20:46.347138Z","shell.execute_reply":"2022-12-01T11:20:46.618359Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df = pd.read_csv('/kaggle/input/quora-insincere-questions-classification/test.csv')","metadata":{"executionInfo":{"elapsed":1646,"status":"ok","timestamp":1667912505443,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"_lA6Vb6YVN1p","execution":{"iopub.status.busy":"2022-12-01T11:20:46.621721Z","iopub.execute_input":"2022-12-01T11:20:46.622534Z","iopub.status.idle":"2022-12-01T11:20:48.046568Z","shell.execute_reply.started":"2022-12-01T11:20:46.622493Z","shell.execute_reply":"2022-12-01T11:20:48.045281Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df","metadata":{"executionInfo":{"elapsed":3,"status":"ok","timestamp":1667912507665,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"gH0ij_pmVNzV","outputId":"ad30dad3-ffd4-43ac-c04d-659ac7cf006d","execution":{"iopub.status.busy":"2022-12-01T11:20:48.048759Z","iopub.execute_input":"2022-12-01T11:20:48.049296Z","iopub.status.idle":"2022-12-01T11:20:48.064485Z","shell.execute_reply.started":"2022-12-01T11:20:48.049243Z","shell.execute_reply":"2022-12-01T11:20:48.063072Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub_df = pd.read_csv('/kaggle/input/quora-insincere-questions-classification/sample_submission.csv')","metadata":{"executionInfo":{"elapsed":532,"status":"ok","timestamp":1667912554978,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"exDbcT9bVNw0","execution":{"iopub.status.busy":"2022-12-01T11:20:48.066359Z","iopub.execute_input":"2022-12-01T11:20:48.066810Z","iopub.status.idle":"2022-12-01T11:20:48.501153Z","shell.execute_reply.started":"2022-12-01T11:20:48.066769Z","shell.execute_reply":"2022-12-01T11:20:48.499634Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub_df","metadata":{"executionInfo":{"elapsed":571,"status":"ok","timestamp":1667912558749,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"vLCvMxN3UL7l","outputId":"75dcfd90-504e-4612-b578-dcff4978dc00","execution":{"iopub.status.busy":"2022-12-01T11:20:48.502771Z","iopub.execute_input":"2022-12-01T11:20:48.503147Z","iopub.status.idle":"2022-12-01T11:20:48.516998Z","shell.execute_reply.started":"2022-12-01T11:20:48.503113Z","shell.execute_reply":"2022-12-01T11:20:48.515864Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub_df.prediction.value_counts()","metadata":{"executionInfo":{"elapsed":394,"status":"ok","timestamp":1667912639872,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"BvL2NJuSVv4l","outputId":"d362dd88-f047-4ca6-e729-54b153126cb1","execution":{"iopub.status.busy":"2022-12-01T11:20:48.518455Z","iopub.execute_input":"2022-12-01T11:20:48.519141Z","iopub.status.idle":"2022-12-01T11:20:48.539800Z","shell.execute_reply.started":"2022-12-01T11:20:48.519106Z","shell.execute_reply":"2022-12-01T11:20:48.537968Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Create a Working Sample\n","metadata":{"id":"v7sWG0KCV0g-"}},{"cell_type":"code","source":"#sample size \"here '_' work as ',' you can check by own\"\nSAMPLE_SIZE = 100_000","metadata":{"executionInfo":{"elapsed":653,"status":"ok","timestamp":1667912690959,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"hu5s7hLKV7Kr","execution":{"iopub.status.busy":"2022-12-01T11:22:18.282077Z","iopub.execute_input":"2022-12-01T11:22:18.282631Z","iopub.status.idle":"2022-12-01T11:22:18.288673Z","shell.execute_reply.started":"2022-12-01T11:22:18.282589Z","shell.execute_reply":"2022-12-01T11:22:18.287572Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#take some sample data for testing our model\n#sample_df = raw_df.sample(SAMPLE_SIZE, random_state=42)","metadata":{"executionInfo":{"elapsed":702,"status":"ok","timestamp":1667912713848,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"0xzqyYo_V31J","execution":{"iopub.status.busy":"2022-12-01T11:21:32.294651Z","iopub.execute_input":"2022-12-01T11:21:32.295129Z","iopub.status.idle":"2022-12-01T11:21:32.301296Z","shell.execute_reply.started":"2022-12-01T11:21:32.295093Z","shell.execute_reply":"2022-12-01T11:21:32.299865Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_df = raw_df.copy()","metadata":{"execution":{"iopub.status.busy":"2022-12-01T11:21:04.589663Z","iopub.execute_input":"2022-12-01T11:21:04.590064Z","iopub.status.idle":"2022-12-01T11:21:04.658333Z","shell.execute_reply.started":"2022-12-01T11:21:04.590032Z","shell.execute_reply":"2022-12-01T11:21:04.656913Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_df.shape","metadata":{"executionInfo":{"elapsed":886,"status":"ok","timestamp":1667912722831,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"4xFSOIfdV3yh","outputId":"a1f8d588-1005-475a-bfa9-04df72110179","execution":{"iopub.status.busy":"2022-12-01T11:21:05.610900Z","iopub.execute_input":"2022-12-01T11:21:05.611623Z","iopub.status.idle":"2022-12-01T11:21:05.619501Z","shell.execute_reply.started":"2022-12-01T11:21:05.611442Z","shell.execute_reply":"2022-12-01T11:21:05.618314Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Text Preprocessing Techniques\n\nOutline:\n\n1. Understand the bag of words model\n2. Tokenization\n3. Stop word removal\n4. Stemming","metadata":{"id":"-gZy068pQoPg"}},{"cell_type":"markdown","source":"### Bag of Words Intuition\n\n1. Create a list of all the words across all the text documents\n2. You convert each document into vector counts of each word\n\n\nLimitations:\n1. There may be too many words in the dataset\n2. Some words may occur too frequently\n3. Some words may occur very rarely or only once\n4. A single word may have many forms (go, gone, going or bird vs. birds)","metadata":{"id":"pJ0mh3gjXH9e"}},{"cell_type":"code","source":"q0 = sincere_df.question_text.values[1]","metadata":{"executionInfo":{"elapsed":628,"status":"ok","timestamp":1667913361214,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"PDb6D4pfYWPu","execution":{"iopub.status.busy":"2022-12-01T11:22:58.283986Z","iopub.execute_input":"2022-12-01T11:22:58.286146Z","iopub.status.idle":"2022-12-01T11:22:58.294789Z","shell.execute_reply.started":"2022-12-01T11:22:58.286057Z","shell.execute_reply":"2022-12-01T11:22:58.292979Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"q0","metadata":{"executionInfo":{"elapsed":546,"status":"ok","timestamp":1667913367151,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"mQjFxTKFYjJk","outputId":"3012387d-9827-46bf-a3bb-d9d077ee6a9f","execution":{"iopub.status.busy":"2022-12-01T11:22:58.645205Z","iopub.execute_input":"2022-12-01T11:22:58.645977Z","iopub.status.idle":"2022-12-01T11:22:58.653381Z","shell.execute_reply.started":"2022-12-01T11:22:58.645935Z","shell.execute_reply":"2022-12-01T11:22:58.652296Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"q1 = raw_df[raw_df.target == 1].question_text.values[0]","metadata":{"executionInfo":{"elapsed":478,"status":"ok","timestamp":1667913449142,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"GIvt_M-yYlJH","execution":{"iopub.status.busy":"2022-12-01T11:22:59.097597Z","iopub.execute_input":"2022-12-01T11:22:59.098381Z","iopub.status.idle":"2022-12-01T11:22:59.137811Z","shell.execute_reply.started":"2022-12-01T11:22:59.098327Z","shell.execute_reply":"2022-12-01T11:22:59.136523Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"q1","metadata":{"executionInfo":{"elapsed":3,"status":"ok","timestamp":1667913449769,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"86bHxTMbY182","outputId":"ffafd417-1607-420d-a2ff-9169707470b2","execution":{"iopub.status.busy":"2022-12-01T11:23:01.381060Z","iopub.execute_input":"2022-12-01T11:23:01.382275Z","iopub.status.idle":"2022-12-01T11:23:01.391670Z","shell.execute_reply.started":"2022-12-01T11:23:01.382203Z","shell.execute_reply":"2022-12-01T11:23:01.389936Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Tokenization\n\nsplitting a document into words and separators","metadata":{"id":"qj4Zg1yaYPJv"}},{"cell_type":"code","source":"import nltk","metadata":{"executionInfo":{"elapsed":474,"status":"ok","timestamp":1667913549193,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"EqacxiuyZPds","execution":{"iopub.status.busy":"2022-12-01T11:23:02.061532Z","iopub.execute_input":"2022-12-01T11:23:02.061925Z","iopub.status.idle":"2022-12-01T11:23:03.710772Z","shell.execute_reply.started":"2022-12-01T11:23:02.061892Z","shell.execute_reply":"2022-12-01T11:23:03.709298Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from nltk.tokenize import word_tokenize","metadata":{"executionInfo":{"elapsed":1852,"status":"ok","timestamp":1667913505183,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"2vQDMEq0Q_lP","execution":{"iopub.status.busy":"2022-12-01T11:23:03.713101Z","iopub.execute_input":"2022-12-01T11:23:03.714047Z","iopub.status.idle":"2022-12-01T11:23:03.719722Z","shell.execute_reply.started":"2022-12-01T11:23:03.713995Z","shell.execute_reply":"2022-12-01T11:23:03.718440Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"q0","metadata":{"executionInfo":{"elapsed":1019,"status":"ok","timestamp":1667913515768,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"VyIaEATyQ_ip","outputId":"655a30fa-ccf0-4839-aa2a-15f4437a5562","execution":{"iopub.status.busy":"2022-12-01T11:23:58.877564Z","iopub.execute_input":"2022-12-01T11:23:58.877958Z","iopub.status.idle":"2022-12-01T11:23:58.887545Z","shell.execute_reply.started":"2022-12-01T11:23:58.877926Z","shell.execute_reply":"2022-12-01T11:23:58.885680Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"word_tokenize(q0)","metadata":{"executionInfo":{"elapsed":458,"status":"ok","timestamp":1667913557740,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"1yULG5nVQ_gk","outputId":"83e3941f-3d29-4008-c885-b9a5eaf93912","execution":{"iopub.status.busy":"2022-12-01T11:23:59.521563Z","iopub.execute_input":"2022-12-01T11:23:59.522036Z","iopub.status.idle":"2022-12-01T11:23:59.531560Z","shell.execute_reply.started":"2022-12-01T11:23:59.522000Z","shell.execute_reply":"2022-12-01T11:23:59.530001Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"word_tokenize(' this is (something) with, a lot of, punctuation;')","metadata":{"executionInfo":{"elapsed":451,"status":"ok","timestamp":1667913596244,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"gJ_lrgEpZIt8","outputId":"aa250716-93b6-47ab-cbd1-d9ba9076c620","execution":{"iopub.status.busy":"2022-12-01T11:24:00.023612Z","iopub.execute_input":"2022-12-01T11:24:00.024451Z","iopub.status.idle":"2022-12-01T11:24:00.033799Z","shell.execute_reply.started":"2022-12-01T11:24:00.024396Z","shell.execute_reply":"2022-12-01T11:24:00.032395Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"q1","metadata":{"executionInfo":{"elapsed":913,"status":"ok","timestamp":1667913614596,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"uoP8H4KcZIr1","outputId":"0800baed-d874-4f5c-ba0b-fb5aad5b0fc0","execution":{"iopub.status.busy":"2022-12-01T11:24:00.432547Z","iopub.execute_input":"2022-12-01T11:24:00.433252Z","iopub.status.idle":"2022-12-01T11:24:00.442374Z","shell.execute_reply.started":"2022-12-01T11:24:00.433189Z","shell.execute_reply":"2022-12-01T11:24:00.440928Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"word_tokenize(q1)","metadata":{"executionInfo":{"elapsed":977,"status":"ok","timestamp":1667913623231,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"Bs_0YAawZIpf","outputId":"e97ea6f2-c22c-4f06-fe1f-160df99340ad","execution":{"iopub.status.busy":"2022-12-01T11:24:00.843185Z","iopub.execute_input":"2022-12-01T11:24:00.843722Z","iopub.status.idle":"2022-12-01T11:24:00.853206Z","shell.execute_reply.started":"2022-12-01T11:24:00.843683Z","shell.execute_reply":"2022-12-01T11:24:00.851524Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"q0_tok = word_tokenize(q0)\nq1_tok = word_tokenize(q1)","metadata":{"executionInfo":{"elapsed":1,"status":"ok","timestamp":1667913656571,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"7HNGqFYtZInd","execution":{"iopub.status.busy":"2022-12-01T11:24:01.126427Z","iopub.execute_input":"2022-12-01T11:24:01.126871Z","iopub.status.idle":"2022-12-01T11:24:01.133297Z","shell.execute_reply.started":"2022-12-01T11:24:01.126835Z","shell.execute_reply":"2022-12-01T11:24:01.132003Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Stop Word Removal\n\nRemoving commonly occuring words","metadata":{"id":"Mf5aqvkGZrR-"}},{"cell_type":"code","source":"q1_tok","metadata":{"executionInfo":{"elapsed":385,"status":"ok","timestamp":1667913691780,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"7yZhlXV9Zxo-","outputId":"408a649d-3883-4f70-e1b5-a052ada07d60","execution":{"iopub.status.busy":"2022-12-01T11:24:02.368904Z","iopub.execute_input":"2022-12-01T11:24:02.369354Z","iopub.status.idle":"2022-12-01T11:24:02.377510Z","shell.execute_reply.started":"2022-12-01T11:24:02.369318Z","shell.execute_reply":"2022-12-01T11:24:02.376213Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from nltk.corpus import stopwords","metadata":{"executionInfo":{"elapsed":1009,"status":"ok","timestamp":1667913739715,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"s1qsENBhZrD0","execution":{"iopub.status.busy":"2022-12-01T11:24:04.713462Z","iopub.execute_input":"2022-12-01T11:24:04.715471Z","iopub.status.idle":"2022-12-01T11:24:04.723605Z","shell.execute_reply.started":"2022-12-01T11:24:04.715269Z","shell.execute_reply":"2022-12-01T11:24:04.721503Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"english_stopwords = stopwords.words('english')","metadata":{"executionInfo":{"elapsed":430,"status":"ok","timestamp":1667913795278,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"ejhkI9oVZrCD","execution":{"iopub.status.busy":"2022-12-01T11:24:05.733307Z","iopub.execute_input":"2022-12-01T11:24:05.734907Z","iopub.status.idle":"2022-12-01T11:24:05.742000Z","shell.execute_reply.started":"2022-12-01T11:24:05.734836Z","shell.execute_reply":"2022-12-01T11:24:05.740497Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\", \".join(english_stopwords)","metadata":{"executionInfo":{"elapsed":357,"status":"ok","timestamp":1667913817359,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"jB5avxBsZqxB","outputId":"27c5b385-e8e1-4987-cc2a-19d43eb70308","execution":{"iopub.status.busy":"2022-12-01T11:24:08.011412Z","iopub.execute_input":"2022-12-01T11:24:08.011839Z","iopub.status.idle":"2022-12-01T11:24:08.020789Z","shell.execute_reply.started":"2022-12-01T11:24:08.011806Z","shell.execute_reply":"2022-12-01T11:24:08.019245Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def remove_stopwords(tokens):\n    return [word for word in tokens if word.lower() not in english_stopwords]","metadata":{"executionInfo":{"elapsed":352,"status":"ok","timestamp":1667913917513,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"7umWfNCvZqsF","execution":{"iopub.status.busy":"2022-12-01T11:24:10.676756Z","iopub.execute_input":"2022-12-01T11:24:10.677327Z","iopub.status.idle":"2022-12-01T11:24:10.684551Z","shell.execute_reply.started":"2022-12-01T11:24:10.677268Z","shell.execute_reply":"2022-12-01T11:24:10.682776Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"q0_tok","metadata":{"executionInfo":{"elapsed":930,"status":"ok","timestamp":1667913931371,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"15NSoE_Saq6W","outputId":"4abcb5d0-fae7-4c59-f01d-c8d641d798e9","execution":{"iopub.status.busy":"2022-12-01T11:24:12.416140Z","iopub.execute_input":"2022-12-01T11:24:12.417296Z","iopub.status.idle":"2022-12-01T11:24:12.426107Z","shell.execute_reply.started":"2022-12-01T11:24:12.417243Z","shell.execute_reply":"2022-12-01T11:24:12.424203Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"q0_stp = remove_stopwords(q0_tok)","metadata":{"executionInfo":{"elapsed":667,"status":"ok","timestamp":1667913944997,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"b59BPAClaq4B","execution":{"iopub.status.busy":"2022-12-01T11:24:16.312186Z","iopub.execute_input":"2022-12-01T11:24:16.312619Z","iopub.status.idle":"2022-12-01T11:24:16.317823Z","shell.execute_reply.started":"2022-12-01T11:24:16.312585Z","shell.execute_reply":"2022-12-01T11:24:16.316295Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"q0_stp","metadata":{"executionInfo":{"elapsed":502,"status":"ok","timestamp":1667913950071,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"UJpOKfd-Zqp_","outputId":"d9cd0cfc-26a5-45a2-eb18-3517d13bedda","execution":{"iopub.status.busy":"2022-12-01T11:24:16.687710Z","iopub.execute_input":"2022-12-01T11:24:16.688256Z","iopub.status.idle":"2022-12-01T11:24:16.699272Z","shell.execute_reply.started":"2022-12-01T11:24:16.688200Z","shell.execute_reply":"2022-12-01T11:24:16.697778Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"q1_stp = remove_stopwords(q1_tok)","metadata":{"executionInfo":{"elapsed":476,"status":"ok","timestamp":1667913994939,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"IKR9dyGVa6OR","execution":{"iopub.status.busy":"2022-12-01T11:24:19.078488Z","iopub.execute_input":"2022-12-01T11:24:19.078954Z","iopub.status.idle":"2022-12-01T11:24:19.085606Z","shell.execute_reply.started":"2022-12-01T11:24:19.078914Z","shell.execute_reply":"2022-12-01T11:24:19.084264Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"q1_tok","metadata":{"executionInfo":{"elapsed":699,"status":"ok","timestamp":1667913999121,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"_RvMiZAka6Lv","outputId":"4f446d13-cf60-4da4-e6d6-50b9f46a1ad0","execution":{"iopub.status.busy":"2022-12-01T11:24:19.431183Z","iopub.execute_input":"2022-12-01T11:24:19.431759Z","iopub.status.idle":"2022-12-01T11:24:19.441603Z","shell.execute_reply.started":"2022-12-01T11:24:19.431711Z","shell.execute_reply":"2022-12-01T11:24:19.440260Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"q1_stp","metadata":{"executionInfo":{"elapsed":485,"status":"ok","timestamp":1667914001968,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"GqpqLKw9a-A8","outputId":"9280f918-57dc-4673-bb3b-274107f8b62b","execution":{"iopub.status.busy":"2022-12-01T11:24:19.951369Z","iopub.execute_input":"2022-12-01T11:24:19.951844Z","iopub.status.idle":"2022-12-01T11:24:19.963014Z","shell.execute_reply.started":"2022-12-01T11:24:19.951809Z","shell.execute_reply":"2022-12-01T11:24:19.961541Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Stemming\n\n\"go\", \"gone\", \"going\" -> \"go\"\n\"birds\", \"bird\" -> \"bird\"","metadata":{"id":"FeVx68x5bHEF"}},{"cell_type":"code","source":"from nltk.stem.snowball import SnowballStemmer","metadata":{"executionInfo":{"elapsed":1054,"status":"ok","timestamp":1667914103321,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"9jFOfSqtbG6y","execution":{"iopub.status.busy":"2022-12-01T11:24:22.065742Z","iopub.execute_input":"2022-12-01T11:24:22.067178Z","iopub.status.idle":"2022-12-01T11:24:22.072925Z","shell.execute_reply.started":"2022-12-01T11:24:22.067122Z","shell.execute_reply":"2022-12-01T11:24:22.071967Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"stemmer = SnowballStemmer(language='english')","metadata":{"executionInfo":{"elapsed":544,"status":"ok","timestamp":1667914116697,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"SC0PMIy3bG4r","execution":{"iopub.status.busy":"2022-12-01T11:24:25.030909Z","iopub.execute_input":"2022-12-01T11:24:25.031756Z","iopub.status.idle":"2022-12-01T11:24:25.038461Z","shell.execute_reply.started":"2022-12-01T11:24:25.031691Z","shell.execute_reply":"2022-12-01T11:24:25.036738Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"stemmer.stem('going')","metadata":{"executionInfo":{"elapsed":792,"status":"ok","timestamp":1667914135778,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"apl0nhFqbG2F","outputId":"fc70d0f5-0f7a-49b4-ace3-ea1113cb8c66","execution":{"iopub.status.busy":"2022-12-01T11:24:26.787721Z","iopub.execute_input":"2022-12-01T11:24:26.788237Z","iopub.status.idle":"2022-12-01T11:24:26.797731Z","shell.execute_reply.started":"2022-12-01T11:24:26.788180Z","shell.execute_reply":"2022-12-01T11:24:26.795714Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"stemmer.stem('supposedly')","metadata":{"executionInfo":{"elapsed":386,"status":"ok","timestamp":1667914142907,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"wtIoHT1ybGz2","outputId":"3609cf3d-2c9f-4638-db1f-07ce31e63f20","execution":{"iopub.status.busy":"2022-12-01T11:24:28.218262Z","iopub.execute_input":"2022-12-01T11:24:28.219924Z","iopub.status.idle":"2022-12-01T11:24:28.229462Z","shell.execute_reply.started":"2022-12-01T11:24:28.219853Z","shell.execute_reply":"2022-12-01T11:24:28.227835Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"q0_stm = [stemmer.stem(word) for word in q0_stp]","metadata":{"executionInfo":{"elapsed":364,"status":"ok","timestamp":1667914180406,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"hGdECu-rbmbZ","execution":{"iopub.status.busy":"2022-12-01T11:24:31.173850Z","iopub.execute_input":"2022-12-01T11:24:31.174561Z","iopub.status.idle":"2022-12-01T11:24:31.182609Z","shell.execute_reply.started":"2022-12-01T11:24:31.174502Z","shell.execute_reply":"2022-12-01T11:24:31.181003Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"q0_stp","metadata":{"executionInfo":{"elapsed":817,"status":"ok","timestamp":1667914186316,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"BY0fs95pbqtH","outputId":"5e0d4927-385f-40c7-ad88-82ec30049890","execution":{"iopub.status.busy":"2022-12-01T11:24:33.088600Z","iopub.execute_input":"2022-12-01T11:24:33.089760Z","iopub.status.idle":"2022-12-01T11:24:33.103572Z","shell.execute_reply.started":"2022-12-01T11:24:33.089707Z","shell.execute_reply":"2022-12-01T11:24:33.102429Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"q0_stm","metadata":{"executionInfo":{"elapsed":573,"status":"ok","timestamp":1667914193742,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"ARuFpGkHbqn2","outputId":"3e4e85e5-bb93-4feb-97f2-5141ebbcd1db","execution":{"iopub.status.busy":"2022-12-01T11:24:35.198410Z","iopub.execute_input":"2022-12-01T11:24:35.199572Z","iopub.status.idle":"2022-12-01T11:24:35.207512Z","shell.execute_reply.started":"2022-12-01T11:24:35.199531Z","shell.execute_reply":"2022-12-01T11:24:35.206055Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"q1_stm = [stemmer.stem(word) for word in q1_stp]","metadata":{"executionInfo":{"elapsed":432,"status":"ok","timestamp":1667914233752,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"kiFiGdCpbrot","execution":{"iopub.status.busy":"2022-12-01T11:24:37.678619Z","iopub.execute_input":"2022-12-01T11:24:37.679121Z","iopub.status.idle":"2022-12-01T11:24:37.686293Z","shell.execute_reply.started":"2022-12-01T11:24:37.679083Z","shell.execute_reply":"2022-12-01T11:24:37.684661Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"q1_stp","metadata":{"executionInfo":{"elapsed":458,"status":"ok","timestamp":1667914238715,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"WweHQFO4brmn","outputId":"58cc1dae-68c0-4778-9fb1-27e37e054e86","execution":{"iopub.status.busy":"2022-12-01T11:24:37.970454Z","iopub.execute_input":"2022-12-01T11:24:37.970909Z","iopub.status.idle":"2022-12-01T11:24:37.980534Z","shell.execute_reply.started":"2022-12-01T11:24:37.970873Z","shell.execute_reply":"2022-12-01T11:24:37.978715Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"q1_stm","metadata":{"executionInfo":{"elapsed":906,"status":"ok","timestamp":1667914244006,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"AVu5FiAbbrkV","outputId":"4a4c2193-e5b7-4f1c-8781-127e2f84efb0","execution":{"iopub.status.busy":"2022-12-01T11:24:40.164592Z","iopub.execute_input":"2022-12-01T11:24:40.165335Z","iopub.status.idle":"2022-12-01T11:24:40.172532Z","shell.execute_reply.started":"2022-12-01T11:24:40.165293Z","shell.execute_reply":"2022-12-01T11:24:40.171179Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Lemmatization\n\n\"love\" -> \"love\"\n\"loving\" -> \"love\"\n\"lovable\" -> \"love\"\n","metadata":{"id":"VWWkiTcWcAR7"}},{"cell_type":"markdown","source":"## Implement Bag of Words \n\n\nOutline:\n\n1. Create a vocabulary using Count Vectorizer\n2. Transform text to vectors using Count Vectorizer\n3. Configure text preprocessing in Count Vectorizer","metadata":{"id":"CuTR9sOWQoMq"}},{"cell_type":"markdown","source":"### Create a Vocabulary","metadata":{"id":"0HWShc6jdPFy"}},{"cell_type":"code","source":"sample_df","metadata":{"executionInfo":{"elapsed":419,"status":"ok","timestamp":1667914532328,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"uGScEAd9c_dQ","outputId":"e923813b-b8fd-438b-a6f0-743c280a204d","execution":{"iopub.status.busy":"2022-12-01T11:24:47.187797Z","iopub.execute_input":"2022-12-01T11:24:47.188948Z","iopub.status.idle":"2022-12-01T11:24:47.209011Z","shell.execute_reply.started":"2022-12-01T11:24:47.188894Z","shell.execute_reply":"2022-12-01T11:24:47.207069Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#taking snall data for test our count vectorizer\nsmall_df = sample_df[:5]","metadata":{"executionInfo":{"elapsed":707,"status":"ok","timestamp":1667914559612,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"IKj0BM2NRATm","execution":{"iopub.status.busy":"2022-12-01T11:25:40.597262Z","iopub.execute_input":"2022-12-01T11:25:40.597716Z","iopub.status.idle":"2022-12-01T11:25:40.605028Z","shell.execute_reply.started":"2022-12-01T11:25:40.597683Z","shell.execute_reply":"2022-12-01T11:25:40.603355Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"small_df","metadata":{"executionInfo":{"elapsed":1222,"status":"ok","timestamp":1667914576774,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"awqkIOZEdKFu","outputId":"9c110405-71e1-4c72-c02c-dfbc1d5be43c","execution":{"iopub.status.busy":"2022-12-01T11:25:42.374540Z","iopub.execute_input":"2022-12-01T11:25:42.375024Z","iopub.status.idle":"2022-12-01T11:25:42.389335Z","shell.execute_reply.started":"2022-12-01T11:25:42.374986Z","shell.execute_reply":"2022-12-01T11:25:42.387829Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"small_df.question_text.values","metadata":{"executionInfo":{"elapsed":430,"status":"ok","timestamp":1667914566540,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"XZOIXmC3RARh","outputId":"330c3a06-1368-40de-ae9c-40f0061e61d6","execution":{"iopub.status.busy":"2022-12-01T11:25:42.656522Z","iopub.execute_input":"2022-12-01T11:25:42.657324Z","iopub.status.idle":"2022-12-01T11:25:42.667084Z","shell.execute_reply.started":"2022-12-01T11:25:42.657276Z","shell.execute_reply":"2022-12-01T11:25:42.665673Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.feature_extraction.text import CountVectorizer","metadata":{"executionInfo":{"elapsed":362,"status":"ok","timestamp":1667914669258,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"Yr3XgzBWdNNx","execution":{"iopub.status.busy":"2022-12-01T11:25:43.080354Z","iopub.execute_input":"2022-12-01T11:25:43.080809Z","iopub.status.idle":"2022-12-01T11:25:43.088205Z","shell.execute_reply.started":"2022-12-01T11:25:43.080774Z","shell.execute_reply":"2022-12-01T11:25:43.086698Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"small_vect = CountVectorizer()","metadata":{"executionInfo":{"elapsed":351,"status":"ok","timestamp":1667914677547,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"-sE8WL3-dNLf","execution":{"iopub.status.busy":"2022-12-01T11:25:46.180552Z","iopub.execute_input":"2022-12-01T11:25:46.181457Z","iopub.status.idle":"2022-12-01T11:25:46.187902Z","shell.execute_reply.started":"2022-12-01T11:25:46.181304Z","shell.execute_reply":"2022-12-01T11:25:46.186696Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"small_vect.fit(small_df.question_text)","metadata":{"executionInfo":{"elapsed":409,"status":"ok","timestamp":1667914713345,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"ba9oOkD6RAPP","outputId":"eccc7e9b-266b-4aea-90fa-654fa176b504","execution":{"iopub.status.busy":"2022-12-01T11:25:46.769867Z","iopub.execute_input":"2022-12-01T11:25:46.770415Z","iopub.status.idle":"2022-12-01T11:25:46.792043Z","shell.execute_reply.started":"2022-12-01T11:25:46.770373Z","shell.execute_reply":"2022-12-01T11:25:46.790613Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"small_vect.get_feature_names_out()","metadata":{"executionInfo":{"elapsed":651,"status":"ok","timestamp":1667914761316,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"-QPGkxLZdkPy","outputId":"7de3797e-3a41-4bb1-c581-ad7bd7610eb4","execution":{"iopub.status.busy":"2022-12-01T11:25:47.520966Z","iopub.execute_input":"2022-12-01T11:25:47.522148Z","iopub.status.idle":"2022-12-01T11:25:47.530057Z","shell.execute_reply.started":"2022-12-01T11:25:47.522106Z","shell.execute_reply":"2022-12-01T11:25:47.528577Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Transform documents into Vectors","metadata":{"id":"WXsUWz_hd9KE"}},{"cell_type":"code","source":"vectors = small_vect.transform(small_df.question_text)","metadata":{"executionInfo":{"elapsed":533,"status":"ok","timestamp":1667914847949,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"nZVZ3jIddkKT","execution":{"iopub.status.busy":"2022-12-01T11:25:50.292511Z","iopub.execute_input":"2022-12-01T11:25:50.292978Z","iopub.status.idle":"2022-12-01T11:25:50.299895Z","shell.execute_reply.started":"2022-12-01T11:25:50.292942Z","shell.execute_reply":"2022-12-01T11:25:50.298384Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"vectors","metadata":{"executionInfo":{"elapsed":638,"status":"ok","timestamp":1667914853853,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"MIGbsQEQdkHt","outputId":"1ad2eff7-d795-4c0e-ab60-f88ce7b4ed8c","execution":{"iopub.status.busy":"2022-12-01T11:25:50.653924Z","iopub.execute_input":"2022-12-01T11:25:50.654833Z","iopub.status.idle":"2022-12-01T11:25:50.663890Z","shell.execute_reply.started":"2022-12-01T11:25:50.654768Z","shell.execute_reply":"2022-12-01T11:25:50.662360Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"vectors.shape","metadata":{"executionInfo":{"elapsed":630,"status":"ok","timestamp":1667914900662,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"wriwC-1DeZPA","outputId":"9b7a3cba-e4e6-4023-851b-7cb6b6029f40","execution":{"iopub.status.busy":"2022-12-01T11:25:51.114963Z","iopub.execute_input":"2022-12-01T11:25:51.116062Z","iopub.status.idle":"2022-12-01T11:25:51.125306Z","shell.execute_reply.started":"2022-12-01T11:25:51.116020Z","shell.execute_reply":"2022-12-01T11:25:51.123890Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"small_df.question_text.values[0]","metadata":{"executionInfo":{"elapsed":495,"status":"ok","timestamp":1667914953574,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"bXtFEWjtekqt","outputId":"a4c659e8-4c86-478e-c827-ddbacde28bd9","execution":{"iopub.status.busy":"2022-12-01T11:25:54.305577Z","iopub.execute_input":"2022-12-01T11:25:54.306551Z","iopub.status.idle":"2022-12-01T11:25:54.314707Z","shell.execute_reply.started":"2022-12-01T11:25:54.306503Z","shell.execute_reply":"2022-12-01T11:25:54.313330Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"vectors[0].toarray()","metadata":{"executionInfo":{"elapsed":982,"status":"ok","timestamp":1667915020439,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"etQ-T7_1e0vF","outputId":"9446f31f-ae80-4fac-857f-42f9481bafed","execution":{"iopub.status.busy":"2022-12-01T11:25:55.969671Z","iopub.execute_input":"2022-12-01T11:25:55.970165Z","iopub.status.idle":"2022-12-01T11:25:55.979759Z","shell.execute_reply.started":"2022-12-01T11:25:55.970124Z","shell.execute_reply":"2022-12-01T11:25:55.978678Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"vectors.toarray()","metadata":{"executionInfo":{"elapsed":483,"status":"ok","timestamp":1667914889375,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"hSrf1ZPndkFX","outputId":"63dd68aa-56d5-4838-9f41-9f12c165e9b2","execution":{"iopub.status.busy":"2022-12-01T11:25:56.330321Z","iopub.execute_input":"2022-12-01T11:25:56.330833Z","iopub.status.idle":"2022-12-01T11:25:56.340092Z","shell.execute_reply.started":"2022-12-01T11:25:56.330787Z","shell.execute_reply":"2022-12-01T11:25:56.338270Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Configure Count Vectorizer Parameters","metadata":{"id":"vZPCOwobe485"}},{"cell_type":"code","source":"stemmer = SnowballStemmer(language='english')","metadata":{"executionInfo":{"elapsed":391,"status":"ok","timestamp":1667915246398,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"xUjZDsdqfq4u","execution":{"iopub.status.busy":"2022-12-01T11:25:59.231570Z","iopub.execute_input":"2022-12-01T11:25:59.232103Z","iopub.status.idle":"2022-12-01T11:25:59.237977Z","shell.execute_reply.started":"2022-12-01T11:25:59.232055Z","shell.execute_reply":"2022-12-01T11:25:59.237005Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def tokenize(text):\n    return [stemmer.stem(word) for word in word_tokenize(text)]","metadata":{"executionInfo":{"elapsed":368,"status":"ok","timestamp":1667915256240,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"9hZzn1LBfPoM","execution":{"iopub.status.busy":"2022-12-01T11:25:59.860690Z","iopub.execute_input":"2022-12-01T11:25:59.861063Z","iopub.status.idle":"2022-12-01T11:25:59.867369Z","shell.execute_reply.started":"2022-12-01T11:25:59.861031Z","shell.execute_reply":"2022-12-01T11:25:59.865801Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tokenize('What is the really (dealing) here?')","metadata":{"executionInfo":{"elapsed":358,"status":"ok","timestamp":1667915263406,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"OAausoIrfa2I","outputId":"96c1e3a3-1f5c-40d0-e05b-1c32046aff8c","execution":{"iopub.status.busy":"2022-12-01T11:26:00.447064Z","iopub.execute_input":"2022-12-01T11:26:00.448160Z","iopub.status.idle":"2022-12-01T11:26:00.455639Z","shell.execute_reply.started":"2022-12-01T11:26:00.448115Z","shell.execute_reply":"2022-12-01T11:26:00.454476Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"vectorizer = CountVectorizer(lowercase=True, \n                             tokenizer=tokenize,\n                             stop_words=english_stopwords,\n                             max_features=1000)","metadata":{"executionInfo":{"elapsed":559,"status":"ok","timestamp":1667915360958,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"pJGAErs7e9Fh","execution":{"iopub.status.busy":"2022-12-01T11:26:04.416280Z","iopub.execute_input":"2022-12-01T11:26:04.416750Z","iopub.status.idle":"2022-12-01T11:26:04.422787Z","shell.execute_reply.started":"2022-12-01T11:26:04.416715Z","shell.execute_reply":"2022-12-01T11:26:04.421323Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nvectorizer.fit(sample_df.question_text)","metadata":{"executionInfo":{"elapsed":32457,"status":"ok","timestamp":1667915423839,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"pI8nbkpAe9Dg","outputId":"e6151146-4e99-4bb5-bf70-c7336b5ed226","execution":{"iopub.status.busy":"2022-12-01T11:26:07.226847Z","iopub.execute_input":"2022-12-01T11:26:07.228052Z","iopub.status.idle":"2022-12-01T11:33:44.173911Z","shell.execute_reply.started":"2022-12-01T11:26:07.228007Z","shell.execute_reply":"2022-12-01T11:33:44.172617Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(vectorizer.vocabulary_)","metadata":{"executionInfo":{"elapsed":2068,"status":"ok","timestamp":1667915449808,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"sJBlLm5Ye88Z","outputId":"e8ad80f8-cd4d-4289-fd26-163dc808e69f","execution":{"iopub.status.busy":"2022-12-01T11:33:44.176628Z","iopub.execute_input":"2022-12-01T11:33:44.177509Z","iopub.status.idle":"2022-12-01T11:33:44.186606Z","shell.execute_reply.started":"2022-12-01T11:33:44.177447Z","shell.execute_reply":"2022-12-01T11:33:44.185052Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"vectorizer.get_feature_names_out()[:100]","metadata":{"executionInfo":{"elapsed":1059,"status":"ok","timestamp":1667915475499,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"b9K638h1gg5_","outputId":"955fe2e0-52ab-4137-9ecc-f18a64542e45","execution":{"iopub.status.busy":"2022-12-01T11:33:44.188070Z","iopub.execute_input":"2022-12-01T11:33:44.188482Z","iopub.status.idle":"2022-12-01T11:33:44.203858Z","shell.execute_reply.started":"2022-12-01T11:33:44.188446Z","shell.execute_reply":"2022-12-01T11:33:44.202541Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ninputs = vectorizer.transform(sample_df.question_text)","metadata":{"executionInfo":{"elapsed":30784,"status":"ok","timestamp":1667915566857,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"n3iq65OsguTR","outputId":"35c22bcd-b6c2-4629-9e83-ad81a29456ee","execution":{"iopub.status.busy":"2022-12-01T11:33:44.206808Z","iopub.execute_input":"2022-12-01T11:33:44.207495Z","iopub.status.idle":"2022-12-01T11:41:20.977488Z","shell.execute_reply.started":"2022-12-01T11:33:44.207460Z","shell.execute_reply":"2022-12-01T11:41:20.976051Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"inputs.shape","metadata":{"executionInfo":{"elapsed":661,"status":"ok","timestamp":1667915574286,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"9aEbKIC0gg3o","outputId":"a0f9fdcf-a53d-455d-bd49-691f1f737a47","execution":{"iopub.status.busy":"2022-12-01T11:41:20.979277Z","iopub.execute_input":"2022-12-01T11:41:20.979715Z","iopub.status.idle":"2022-12-01T11:41:20.988436Z","shell.execute_reply.started":"2022-12-01T11:41:20.979673Z","shell.execute_reply":"2022-12-01T11:41:20.987072Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"inputs","metadata":{"executionInfo":{"elapsed":2,"status":"ok","timestamp":1667915578819,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"meslQK7Le86I","outputId":"d6cdab93-2640-43d0-d5fd-b907c201177d","execution":{"iopub.status.busy":"2022-12-01T11:41:20.990269Z","iopub.execute_input":"2022-12-01T11:41:20.990688Z","iopub.status.idle":"2022-12-01T11:41:21.006389Z","shell.execute_reply.started":"2022-12-01T11:41:20.990652Z","shell.execute_reply":"2022-12-01T11:41:21.004568Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_df.question_text.values[0]","metadata":{"executionInfo":{"elapsed":652,"status":"ok","timestamp":1667915621850,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"ARTJ68oehHVR","outputId":"83f27c51-7c51-4808-9450-489e5599776b","execution":{"iopub.status.busy":"2022-12-01T11:41:21.009872Z","iopub.execute_input":"2022-12-01T11:41:21.010546Z","iopub.status.idle":"2022-12-01T11:41:21.020173Z","shell.execute_reply.started":"2022-12-01T11:41:21.010490Z","shell.execute_reply":"2022-12-01T11:41:21.018795Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#checking with test data\ntest_df","metadata":{"executionInfo":{"elapsed":1081,"status":"ok","timestamp":1667915683093,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"fJ497jJyhCaK","outputId":"cdb77624-843c-4fe1-ff52-5f93794be092","execution":{"iopub.status.busy":"2022-12-01T11:41:21.022373Z","iopub.execute_input":"2022-12-01T11:41:21.022751Z","iopub.status.idle":"2022-12-01T11:41:21.042113Z","shell.execute_reply.started":"2022-12-01T11:41:21.022718Z","shell.execute_reply":"2022-12-01T11:41:21.041075Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ntest_inputs = vectorizer.transform(test_df.question_text)","metadata":{"executionInfo":{"elapsed":112943,"status":"ok","timestamp":1667915815920,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"UDu0wUf9g--d","outputId":"3db731b6-8002-435a-bfb5-7500c8921da8","execution":{"iopub.status.busy":"2022-12-01T11:41:21.047440Z","iopub.execute_input":"2022-12-01T11:41:21.047844Z","iopub.status.idle":"2022-12-01T11:43:32.661940Z","shell.execute_reply.started":"2022-12-01T11:41:21.047811Z","shell.execute_reply":"2022-12-01T11:43:32.660527Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_inputs.shape","metadata":{"id":"CKLhsg9GdkDW","execution":{"iopub.status.busy":"2022-12-01T11:43:32.663685Z","iopub.execute_input":"2022-12-01T11:43:32.664090Z","iopub.status.idle":"2022-12-01T11:43:32.672255Z","shell.execute_reply.started":"2022-12-01T11:43:32.664055Z","shell.execute_reply":"2022-12-01T11:43:32.671112Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## ML Models for Text Classification\n\nOutline:\n\n- Create a training & validation set\n- Train a logistic regression model\n- Make predictions on training, validation & test data","metadata":{"id":"AyVwmOboQoJn"}},{"cell_type":"markdown","source":"### Split into Training and Validation Set","metadata":{"id":"1CPsktK-iPgK"}},{"cell_type":"code","source":"sample_df","metadata":{"executionInfo":{"elapsed":3,"status":"ok","timestamp":1667915931527,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"hoIv721biUVy","outputId":"73a07b8c-ec27-495e-8ccd-9bb4c53e40da","execution":{"iopub.status.busy":"2022-12-01T11:43:32.674246Z","iopub.execute_input":"2022-12-01T11:43:32.674624Z","iopub.status.idle":"2022-12-01T11:43:32.696471Z","shell.execute_reply.started":"2022-12-01T11:43:32.674592Z","shell.execute_reply":"2022-12-01T11:43:32.695315Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"inputs.shape","metadata":{"executionInfo":{"elapsed":613,"status":"ok","timestamp":1667915949798,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"ckPyKMcMRBET","outputId":"0bb74251-5275-42ed-b60a-eb763113e3c0","execution":{"iopub.status.busy":"2022-12-01T11:43:32.698133Z","iopub.execute_input":"2022-12-01T11:43:32.698618Z","iopub.status.idle":"2022-12-01T11:43:32.708373Z","shell.execute_reply.started":"2022-12-01T11:43:32.698573Z","shell.execute_reply":"2022-12-01T11:43:32.707022Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split","metadata":{"executionInfo":{"elapsed":348,"status":"ok","timestamp":1667916033391,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"OB2jqu7dirOn","execution":{"iopub.status.busy":"2022-12-01T11:43:32.712169Z","iopub.execute_input":"2022-12-01T11:43:32.712567Z","iopub.status.idle":"2022-12-01T11:43:32.719912Z","shell.execute_reply.started":"2022-12-01T11:43:32.712534Z","shell.execute_reply":"2022-12-01T11:43:32.718523Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_inputs, val_inputs, train_targets, val_targets = train_test_split(inputs, sample_df.target, \n                                                                        test_size=0.3, random_state=42)","metadata":{"executionInfo":{"elapsed":364,"status":"ok","timestamp":1667916089349,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"m3eE_6y6ieLP","execution":{"iopub.status.busy":"2022-12-01T11:43:32.721398Z","iopub.execute_input":"2022-12-01T11:43:32.721756Z","iopub.status.idle":"2022-12-01T11:43:33.060206Z","shell.execute_reply.started":"2022-12-01T11:43:32.721726Z","shell.execute_reply":"2022-12-01T11:43:33.058764Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_inputs.shape","metadata":{"executionInfo":{"elapsed":908,"status":"ok","timestamp":1667916115211,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"1hMoWlcrRBBt","outputId":"c604cdcc-ed79-4c04-dba0-8d46713cab38","execution":{"iopub.status.busy":"2022-12-01T11:43:33.061509Z","iopub.execute_input":"2022-12-01T11:43:33.061839Z","iopub.status.idle":"2022-12-01T11:43:33.070736Z","shell.execute_reply.started":"2022-12-01T11:43:33.061810Z","shell.execute_reply":"2022-12-01T11:43:33.068198Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_targets.shape","metadata":{"executionInfo":{"elapsed":986,"status":"ok","timestamp":1667916129267,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"ioOd41xKjCkX","outputId":"f3c280ca-aa5c-4f49-d551-3ffebe05c89a","execution":{"iopub.status.busy":"2022-12-01T11:43:33.072792Z","iopub.execute_input":"2022-12-01T11:43:33.074114Z","iopub.status.idle":"2022-12-01T11:43:33.083897Z","shell.execute_reply.started":"2022-12-01T11:43:33.074052Z","shell.execute_reply":"2022-12-01T11:43:33.082294Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"val_inputs.shape","metadata":{"executionInfo":{"elapsed":493,"status":"ok","timestamp":1667916138104,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"d2ipJkxRjChw","outputId":"798265ae-8305-41b5-f96c-8611c45aef4f","execution":{"iopub.status.busy":"2022-12-01T11:43:33.086750Z","iopub.execute_input":"2022-12-01T11:43:33.087101Z","iopub.status.idle":"2022-12-01T11:43:33.101447Z","shell.execute_reply.started":"2022-12-01T11:43:33.087071Z","shell.execute_reply":"2022-12-01T11:43:33.100211Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"val_targets.shape","metadata":{"executionInfo":{"elapsed":1040,"status":"ok","timestamp":1667916148487,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"TZuuLryBjIWZ","outputId":"caa99599-0889-4588-ec20-3ad21881b1d6","execution":{"iopub.status.busy":"2022-12-01T11:43:33.103096Z","iopub.execute_input":"2022-12-01T11:43:33.104347Z","iopub.status.idle":"2022-12-01T11:43:33.112125Z","shell.execute_reply.started":"2022-12-01T11:43:33.104303Z","shell.execute_reply":"2022-12-01T11:43:33.110991Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Train Logistic Regression model","metadata":{"id":"bBAcDSenjMSk"}},{"cell_type":"code","source":"from sklearn.linear_model import LogisticRegression","metadata":{"executionInfo":{"elapsed":535,"status":"ok","timestamp":1667916194680,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"tURg-O4pjMmX","execution":{"iopub.status.busy":"2022-12-01T11:43:33.113584Z","iopub.execute_input":"2022-12-01T11:43:33.113910Z","iopub.status.idle":"2022-12-01T11:43:33.122034Z","shell.execute_reply.started":"2022-12-01T11:43:33.113880Z","shell.execute_reply":"2022-12-01T11:43:33.121030Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"MAX_ITER = 1000","metadata":{"executionInfo":{"elapsed":373,"status":"ok","timestamp":1667918920336,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"aqk9RFIYj9tu","execution":{"iopub.status.busy":"2022-12-01T11:43:33.123260Z","iopub.execute_input":"2022-12-01T11:43:33.124185Z","iopub.status.idle":"2022-12-01T11:43:33.135523Z","shell.execute_reply.started":"2022-12-01T11:43:33.124142Z","shell.execute_reply":"2022-12-01T11:43:33.134469Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = LogisticRegression(max_iter=MAX_ITER, solver='sag')","metadata":{"executionInfo":{"elapsed":626,"status":"ok","timestamp":1667916401162,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"uUQ0wzNIjNHr","execution":{"iopub.status.busy":"2022-12-01T11:43:33.136501Z","iopub.execute_input":"2022-12-01T11:43:33.136820Z","iopub.status.idle":"2022-12-01T11:43:33.148965Z","shell.execute_reply.started":"2022-12-01T11:43:33.136791Z","shell.execute_reply":"2022-12-01T11:43:33.147985Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nmodel.fit(train_inputs, train_targets)","metadata":{"executionInfo":{"elapsed":27440,"status":"ok","timestamp":1667916434501,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"fOehu5rLjNFl","outputId":"319e0d60-e7db-4851-d7a5-1bcefd58f239","execution":{"iopub.status.busy":"2022-12-01T11:43:33.149925Z","iopub.execute_input":"2022-12-01T11:43:33.150254Z","iopub.status.idle":"2022-12-01T11:50:42.916242Z","shell.execute_reply.started":"2022-12-01T11:43:33.150212Z","shell.execute_reply":"2022-12-01T11:50:42.914935Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Make predictions using the model","metadata":{"id":"o16-8HmXmQT0"}},{"cell_type":"code","source":"train_preds = model.predict(train_inputs)","metadata":{"executionInfo":{"elapsed":1020,"status":"ok","timestamp":1667916460185,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"i12DBlkVkThU","execution":{"iopub.status.busy":"2022-12-01T11:50:42.923541Z","iopub.execute_input":"2022-12-01T11:50:42.923962Z","iopub.status.idle":"2022-12-01T11:50:42.957925Z","shell.execute_reply.started":"2022-12-01T11:50:42.923927Z","shell.execute_reply":"2022-12-01T11:50:42.956633Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_targets","metadata":{"executionInfo":{"elapsed":363,"status":"ok","timestamp":1667916573761,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"urNCyppPkxZB","outputId":"71349a02-b3a6-4b44-d87f-727dde95605e","execution":{"iopub.status.busy":"2022-12-01T11:50:42.959516Z","iopub.execute_input":"2022-12-01T11:50:42.959988Z","iopub.status.idle":"2022-12-01T11:50:42.972497Z","shell.execute_reply.started":"2022-12-01T11:50:42.959941Z","shell.execute_reply":"2022-12-01T11:50:42.970734Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_preds","metadata":{"executionInfo":{"elapsed":861,"status":"ok","timestamp":1667916464475,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"qS8PDYxljNDP","outputId":"9216c5a9-c9aa-4d6a-d540-e804d4d3c925","execution":{"iopub.status.busy":"2022-12-01T11:50:42.973917Z","iopub.execute_input":"2022-12-01T11:50:42.974281Z","iopub.status.idle":"2022-12-01T11:50:42.985372Z","shell.execute_reply.started":"2022-12-01T11:50:42.974249Z","shell.execute_reply":"2022-12-01T11:50:42.984386Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.Series(train_preds).value_counts()","metadata":{"executionInfo":{"elapsed":567,"status":"ok","timestamp":1667916484628,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"W74eTiJgjNA9","outputId":"4f1aa29b-d959-4561-9384-a83fc36ca13d","execution":{"iopub.status.busy":"2022-12-01T11:50:42.986867Z","iopub.execute_input":"2022-12-01T11:50:42.987331Z","iopub.status.idle":"2022-12-01T11:50:43.010784Z","shell.execute_reply.started":"2022-12-01T11:50:42.987287Z","shell.execute_reply":"2022-12-01T11:50:43.009245Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.Series(train_targets).value_counts()","metadata":{"executionInfo":{"elapsed":1125,"status":"ok","timestamp":1667916512008,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"pixsvLxrkhK4","outputId":"44819bce-587f-41e3-cb6b-55e67b8c7c84","execution":{"iopub.status.busy":"2022-12-01T11:50:43.012570Z","iopub.execute_input":"2022-12-01T11:50:43.012986Z","iopub.status.idle":"2022-12-01T11:50:43.037950Z","shell.execute_reply.started":"2022-12-01T11:50:43.012933Z","shell.execute_reply":"2022-12-01T11:50:43.036447Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import accuracy_score","metadata":{"executionInfo":{"elapsed":370,"status":"ok","timestamp":1667916547769,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"YxH33IpFkpXJ","execution":{"iopub.status.busy":"2022-12-01T11:50:43.040259Z","iopub.execute_input":"2022-12-01T11:50:43.040781Z","iopub.status.idle":"2022-12-01T11:50:43.048844Z","shell.execute_reply.started":"2022-12-01T11:50:43.040734Z","shell.execute_reply":"2022-12-01T11:50:43.047461Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"accuracy_score(train_targets, train_preds)","metadata":{"executionInfo":{"elapsed":893,"status":"ok","timestamp":1667916561938,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"oUgKam9RkYD6","outputId":"dfc12873-4db8-4993-ad67-bbf6456d8b18","execution":{"iopub.status.busy":"2022-12-01T11:50:43.051046Z","iopub.execute_input":"2022-12-01T11:50:43.052584Z","iopub.status.idle":"2022-12-01T11:50:43.164155Z","shell.execute_reply.started":"2022-12-01T11:50:43.052527Z","shell.execute_reply":"2022-12-01T11:50:43.162244Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np","metadata":{"executionInfo":{"elapsed":373,"status":"ok","timestamp":1667916643295,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"ipzg1inClCcJ","execution":{"iopub.status.busy":"2022-12-01T11:50:43.166540Z","iopub.execute_input":"2022-12-01T11:50:43.167018Z","iopub.status.idle":"2022-12-01T11:50:43.176847Z","shell.execute_reply.started":"2022-12-01T11:50:43.166981Z","shell.execute_reply":"2022-12-01T11:50:43.175386Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"accuracy_score(train_targets, np.zeros(len(train_targets)))","metadata":{"executionInfo":{"elapsed":620,"status":"ok","timestamp":1667916647313,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"_P-IsOtTk5x_","outputId":"3aed16ea-c1ab-4c6c-db22-cc083ebe1278","execution":{"iopub.status.busy":"2022-12-01T11:50:43.179023Z","iopub.execute_input":"2022-12-01T11:50:43.180190Z","iopub.status.idle":"2022-12-01T11:50:43.302963Z","shell.execute_reply.started":"2022-12-01T11:50:43.180148Z","shell.execute_reply":"2022-12-01T11:50:43.301549Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import f1_score","metadata":{"executionInfo":{"elapsed":896,"status":"ok","timestamp":1667916682226,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"89TtZhqclKwb","execution":{"iopub.status.busy":"2022-12-01T11:50:43.304012Z","iopub.execute_input":"2022-12-01T11:50:43.304397Z","iopub.status.idle":"2022-12-01T11:50:43.310431Z","shell.execute_reply.started":"2022-12-01T11:50:43.304363Z","shell.execute_reply":"2022-12-01T11:50:43.309308Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"f1_score(train_targets, train_preds)","metadata":{"executionInfo":{"elapsed":476,"status":"ok","timestamp":1667916695952,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"pim_sjENlKn0","outputId":"a919cf7d-3c6b-42ce-c99e-e2da856291ab","execution":{"iopub.status.busy":"2022-12-01T11:50:43.311687Z","iopub.execute_input":"2022-12-01T11:50:43.312009Z","iopub.status.idle":"2022-12-01T11:50:43.719022Z","shell.execute_reply.started":"2022-12-01T11:50:43.311981Z","shell.execute_reply":"2022-12-01T11:50:43.717875Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"f1_score(train_targets, np.zeros(len(train_targets)))","metadata":{"executionInfo":{"elapsed":375,"status":"ok","timestamp":1667916720434,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"RembBU-Fksbk","outputId":"9c815608-6f06-45d9-c352-d52ad659c965","execution":{"iopub.status.busy":"2022-12-01T11:50:43.720674Z","iopub.execute_input":"2022-12-01T11:50:43.721199Z","iopub.status.idle":"2022-12-01T11:50:44.214478Z","shell.execute_reply.started":"2022-12-01T11:50:43.721153Z","shell.execute_reply":"2022-12-01T11:50:44.212992Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"random_preds = np.random.choice((0, 1), len(train_targets))\nf1_score(train_targets, random_preds)","metadata":{"executionInfo":{"elapsed":530,"status":"ok","timestamp":1667916804410,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"43hI8WB8ld0e","outputId":"36a47c48-a715-47a9-9698-92c5eec0cd7f","execution":{"iopub.status.busy":"2022-12-01T11:50:44.216379Z","iopub.execute_input":"2022-12-01T11:50:44.216734Z","iopub.status.idle":"2022-12-01T11:50:44.696193Z","shell.execute_reply.started":"2022-12-01T11:50:44.216690Z","shell.execute_reply":"2022-12-01T11:50:44.694701Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"val_preds = model.predict(val_inputs)","metadata":{"executionInfo":{"elapsed":394,"status":"ok","timestamp":1667916889103,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"FM0X5ImAl7gB","execution":{"iopub.status.busy":"2022-12-01T11:50:44.697626Z","iopub.execute_input":"2022-12-01T11:50:44.699795Z","iopub.status.idle":"2022-12-01T11:50:44.717473Z","shell.execute_reply.started":"2022-12-01T11:50:44.699754Z","shell.execute_reply":"2022-12-01T11:50:44.715777Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"accuracy_score(val_targets, val_preds)","metadata":{"executionInfo":{"elapsed":592,"status":"ok","timestamp":1667916920449,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"WiXz18lal7db","outputId":"1199b1c7-51b2-4c95-d83e-d945e1541bf9","execution":{"iopub.status.busy":"2022-12-01T11:50:44.718900Z","iopub.execute_input":"2022-12-01T11:50:44.721159Z","iopub.status.idle":"2022-12-01T11:50:44.766231Z","shell.execute_reply.started":"2022-12-01T11:50:44.721118Z","shell.execute_reply":"2022-12-01T11:50:44.765314Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"f1_score(val_targets, val_preds)","metadata":{"executionInfo":{"elapsed":902,"status":"ok","timestamp":1667916932147,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"Kj-oIhIHksY-","outputId":"de4648a5-5f9c-48fb-f86e-aad601d0a956","execution":{"iopub.status.busy":"2022-12-01T11:50:44.767655Z","iopub.execute_input":"2022-12-01T11:50:44.768002Z","iopub.status.idle":"2022-12-01T11:50:44.938428Z","shell.execute_reply.started":"2022-12-01T11:50:44.767969Z","shell.execute_reply":"2022-12-01T11:50:44.937260Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sincere_df.question_text.values[:10]","metadata":{"executionInfo":{"elapsed":653,"status":"ok","timestamp":1667917017459,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"qxCSeXZQmHpS","outputId":"7cb895cb-7605-45d2-b650-b61b02cecffe","execution":{"iopub.status.busy":"2022-12-01T11:50:44.939806Z","iopub.execute_input":"2022-12-01T11:50:44.940163Z","iopub.status.idle":"2022-12-01T11:50:44.949148Z","shell.execute_reply.started":"2022-12-01T11:50:44.940131Z","shell.execute_reply":"2022-12-01T11:50:44.948295Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sincere_df.target.values[:10]","metadata":{"executionInfo":{"elapsed":1051,"status":"ok","timestamp":1667917031436,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"fwlbPtLDmgDf","outputId":"a6360495-7f8f-489e-b955-56fbcffa88bd","execution":{"iopub.status.busy":"2022-12-01T11:50:44.950475Z","iopub.execute_input":"2022-12-01T11:50:44.950986Z","iopub.status.idle":"2022-12-01T11:50:44.963678Z","shell.execute_reply.started":"2022-12-01T11:50:44.950954Z","shell.execute_reply":"2022-12-01T11:50:44.962278Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.predict(vectorizer.transform(sincere_df.question_text.values[:10]))","metadata":{"executionInfo":{"elapsed":375,"status":"ok","timestamp":1667917062825,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"qHE1gR0mmV0l","outputId":"c1e5f7f9-e5a3-4ea7-ff65-caa1c9280ce0","execution":{"iopub.status.busy":"2022-12-01T11:50:44.965736Z","iopub.execute_input":"2022-12-01T11:50:44.966849Z","iopub.status.idle":"2022-12-01T11:50:44.982349Z","shell.execute_reply.started":"2022-12-01T11:50:44.966791Z","shell.execute_reply":"2022-12-01T11:50:44.980787Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"insincere_df.question_text.values[:10]","metadata":{"executionInfo":{"elapsed":411,"status":"ok","timestamp":1667917165691,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"k58qFC-kmVyR","outputId":"d6b34cf6-182e-4375-d37e-01c3cfae8844","execution":{"iopub.status.busy":"2022-12-01T11:50:44.984083Z","iopub.execute_input":"2022-12-01T11:50:44.985100Z","iopub.status.idle":"2022-12-01T11:50:44.996979Z","shell.execute_reply.started":"2022-12-01T11:50:44.985035Z","shell.execute_reply":"2022-12-01T11:50:44.995651Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"insincere_df.target.values[:10]","metadata":{"executionInfo":{"elapsed":389,"status":"ok","timestamp":1667917180899,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"lFwis7sOmVv7","outputId":"f5b1eb04-8277-4b61-e0d8-6a08b7840770","execution":{"iopub.status.busy":"2022-12-01T11:50:44.998884Z","iopub.execute_input":"2022-12-01T11:50:44.999372Z","iopub.status.idle":"2022-12-01T11:50:45.014080Z","shell.execute_reply.started":"2022-12-01T11:50:44.999331Z","shell.execute_reply":"2022-12-01T11:50:45.012665Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.predict(vectorizer.transform(insincere_df.question_text.values[:10]))","metadata":{"executionInfo":{"elapsed":572,"status":"ok","timestamp":1667917197399,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"2T_zvDYvmVt6","outputId":"36b18600-75b3-4287-de0d-b963b97602f6","execution":{"iopub.status.busy":"2022-12-01T11:50:45.015581Z","iopub.execute_input":"2022-12-01T11:50:45.016066Z","iopub.status.idle":"2022-12-01T11:50:45.035421Z","shell.execute_reply.started":"2022-12-01T11:50:45.016031Z","shell.execute_reply":"2022-12-01T11:50:45.033679Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Make Predictions and Submit to Kaggle","metadata":{"id":"TSFA3r48Q2ec"}},{"cell_type":"code","source":"test_df","metadata":{"executionInfo":{"elapsed":1103,"status":"ok","timestamp":1667917318912,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"bIAZhmcWQDd4","outputId":"e92eb5e4-41ad-4590-da74-d9b14dc9697a","execution":{"iopub.status.busy":"2022-12-01T11:50:45.036927Z","iopub.execute_input":"2022-12-01T11:50:45.038032Z","iopub.status.idle":"2022-12-01T11:50:45.055979Z","shell.execute_reply.started":"2022-12-01T11:50:45.037988Z","shell.execute_reply":"2022-12-01T11:50:45.054324Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_inputs.shape","metadata":{"executionInfo":{"elapsed":460,"status":"ok","timestamp":1667917332470,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"jzb8x69DnoHz","outputId":"23a5695e-d7b4-49ae-c715-2360925ad007","execution":{"iopub.status.busy":"2022-12-01T11:50:45.057765Z","iopub.execute_input":"2022-12-01T11:50:45.058491Z","iopub.status.idle":"2022-12-01T11:50:45.069543Z","shell.execute_reply.started":"2022-12-01T11:50:45.058452Z","shell.execute_reply":"2022-12-01T11:50:45.068599Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_preds = model.predict(test_inputs)","metadata":{"executionInfo":{"elapsed":1112,"status":"ok","timestamp":1667917351071,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"oiY8Ai-anqRP","execution":{"iopub.status.busy":"2022-12-01T11:50:45.070601Z","iopub.execute_input":"2022-12-01T11:50:45.070947Z","iopub.status.idle":"2022-12-01T11:50:45.093070Z","shell.execute_reply.started":"2022-12-01T11:50:45.070917Z","shell.execute_reply":"2022-12-01T11:50:45.092057Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub_df.head(10)","metadata":{"executionInfo":{"elapsed":558,"status":"ok","timestamp":1667917386054,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"W8h1xhbZnwBt","outputId":"6698c104-0bd3-4c8a-b12f-6e59feac67a3","execution":{"iopub.status.busy":"2022-12-01T12:01:09.626965Z","iopub.execute_input":"2022-12-01T12:01:09.627403Z","iopub.status.idle":"2022-12-01T12:01:09.641359Z","shell.execute_reply.started":"2022-12-01T12:01:09.627367Z","shell.execute_reply":"2022-12-01T12:01:09.639082Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub_df.prediction = test_preds","metadata":{"executionInfo":{"elapsed":4,"status":"ok","timestamp":1667917429135,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"t6xxbg4yn9pT","execution":{"iopub.status.busy":"2022-12-01T11:50:45.115534Z","iopub.execute_input":"2022-12-01T11:50:45.116157Z","iopub.status.idle":"2022-12-01T11:50:45.131233Z","shell.execute_reply.started":"2022-12-01T11:50:45.116120Z","shell.execute_reply":"2022-12-01T11:50:45.130054Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub_df.prediction.value_counts()","metadata":{"executionInfo":{"elapsed":2,"status":"ok","timestamp":1667917437994,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"P1vWhwOIn4id","outputId":"4822357f-2f37-4a73-c939-ca1d3f981454","execution":{"iopub.status.busy":"2022-12-01T11:50:45.132931Z","iopub.execute_input":"2022-12-01T11:50:45.133805Z","iopub.status.idle":"2022-12-01T11:50:45.149675Z","shell.execute_reply.started":"2022-12-01T11:50:45.133765Z","shell.execute_reply":"2022-12-01T11:50:45.147891Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub_df.head(10)","metadata":{"executionInfo":{"elapsed":371,"status":"ok","timestamp":1667917442478,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"ZSHxruEXoFWj","outputId":"231bcd4a-0219-46cc-935a-a83b4e501672","execution":{"iopub.status.busy":"2022-12-01T12:01:20.079884Z","iopub.execute_input":"2022-12-01T12:01:20.081515Z","iopub.status.idle":"2022-12-01T12:01:20.094658Z","shell.execute_reply.started":"2022-12-01T12:01:20.081454Z","shell.execute_reply":"2022-12-01T12:01:20.093253Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub_df.to_csv('submission.csv', index=None)","metadata":{"executionInfo":{"elapsed":771,"status":"ok","timestamp":1667917485705,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"r_7JT4-JoGkL","execution":{"iopub.status.busy":"2022-12-01T11:50:45.169819Z","iopub.execute_input":"2022-12-01T11:50:45.170780Z","iopub.status.idle":"2022-12-01T11:50:45.661061Z","shell.execute_reply.started":"2022-12-01T11:50:45.170739Z","shell.execute_reply":"2022-12-01T11:50:45.659846Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!head submission.csv","metadata":{"executionInfo":{"elapsed":1077,"status":"ok","timestamp":1667917506958,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"496wLhWjoRA3","outputId":"24c74944-ee72-4168-8342-0c52decb77c4","execution":{"iopub.status.busy":"2022-12-01T11:50:45.662784Z","iopub.execute_input":"2022-12-01T11:50:45.663599Z","iopub.status.idle":"2022-12-01T11:50:46.809374Z","shell.execute_reply.started":"2022-12-01T11:50:45.663561Z","shell.execute_reply":"2022-12-01T11:50:46.808141Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"id":"o84S545HoWE4"},"execution_count":null,"outputs":[]}]}