{"metadata":{"colab":{"collapsed_sections":[],"machine_shape":"hm","provenance":[]},"gpuClass":"standard","kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":10737,"databundleVersionId":290346,"sourceType":"competition"}],"dockerImageVersionId":30732,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Text Classification with Bag of Words - Natural Language Processing\n\n![](https://i.imgur.com/hlEQ5X8.png)\n\n> _\"Natural language processing (NLP) is a subfield of linguistics, computer science, and artificial intelligence concerned with the interactions between computers and human language, in particular how to program computers to process and analyze large amounts of natural language data.\"_ - Wikipedia\n\n> _**Bag of Words**: The bag-of-words (BOW) model is a representation that turns arbitrary text into fixed-length vectors by counting how many times each word appears._\n \nOutline:\n\n1. Download and explore a real-world dataset\n2. Apply text preprocessing techniques\n3. Implement the bag of words model\n4. Train ML models for text classification\n5. Make predictions and submit to Kaggle\n\n\nDataset: https://www.kaggle.com/c/quora-insincere-questions-classification\n\n","metadata":{"id":"tBxC0flTQDd3"}},{"cell_type":"markdown","source":"## Download and Explore the Data\n\nOutline:\n\n1. Download the dataset from Kaggle to Colab\n2. Explore the data using Pandas\n3. Create a small working sample","metadata":{"id":"ZOcWwlvxQoSJ"}},{"cell_type":"markdown","source":"### Explore the Data using Pandas","metadata":{"id":"Ql6Gv4shTSBJ"}},{"cell_type":"code","source":"train_fname = '/kaggle/input/quora-insincere-questions-classification/train.csv'\ntest_fname = '/kaggle/input/quora-insincere-questions-classification/test.csv'\nsample_fname = '/kaggle/input/quora-insincere-questions-classification/sample_submission.csv' ","metadata":{"execution":{"iopub.status.busy":"2024-07-04T18:15:57.426113Z","iopub.execute_input":"2024-07-04T18:15:57.426532Z","iopub.status.idle":"2024-07-04T18:15:57.432604Z","shell.execute_reply.started":"2024-07-04T18:15:57.426477Z","shell.execute_reply":"2024-07-04T18:15:57.431360Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd","metadata":{"executionInfo":{"elapsed":358,"status":"ok","timestamp":1667912043319,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"pf65NRzdQ-hU","execution":{"iopub.status.busy":"2024-07-04T18:15:57.442234Z","iopub.execute_input":"2024-07-04T18:15:57.442619Z","iopub.status.idle":"2024-07-04T18:15:57.447694Z","shell.execute_reply.started":"2024-07-04T18:15:57.442585Z","shell.execute_reply":"2024-07-04T18:15:57.446414Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"raw_df = pd.read_csv(train_fname)","metadata":{"executionInfo":{"elapsed":4754,"status":"ok","timestamp":1667912065510,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"H1FjN5gDTfUY","execution":{"iopub.status.busy":"2024-07-04T18:15:57.466253Z","iopub.execute_input":"2024-07-04T18:15:57.466676Z","iopub.status.idle":"2024-07-04T18:16:01.107888Z","shell.execute_reply.started":"2024-07-04T18:15:57.466644Z","shell.execute_reply":"2024-07-04T18:16:01.106663Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"raw_df","metadata":{"executionInfo":{"elapsed":474,"status":"ok","timestamp":1667912069885,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"CHMgt_5iTfRt","outputId":"7ad71a7a-b478-4a42-bce0-ee467479480f","execution":{"iopub.status.busy":"2024-07-04T18:16:01.110035Z","iopub.execute_input":"2024-07-04T18:16:01.110394Z","iopub.status.idle":"2024-07-04T18:16:01.123805Z","shell.execute_reply.started":"2024-07-04T18:16:01.110362Z","shell.execute_reply":"2024-07-04T18:16:01.122733Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sincere_df = raw_df[raw_df.target == 0]","metadata":{"executionInfo":{"elapsed":382,"status":"ok","timestamp":1667912255712,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"wV2xGZ3STfPx","execution":{"iopub.status.busy":"2024-07-04T18:16:01.125523Z","iopub.execute_input":"2024-07-04T18:16:01.126021Z","iopub.status.idle":"2024-07-04T18:16:01.206088Z","shell.execute_reply.started":"2024-07-04T18:16:01.125979Z","shell.execute_reply":"2024-07-04T18:16:01.204978Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sincere_df.question_text.values[:10]","metadata":{"executionInfo":{"elapsed":375,"status":"ok","timestamp":1667912261902,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"me4srOFdUMCQ","outputId":"6f5e9340-bdb2-4c22-b75e-57177ccf6ea9","execution":{"iopub.status.busy":"2024-07-04T18:16:01.208818Z","iopub.execute_input":"2024-07-04T18:16:01.209189Z","iopub.status.idle":"2024-07-04T18:16:01.216810Z","shell.execute_reply.started":"2024-07-04T18:16:01.209157Z","shell.execute_reply":"2024-07-04T18:16:01.215643Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"insincere_df = raw_df[raw_df.target == 1]","metadata":{"executionInfo":{"elapsed":572,"status":"ok","timestamp":1667912293701,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"cTApBtxIUMAJ","execution":{"iopub.status.busy":"2024-07-04T18:16:01.218223Z","iopub.execute_input":"2024-07-04T18:16:01.218645Z","iopub.status.idle":"2024-07-04T18:16:01.245392Z","shell.execute_reply.started":"2024-07-04T18:16:01.218606Z","shell.execute_reply":"2024-07-04T18:16:01.244332Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"insincere_df.question_text.values[:10]","metadata":{"executionInfo":{"elapsed":652,"status":"ok","timestamp":1667912301181,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"-nFN5bZVUL91","outputId":"ce551e6c-3255-408d-9182-4408269d67bf","execution":{"iopub.status.busy":"2024-07-04T18:16:01.246814Z","iopub.execute_input":"2024-07-04T18:16:01.247216Z","iopub.status.idle":"2024-07-04T18:16:01.255528Z","shell.execute_reply.started":"2024-07-04T18:16:01.247135Z","shell.execute_reply":"2024-07-04T18:16:01.254302Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"raw_df.target.value_counts(normalize=True)","metadata":{"executionInfo":{"elapsed":431,"status":"ok","timestamp":1667912372012,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"n2VQSfKmUn7Z","outputId":"abe27eed-82e0-472d-f0bb-3c28a5a77b06","execution":{"iopub.status.busy":"2024-07-04T18:16:01.257728Z","iopub.execute_input":"2024-07-04T18:16:01.258240Z","iopub.status.idle":"2024-07-04T18:16:01.282547Z","shell.execute_reply.started":"2024-07-04T18:16:01.258198Z","shell.execute_reply":"2024-07-04T18:16:01.281243Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"raw_df.target.value_counts(normalize=True).plot(kind='bar')","metadata":{"executionInfo":{"elapsed":395,"status":"ok","timestamp":1667912455099,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"ZSzHylWuU0MF","outputId":"557af698-754b-49e6-a8f7-a504aaa74737","execution":{"iopub.status.busy":"2024-07-04T18:16:01.285307Z","iopub.execute_input":"2024-07-04T18:16:01.285769Z","iopub.status.idle":"2024-07-04T18:16:01.484014Z","shell.execute_reply.started":"2024-07-04T18:16:01.285728Z","shell.execute_reply":"2024-07-04T18:16:01.482894Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df = pd.read_csv(test_fname)","metadata":{"executionInfo":{"elapsed":1646,"status":"ok","timestamp":1667912505443,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"_lA6Vb6YVN1p","execution":{"iopub.status.busy":"2024-07-04T18:16:01.485397Z","iopub.execute_input":"2024-07-04T18:16:01.485827Z","iopub.status.idle":"2024-07-04T18:16:02.488301Z","shell.execute_reply.started":"2024-07-04T18:16:01.485788Z","shell.execute_reply":"2024-07-04T18:16:02.487147Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df","metadata":{"executionInfo":{"elapsed":3,"status":"ok","timestamp":1667912507665,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"gH0ij_pmVNzV","outputId":"ad30dad3-ffd4-43ac-c04d-659ac7cf006d","execution":{"iopub.status.busy":"2024-07-04T18:16:02.493005Z","iopub.execute_input":"2024-07-04T18:16:02.493370Z","iopub.status.idle":"2024-07-04T18:16:02.506463Z","shell.execute_reply.started":"2024-07-04T18:16:02.493340Z","shell.execute_reply":"2024-07-04T18:16:02.505220Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub_df = pd.read_csv(sample_fname)","metadata":{"executionInfo":{"elapsed":532,"status":"ok","timestamp":1667912554978,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"exDbcT9bVNw0","execution":{"iopub.status.busy":"2024-07-04T18:16:02.508089Z","iopub.execute_input":"2024-07-04T18:16:02.508606Z","iopub.status.idle":"2024-07-04T18:16:02.893818Z","shell.execute_reply.started":"2024-07-04T18:16:02.508566Z","shell.execute_reply":"2024-07-04T18:16:02.892416Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub_df","metadata":{"executionInfo":{"elapsed":571,"status":"ok","timestamp":1667912558749,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"vLCvMxN3UL7l","outputId":"75dcfd90-504e-4612-b578-dcff4978dc00","execution":{"iopub.status.busy":"2024-07-04T18:16:02.895492Z","iopub.execute_input":"2024-07-04T18:16:02.895958Z","iopub.status.idle":"2024-07-04T18:16:02.910866Z","shell.execute_reply.started":"2024-07-04T18:16:02.895917Z","shell.execute_reply":"2024-07-04T18:16:02.909772Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub_df.prediction.value_counts()","metadata":{"executionInfo":{"elapsed":394,"status":"ok","timestamp":1667912639872,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"BvL2NJuSVv4l","outputId":"d362dd88-f047-4ca6-e729-54b153126cb1","execution":{"iopub.status.busy":"2024-07-04T18:16:02.912469Z","iopub.execute_input":"2024-07-04T18:16:02.912850Z","iopub.status.idle":"2024-07-04T18:16:02.928437Z","shell.execute_reply.started":"2024-07-04T18:16:02.912820Z","shell.execute_reply":"2024-07-04T18:16:02.927298Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Create a Working Sample\n","metadata":{"id":"v7sWG0KCV0g-"}},{"cell_type":"code","source":"SAMPLE_SIZE = 100_000","metadata":{"executionInfo":{"elapsed":653,"status":"ok","timestamp":1667912690959,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"hu5s7hLKV7Kr","execution":{"iopub.status.busy":"2024-07-04T18:16:39.346831Z","iopub.execute_input":"2024-07-04T18:16:39.347255Z","iopub.status.idle":"2024-07-04T18:16:39.353890Z","shell.execute_reply.started":"2024-07-04T18:16:39.347221Z","shell.execute_reply":"2024-07-04T18:16:39.351974Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_df = raw_df.sample(SAMPLE_SIZE, random_state=42)","metadata":{"executionInfo":{"elapsed":702,"status":"ok","timestamp":1667912713848,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"0xzqyYo_V31J","execution":{"iopub.status.busy":"2024-07-04T18:16:39.860831Z","iopub.execute_input":"2024-07-04T18:16:39.861273Z","iopub.status.idle":"2024-07-04T18:16:39.966200Z","shell.execute_reply.started":"2024-07-04T18:16:39.861239Z","shell.execute_reply":"2024-07-04T18:16:39.965031Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_df","metadata":{"executionInfo":{"elapsed":886,"status":"ok","timestamp":1667912722831,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"4xFSOIfdV3yh","outputId":"a1f8d588-1005-475a-bfa9-04df72110179","execution":{"iopub.status.busy":"2024-07-04T18:16:39.968479Z","iopub.execute_input":"2024-07-04T18:16:39.968887Z","iopub.status.idle":"2024-07-04T18:16:39.983067Z","shell.execute_reply.started":"2024-07-04T18:16:39.968843Z","shell.execute_reply":"2024-07-04T18:16:39.981831Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Text Preprocessing Techniques\n\nOutline:\n\n1. Understand the bag of words model\n2. Tokenization\n3. Stop word removal\n4. Stemming","metadata":{"id":"-gZy068pQoPg"}},{"cell_type":"markdown","source":"### Bag of Words Intuition\n\n1. Create a list of all the words across all the text documents\n2. You convert each document into vector counts of each word\n\n\nLimitations:\n1. There may be too many words in the dataset\n2. Some words may occur too frequently\n3. Some words may occur very rarely or only once\n4. A single word may have many forms (go, gone, going or bird vs. birds)","metadata":{"id":"pJ0mh3gjXH9e"}},{"cell_type":"code","source":"q0 = sincere_df.question_text.values[1]","metadata":{"executionInfo":{"elapsed":628,"status":"ok","timestamp":1667913361214,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"PDb6D4pfYWPu","execution":{"iopub.status.busy":"2024-07-04T18:16:40.005565Z","iopub.execute_input":"2024-07-04T18:16:40.006045Z","iopub.status.idle":"2024-07-04T18:16:40.013798Z","shell.execute_reply.started":"2024-07-04T18:16:40.006010Z","shell.execute_reply":"2024-07-04T18:16:40.011470Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"q0","metadata":{"executionInfo":{"elapsed":546,"status":"ok","timestamp":1667913367151,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"mQjFxTKFYjJk","outputId":"3012387d-9827-46bf-a3bb-d9d077ee6a9f","execution":{"iopub.status.busy":"2024-07-04T18:16:40.017427Z","iopub.execute_input":"2024-07-04T18:16:40.018054Z","iopub.status.idle":"2024-07-04T18:16:40.032885Z","shell.execute_reply.started":"2024-07-04T18:16:40.018004Z","shell.execute_reply":"2024-07-04T18:16:40.030743Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"q1 = raw_df[raw_df.target == 1].question_text.values[0]","metadata":{"executionInfo":{"elapsed":478,"status":"ok","timestamp":1667913449142,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"GIvt_M-yYlJH","execution":{"iopub.status.busy":"2024-07-04T18:16:40.040930Z","iopub.execute_input":"2024-07-04T18:16:40.041438Z","iopub.status.idle":"2024-07-04T18:16:40.078285Z","shell.execute_reply.started":"2024-07-04T18:16:40.041401Z","shell.execute_reply":"2024-07-04T18:16:40.075782Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"q1","metadata":{"executionInfo":{"elapsed":3,"status":"ok","timestamp":1667913449769,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"86bHxTMbY182","outputId":"ffafd417-1607-420d-a2ff-9169707470b2","execution":{"iopub.status.busy":"2024-07-04T18:16:40.080830Z","iopub.execute_input":"2024-07-04T18:16:40.081266Z","iopub.status.idle":"2024-07-04T18:16:40.093021Z","shell.execute_reply.started":"2024-07-04T18:16:40.081231Z","shell.execute_reply":"2024-07-04T18:16:40.090750Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Tokenization\n\nsplitting a document into words and separators","metadata":{"id":"qj4Zg1yaYPJv"}},{"cell_type":"code","source":"import nltk","metadata":{"executionInfo":{"elapsed":474,"status":"ok","timestamp":1667913549193,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"EqacxiuyZPds","execution":{"iopub.status.busy":"2024-07-04T18:16:40.138428Z","iopub.execute_input":"2024-07-04T18:16:40.138867Z","iopub.status.idle":"2024-07-04T18:16:41.747916Z","shell.execute_reply.started":"2024-07-04T18:16:40.138836Z","shell.execute_reply":"2024-07-04T18:16:41.746712Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from nltk.tokenize import word_tokenize","metadata":{"executionInfo":{"elapsed":1852,"status":"ok","timestamp":1667913505183,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"2vQDMEq0Q_lP","execution":{"iopub.status.busy":"2024-07-04T18:16:41.750260Z","iopub.execute_input":"2024-07-04T18:16:41.750727Z","iopub.status.idle":"2024-07-04T18:16:41.756260Z","shell.execute_reply.started":"2024-07-04T18:16:41.750685Z","shell.execute_reply":"2024-07-04T18:16:41.755031Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"nltk.download('punkt')","metadata":{"executionInfo":{"elapsed":639,"status":"ok","timestamp":1667913551680,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"5eaAldKAZN-4","outputId":"d419ecd2-0540-4ed5-f0fc-32e68bd287dc","execution":{"iopub.status.busy":"2024-07-04T18:16:41.757598Z","iopub.execute_input":"2024-07-04T18:16:41.757933Z","iopub.status.idle":"2024-07-04T18:16:42.080764Z","shell.execute_reply.started":"2024-07-04T18:16:41.757904Z","shell.execute_reply":"2024-07-04T18:16:42.079660Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"q0","metadata":{"executionInfo":{"elapsed":1019,"status":"ok","timestamp":1667913515768,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"VyIaEATyQ_ip","outputId":"655a30fa-ccf0-4839-aa2a-15f4437a5562","execution":{"iopub.status.busy":"2024-07-04T18:16:42.084197Z","iopub.execute_input":"2024-07-04T18:16:42.084682Z","iopub.status.idle":"2024-07-04T18:16:42.091750Z","shell.execute_reply.started":"2024-07-04T18:16:42.084640Z","shell.execute_reply":"2024-07-04T18:16:42.090591Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"word_tokenize(q0)","metadata":{"executionInfo":{"elapsed":458,"status":"ok","timestamp":1667913557740,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"1yULG5nVQ_gk","outputId":"83e3941f-3d29-4008-c885-b9a5eaf93912","execution":{"iopub.status.busy":"2024-07-04T18:16:42.093348Z","iopub.execute_input":"2024-07-04T18:16:42.093779Z","iopub.status.idle":"2024-07-04T18:16:42.118082Z","shell.execute_reply.started":"2024-07-04T18:16:42.093740Z","shell.execute_reply":"2024-07-04T18:16:42.116955Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"word_tokenize(' this is (something) with, a lot of, punctuation;')","metadata":{"executionInfo":{"elapsed":451,"status":"ok","timestamp":1667913596244,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"gJ_lrgEpZIt8","outputId":"aa250716-93b6-47ab-cbd1-d9ba9076c620","execution":{"iopub.status.busy":"2024-07-04T18:16:42.119915Z","iopub.execute_input":"2024-07-04T18:16:42.120267Z","iopub.status.idle":"2024-07-04T18:16:42.128677Z","shell.execute_reply.started":"2024-07-04T18:16:42.120236Z","shell.execute_reply":"2024-07-04T18:16:42.127394Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"q1","metadata":{"executionInfo":{"elapsed":913,"status":"ok","timestamp":1667913614596,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"uoP8H4KcZIr1","outputId":"0800baed-d874-4f5c-ba0b-fb5aad5b0fc0","execution":{"iopub.status.busy":"2024-07-04T18:16:42.130199Z","iopub.execute_input":"2024-07-04T18:16:42.130601Z","iopub.status.idle":"2024-07-04T18:16:42.141555Z","shell.execute_reply.started":"2024-07-04T18:16:42.130568Z","shell.execute_reply":"2024-07-04T18:16:42.140331Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"word_tokenize(q1)","metadata":{"executionInfo":{"elapsed":977,"status":"ok","timestamp":1667913623231,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"Bs_0YAawZIpf","outputId":"e97ea6f2-c22c-4f06-fe1f-160df99340ad","execution":{"iopub.status.busy":"2024-07-04T18:16:42.143990Z","iopub.execute_input":"2024-07-04T18:16:42.144467Z","iopub.status.idle":"2024-07-04T18:16:42.156870Z","shell.execute_reply.started":"2024-07-04T18:16:42.144434Z","shell.execute_reply":"2024-07-04T18:16:42.155722Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"q0_tok = word_tokenize(q0)\nq1_tok = word_tokenize(q1)","metadata":{"executionInfo":{"elapsed":1,"status":"ok","timestamp":1667913656571,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"7HNGqFYtZInd","execution":{"iopub.status.busy":"2024-07-04T18:16:42.159017Z","iopub.execute_input":"2024-07-04T18:16:42.159451Z","iopub.status.idle":"2024-07-04T18:16:42.169827Z","shell.execute_reply.started":"2024-07-04T18:16:42.159418Z","shell.execute_reply":"2024-07-04T18:16:42.168563Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Stop Word Removal\n\nRemoving commonly occuring words","metadata":{"id":"Mf5aqvkGZrR-"}},{"cell_type":"code","source":"q1_tok","metadata":{"executionInfo":{"elapsed":385,"status":"ok","timestamp":1667913691780,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"7yZhlXV9Zxo-","outputId":"408a649d-3883-4f70-e1b5-a052ada07d60","execution":{"iopub.status.busy":"2024-07-04T18:16:42.174104Z","iopub.execute_input":"2024-07-04T18:16:42.174525Z","iopub.status.idle":"2024-07-04T18:16:42.184085Z","shell.execute_reply.started":"2024-07-04T18:16:42.174464Z","shell.execute_reply":"2024-07-04T18:16:42.182826Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from nltk.corpus import stopwords","metadata":{"executionInfo":{"elapsed":1009,"status":"ok","timestamp":1667913739715,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"s1qsENBhZrD0","execution":{"iopub.status.busy":"2024-07-04T18:16:42.185677Z","iopub.execute_input":"2024-07-04T18:16:42.186050Z","iopub.status.idle":"2024-07-04T18:16:42.195639Z","shell.execute_reply.started":"2024-07-04T18:16:42.186010Z","shell.execute_reply":"2024-07-04T18:16:42.194363Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"nltk.download('stopwords')","metadata":{"executionInfo":{"elapsed":459,"status":"ok","timestamp":1667913791292,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"4kjpGIjraK7r","outputId":"dc16bc42-9e3e-4796-8937-5d9af34d099e","execution":{"iopub.status.busy":"2024-07-04T18:16:42.197186Z","iopub.execute_input":"2024-07-04T18:16:42.197610Z","iopub.status.idle":"2024-07-04T18:16:42.219298Z","shell.execute_reply.started":"2024-07-04T18:16:42.197576Z","shell.execute_reply":"2024-07-04T18:16:42.218095Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"english_stopwords = stopwords.words('english')","metadata":{"executionInfo":{"elapsed":430,"status":"ok","timestamp":1667913795278,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"ejhkI9oVZrCD","execution":{"iopub.status.busy":"2024-07-04T18:16:42.221016Z","iopub.execute_input":"2024-07-04T18:16:42.221470Z","iopub.status.idle":"2024-07-04T18:16:42.230023Z","shell.execute_reply.started":"2024-07-04T18:16:42.221431Z","shell.execute_reply":"2024-07-04T18:16:42.228957Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\", \".join(english_stopwords)","metadata":{"executionInfo":{"elapsed":357,"status":"ok","timestamp":1667913817359,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"jB5avxBsZqxB","outputId":"27c5b385-e8e1-4987-cc2a-19d43eb70308","execution":{"iopub.status.busy":"2024-07-04T18:16:42.231555Z","iopub.execute_input":"2024-07-04T18:16:42.232040Z","iopub.status.idle":"2024-07-04T18:16:42.240736Z","shell.execute_reply.started":"2024-07-04T18:16:42.232000Z","shell.execute_reply":"2024-07-04T18:16:42.239484Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def remove_stopwords(tokens):\n    return [word for word in tokens if word.lower() not in english_stopwords]","metadata":{"executionInfo":{"elapsed":352,"status":"ok","timestamp":1667913917513,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"7umWfNCvZqsF","execution":{"iopub.status.busy":"2024-07-04T18:16:42.242407Z","iopub.execute_input":"2024-07-04T18:16:42.242868Z","iopub.status.idle":"2024-07-04T18:16:42.251455Z","shell.execute_reply.started":"2024-07-04T18:16:42.242811Z","shell.execute_reply":"2024-07-04T18:16:42.249914Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"q0_tok","metadata":{"executionInfo":{"elapsed":930,"status":"ok","timestamp":1667913931371,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"15NSoE_Saq6W","outputId":"4abcb5d0-fae7-4c59-f01d-c8d641d798e9","execution":{"iopub.status.busy":"2024-07-04T18:16:42.253326Z","iopub.execute_input":"2024-07-04T18:16:42.254260Z","iopub.status.idle":"2024-07-04T18:16:42.267155Z","shell.execute_reply.started":"2024-07-04T18:16:42.254212Z","shell.execute_reply":"2024-07-04T18:16:42.265826Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"q0_stp = remove_stopwords(q0_tok)","metadata":{"executionInfo":{"elapsed":667,"status":"ok","timestamp":1667913944997,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"b59BPAClaq4B","execution":{"iopub.status.busy":"2024-07-04T18:16:42.269079Z","iopub.execute_input":"2024-07-04T18:16:42.270012Z","iopub.status.idle":"2024-07-04T18:16:42.279078Z","shell.execute_reply.started":"2024-07-04T18:16:42.269973Z","shell.execute_reply":"2024-07-04T18:16:42.277901Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"q0_stp","metadata":{"executionInfo":{"elapsed":502,"status":"ok","timestamp":1667913950071,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"UJpOKfd-Zqp_","outputId":"d9cd0cfc-26a5-45a2-eb18-3517d13bedda","execution":{"iopub.status.busy":"2024-07-04T18:16:43.218546Z","iopub.execute_input":"2024-07-04T18:16:43.219530Z","iopub.status.idle":"2024-07-04T18:16:43.227335Z","shell.execute_reply.started":"2024-07-04T18:16:43.219466Z","shell.execute_reply":"2024-07-04T18:16:43.225995Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"q1_stp = remove_stopwords(q1_tok)","metadata":{"executionInfo":{"elapsed":476,"status":"ok","timestamp":1667913994939,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"IKR9dyGVa6OR","execution":{"iopub.status.busy":"2024-07-04T18:16:43.716523Z","iopub.execute_input":"2024-07-04T18:16:43.717538Z","iopub.status.idle":"2024-07-04T18:16:43.722148Z","shell.execute_reply.started":"2024-07-04T18:16:43.717473Z","shell.execute_reply":"2024-07-04T18:16:43.720965Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"q1_tok","metadata":{"executionInfo":{"elapsed":699,"status":"ok","timestamp":1667913999121,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"_RvMiZAka6Lv","outputId":"4f446d13-cf60-4da4-e6d6-50b9f46a1ad0","execution":{"iopub.status.busy":"2024-07-04T18:16:43.736425Z","iopub.execute_input":"2024-07-04T18:16:43.736874Z","iopub.status.idle":"2024-07-04T18:16:43.744525Z","shell.execute_reply.started":"2024-07-04T18:16:43.736840Z","shell.execute_reply":"2024-07-04T18:16:43.742988Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"q1_stp","metadata":{"executionInfo":{"elapsed":485,"status":"ok","timestamp":1667914001968,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"GqpqLKw9a-A8","outputId":"9280f918-57dc-4673-bb3b-274107f8b62b","execution":{"iopub.status.busy":"2024-07-04T18:16:43.766633Z","iopub.execute_input":"2024-07-04T18:16:43.767059Z","iopub.status.idle":"2024-07-04T18:16:43.775273Z","shell.execute_reply.started":"2024-07-04T18:16:43.767021Z","shell.execute_reply":"2024-07-04T18:16:43.774185Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Stemming\n\n\"go\", \"gone\", \"going\" -> \"go\"\n\"birds\", \"bird\" -> \"bird\"","metadata":{"id":"FeVx68x5bHEF"}},{"cell_type":"code","source":"from nltk.stem.snowball import SnowballStemmer","metadata":{"executionInfo":{"elapsed":1054,"status":"ok","timestamp":1667914103321,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"9jFOfSqtbG6y","execution":{"iopub.status.busy":"2024-07-04T18:16:43.836572Z","iopub.execute_input":"2024-07-04T18:16:43.837761Z","iopub.status.idle":"2024-07-04T18:16:43.842942Z","shell.execute_reply.started":"2024-07-04T18:16:43.837722Z","shell.execute_reply":"2024-07-04T18:16:43.841648Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"stemmer = SnowballStemmer(language='english')","metadata":{"executionInfo":{"elapsed":544,"status":"ok","timestamp":1667914116697,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"SC0PMIy3bG4r","execution":{"iopub.status.busy":"2024-07-04T18:16:43.875283Z","iopub.execute_input":"2024-07-04T18:16:43.875707Z","iopub.status.idle":"2024-07-04T18:16:43.880838Z","shell.execute_reply.started":"2024-07-04T18:16:43.875673Z","shell.execute_reply":"2024-07-04T18:16:43.879458Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"stemmer.stem('going')","metadata":{"executionInfo":{"elapsed":792,"status":"ok","timestamp":1667914135778,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"apl0nhFqbG2F","outputId":"fc70d0f5-0f7a-49b4-ace3-ea1113cb8c66","execution":{"iopub.status.busy":"2024-07-04T18:16:43.914013Z","iopub.execute_input":"2024-07-04T18:16:43.914589Z","iopub.status.idle":"2024-07-04T18:16:43.922594Z","shell.execute_reply.started":"2024-07-04T18:16:43.914551Z","shell.execute_reply":"2024-07-04T18:16:43.921352Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"stemmer.stem('supposedly')","metadata":{"executionInfo":{"elapsed":386,"status":"ok","timestamp":1667914142907,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"wtIoHT1ybGz2","outputId":"3609cf3d-2c9f-4638-db1f-07ce31e63f20","execution":{"iopub.status.busy":"2024-07-04T18:16:43.951545Z","iopub.execute_input":"2024-07-04T18:16:43.952231Z","iopub.status.idle":"2024-07-04T18:16:43.961527Z","shell.execute_reply.started":"2024-07-04T18:16:43.952196Z","shell.execute_reply":"2024-07-04T18:16:43.959192Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"q0_stm = [stemmer.stem(word) for word in q0_stp]","metadata":{"executionInfo":{"elapsed":364,"status":"ok","timestamp":1667914180406,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"hGdECu-rbmbZ","execution":{"iopub.status.busy":"2024-07-04T18:16:43.967453Z","iopub.execute_input":"2024-07-04T18:16:43.968211Z","iopub.status.idle":"2024-07-04T18:16:43.974311Z","shell.execute_reply.started":"2024-07-04T18:16:43.968173Z","shell.execute_reply":"2024-07-04T18:16:43.972946Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"q0_stp","metadata":{"executionInfo":{"elapsed":817,"status":"ok","timestamp":1667914186316,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"BY0fs95pbqtH","outputId":"5e0d4927-385f-40c7-ad88-82ec30049890","execution":{"iopub.status.busy":"2024-07-04T18:16:44.018599Z","iopub.execute_input":"2024-07-04T18:16:44.018991Z","iopub.status.idle":"2024-07-04T18:16:44.026380Z","shell.execute_reply.started":"2024-07-04T18:16:44.018960Z","shell.execute_reply":"2024-07-04T18:16:44.025124Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"q0_stm","metadata":{"executionInfo":{"elapsed":573,"status":"ok","timestamp":1667914193742,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"ARuFpGkHbqn2","outputId":"3e4e85e5-bb93-4feb-97f2-5141ebbcd1db","execution":{"iopub.status.busy":"2024-07-04T18:16:44.034083Z","iopub.execute_input":"2024-07-04T18:16:44.035416Z","iopub.status.idle":"2024-07-04T18:16:44.042713Z","shell.execute_reply.started":"2024-07-04T18:16:44.035376Z","shell.execute_reply":"2024-07-04T18:16:44.041407Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"q1_stm = [stemmer.stem(word) for word in q1_stp]","metadata":{"executionInfo":{"elapsed":432,"status":"ok","timestamp":1667914233752,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"kiFiGdCpbrot","execution":{"iopub.status.busy":"2024-07-04T18:16:44.353527Z","iopub.execute_input":"2024-07-04T18:16:44.353899Z","iopub.status.idle":"2024-07-04T18:16:44.359815Z","shell.execute_reply.started":"2024-07-04T18:16:44.353870Z","shell.execute_reply":"2024-07-04T18:16:44.358469Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"q1_stp","metadata":{"executionInfo":{"elapsed":458,"status":"ok","timestamp":1667914238715,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"WweHQFO4brmn","outputId":"58cc1dae-68c0-4778-9fb1-27e37e054e86","execution":{"iopub.status.busy":"2024-07-04T18:16:44.796430Z","iopub.execute_input":"2024-07-04T18:16:44.797310Z","iopub.status.idle":"2024-07-04T18:16:44.805961Z","shell.execute_reply.started":"2024-07-04T18:16:44.797269Z","shell.execute_reply":"2024-07-04T18:16:44.804382Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"q1_stm","metadata":{"executionInfo":{"elapsed":906,"status":"ok","timestamp":1667914244006,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"AVu5FiAbbrkV","outputId":"4a4c2193-e5b7-4f1c-8781-127e2f84efb0","execution":{"iopub.status.busy":"2024-07-04T18:16:44.810167Z","iopub.execute_input":"2024-07-04T18:16:44.811356Z","iopub.status.idle":"2024-07-04T18:16:44.822697Z","shell.execute_reply.started":"2024-07-04T18:16:44.811289Z","shell.execute_reply":"2024-07-04T18:16:44.821242Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Lemmatization\n\n\"love\" -> \"love\"\n\"loving\" -> \"love\"\n\"lovable\" -> \"love\"\n","metadata":{"id":"VWWkiTcWcAR7"}},{"cell_type":"markdown","source":"## Implement Bag of Words \n\n\nOutline:\n\n1. Create a vocabulary using Count Vectorizer\n2. Transform text to vectors using Count Vectorizer\n3. Configure text preprocessing in Count Vectorizer","metadata":{"id":"CuTR9sOWQoMq"}},{"cell_type":"markdown","source":"### Create a Vocabulary","metadata":{"id":"0HWShc6jdPFy"}},{"cell_type":"code","source":"sample_df","metadata":{"executionInfo":{"elapsed":419,"status":"ok","timestamp":1667914532328,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"uGScEAd9c_dQ","outputId":"e923813b-b8fd-438b-a6f0-743c280a204d","execution":{"iopub.status.busy":"2024-07-04T18:16:44.978378Z","iopub.execute_input":"2024-07-04T18:16:44.978811Z","iopub.status.idle":"2024-07-04T18:16:44.994302Z","shell.execute_reply.started":"2024-07-04T18:16:44.978779Z","shell.execute_reply":"2024-07-04T18:16:44.992729Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"small_df = sample_df[:5]","metadata":{"executionInfo":{"elapsed":707,"status":"ok","timestamp":1667914559612,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"IKj0BM2NRATm","execution":{"iopub.status.busy":"2024-07-04T18:16:44.996405Z","iopub.execute_input":"2024-07-04T18:16:44.996843Z","iopub.status.idle":"2024-07-04T18:16:45.004723Z","shell.execute_reply.started":"2024-07-04T18:16:44.996811Z","shell.execute_reply":"2024-07-04T18:16:45.003397Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"small_df","metadata":{"executionInfo":{"elapsed":1222,"status":"ok","timestamp":1667914576774,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"awqkIOZEdKFu","outputId":"9c110405-71e1-4c72-c02c-dfbc1d5be43c","execution":{"iopub.status.busy":"2024-07-04T18:16:45.016852Z","iopub.execute_input":"2024-07-04T18:16:45.017267Z","iopub.status.idle":"2024-07-04T18:16:45.030578Z","shell.execute_reply.started":"2024-07-04T18:16:45.017236Z","shell.execute_reply":"2024-07-04T18:16:45.029157Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"small_df.question_text.values","metadata":{"executionInfo":{"elapsed":430,"status":"ok","timestamp":1667914566540,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"XZOIXmC3RARh","outputId":"330c3a06-1368-40de-ae9c-40f0061e61d6","execution":{"iopub.status.busy":"2024-07-04T18:16:45.056456Z","iopub.execute_input":"2024-07-04T18:16:45.057318Z","iopub.status.idle":"2024-07-04T18:16:45.065446Z","shell.execute_reply.started":"2024-07-04T18:16:45.057275Z","shell.execute_reply":"2024-07-04T18:16:45.063821Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.feature_extraction.text import CountVectorizer","metadata":{"executionInfo":{"elapsed":362,"status":"ok","timestamp":1667914669258,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"Yr3XgzBWdNNx","execution":{"iopub.status.busy":"2024-07-04T18:16:45.096337Z","iopub.execute_input":"2024-07-04T18:16:45.096756Z","iopub.status.idle":"2024-07-04T18:16:45.102629Z","shell.execute_reply.started":"2024-07-04T18:16:45.096723Z","shell.execute_reply":"2024-07-04T18:16:45.101062Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"small_vect = CountVectorizer()","metadata":{"executionInfo":{"elapsed":351,"status":"ok","timestamp":1667914677547,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"-sE8WL3-dNLf","execution":{"iopub.status.busy":"2024-07-04T18:16:45.147050Z","iopub.execute_input":"2024-07-04T18:16:45.147972Z","iopub.status.idle":"2024-07-04T18:16:45.153311Z","shell.execute_reply.started":"2024-07-04T18:16:45.147921Z","shell.execute_reply":"2024-07-04T18:16:45.151994Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"small_vect.fit(small_df.question_text)","metadata":{"executionInfo":{"elapsed":409,"status":"ok","timestamp":1667914713345,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"ba9oOkD6RAPP","outputId":"eccc7e9b-266b-4aea-90fa-654fa176b504","execution":{"iopub.status.busy":"2024-07-04T18:16:45.546604Z","iopub.execute_input":"2024-07-04T18:16:45.547011Z","iopub.status.idle":"2024-07-04T18:16:45.568901Z","shell.execute_reply.started":"2024-07-04T18:16:45.546978Z","shell.execute_reply":"2024-07-04T18:16:45.567308Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"small_vect.get_feature_names_out()","metadata":{"executionInfo":{"elapsed":651,"status":"ok","timestamp":1667914761316,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"-QPGkxLZdkPy","outputId":"7de3797e-3a41-4bb1-c581-ad7bd7610eb4","execution":{"iopub.status.busy":"2024-07-04T18:16:46.076682Z","iopub.execute_input":"2024-07-04T18:16:46.077076Z","iopub.status.idle":"2024-07-04T18:16:46.084884Z","shell.execute_reply.started":"2024-07-04T18:16:46.077046Z","shell.execute_reply":"2024-07-04T18:16:46.083678Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Transform documents into Vectors","metadata":{"id":"WXsUWz_hd9KE"}},{"cell_type":"code","source":"vectors = small_vect.transform(small_df.question_text)","metadata":{"executionInfo":{"elapsed":533,"status":"ok","timestamp":1667914847949,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"nZVZ3jIddkKT","execution":{"iopub.status.busy":"2024-07-04T18:16:46.145696Z","iopub.execute_input":"2024-07-04T18:16:46.146153Z","iopub.status.idle":"2024-07-04T18:16:46.152223Z","shell.execute_reply.started":"2024-07-04T18:16:46.146119Z","shell.execute_reply":"2024-07-04T18:16:46.151097Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"vectors","metadata":{"executionInfo":{"elapsed":638,"status":"ok","timestamp":1667914853853,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"MIGbsQEQdkHt","outputId":"1ad2eff7-d795-4c0e-ab60-f88ce7b4ed8c","execution":{"iopub.status.busy":"2024-07-04T18:16:46.179614Z","iopub.execute_input":"2024-07-04T18:16:46.180330Z","iopub.status.idle":"2024-07-04T18:16:46.187728Z","shell.execute_reply.started":"2024-07-04T18:16:46.180295Z","shell.execute_reply":"2024-07-04T18:16:46.186516Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"vectors.shape","metadata":{"executionInfo":{"elapsed":630,"status":"ok","timestamp":1667914900662,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"wriwC-1DeZPA","outputId":"9b7a3cba-e4e6-4023-851b-7cb6b6029f40","execution":{"iopub.status.busy":"2024-07-04T18:16:46.202428Z","iopub.execute_input":"2024-07-04T18:16:46.202894Z","iopub.status.idle":"2024-07-04T18:16:46.211170Z","shell.execute_reply.started":"2024-07-04T18:16:46.202842Z","shell.execute_reply":"2024-07-04T18:16:46.209958Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"small_df.question_text.values[0]","metadata":{"executionInfo":{"elapsed":495,"status":"ok","timestamp":1667914953574,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"bXtFEWjtekqt","outputId":"a4c659e8-4c86-478e-c827-ddbacde28bd9","execution":{"iopub.status.busy":"2024-07-04T18:16:46.226735Z","iopub.execute_input":"2024-07-04T18:16:46.227221Z","iopub.status.idle":"2024-07-04T18:16:46.236251Z","shell.execute_reply.started":"2024-07-04T18:16:46.227185Z","shell.execute_reply":"2024-07-04T18:16:46.234480Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"vectors[0].toarray()","metadata":{"executionInfo":{"elapsed":982,"status":"ok","timestamp":1667915020439,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"etQ-T7_1e0vF","outputId":"9446f31f-ae80-4fac-857f-42f9481bafed","execution":{"iopub.status.busy":"2024-07-04T18:16:46.257696Z","iopub.execute_input":"2024-07-04T18:16:46.258158Z","iopub.status.idle":"2024-07-04T18:16:46.268078Z","shell.execute_reply.started":"2024-07-04T18:16:46.258123Z","shell.execute_reply":"2024-07-04T18:16:46.266639Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"vectors.toarray()","metadata":{"executionInfo":{"elapsed":483,"status":"ok","timestamp":1667914889375,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"hSrf1ZPndkFX","outputId":"63dd68aa-56d5-4838-9f41-9f12c165e9b2","execution":{"iopub.status.busy":"2024-07-04T18:16:46.296589Z","iopub.execute_input":"2024-07-04T18:16:46.297012Z","iopub.status.idle":"2024-07-04T18:16:46.306265Z","shell.execute_reply.started":"2024-07-04T18:16:46.296978Z","shell.execute_reply":"2024-07-04T18:16:46.304965Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Configure Count Vectorizer Parameters","metadata":{"id":"vZPCOwobe485"}},{"cell_type":"code","source":"stemmer = SnowballStemmer(language='english')","metadata":{"executionInfo":{"elapsed":391,"status":"ok","timestamp":1667915246398,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"xUjZDsdqfq4u","execution":{"iopub.status.busy":"2024-07-04T18:16:46.378639Z","iopub.execute_input":"2024-07-04T18:16:46.379073Z","iopub.status.idle":"2024-07-04T18:16:46.384980Z","shell.execute_reply.started":"2024-07-04T18:16:46.379038Z","shell.execute_reply":"2024-07-04T18:16:46.383370Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def tokenize(text):\n    return [stemmer.stem(word) for word in word_tokenize(text)]","metadata":{"executionInfo":{"elapsed":368,"status":"ok","timestamp":1667915256240,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"9hZzn1LBfPoM","execution":{"iopub.status.busy":"2024-07-04T18:16:46.391964Z","iopub.execute_input":"2024-07-04T18:16:46.392476Z","iopub.status.idle":"2024-07-04T18:16:46.399550Z","shell.execute_reply.started":"2024-07-04T18:16:46.392440Z","shell.execute_reply":"2024-07-04T18:16:46.398121Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tokenize('What is the really (dealing) here?')","metadata":{"executionInfo":{"elapsed":358,"status":"ok","timestamp":1667915263406,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"OAausoIrfa2I","outputId":"96c1e3a3-1f5c-40d0-e05b-1c32046aff8c","execution":{"iopub.status.busy":"2024-07-04T18:16:46.459153Z","iopub.execute_input":"2024-07-04T18:16:46.459565Z","iopub.status.idle":"2024-07-04T18:16:46.467671Z","shell.execute_reply.started":"2024-07-04T18:16:46.459533Z","shell.execute_reply":"2024-07-04T18:16:46.466289Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"vectorizer = CountVectorizer(lowercase=True, \n                             tokenizer=tokenize,\n                             stop_words=english_stopwords,\n                             max_features=1000)","metadata":{"executionInfo":{"elapsed":559,"status":"ok","timestamp":1667915360958,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"pJGAErs7e9Fh","execution":{"iopub.status.busy":"2024-07-04T18:16:46.996753Z","iopub.execute_input":"2024-07-04T18:16:46.997149Z","iopub.status.idle":"2024-07-04T18:16:47.003371Z","shell.execute_reply.started":"2024-07-04T18:16:46.997116Z","shell.execute_reply":"2024-07-04T18:16:47.001912Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nvectorizer.fit(sample_df.question_text)","metadata":{"executionInfo":{"elapsed":32457,"status":"ok","timestamp":1667915423839,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"pI8nbkpAe9Dg","outputId":"e6151146-4e99-4bb5-bf70-c7336b5ed226","execution":{"iopub.status.busy":"2024-07-04T18:16:47.497754Z","iopub.execute_input":"2024-07-04T18:16:47.498160Z","iopub.status.idle":"2024-07-04T18:17:37.847886Z","shell.execute_reply.started":"2024-07-04T18:16:47.498127Z","shell.execute_reply":"2024-07-04T18:17:37.846422Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(vectorizer.vocabulary_)","metadata":{"executionInfo":{"elapsed":2068,"status":"ok","timestamp":1667915449808,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"sJBlLm5Ye88Z","outputId":"e8ad80f8-cd4d-4289-fd26-163dc808e69f","execution":{"iopub.status.busy":"2024-07-04T18:17:37.850430Z","iopub.execute_input":"2024-07-04T18:17:37.850909Z","iopub.status.idle":"2024-07-04T18:17:37.858714Z","shell.execute_reply.started":"2024-07-04T18:17:37.850875Z","shell.execute_reply":"2024-07-04T18:17:37.857277Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"vectorizer.get_feature_names_out()[:100]","metadata":{"executionInfo":{"elapsed":1059,"status":"ok","timestamp":1667915475499,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"b9K638h1gg5_","outputId":"955fe2e0-52ab-4137-9ecc-f18a64542e45","execution":{"iopub.status.busy":"2024-07-04T18:17:37.860618Z","iopub.execute_input":"2024-07-04T18:17:37.861064Z","iopub.status.idle":"2024-07-04T18:17:37.874095Z","shell.execute_reply.started":"2024-07-04T18:17:37.861022Z","shell.execute_reply":"2024-07-04T18:17:37.872713Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ninputs = vectorizer.transform(sample_df.question_text)","metadata":{"executionInfo":{"elapsed":30784,"status":"ok","timestamp":1667915566857,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"n3iq65OsguTR","outputId":"35c22bcd-b6c2-4629-9e83-ad81a29456ee","execution":{"iopub.status.busy":"2024-07-04T18:17:37.877299Z","iopub.execute_input":"2024-07-04T18:17:37.877786Z","iopub.status.idle":"2024-07-04T18:18:27.863157Z","shell.execute_reply.started":"2024-07-04T18:17:37.877743Z","shell.execute_reply":"2024-07-04T18:18:27.861697Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"inputs.shape","metadata":{"executionInfo":{"elapsed":661,"status":"ok","timestamp":1667915574286,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"9aEbKIC0gg3o","outputId":"a0f9fdcf-a53d-455d-bd49-691f1f737a47","execution":{"iopub.status.busy":"2024-07-04T18:18:27.864845Z","iopub.execute_input":"2024-07-04T18:18:27.865402Z","iopub.status.idle":"2024-07-04T18:18:27.874371Z","shell.execute_reply.started":"2024-07-04T18:18:27.865348Z","shell.execute_reply":"2024-07-04T18:18:27.872674Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"inputs","metadata":{"executionInfo":{"elapsed":2,"status":"ok","timestamp":1667915578819,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"meslQK7Le86I","outputId":"d6cdab93-2640-43d0-d5fd-b907c201177d","execution":{"iopub.status.busy":"2024-07-04T18:18:27.876989Z","iopub.execute_input":"2024-07-04T18:18:27.877915Z","iopub.status.idle":"2024-07-04T18:18:27.890426Z","shell.execute_reply.started":"2024-07-04T18:18:27.877878Z","shell.execute_reply":"2024-07-04T18:18:27.888901Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_df.question_text.values[0]","metadata":{"executionInfo":{"elapsed":652,"status":"ok","timestamp":1667915621850,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"ARTJ68oehHVR","outputId":"83f27c51-7c51-4808-9450-489e5599776b","execution":{"iopub.status.busy":"2024-07-04T18:18:27.894082Z","iopub.execute_input":"2024-07-04T18:18:27.894738Z","iopub.status.idle":"2024-07-04T18:18:27.910979Z","shell.execute_reply.started":"2024-07-04T18:18:27.894681Z","shell.execute_reply":"2024-07-04T18:18:27.908789Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df","metadata":{"executionInfo":{"elapsed":1081,"status":"ok","timestamp":1667915683093,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"fJ497jJyhCaK","outputId":"cdb77624-843c-4fe1-ff52-5f93794be092","execution":{"iopub.status.busy":"2024-07-04T18:18:27.913392Z","iopub.execute_input":"2024-07-04T18:18:27.913952Z","iopub.status.idle":"2024-07-04T18:18:27.929477Z","shell.execute_reply.started":"2024-07-04T18:18:27.913907Z","shell.execute_reply":"2024-07-04T18:18:27.928117Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ntest_inputs = vectorizer.transform(test_df.question_text)","metadata":{"executionInfo":{"elapsed":112943,"status":"ok","timestamp":1667915815920,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"UDu0wUf9g--d","outputId":"3db731b6-8002-435a-bfb5-7500c8921da8","execution":{"iopub.status.busy":"2024-07-04T18:18:27.931652Z","iopub.execute_input":"2024-07-04T18:18:27.932141Z","iopub.status.idle":"2024-07-04T18:21:33.705233Z","shell.execute_reply.started":"2024-07-04T18:18:27.932100Z","shell.execute_reply":"2024-07-04T18:21:33.703816Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"id":"CKLhsg9GdkDW"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## ML Models for Text Classification\n\nOutline:\n\n- Create a training & validation set\n- Train a logistic regression model\n- Make predictions on training, validation & test data","metadata":{"id":"AyVwmOboQoJn"}},{"cell_type":"markdown","source":"### Split into Training and Validation Set","metadata":{"id":"1CPsktK-iPgK"}},{"cell_type":"code","source":"sample_df","metadata":{"executionInfo":{"elapsed":3,"status":"ok","timestamp":1667915931527,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"hoIv721biUVy","outputId":"73a07b8c-ec27-495e-8ccd-9bb4c53e40da","execution":{"iopub.status.busy":"2024-07-04T18:21:33.710668Z","iopub.execute_input":"2024-07-04T18:21:33.711061Z","iopub.status.idle":"2024-07-04T18:21:33.725421Z","shell.execute_reply.started":"2024-07-04T18:21:33.711029Z","shell.execute_reply":"2024-07-04T18:21:33.724289Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"inputs.shape","metadata":{"executionInfo":{"elapsed":613,"status":"ok","timestamp":1667915949798,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"ckPyKMcMRBET","outputId":"0bb74251-5275-42ed-b60a-eb763113e3c0","execution":{"iopub.status.busy":"2024-07-04T18:21:33.727520Z","iopub.execute_input":"2024-07-04T18:21:33.728116Z","iopub.status.idle":"2024-07-04T18:21:33.742353Z","shell.execute_reply.started":"2024-07-04T18:21:33.728081Z","shell.execute_reply":"2024-07-04T18:21:33.740954Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split","metadata":{"executionInfo":{"elapsed":348,"status":"ok","timestamp":1667916033391,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"OB2jqu7dirOn","execution":{"iopub.status.busy":"2024-07-04T18:21:33.743947Z","iopub.execute_input":"2024-07-04T18:21:33.744735Z","iopub.status.idle":"2024-07-04T18:21:33.755011Z","shell.execute_reply.started":"2024-07-04T18:21:33.744669Z","shell.execute_reply":"2024-07-04T18:21:33.753812Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_inputs, val_inputs, train_targets, val_targets = train_test_split(inputs, sample_df.target, \n                                                                        test_size=0.3, random_state=42)","metadata":{"executionInfo":{"elapsed":364,"status":"ok","timestamp":1667916089349,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"m3eE_6y6ieLP","execution":{"iopub.status.busy":"2024-07-04T18:21:33.756765Z","iopub.execute_input":"2024-07-04T18:21:33.757165Z","iopub.status.idle":"2024-07-04T18:21:33.784609Z","shell.execute_reply.started":"2024-07-04T18:21:33.757132Z","shell.execute_reply":"2024-07-04T18:21:33.783408Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_inputs.shape","metadata":{"executionInfo":{"elapsed":908,"status":"ok","timestamp":1667916115211,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"1hMoWlcrRBBt","outputId":"c604cdcc-ed79-4c04-dba0-8d46713cab38","execution":{"iopub.status.busy":"2024-07-04T18:21:33.786066Z","iopub.execute_input":"2024-07-04T18:21:33.786428Z","iopub.status.idle":"2024-07-04T18:21:33.794081Z","shell.execute_reply.started":"2024-07-04T18:21:33.786397Z","shell.execute_reply":"2024-07-04T18:21:33.792615Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_targets.shape","metadata":{"executionInfo":{"elapsed":986,"status":"ok","timestamp":1667916129267,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"ioOd41xKjCkX","outputId":"f3c280ca-aa5c-4f49-d551-3ffebe05c89a","execution":{"iopub.status.busy":"2024-07-04T18:21:33.795526Z","iopub.execute_input":"2024-07-04T18:21:33.795875Z","iopub.status.idle":"2024-07-04T18:21:33.809986Z","shell.execute_reply.started":"2024-07-04T18:21:33.795845Z","shell.execute_reply":"2024-07-04T18:21:33.807834Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"val_inputs.shape","metadata":{"executionInfo":{"elapsed":493,"status":"ok","timestamp":1667916138104,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"d2ipJkxRjChw","outputId":"798265ae-8305-41b5-f96c-8611c45aef4f","execution":{"iopub.status.busy":"2024-07-04T18:21:33.812005Z","iopub.execute_input":"2024-07-04T18:21:33.812462Z","iopub.status.idle":"2024-07-04T18:21:33.824473Z","shell.execute_reply.started":"2024-07-04T18:21:33.812421Z","shell.execute_reply":"2024-07-04T18:21:33.822621Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"val_targets.shape","metadata":{"executionInfo":{"elapsed":1040,"status":"ok","timestamp":1667916148487,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"TZuuLryBjIWZ","outputId":"caa99599-0889-4588-ec20-3ad21881b1d6","execution":{"iopub.status.busy":"2024-07-04T18:21:33.827423Z","iopub.execute_input":"2024-07-04T18:21:33.828790Z","iopub.status.idle":"2024-07-04T18:21:33.838068Z","shell.execute_reply.started":"2024-07-04T18:21:33.828751Z","shell.execute_reply":"2024-07-04T18:21:33.836855Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Train Logistic Regression model","metadata":{"id":"bBAcDSenjMSk"}},{"cell_type":"code","source":"from sklearn.linear_model import LogisticRegression","metadata":{"executionInfo":{"elapsed":535,"status":"ok","timestamp":1667916194680,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"tURg-O4pjMmX","execution":{"iopub.status.busy":"2024-07-04T18:21:33.839503Z","iopub.execute_input":"2024-07-04T18:21:33.839856Z","iopub.status.idle":"2024-07-04T18:21:33.848658Z","shell.execute_reply.started":"2024-07-04T18:21:33.839826Z","shell.execute_reply":"2024-07-04T18:21:33.847416Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"MAX_ITER = 1000","metadata":{"executionInfo":{"elapsed":373,"status":"ok","timestamp":1667918920336,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"aqk9RFIYj9tu","execution":{"iopub.status.busy":"2024-07-04T18:21:33.850586Z","iopub.execute_input":"2024-07-04T18:21:33.851049Z","iopub.status.idle":"2024-07-04T18:21:33.862417Z","shell.execute_reply.started":"2024-07-04T18:21:33.851007Z","shell.execute_reply":"2024-07-04T18:21:33.861194Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = LogisticRegression(max_iter=MAX_ITER, solver='sag')","metadata":{"executionInfo":{"elapsed":626,"status":"ok","timestamp":1667916401162,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"uUQ0wzNIjNHr","execution":{"iopub.status.busy":"2024-07-04T18:21:33.864174Z","iopub.execute_input":"2024-07-04T18:21:33.865034Z","iopub.status.idle":"2024-07-04T18:21:33.876361Z","shell.execute_reply.started":"2024-07-04T18:21:33.864959Z","shell.execute_reply":"2024-07-04T18:21:33.874729Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nmodel.fit(train_inputs, train_targets)","metadata":{"executionInfo":{"elapsed":27440,"status":"ok","timestamp":1667916434501,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"fOehu5rLjNFl","outputId":"319e0d60-e7db-4851-d7a5-1bcefd58f239","execution":{"iopub.status.busy":"2024-07-04T18:21:33.878477Z","iopub.execute_input":"2024-07-04T18:21:33.879069Z","iopub.status.idle":"2024-07-04T18:22:02.132281Z","shell.execute_reply.started":"2024-07-04T18:21:33.879030Z","shell.execute_reply":"2024-07-04T18:22:02.131299Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Make predictions using the model","metadata":{"id":"o16-8HmXmQT0"}},{"cell_type":"code","source":"train_preds = model.predict(train_inputs)","metadata":{"executionInfo":{"elapsed":1020,"status":"ok","timestamp":1667916460185,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"i12DBlkVkThU","execution":{"iopub.status.busy":"2024-07-04T18:22:02.133965Z","iopub.execute_input":"2024-07-04T18:22:02.134309Z","iopub.status.idle":"2024-07-04T18:22:02.143379Z","shell.execute_reply.started":"2024-07-04T18:22:02.134279Z","shell.execute_reply":"2024-07-04T18:22:02.142215Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_targets","metadata":{"executionInfo":{"elapsed":363,"status":"ok","timestamp":1667916573761,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"urNCyppPkxZB","outputId":"71349a02-b3a6-4b44-d87f-727dde95605e","execution":{"iopub.status.busy":"2024-07-04T18:22:02.144820Z","iopub.execute_input":"2024-07-04T18:22:02.145175Z","iopub.status.idle":"2024-07-04T18:22:02.158059Z","shell.execute_reply.started":"2024-07-04T18:22:02.145142Z","shell.execute_reply":"2024-07-04T18:22:02.156735Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_preds","metadata":{"executionInfo":{"elapsed":861,"status":"ok","timestamp":1667916464475,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"qS8PDYxljNDP","outputId":"9216c5a9-c9aa-4d6a-d540-e804d4d3c925","execution":{"iopub.status.busy":"2024-07-04T18:22:02.160043Z","iopub.execute_input":"2024-07-04T18:22:02.160508Z","iopub.status.idle":"2024-07-04T18:22:02.167828Z","shell.execute_reply.started":"2024-07-04T18:22:02.160449Z","shell.execute_reply":"2024-07-04T18:22:02.166812Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.Series(train_preds).value_counts()","metadata":{"executionInfo":{"elapsed":567,"status":"ok","timestamp":1667916484628,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"W74eTiJgjNA9","outputId":"4f1aa29b-d959-4561-9384-a83fc36ca13d","execution":{"iopub.status.busy":"2024-07-04T18:22:02.169194Z","iopub.execute_input":"2024-07-04T18:22:02.169620Z","iopub.status.idle":"2024-07-04T18:22:02.181123Z","shell.execute_reply.started":"2024-07-04T18:22:02.169580Z","shell.execute_reply":"2024-07-04T18:22:02.180051Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.Series(train_targets).value_counts()","metadata":{"executionInfo":{"elapsed":1125,"status":"ok","timestamp":1667916512008,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"pixsvLxrkhK4","outputId":"44819bce-587f-41e3-cb6b-55e67b8c7c84","execution":{"iopub.status.busy":"2024-07-04T18:22:02.182744Z","iopub.execute_input":"2024-07-04T18:22:02.183063Z","iopub.status.idle":"2024-07-04T18:22:02.196329Z","shell.execute_reply.started":"2024-07-04T18:22:02.183035Z","shell.execute_reply":"2024-07-04T18:22:02.195206Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import accuracy_score","metadata":{"executionInfo":{"elapsed":370,"status":"ok","timestamp":1667916547769,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"YxH33IpFkpXJ","execution":{"iopub.status.busy":"2024-07-04T18:22:02.197618Z","iopub.execute_input":"2024-07-04T18:22:02.197990Z","iopub.status.idle":"2024-07-04T18:22:02.206287Z","shell.execute_reply.started":"2024-07-04T18:22:02.197959Z","shell.execute_reply":"2024-07-04T18:22:02.205211Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"accuracy_score(train_targets, train_preds)","metadata":{"executionInfo":{"elapsed":893,"status":"ok","timestamp":1667916561938,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"oUgKam9RkYD6","outputId":"dfc12873-4db8-4993-ad67-bbf6456d8b18","execution":{"iopub.status.busy":"2024-07-04T18:22:02.214467Z","iopub.execute_input":"2024-07-04T18:22:02.215138Z","iopub.status.idle":"2024-07-04T18:22:02.229364Z","shell.execute_reply.started":"2024-07-04T18:22:02.215100Z","shell.execute_reply":"2024-07-04T18:22:02.228250Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np","metadata":{"executionInfo":{"elapsed":373,"status":"ok","timestamp":1667916643295,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"ipzg1inClCcJ","execution":{"iopub.status.busy":"2024-07-04T18:22:02.230994Z","iopub.execute_input":"2024-07-04T18:22:02.231582Z","iopub.status.idle":"2024-07-04T18:22:02.238778Z","shell.execute_reply.started":"2024-07-04T18:22:02.231529Z","shell.execute_reply":"2024-07-04T18:22:02.236879Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"accuracy_score(train_targets, np.zeros(len(train_targets)))","metadata":{"executionInfo":{"elapsed":620,"status":"ok","timestamp":1667916647313,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"_P-IsOtTk5x_","outputId":"3aed16ea-c1ab-4c6c-db22-cc083ebe1278","execution":{"iopub.status.busy":"2024-07-04T18:22:02.240461Z","iopub.execute_input":"2024-07-04T18:22:02.240912Z","iopub.status.idle":"2024-07-04T18:22:02.262151Z","shell.execute_reply.started":"2024-07-04T18:22:02.240878Z","shell.execute_reply":"2024-07-04T18:22:02.260920Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import f1_score","metadata":{"executionInfo":{"elapsed":896,"status":"ok","timestamp":1667916682226,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"89TtZhqclKwb","execution":{"iopub.status.busy":"2024-07-04T18:22:02.263471Z","iopub.execute_input":"2024-07-04T18:22:02.263831Z","iopub.status.idle":"2024-07-04T18:22:02.270007Z","shell.execute_reply.started":"2024-07-04T18:22:02.263801Z","shell.execute_reply":"2024-07-04T18:22:02.268652Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"f1_score(train_targets, train_preds)","metadata":{"executionInfo":{"elapsed":476,"status":"ok","timestamp":1667916695952,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"pim_sjENlKn0","outputId":"a919cf7d-3c6b-42ce-c99e-e2da856291ab","execution":{"iopub.status.busy":"2024-07-04T18:22:02.271207Z","iopub.execute_input":"2024-07-04T18:22:02.271649Z","iopub.status.idle":"2024-07-04T18:22:02.309978Z","shell.execute_reply.started":"2024-07-04T18:22:02.271613Z","shell.execute_reply":"2024-07-04T18:22:02.308790Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"f1_score(train_targets, np.zeros(len(train_targets)))","metadata":{"executionInfo":{"elapsed":375,"status":"ok","timestamp":1667916720434,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"RembBU-Fksbk","outputId":"9c815608-6f06-45d9-c352-d52ad659c965","execution":{"iopub.status.busy":"2024-07-04T18:22:02.311418Z","iopub.execute_input":"2024-07-04T18:22:02.311859Z","iopub.status.idle":"2024-07-04T18:22:02.353321Z","shell.execute_reply.started":"2024-07-04T18:22:02.311822Z","shell.execute_reply":"2024-07-04T18:22:02.352176Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"random_preds = np.random.choice((0, 1), len(train_targets))\nf1_score(train_targets, random_preds)","metadata":{"executionInfo":{"elapsed":530,"status":"ok","timestamp":1667916804410,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"43hI8WB8ld0e","outputId":"36a47c48-a715-47a9-9698-92c5eec0cd7f","execution":{"iopub.status.busy":"2024-07-04T18:22:02.354824Z","iopub.execute_input":"2024-07-04T18:22:02.355259Z","iopub.status.idle":"2024-07-04T18:22:02.397248Z","shell.execute_reply.started":"2024-07-04T18:22:02.355218Z","shell.execute_reply":"2024-07-04T18:22:02.395882Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"val_preds = model.predict(val_inputs)","metadata":{"executionInfo":{"elapsed":394,"status":"ok","timestamp":1667916889103,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"FM0X5ImAl7gB","execution":{"iopub.status.busy":"2024-07-04T18:22:02.399447Z","iopub.execute_input":"2024-07-04T18:22:02.400525Z","iopub.status.idle":"2024-07-04T18:22:02.406389Z","shell.execute_reply.started":"2024-07-04T18:22:02.400451Z","shell.execute_reply":"2024-07-04T18:22:02.405191Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"accuracy_score(val_targets, val_preds)","metadata":{"executionInfo":{"elapsed":592,"status":"ok","timestamp":1667916920449,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"WiXz18lal7db","outputId":"1199b1c7-51b2-4c95-d83e-d945e1541bf9","execution":{"iopub.status.busy":"2024-07-04T18:22:02.407997Z","iopub.execute_input":"2024-07-04T18:22:02.408393Z","iopub.status.idle":"2024-07-04T18:22:02.422568Z","shell.execute_reply.started":"2024-07-04T18:22:02.408361Z","shell.execute_reply":"2024-07-04T18:22:02.421257Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"f1_score(val_targets, val_preds)","metadata":{"executionInfo":{"elapsed":902,"status":"ok","timestamp":1667916932147,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"Kj-oIhIHksY-","outputId":"de4648a5-5f9c-48fb-f86e-aad601d0a956","execution":{"iopub.status.busy":"2024-07-04T18:22:02.424374Z","iopub.execute_input":"2024-07-04T18:22:02.424874Z","iopub.status.idle":"2024-07-04T18:22:02.445759Z","shell.execute_reply.started":"2024-07-04T18:22:02.424835Z","shell.execute_reply":"2024-07-04T18:22:02.444600Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sincere_df.question_text.values[:10]","metadata":{"executionInfo":{"elapsed":653,"status":"ok","timestamp":1667917017459,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"qxCSeXZQmHpS","outputId":"7cb895cb-7605-45d2-b650-b61b02cecffe","execution":{"iopub.status.busy":"2024-07-04T18:22:02.447165Z","iopub.execute_input":"2024-07-04T18:22:02.447582Z","iopub.status.idle":"2024-07-04T18:22:02.455122Z","shell.execute_reply.started":"2024-07-04T18:22:02.447551Z","shell.execute_reply":"2024-07-04T18:22:02.453983Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sincere_df.target.values[:10]","metadata":{"executionInfo":{"elapsed":1051,"status":"ok","timestamp":1667917031436,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"fwlbPtLDmgDf","outputId":"a6360495-7f8f-489e-b955-56fbcffa88bd","execution":{"iopub.status.busy":"2024-07-04T18:22:02.456608Z","iopub.execute_input":"2024-07-04T18:22:02.456953Z","iopub.status.idle":"2024-07-04T18:22:02.467921Z","shell.execute_reply.started":"2024-07-04T18:22:02.456924Z","shell.execute_reply":"2024-07-04T18:22:02.466811Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.predict(vectorizer.transform(sincere_df.question_text.values[:10]))","metadata":{"executionInfo":{"elapsed":375,"status":"ok","timestamp":1667917062825,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"qHE1gR0mmV0l","outputId":"c1e5f7f9-e5a3-4ea7-ff65-caa1c9280ce0","execution":{"iopub.status.busy":"2024-07-04T18:22:02.469453Z","iopub.execute_input":"2024-07-04T18:22:02.469904Z","iopub.status.idle":"2024-07-04T18:22:02.488993Z","shell.execute_reply.started":"2024-07-04T18:22:02.469861Z","shell.execute_reply":"2024-07-04T18:22:02.487517Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"insincere_df.question_text.values[:10]","metadata":{"executionInfo":{"elapsed":411,"status":"ok","timestamp":1667917165691,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"k58qFC-kmVyR","outputId":"d6b34cf6-182e-4375-d37e-01c3cfae8844","execution":{"iopub.status.busy":"2024-07-04T18:22:02.490345Z","iopub.execute_input":"2024-07-04T18:22:02.490778Z","iopub.status.idle":"2024-07-04T18:22:02.500782Z","shell.execute_reply.started":"2024-07-04T18:22:02.490748Z","shell.execute_reply":"2024-07-04T18:22:02.499658Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"insincere_df.target.values[:10]","metadata":{"executionInfo":{"elapsed":389,"status":"ok","timestamp":1667917180899,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"lFwis7sOmVv7","outputId":"f5b1eb04-8277-4b61-e0d8-6a08b7840770","execution":{"iopub.status.busy":"2024-07-04T18:22:02.502121Z","iopub.execute_input":"2024-07-04T18:22:02.502510Z","iopub.status.idle":"2024-07-04T18:22:02.512265Z","shell.execute_reply.started":"2024-07-04T18:22:02.502451Z","shell.execute_reply":"2024-07-04T18:22:02.511183Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.predict(vectorizer.transform(insincere_df.question_text.values[:10]))","metadata":{"executionInfo":{"elapsed":572,"status":"ok","timestamp":1667917197399,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"2T_zvDYvmVt6","outputId":"36b18600-75b3-4287-de0d-b963b97602f6","execution":{"iopub.status.busy":"2024-07-04T18:22:02.513726Z","iopub.execute_input":"2024-07-04T18:22:02.514077Z","iopub.status.idle":"2024-07-04T18:22:02.532859Z","shell.execute_reply.started":"2024-07-04T18:22:02.514048Z","shell.execute_reply":"2024-07-04T18:22:02.531588Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Make Predictions and Submit to Kaggle","metadata":{"id":"TSFA3r48Q2ec"}},{"cell_type":"code","source":"test_df","metadata":{"executionInfo":{"elapsed":1103,"status":"ok","timestamp":1667917318912,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"bIAZhmcWQDd4","outputId":"e92eb5e4-41ad-4590-da74-d9b14dc9697a","execution":{"iopub.status.busy":"2024-07-04T18:22:02.534177Z","iopub.execute_input":"2024-07-04T18:22:02.534551Z","iopub.status.idle":"2024-07-04T18:22:02.553229Z","shell.execute_reply.started":"2024-07-04T18:22:02.534513Z","shell.execute_reply":"2024-07-04T18:22:02.551834Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_inputs.shape","metadata":{"executionInfo":{"elapsed":460,"status":"ok","timestamp":1667917332470,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"jzb8x69DnoHz","outputId":"23a5695e-d7b4-49ae-c715-2360925ad007","execution":{"iopub.status.busy":"2024-07-04T18:22:02.554824Z","iopub.execute_input":"2024-07-04T18:22:02.555385Z","iopub.status.idle":"2024-07-04T18:22:02.568197Z","shell.execute_reply.started":"2024-07-04T18:22:02.555343Z","shell.execute_reply":"2024-07-04T18:22:02.567054Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_preds = model.predict(test_inputs)","metadata":{"executionInfo":{"elapsed":1112,"status":"ok","timestamp":1667917351071,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"oiY8Ai-anqRP","execution":{"iopub.status.busy":"2024-07-04T18:22:02.569589Z","iopub.execute_input":"2024-07-04T18:22:02.570067Z","iopub.status.idle":"2024-07-04T18:22:02.595869Z","shell.execute_reply.started":"2024-07-04T18:22:02.570036Z","shell.execute_reply":"2024-07-04T18:22:02.594350Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub_df","metadata":{"executionInfo":{"elapsed":558,"status":"ok","timestamp":1667917386054,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"W8h1xhbZnwBt","outputId":"6698c104-0bd3-4c8a-b12f-6e59feac67a3","execution":{"iopub.status.busy":"2024-07-04T18:22:02.597193Z","iopub.execute_input":"2024-07-04T18:22:02.597540Z","iopub.status.idle":"2024-07-04T18:22:02.611056Z","shell.execute_reply.started":"2024-07-04T18:22:02.597483Z","shell.execute_reply":"2024-07-04T18:22:02.609831Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub_df.prediction = test_preds","metadata":{"executionInfo":{"elapsed":4,"status":"ok","timestamp":1667917429135,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"t6xxbg4yn9pT","execution":{"iopub.status.busy":"2024-07-04T18:22:02.612771Z","iopub.execute_input":"2024-07-04T18:22:02.613235Z","iopub.status.idle":"2024-07-04T18:22:02.622216Z","shell.execute_reply.started":"2024-07-04T18:22:02.613191Z","shell.execute_reply":"2024-07-04T18:22:02.621082Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub_df.prediction.value_counts()","metadata":{"executionInfo":{"elapsed":2,"status":"ok","timestamp":1667917437994,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"P1vWhwOIn4id","outputId":"4822357f-2f37-4a73-c939-ca1d3f981454","execution":{"iopub.status.busy":"2024-07-04T18:22:02.624019Z","iopub.execute_input":"2024-07-04T18:22:02.624440Z","iopub.status.idle":"2024-07-04T18:22:02.639615Z","shell.execute_reply.started":"2024-07-04T18:22:02.624409Z","shell.execute_reply":"2024-07-04T18:22:02.638316Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub_df","metadata":{"executionInfo":{"elapsed":371,"status":"ok","timestamp":1667917442478,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"ZSHxruEXoFWj","outputId":"231bcd4a-0219-46cc-935a-a83b4e501672","execution":{"iopub.status.busy":"2024-07-04T18:22:02.641027Z","iopub.execute_input":"2024-07-04T18:22:02.641415Z","iopub.status.idle":"2024-07-04T18:22:02.657880Z","shell.execute_reply.started":"2024-07-04T18:22:02.641382Z","shell.execute_reply":"2024-07-04T18:22:02.656733Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub_df.to_csv('submission.csv', index=None)","metadata":{"executionInfo":{"elapsed":771,"status":"ok","timestamp":1667917485705,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"r_7JT4-JoGkL","execution":{"iopub.status.busy":"2024-07-04T18:22:02.659281Z","iopub.execute_input":"2024-07-04T18:22:02.659671Z","iopub.status.idle":"2024-07-04T18:22:03.427064Z","shell.execute_reply.started":"2024-07-04T18:22:02.659641Z","shell.execute_reply":"2024-07-04T18:22:03.425598Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!head submission.csv","metadata":{"executionInfo":{"elapsed":1077,"status":"ok","timestamp":1667917506958,"user":{"displayName":"Aakash N S","userId":"03254185060287524023"},"user_tz":-330},"id":"496wLhWjoRA3","outputId":"24c74944-ee72-4168-8342-0c52decb77c4","execution":{"iopub.status.busy":"2024-07-04T18:22:03.428340Z","iopub.execute_input":"2024-07-04T18:22:03.428694Z","iopub.status.idle":"2024-07-04T18:22:04.648952Z","shell.execute_reply.started":"2024-07-04T18:22:03.428664Z","shell.execute_reply":"2024-07-04T18:22:04.647593Z"},"trusted":true},"execution_count":null,"outputs":[]}]}