{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"colab":{"provenance":[]},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":10737,"databundleVersionId":290346,"sourceType":"competition"}],"dockerImageVersionId":30786,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Text Classification with Bag of words\n\n\n\nOutline:\n\n* Download and explore the data\n\n* Apply text preprocessing techniques\n\n* Implement the bag of words model\n\n* Train ML models for text classification\n\n* Make predictions and submit to kaggle","metadata":{"id":"kt9V71Sqh3j4"}},{"cell_type":"markdown","source":"# Download and explore the deta\n\n\n\nOutline:\n\n* Download the dataset form Kaggle to colab\n\n* Explore the data using pandas\n\n* Create a small working sample","metadata":{"id":"NDIuzT_2jiio"}},{"cell_type":"markdown","source":"### Download the dataset form Kaggle to colab\n","metadata":{"id":"WRWhu4kVlFKe"}},{"cell_type":"code","source":"!ls .","metadata":{"id":"Th4WuLNkkTiP","outputId":"61334f28-8d56-4e12-f092-76f4fbb075ec","trusted":true,"execution":{"iopub.status.busy":"2024-11-06T20:18:34.724872Z","iopub.execute_input":"2024-11-06T20:18:34.725318Z","iopub.status.idle":"2024-11-06T20:18:35.968285Z","shell.execute_reply.started":"2024-11-06T20:18:34.725275Z","shell.execute_reply":"2024-11-06T20:18:35.966460Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os","metadata":{"id":"rJ5PJud7txLR","trusted":true,"execution":{"iopub.status.busy":"2024-11-06T20:18:35.971563Z","iopub.execute_input":"2024-11-06T20:18:35.972014Z","iopub.status.idle":"2024-11-06T20:18:35.978856Z","shell.execute_reply.started":"2024-11-06T20:18:35.971969Z","shell.execute_reply":"2024-11-06T20:18:35.977333Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"os.environ['KAGGLE_CONFIG_DIR'] ='.'","metadata":{"id":"tDUA-pzLuCoK","trusted":true,"execution":{"iopub.status.busy":"2024-11-06T20:18:35.981565Z","iopub.execute_input":"2024-11-06T20:18:35.981995Z","iopub.status.idle":"2024-11-06T20:18:35.992832Z","shell.execute_reply.started":"2024-11-06T20:18:35.981954Z","shell.execute_reply":"2024-11-06T20:18:35.991201Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# !kaggle competitions download -c quora-insincere-questions-classification -f train.csv -p data","metadata":{"id":"faKWVv8lkTlW","outputId":"635dca9e-13d0-4ef0-c2b2-ee82f8ecca92","trusted":true,"execution":{"iopub.status.busy":"2024-11-06T20:18:35.996622Z","iopub.execute_input":"2024-11-06T20:18:35.997168Z","iopub.status.idle":"2024-11-06T20:18:36.005274Z","shell.execute_reply.started":"2024-11-06T20:18:35.997105Z","shell.execute_reply":"2024-11-06T20:18:36.003934Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# !kaggle competitions download -c quora-insincere-questions-classification -f test.csv -p data","metadata":{"id":"dR-kJ6vw1zSA","outputId":"6870ed74-0585-44b3-e86c-319cfbf8cdbc","trusted":true,"execution":{"iopub.status.busy":"2024-11-06T20:18:36.006957Z","iopub.execute_input":"2024-11-06T20:18:36.007365Z","iopub.status.idle":"2024-11-06T20:18:36.026141Z","shell.execute_reply.started":"2024-11-06T20:18:36.007321Z","shell.execute_reply":"2024-11-06T20:18:36.024835Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# !kaggle competitions download -c quora-insincere-questions-classification -f train.csv -p data\n# !kaggle competitions download -c quora-insincere-questions-classification -f test.csv -p data\n# !kaggle competitions download -c quora-insincere-questions-classification -f sample_submission.csv -p data","metadata":{"id":"xNqjesIZ1zU_","outputId":"4b4db45d-7b69-4ed2-85b7-409b6d05b14c","trusted":true,"execution":{"iopub.status.busy":"2024-11-06T20:18:36.027753Z","iopub.execute_input":"2024-11-06T20:18:36.028170Z","iopub.status.idle":"2024-11-06T20:18:36.040911Z","shell.execute_reply.started":"2024-11-06T20:18:36.028128Z","shell.execute_reply":"2024-11-06T20:18:36.039557Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Explore the data using pandas","metadata":{"id":"Yu4pWtaM5Qej"}},{"cell_type":"code","source":"# train_fname = '/content/data/train.csv.zip'\n\n# test_fname = '/content/data/test.csv.zip'\n\n# sample_fname = '/content/data/sample_submission.csv.zip'","metadata":{"id":"F9t0OZYQ5YYa","trusted":true,"execution":{"iopub.status.busy":"2024-11-06T20:18:36.043167Z","iopub.execute_input":"2024-11-06T20:18:36.043635Z","iopub.status.idle":"2024-11-06T20:18:36.053686Z","shell.execute_reply.started":"2024-11-06T20:18:36.043564Z","shell.execute_reply":"2024-11-06T20:18:36.052309Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-06T20:18:36.055364Z","iopub.execute_input":"2024-11-06T20:18:36.055807Z","iopub.status.idle":"2024-11-06T20:18:36.068161Z","shell.execute_reply.started":"2024-11-06T20:18:36.055766Z","shell.execute_reply":"2024-11-06T20:18:36.066511Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"IS_KAGGLE = 'KAGGLE_KERNEL_RUN_TYPE' in os.environ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-06T20:18:36.070090Z","iopub.execute_input":"2024-11-06T20:18:36.070499Z","iopub.status.idle":"2024-11-06T20:18:36.079941Z","shell.execute_reply.started":"2024-11-06T20:18:36.070459Z","shell.execute_reply":"2024-11-06T20:18:36.078539Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"if IS_KAGGLE:\n   data_dir = '/kaggle/input/quora-insincere-questions-classification'\n   train_fname = data_dir + '/train.csv'\n   test_fname = data_dir + 'test.csv'\n   sample_fname = data_dir + 'sample_submission.csv'\nelse:\n  os.environ['KAGGLE_CONFIG_DIR'] ='.'\n  !kaggle competitions download -c quora-insincere-questions-classification -f train.csv -p data\n  !kaggle competitions download -c quora-insincere-questions-classification -f test.csv -p data\n  !kaggle competitions download -c quora-insincere-questions-classification -f sample_submission.csv -p data\n  train_fname = '/content/data/train.csv.zip'\n  test_fname = '/content/data/test.csv.zip'\n  sample_fname = '/content/data/sample_submission.csv.zip'\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-06T20:18:36.084938Z","iopub.execute_input":"2024-11-06T20:18:36.085368Z","iopub.status.idle":"2024-11-06T20:18:36.100120Z","shell.execute_reply.started":"2024-11-06T20:18:36.085326Z","shell.execute_reply":"2024-11-06T20:18:36.098318Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd","metadata":{"id":"vZI_aE0BxZYn","trusted":true,"execution":{"iopub.status.busy":"2024-11-06T20:18:36.101791Z","iopub.execute_input":"2024-11-06T20:18:36.102305Z","iopub.status.idle":"2024-11-06T20:18:36.113434Z","shell.execute_reply.started":"2024-11-06T20:18:36.102246Z","shell.execute_reply":"2024-11-06T20:18:36.112023Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"raw_df = pd.read_csv(train_fname)","metadata":{"id":"RBe21trb6u09","trusted":true,"execution":{"iopub.status.busy":"2024-11-06T20:18:36.114831Z","iopub.execute_input":"2024-11-06T20:18:36.115323Z","iopub.status.idle":"2024-11-06T20:18:39.640067Z","shell.execute_reply.started":"2024-11-06T20:18:36.115265Z","shell.execute_reply":"2024-11-06T20:18:39.638884Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"raw_df","metadata":{"id":"xcp_9EKG6u87","outputId":"4155aa86-7a4a-4e2d-d2fc-133292b5d82a","trusted":true,"execution":{"iopub.status.busy":"2024-11-06T20:18:39.641568Z","iopub.execute_input":"2024-11-06T20:18:39.641972Z","iopub.status.idle":"2024-11-06T20:18:39.657368Z","shell.execute_reply.started":"2024-11-06T20:18:39.641931Z","shell.execute_reply":"2024-11-06T20:18:39.656206Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sincere_df = raw_df[raw_df.target ==0]","metadata":{"id":"t0aRmCawdUr5","trusted":true,"execution":{"iopub.status.busy":"2024-11-06T20:18:39.659138Z","iopub.execute_input":"2024-11-06T20:18:39.659629Z","iopub.status.idle":"2024-11-06T20:18:39.744332Z","shell.execute_reply.started":"2024-11-06T20:18:39.659541Z","shell.execute_reply":"2024-11-06T20:18:39.742858Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sincere_df.question_text.values[:10]","metadata":{"id":"aSPChVxbdUu6","outputId":"fc766a1b-c084-4104-9be5-45aee224ec96","trusted":true,"execution":{"iopub.status.busy":"2024-11-06T20:18:39.746058Z","iopub.execute_input":"2024-11-06T20:18:39.746455Z","iopub.status.idle":"2024-11-06T20:18:39.754806Z","shell.execute_reply.started":"2024-11-06T20:18:39.746399Z","shell.execute_reply":"2024-11-06T20:18:39.753541Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"insincere_df = raw_df[raw_df.target ==1]","metadata":{"id":"htcjCAUhdUxw","trusted":true,"execution":{"iopub.status.busy":"2024-11-06T20:18:39.756254Z","iopub.execute_input":"2024-11-06T20:18:39.756689Z","iopub.status.idle":"2024-11-06T20:18:39.784563Z","shell.execute_reply.started":"2024-11-06T20:18:39.756645Z","shell.execute_reply":"2024-11-06T20:18:39.783494Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"insincere_df.question_text.values[:10]","metadata":{"id":"LSfae6r2jD04","outputId":"7c7c48cb-d19b-4dc7-eab1-2666c5895b64","trusted":true,"execution":{"iopub.status.busy":"2024-11-06T20:18:39.786113Z","iopub.execute_input":"2024-11-06T20:18:39.786481Z","iopub.status.idle":"2024-11-06T20:18:39.794143Z","shell.execute_reply.started":"2024-11-06T20:18:39.786439Z","shell.execute_reply":"2024-11-06T20:18:39.792924Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"raw_df.target.value_counts()","metadata":{"id":"tToo-srBjD3_","outputId":"d22f3182-bad5-4e74-b1bc-bce71aad2687","trusted":true,"execution":{"iopub.status.busy":"2024-11-06T20:18:39.795433Z","iopub.execute_input":"2024-11-06T20:18:39.796011Z","iopub.status.idle":"2024-11-06T20:18:39.822168Z","shell.execute_reply.started":"2024-11-06T20:18:39.795954Z","shell.execute_reply":"2024-11-06T20:18:39.820949Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"raw_df.target.value_counts(normalize=True)","metadata":{"id":"i-IYLhI2kfRY","outputId":"45dda282-14ed-46ff-a4e0-7eb2390c9aac","trusted":true,"execution":{"iopub.status.busy":"2024-11-06T20:18:39.824053Z","iopub.execute_input":"2024-11-06T20:18:39.824534Z","iopub.status.idle":"2024-11-06T20:18:39.850866Z","shell.execute_reply.started":"2024-11-06T20:18:39.824478Z","shell.execute_reply":"2024-11-06T20:18:39.849521Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import matplotlib.pyplot as plt\n\nraw_df.target.value_counts(normalize=True).plot(kind='bar')","metadata":{"id":"_tLqQJP4lI-G","outputId":"e976d9e6-1643-4220-eb40-acda7795c4d9","trusted":true,"execution":{"iopub.status.busy":"2024-11-06T20:18:39.852421Z","iopub.execute_input":"2024-11-06T20:18:39.852865Z","iopub.status.idle":"2024-11-06T20:18:40.108527Z","shell.execute_reply.started":"2024-11-06T20:18:39.852821Z","shell.execute_reply":"2024-11-06T20:18:40.107327Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":" test_df = pd.read_csv('/kaggle/input/quora-insincere-questions-classification/test.csv')","metadata":{"id":"PlvXab0emtyr","trusted":true,"execution":{"iopub.status.busy":"2024-11-06T20:18:40.110143Z","iopub.execute_input":"2024-11-06T20:18:40.110536Z","iopub.status.idle":"2024-11-06T20:18:41.151845Z","shell.execute_reply.started":"2024-11-06T20:18:40.110494Z","shell.execute_reply":"2024-11-06T20:18:41.150774Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_df","metadata":{"id":"Rm6zwo1xmt1o","outputId":"4a84f6f9-05b2-467b-ebed-484f161d4627","trusted":true,"execution":{"iopub.status.busy":"2024-11-06T20:18:41.153625Z","iopub.execute_input":"2024-11-06T20:18:41.154113Z","iopub.status.idle":"2024-11-06T20:18:41.169010Z","shell.execute_reply.started":"2024-11-06T20:18:41.154057Z","shell.execute_reply":"2024-11-06T20:18:41.167610Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"insincere_df = raw_df[raw_df.target==1]","metadata":{"id":"A-eqZn73JiTl","trusted":true,"execution":{"iopub.status.busy":"2024-11-06T20:18:41.170752Z","iopub.execute_input":"2024-11-06T20:18:41.171780Z","iopub.status.idle":"2024-11-06T20:18:41.198297Z","shell.execute_reply.started":"2024-11-06T20:18:41.171710Z","shell.execute_reply":"2024-11-06T20:18:41.197115Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"insincere_df.question_text.values[:10]","metadata":{"id":"iteH_kH8KDT-","outputId":"95479033-edb7-4494-81f0-8167f26fd953","trusted":true,"execution":{"iopub.status.busy":"2024-11-06T20:18:41.199751Z","iopub.execute_input":"2024-11-06T20:18:41.200142Z","iopub.status.idle":"2024-11-06T20:18:41.209673Z","shell.execute_reply.started":"2024-11-06T20:18:41.200100Z","shell.execute_reply":"2024-11-06T20:18:41.207750Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"raw_df.target.value_counts()","metadata":{"id":"ITV1aKDhJidd","outputId":"cb0811ba-f198-40af-f840-428310edbba7","trusted":true,"execution":{"iopub.status.busy":"2024-11-06T20:18:41.211247Z","iopub.execute_input":"2024-11-06T20:18:41.211678Z","iopub.status.idle":"2024-11-06T20:18:41.241574Z","shell.execute_reply.started":"2024-11-06T20:18:41.211635Z","shell.execute_reply":"2024-11-06T20:18:41.240277Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"raw_df.target.value_counts(normalize=True)","metadata":{"id":"R3bLo6yZK6Y4","outputId":"d982fe96-38ae-4f93-ee6a-e78121eca494","trusted":true,"execution":{"iopub.status.busy":"2024-11-06T20:18:41.243627Z","iopub.execute_input":"2024-11-06T20:18:41.244346Z","iopub.status.idle":"2024-11-06T20:18:41.274666Z","shell.execute_reply.started":"2024-11-06T20:18:41.244252Z","shell.execute_reply":"2024-11-06T20:18:41.273285Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"raw_df.target.value_counts(normalize=True).plot(kind='bar')","metadata":{"id":"8iRKoMm8LTXl","outputId":"4e035734-dc51-4c18-b99c-10c5f016700b","trusted":true,"execution":{"iopub.status.busy":"2024-11-06T20:18:41.276955Z","iopub.execute_input":"2024-11-06T20:18:41.277465Z","iopub.status.idle":"2024-11-06T20:18:41.482287Z","shell.execute_reply.started":"2024-11-06T20:18:41.277390Z","shell.execute_reply":"2024-11-06T20:18:41.480974Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_df = pd.read_csv('/kaggle/input/quora-insincere-questions-classification/test.csv')","metadata":{"id":"28OpwbanMita","trusted":true,"execution":{"iopub.status.busy":"2024-11-06T20:18:41.492290Z","iopub.execute_input":"2024-11-06T20:18:41.493902Z","iopub.status.idle":"2024-11-06T20:18:42.453137Z","shell.execute_reply.started":"2024-11-06T20:18:41.493843Z","shell.execute_reply":"2024-11-06T20:18:42.451833Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_df","metadata":{"id":"idqSpV1hMiwZ","outputId":"7e35c7bc-c0b4-4e0e-d77c-553dd817e7c7","trusted":true,"execution":{"iopub.status.busy":"2024-11-06T20:18:42.454612Z","iopub.execute_input":"2024-11-06T20:18:42.455023Z","iopub.status.idle":"2024-11-06T20:18:42.469170Z","shell.execute_reply.started":"2024-11-06T20:18:42.454981Z","shell.execute_reply":"2024-11-06T20:18:42.467875Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sub_df= pd.read_csv('/kaggle/input/quora-insincere-questions-classification/sample_submission.csv')\n\nsub_df","metadata":{"id":"SATFr3pwMiz-","outputId":"7c70a49b-b205-4a05-a116-cbab1a8761ce","trusted":true,"execution":{"iopub.status.busy":"2024-11-06T20:18:42.471395Z","iopub.execute_input":"2024-11-06T20:18:42.472094Z","iopub.status.idle":"2024-11-06T20:18:42.814651Z","shell.execute_reply.started":"2024-11-06T20:18:42.472033Z","shell.execute_reply":"2024-11-06T20:18:42.812799Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sub_df.prediction.value_counts(normalize=False)","metadata":{"id":"xGMfuphsMi24","outputId":"6999439c-40ee-4983-f1f6-d2bb7e0deadb","trusted":true,"execution":{"iopub.status.busy":"2024-11-06T20:18:42.816342Z","iopub.execute_input":"2024-11-06T20:18:42.816875Z","iopub.status.idle":"2024-11-06T20:18:42.831954Z","shell.execute_reply.started":"2024-11-06T20:18:42.816812Z","shell.execute_reply":"2024-11-06T20:18:42.830760Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Create a small working sample","metadata":{"id":"1yGsezuGOw9o"}},{"cell_type":"code","source":"SAMPLE_SIZE = 100_000\n\nSAMPLE_SIZE","metadata":{"id":"cAPms68VO5wx","outputId":"eaaedf1f-b4e3-4bf1-e174-17df1cb734dd","trusted":true,"execution":{"iopub.status.busy":"2024-11-06T20:18:42.833288Z","iopub.execute_input":"2024-11-06T20:18:42.834247Z","iopub.status.idle":"2024-11-06T20:18:42.845933Z","shell.execute_reply.started":"2024-11-06T20:18:42.834191Z","shell.execute_reply":"2024-11-06T20:18:42.844717Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sample_df = raw_df.sample(SAMPLE_SIZE, random_state= 42)","metadata":{"id":"taQqiyI0OCG6","trusted":true,"execution":{"iopub.status.busy":"2024-11-06T20:18:42.847571Z","iopub.execute_input":"2024-11-06T20:18:42.848735Z","iopub.status.idle":"2024-11-06T20:18:42.943080Z","shell.execute_reply.started":"2024-11-06T20:18:42.848676Z","shell.execute_reply":"2024-11-06T20:18:42.941875Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sample_df","metadata":{"id":"aBvoUsJ9OCJ-","outputId":"1e7e34e3-e964-422d-a066-d3fcb872c352","trusted":true,"execution":{"iopub.status.busy":"2024-11-06T20:22:32.230858Z","iopub.execute_input":"2024-11-06T20:22:32.231288Z","iopub.status.idle":"2024-11-06T20:22:32.250040Z","shell.execute_reply.started":"2024-11-06T20:22:32.231247Z","shell.execute_reply":"2024-11-06T20:22:32.248745Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# text preprocessing techniques\n\nOutline:\n\n\n\n1. Understand the bag of words model\n\n2. Toknization\n\n3. Stop word removal\n\n4. Stemming\n\n\n\n### Bag of Words Intuition\n\n\n\n1. Create a list of all the words across all the text documents\n\n2. You convert each document into vector counts of each word\n\n\n\n\n\nLimitations:\n\n1. There may be too many words in the dataset\n\n2. Some words may occur too frequently\n\n3. Some words may occur very rarely or only once\n\n4. A single word may have many forms (go, gone, going or bird vs. birds)","metadata":{"id":"Y3K5hsBOjsxb"}},{"cell_type":"code","source":"q0 = sincere_df.question_text.values[1]","metadata":{"id":"_QcIuc49GT9c","trusted":true,"execution":{"iopub.status.busy":"2024-11-06T20:22:32.206543Z","iopub.execute_input":"2024-11-06T20:22:32.207027Z","iopub.status.idle":"2024-11-06T20:22:32.214337Z","shell.execute_reply.started":"2024-11-06T20:22:32.206981Z","shell.execute_reply":"2024-11-06T20:22:32.213234Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"q0","metadata":{"id":"eyBjyRr9GUAu","outputId":"f73da040-b151-4ef5-f717-ab7cca425c0c","trusted":true,"execution":{"iopub.status.busy":"2024-11-06T20:22:32.216070Z","iopub.execute_input":"2024-11-06T20:22:32.216467Z","iopub.status.idle":"2024-11-06T20:22:32.229182Z","shell.execute_reply.started":"2024-11-06T20:22:32.216417Z","shell.execute_reply":"2024-11-06T20:22:32.227974Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"q1 = raw_df[raw_df.target == 1].question_text.values[0]\n\nq1","metadata":{"id":"HPDs6Ei7GUGB","outputId":"c9f1daeb-888e-47f3-82d7-a6d920251bed","trusted":true,"execution":{"iopub.status.busy":"2024-11-06T20:18:42.990799Z","iopub.execute_input":"2024-11-06T20:18:42.991294Z","iopub.status.idle":"2024-11-06T20:18:43.024245Z","shell.execute_reply.started":"2024-11-06T20:18:42.991246Z","shell.execute_reply":"2024-11-06T20:18:43.022902Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Tokennization\n\nsplitting a document into word and separators","metadata":{"id":"LfX5b-A7S2hy"}},{"cell_type":"code","source":"import nltk","metadata":{"id":"W80tQ72TOcZd","trusted":true,"execution":{"iopub.status.busy":"2024-11-06T20:22:32.266770Z","iopub.execute_input":"2024-11-06T20:22:32.267227Z","iopub.status.idle":"2024-11-06T20:22:32.272644Z","shell.execute_reply.started":"2024-11-06T20:22:32.267182Z","shell.execute_reply":"2024-11-06T20:22:32.271426Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from nltk.tokenize import word_tokenize","metadata":{"id":"-NjIYjUgGUIo","trusted":true,"execution":{"iopub.status.busy":"2024-11-06T20:18:43.803561Z","iopub.execute_input":"2024-11-06T20:18:43.804114Z","iopub.status.idle":"2024-11-06T20:18:43.810229Z","shell.execute_reply.started":"2024-11-06T20:18:43.804054Z","shell.execute_reply":"2024-11-06T20:18:43.808396Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"nltk.download('punkt')","metadata":{"id":"J__VkSJ6OLF6","outputId":"801dc8ae-7aec-40e9-fadc-c69606647652","trusted":true,"execution":{"iopub.status.busy":"2024-11-06T20:18:43.812337Z","iopub.execute_input":"2024-11-06T20:18:43.813408Z","iopub.status.idle":"2024-11-06T20:19:03.873325Z","shell.execute_reply.started":"2024-11-06T20:18:43.813345Z","shell.execute_reply":"2024-11-06T20:19:03.871936Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"q0","metadata":{"id":"m0qSX7TjNr7r","outputId":"3924b77f-f264-43f0-af18-109daf0c249b","trusted":true,"execution":{"iopub.status.busy":"2024-11-06T20:19:03.874995Z","iopub.execute_input":"2024-11-06T20:19:03.875405Z","iopub.status.idle":"2024-11-06T20:19:03.883797Z","shell.execute_reply.started":"2024-11-06T20:19:03.875363Z","shell.execute_reply":"2024-11-06T20:19:03.882513Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"word_tokenize(q0)","metadata":{"id":"BvgLKSTrNr-1","outputId":"f32fea38-50a8-4192-8e59-032d46735d49","trusted":true,"execution":{"iopub.status.busy":"2024-11-06T20:19:03.885879Z","iopub.execute_input":"2024-11-06T20:19:03.886401Z","iopub.status.idle":"2024-11-06T20:19:03.916717Z","shell.execute_reply.started":"2024-11-06T20:19:03.886338Z","shell.execute_reply":"2024-11-06T20:19:03.915480Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"q1","metadata":{"id":"BjWoofMPNsB-","outputId":"8cbe6307-d68a-4a1f-b080-68fe49242c08","trusted":true,"execution":{"iopub.status.busy":"2024-11-06T20:19:03.918233Z","iopub.execute_input":"2024-11-06T20:19:03.918751Z","iopub.status.idle":"2024-11-06T20:19:03.926859Z","shell.execute_reply.started":"2024-11-06T20:19:03.918697Z","shell.execute_reply":"2024-11-06T20:19:03.925685Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"word_tokenize(q1)","metadata":{"id":"WSd28gG8O277","outputId":"892c433d-c6e6-484b-9111-93c492419aa9","trusted":true,"execution":{"iopub.status.busy":"2024-11-06T20:19:03.928743Z","iopub.execute_input":"2024-11-06T20:19:03.929286Z","iopub.status.idle":"2024-11-06T20:19:03.941709Z","shell.execute_reply.started":"2024-11-06T20:19:03.929228Z","shell.execute_reply":"2024-11-06T20:19:03.939460Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"q0_tok = word_tokenize(q0)\n\nq1_tok = word_tokenize(q1)","metadata":{"id":"16-0tDZUO2_A","trusted":true,"execution":{"iopub.status.busy":"2024-11-06T20:19:03.943586Z","iopub.execute_input":"2024-11-06T20:19:03.944108Z","iopub.status.idle":"2024-11-06T20:19:03.957144Z","shell.execute_reply.started":"2024-11-06T20:19:03.944061Z","shell.execute_reply":"2024-11-06T20:19:03.955911Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### stop Word Removl\n\nRomoving commonly occuring words\n","metadata":{"id":"T-ZHqxKkPX6Y"}},{"cell_type":"code","source":"q1_tok","metadata":{"id":"ygw4I9GlO3B3","outputId":"1d9ddfb3-f04c-43d4-99b1-0c869029c14a","trusted":true,"execution":{"iopub.status.busy":"2024-11-06T20:19:03.958714Z","iopub.execute_input":"2024-11-06T20:19:03.959117Z","iopub.status.idle":"2024-11-06T20:19:03.972381Z","shell.execute_reply.started":"2024-11-06T20:19:03.959074Z","shell.execute_reply":"2024-11-06T20:19:03.971077Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from nltk.corpus import stopwords","metadata":{"id":"tGlA97YrO3Fg","trusted":true,"execution":{"iopub.status.busy":"2024-11-06T20:19:03.974088Z","iopub.execute_input":"2024-11-06T20:19:03.974592Z","iopub.status.idle":"2024-11-06T20:19:03.985369Z","shell.execute_reply.started":"2024-11-06T20:19:03.974532Z","shell.execute_reply":"2024-11-06T20:19:03.984157Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"nltk.download('stopwords')","metadata":{"id":"-TF60Y4JRF6f","outputId":"a3e94332-c6fc-4985-8374-d468a379df55","trusted":true,"execution":{"iopub.status.busy":"2024-11-06T20:19:03.986888Z","iopub.execute_input":"2024-11-06T20:19:03.987263Z","iopub.status.idle":"2024-11-06T20:19:24.033960Z","shell.execute_reply.started":"2024-11-06T20:19:03.987222Z","shell.execute_reply":"2024-11-06T20:19:24.032840Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"english_stopwords = stopwords.words('english')","metadata":{"id":"EzxDESupO3I0","trusted":true,"execution":{"iopub.status.busy":"2024-11-06T20:19:24.035265Z","iopub.execute_input":"2024-11-06T20:19:24.035704Z","iopub.status.idle":"2024-11-06T20:19:24.047010Z","shell.execute_reply.started":"2024-11-06T20:19:24.035659Z","shell.execute_reply":"2024-11-06T20:19:24.045528Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\", \".join(english_stopwords)","metadata":{"id":"L-6BxK-WNsFH","outputId":"70631553-6bbf-4969-8a48-d29cb742c8e6","trusted":true,"execution":{"iopub.status.busy":"2024-11-06T20:19:24.048517Z","iopub.execute_input":"2024-11-06T20:19:24.049166Z","iopub.status.idle":"2024-11-06T20:19:24.057915Z","shell.execute_reply.started":"2024-11-06T20:19:24.049120Z","shell.execute_reply":"2024-11-06T20:19:24.056559Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def remove_stopwords(tokens):\n\n  return[word for word in tokens if word.lower() not in english_stopwords]","metadata":{"id":"tXcVdr0uRnwU","trusted":true,"execution":{"iopub.status.busy":"2024-11-06T20:19:24.059481Z","iopub.execute_input":"2024-11-06T20:19:24.060393Z","iopub.status.idle":"2024-11-06T20:19:24.069633Z","shell.execute_reply.started":"2024-11-06T20:19:24.060333Z","shell.execute_reply":"2024-11-06T20:19:24.068326Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"q0_tok","metadata":{"id":"4ThPD1uZSKON","outputId":"1aa5e7da-eb36-4b3d-9c71-faf31f95e9e2","trusted":true,"execution":{"iopub.status.busy":"2024-11-06T20:19:24.071751Z","iopub.execute_input":"2024-11-06T20:19:24.072216Z","iopub.status.idle":"2024-11-06T20:19:24.085315Z","shell.execute_reply.started":"2024-11-06T20:19:24.072150Z","shell.execute_reply":"2024-11-06T20:19:24.083966Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"q0_stp = remove_stopwords(q0_tok)","metadata":{"id":"FWXCqPsdSKTu","trusted":true,"execution":{"iopub.status.busy":"2024-11-06T20:19:24.088017Z","iopub.execute_input":"2024-11-06T20:19:24.088558Z","iopub.status.idle":"2024-11-06T20:19:24.099713Z","shell.execute_reply.started":"2024-11-06T20:19:24.088507Z","shell.execute_reply":"2024-11-06T20:19:24.097735Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"q0_stp","metadata":{"id":"G0QOWNifSKZE","outputId":"ca29b67a-e3ee-4eca-c473-c9360b6e2d41","trusted":true,"execution":{"iopub.status.busy":"2024-11-06T20:19:24.101886Z","iopub.execute_input":"2024-11-06T20:19:24.102361Z","iopub.status.idle":"2024-11-06T20:19:24.114495Z","shell.execute_reply.started":"2024-11-06T20:19:24.102308Z","shell.execute_reply":"2024-11-06T20:19:24.113180Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"q1_stp = remove_stopwords(q1_tok)","metadata":{"id":"OzqOkqimSKiA","trusted":true,"execution":{"iopub.status.busy":"2024-11-06T20:19:24.115878Z","iopub.execute_input":"2024-11-06T20:19:24.116220Z","iopub.status.idle":"2024-11-06T20:19:24.125056Z","shell.execute_reply.started":"2024-11-06T20:19:24.116183Z","shell.execute_reply":"2024-11-06T20:19:24.123857Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"q1_tok","metadata":{"id":"MUGC4GLtJ6N-","outputId":"2f134410-e549-447c-a7f4-9e086084c23c","trusted":true,"execution":{"iopub.status.busy":"2024-11-06T20:19:24.127085Z","iopub.execute_input":"2024-11-06T20:19:24.127540Z","iopub.status.idle":"2024-11-06T20:19:24.143002Z","shell.execute_reply.started":"2024-11-06T20:19:24.127465Z","shell.execute_reply":"2024-11-06T20:19:24.141574Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"q1_stp","metadata":{"id":"HeOfCOyZRn0U","outputId":"333ddb68-6580-44ec-d610-43068587c4fc","trusted":true,"execution":{"iopub.status.busy":"2024-11-06T20:19:24.144536Z","iopub.execute_input":"2024-11-06T20:19:24.145037Z","iopub.status.idle":"2024-11-06T20:19:24.158904Z","shell.execute_reply.started":"2024-11-06T20:19:24.144991Z","shell.execute_reply":"2024-11-06T20:19:24.157584Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### stemming\n\n'go', 'gone', 'going'==>'go', 'brids, 'brid' '=>'brid'","metadata":{"id":"XHOh1mwkN_gI"}},{"cell_type":"code","source":"from nltk.stem.snowball import SnowballStemmer","metadata":{"id":"yfyNuggYRn4E","trusted":true,"execution":{"iopub.status.busy":"2024-11-06T20:19:24.160579Z","iopub.execute_input":"2024-11-06T20:19:24.161031Z","iopub.status.idle":"2024-11-06T20:19:24.170381Z","shell.execute_reply.started":"2024-11-06T20:19:24.160987Z","shell.execute_reply":"2024-11-06T20:19:24.169082Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"stemmer = SnowballStemmer(language='english')","metadata":{"id":"EoIIMUeoRn_E","trusted":true,"execution":{"iopub.status.busy":"2024-11-06T20:19:24.172183Z","iopub.execute_input":"2024-11-06T20:19:24.172873Z","iopub.status.idle":"2024-11-06T20:19:24.183457Z","shell.execute_reply.started":"2024-11-06T20:19:24.172810Z","shell.execute_reply":"2024-11-06T20:19:24.182141Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"stemmer.stem('going')","metadata":{"id":"tUy7Bm_jRoCa","outputId":"3de11981-7cdc-4f4e-dc78-1b562c3c53e4","trusted":true,"execution":{"iopub.status.busy":"2024-11-06T20:19:24.185236Z","iopub.execute_input":"2024-11-06T20:19:24.185715Z","iopub.status.idle":"2024-11-06T20:19:24.202223Z","shell.execute_reply.started":"2024-11-06T20:19:24.185668Z","shell.execute_reply":"2024-11-06T20:19:24.200913Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"stemmer.stem('supposedly')","metadata":{"id":"DWMWDd1nRoHJ","outputId":"53cec8d5-f7c9-4750-bbfc-3aa983bfca33","trusted":true,"execution":{"iopub.status.busy":"2024-11-06T20:19:24.203752Z","iopub.execute_input":"2024-11-06T20:19:24.204488Z","iopub.status.idle":"2024-11-06T20:19:24.217062Z","shell.execute_reply.started":"2024-11-06T20:19:24.204390Z","shell.execute_reply":"2024-11-06T20:19:24.215694Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"q0_stm = [stemmer.stem(word)for word in q0_stp]","metadata":{"id":"Zc9EqSurPObv","trusted":true,"execution":{"iopub.status.busy":"2024-11-06T20:19:24.218843Z","iopub.execute_input":"2024-11-06T20:19:24.219342Z","iopub.status.idle":"2024-11-06T20:19:24.230765Z","shell.execute_reply.started":"2024-11-06T20:19:24.219285Z","shell.execute_reply":"2024-11-06T20:19:24.229227Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"q0_stp","metadata":{"id":"TJ5DssiTPOem","outputId":"40374909-115b-44a4-d2e0-4fa679381f0b","trusted":true,"execution":{"iopub.status.busy":"2024-11-06T20:19:24.232408Z","iopub.execute_input":"2024-11-06T20:19:24.232921Z","iopub.status.idle":"2024-11-06T20:19:24.249126Z","shell.execute_reply.started":"2024-11-06T20:19:24.232864Z","shell.execute_reply":"2024-11-06T20:19:24.247971Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"q0_stm","metadata":{"id":"TFkZlXp4QE88","outputId":"a6c80e28-7f41-42fa-80ac-8f4d2ea864bb","trusted":true,"execution":{"iopub.status.busy":"2024-11-06T20:19:24.250942Z","iopub.execute_input":"2024-11-06T20:19:24.251305Z","iopub.status.idle":"2024-11-06T20:19:24.264123Z","shell.execute_reply.started":"2024-11-06T20:19:24.251267Z","shell.execute_reply":"2024-11-06T20:19:24.262913Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"q1_stm = [stemmer.stem(word) for word in q1_stp]","metadata":{"id":"KYxPcZSpQXqs","trusted":true,"execution":{"iopub.status.busy":"2024-11-06T20:19:24.265526Z","iopub.execute_input":"2024-11-06T20:19:24.266070Z","iopub.status.idle":"2024-11-06T20:19:24.279498Z","shell.execute_reply.started":"2024-11-06T20:19:24.266023Z","shell.execute_reply":"2024-11-06T20:19:24.277951Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"q1_stp","metadata":{"id":"lh91_XkNQXt1","outputId":"8f42c7b8-9ec2-4f4a-b499-a7dbb5df9084","trusted":true,"execution":{"iopub.status.busy":"2024-11-06T20:19:24.281537Z","iopub.execute_input":"2024-11-06T20:19:24.282214Z","iopub.status.idle":"2024-11-06T20:19:24.298124Z","shell.execute_reply.started":"2024-11-06T20:19:24.282154Z","shell.execute_reply":"2024-11-06T20:19:24.296704Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"q1_stm","metadata":{"id":"ZnLQrRHnQXw-","outputId":"0eb21852-f44a-4823-a5b8-f1c5b19d59f7","trusted":true,"execution":{"iopub.status.busy":"2024-11-06T20:19:24.300352Z","iopub.execute_input":"2024-11-06T20:19:24.302543Z","iopub.status.idle":"2024-11-06T20:19:24.315282Z","shell.execute_reply.started":"2024-11-06T20:19:24.302285Z","shell.execute_reply":"2024-11-06T20:19:24.314042Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Lemmatization\n\n\"love\" ->love\n\n\"loving\"  ->love\n\n\"lovable\"  ->love","metadata":{"id":"ICHmUU61RW4X"}},{"cell_type":"markdown","source":"## Bag  of Words Intuition\n\n1. There may be too many words in the dataset\n\n2. Some words may occur too frequently\n\n3. Some words may occur very rarely or only once\n\n4. A single word may have many forms (go, gone, going or bird vs. birds)","metadata":{"id":"JSbYfjicDeKR"}},{"cell_type":"markdown","source":"## Implement Bag of Words\n\n\n\n\n\nOutline:\n\n\n\n1. Create a vocabulary using Count Vectorizer\n\n2. Transform text to vectors using Count Vectorizer\n\n3. Configure text preprocessing in Count Vectorizer","metadata":{"id":"qW5Thqi8j4Eu"}},{"cell_type":"markdown","source":"### Create a vocabulary","metadata":{"id":"KpxLOs8qWkmp"}},{"cell_type":"code","source":"small_df = sample_df[:5]","metadata":{"id":"-P-Fje2WVmJq","trusted":true,"execution":{"iopub.status.busy":"2024-11-06T20:19:24.317508Z","iopub.execute_input":"2024-11-06T20:19:24.317960Z","iopub.status.idle":"2024-11-06T20:19:24.329542Z","shell.execute_reply.started":"2024-11-06T20:19:24.317917Z","shell.execute_reply":"2024-11-06T20:19:24.327929Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"small_df","metadata":{"id":"yL-UcG3ZVmMY","outputId":"83e552a2-3ba8-481d-bd75-429c0f007829","trusted":true,"execution":{"iopub.status.busy":"2024-11-06T20:19:24.331652Z","iopub.execute_input":"2024-11-06T20:19:24.332768Z","iopub.status.idle":"2024-11-06T20:19:24.354510Z","shell.execute_reply.started":"2024-11-06T20:19:24.332693Z","shell.execute_reply":"2024-11-06T20:19:24.353188Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"small_df.question_text.values","metadata":{"id":"l4aplzYmWE9Z","outputId":"90b06015-6956-4544-8050-bb47ed9990f4","trusted":true,"execution":{"iopub.status.busy":"2024-11-06T20:19:24.356194Z","iopub.execute_input":"2024-11-06T20:19:24.356592Z","iopub.status.idle":"2024-11-06T20:19:24.370395Z","shell.execute_reply.started":"2024-11-06T20:19:24.356548Z","shell.execute_reply":"2024-11-06T20:19:24.369065Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.feature_extraction.text import CountVectorizer","metadata":{"id":"0nkXU5BxVmPa","trusted":true,"execution":{"iopub.status.busy":"2024-11-06T20:19:24.372686Z","iopub.execute_input":"2024-11-06T20:19:24.373214Z","iopub.status.idle":"2024-11-06T20:19:24.389187Z","shell.execute_reply.started":"2024-11-06T20:19:24.373154Z","shell.execute_reply":"2024-11-06T20:19:24.387487Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"small_vect = CountVectorizer()","metadata":{"id":"naL3a2ZqVmSD","trusted":true,"execution":{"iopub.status.busy":"2024-11-06T20:19:24.391152Z","iopub.execute_input":"2024-11-06T20:19:24.391585Z","iopub.status.idle":"2024-11-06T20:19:24.404225Z","shell.execute_reply.started":"2024-11-06T20:19:24.391536Z","shell.execute_reply":"2024-11-06T20:19:24.402898Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"small_vect.fit(small_df.question_text)","metadata":{"id":"p1y6iZ5OYgaQ","outputId":"657f2ff0-a622-4d5d-859b-0363b7f22ebc","trusted":true,"execution":{"iopub.status.busy":"2024-11-06T20:19:24.405537Z","iopub.execute_input":"2024-11-06T20:19:24.405990Z","iopub.status.idle":"2024-11-06T20:19:24.433235Z","shell.execute_reply.started":"2024-11-06T20:19:24.405931Z","shell.execute_reply":"2024-11-06T20:19:24.431696Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"small_vect.get_feature_names_out()","metadata":{"id":"tHF3qjV6YgdB","outputId":"9a8c1011-4de7-43d3-dc01-164bce1cb2cd","trusted":true,"execution":{"iopub.status.busy":"2024-11-06T20:19:24.435022Z","iopub.execute_input":"2024-11-06T20:19:24.435539Z","iopub.status.idle":"2024-11-06T20:19:24.450266Z","shell.execute_reply.started":"2024-11-06T20:19:24.435482Z","shell.execute_reply":"2024-11-06T20:19:24.448832Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Transform documents into Vectors","metadata":{"id":"h4zhNIRlPGuO"}},{"cell_type":"code","source":"small_df.question_text.values","metadata":{"id":"mnTpRbkXKD7x","outputId":"a7b16ec1-fcba-4489-cec9-e8cb4a7bfe4a","trusted":true,"execution":{"iopub.status.busy":"2024-11-06T20:19:24.451986Z","iopub.execute_input":"2024-11-06T20:19:24.452412Z","iopub.status.idle":"2024-11-06T20:19:24.467395Z","shell.execute_reply.started":"2024-11-06T20:19:24.452360Z","shell.execute_reply":"2024-11-06T20:19:24.466112Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"vectors = small_vect.transform(small_df.question_text)","metadata":{"id":"wGaDulUBYghF","trusted":true,"execution":{"iopub.status.busy":"2024-11-06T20:19:24.469037Z","iopub.execute_input":"2024-11-06T20:19:24.469525Z","iopub.status.idle":"2024-11-06T20:19:24.480126Z","shell.execute_reply.started":"2024-11-06T20:19:24.469468Z","shell.execute_reply":"2024-11-06T20:19:24.478552Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"vectors","metadata":{"id":"c51_Qm6udins","outputId":"d5a801e7-f722-42af-bdbb-df3cb2e0ebc9","trusted":true,"execution":{"iopub.status.busy":"2024-11-06T20:19:24.482045Z","iopub.execute_input":"2024-11-06T20:19:24.482796Z","iopub.status.idle":"2024-11-06T20:19:24.505047Z","shell.execute_reply.started":"2024-11-06T20:19:24.482734Z","shell.execute_reply":"2024-11-06T20:19:24.503706Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"vectors.shape","metadata":{"id":"8oEaPJqZaPAn","outputId":"fd39d7c7-f273-4a39-bd65-b3e90523846f","trusted":true,"execution":{"iopub.status.busy":"2024-11-06T20:19:24.506448Z","iopub.execute_input":"2024-11-06T20:19:24.506892Z","iopub.status.idle":"2024-11-06T20:19:24.519206Z","shell.execute_reply.started":"2024-11-06T20:19:24.506847Z","shell.execute_reply":"2024-11-06T20:19:24.517916Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"small_df.question_text.values[0]","metadata":{"id":"opCJ1TFeR8yv","outputId":"04ae99e1-db04-495b-cf97-df00a471962d","trusted":true,"execution":{"iopub.status.busy":"2024-11-06T20:19:24.520975Z","iopub.execute_input":"2024-11-06T20:19:24.521514Z","iopub.status.idle":"2024-11-06T20:19:24.537364Z","shell.execute_reply.started":"2024-11-06T20:19:24.521445Z","shell.execute_reply":"2024-11-06T20:19:24.536107Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"vectors.toarray()","metadata":{"id":"eKiUudi0aPD-","outputId":"fce40b0d-2279-4fa1-c112-4d842c32e688","trusted":true,"execution":{"iopub.status.busy":"2024-11-06T20:19:24.539927Z","iopub.execute_input":"2024-11-06T20:19:24.540379Z","iopub.status.idle":"2024-11-06T20:19:24.558231Z","shell.execute_reply.started":"2024-11-06T20:19:24.540308Z","shell.execute_reply":"2024-11-06T20:19:24.556689Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### configure Count Vectorizer Parameters","metadata":{"id":"egA4dZ2VMFJ7"}},{"cell_type":"code","source":"stemmer = SnowballStemmer(language = 'english')","metadata":{"id":"DmEwFvY9QTLc","trusted":true,"execution":{"iopub.status.busy":"2024-11-06T20:19:24.560187Z","iopub.execute_input":"2024-11-06T20:19:24.560729Z","iopub.status.idle":"2024-11-06T20:19:24.570464Z","shell.execute_reply.started":"2024-11-06T20:19:24.560669Z","shell.execute_reply":"2024-11-06T20:19:24.569149Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def tokenize(text):\n\n  return[word for word in word_tokenize(text) if word.lower() not in english_stopwords]\n","metadata":{"id":"YGPDnbm1NBVs","trusted":true,"execution":{"iopub.status.busy":"2024-11-06T20:19:24.591349Z","iopub.execute_input":"2024-11-06T20:19:24.591800Z","iopub.status.idle":"2024-11-06T20:19:24.598002Z","shell.execute_reply.started":"2024-11-06T20:19:24.591759Z","shell.execute_reply":"2024-11-06T20:19:24.596663Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"tokenize ('what is the real (deal)here ?')","metadata":{"id":"417prjJqODz6","outputId":"495ac102-a3e0-4e9a-9ea6-c35ffefd4a29","trusted":true,"execution":{"iopub.status.busy":"2024-11-06T20:19:24.599708Z","iopub.execute_input":"2024-11-06T20:19:24.600198Z","iopub.status.idle":"2024-11-06T20:19:24.613838Z","shell.execute_reply.started":"2024-11-06T20:19:24.600143Z","shell.execute_reply":"2024-11-06T20:19:24.612417Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"vectorizer = CountVectorizer(lowercase=True,\n\n                             tokenizer=tokenize,\n\n                             stop_words=english_stopwords,\n\n                             max_features = 1000)","metadata":{"id":"8NFG_6MJMVEe","trusted":true,"execution":{"iopub.status.busy":"2024-11-06T20:19:24.615325Z","iopub.execute_input":"2024-11-06T20:19:24.615765Z","iopub.status.idle":"2024-11-06T20:19:24.625007Z","shell.execute_reply.started":"2024-11-06T20:19:24.615701Z","shell.execute_reply":"2024-11-06T20:19:24.623726Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"%%time\n\nvectorizer.fit(sample_df.question_text)","metadata":{"id":"8BEj9dFhcIsh","outputId":"a2f1067d-f6f6-43c0-a656-dcc12bf3e3a7","trusted":true,"execution":{"iopub.status.busy":"2024-11-06T20:19:24.626515Z","iopub.execute_input":"2024-11-06T20:19:24.626998Z","iopub.status.idle":"2024-11-06T20:19:53.566565Z","shell.execute_reply.started":"2024-11-06T20:19:24.626948Z","shell.execute_reply":"2024-11-06T20:19:53.565303Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"len(vectorizer.vocabulary_)","metadata":{"id":"m18S_T6ycIwC","outputId":"98a8fe7e-3d6d-4665-d6dd-3aecda98529b","trusted":true,"execution":{"iopub.status.busy":"2024-11-06T20:19:53.568117Z","iopub.execute_input":"2024-11-06T20:19:53.568511Z","iopub.status.idle":"2024-11-06T20:19:53.576246Z","shell.execute_reply.started":"2024-11-06T20:19:53.568468Z","shell.execute_reply":"2024-11-06T20:19:53.575024Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"vectorizer.get_feature_names_out()[:100]","metadata":{"id":"PXZg2Oe2cIzo","outputId":"29454bf2-6676-44cf-9421-7e32e790c4bc","trusted":true,"execution":{"iopub.status.busy":"2024-11-06T20:19:53.577977Z","iopub.execute_input":"2024-11-06T20:19:53.578483Z","iopub.status.idle":"2024-11-06T20:19:53.591166Z","shell.execute_reply.started":"2024-11-06T20:19:53.578427Z","shell.execute_reply":"2024-11-06T20:19:53.589891Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"%%time\n\ninputs = vectorizer.transform(sample_df.question_text)","metadata":{"id":"1H-KgTf1cI43","outputId":"e21a4057-c87d-4555-f44c-5519b0b1a9f9","trusted":true,"execution":{"iopub.status.busy":"2024-11-06T20:19:53.592703Z","iopub.execute_input":"2024-11-06T20:19:53.593147Z","iopub.status.idle":"2024-11-06T20:20:22.135337Z","shell.execute_reply.started":"2024-11-06T20:19:53.593096Z","shell.execute_reply":"2024-11-06T20:20:22.134088Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"inputs.shape","metadata":{"id":"7cZaXNiTcI8h","outputId":"19e50f3f-1dcc-4fbd-d25d-4db54388fad7","trusted":true,"execution":{"iopub.status.busy":"2024-11-06T20:20:22.136828Z","iopub.execute_input":"2024-11-06T20:20:22.137194Z","iopub.status.idle":"2024-11-06T20:20:22.145140Z","shell.execute_reply.started":"2024-11-06T20:20:22.137156Z","shell.execute_reply":"2024-11-06T20:20:22.143876Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"inputs","metadata":{"id":"NiOR_RvScJAI","outputId":"36a526fa-0585-4060-c641-8d6a20675875","trusted":true,"execution":{"iopub.status.busy":"2024-11-06T20:20:22.146565Z","iopub.execute_input":"2024-11-06T20:20:22.147079Z","iopub.status.idle":"2024-11-06T20:20:22.161262Z","shell.execute_reply.started":"2024-11-06T20:20:22.147020Z","shell.execute_reply":"2024-11-06T20:20:22.159955Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sample_df.question_text.values[0]","metadata":{"id":"F3ITapULfguq","outputId":"6f18b5d4-beb1-42e9-b5fa-f8c19fc57873","trusted":true,"execution":{"iopub.status.busy":"2024-11-06T20:20:22.163186Z","iopub.execute_input":"2024-11-06T20:20:22.163722Z","iopub.status.idle":"2024-11-06T20:20:22.177227Z","shell.execute_reply.started":"2024-11-06T20:20:22.163633Z","shell.execute_reply":"2024-11-06T20:20:22.175986Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"inputs[0].toarray()","metadata":{"id":"y4ITys5EfLDm","outputId":"3f13d15f-eb1c-481c-d546-4cf63f49a89b","trusted":true,"execution":{"iopub.status.busy":"2024-11-06T20:20:22.178960Z","iopub.execute_input":"2024-11-06T20:20:22.179494Z","iopub.status.idle":"2024-11-06T20:20:22.195589Z","shell.execute_reply.started":"2024-11-06T20:20:22.179419Z","shell.execute_reply":"2024-11-06T20:20:22.194303Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_df","metadata":{"id":"yjKgg5wFgAxV","outputId":"d8b16375-c864-485b-900e-117fc6281740","trusted":true,"execution":{"iopub.status.busy":"2024-11-06T20:20:22.197498Z","iopub.execute_input":"2024-11-06T20:20:22.197909Z","iopub.status.idle":"2024-11-06T20:20:22.212585Z","shell.execute_reply.started":"2024-11-06T20:20:22.197866Z","shell.execute_reply":"2024-11-06T20:20:22.211293Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"%%time\n\ntest_inputs = vectorizer.transform(test_df.question_text)","metadata":{"id":"NWIvmSVXgBCY","outputId":"1663b577-7040-4187-a38a-3b02fbecf320","trusted":true,"execution":{"iopub.status.busy":"2024-11-06T20:20:22.214422Z","iopub.execute_input":"2024-11-06T20:20:22.215276Z","iopub.status.idle":"2024-11-06T20:22:06.992272Z","shell.execute_reply.started":"2024-11-06T20:20:22.215218Z","shell.execute_reply":"2024-11-06T20:22:06.991034Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# ML Models for Text Classification\n\n\n\nOutline :\n\n* Create a Training  and Validation set\n\n* Train e Logistic regression model\n\n* Make prediction on training validation and test data\n","metadata":{"id":"JnrUdI8UkAlb"}},{"cell_type":"markdown","source":"### Split into Training and Validation set","metadata":{"id":"VQklVdUv_epg"}},{"cell_type":"code","source":"sample_df","metadata":{"id":"-qjt3Iw4_dkQ","outputId":"9974bc2e-9857-4272-a51b-1399e7e2a218","trusted":true,"execution":{"iopub.status.busy":"2024-11-06T20:22:06.994006Z","iopub.execute_input":"2024-11-06T20:22:06.994506Z","iopub.status.idle":"2024-11-06T20:22:07.010256Z","shell.execute_reply.started":"2024-11-06T20:22:06.994440Z","shell.execute_reply":"2024-11-06T20:22:07.008851Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"inputs.shape","metadata":{"id":"oNged9jn_trS","outputId":"ec955bbe-23f4-45af-a4b2-922976bc85fa","trusted":true,"execution":{"iopub.status.busy":"2024-11-06T20:22:07.012123Z","iopub.execute_input":"2024-11-06T20:22:07.012906Z","iopub.status.idle":"2024-11-06T20:22:07.022096Z","shell.execute_reply.started":"2024-11-06T20:22:07.012848Z","shell.execute_reply":"2024-11-06T20:22:07.020982Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split","metadata":{"id":"ILPRRfmoAP1M","trusted":true,"execution":{"iopub.status.busy":"2024-11-06T20:22:07.023712Z","iopub.execute_input":"2024-11-06T20:22:07.024190Z","iopub.status.idle":"2024-11-06T20:22:07.035152Z","shell.execute_reply.started":"2024-11-06T20:22:07.024135Z","shell.execute_reply":"2024-11-06T20:22:07.033910Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_inputs, val_inputs, train_targets, val_targets = train_test_split(inputs, sample_df.target, test_size=0.3, random_state=42)","metadata":{"id":"mfFzcwl9_zTs","trusted":true,"execution":{"iopub.status.busy":"2024-11-06T20:22:07.036651Z","iopub.execute_input":"2024-11-06T20:22:07.037097Z","iopub.status.idle":"2024-11-06T20:22:07.062424Z","shell.execute_reply.started":"2024-11-06T20:22:07.037038Z","shell.execute_reply":"2024-11-06T20:22:07.061260Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_inputs.shape","metadata":{"id":"2khrrIChAf78","outputId":"29966a45-794a-4c67-bbfc-ae408b657f6d","trusted":true,"execution":{"iopub.status.busy":"2024-11-06T20:22:07.063951Z","iopub.execute_input":"2024-11-06T20:22:07.064305Z","iopub.status.idle":"2024-11-06T20:22:07.072308Z","shell.execute_reply.started":"2024-11-06T20:22:07.064267Z","shell.execute_reply":"2024-11-06T20:22:07.071110Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_targets.shape","metadata":{"id":"SUj46oe7C6zt","outputId":"6dc5fbaf-732d-45ff-f293-2fbd00b8aafe","trusted":true,"execution":{"iopub.status.busy":"2024-11-06T20:22:07.074102Z","iopub.execute_input":"2024-11-06T20:22:07.074614Z","iopub.status.idle":"2024-11-06T20:22:07.083988Z","shell.execute_reply.started":"2024-11-06T20:22:07.074540Z","shell.execute_reply":"2024-11-06T20:22:07.082688Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"val_inputs.shape","metadata":{"id":"r6ENmGXo_zW1","outputId":"6a19b403-21a8-4ef0-b145-a44f749a3283","trusted":true,"execution":{"iopub.status.busy":"2024-11-06T20:22:07.086103Z","iopub.execute_input":"2024-11-06T20:22:07.086519Z","iopub.status.idle":"2024-11-06T20:22:07.097163Z","shell.execute_reply.started":"2024-11-06T20:22:07.086477Z","shell.execute_reply":"2024-11-06T20:22:07.095958Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"val_targets.shape","metadata":{"id":"F7EjePSF_zaK","outputId":"fcc835aa-0402-440a-b879-594022290f4e","trusted":true,"execution":{"iopub.status.busy":"2024-11-06T20:22:07.098562Z","iopub.execute_input":"2024-11-06T20:22:07.099007Z","iopub.status.idle":"2024-11-06T20:22:07.109434Z","shell.execute_reply.started":"2024-11-06T20:22:07.098965Z","shell.execute_reply":"2024-11-06T20:22:07.108265Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Train Logistic Regression model","metadata":{"id":"FpBt0PCYFGLt"}},{"cell_type":"code","source":"from sklearn.linear_model import LogisticRegression","metadata":{"id":"t9A_vIHwFPJd","trusted":true,"execution":{"iopub.status.busy":"2024-11-06T20:22:07.110992Z","iopub.execute_input":"2024-11-06T20:22:07.112158Z","iopub.status.idle":"2024-11-06T20:22:07.119565Z","shell.execute_reply.started":"2024-11-06T20:22:07.112100Z","shell.execute_reply":"2024-11-06T20:22:07.118553Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"MAX_ITER = 1000","metadata":{"id":"ZON5H7U2HVqz","trusted":true,"execution":{"iopub.status.busy":"2024-11-06T20:22:07.121139Z","iopub.execute_input":"2024-11-06T20:22:07.121580Z","iopub.status.idle":"2024-11-06T20:22:07.133077Z","shell.execute_reply.started":"2024-11-06T20:22:07.121528Z","shell.execute_reply":"2024-11-06T20:22:07.131705Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"model =  LogisticRegression(max_iter = 1000, solver='sag')\n\nmodel","metadata":{"id":"NoyXR-4fFfk7","outputId":"60c59493-37af-4258-d7bb-522363ec1492","trusted":true,"execution":{"iopub.status.busy":"2024-11-06T20:22:07.135131Z","iopub.execute_input":"2024-11-06T20:22:07.135558Z","iopub.status.idle":"2024-11-06T20:22:07.151994Z","shell.execute_reply.started":"2024-11-06T20:22:07.135513Z","shell.execute_reply":"2024-11-06T20:22:07.150801Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"%%time\n\nmodel.fit(train_inputs,train_targets)","metadata":{"id":"0TJOkoasFPMj","outputId":"6317f5e7-d882-4c23-9c70-cf5e8e952ebc","trusted":true,"execution":{"iopub.status.busy":"2024-11-06T20:22:07.153654Z","iopub.execute_input":"2024-11-06T20:22:07.154500Z","iopub.status.idle":"2024-11-06T20:22:29.215745Z","shell.execute_reply.started":"2024-11-06T20:22:07.154433Z","shell.execute_reply":"2024-11-06T20:22:29.214648Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_preds = model.predict(train_inputs)","metadata":{"id":"I1H02ItEIVtJ","trusted":true,"execution":{"iopub.status.busy":"2024-11-06T20:22:29.217148Z","iopub.execute_input":"2024-11-06T20:22:29.217530Z","iopub.status.idle":"2024-11-06T20:22:29.225773Z","shell.execute_reply.started":"2024-11-06T20:22:29.217488Z","shell.execute_reply":"2024-11-06T20:22:29.224564Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_preds","metadata":{"id":"K88veg4MIV6L","outputId":"3f01bcfb-0d51-4ded-e6d7-d642c24048cc","trusted":true,"execution":{"iopub.status.busy":"2024-11-06T20:22:29.227207Z","iopub.execute_input":"2024-11-06T20:22:29.227657Z","iopub.status.idle":"2024-11-06T20:22:29.246229Z","shell.execute_reply.started":"2024-11-06T20:22:29.227580Z","shell.execute_reply":"2024-11-06T20:22:29.245039Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"pd.Series(train_preds).value_counts()","metadata":{"id":"-GQiZfX-KKm3","outputId":"408c8293-53f8-43c9-a599-aad63dbc6bf3","trusted":true,"execution":{"iopub.status.busy":"2024-11-06T20:22:29.247921Z","iopub.execute_input":"2024-11-06T20:22:29.248297Z","iopub.status.idle":"2024-11-06T20:22:29.262509Z","shell.execute_reply.started":"2024-11-06T20:22:29.248257Z","shell.execute_reply":"2024-11-06T20:22:29.261357Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"pd.Series(train_preds).value_counts().plot(kind='bar')","metadata":{"id":"2Lhh6KmNFPPt","outputId":"084a9c64-bd34-4c2d-da60-9be970a84485","trusted":true,"execution":{"iopub.status.busy":"2024-11-06T20:22:29.265039Z","iopub.execute_input":"2024-11-06T20:22:29.265428Z","iopub.status.idle":"2024-11-06T20:22:29.464740Z","shell.execute_reply.started":"2024-11-06T20:22:29.265387Z","shell.execute_reply":"2024-11-06T20:22:29.463615Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"pd.Series(train_targets).value_counts()","metadata":{"id":"vPPt14NcLEn6","outputId":"2b81afa1-9480-4925-8b21-5661d353f108","trusted":true,"execution":{"iopub.status.busy":"2024-11-06T20:22:29.466178Z","iopub.execute_input":"2024-11-06T20:22:29.466526Z","iopub.status.idle":"2024-11-06T20:22:29.477137Z","shell.execute_reply.started":"2024-11-06T20:22:29.466487Z","shell.execute_reply":"2024-11-06T20:22:29.476045Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"pd.Series(train_targets).value_counts().plot(kind='bar')","metadata":{"id":"LeRAnfuXF4u_","outputId":"3015c558-cf4c-4c2a-f72f-654f07a01a44","trusted":true,"execution":{"iopub.status.busy":"2024-11-06T20:22:29.478788Z","iopub.execute_input":"2024-11-06T20:22:29.479247Z","iopub.status.idle":"2024-11-06T20:22:29.724798Z","shell.execute_reply.started":"2024-11-06T20:22:29.479191Z","shell.execute_reply":"2024-11-06T20:22:29.723622Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.metrics import accuracy_score","metadata":{"id":"EoGrhgbSF4zG","trusted":true,"execution":{"iopub.status.busy":"2024-11-06T20:22:29.726810Z","iopub.execute_input":"2024-11-06T20:22:29.727182Z","iopub.status.idle":"2024-11-06T20:22:29.731971Z","shell.execute_reply.started":"2024-11-06T20:22:29.727141Z","shell.execute_reply":"2024-11-06T20:22:29.730766Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"accuracy_score(train_targets, train_preds)","metadata":{"id":"Ydhq2dyAF42I","outputId":"8c5c21f5-90d0-45f3-d16e-f06ec9529adb","trusted":true,"execution":{"iopub.status.busy":"2024-11-06T20:22:29.733471Z","iopub.execute_input":"2024-11-06T20:22:29.733849Z","iopub.status.idle":"2024-11-06T20:22:29.755629Z","shell.execute_reply.started":"2024-11-06T20:22:29.733808Z","shell.execute_reply":"2024-11-06T20:22:29.754109Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np","metadata":{"id":"feP_MC8JMTjd","trusted":true,"execution":{"iopub.status.busy":"2024-11-06T20:22:29.757161Z","iopub.execute_input":"2024-11-06T20:22:29.757570Z","iopub.status.idle":"2024-11-06T20:22:29.762909Z","shell.execute_reply.started":"2024-11-06T20:22:29.757528Z","shell.execute_reply":"2024-11-06T20:22:29.761550Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"accuracy_score(train_targets, np.zeros(len(train_targets)))","metadata":{"id":"gL82sSUjF5AP","outputId":"7e60c4ff-2640-4400-d8ce-7dac988ebe56","trusted":true,"execution":{"iopub.status.busy":"2024-11-06T20:22:29.764470Z","iopub.execute_input":"2024-11-06T20:22:29.766066Z","iopub.status.idle":"2024-11-06T20:22:29.785044Z","shell.execute_reply.started":"2024-11-06T20:22:29.766008Z","shell.execute_reply":"2024-11-06T20:22:29.783783Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"accuracy_score(train_targets, np.ones(len(train_targets)))","metadata":{"id":"2oq9uxJm_zeG","outputId":"db4f934a-ca9d-4123-c9e7-48d3fa2af0e9","trusted":true,"execution":{"iopub.status.busy":"2024-11-06T20:22:29.786724Z","iopub.execute_input":"2024-11-06T20:22:29.787088Z","iopub.status.idle":"2024-11-06T20:22:29.804235Z","shell.execute_reply.started":"2024-11-06T20:22:29.787048Z","shell.execute_reply":"2024-11-06T20:22:29.802991Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.metrics import f1_score","metadata":{"id":"jFNa_PJ_MlvX","trusted":true,"execution":{"iopub.status.busy":"2024-11-06T20:22:29.807525Z","iopub.execute_input":"2024-11-06T20:22:29.807949Z","iopub.status.idle":"2024-11-06T20:22:29.813542Z","shell.execute_reply.started":"2024-11-06T20:22:29.807909Z","shell.execute_reply":"2024-11-06T20:22:29.812218Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"f1_score(train_targets,train_preds)","metadata":{"id":"qsTxz9SeNSf5","outputId":"716994f7-3e55-4a67-adb3-112d43cf8854","trusted":true,"execution":{"iopub.status.busy":"2024-11-06T20:22:29.815108Z","iopub.execute_input":"2024-11-06T20:22:29.815474Z","iopub.status.idle":"2024-11-06T20:22:29.855107Z","shell.execute_reply.started":"2024-11-06T20:22:29.815433Z","shell.execute_reply":"2024-11-06T20:22:29.854005Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"f1_score(train_targets,np.zeros(len(train_targets)))","metadata":{"id":"HzG1K_dQMlyr","outputId":"7e170192-626c-479d-fdc3-4b0af26cac32","trusted":true,"execution":{"iopub.status.busy":"2024-11-06T20:22:29.856416Z","iopub.execute_input":"2024-11-06T20:22:29.857036Z","iopub.status.idle":"2024-11-06T20:22:29.895816Z","shell.execute_reply.started":"2024-11-06T20:22:29.856993Z","shell.execute_reply":"2024-11-06T20:22:29.894688Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"random_preds = np.random.choice((0,1),len(train_targets))\n\nf1_score(train_targets, random_preds)","metadata":{"id":"g7WSiPamMl4-","outputId":"ea7bea35-a1c8-4a1d-b413-a13e5ade9237","trusted":true,"execution":{"iopub.status.busy":"2024-11-06T20:22:29.897241Z","iopub.execute_input":"2024-11-06T20:22:29.897751Z","iopub.status.idle":"2024-11-06T20:22:29.937894Z","shell.execute_reply.started":"2024-11-06T20:22:29.897693Z","shell.execute_reply":"2024-11-06T20:22:29.936680Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"val_preds = model.predict(val_inputs)\n\nval_preds","metadata":{"id":"kXr3FoB7UqfT","outputId":"ad14fcc5-da66-41f1-ea06-cb76a90f5d99","trusted":true,"execution":{"iopub.status.busy":"2024-11-06T20:22:29.939293Z","iopub.execute_input":"2024-11-06T20:22:29.939676Z","iopub.status.idle":"2024-11-06T20:22:29.949130Z","shell.execute_reply.started":"2024-11-06T20:22:29.939634Z","shell.execute_reply":"2024-11-06T20:22:29.947862Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"accuracy_score(val_targets, val_preds)","metadata":{"id":"bY98eHveUqjm","outputId":"af1eb646-f791-418e-90fd-67052677b27a","trusted":true,"execution":{"iopub.status.busy":"2024-11-06T20:22:29.950929Z","iopub.execute_input":"2024-11-06T20:22:29.951372Z","iopub.status.idle":"2024-11-06T20:22:29.965818Z","shell.execute_reply.started":"2024-11-06T20:22:29.951308Z","shell.execute_reply":"2024-11-06T20:22:29.964638Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sincere_df.question_text.values[:10]","metadata":{"id":"sW3cF3upVkcZ","outputId":"8aca6ad8-1b95-4f1e-bd81-b83e14553fff","trusted":true,"execution":{"iopub.status.busy":"2024-11-06T20:22:29.967043Z","iopub.execute_input":"2024-11-06T20:22:29.967576Z","iopub.status.idle":"2024-11-06T20:22:29.976227Z","shell.execute_reply.started":"2024-11-06T20:22:29.967531Z","shell.execute_reply":"2024-11-06T20:22:29.974978Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sincere_df.target.values[:10]","metadata":{"id":"Cl3NEHEPVkgj","outputId":"b62d7eed-06a0-4047-c626-89d32ab0f6f0","trusted":true,"execution":{"iopub.status.busy":"2024-11-06T20:22:29.978015Z","iopub.execute_input":"2024-11-06T20:22:29.978399Z","iopub.status.idle":"2024-11-06T20:22:29.988118Z","shell.execute_reply.started":"2024-11-06T20:22:29.978356Z","shell.execute_reply":"2024-11-06T20:22:29.986888Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"model.predict(vectorizer.transform(sincere_df.question_text.values[:10]))","metadata":{"id":"eev5iIzXVkk_","outputId":"605d4f35-c19b-458a-a249-f09685a19667","trusted":true,"execution":{"iopub.status.busy":"2024-11-06T20:22:29.989292Z","iopub.execute_input":"2024-11-06T20:22:29.989711Z","iopub.status.idle":"2024-11-06T20:22:30.008148Z","shell.execute_reply.started":"2024-11-06T20:22:29.989655Z","shell.execute_reply":"2024-11-06T20:22:30.006891Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"f1_score(val_targets, val_preds)","metadata":{"id":"6EZBeDf9UqoA","outputId":"905fa14a-d9b2-47f3-90b0-25bdf99abb43","trusted":true,"execution":{"iopub.status.busy":"2024-11-06T20:22:30.009896Z","iopub.execute_input":"2024-11-06T20:22:30.010372Z","iopub.status.idle":"2024-11-06T20:22:30.035192Z","shell.execute_reply.started":"2024-11-06T20:22:30.010326Z","shell.execute_reply":"2024-11-06T20:22:30.034062Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"insincere_df.target.values[:10]","metadata":{"id":"nkkwPdS_WpjX","outputId":"d8e48096-0200-48f5-e85e-6849fde7aa44","trusted":true,"execution":{"iopub.status.busy":"2024-11-06T20:22:30.037158Z","iopub.execute_input":"2024-11-06T20:22:30.037569Z","iopub.status.idle":"2024-11-06T20:22:30.045932Z","shell.execute_reply.started":"2024-11-06T20:22:30.037527Z","shell.execute_reply":"2024-11-06T20:22:30.044767Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"model.predict(vectorizer.transform(insincere_df.question_text.values[:10]))","metadata":{"id":"nCAseeLPWpoZ","outputId":"57a4bf4d-2631-4da1-8c9d-86333bf3f66e","trusted":true,"execution":{"iopub.status.busy":"2024-11-06T20:22:30.047422Z","iopub.execute_input":"2024-11-06T20:22:30.047816Z","iopub.status.idle":"2024-11-06T20:22:30.065989Z","shell.execute_reply.started":"2024-11-06T20:22:30.047775Z","shell.execute_reply":"2024-11-06T20:22:30.064758Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Make predictions and submit to kaggle","metadata":{"id":"l4csqycRkJaj"}},{"cell_type":"code","source":"test_df","metadata":{"id":"FGnM47_m-wVx","outputId":"e549f6e1-82e5-40a4-9261-5d9f64e05734","trusted":true,"execution":{"iopub.status.busy":"2024-11-06T20:22:30.067682Z","iopub.execute_input":"2024-11-06T20:22:30.068346Z","iopub.status.idle":"2024-11-06T20:22:30.084490Z","shell.execute_reply.started":"2024-11-06T20:22:30.068279Z","shell.execute_reply":"2024-11-06T20:22:30.083256Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_inputs.shape","metadata":{"id":"c-3u3cOrXuY6","outputId":"d87febb2-04a4-4923-fc5c-984b8b75a298","trusted":true,"execution":{"iopub.status.busy":"2024-11-06T20:22:30.086277Z","iopub.execute_input":"2024-11-06T20:22:30.086943Z","iopub.status.idle":"2024-11-06T20:22:30.097138Z","shell.execute_reply.started":"2024-11-06T20:22:30.086885Z","shell.execute_reply":"2024-11-06T20:22:30.095946Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_preds =model.predict(test_inputs)","metadata":{"id":"FDWqhUJBXud5","trusted":true,"execution":{"iopub.status.busy":"2024-11-06T20:22:30.098583Z","iopub.execute_input":"2024-11-06T20:22:30.098975Z","iopub.status.idle":"2024-11-06T20:22:30.124885Z","shell.execute_reply.started":"2024-11-06T20:22:30.098935Z","shell.execute_reply":"2024-11-06T20:22:30.123697Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sub_df","metadata":{"id":"FjqYresMXujI","outputId":"7a60a75a-8abe-448e-9fe6-c729a3213f0a","trusted":true,"execution":{"iopub.status.busy":"2024-11-06T20:22:30.126401Z","iopub.execute_input":"2024-11-06T20:22:30.126801Z","iopub.status.idle":"2024-11-06T20:22:30.139848Z","shell.execute_reply.started":"2024-11-06T20:22:30.126759Z","shell.execute_reply":"2024-11-06T20:22:30.138645Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sub_df.prediction = test_preds","metadata":{"id":"yRH5MukJXun5","trusted":true,"execution":{"iopub.status.busy":"2024-11-06T20:22:30.141348Z","iopub.execute_input":"2024-11-06T20:22:30.141760Z","iopub.status.idle":"2024-11-06T20:22:30.149836Z","shell.execute_reply.started":"2024-11-06T20:22:30.141711Z","shell.execute_reply":"2024-11-06T20:22:30.148665Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sub_df.prediction.value_counts()","metadata":{"id":"LKw391CGXusm","outputId":"c9780807-67b7-4c58-a6c4-46011d279924","trusted":true,"execution":{"iopub.status.busy":"2024-11-06T20:22:30.151378Z","iopub.execute_input":"2024-11-06T20:22:30.151797Z","iopub.status.idle":"2024-11-06T20:22:30.168559Z","shell.execute_reply.started":"2024-11-06T20:22:30.151741Z","shell.execute_reply":"2024-11-06T20:22:30.167177Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sub_df","metadata":{"id":"umfe8NEk-wZ3","outputId":"619d9be9-9bc9-41cc-91b8-d36a3a9a0c36","trusted":true,"execution":{"iopub.status.busy":"2024-11-06T20:22:30.170056Z","iopub.execute_input":"2024-11-06T20:22:30.170526Z","iopub.status.idle":"2024-11-06T20:22:30.187144Z","shell.execute_reply.started":"2024-11-06T20:22:30.170471Z","shell.execute_reply":"2024-11-06T20:22:30.185531Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sub_df.to_csv('submission.csv', index=None)","metadata":{"id":"MjJp_uyJ-wdg","trusted":true,"execution":{"iopub.status.busy":"2024-11-06T20:22:30.188786Z","iopub.execute_input":"2024-11-06T20:22:30.189320Z","iopub.status.idle":"2024-11-06T20:22:30.958518Z","shell.execute_reply.started":"2024-11-06T20:22:30.189262Z","shell.execute_reply":"2024-11-06T20:22:30.957184Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!head submission.csv","metadata":{"id":"cUb51NqNivsR","outputId":"31cf53b0-0c6d-4a63-a8c2-3776595057b2","trusted":true,"execution":{"iopub.status.busy":"2024-11-06T20:22:30.960042Z","iopub.execute_input":"2024-11-06T20:22:30.960394Z","iopub.status.idle":"2024-11-06T20:22:32.204317Z","shell.execute_reply.started":"2024-11-06T20:22:30.960357Z","shell.execute_reply":"2024-11-06T20:22:32.202925Z"}},"outputs":[],"execution_count":null}]}