{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":10737,"databundleVersionId":290346,"sourceType":"competition"}],"dockerImageVersionId":30626,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Neural Networks and Embeddings for Natural Language Processing\n\nOutline:\n- Download the Data\n- Prepare Data for Training\n- Logistic Regression Model\n- Feed Forward Neural Network\n\n\nDataset: https://www.kaggle.com/c/quora-insincere-questions-classification","metadata":{}},{"cell_type":"markdown","source":"## Download the Data\n\nUpload your `kaggle.json` file to Colab","metadata":{}},{"cell_type":"code","source":"!ls","metadata":{"execution":{"iopub.status.busy":"2023-12-26T06:36:17.675782Z","iopub.execute_input":"2023-12-26T06:36:17.676747Z","iopub.status.idle":"2023-12-26T06:36:18.563790Z","shell.execute_reply.started":"2023-12-26T06:36:17.676710Z","shell.execute_reply":"2023-12-26T06:36:18.562474Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os","metadata":{"execution":{"iopub.status.busy":"2023-12-26T06:36:18.566886Z","iopub.execute_input":"2023-12-26T06:36:18.567619Z","iopub.status.idle":"2023-12-26T06:36:18.572827Z","shell.execute_reply.started":"2023-12-26T06:36:18.567583Z","shell.execute_reply":"2023-12-26T06:36:18.571686Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pwd","metadata":{"execution":{"iopub.status.busy":"2023-12-26T06:36:18.574627Z","iopub.execute_input":"2023-12-26T06:36:18.575030Z","iopub.status.idle":"2023-12-26T06:36:19.004737Z","shell.execute_reply.started":"2023-12-26T06:36:18.574998Z","shell.execute_reply":"2023-12-26T06:36:19.003253Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"IS_KAGGLE = 'KAGGLE_KERNEL_RUN_TYPE' in os.environ","metadata":{"execution":{"iopub.status.busy":"2023-12-26T06:36:19.007882Z","iopub.execute_input":"2023-12-26T06:36:19.008891Z","iopub.status.idle":"2023-12-26T06:36:19.014188Z","shell.execute_reply.started":"2023-12-26T06:36:19.008851Z","shell.execute_reply":"2023-12-26T06:36:19.013150Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if IS_KAGGLE:\n    data_dir = '../input/quora-insincere-questions-classification'\n    train_fname = data_dir + '/train.csv'\n    test_fname = data_dir + '/test.csv'\n    sub_fname = data_dir + '/sample_submission.csv'\nelse:\n    os.environ['KAGGLE_CONFIG_DIR'] = '.'\n    !kaggle competitions download -c quora-insincere-questions-classification -f train.csv -p data\n    !kaggle competitions download -c quora-insincere-questions-classification -f test.csv -p data\n    !kaggle competitions download -c quora-insincere-questions-classification -f sample_submission.csv -p data\n    train_fname = 'data/train.csv.zip'\n    test_fname = 'data/test.csv.zip'\n    sub_fname = 'data/sample_submission.csv.zip' ","metadata":{"execution":{"iopub.status.busy":"2023-12-26T06:36:19.016006Z","iopub.execute_input":"2023-12-26T06:36:19.016849Z","iopub.status.idle":"2023-12-26T06:36:19.027492Z","shell.execute_reply.started":"2023-12-26T06:36:19.016791Z","shell.execute_reply":"2023-12-26T06:36:19.026376Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd","metadata":{"execution":{"iopub.status.busy":"2023-12-26T06:36:19.028979Z","iopub.execute_input":"2023-12-26T06:36:19.029972Z","iopub.status.idle":"2023-12-26T06:36:19.044455Z","shell.execute_reply.started":"2023-12-26T06:36:19.029937Z","shell.execute_reply":"2023-12-26T06:36:19.043631Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"raw_df = pd.read_csv(train_fname)\ntest_df = pd.read_csv(test_fname)\nsub_df = pd.read_csv(sub_fname)","metadata":{"execution":{"iopub.status.busy":"2023-12-26T06:36:19.045899Z","iopub.execute_input":"2023-12-26T06:36:19.046915Z","iopub.status.idle":"2023-12-26T06:36:23.734717Z","shell.execute_reply.started":"2023-12-26T06:36:19.046884Z","shell.execute_reply":"2023-12-26T06:36:23.733501Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"raw_df","metadata":{"execution":{"iopub.status.busy":"2023-12-26T06:36:23.736084Z","iopub.execute_input":"2023-12-26T06:36:23.736428Z","iopub.status.idle":"2023-12-26T06:36:23.750177Z","shell.execute_reply.started":"2023-12-26T06:36:23.736401Z","shell.execute_reply":"2023-12-26T06:36:23.749103Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df","metadata":{"execution":{"iopub.status.busy":"2023-12-26T06:36:23.751437Z","iopub.execute_input":"2023-12-26T06:36:23.751759Z","iopub.status.idle":"2023-12-26T06:36:23.767601Z","shell.execute_reply.started":"2023-12-26T06:36:23.751733Z","shell.execute_reply":"2023-12-26T06:36:23.766257Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df.question_text.values[0]","metadata":{"execution":{"iopub.status.busy":"2023-12-26T06:36:23.770935Z","iopub.execute_input":"2023-12-26T06:36:23.771322Z","iopub.status.idle":"2023-12-26T06:36:23.780421Z","shell.execute_reply.started":"2023-12-26T06:36:23.771289Z","shell.execute_reply":"2023-12-26T06:36:23.779318Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub_df","metadata":{"execution":{"iopub.status.busy":"2023-12-26T06:36:23.782123Z","iopub.execute_input":"2023-12-26T06:36:23.782465Z","iopub.status.idle":"2023-12-26T06:36:23.798950Z","shell.execute_reply.started":"2023-12-26T06:36:23.782433Z","shell.execute_reply":"2023-12-26T06:36:23.797406Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if IS_KAGGLE:\n    sample_df = raw_df\nelse:\n    sample_df = raw_df.sample(100_000, random_state=42)","metadata":{"execution":{"iopub.status.busy":"2023-12-26T06:36:23.800444Z","iopub.execute_input":"2023-12-26T06:36:23.800749Z","iopub.status.idle":"2023-12-26T06:36:23.807861Z","shell.execute_reply.started":"2023-12-26T06:36:23.800725Z","shell.execute_reply":"2023-12-26T06:36:23.806964Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Prepare Data for Training\n\n\nOutline:\n- Convert text to TF-IDF Vectors\n- Split training & validation set\n- Convert to PyTorch tensors","metadata":{}},{"cell_type":"markdown","source":"### Conversion to TF-IDF Vectors","metadata":{}},{"cell_type":"code","source":"import nltk\nfrom nltk.tokenize import word_tokenize\nfrom nltk.stem import SnowballStemmer\nfrom nltk.corpus import stopwords\nfrom sklearn.feature_extraction.text import TfidfVectorizer","metadata":{"execution":{"iopub.status.busy":"2023-12-26T06:36:23.809188Z","iopub.execute_input":"2023-12-26T06:36:23.810356Z","iopub.status.idle":"2023-12-26T06:36:23.819679Z","shell.execute_reply.started":"2023-12-26T06:36:23.810314Z","shell.execute_reply":"2023-12-26T06:36:23.818565Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"nltk.download('punkt')","metadata":{"execution":{"iopub.status.busy":"2023-12-26T06:36:23.821353Z","iopub.execute_input":"2023-12-26T06:36:23.821673Z","iopub.status.idle":"2023-12-26T06:36:43.871471Z","shell.execute_reply.started":"2023-12-26T06:36:23.821648Z","shell.execute_reply":"2023-12-26T06:36:43.869961Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"stemmer = SnowballStemmer(language='english')\n\ndef tokenize(text):\n    return [stemmer.stem(token) for token in word_tokenize(text)]","metadata":{"execution":{"iopub.status.busy":"2023-12-26T06:36:43.873150Z","iopub.execute_input":"2023-12-26T06:36:43.873568Z","iopub.status.idle":"2023-12-26T06:36:43.879237Z","shell.execute_reply.started":"2023-12-26T06:36:43.873518Z","shell.execute_reply":"2023-12-26T06:36:43.878112Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tokenize(\"Ain't nothin' (but a heartache)!\")","metadata":{"execution":{"iopub.status.busy":"2023-12-26T06:36:43.880759Z","iopub.execute_input":"2023-12-26T06:36:43.881155Z","iopub.status.idle":"2023-12-26T06:36:43.894882Z","shell.execute_reply.started":"2023-12-26T06:36:43.881122Z","shell.execute_reply":"2023-12-26T06:36:43.893594Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"nltk.download('stopwords')","metadata":{"execution":{"iopub.status.busy":"2023-12-26T06:36:43.896503Z","iopub.execute_input":"2023-12-26T06:36:43.896921Z","iopub.status.idle":"2023-12-26T06:37:03.940308Z","shell.execute_reply.started":"2023-12-26T06:36:43.896857Z","shell.execute_reply":"2023-12-26T06:37:03.939113Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"english_stopwords = stopwords.words('english')","metadata":{"execution":{"iopub.status.busy":"2023-12-26T06:37:03.941491Z","iopub.execute_input":"2023-12-26T06:37:03.941765Z","iopub.status.idle":"2023-12-26T06:37:03.947613Z","shell.execute_reply.started":"2023-12-26T06:37:03.941742Z","shell.execute_reply":"2023-12-26T06:37:03.946203Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"vectorizer = TfidfVectorizer(lowercase=True, \n                             tokenizer=tokenize,\n                             stop_words=english_stopwords,\n                             max_features=1000)","metadata":{"execution":{"iopub.status.busy":"2023-12-26T06:37:03.949307Z","iopub.execute_input":"2023-12-26T06:37:03.949662Z","iopub.status.idle":"2023-12-26T06:37:03.960993Z","shell.execute_reply.started":"2023-12-26T06:37:03.949632Z","shell.execute_reply":"2023-12-26T06:37:03.959994Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nvectorizer.fit(sample_df.question_text)","metadata":{"execution":{"iopub.status.busy":"2023-12-26T06:37:03.961870Z","iopub.execute_input":"2023-12-26T06:37:03.962193Z","iopub.status.idle":"2023-12-26T06:43:07.779322Z","shell.execute_reply.started":"2023-12-26T06:37:03.962169Z","shell.execute_reply":"2023-12-26T06:43:07.778578Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"vectorizer.get_feature_names_out()[:100]","metadata":{"execution":{"iopub.status.busy":"2023-12-26T06:43:07.780303Z","iopub.execute_input":"2023-12-26T06:43:07.781262Z","iopub.status.idle":"2023-12-26T06:43:07.790858Z","shell.execute_reply.started":"2023-12-26T06:43:07.781235Z","shell.execute_reply":"2023-12-26T06:43:07.788985Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Transform the questions into vectors","metadata":{}},{"cell_type":"code","source":"%%time\ninputs = vectorizer.transform(sample_df.question_text)","metadata":{"execution":{"iopub.status.busy":"2023-12-26T06:43:07.792880Z","iopub.execute_input":"2023-12-26T06:43:07.793339Z","iopub.status.idle":"2023-12-26T06:49:11.223857Z","shell.execute_reply.started":"2023-12-26T06:43:07.793310Z","shell.execute_reply":"2023-12-26T06:49:11.222606Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"inputs.shape","metadata":{"execution":{"iopub.status.busy":"2023-12-26T06:49:11.225362Z","iopub.execute_input":"2023-12-26T06:49:11.225671Z","iopub.status.idle":"2023-12-26T06:49:11.231797Z","shell.execute_reply.started":"2023-12-26T06:49:11.225645Z","shell.execute_reply":"2023-12-26T06:49:11.230865Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"inputs[0].toarray()","metadata":{"execution":{"iopub.status.busy":"2023-12-26T06:49:11.232840Z","iopub.execute_input":"2023-12-26T06:49:11.233136Z","iopub.status.idle":"2023-12-26T06:49:11.251186Z","shell.execute_reply.started":"2023-12-26T06:49:11.233105Z","shell.execute_reply":"2023-12-26T06:49:11.250122Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ntest_inputs = vectorizer.transform(test_df.question_text)","metadata":{"execution":{"iopub.status.busy":"2023-12-26T06:49:11.252581Z","iopub.execute_input":"2023-12-26T06:49:11.252991Z","iopub.status.idle":"2023-12-26T06:50:55.805054Z","shell.execute_reply.started":"2023-12-26T06:49:11.252958Z","shell.execute_reply":"2023-12-26T06:50:55.804017Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_inputs.shape","metadata":{"execution":{"iopub.status.busy":"2023-12-26T06:50:55.806611Z","iopub.execute_input":"2023-12-26T06:50:55.807021Z","iopub.status.idle":"2023-12-26T06:50:55.813584Z","shell.execute_reply.started":"2023-12-26T06:50:55.806996Z","shell.execute_reply":"2023-12-26T06:50:55.812473Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Split the Training and Validation set","metadata":{}},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split","metadata":{"execution":{"iopub.status.busy":"2023-12-26T06:50:55.814505Z","iopub.execute_input":"2023-12-26T06:50:55.814744Z","iopub.status.idle":"2023-12-26T06:50:55.824000Z","shell.execute_reply.started":"2023-12-26T06:50:55.814723Z","shell.execute_reply":"2023-12-26T06:50:55.823025Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"targets = sample_df.target\ntargets","metadata":{"execution":{"iopub.status.busy":"2023-12-26T06:50:55.830404Z","iopub.execute_input":"2023-12-26T06:50:55.830691Z","iopub.status.idle":"2023-12-26T06:50:55.840799Z","shell.execute_reply.started":"2023-12-26T06:50:55.830668Z","shell.execute_reply":"2023-12-26T06:50:55.839524Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_inputs, val_inputs, train_targets, val_targets = train_test_split(inputs, targets, test_size = 0.3)","metadata":{"execution":{"iopub.status.busy":"2023-12-26T06:50:55.842184Z","iopub.execute_input":"2023-12-26T06:50:55.842545Z","iopub.status.idle":"2023-12-26T06:50:56.136301Z","shell.execute_reply.started":"2023-12-26T06:50:55.842520Z","shell.execute_reply":"2023-12-26T06:50:56.135267Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_inputs.shape, train_targets.shape, val_inputs.shape, val_targets.shape","metadata":{"execution":{"iopub.status.busy":"2023-12-26T06:50:56.138291Z","iopub.execute_input":"2023-12-26T06:50:56.139762Z","iopub.status.idle":"2023-12-26T06:50:56.148813Z","shell.execute_reply.started":"2023-12-26T06:50:56.139702Z","shell.execute_reply":"2023-12-26T06:50:56.147537Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Convert to PyTorch Tensors","metadata":{}},{"cell_type":"code","source":"import torch","metadata":{"execution":{"iopub.status.busy":"2023-12-26T06:50:56.150266Z","iopub.execute_input":"2023-12-26T06:50:56.150505Z","iopub.status.idle":"2023-12-26T06:50:56.156580Z","shell.execute_reply.started":"2023-12-26T06:50:56.150484Z","shell.execute_reply":"2023-12-26T06:50:56.155354Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_input_tensors = torch.tensor(train_inputs.toarray()).float()\nval_input_tensors = torch.tensor(val_inputs.toarray()).float()","metadata":{"execution":{"iopub.status.busy":"2023-12-26T06:50:56.158006Z","iopub.execute_input":"2023-12-26T06:50:56.158446Z","iopub.status.idle":"2023-12-26T06:51:12.622769Z","shell.execute_reply.started":"2023-12-26T06:50:56.158415Z","shell.execute_reply":"2023-12-26T06:51:12.621571Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_input_tensors.shape","metadata":{"execution":{"iopub.status.busy":"2023-12-26T06:51:12.624136Z","iopub.execute_input":"2023-12-26T06:51:12.624473Z","iopub.status.idle":"2023-12-26T06:51:12.631888Z","shell.execute_reply.started":"2023-12-26T06:51:12.624444Z","shell.execute_reply":"2023-12-26T06:51:12.630141Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"val_input_tensors.shape","metadata":{"execution":{"iopub.status.busy":"2023-12-26T06:51:12.635209Z","iopub.execute_input":"2023-12-26T06:51:12.635634Z","iopub.status.idle":"2023-12-26T06:51:12.644008Z","shell.execute_reply.started":"2023-12-26T06:51:12.635597Z","shell.execute_reply":"2023-12-26T06:51:12.642761Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_target_tensors = torch.tensor(train_targets.values).float()\nval_target_tensors = torch.tensor(val_targets.values).float()","metadata":{"execution":{"iopub.status.busy":"2023-12-26T06:51:12.646055Z","iopub.execute_input":"2023-12-26T06:51:12.646873Z","iopub.status.idle":"2023-12-26T06:51:12.654175Z","shell.execute_reply.started":"2023-12-26T06:51:12.646826Z","shell.execute_reply":"2023-12-26T06:51:12.653074Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_target_tensors","metadata":{"execution":{"iopub.status.busy":"2023-12-26T06:51:12.655303Z","iopub.execute_input":"2023-12-26T06:51:12.655571Z","iopub.status.idle":"2023-12-26T06:51:12.665489Z","shell.execute_reply.started":"2023-12-26T06:51:12.655548Z","shell.execute_reply":"2023-12-26T06:51:12.664433Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_input_tensors = torch.tensor(test_inputs.toarray()).float()","metadata":{"execution":{"iopub.status.busy":"2023-12-26T06:51:12.666494Z","iopub.execute_input":"2023-12-26T06:51:12.666740Z","iopub.status.idle":"2023-12-26T06:51:15.075196Z","shell.execute_reply.started":"2023-12-26T06:51:12.666720Z","shell.execute_reply":"2023-12-26T06:51:15.073932Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_input_tensors","metadata":{"execution":{"iopub.status.busy":"2023-12-26T06:51:15.076814Z","iopub.execute_input":"2023-12-26T06:51:15.077380Z","iopub.status.idle":"2023-12-26T06:51:15.083425Z","shell.execute_reply.started":"2023-12-26T06:51:15.077351Z","shell.execute_reply":"2023-12-26T06:51:15.082454Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Create PyTorch Data Loaders","metadata":{}},{"cell_type":"code","source":"from torch.utils.data import TensorDataset, DataLoader","metadata":{"execution":{"iopub.status.busy":"2023-12-26T06:51:15.084375Z","iopub.execute_input":"2023-12-26T06:51:15.084616Z","iopub.status.idle":"2023-12-26T06:51:15.093119Z","shell.execute_reply.started":"2023-12-26T06:51:15.084595Z","shell.execute_reply":"2023-12-26T06:51:15.092110Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_ds = TensorDataset(train_input_tensors, train_target_tensors)\nval_ds = TensorDataset(val_input_tensors, val_target_tensors)\ntest_ds = TensorDataset(test_input_tensors)","metadata":{"execution":{"iopub.status.busy":"2023-12-26T06:51:15.094421Z","iopub.execute_input":"2023-12-26T06:51:15.094752Z","iopub.status.idle":"2023-12-26T06:51:15.103140Z","shell.execute_reply.started":"2023-12-26T06:51:15.094719Z","shell.execute_reply":"2023-12-26T06:51:15.102210Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_ds[:10]","metadata":{"execution":{"iopub.status.busy":"2023-12-26T06:51:15.104919Z","iopub.execute_input":"2023-12-26T06:51:15.105446Z","iopub.status.idle":"2023-12-26T06:51:15.115331Z","shell.execute_reply.started":"2023-12-26T06:51:15.105414Z","shell.execute_reply":"2023-12-26T06:51:15.114277Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"BATCH_SIZE = 128","metadata":{"execution":{"iopub.status.busy":"2023-12-26T06:51:15.116433Z","iopub.execute_input":"2023-12-26T06:51:15.116752Z","iopub.status.idle":"2023-12-26T06:51:15.122564Z","shell.execute_reply.started":"2023-12-26T06:51:15.116720Z","shell.execute_reply":"2023-12-26T06:51:15.121661Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_dl = DataLoader(train_ds, batch_size = BATCH_SIZE, shuffle = True)\nval_dl = DataLoader(val_ds, batch_size = BATCH_SIZE)\ntest_dl = DataLoader(test_ds, batch_size = BATCH_SIZE)","metadata":{"execution":{"iopub.status.busy":"2023-12-26T06:51:15.123713Z","iopub.execute_input":"2023-12-26T06:51:15.124258Z","iopub.status.idle":"2023-12-26T06:51:15.133663Z","shell.execute_reply.started":"2023-12-26T06:51:15.124233Z","shell.execute_reply":"2023-12-26T06:51:15.132490Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for batch in train_dl:\n    batch_inputs = batch[0]\n    batch_targets = batch[1]\n    print('batch_inputs.shape', batch_inputs.shape)\n    print('batch_targets.shape', batch_targets.shape)\n    break","metadata":{"execution":{"iopub.status.busy":"2023-12-26T06:51:15.135174Z","iopub.execute_input":"2023-12-26T06:51:15.135481Z","iopub.status.idle":"2023-12-26T06:51:15.220715Z","shell.execute_reply.started":"2023-12-26T06:51:15.135458Z","shell.execute_reply":"2023-12-26T06:51:15.219454Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('Number of batches:' , len(train_dl))","metadata":{"execution":{"iopub.status.busy":"2023-12-26T06:51:15.221935Z","iopub.execute_input":"2023-12-26T06:51:15.222368Z","iopub.status.idle":"2023-12-26T06:51:15.227651Z","shell.execute_reply.started":"2023-12-26T06:51:15.222343Z","shell.execute_reply":"2023-12-26T06:51:15.226470Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Train a Deep Learning Neural Network","metadata":{}},{"cell_type":"code","source":"import torch.nn as nn\nimport torch.nn.functional as F","metadata":{"execution":{"iopub.status.busy":"2023-12-26T06:51:15.228900Z","iopub.execute_input":"2023-12-26T06:51:15.229229Z","iopub.status.idle":"2023-12-26T06:51:15.237751Z","shell.execute_reply.started":"2023-12-26T06:51:15.229205Z","shell.execute_reply":"2023-12-26T06:51:15.236632Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class QuoraNet(nn.Module):\n    def __init__(self):\n        super().__init__()\n        self.layer1 = nn.Linear(1000, 512)\n        self.layer2 = nn.Linear(512, 256)\n        self.layer3 = nn.Linear(256, 128)\n        self.layer4 = nn.Linear(128, 1)\n    \n    def forward(self, input):\n        out = self.layer1(input)\n        out = F.relu(out)\n        out = self.layer2(out)\n        out = F.relu(out)\n        out = self.layer3(out)\n        out = F.relu(out)\n        out = self.layer4(out)\n        return out","metadata":{"execution":{"iopub.status.busy":"2023-12-26T06:51:15.239726Z","iopub.execute_input":"2023-12-26T06:51:15.240207Z","iopub.status.idle":"2023-12-26T06:51:15.248990Z","shell.execute_reply.started":"2023-12-26T06:51:15.240177Z","shell.execute_reply":"2023-12-26T06:51:15.248197Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = QuoraNet()","metadata":{"execution":{"iopub.status.busy":"2023-12-26T06:51:15.250025Z","iopub.execute_input":"2023-12-26T06:51:15.250438Z","iopub.status.idle":"2023-12-26T06:51:15.268889Z","shell.execute_reply.started":"2023-12-26T06:51:15.250406Z","shell.execute_reply":"2023-12-26T06:51:15.267953Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import f1_score, accuracy_score","metadata":{"execution":{"iopub.status.busy":"2023-12-26T06:51:15.270482Z","iopub.execute_input":"2023-12-26T06:51:15.270949Z","iopub.status.idle":"2023-12-26T06:51:15.276793Z","shell.execute_reply.started":"2023-12-26T06:51:15.270913Z","shell.execute_reply":"2023-12-26T06:51:15.275438Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for batch in train_dl:\n    bi, bt = batch\n    print('inputs.shape', bi.shape)\n    print('targets.shape', bt.shape)\n    \n    bo = model(bi)\n    print('outputs.shape', bo.shape)\n    print('bo', bo.shape)\n    \n    #convert outputs to probabilities\n    probs = torch.sigmoid(bo[:,0])\n    print('probs', probs[:10])\n    \n#     Convert probabilities to predictions\n    preds = (probs > 0.5).int()\n    print('preds', preds[:10])\n    print('targets', bt[:10])\n    \n    #check metrics\n    print('accuracy', accuracy_score(bt, preds))\n    print('f1_score', f1_score(bt, preds))\n    \n    # print Loss\n    print('Loss', F.binary_cross_entropy(probs.float(), bt))\n    break","metadata":{"execution":{"iopub.status.busy":"2023-12-26T06:51:15.278141Z","iopub.execute_input":"2023-12-26T06:51:15.278422Z","iopub.status.idle":"2023-12-26T06:51:15.362890Z","shell.execute_reply.started":"2023-12-26T06:51:15.278398Z","shell.execute_reply":"2023-12-26T06:51:15.361966Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Evaluate model performance\ndef evaluate(model, dl):\n    \n    losses, accs, f1s = [], [], []\n#     Loop over batches\n    for batch in dl:\n        \n        # Get inputs and targets\n        inputs, targets = batch\n        \n        # Pass inputs through the model\n        outputs = model(inputs)\n        \n        #Convert to probabilities\n        probs = torch.sigmoid(outputs[:,0])\n        \n        #Compute loss\n        loss = F.binary_cross_entropy(probs, targets.float(),\n                                     weight = torch.tensor(20))\n        \n        #compute preds\n        preds = (probs>0.5).int()\n        \n        #compute accuracy and f1 score\n        acc = accuracy_score(targets, preds)\n        f1 = f1_score(targets, preds)\n        \n        losses.append(loss.item())\n        accs.append(acc)\n        f1s.append(f1)\n        \n        return (torch.mean(torch.tensor(losses)).item(), \n                torch.mean(torch.tensor(accs)).item(), \n                torch.mean(torch.tensor(f1s)).item())","metadata":{"execution":{"iopub.status.busy":"2023-12-26T06:51:15.364095Z","iopub.execute_input":"2023-12-26T06:51:15.364377Z","iopub.status.idle":"2023-12-26T06:51:15.371991Z","shell.execute_reply.started":"2023-12-26T06:51:15.364352Z","shell.execute_reply":"2023-12-26T06:51:15.371091Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Train the model batch by batch\ndef fit(epochs, lr, model, train_dl, val_dl):\n    \n    history = []\n\n    optimizer = torch.optim.Adam(model.parameters(), lr,weight_decay = 1e-5)\n    \n    for epoch in range(epochs):\n        # Training Phase\n        for batch in train_dl:\n            inputs, targets = batch\n            outputs = model(inputs)\n            probs = torch.sigmoid(outputs[:,0])\n            loss = F.binary_cross_entropy(probs, targets.float(),\n                                         weight = torch.tensor(20))\n            loss.backward()\n            optimizer.step()\n            optimizer.zero_grad()\n            \n        #Evaluation phase\n        loss, acc, f1 = evaluate(model, val_dl)\n        print('Epoch {:}: Loss: {:.4f}: Accuracy {:.4f}: F1_score: {:.4f}'.format(\n        epoch+1, loss, acc, f1))\n        history.append([loss, acc, f1])\n        \n    return history","metadata":{"execution":{"iopub.status.busy":"2023-12-26T06:51:15.373217Z","iopub.execute_input":"2023-12-26T06:51:15.373834Z","iopub.status.idle":"2023-12-26T06:51:15.385841Z","shell.execute_reply.started":"2023-12-26T06:51:15.373795Z","shell.execute_reply":"2023-12-26T06:51:15.384977Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = QuoraNet()","metadata":{"execution":{"iopub.status.busy":"2023-12-26T06:51:15.386790Z","iopub.execute_input":"2023-12-26T06:51:15.387024Z","iopub.status.idle":"2023-12-26T06:51:15.401127Z","shell.execute_reply.started":"2023-12-26T06:51:15.387001Z","shell.execute_reply":"2023-12-26T06:51:15.400008Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history = []","metadata":{"execution":{"iopub.status.busy":"2023-12-26T06:51:15.402001Z","iopub.execute_input":"2023-12-26T06:51:15.402355Z","iopub.status.idle":"2023-12-26T06:51:15.406766Z","shell.execute_reply.started":"2023-12-26T06:51:15.402326Z","shell.execute_reply":"2023-12-26T06:51:15.405740Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history.append(evaluate(model, val_dl))\nhistory","metadata":{"execution":{"iopub.status.busy":"2023-12-26T06:51:15.407771Z","iopub.execute_input":"2023-12-26T06:51:15.408083Z","iopub.status.idle":"2023-12-26T06:51:15.423969Z","shell.execute_reply.started":"2023-12-26T06:51:15.408032Z","shell.execute_reply":"2023-12-26T06:51:15.423293Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history += fit(10, 0.001, model, train_dl, val_dl)","metadata":{"execution":{"iopub.status.busy":"2023-12-26T06:51:15.425124Z","iopub.execute_input":"2023-12-26T06:51:15.425793Z","iopub.status.idle":"2023-12-26T06:59:26.014637Z","shell.execute_reply.started":"2023-12-26T06:51:15.425769Z","shell.execute_reply":"2023-12-26T06:59:26.013414Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"losses = [item[0] for item in history]\naccs = [item[1] for item in history]\nf1s = [item[2] for item in history]","metadata":{"execution":{"iopub.status.busy":"2023-12-26T06:59:26.016222Z","iopub.execute_input":"2023-12-26T06:59:26.017445Z","iopub.status.idle":"2023-12-26T06:59:26.023433Z","shell.execute_reply.started":"2023-12-26T06:59:26.017408Z","shell.execute_reply":"2023-12-26T06:59:26.022263Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt","metadata":{"execution":{"iopub.status.busy":"2023-12-26T06:59:26.025050Z","iopub.execute_input":"2023-12-26T06:59:26.025455Z","iopub.status.idle":"2023-12-26T06:59:26.035187Z","shell.execute_reply.started":"2023-12-26T06:59:26.025423Z","shell.execute_reply":"2023-12-26T06:59:26.034229Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.plot(losses);\nplt.title('Loss')","metadata":{"execution":{"iopub.status.busy":"2023-12-26T06:59:26.036403Z","iopub.execute_input":"2023-12-26T06:59:26.036711Z","iopub.status.idle":"2023-12-26T06:59:26.262869Z","shell.execute_reply.started":"2023-12-26T06:59:26.036685Z","shell.execute_reply":"2023-12-26T06:59:26.260944Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.plot(f1s)\nplt.title('F1 Score')","metadata":{"execution":{"iopub.status.busy":"2023-12-26T06:59:26.264486Z","iopub.execute_input":"2023-12-26T06:59:26.264834Z","iopub.status.idle":"2023-12-26T06:59:26.471284Z","shell.execute_reply.started":"2023-12-26T06:59:26.264798Z","shell.execute_reply":"2023-12-26T06:59:26.470284Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Make predictions on sample data","metadata":{}},{"cell_type":"code","source":"small_df = raw_df.sample(20)\nsmall_df","metadata":{"execution":{"iopub.status.busy":"2023-12-26T06:59:26.473941Z","iopub.execute_input":"2023-12-26T06:59:26.474390Z","iopub.status.idle":"2023-12-26T06:59:26.525095Z","shell.execute_reply.started":"2023-12-26T06:59:26.474354Z","shell.execute_reply":"2023-12-26T06:59:26.523858Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def predict_df(df):\n    inputs = vectorizer.transform(df.question_text)\n    input_tensors = torch.tensor(inputs.toarray()).float()\n    outputs = model(input_tensors)\n    probs = torch.sigmoid(outputs[:,0])\n    preds= (probs > 0.5).int()\n    return preds","metadata":{"execution":{"iopub.status.busy":"2023-12-26T06:59:26.527934Z","iopub.execute_input":"2023-12-26T06:59:26.528892Z","iopub.status.idle":"2023-12-26T06:59:26.535881Z","shell.execute_reply.started":"2023-12-26T06:59:26.528849Z","shell.execute_reply":"2023-12-26T06:59:26.534518Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"small_df.target.values","metadata":{"execution":{"iopub.status.busy":"2023-12-26T06:59:26.537465Z","iopub.execute_input":"2023-12-26T06:59:26.537855Z","iopub.status.idle":"2023-12-26T06:59:26.547141Z","shell.execute_reply.started":"2023-12-26T06:59:26.537821Z","shell.execute_reply":"2023-12-26T06:59:26.545917Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predict_df(small_df)","metadata":{"execution":{"iopub.status.busy":"2023-12-26T06:59:26.548651Z","iopub.execute_input":"2023-12-26T06:59:26.549076Z","iopub.status.idle":"2023-12-26T06:59:26.569356Z","shell.execute_reply.started":"2023-12-26T06:59:26.549018Z","shell.execute_reply":"2023-12-26T06:59:26.568120Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def predict_text(text):\n    df = pd.DataFrame({'question_text': [text]})\n    inputs = vectorizer.transform(df.question_text)\n    input_tensors = torch.tensor(inputs.toarray()).float()\n    outputs = model(input_tensors)\n    probs = torch.sigmoid(outputs[:,0])\n    preds= (probs > 0.5).int()\n    return preds","metadata":{"execution":{"iopub.status.busy":"2023-12-26T06:59:26.570797Z","iopub.execute_input":"2023-12-26T06:59:26.571127Z","iopub.status.idle":"2023-12-26T06:59:26.577905Z","shell.execute_reply.started":"2023-12-26T06:59:26.571097Z","shell.execute_reply":"2023-12-26T06:59:26.576827Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predict_text('Why is Christmas so beautiful')","metadata":{"execution":{"iopub.status.busy":"2023-12-26T06:59:26.579448Z","iopub.execute_input":"2023-12-26T06:59:26.579738Z","iopub.status.idle":"2023-12-26T06:59:26.592405Z","shell.execute_reply.started":"2023-12-26T06:59:26.579711Z","shell.execute_reply":"2023-12-26T06:59:26.590700Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Generate Predictions for test","metadata":{}},{"cell_type":"code","source":"test_dl","metadata":{"execution":{"iopub.status.busy":"2023-12-26T06:59:26.593688Z","iopub.execute_input":"2023-12-26T06:59:26.593993Z","iopub.status.idle":"2023-12-26T06:59:26.600421Z","shell.execute_reply.started":"2023-12-26T06:59:26.593969Z","shell.execute_reply":"2023-12-26T06:59:26.599476Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def predict(model, dl):\n    all_preds = []\n    for batch in dl:\n        inputs, = batch\n        out = model(inputs)\n        probs = torch.sigmoid(out)[:,0]\n        preds = (probs > 0.5).int()\n        all_preds += list(preds.numpy())\n    return all_preds","metadata":{"execution":{"iopub.status.busy":"2023-12-26T06:59:26.601561Z","iopub.execute_input":"2023-12-26T06:59:26.601821Z","iopub.status.idle":"2023-12-26T06:59:26.609340Z","shell.execute_reply.started":"2023-12-26T06:59:26.601798Z","shell.execute_reply":"2023-12-26T06:59:26.608363Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_preds = predict(model, test_dl)","metadata":{"execution":{"iopub.status.busy":"2023-12-26T06:59:26.610611Z","iopub.execute_input":"2023-12-26T06:59:26.610875Z","iopub.status.idle":"2023-12-26T06:59:32.527197Z","shell.execute_reply.started":"2023-12-26T06:59:26.610852Z","shell.execute_reply":"2023-12-26T06:59:32.525786Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_preds[:20]","metadata":{"execution":{"iopub.status.busy":"2023-12-26T06:59:32.528566Z","iopub.execute_input":"2023-12-26T06:59:32.528849Z","iopub.status.idle":"2023-12-26T06:59:32.533984Z","shell.execute_reply.started":"2023-12-26T06:59:32.528824Z","shell.execute_reply":"2023-12-26T06:59:32.533157Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub_df.prediction = test_preds","metadata":{"execution":{"iopub.status.busy":"2023-12-26T06:59:32.534983Z","iopub.execute_input":"2023-12-26T06:59:32.535508Z","iopub.status.idle":"2023-12-26T06:59:34.220703Z","shell.execute_reply.started":"2023-12-26T06:59:32.535483Z","shell.execute_reply":"2023-12-26T06:59:34.219707Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub_df.to_csv('submission.csv', index=None)","metadata":{"execution":{"iopub.status.busy":"2023-12-26T06:59:34.221850Z","iopub.execute_input":"2023-12-26T06:59:34.222721Z","iopub.status.idle":"2023-12-26T06:59:34.649695Z","shell.execute_reply.started":"2023-12-26T06:59:34.222695Z","shell.execute_reply":"2023-12-26T06:59:34.648500Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!head submission.csv","metadata":{"execution":{"iopub.status.busy":"2023-12-26T06:59:34.651206Z","iopub.execute_input":"2023-12-26T06:59:34.651509Z","iopub.status.idle":"2023-12-26T06:59:35.211692Z","shell.execute_reply.started":"2023-12-26T06:59:34.651483Z","shell.execute_reply":"2023-12-26T06:59:35.210473Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}