{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":10737,"databundleVersionId":290346,"sourceType":"competition"}],"dockerImageVersionId":30635,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        \n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-02-05T05:53:52.438045Z","iopub.execute_input":"2024-02-05T05:53:52.438526Z","iopub.status.idle":"2024-02-05T05:53:52.907524Z","shell.execute_reply.started":"2024-02-05T05:53:52.438487Z","shell.execute_reply":"2024-02-05T05:53:52.905992Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Explore Dataset","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nraw_df=pd.read_csv('/kaggle/input/quora-insincere-questions-classification/train.csv')\nraw_df","metadata":{"execution":{"iopub.status.busy":"2024-02-05T05:53:54.086410Z","iopub.execute_input":"2024-02-05T05:53:54.087660Z","iopub.status.idle":"2024-02-05T05:53:59.499771Z","shell.execute_reply.started":"2024-02-05T05:53:54.087620Z","shell.execute_reply":"2024-02-05T05:53:59.498846Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df=pd.read_csv('/kaggle/input/quora-insincere-questions-classification/test.csv')\ntest_df","metadata":{"execution":{"iopub.status.busy":"2024-02-05T05:53:59.501592Z","iopub.execute_input":"2024-02-05T05:53:59.502124Z","iopub.status.idle":"2024-02-05T05:54:01.025342Z","shell.execute_reply.started":"2024-02-05T05:53:59.502089Z","shell.execute_reply":"2024-02-05T05:54:01.023935Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub_df=pd.read_csv('/kaggle/input/quora-insincere-questions-classification/sample_submission.csv')\nsub_df","metadata":{"execution":{"iopub.status.busy":"2024-02-05T05:54:01.027189Z","iopub.execute_input":"2024-02-05T05:54:01.027692Z","iopub.status.idle":"2024-02-05T05:54:01.471686Z","shell.execute_reply.started":"2024-02-05T05:54:01.027647Z","shell.execute_reply":"2024-02-05T05:54:01.470427Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Prepare Data for Training\nOutline:\n1. Convert text to TF-IDF Vectors\n2. Split training & validation set\n3. Convert to PyTorch tensors","metadata":{}},{"cell_type":"code","source":"import nltk\nfrom nltk.tokenize import word_tokenize\nfrom nltk.stem import SnowballStemmer\nfrom nltk.corpus import stopwords\nfrom sklearn.feature_extraction.text import TfidfVectorizer","metadata":{"execution":{"iopub.status.busy":"2024-02-05T05:54:01.474782Z","iopub.execute_input":"2024-02-05T05:54:01.475118Z","iopub.status.idle":"2024-02-05T05:54:02.924910Z","shell.execute_reply.started":"2024-02-05T05:54:01.475088Z","shell.execute_reply":"2024-02-05T05:54:02.923422Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"nltk.download('punkt')\n","metadata":{"execution":{"iopub.status.busy":"2024-02-05T05:54:02.926619Z","iopub.execute_input":"2024-02-05T05:54:02.926991Z","iopub.status.idle":"2024-02-05T05:54:03.101115Z","shell.execute_reply.started":"2024-02-05T05:54:02.926950Z","shell.execute_reply":"2024-02-05T05:54:03.099717Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"stemmer = SnowballStemmer(language='english')\n\ndef tokenize(text):\n    return [stemmer.stem(token) for token in word_tokenize(text)]","metadata":{"execution":{"iopub.status.busy":"2024-02-05T05:54:03.102995Z","iopub.execute_input":"2024-02-05T05:54:03.103383Z","iopub.status.idle":"2024-02-05T05:54:03.109511Z","shell.execute_reply.started":"2024-02-05T05:54:03.103349Z","shell.execute_reply":"2024-02-05T05:54:03.108002Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tokenize(\"Ain't nothin' (but a heartache)!\")","metadata":{"execution":{"iopub.status.busy":"2024-02-05T05:54:03.111546Z","iopub.execute_input":"2024-02-05T05:54:03.112043Z","iopub.status.idle":"2024-02-05T05:54:03.139196Z","shell.execute_reply.started":"2024-02-05T05:54:03.111997Z","shell.execute_reply":"2024-02-05T05:54:03.137737Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"nltk.download('stopwords')","metadata":{"execution":{"iopub.status.busy":"2024-02-05T05:54:03.141045Z","iopub.execute_input":"2024-02-05T05:54:03.141453Z","iopub.status.idle":"2024-02-05T05:54:03.155322Z","shell.execute_reply.started":"2024-02-05T05:54:03.141419Z","shell.execute_reply":"2024-02-05T05:54:03.154066Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"english_stopwords = stopwords.words('english')","metadata":{"execution":{"iopub.status.busy":"2024-02-05T05:54:07.461625Z","iopub.execute_input":"2024-02-05T05:54:07.462109Z","iopub.status.idle":"2024-02-05T05:54:07.470182Z","shell.execute_reply.started":"2024-02-05T05:54:07.462073Z","shell.execute_reply":"2024-02-05T05:54:07.468726Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"vectorizer = TfidfVectorizer(lowercase=True, \n                             tokenizer=None,\n                             stop_words=english_stopwords,\n                             max_features=1000)","metadata":{"execution":{"iopub.status.busy":"2024-02-05T05:54:10.091355Z","iopub.execute_input":"2024-02-05T05:54:10.091763Z","iopub.status.idle":"2024-02-05T05:54:10.098081Z","shell.execute_reply.started":"2024-02-05T05:54:10.091729Z","shell.execute_reply":"2024-02-05T05:54:10.096784Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nvectorizer.fit(sample_df.question_text)","metadata":{"execution":{"iopub.status.busy":"2024-02-05T05:54:26.012695Z","iopub.execute_input":"2024-02-05T05:54:26.013999Z","iopub.status.idle":"2024-02-05T05:54:52.913420Z","shell.execute_reply.started":"2024-02-05T05:54:26.013937Z","shell.execute_reply":"2024-02-05T05:54:52.912071Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if IS_KAGGLE:\n    sample_df = raw_df\nelse:\n    sample_df = raw_df.sample(100_000, random_state=42)","metadata":{"execution":{"iopub.status.busy":"2024-02-05T05:54:20.477866Z","iopub.execute_input":"2024-02-05T05:54:20.478446Z","iopub.status.idle":"2024-02-05T05:54:20.486758Z","shell.execute_reply.started":"2024-02-05T05:54:20.478390Z","shell.execute_reply":"2024-02-05T05:54:20.485174Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!ls","metadata":{"execution":{"iopub.status.busy":"2024-02-05T05:54:13.385829Z","iopub.execute_input":"2024-02-05T05:54:13.386258Z","iopub.status.idle":"2024-02-05T05:54:14.501581Z","shell.execute_reply.started":"2024-02-05T05:54:13.386207Z","shell.execute_reply":"2024-02-05T05:54:14.499864Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os","metadata":{"execution":{"iopub.status.busy":"2024-02-05T05:54:14.504614Z","iopub.execute_input":"2024-02-05T05:54:14.505050Z","iopub.status.idle":"2024-02-05T05:54:14.510607Z","shell.execute_reply.started":"2024-02-05T05:54:14.505007Z","shell.execute_reply":"2024-02-05T05:54:14.509497Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"IS_KAGGLE = 'KAGGLE_KERNEL_RUN_TYPE' in os.environ","metadata":{"execution":{"iopub.status.busy":"2024-02-05T05:54:14.511942Z","iopub.execute_input":"2024-02-05T05:54:14.512603Z","iopub.status.idle":"2024-02-05T05:54:14.524327Z","shell.execute_reply.started":"2024-02-05T05:54:14.512556Z","shell.execute_reply":"2024-02-05T05:54:14.523045Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ninputs = vectorizer.transform(sample_df.question_text)","metadata":{"execution":{"iopub.status.busy":"2024-02-05T05:54:52.916076Z","iopub.execute_input":"2024-02-05T05:54:52.916730Z","iopub.status.idle":"2024-02-05T05:55:18.760419Z","shell.execute_reply.started":"2024-02-05T05:54:52.916692Z","shell.execute_reply":"2024-02-05T05:55:18.758975Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"inputs.shape","metadata":{"execution":{"iopub.status.busy":"2024-02-05T05:55:27.777697Z","iopub.execute_input":"2024-02-05T05:55:27.778116Z","iopub.status.idle":"2024-02-05T05:55:27.786890Z","shell.execute_reply.started":"2024-02-05T05:55:27.778083Z","shell.execute_reply":"2024-02-05T05:55:27.785363Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"targets = sample_df.target.values","metadata":{"execution":{"iopub.status.busy":"2024-02-05T05:55:28.139794Z","iopub.execute_input":"2024-02-05T05:55:28.140318Z","iopub.status.idle":"2024-02-05T05:55:28.146235Z","shell.execute_reply.started":"2024-02-05T05:55:28.140271Z","shell.execute_reply":"2024-02-05T05:55:28.145032Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"targets.shape","metadata":{"execution":{"iopub.status.busy":"2024-02-05T05:55:28.922385Z","iopub.execute_input":"2024-02-05T05:55:28.922787Z","iopub.status.idle":"2024-02-05T05:55:28.930596Z","shell.execute_reply.started":"2024-02-05T05:55:28.922753Z","shell.execute_reply":"2024-02-05T05:55:28.929446Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ntest_inputs = vectorizer.transform(test_df.question_text)","metadata":{"execution":{"iopub.status.busy":"2024-02-05T05:55:29.327939Z","iopub.execute_input":"2024-02-05T05:55:29.328362Z","iopub.status.idle":"2024-02-05T05:55:36.736111Z","shell.execute_reply.started":"2024-02-05T05:55:29.328328Z","shell.execute_reply":"2024-02-05T05:55:36.734912Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"##  Split training and validation set","metadata":{}},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split","metadata":{"execution":{"iopub.status.busy":"2024-02-05T05:55:36.738726Z","iopub.execute_input":"2024-02-05T05:55:36.739162Z","iopub.status.idle":"2024-02-05T05:55:36.743777Z","shell.execute_reply.started":"2024-02-05T05:55:36.739129Z","shell.execute_reply":"2024-02-05T05:55:36.742911Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_inputs, val_inputs, train_targets, val_targets = train_test_split(inputs, targets, test_size=0.3, random_state=42)","metadata":{"execution":{"iopub.status.busy":"2024-02-05T05:55:36.745027Z","iopub.execute_input":"2024-02-05T05:55:36.746078Z","iopub.status.idle":"2024-02-05T05:55:37.004538Z","shell.execute_reply.started":"2024-02-05T05:55:36.746043Z","shell.execute_reply":"2024-02-05T05:55:37.002750Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_inputs.shape, val_inputs.shape","metadata":{"execution":{"iopub.status.busy":"2024-02-05T05:55:37.008074Z","iopub.execute_input":"2024-02-05T05:55:37.008679Z","iopub.status.idle":"2024-02-05T05:55:37.017555Z","shell.execute_reply.started":"2024-02-05T05:55:37.008633Z","shell.execute_reply":"2024-02-05T05:55:37.016264Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_targets.shape","metadata":{"execution":{"iopub.status.busy":"2024-02-05T05:55:37.019647Z","iopub.execute_input":"2024-02-05T05:55:37.020789Z","iopub.status.idle":"2024-02-05T05:55:37.030331Z","shell.execute_reply.started":"2024-02-05T05:55:37.020742Z","shell.execute_reply":"2024-02-05T05:55:37.028743Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Convert to PyTorch Tensors","metadata":{}},{"cell_type":"code","source":"import torch\nfrom torch.utils.data import TensorDataset, DataLoader\nimport torch.nn.functional as F","metadata":{"execution":{"iopub.status.busy":"2024-02-05T05:55:37.034424Z","iopub.execute_input":"2024-02-05T05:55:37.034821Z","iopub.status.idle":"2024-02-05T05:55:40.203649Z","shell.execute_reply.started":"2024-02-05T05:55:37.034791Z","shell.execute_reply":"2024-02-05T05:55:40.202139Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_tensors = F.normalize(torch.tensor(train_inputs.toarray()).float(), dim=0)\nval_tensors = F.normalize(torch.tensor(val_inputs.toarray()).float(), dim=0)","metadata":{"execution":{"iopub.status.busy":"2024-02-05T05:55:40.205852Z","iopub.execute_input":"2024-02-05T05:55:40.207266Z","iopub.status.idle":"2024-02-05T05:56:19.132900Z","shell.execute_reply.started":"2024-02-05T05:55:40.207151Z","shell.execute_reply":"2024-02-05T05:56:19.131663Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_tensors.shape, val_tensors.shape","metadata":{"execution":{"iopub.status.busy":"2024-02-05T05:56:19.134418Z","iopub.execute_input":"2024-02-05T05:56:19.134886Z","iopub.status.idle":"2024-02-05T05:56:19.144416Z","shell.execute_reply.started":"2024-02-05T05:56:19.134840Z","shell.execute_reply":"2024-02-05T05:56:19.142895Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_ds = TensorDataset(train_tensors, torch.tensor(train_targets))\nval_ds = TensorDataset(val_tensors, torch.tensor(val_targets))","metadata":{"execution":{"iopub.status.busy":"2024-02-05T05:56:19.146660Z","iopub.execute_input":"2024-02-05T05:56:19.147113Z","iopub.status.idle":"2024-02-05T05:56:19.160281Z","shell.execute_reply.started":"2024-02-05T05:56:19.147070Z","shell.execute_reply":"2024-02-05T05:56:19.158876Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"batch_size = 128\ntrain_dl = DataLoader(train_ds, batch_size, shuffle=True)\nval_dl = DataLoader(val_ds, batch_size)","metadata":{"execution":{"iopub.status.busy":"2024-02-05T05:56:19.166191Z","iopub.execute_input":"2024-02-05T05:56:19.166713Z","iopub.status.idle":"2024-02-05T05:56:19.174234Z","shell.execute_reply.started":"2024-02-05T05:56:19.166679Z","shell.execute_reply":"2024-02-05T05:56:19.172628Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for inputs_batch, targets_batch in train_dl:\n    print('inputs.shape', inputs_batch.shape)\n    print('targets.shape', targets_batch.shape)\n    print(targets_batch)\n    break","metadata":{"execution":{"iopub.status.busy":"2024-02-05T05:56:19.176213Z","iopub.execute_input":"2024-02-05T05:56:19.176752Z","iopub.status.idle":"2024-02-05T05:56:19.334083Z","shell.execute_reply.started":"2024-02-05T05:56:19.176708Z","shell.execute_reply":"2024-02-05T05:56:19.332796Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Logistic Regression Model","metadata":{}},{"cell_type":"code","source":"import torch.nn as nn","metadata":{"execution":{"iopub.status.busy":"2024-02-05T05:56:19.335629Z","iopub.execute_input":"2024-02-05T05:56:19.336022Z","iopub.status.idle":"2024-02-05T05:56:19.342115Z","shell.execute_reply.started":"2024-02-05T05:56:19.335923Z","shell.execute_reply":"2024-02-05T05:56:19.340638Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class LogReg(nn.Module):\n    def __init__(self):\n        super().__init__()\n        self.linear1 = nn.Linear(1000, 1)\n        \n    def forward(self, xb):\n        out = self.linear1(xb)\n        return out","metadata":{"execution":{"iopub.status.busy":"2024-02-05T05:56:19.343729Z","iopub.execute_input":"2024-02-05T05:56:19.344077Z","iopub.status.idle":"2024-02-05T05:56:19.354184Z","shell.execute_reply.started":"2024-02-05T05:56:19.344047Z","shell.execute_reply":"2024-02-05T05:56:19.352619Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nfrom sklearn.metrics import accuracy_score, f1_score","metadata":{"execution":{"iopub.status.busy":"2024-02-05T05:56:19.355780Z","iopub.execute_input":"2024-02-05T05:56:19.356109Z","iopub.status.idle":"2024-02-05T05:56:19.364422Z","shell.execute_reply.started":"2024-02-05T05:56:19.356080Z","shell.execute_reply":"2024-02-05T05:56:19.363169Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"logreg_model = LogReg()","metadata":{"execution":{"iopub.status.busy":"2024-02-05T05:56:19.365882Z","iopub.execute_input":"2024-02-05T05:56:19.366192Z","iopub.status.idle":"2024-02-05T05:56:19.376993Z","shell.execute_reply.started":"2024-02-05T05:56:19.366163Z","shell.execute_reply":"2024-02-05T05:56:19.375719Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for batch in val_dl:\n    batch_inputs, batch_targets = batch\n    print('inputs.shape', batch_inputs.shape)\n    print('targets', batch_targets)\n    \n    batch_out = logreg_model(batch_inputs)\n    probs = torch.sigmoid(batch_out[:,0])\n    preds = (probs >= 0.5).int()\n    \n    print('outputs', preds)\n    print('accuracy', accuracy_score(batch_targets, preds))\n    print('f1_score', f1_score(batch_targets, preds))\n    break","metadata":{"execution":{"iopub.status.busy":"2024-02-05T05:56:19.378561Z","iopub.execute_input":"2024-02-05T05:56:19.378912Z","iopub.status.idle":"2024-02-05T05:56:19.424861Z","shell.execute_reply.started":"2024-02-05T05:56:19.378880Z","shell.execute_reply":"2024-02-05T05:56:19.423511Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def evaluate(model, dl):\n    losses, accs, f1s = [], [], []\n    for batch in dl:\n        inputs, targets = batch\n        out = model(inputs)\n        \n        probs = torch.sigmoid(out[:,0])\n        loss = F.binary_cross_entropy(probs, targets.float(), weight=torch.tensor(20.))\n        losses.append(loss.item())\n\n        preds = (probs > 0.5).int()\n        acc = accuracy_score(targets, preds)\n        f1 = f1_score(targets, preds)\n        \n        accs.append(acc)\n        f1s.append(f1)\n\n    return np.mean(losses), np.mean(accs), np.mean(f1s)","metadata":{"execution":{"iopub.status.busy":"2024-02-05T05:56:19.426827Z","iopub.execute_input":"2024-02-05T05:56:19.429568Z","iopub.status.idle":"2024-02-05T05:56:19.439429Z","shell.execute_reply.started":"2024-02-05T05:56:19.429515Z","shell.execute_reply":"2024-02-05T05:56:19.437725Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def fit(epochs, lr, model, train_loader, val_loader):\n    optimizer = torch.optim.Adam(model.parameters(), lr, weight_decay=1e-5)\n    history = [] # for recording epoch-wise results\n    \n    for epoch in range(epochs):\n        \n        # Training Phase \n        for batch in train_loader:\n            inputs, targets = batch\n            out = model(inputs)\n            probs = torch.sigmoid(out[:,0])\n            loss = F.binary_cross_entropy(probs, \n                                          targets.float(), \n                                          weight=torch.tensor(20.))\n            loss.backward()\n            optimizer.step()\n            optimizer.zero_grad()\n        \n        # Validation phase\n        result = evaluate(model, val_loader)\n        loss, acc, f1 = result\n        print('Epoch: {}; Loss: {:.4f}; Accuracy: {:.4f}; F1 Score: {:.4f}'.format(\n            epoch, loss, acc, f1))\n        history.append(result)\n\n    return history","metadata":{"execution":{"iopub.status.busy":"2024-02-05T05:56:19.441190Z","iopub.execute_input":"2024-02-05T05:56:19.441682Z","iopub.status.idle":"2024-02-05T05:56:19.453296Z","shell.execute_reply.started":"2024-02-05T05:56:19.441642Z","shell.execute_reply":"2024-02-05T05:56:19.451752Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"logreg_model = LogReg()","metadata":{"execution":{"iopub.status.busy":"2024-02-05T05:56:19.455047Z","iopub.execute_input":"2024-02-05T05:56:19.455436Z","iopub.status.idle":"2024-02-05T05:56:19.462504Z","shell.execute_reply.started":"2024-02-05T05:56:19.455393Z","shell.execute_reply":"2024-02-05T05:56:19.461286Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history = [evaluate(logreg_model, val_dl)]","metadata":{"execution":{"iopub.status.busy":"2024-02-05T05:56:19.464415Z","iopub.execute_input":"2024-02-05T05:56:19.464788Z","iopub.status.idle":"2024-02-05T05:56:32.570723Z","shell.execute_reply.started":"2024-02-05T05:56:19.464756Z","shell.execute_reply":"2024-02-05T05:56:32.569699Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history","metadata":{"execution":{"iopub.status.busy":"2024-02-05T05:56:32.572352Z","iopub.execute_input":"2024-02-05T05:56:32.573206Z","iopub.status.idle":"2024-02-05T05:56:32.580794Z","shell.execute_reply.started":"2024-02-05T05:56:32.573161Z","shell.execute_reply":"2024-02-05T05:56:32.579682Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history += fit(5, 0.01, logreg_model, train_dl, val_dl)","metadata":{"execution":{"iopub.status.busy":"2024-02-05T05:59:47.439775Z","iopub.execute_input":"2024-02-05T05:59:47.440289Z","iopub.status.idle":"2024-02-05T06:02:11.755860Z","shell.execute_reply.started":"2024-02-05T05:59:47.440225Z","shell.execute_reply":"2024-02-05T06:02:11.754507Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history += fit(5, 0.01, logreg_model, train_dl, val_dl)","metadata":{"execution":{"iopub.status.busy":"2024-02-05T06:03:57.617271Z","iopub.execute_input":"2024-02-05T06:03:57.617760Z","iopub.status.idle":"2024-02-05T06:06:25.681828Z","shell.execute_reply.started":"2024-02-05T06:03:57.617726Z","shell.execute_reply":"2024-02-05T06:06:25.680371Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history += fit(5, 0.01, logreg_model, train_dl, val_dl)","metadata":{"execution":{"iopub.status.busy":"2024-02-05T06:06:51.741990Z","iopub.execute_input":"2024-02-05T06:06:51.742465Z","iopub.status.idle":"2024-02-05T06:09:17.393165Z","shell.execute_reply.started":"2024-02-05T06:06:51.742417Z","shell.execute_reply":"2024-02-05T06:09:17.391832Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"losses = [item[0] for item in history]","metadata":{"execution":{"iopub.status.busy":"2024-02-05T06:15:20.456820Z","iopub.execute_input":"2024-02-05T06:15:20.457284Z","iopub.status.idle":"2024-02-05T06:15:20.463814Z","shell.execute_reply.started":"2024-02-05T06:15:20.457233Z","shell.execute_reply":"2024-02-05T06:15:20.462474Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt","metadata":{"execution":{"iopub.status.busy":"2024-02-05T06:15:20.842594Z","iopub.execute_input":"2024-02-05T06:15:20.843019Z","iopub.status.idle":"2024-02-05T06:15:20.849269Z","shell.execute_reply.started":"2024-02-05T06:15:20.842977Z","shell.execute_reply":"2024-02-05T06:15:20.847935Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.plot(losses);\nplt.title('Loss')","metadata":{"execution":{"iopub.status.busy":"2024-02-05T06:15:21.322483Z","iopub.execute_input":"2024-02-05T06:15:21.323018Z","iopub.status.idle":"2024-02-05T06:15:21.715205Z","shell.execute_reply.started":"2024-02-05T06:15:21.322978Z","shell.execute_reply":"2024-02-05T06:15:21.714258Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"f1s = [item[2] for item in history]","metadata":{"execution":{"iopub.status.busy":"2024-02-05T06:15:21.894193Z","iopub.execute_input":"2024-02-05T06:15:21.895017Z","iopub.status.idle":"2024-02-05T06:15:21.901551Z","shell.execute_reply.started":"2024-02-05T06:15:21.894977Z","shell.execute_reply":"2024-02-05T06:15:21.900728Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.plot(f1s)\nplt.title('F1 Score')","metadata":{"execution":{"iopub.status.busy":"2024-02-05T06:15:22.119936Z","iopub.execute_input":"2024-02-05T06:15:22.120380Z","iopub.status.idle":"2024-02-05T06:15:22.390694Z","shell.execute_reply.started":"2024-02-05T06:15:22.120344Z","shell.execute_reply":"2024-02-05T06:15:22.389718Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Feed Forward Neural Network","metadata":{}},{"cell_type":"code","source":"import torch.nn.functional as F","metadata":{"execution":{"iopub.status.busy":"2024-02-05T06:15:22.727692Z","iopub.execute_input":"2024-02-05T06:15:22.728198Z","iopub.status.idle":"2024-02-05T06:15:22.734978Z","shell.execute_reply.started":"2024-02-05T06:15:22.728153Z","shell.execute_reply":"2024-02-05T06:15:22.733101Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class FeedForwardModel(nn.Module):\n    def __init__(self):\n        super().__init__()\n        self.linear1 = nn.Linear(1000, 512)\n        self.linear2 = nn.Linear(512, 256)\n        self.linear3 = nn.Linear(256, 128)\n        self.linear4 = nn.Linear(128, 1)\n        \n    def forward(self, xb):\n        out = F.relu(self.linear1(xb))\n        out = F.relu(self.linear2(out))\n        out = F.relu(self.linear3(out))\n        out = self.linear4(out)\n        return out","metadata":{"execution":{"iopub.status.busy":"2024-02-05T06:15:23.317913Z","iopub.execute_input":"2024-02-05T06:15:23.318401Z","iopub.status.idle":"2024-02-05T06:15:23.327587Z","shell.execute_reply.started":"2024-02-05T06:15:23.318365Z","shell.execute_reply":"2024-02-05T06:15:23.326292Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ff_model = FeedForwardModel()","metadata":{"execution":{"iopub.status.busy":"2024-02-05T06:15:23.729046Z","iopub.execute_input":"2024-02-05T06:15:23.730215Z","iopub.status.idle":"2024-02-05T06:15:23.745852Z","shell.execute_reply.started":"2024-02-05T06:15:23.730172Z","shell.execute_reply":"2024-02-05T06:15:23.744318Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history = [evaluate(ff_model, val_dl)]","metadata":{"execution":{"iopub.status.busy":"2024-02-05T06:15:24.231771Z","iopub.execute_input":"2024-02-05T06:15:24.232288Z","iopub.status.idle":"2024-02-05T06:15:45.529061Z","shell.execute_reply.started":"2024-02-05T06:15:24.232222Z","shell.execute_reply":"2024-02-05T06:15:45.527827Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history","metadata":{"execution":{"iopub.status.busy":"2024-02-05T06:15:45.532025Z","iopub.execute_input":"2024-02-05T06:15:45.532541Z","iopub.status.idle":"2024-02-05T06:15:45.540912Z","shell.execute_reply.started":"2024-02-05T06:15:45.532494Z","shell.execute_reply":"2024-02-05T06:15:45.539633Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nhistory += fit(5, 0.001, ff_model, train_dl, val_dl)","metadata":{"execution":{"iopub.status.busy":"2024-02-05T06:15:45.542842Z","iopub.execute_input":"2024-02-05T06:15:45.543566Z","iopub.status.idle":"2024-02-05T07:00:02.360541Z","shell.execute_reply.started":"2024-02-05T06:15:45.543522Z","shell.execute_reply":"2024-02-05T07:00:02.359138Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Make Predictions and Submit","metadata":{}},{"cell_type":"code","source":"test_tensors = torch.tensor(test_inputs.toarray()).float()","metadata":{"execution":{"iopub.status.busy":"2024-02-05T07:07:39.551825Z","iopub.execute_input":"2024-02-05T07:07:39.552357Z","iopub.status.idle":"2024-02-05T07:07:46.186386Z","shell.execute_reply.started":"2024-02-05T07:07:39.552316Z","shell.execute_reply":"2024-02-05T07:07:46.185112Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_ds = TensorDataset(test_tensors)","metadata":{"execution":{"iopub.status.busy":"2024-02-05T07:07:50.629293Z","iopub.execute_input":"2024-02-05T07:07:50.630042Z","iopub.status.idle":"2024-02-05T07:07:50.635200Z","shell.execute_reply.started":"2024-02-05T07:07:50.630000Z","shell.execute_reply":"2024-02-05T07:07:50.633606Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_dl = DataLoader(test_ds, batch_size)","metadata":{"execution":{"iopub.status.busy":"2024-02-05T07:07:51.533122Z","iopub.execute_input":"2024-02-05T07:07:51.533561Z","iopub.status.idle":"2024-02-05T07:07:51.540682Z","shell.execute_reply.started":"2024-02-05T07:07:51.533527Z","shell.execute_reply":"2024-02-05T07:07:51.539351Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def predict(model, dl):\n    all_preds = []\n    for batch in dl:\n        inputs, = batch\n        out = model(inputs)\n        probs = torch.sigmoid(out)[:,0]\n        preds = (probs > 0.5).int()\n        all_preds += list(preds.numpy())\n    return all_preds","metadata":{"execution":{"iopub.status.busy":"2024-02-05T07:07:52.727925Z","iopub.execute_input":"2024-02-05T07:07:52.728412Z","iopub.status.idle":"2024-02-05T07:07:52.735835Z","shell.execute_reply.started":"2024-02-05T07:07:52.728375Z","shell.execute_reply":"2024-02-05T07:07:52.734202Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_preds = predict(ff_model, test_dl)","metadata":{"execution":{"iopub.status.busy":"2024-02-05T07:07:53.644395Z","iopub.execute_input":"2024-02-05T07:07:53.644797Z","iopub.status.idle":"2024-02-05T07:08:42.521508Z","shell.execute_reply.started":"2024-02-05T07:07:53.644766Z","shell.execute_reply":"2024-02-05T07:08:42.520384Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_preds[:20]","metadata":{"execution":{"iopub.status.busy":"2024-02-05T07:09:21.240982Z","iopub.execute_input":"2024-02-05T07:09:21.241776Z","iopub.status.idle":"2024-02-05T07:09:21.251776Z","shell.execute_reply.started":"2024-02-05T07:09:21.241703Z","shell.execute_reply":"2024-02-05T07:09:21.250315Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub_df","metadata":{"execution":{"iopub.status.busy":"2024-02-05T07:09:22.155571Z","iopub.execute_input":"2024-02-05T07:09:22.156025Z","iopub.status.idle":"2024-02-05T07:09:22.172887Z","shell.execute_reply.started":"2024-02-05T07:09:22.155991Z","shell.execute_reply":"2024-02-05T07:09:22.171595Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub_df.prediction = test_preds","metadata":{"execution":{"iopub.status.busy":"2024-02-05T07:09:23.139079Z","iopub.execute_input":"2024-02-05T07:09:23.139994Z","iopub.status.idle":"2024-02-05T07:09:27.064898Z","shell.execute_reply.started":"2024-02-05T07:09:23.139957Z","shell.execute_reply":"2024-02-05T07:09:27.063691Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub_df.to_csv('submission.csv', index=None)","metadata":{"execution":{"iopub.status.busy":"2024-02-05T07:09:30.991747Z","iopub.execute_input":"2024-02-05T07:09:30.992777Z","iopub.status.idle":"2024-02-05T07:09:32.143921Z","shell.execute_reply.started":"2024-02-05T07:09:30.992734Z","shell.execute_reply":"2024-02-05T07:09:32.142626Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!head submission.csv","metadata":{"execution":{"iopub.status.busy":"2024-02-05T07:09:33.225875Z","iopub.execute_input":"2024-02-05T07:09:33.226305Z","iopub.status.idle":"2024-02-05T07:09:34.739009Z","shell.execute_reply.started":"2024-02-05T07:09:33.226272Z","shell.execute_reply":"2024-02-05T07:09:34.737009Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}