{"metadata":{"kernelspec":{"name":"python3","display_name":"Python 3","language":"python"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.10.14"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":10737,"databundleVersionId":290346,"sourceType":"competition"}],"dockerImageVersionId":30786,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Libraries","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nplt.rc('figure', figsize=(6,4))\n%matplotlib inline\n\nimport nltk\nfrom nltk.tokenize import word_tokenize\nfrom nltk.stem import SnowballStemmer\nfrom nltk.corpus import stopwords","metadata":{"execution":{"iopub.status.busy":"2024-10-10T08:01:23.990745Z","iopub.execute_input":"2024-10-10T08:01:23.991404Z","iopub.status.idle":"2024-10-10T08:01:26.858954Z","shell.execute_reply.started":"2024-10-10T08:01:23.991331Z","shell.execute_reply":"2024-10-10T08:01:26.857568Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.feature_extraction.text import TfidfVectorizer\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import f1_score, accuracy_score","metadata":{"execution":{"iopub.status.busy":"2024-10-10T08:01:26.861767Z","iopub.execute_input":"2024-10-10T08:01:26.862843Z","iopub.status.idle":"2024-10-10T08:01:26.869804Z","shell.execute_reply.started":"2024-10-10T08:01:26.862762Z","shell.execute_reply":"2024-10-10T08:01:26.868257Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nimport torch.nn as nn \nimport torch.nn.functional as F\nfrom torch.utils.data import DataLoader, TensorDataset","metadata":{"execution":{"iopub.status.busy":"2024-10-10T08:01:26.872134Z","iopub.execute_input":"2024-10-10T08:01:26.872724Z","iopub.status.idle":"2024-10-10T08:01:29.351941Z","shell.execute_reply.started":"2024-10-10T08:01:26.872659Z","shell.execute_reply":"2024-10-10T08:01:29.349954Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_dir = '../input/quora-insincere-questions-classification'\ntrain_fname= data_dir + '/train.csv'\ntest_fname= data_dir + '/test.csv'\nsub_fname= data_dir + '/sample_submission.csv'","metadata":{"execution":{"iopub.status.busy":"2024-10-10T08:01:29.354081Z","iopub.execute_input":"2024-10-10T08:01:29.354923Z","iopub.status.idle":"2024-10-10T08:01:29.362424Z","shell.execute_reply.started":"2024-10-10T08:01:29.354861Z","shell.execute_reply":"2024-10-10T08:01:29.360795Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Analysis and Loading","metadata":{}},{"cell_type":"code","source":"raw_df=pd.read_csv(train_fname)\ntest_df=pd.read_csv(test_fname)","metadata":{"execution":{"iopub.status.busy":"2024-10-10T08:01:29.366248Z","iopub.execute_input":"2024-10-10T08:01:29.366795Z","iopub.status.idle":"2024-10-10T08:01:37.259573Z","shell.execute_reply.started":"2024-10-10T08:01:29.366743Z","shell.execute_reply":"2024-10-10T08:01:37.257688Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"raw_df.question_text.sample(10).values[:5]","metadata":{"execution":{"iopub.status.busy":"2024-10-10T08:01:37.261747Z","iopub.execute_input":"2024-10-10T08:01:37.262259Z","iopub.status.idle":"2024-10-10T08:01:37.338991Z","shell.execute_reply.started":"2024-10-10T08:01:37.262210Z","shell.execute_reply":"2024-10-10T08:01:37.337642Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df.question_text.sample(10).values[:5]","metadata":{"execution":{"iopub.status.busy":"2024-10-10T08:01:37.340614Z","iopub.execute_input":"2024-10-10T08:01:37.341070Z","iopub.status.idle":"2024-10-10T08:01:37.364206Z","shell.execute_reply.started":"2024-10-10T08:01:37.341025Z","shell.execute_reply":"2024-10-10T08:01:37.362882Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_size=len(raw_df)\nsample_df=raw_df.sample(sample_size)","metadata":{"execution":{"iopub.status.busy":"2024-10-10T08:01:37.365595Z","iopub.execute_input":"2024-10-10T08:01:37.366007Z","iopub.status.idle":"2024-10-10T08:01:37.868704Z","shell.execute_reply.started":"2024-10-10T08:01:37.365965Z","shell.execute_reply":"2024-10-10T08:01:37.867221Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_df.question_text.sample(10).values[:5]","metadata":{"execution":{"iopub.status.busy":"2024-10-10T08:01:37.870271Z","iopub.execute_input":"2024-10-10T08:01:37.870715Z","iopub.status.idle":"2024-10-10T08:01:37.935246Z","shell.execute_reply.started":"2024-10-10T08:01:37.870670Z","shell.execute_reply":"2024-10-10T08:01:37.933572Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_df.target.value_counts(normalize=True)","metadata":{"execution":{"iopub.status.busy":"2024-10-10T08:01:37.937101Z","iopub.execute_input":"2024-10-10T08:01:37.937655Z","iopub.status.idle":"2024-10-10T08:01:37.976756Z","shell.execute_reply.started":"2024-10-10T08:01:37.937598Z","shell.execute_reply":"2024-10-10T08:01:37.975366Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Preprocessing","metadata":{}},{"cell_type":"markdown","source":"## TF-IDF Vectorization","metadata":{}},{"cell_type":"code","source":"word_tokenize(\"testing, a lot, of punctuations!!\")","metadata":{"execution":{"iopub.status.busy":"2024-10-10T08:01:37.978422Z","iopub.execute_input":"2024-10-10T08:01:37.978988Z","iopub.status.idle":"2024-10-10T08:01:38.013271Z","shell.execute_reply.started":"2024-10-10T08:01:37.978921Z","shell.execute_reply":"2024-10-10T08:01:38.011521Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"stemmer= SnowballStemmer(language='english')","metadata":{"execution":{"iopub.status.busy":"2024-10-10T08:01:38.015300Z","iopub.execute_input":"2024-10-10T08:01:38.015943Z","iopub.status.idle":"2024-10-10T08:01:38.022688Z","shell.execute_reply.started":"2024-10-10T08:01:38.015868Z","shell.execute_reply":"2024-10-10T08:01:38.020942Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"stemmer.stem(\"butterflies\"), stemmer.stem(\"doing\"), stemmer.stem(\"really\")","metadata":{"execution":{"iopub.status.busy":"2024-10-10T08:01:38.024802Z","iopub.execute_input":"2024-10-10T08:01:38.025487Z","iopub.status.idle":"2024-10-10T08:01:38.040929Z","shell.execute_reply.started":"2024-10-10T08:01:38.025420Z","shell.execute_reply":"2024-10-10T08:01:38.039382Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def tokenize(text):\n    return [stemmer.stem(token) for token in word_tokenize(text)]","metadata":{"execution":{"iopub.status.busy":"2024-10-10T08:01:38.050149Z","iopub.execute_input":"2024-10-10T08:01:38.050718Z","iopub.status.idle":"2024-10-10T08:01:38.058681Z","shell.execute_reply.started":"2024-10-10T08:01:38.050673Z","shell.execute_reply":"2024-10-10T08:01:38.056652Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tokenize(\"testing tokenizer, really good enough!\")","metadata":{"execution":{"iopub.status.busy":"2024-10-10T08:01:38.060693Z","iopub.execute_input":"2024-10-10T08:01:38.061193Z","iopub.status.idle":"2024-10-10T08:01:38.078571Z","shell.execute_reply.started":"2024-10-10T08:01:38.061145Z","shell.execute_reply":"2024-10-10T08:01:38.076707Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"english_stopwords= stopwords.words('english')","metadata":{"execution":{"iopub.status.busy":"2024-10-10T08:01:38.080502Z","iopub.execute_input":"2024-10-10T08:01:38.081038Z","iopub.status.idle":"2024-10-10T08:01:38.104731Z","shell.execute_reply.started":"2024-10-10T08:01:38.080979Z","shell.execute_reply":"2024-10-10T08:01:38.102906Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"vectorizer= TfidfVectorizer(tokenizer=tokenize,\n                            stop_words=english_stopwords, max_features=1024)","metadata":{"execution":{"iopub.status.busy":"2024-10-10T10:04:32.208282Z","iopub.execute_input":"2024-10-10T10:04:32.208804Z","iopub.status.idle":"2024-10-10T10:04:32.216042Z","shell.execute_reply.started":"2024-10-10T10:04:32.208761Z","shell.execute_reply":"2024-10-10T10:04:32.214170Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nvectorizer.fit(sample_df.question_text)","metadata":{"execution":{"iopub.status.busy":"2024-10-10T10:04:33.418273Z","iopub.execute_input":"2024-10-10T10:04:33.418772Z","iopub.status.idle":"2024-10-10T10:13:21.051771Z","shell.execute_reply.started":"2024-10-10T10:04:33.418726Z","shell.execute_reply":"2024-10-10T10:13:21.049330Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"vectorizer.get_feature_names_out()[:50]","metadata":{"execution":{"iopub.status.busy":"2024-10-10T10:13:21.056334Z","iopub.execute_input":"2024-10-10T10:13:21.058568Z","iopub.status.idle":"2024-10-10T10:13:21.071865Z","shell.execute_reply.started":"2024-10-10T10:13:21.058489Z","shell.execute_reply":"2024-10-10T10:13:21.070105Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Transforming questions to vectors","metadata":{}},{"cell_type":"code","source":"%%time\ninput = vectorizer.transform(sample_df['question_text'])","metadata":{"execution":{"iopub.status.busy":"2024-10-10T10:13:21.075064Z","iopub.execute_input":"2024-10-10T10:13:21.075674Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ntest_input = vectorizer.transform(test_df['question_text'])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Preparation for training","metadata":{}},{"cell_type":"code","source":"train_df, val_df, train_target, val_target= train_test_split(input, sample_df['target'], test_size=0.2)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Converting to Pytorch Tensor","metadata":{}},{"cell_type":"code","source":"train_input_tensor= torch.tensor(train_df.toarray()).float()\nval_input_tensor= torch.tensor(val_df.toarray()).float()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_target_tensor= torch.tensor(train_target.values).float()\nval_target_tensor=torch.tensor(val_target.values).float()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_input_tensor= torch.tensor(test_input.toarray()).float()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## DataLoader","metadata":{}},{"cell_type":"code","source":"print(type(train_input_tensor))\nprint(type(train_target_tensor))\nprint(type(val_input_tensor))\nprint(type(val_target_tensor))\nprint(type(test_input_tensor))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_ds= TensorDataset(train_input_tensor, train_target_tensor)\nval_ds= TensorDataset(val_input_tensor, val_target_tensor)\ntest_ds= TensorDataset(test_input_tensor)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_dl= DataLoader(train_ds, batch_size=128, shuffle=True)\nval_dl= DataLoader(val_ds, batch_size=128)\ntest_dl= DataLoader(test_ds, batch_size=128)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model","metadata":{}},{"cell_type":"code","source":"class QuoraNet(nn.Module):\n    def __init__(self):\n        super(QuoraNet, self).__init__()\n        self.layer1=nn.Linear(1024, 512)\n        self.layer2=nn.Linear(512, 256)\n        self.layer3=nn.Linear(256, 128)\n        self.layer4=nn.Linear(128, 32)\n        self.layer5=nn.Linear(32, 1)\n\n    def forward(self, inputs):\n        out=self.layer1(inputs)\n        out= F.relu(out)\n        out=self.layer2(out)\n        out= F.relu(out)\n        out=self.layer3(out)\n        out= F.relu(out)\n        out=self.layer4(out)\n        out= F.relu(out)\n        out=self.layer5(out)\n        return out\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model=QuoraNet()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for batch in train_dl:\n    batch_inputs, batch_targets = batch\n    print('inputs.shape:', batch_inputs.shape)\n    print('targets.shape:', batch_targets.shape)\n    \n    batch_out = model(batch_inputs)\n    print(\"batch_out.shape: \", batch_out.shape)\n\n\n    #probability conversion\n    probs = torch.sigmoid(batch_out[:,0])\n    print(\"probs:\", probs[:10])\n\n    preds = (probs >= 0.5).int()\n    print(\"preds: \", preds[:10])\n    print(\"targets: \", batch_targets[:10])\n    \n    print('outputs', preds)\n    print('accuracy', accuracy_score(batch_targets, preds))\n    print('f1_score', f1_score(batch_targets, preds))\n\n    #loss\n    print(\"loss: \", F.binary_cross_entropy(preds.float(), batch_targets))\n    break","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def evaluate(model, dl):\n    losses, accs, f1s = [], [], []\n    for batch in dl:\n        inputs, targets = batch\n        out = model(inputs)\n        \n        probs = torch.sigmoid(out[:,0])\n        loss = F.binary_cross_entropy(probs, targets.float(), weight=torch.tensor(20.))\n        losses.append(loss.item())\n\n        preds = (probs > 0.5).int()\n        acc = accuracy_score(targets, preds)\n        f1 = f1_score(targets, preds)\n        \n        accs.append(acc)\n        f1s.append(f1)\n\n    return np.mean(losses), np.mean(accs), np.mean(f1s)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"evaluate(model, train_dl), evaluate(model, val_dl)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def fit(epochs, lr, model, train_loader, val_loader):\n    optimizer = torch.optim.Adam(model.parameters(), lr, weight_decay=1e-5)\n    history = [] # for recording epoch-wise results\n    \n    for epoch in range(epochs):\n        \n        # Training Phase \n        for batch in train_loader:\n            inputs, targets = batch\n            out = model(inputs)\n            probs = torch.sigmoid(out[:,0])\n            loss = F.binary_cross_entropy(probs, \n                                          targets.float(), \n                                          weight=torch.tensor(20.))\n            loss.backward()\n            optimizer.step()\n            optimizer.zero_grad()\n        \n        # Validation phase\n        result = evaluate(model, val_loader)\n        loss, acc, f1 = result\n        print('Epoch: {}; Loss: {:.4f}; Accuracy: {:.4f}; F1 Score: {:.4f}'.format(\n            epoch, loss, acc, f1))\n        history.append(result)\n\n    return history","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model=QuoraNet()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history = [evaluate(model, val_dl)]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history+=fit(10, 0.01, model, train_dl, val_dl)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history+=fit(5, 0.001, model, train_dl, val_dl)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"losses = [item[0] for item in history]\nacc=[item[1] for item in history]\nf1s = [item[2] for item in history]\n\nfig, axes = plt.subplots(nrows=1, ncols=3, figsize=(22, 6))\nfig.suptitle('Losses, Accuracy and F1 Score Vs Epochs')\n\nsns.lineplot(losses, ax=axes[0])\naxes[0].set_title('Losses')\nsns.lineplot(acc, ax=axes[1])\naxes[1].set_title('Accuracy')\nsns.lineplot(f1s, ax=axes[2])\naxes[2].set_title('F1 Score')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-10-10T09:34:32.878439Z","iopub.status.idle":"2024-10-10T09:34:32.878974Z","shell.execute_reply.started":"2024-10-10T09:34:32.878705Z","shell.execute_reply":"2024-10-10T09:34:32.878731Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Predictions on Kaggle Data","metadata":{}},{"cell_type":"code","source":"test_tensors = torch.tensor(test_input.toarray()).float()","metadata":{"execution":{"iopub.status.busy":"2024-10-10T09:34:32.881038Z","iopub.status.idle":"2024-10-10T09:34:32.881549Z","shell.execute_reply.started":"2024-10-10T09:34:32.881290Z","shell.execute_reply":"2024-10-10T09:34:32.881313Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_ds = TensorDataset(test_tensors)\ntest_dl = DataLoader(test_ds, batch_size=128)","metadata":{"execution":{"iopub.status.busy":"2024-10-10T09:34:32.883741Z","iopub.status.idle":"2024-10-10T09:34:32.884303Z","shell.execute_reply.started":"2024-10-10T09:34:32.884051Z","shell.execute_reply":"2024-10-10T09:34:32.884076Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def predict(model, dl):\n    all_preds = []\n    for batch in dl:\n        inputs, = batch\n        out = model(inputs)\n        probs = torch.sigmoid(out)[:,0]\n        preds = (probs > 0.5).int()\n        all_preds += list(preds.numpy())\n    return all_preds","metadata":{"execution":{"iopub.status.busy":"2024-10-10T09:34:32.888068Z","iopub.status.idle":"2024-10-10T09:34:32.888836Z","shell.execute_reply.started":"2024-10-10T09:34:32.888451Z","shell.execute_reply":"2024-10-10T09:34:32.888489Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_preds = predict(model, test_dl)","metadata":{"execution":{"iopub.status.busy":"2024-10-10T09:34:32.890764Z","iopub.status.idle":"2024-10-10T09:34:32.891487Z","shell.execute_reply.started":"2024-10-10T09:34:32.891135Z","shell.execute_reply":"2024-10-10T09:34:32.891170Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_preds[:20]","metadata":{"execution":{"iopub.status.busy":"2024-10-10T09:34:32.893359Z","iopub.status.idle":"2024-10-10T09:34:32.894063Z","shell.execute_reply.started":"2024-10-10T09:34:32.893698Z","shell.execute_reply":"2024-10-10T09:34:32.893733Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub_df=pd.read_csv(sub_fname)","metadata":{"execution":{"iopub.status.busy":"2024-10-10T09:34:32.896218Z","iopub.status.idle":"2024-10-10T09:34:32.896732Z","shell.execute_reply.started":"2024-10-10T09:34:32.896498Z","shell.execute_reply":"2024-10-10T09:34:32.896523Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub_df.prediction = test_preds\nsub_df.to_csv('submission.csv', index=None)","metadata":{"execution":{"iopub.status.busy":"2024-10-10T09:34:32.898193Z","iopub.status.idle":"2024-10-10T09:34:32.898694Z","shell.execute_reply.started":"2024-10-10T09:34:32.898473Z","shell.execute_reply":"2024-10-10T09:34:32.898497Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}