{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## Language Only\nGlove RNN or BERT model (WIP)\n\n* https://towardsdatascience.com/sentiment-analysis-using-lstm-and-glove-embeddings-99223a87fe8e\n* https://victordibia.com/blog/text-classification-hf-tf2/\n\n#### The following features were used\n\n* Number of words in title\n* Number of words in body\n* Number of tags (_'java', 'front-end' etc._)\n* User reputation at post creation \n* Account age (in days)\n* Post age (in days)","metadata":{"tags":[],"cell_id":"56ded819-da00-40ee-ae38-9b9bceaa2c88","deepnote_cell_type":"markdown","deepnote_cell_height":405.984375}},{"cell_type":"code","source":"!python -m spacy download en_core_web_sm","metadata":{"cell_id":"100f0aec135b4f189121ea4498fabbe5","tags":[],"deepnote_to_be_reexecuted":false,"source_hash":"5a834a00","execution_start":1652135871670,"execution_millis":22416,"deepnote_cell_type":"code","deepnote_cell_height":701,"execution":{"iopub.status.busy":"2022-05-10T08:39:29.287716Z","iopub.execute_input":"2022-05-10T08:39:29.288337Z","iopub.status.idle":"2022-05-10T08:40:13.302206Z","shell.execute_reply.started":"2022-05-10T08:39:29.288218Z","shell.execute_reply":"2022-05-10T08:40:13.301119Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport sys\nfrom datetime import datetime\nimport pickle","metadata":{"cell_id":"d115f9e9500246eda62c1b8043355ebe","tags":[],"deepnote_to_be_reexecuted":false,"source_hash":"71bf9a2","execution_start":1652135894097,"execution_millis":7,"deepnote_cell_type":"code","deepnote_cell_height":135,"execution":{"iopub.status.busy":"2022-05-10T08:40:13.305949Z","iopub.execute_input":"2022-05-10T08:40:13.306773Z","iopub.status.idle":"2022-05-10T08:40:13.313909Z","shell.execute_reply.started":"2022-05-10T08:40:13.306737Z","shell.execute_reply":"2022-05-10T08:40:13.312827Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport numpy as np\nimport pandas as pd\nfrom collections import Counter, defaultdict\nfrom nltk.corpus import stopwords\nimport nltk\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import classification_report\nfrom sklearn.preprocessing import Normalizer","metadata":{"tags":[],"deepnote_to_be_reexecuted":false,"source_hash":"2e37dfb0","execution_start":1652135894110,"execution_millis":8790,"cell_id":"00001-6b62d446-3566-46fe-beef-32c27b6bdd7e","deepnote_cell_type":"code","deepnote_cell_height":863,"execution":{"iopub.status.busy":"2022-05-10T08:40:13.316004Z","iopub.execute_input":"2022-05-10T08:40:13.316425Z","iopub.status.idle":"2022-05-10T08:40:14.311425Z","shell.execute_reply.started":"2022-05-10T08:40:13.316378Z","shell.execute_reply":"2022-05-10T08:40:14.310437Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\n# from torchtext import data\n\nSEED = 1234\ntorch.manual_seed(SEED)\ntorch.cuda.manual_seed(SEED)\ntorch.backends.cudnn.deterministic = True\nimport torch.nn as nn\nimport torch.optim as optim\nimport torch.nn.functional as F\n\n\nimport torchtext\nimport re\nfrom torch.utils.data import DataLoader,TensorDataset\n\nimport pyprind\n%matplotlib inline  \n\nnltk.download('punkt')\nnltk.download('stopwords')\n\n# import spacy\n# tokenizer = spacy.load('en_core_web_sm')\nfrom torchtext.data import get_tokenizer\nfrom torchtext.vocab import build_vocab_from_iterator, GloVe","metadata":{"execution":{"iopub.status.busy":"2022-05-10T08:40:14.313881Z","iopub.execute_input":"2022-05-10T08:40:14.314159Z","iopub.status.idle":"2022-05-10T08:40:15.510508Z","shell.execute_reply.started":"2022-05-10T08:40:14.314128Z","shell.execute_reply":"2022-05-10T08:40:15.509458Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"execution":{"iopub.status.busy":"2022-05-10T08:40:15.512376Z","iopub.execute_input":"2022-05-10T08:40:15.512943Z","iopub.status.idle":"2022-05-10T08:40:15.531405Z","shell.execute_reply.started":"2022-05-10T08:40:15.512873Z","shell.execute_reply":"2022-05-10T08:40:15.530355Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# DATAFILE='../input/predict-closed-questions-on-stack-overflow/train.csv'\nDATAFILE='../input/predict-closed-questions-on-stack-overflow/train-sample.csv'\ndf = pd.read_csv(DATAFILE)\n\nopen_idx = df['OpenStatus'] == 'open'\nclosed_idx = (1-open_idx).astype(bool)","metadata":{"tags":[],"deepnote_to_be_reexecuted":false,"source_hash":"64fce11b","execution_start":1652135902903,"execution_millis":5792,"cell_id":"00002-d2d49d73-5b22-4a4a-bec2-5bbfa5acdbfe","deepnote_cell_type":"code","deepnote_cell_height":153,"execution":{"iopub.status.busy":"2022-05-10T08:40:15.53424Z","iopub.execute_input":"2022-05-10T08:40:15.534579Z","iopub.status.idle":"2022-05-10T08:40:19.278722Z","shell.execute_reply.started":"2022-05-10T08:40:15.534531Z","shell.execute_reply":"2022-05-10T08:40:19.277761Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"QUESTION_SAMPLE_LIMIT = 10000\nTRAIN_SAMPLE_LIMIT=7500","metadata":{"execution":{"iopub.status.busy":"2022-05-10T08:40:19.280657Z","iopub.execute_input":"2022-05-10T08:40:19.280993Z","iopub.status.idle":"2022-05-10T08:40:19.28668Z","shell.execute_reply.started":"2022-05-10T08:40:19.280941Z","shell.execute_reply":"2022-05-10T08:40:19.285169Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = df.sample(frac=1)","metadata":{"execution":{"iopub.status.busy":"2022-05-10T08:40:19.288884Z","iopub.execute_input":"2022-05-10T08:40:19.289246Z","iopub.status.idle":"2022-05-10T08:40:19.37251Z","shell.execute_reply.started":"2022-05-10T08:40:19.289198Z","shell.execute_reply":"2022-05-10T08:40:19.371542Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Preprocessing","metadata":{"cell_id":"2a68da385c8a45e4af198ba7dd94a5f0","tags":[],"is_collapsed":false,"deepnote_cell_type":"text-cell-h2"}},{"cell_type":"code","source":"# print(df)\n\nprint(\"Tag remover goes first, spacy goes second\")\n\ndef remove_tags(string):\n    # result = re.sub('<.*?>','',string)\n    try:\n        result = re.sub(r\"(, '[\\W\\.]')\",r\"\",string)\n        return result\n    except:\n#         print(string)\n        return \"invalid sentence\"\n\ntext_df = pd.concat( [df['Title'].apply(lambda x: remove_tags(x) ), df['BodyMarkdown'].apply(lambda x: remove_tags(x) )] , axis=1)\n\n#text_df","metadata":{"cell_id":"3143b6349d3a420e87cdc1c0d4e03b24","tags":[],"deepnote_to_be_reexecuted":false,"source_hash":"416abf64","execution_start":1652135908725,"execution_millis":814,"deepnote_cell_type":"code","deepnote_cell_height":309.6875,"execution":{"iopub.status.busy":"2022-05-10T08:40:19.374298Z","iopub.execute_input":"2022-05-10T08:40:19.374613Z","iopub.status.idle":"2022-05-10T08:40:20.022224Z","shell.execute_reply.started":"2022-05-10T08:40:19.374565Z","shell.execute_reply":"2022-05-10T08:40:20.021219Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# examples = ['chip', 'baby', 'Beautiful']\n# vec = GloVe(name='6B', dim=50)\n# # vec = GloVe(name='840B.300d', dim=300)\n# ret = vec.get_vecs_by_tokens(examples, lower_case_backup=True)","metadata":{"cell_id":"2dbc83902f764a4185f416ddd2102015","tags":[],"deepnote_to_be_reexecuted":false,"source_hash":"83c7caf5","execution_start":1652135909556,"execution_millis":3576601,"deepnote_cell_type":"code","deepnote_cell_height":135,"execution":{"iopub.status.busy":"2022-05-10T08:40:20.027034Z","iopub.execute_input":"2022-05-10T08:40:20.027283Z","iopub.status.idle":"2022-05-10T08:40:20.034315Z","shell.execute_reply.started":"2022-05-10T08:40:20.02725Z","shell.execute_reply":"2022-05-10T08:40:20.033214Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Add Title before Question Body\ntext_df['Text']=text_df['Title']+' '+text_df['BodyMarkdown']\ntext_df.drop(['Title', 'BodyMarkdown'], axis=1, inplace=True)","metadata":{"cell_id":"adecaadc56e14ada825bf9da448bbdf8","tags":[],"deepnote_to_be_reexecuted":false,"source_hash":"ad918ee1","execution_start":1652135909588,"execution_millis":125,"deepnote_cell_type":"code","deepnote_cell_height":99,"execution":{"iopub.status.busy":"2022-05-10T08:40:20.035953Z","iopub.execute_input":"2022-05-10T08:40:20.036852Z","iopub.status.idle":"2022-05-10T08:40:20.294635Z","shell.execute_reply.started":"2022-05-10T08:40:20.036801Z","shell.execute_reply":"2022-05-10T08:40:20.293677Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Tokenize First\ntokenizer = get_tokenizer(\"spacy\",'en') #'en_core_web_sm'\n\nGLOVE_NAME = '6B'\nEMBED_DIM = 50\n# GLOVE_NAME = '840B'\n# EMBED_DIM = 300\n\n#Word Encoder\nvec = GloVe(name=GLOVE_NAME, dim=EMBED_DIM)\nprint(\"Switch to 840B.300d when you are ready\")","metadata":{"cell_id":"2f17c3c9a25e4ca4a5c707d47ab289ad","tags":[],"deepnote_to_be_reexecuted":false,"source_hash":"305f9cdf","execution_start":1652135909722,"execution_millis":9218,"deepnote_cell_type":"code","deepnote_cell_height":336,"execution":{"iopub.status.busy":"2022-05-10T08:40:20.296459Z","iopub.execute_input":"2022-05-10T08:40:20.296793Z","iopub.status.idle":"2022-05-10T08:43:48.731625Z","shell.execute_reply.started":"2022-05-10T08:40:20.296742Z","shell.execute_reply":"2022-05-10T08:43:48.729507Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Investigate Sequence Lengths","metadata":{"cell_id":"001aa29997b04a0e81d83a104189fe92","tags":[],"is_collapsed":false,"deepnote_cell_type":"text-cell-h3"}},{"cell_type":"code","source":"# # QUESTION_SAMPLE_LIMIT=50000\n# QUESTION_SAMPLE_LIMIT=None\n\n# textblocks = text_df['Text']\n# if QUESTION_SAMPLE_LIMIT!=None and QUESTION_SAMPLE_LIMIT > 0:\n#     textblocks = textblocks[0:QUESTION_SAMPLE_LIMIT]\n\n# tokens = [tokenizer(x) for x in textblocks]\n# # N x ? x EMBED_DIM","metadata":{"cell_id":"e78a561ad0d04226a9fd193c9a893690","tags":[],"deepnote_to_be_reexecuted":false,"source_hash":"403b7a60","execution_start":1652135918945,"execution_millis":151495,"deepnote_cell_type":"code","deepnote_cell_height":207,"deepnote_output_heights":[21.1875],"execution":{"iopub.status.busy":"2022-05-10T08:43:48.733699Z","iopub.execute_input":"2022-05-10T08:43:48.734067Z","iopub.status.idle":"2022-05-10T08:43:48.739719Z","shell.execute_reply.started":"2022-05-10T08:43:48.73402Z","shell.execute_reply":"2022-05-10T08:43:48.738218Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# token_lengths = pd.DataFrame({'token_lengths': [len(x) for x in tokens]})","metadata":{"cell_id":"3db98fcadd2743868d28e0fed31dd978","tags":[],"deepnote_to_be_reexecuted":false,"source_hash":"732b2f5e","execution_start":1652136070484,"execution_millis":49,"deepnote_cell_type":"code","deepnote_cell_height":81,"execution":{"iopub.status.busy":"2022-05-10T08:43:48.742258Z","iopub.execute_input":"2022-05-10T08:43:48.74287Z","iopub.status.idle":"2022-05-10T08:43:48.753381Z","shell.execute_reply.started":"2022-05-10T08:43:48.742811Z","shell.execute_reply":"2022-05-10T08:43:48.7523Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# token_lengths = pd.DataFrame([0,1,2])\n# pd.DataFrame({'length':token_lengths.to_numpy()})\n# token_lengths","metadata":{"cell_id":"845cd5a4363f45df8ab53e57823e3f80","tags":[],"deepnote_to_be_reexecuted":false,"source_hash":"82882257","execution_start":1652136070548,"execution_millis":1,"deepnote_cell_type":"code","deepnote_cell_height":117,"deepnote_output_heights":[136.359375],"execution":{"iopub.status.busy":"2022-05-10T08:43:48.75495Z","iopub.execute_input":"2022-05-10T08:43:48.757162Z","iopub.status.idle":"2022-05-10T08:43:48.766205Z","shell.execute_reply.started":"2022-05-10T08:43:48.75711Z","shell.execute_reply":"2022-05-10T08:43:48.765025Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#token_lengths","metadata":{"cell_id":"df35b0efbfb0476b852d23425c83f339","tags":[],"deepnote_to_be_reexecuted":false,"source_hash":"d186dd5e","execution_start":1652136070565,"execution_millis":22,"deepnote_cell_type":"code","deepnote_cell_height":81,"execution":{"iopub.status.busy":"2022-05-10T08:43:48.768401Z","iopub.execute_input":"2022-05-10T08:43:48.769067Z","iopub.status.idle":"2022-05-10T08:43:48.777717Z","shell.execute_reply.started":"2022-05-10T08:43:48.769017Z","shell.execute_reply":"2022-05-10T08:43:48.776639Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# text_df","metadata":{"cell_id":"d191ba25e2b94503bfdb511be8f34031","deepnote_variable_name":"token_lengths","deepnote_visualization_spec":{"$schema":"https://vega.github.io/schema/vega-lite/v4.json","mark":{"type":"bar","tooltip":{"content":"data"}},"height":220,"autosize":{"type":"fit"},"data":{"name":"placeholder"},"encoding":{"x":{"field":"token_lengths","type":"quantitative","sort":null,"scale":{"type":"linear","zero":false}},"y":{"field":"COUNT(*)","type":"quantitative","sort":null,"aggregate":"count","scale":{"type":"linear","zero":true}},"color":{"field":"","type":"nominal","sort":null,"scale":{"type":"linear","zero":false}}}},"deepnote_to_be_reexecuted":false,"source_hash":"e42989b9","execution_start":1652136070588,"execution_millis":1183,"deepnote_cell_type":"visualization","execution":{"iopub.status.busy":"2022-05-10T08:43:48.77987Z","iopub.execute_input":"2022-05-10T08:43:48.780389Z","iopub.status.idle":"2022-05-10T08:43:48.788687Z","shell.execute_reply.started":"2022-05-10T08:43:48.780342Z","shell.execute_reply":"2022-05-10T08:43:48.7876Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# MAX_SEQ_LENGTHS = [500,250,100]\n# for MAX_SEQ_LENGTH in MAX_SEQ_LENGTHS:\n#     P=100*len( token_lengths[ token_lengths['token_lengths'] <= MAX_SEQ_LENGTH ] )/QUESTION_SAMPLE_LIMIT\n#     print( f\"Questions under {MAX_SEQ_LENGTH} tokens: ~{P}%\" )\n\nMAX_SEQ_LENGTH = 250\nprint( f\"SET MAX_SEQ_LENGTH to {MAX_SEQ_LENGTH}\")","metadata":{"cell_id":"9cfc2902019c4931badfedbca952f506","tags":[],"deepnote_to_be_reexecuted":false,"source_hash":"8816e866","execution_start":1652136071836,"execution_millis":3678194,"deepnote_cell_type":"code","deepnote_cell_height":278.78125,"deepnote_output_heights":[21.1875],"execution":{"iopub.status.busy":"2022-05-10T08:43:48.790439Z","iopub.execute_input":"2022-05-10T08:43:48.790905Z","iopub.status.idle":"2022-05-10T08:43:48.802482Z","shell.execute_reply.started":"2022-05-10T08:43:48.790856Z","shell.execute_reply":"2022-05-10T08:43:48.801118Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Tokenise","metadata":{"cell_id":"2d88cedab6ec4d1fb4ac865f7df47118","tags":[],"is_collapsed":false,"deepnote_cell_type":"text-cell-h3"}},{"cell_type":"code","source":"# \n# .sample(frac=1)\n\nprint(\"REMEMBER TO UPDATE QUESTION_SAMPLE_LIMIT\")\n\n# test = Tokenizer(X['Text'][0])\n# print(test)\n#tokenizer.fit_on_texts(X['Text'])\n# tokenizer.fit_on_texts(X)\n\n# QUESTION_SAMPLE_LIMIT=10000\nQUESTION_SAMPLE_LIMIT=None\n\ntextblocks = text_df['Text']\nif QUESTION_SAMPLE_LIMIT!=None and QUESTION_SAMPLE_LIMIT > 0:\n    textblocks = textblocks[0:QUESTION_SAMPLE_LIMIT]\n\n\ntokenblocks = [tokenizer(x) for x in textblocks]\n\n# len(Encodings)\n# len(Encodings[0])\n# len(Encodings[0])","metadata":{"cell_id":"0fe58b80aeb144aca51fc43391ef8529","tags":[],"deepnote_to_be_reexecuted":false,"source_hash":"b7ca558c","allow_embed":false,"execution_start":1652136071851,"execution_millis":11126,"deepnote_cell_type":"code","deepnote_cell_height":453.6875,"execution":{"iopub.status.busy":"2022-05-10T08:43:48.804551Z","iopub.execute_input":"2022-05-10T08:43:48.805038Z","iopub.status.idle":"2022-05-10T08:48:18.43403Z","shell.execute_reply.started":"2022-05-10T08:43:48.804988Z","shell.execute_reply":"2022-05-10T08:48:18.432958Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# del text_df","metadata":{"execution":{"iopub.status.busy":"2022-05-10T08:48:18.435873Z","iopub.execute_input":"2022-05-10T08:48:18.436212Z","iopub.status.idle":"2022-05-10T08:48:18.443043Z","shell.execute_reply.started":"2022-05-10T08:48:18.436167Z","shell.execute_reply":"2022-05-10T08:48:18.441963Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Truncate if above limit","metadata":{"cell_id":"b2517b6ab8e44dd4aff0c0d92e6885cb","tags":[],"is_collapsed":false,"deepnote_cell_type":"text-cell-h3"}},{"cell_type":"code","source":"token_lengths = [len(x) for x in tokenblocks]","metadata":{"execution":{"iopub.status.busy":"2022-05-10T09:09:28.1581Z","iopub.execute_input":"2022-05-10T09:09:28.158406Z","iopub.status.idle":"2022-05-10T09:09:28.222977Z","shell.execute_reply.started":"2022-05-10T09:09:28.158372Z","shell.execute_reply":"2022-05-10T09:09:28.221982Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Target Shape: N x MAX_SEQ_LENGTHS x 50\n\nfor i in range(len(tokenblocks)):\n    tokenblocks[i] = tokenblocks[i][0:min(MAX_SEQ_LENGTH, len(tokenblocks[i]))]","metadata":{"cell_id":"4e4958b55a7a4caba1f15f41fea1ea79","tags":[],"deepnote_to_be_reexecuted":false,"source_hash":"40e3f1a","execution_start":1652136082996,"execution_millis":18,"deepnote_cell_type":"code","deepnote_cell_height":135,"execution":{"iopub.status.busy":"2022-05-10T08:48:18.445053Z","iopub.execute_input":"2022-05-10T08:48:18.445664Z","iopub.status.idle":"2022-05-10T08:48:18.915368Z","shell.execute_reply.started":"2022-05-10T08:48:18.445613Z","shell.execute_reply":"2022-05-10T08:48:18.914399Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#get encodings\nwEncodings = [ vec.get_vecs_by_tokens(tk, lower_case_backup=True) for tk in tokenblocks ]","metadata":{"cell_id":"1e94dc865f1c44e58a208e4e009fb668","tags":[],"deepnote_to_be_reexecuted":false,"source_hash":"c6fa2ee3","execution_start":1652136083023,"execution_millis":5371,"deepnote_cell_type":"code","deepnote_cell_height":99,"execution":{"iopub.status.busy":"2022-05-10T08:48:18.916756Z","iopub.execute_input":"2022-05-10T08:48:18.917091Z","iopub.status.idle":"2022-05-10T08:49:34.799721Z","shell.execute_reply.started":"2022-05-10T08:48:18.917047Z","shell.execute_reply":"2022-05-10T08:49:34.798712Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### [SKIPPED] Encode global (non-sequential) features","metadata":{"tags":[],"cell_id":"00003-3ed3a67a-220e-439f-84ef-e0227a076c7f","deepnote_cell_type":"markdown","deepnote_cell_height":62}},{"cell_type":"code","source":"# # 0 if open, 1 if closed\ndf['y'] = 1-(df.OpenStatus == 'open').astype(int)\n\n# # Encode 'post creation date' as its age in days\n# df.PostCreationDate = pd.to_datetime(df.PostCreationDate)\n# origion_date = max(df.PostCreationDate)\n# df['PostAge'] = (origion_date-df.PostCreationDate).dt.days\n\n# # Encode OwnerCreationDate as account age in days\n# df.OwnerCreationDate = pd.to_datetime(df.OwnerCreationDate)\n# origion_date = max(df.OwnerCreationDate)\n# df['AccountAge'] = (origion_date-df.OwnerCreationDate).dt.days\n\n# # Encode title and body text as their length\n# df['TitleLength'] = df.apply(lambda x: len(x['Title'].split()), 1)\n# df['BodyLength'] = df.apply(lambda x: len(x['BodyMarkdown'].split()), 1)\n\n# # Number of tags a post has (between 0-5)\n# df['TotalTags'] = df.apply(lambda x: sum([not pd.isnull(x[f'Tag{a}']) for a in range(1, 6)]), 1)\n\n# features = ['PostAge', 'AccountAge', 'ReputationAtPostCreation', 'OwnerUndeletedAnswerCountAtPostTime', 'TitleLength', 'BodyLength', 'TotalTags']\n","metadata":{"tags":[],"deepnote_to_be_reexecuted":false,"source_hash":"de3f713e","execution_start":1652136088414,"execution_millis":6,"cell_id":"00004-b80fb519-e053-4aa3-9af6-2297194225d9","deepnote_cell_type":"code","deepnote_cell_height":459,"execution":{"iopub.status.busy":"2022-05-10T08:49:34.801458Z","iopub.execute_input":"2022-05-10T08:49:34.801803Z","iopub.status.idle":"2022-05-10T08:49:34.831577Z","shell.execute_reply.started":"2022-05-10T08:49:34.80174Z","shell.execute_reply":"2022-05-10T08:49:34.830517Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Train/Test Split","metadata":{"tags":[],"cell_id":"00005-d75a81f4-1f9d-4044-9024-e3ed1aae6cc9","deepnote_cell_type":"markdown","deepnote_cell_height":62}},{"cell_type":"code","source":"# TRAIN_SAMPLE_LIMIT=200\nprint(\"REMEMBER TO UPDATE AND/OR REMOVE TRAIN_SAMPLE_LIMIT\")\nif TRAIN_SAMPLE_LIMIT!=None and TRAIN_SAMPLE_LIMIT > 0:\n    N = min(TRAIN_SAMPLE_LIMIT,len(wEncodings))\nelse:\n    N = len(wEncodings)\n\n# Padding will be handled HERE\n# copy these tokens into a numpy array of originally all 0s\n\n# N x MAX_SEQ_LENGTH x EMBED_DIM\nX_lan = np.zeros((N,MAX_SEQ_LENGTH,EMBED_DIM))\nX_lan.shape\n\nfor i in range(N):\n    enc = wEncodings[i]\n    X_lan[i][0:0+len(enc)] = enc","metadata":{"cell_id":"b11aa32ccd1c45828db1cabb0ce1858f","tags":[],"deepnote_to_be_reexecuted":false,"source_hash":"29b08ab4","execution_start":1652136088422,"execution_millis":27,"deepnote_cell_type":"code","deepnote_cell_height":399.6875,"deepnote_output_heights":[null,21.1875],"execution":{"iopub.status.busy":"2022-05-10T08:49:34.83381Z","iopub.execute_input":"2022-05-10T08:49:34.834196Z","iopub.status.idle":"2022-05-10T08:49:35.15848Z","shell.execute_reply.started":"2022-05-10T08:49:34.834149Z","shell.execute_reply":"2022-05-10T08:49:35.157489Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#X_meta = df[features][0:N]\ny = df['y'][0:N]\n#X_meta = df[features]\n# y = df['y']\n\n# X_lan = wEncodings[0:N]","metadata":{"cell_id":"f06cb4a312f040beafbd6d4966d83675","tags":[],"deepnote_to_be_reexecuted":false,"source_hash":"188a823","execution_start":1652136088460,"execution_millis":3657693,"deepnote_cell_type":"code","deepnote_cell_height":171,"execution":{"iopub.status.busy":"2022-05-10T08:49:35.160316Z","iopub.execute_input":"2022-05-10T08:49:35.160629Z","iopub.status.idle":"2022-05-10T08:49:35.167064Z","shell.execute_reply.started":"2022-05-10T08:49:35.160579Z","shell.execute_reply":"2022-05-10T08:49:35.165987Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_indices = list(range(N))","metadata":{"cell_id":"021b3c0abd724572b1cf93aba5868136","tags":[],"deepnote_to_be_reexecuted":false,"source_hash":"4f5a4282","execution_start":1652136088508,"execution_millis":3657698,"deepnote_cell_type":"code","deepnote_cell_height":81,"deepnote_output_heights":[21.1875],"execution":{"iopub.status.busy":"2022-05-10T08:49:35.168849Z","iopub.execute_input":"2022-05-10T08:49:35.169582Z","iopub.status.idle":"2022-05-10T08:49:35.178438Z","shell.execute_reply.started":"2022-05-10T08:49:35.169532Z","shell.execute_reply":"2022-05-10T08:49:35.177337Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train_indices, X_test_indices, y_train, y_test = train_test_split(X_indices, y, stratify=y, test_size=0.2, random_state=1)\n#Use Indices Instead, for X, thank me later\n\nX_lan_train = X_lan[X_train_indices]\nX_lan_test = X_lan[X_test_indices]\n","metadata":{"tags":[],"deepnote_to_be_reexecuted":false,"source_hash":"9e08a7e7","execution_start":1652136088509,"execution_millis":5,"cell_id":"00006-7b8df3e5-b657-4bb2-93b3-80f76d2bfc2b","deepnote_cell_type":"code","deepnote_cell_height":171,"deepnote_output_heights":[21.1875],"execution":{"iopub.status.busy":"2022-05-10T08:49:35.180771Z","iopub.execute_input":"2022-05-10T08:49:35.181513Z","iopub.status.idle":"2022-05-10T08:49:35.963075Z","shell.execute_reply.started":"2022-05-10T08:49:35.181463Z","shell.execute_reply":"2022-05-10T08:49:35.962068Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Normalize Global (non-sequential) Features\n# transformer = Normalizer().fit(X_train[global])\n# X_train[global] = transformer.transform(X_train[global])\n# X_test[global] = transformer.transform(X_test[global])","metadata":{"tags":[],"deepnote_to_be_reexecuted":false,"source_hash":"396efa6c","execution_start":1652136088517,"execution_millis":3,"cell_id":"00007-a63a74e9-4bd0-4675-8261-7a58f1267b2e","deepnote_cell_type":"code","deepnote_cell_height":135,"execution":{"iopub.status.busy":"2022-05-10T08:49:35.969309Z","iopub.execute_input":"2022-05-10T08:49:35.969587Z","iopub.status.idle":"2022-05-10T08:49:35.974175Z","shell.execute_reply.started":"2022-05-10T08:49:35.96955Z","shell.execute_reply":"2022-05-10T08:49:35.972973Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_lan_train[:1]","metadata":{"cell_id":"4ae0526f10cf4efa882df1464bf808fa","tags":[],"deepnote_to_be_reexecuted":false,"source_hash":"92a9602e","execution_start":1652136088573,"execution_millis":0,"deepnote_cell_type":"code","deepnote_cell_height":348.53125,"deepnote_output_heights":[251.53125],"execution":{"iopub.status.busy":"2022-05-10T08:49:35.975905Z","iopub.execute_input":"2022-05-10T08:49:35.976477Z","iopub.status.idle":"2022-05-10T08:49:35.998203Z","shell.execute_reply.started":"2022-05-10T08:49:35.976429Z","shell.execute_reply":"2022-05-10T08:49:35.996976Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# X_train\n#y_train_tensor.shape\n# tokenblocks[X_train_indices[1]]","metadata":{"cell_id":"a76abaeafdc044cdb187d12f480b8eaf","tags":[],"deepnote_to_be_reexecuted":false,"source_hash":"725c3d51","execution_start":1652136088574,"execution_millis":3,"deepnote_cell_type":"code","deepnote_cell_height":117,"deepnote_output_heights":[611],"execution":{"iopub.status.busy":"2022-05-10T08:49:35.999802Z","iopub.execute_input":"2022-05-10T08:49:36.000655Z","iopub.status.idle":"2022-05-10T08:49:36.008784Z","shell.execute_reply.started":"2022-05-10T08:49:36.000603Z","shell.execute_reply":"2022-05-10T08:49:36.007511Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"tags":[],"deepnote_to_be_reexecuted":false,"source_hash":"4b95615c","execution_start":1652136088575,"execution_millis":0,"cell_id":"00008-ad81b7bc-aa44-4fa4-be31-fdef5ae28b70","deepnote_cell_type":"code","deepnote_cell_height":135,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"tags":[],"deepnote_to_be_reexecuted":false,"source_hash":"e1f14cf7","execution_start":1652136088608,"execution_millis":3657832,"cell_id":"00009-710e2590-7ad6-40f0-b639-c2b7062155f8","deepnote_cell_type":"code","deepnote_cell_height":117,"deepnote_output_heights":[21.1875],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Model Training","metadata":{"cell_id":"10906b115c254072a614cf8485e2d5e0","tags":[],"is_collapsed":false,"deepnote_cell_type":"text-cell-h2"}},{"cell_type":"code","source":"# add pytorch model here\n\ndevice = \"cuda\" if torch.cuda.is_available() else \"cpu\"\nprint(f\"Using {device} device\")","metadata":{"execution":{"iopub.status.busy":"2022-05-10T09:58:59.92372Z","iopub.execute_input":"2022-05-10T09:58:59.924061Z","iopub.status.idle":"2022-05-10T09:58:59.930168Z","shell.execute_reply.started":"2022-05-10T09:58:59.924029Z","shell.execute_reply":"2022-05-10T09:58:59.929155Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_lan_train_tensor = torch.tensor(X_lan_train).type(torch.FloatTensor).to(device)\nX_lan_test_tensor = torch.tensor(X_lan_test).type(torch.FloatTensor).to(device)\n# y_train_tensor = torch.tensor(y_train.values).to(device)\n# y_test_tensor = torch.tensor(y_test.values).to(device)\ny_train_tensor = torch.tensor(y_train.values).type(torch.FloatTensor).to(device)\ny_test_tensor = torch.tensor(y_test.values).type(torch.FloatTensor).to(device)","metadata":{"tags":[],"deepnote_to_be_reexecuted":false,"source_hash":"eee3dc45","execution_start":1652136088610,"execution_millis":3621976,"cell_id":"00010-e264c7e1-de00-454b-84b8-f0b262920aa0","deepnote_cell_type":"code","deepnote_cell_height":165.6875,"execution":{"iopub.status.busy":"2022-05-10T09:59:01.789784Z","iopub.execute_input":"2022-05-10T09:59:01.79042Z","iopub.status.idle":"2022-05-10T09:59:02.367926Z","shell.execute_reply.started":"2022-05-10T09:59:01.790369Z","shell.execute_reply":"2022-05-10T09:59:02.36671Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# X_lan_train_tensor.shape, y_train_tensor.shape\ntrain_dataset = TensorDataset(X_lan_train_tensor,y_train_tensor)\ntest_dataset = TensorDataset(X_lan_test_tensor,y_test_tensor)","metadata":{"execution":{"iopub.status.busy":"2022-05-10T09:59:08.456063Z","iopub.execute_input":"2022-05-10T09:59:08.456437Z","iopub.status.idle":"2022-05-10T09:59:08.461551Z","shell.execute_reply.started":"2022-05-10T09:59:08.456388Z","shell.execute_reply":"2022-05-10T09:59:08.460602Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# class NeuralNetwork(nn.Module):\n#     def __init__(self):\n#         super(NeuralNetwork, self).__init__()\n#         self.flatten = nn.Flatten()\n#         #Identical implementation to https://scikit-learn.org/stable/modules/generated/sklearn.neural_network.MLPClassifier.html\n#         #Plus ReLU\n#         self.linear_relu_stack = nn.Sequential(\n#             nn.Linear(X.shape[1], 64),\n#             nn.ReLU(),\n#             nn.Linear(64, 128),\n#             nn.ReLU(),\n#             nn.Linear(128, 2),\n#             nn.ReLU(),\n#             nn.Softmax(dim=1)\n#             # nn.Linear(10, 20),\n#             # nn.ReLU(),\n#             # nn.Linear(20, 2),\n#             # nn.ReLU(),\n#         )\n\n#     def forward(self, x):\n#         x = self.flatten(x)\n#         logits = self.linear_relu_stack(x)\n#         return logits","metadata":{"tags":[],"deepnote_to_be_reexecuted":false,"source_hash":"31e7ab0","execution_start":1652136088611,"execution_millis":3,"cell_id":"00011-fdf271e8-3a65-4c6e-ab5d-4727820a6116","deepnote_cell_type":"code","deepnote_cell_height":495,"execution":{"iopub.status.busy":"2022-05-10T09:59:14.388543Z","iopub.execute_input":"2022-05-10T09:59:14.389435Z","iopub.status.idle":"2022-05-10T09:59:14.395697Z","shell.execute_reply.started":"2022-05-10T09:59:14.38938Z","shell.execute_reply":"2022-05-10T09:59:14.394588Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# hidden_size\nLSTM_HIDDEN = EMBED_DIM\n\nclass QuestionClassifier(nn.Module):\n    def __init__(self, num_features, hidden_layer):\n        super(QuestionClassifier, self).__init__()\n        #Encoder\n        self.lstm = nn.LSTM(num_features, hidden_layer, batch_first = True)\n        self.label = nn.Linear(hidden_layer, 1)\n#         self.out, (self.h,self.c) = self.lstm(x)\n\n    def forward(self, embedded_sentence, apply_sigmoid=False):\n        #embeds = self.word_embeddings(sentence)\n        # lstm_out, _ = self.lstm(embeds.view(len(sentence), 1, -1))\n        # print(embedded_sentence.shape)\n#         lstm_out, _ = self.lstm(embedded_sentence)\n#         tag_space = self.label(lstm_out)\n#         tag_scores = F.log_softmax(tag_space, dim=1)\n        # tag_scores = F.softmax(tag_space,dim=1)\n#         return tag_scores\n        # return tag_scores\n        self.out, (self.h,self.c) = self.lstm(embedded_sentence)\n#         return F.log_softmax(self.h, dim=1)\n        tag_space = self.label(self.h)\n        tag_score = F.sigmoid(tag_space)#.type(torch.float)\n        return tag_score\n\nmodel = QuestionClassifier(EMBED_DIM, LSTM_HIDDEN).to(device)\nprint(model)","metadata":{"cell_id":"f2fc553eb8e14b60a8d95afc2758d28e","tags":[],"deepnote_to_be_reexecuted":false,"source_hash":"5df015fd","execution_start":1652136088615,"execution_millis":6,"deepnote_cell_type":"code","deepnote_cell_height":566.78125,"execution":{"iopub.status.busy":"2022-05-10T10:08:39.018877Z","iopub.execute_input":"2022-05-10T10:08:39.019596Z","iopub.status.idle":"2022-05-10T10:08:39.043131Z","shell.execute_reply.started":"2022-05-10T10:08:39.019533Z","shell.execute_reply":"2022-05-10T10:08:39.041628Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# model(X_temp2)","metadata":{"tags":[],"deepnote_to_be_reexecuted":false,"source_hash":"a460e76f","execution_start":1652136088625,"execution_millis":3621895,"cell_id":"00012-51730178-f52e-4218-bdb3-8ae21c796a7a","deepnote_cell_type":"code","deepnote_cell_height":81,"execution":{"iopub.status.busy":"2022-05-10T10:08:41.328826Z","iopub.execute_input":"2022-05-10T10:08:41.329194Z","iopub.status.idle":"2022-05-10T10:08:41.33596Z","shell.execute_reply.started":"2022-05-10T10:08:41.329161Z","shell.execute_reply":"2022-05-10T10:08:41.332726Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# X_temp = torch.rand(10, 7, 1, device=device)\n# logits = model(X_temp)\n# X_temp2 = torch.tensor(X.values).type(torch.FloatTensor)\n# logits = model(X_temp2)\n# X_temp3 = torch.tensor(X_train).type(torch.FloatTensor)\n# logits = model(X_temp3)\n# pred_probab = nn.Softmax(dim=1)(logits)\n# y_pred = pred_probab.argmax(1)\n# print(f\"Predicted class: {y_pred}\")","metadata":{"tags":[],"deepnote_to_be_reexecuted":false,"source_hash":"a1f04945","execution_start":1652136088639,"execution_millis":3618727,"cell_id":"00013-2c13d150-dbdd-4ea1-9d05-e9e2ff70597a","deepnote_cell_type":"code","deepnote_cell_height":225,"execution":{"iopub.status.busy":"2022-05-10T10:08:42.932578Z","iopub.execute_input":"2022-05-10T10:08:42.932895Z","iopub.status.idle":"2022-05-10T10:08:42.937705Z","shell.execute_reply.started":"2022-05-10T10:08:42.932863Z","shell.execute_reply":"2022-05-10T10:08:42.936679Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"BATCH_SIZE = 16\nprint(f\"Specifying Batch Size as {BATCH_SIZE}\")","metadata":{"tags":[],"deepnote_to_be_reexecuted":false,"source_hash":"e2321a2b","execution_start":1652136088648,"execution_millis":3657814,"cell_id":"00014-b460ab60-5e2a-43b2-9945-4974bd947d80","deepnote_cell_type":"code","deepnote_cell_height":129.6875,"execution":{"iopub.status.busy":"2022-05-10T10:08:44.462899Z","iopub.execute_input":"2022-05-10T10:08:44.463271Z","iopub.status.idle":"2022-05-10T10:08:44.469961Z","shell.execute_reply.started":"2022-05-10T10:08:44.463238Z","shell.execute_reply":"2022-05-10T10:08:44.468677Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_dataloader = DataLoader(train_dataset, batch_size=BATCH_SIZE, shuffle=True)\ntest_dataloader = DataLoader(test_dataset, batch_size=BATCH_SIZE, shuffle=True)","metadata":{"tags":[],"deepnote_to_be_reexecuted":false,"source_hash":"8ba0e9bf","execution_start":1652136088684,"execution_millis":3657824,"cell_id":"00015-7911d97a-cb66-40b2-82b8-7b280b4fd8e2","deepnote_cell_type":"code","deepnote_cell_height":99,"execution":{"iopub.status.busy":"2022-05-10T10:08:45.639974Z","iopub.execute_input":"2022-05-10T10:08:45.640659Z","iopub.status.idle":"2022-05-10T10:08:45.646126Z","shell.execute_reply.started":"2022-05-10T10:08:45.640623Z","shell.execute_reply":"2022-05-10T10:08:45.644788Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"REPORT_EVERY_N_BATCHES = 100\neps = 1e-7\n\ndef train_loop(dataloader, model, loss_fn, optimizer):\n    size = len(dataloader.dataset)\n    for batch, (X, y) in enumerate(dataloader):\n        # Compute prediction and loss\n        pred = model(X)[0]\n#         print(pred)\n#         print(pred.shape, y.view(-1,1).shape)\n#         print(pred.dtype, y.view(-1,1).dtype)\n        \n        loss = loss_fn(pred, y.view(-1,1))\n        # break\n\n        # Backpropagation\n        optimizer.zero_grad()\n        loss.backward()\n        optimizer.step()\n\n        if batch % REPORT_EVERY_N_BATCHES == 0:\n            loss, current = loss.item(), batch * len(X)\n            print(f\"loss: {loss:>7f}  [{current:>5d}/{size:>5d}]\")\n\n\ndef test_loop(dataloader, model, loss_fn):\n    size = len(dataloader.dataset)\n    num_batches = len(dataloader)\n    test_loss, correct = 0, 0\n    tp, tn, fp, fn = 0,0,0,0\n    precision, recall = 0,0\n    f1 = 0\n\n    with torch.no_grad():\n        print_ex=True\n        for X, y in dataloader:\n            pred = model(X)[0]\n#             print(\"pred\")\n            # print(pred.shape)\n#             print(pred)\n            # print(pred.mean(axis=1) > 0 )\n            # break\n            # print(\"predict?\")\n            # print((pred > 0))\n            # break\n            y_pred = (pred > 0.5).to(torch.int)\n            y_true = y.view(-1,1)\n#             print(y_pred.shape)\n#             print(y_true.shape)\n            # break\n#             print('\"MEAN\":')\n#             print(pred.mean(axis=1))\n#             break\n#             y_pred = (pred.mean(axis=1) >= np.log(0.5)).to(torch.int)\n            if print_ex:\n#                 print(pred, y_pred, y_true)\n                print_ex = False\n\n            test_loss += loss_fn(pred, y.view(-1,1)).item()\n\n\n            # tp = (y_true * y_pred).sum().to(torch.float32)\n            ttp = (y_true * y_pred).sum().to(torch.float32)\n            tp += ttp\n            ttn = ((1 - y_true) * (1 - y_pred)).sum().to(torch.float32)\n            tn += ttn\n            fp += ((1 - y_true) * y_pred).sum().to(torch.float32)\n            fn += (y_true * (1 - y_pred)).sum().to(torch.float32)\n\n            correct += (ttp+ttn).type(torch.float).sum().item()\n            # print(\"y_pred\")\n            # print(y_pred)\n            # print(\"y_true\")\n            # print(y_true)\n            \n\n    test_loss /= num_batches\n    correct /= size\n\n    precision = tp / (tp + fp + eps)\n    recall = tp / (tp + fn + eps)\n    f1 = 2 * precision * recall / (precision+recall+eps)\n\n\n    print(f\"Test Error: \\n Accuracy: {(100*correct):>0.3f}%, Avg loss: {test_loss:>8f} \\n\")\n    # print(y_pred, y_true)\n    # print(tp, tn, fp, fn, correct, size)\n    # print(tp, correct, size)\n    acc = 100*correct\n    # print(tp, tn, acc, correct)\n    return (acc, test_loss, precision, recall, f1)","metadata":{"tags":[],"deepnote_to_be_reexecuted":false,"source_hash":"7cc1c1cf","execution_start":1652136088685,"execution_millis":3,"cell_id":"00016-bb321400-6ec8-4890-bf42-edd83129b951","deepnote_cell_type":"code","deepnote_cell_height":1485,"execution":{"iopub.status.busy":"2022-05-10T10:08:47.766943Z","iopub.execute_input":"2022-05-10T10:08:47.76729Z","iopub.status.idle":"2022-05-10T10:08:47.787238Z","shell.execute_reply.started":"2022-05-10T10:08:47.767256Z","shell.execute_reply":"2022-05-10T10:08:47.785814Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"now = datetime.now()\n\n# dd/mm/YY H:M:S\ncurrent_time = now.strftime(\"%d-%m-%H-%M\")\n\n\ndef save_performance(epoch):\n\n    torch.save(model, f'output/models/lan_model_{current_time}_ep_{epoch}.pth')\n    torch.save(model.state_dict(), f'output/models/lan_model_state_{current_time}_ep_{epoch}.pth')\n\n    with open(f'output/performances/acc_{current_time}_ep_{epoch}.pkl', 'wb') as filehandler:\n        pickle.dump(epoch_accuracies, filehandler)\n    with open(f'output/performances/loss_{current_time}_ep_{epoch}.pkl', 'wb') as filehandler:\n        pickle.dump(epoch_losses, filehandler)\n    with open(f'output/performances/precisions_{current_time}_ep_{epoch}.pkl', 'wb') as filehandler:\n        pickle.dump(epoch_precisions, filehandler)\n    with open(f'output/performances/recalls_{current_time}_ep_{epoch}.pkl', 'wb') as filehandler:\n        pickle.dump(epoch_recalls, filehandler)\n    with open(f'output/performances/f1s_{current_time}_ep_{epoch}.pkl', 'wb') as filehandler:\n        pickle.dump(epoch_f1s, filehandler)","metadata":{"cell_id":"58734992c6204fab8f8b588579ad0fbf","tags":[],"deepnote_to_be_reexecuted":false,"source_hash":"5f4b711d","execution_start":1652136088686,"execution_millis":4,"owner_user_id":"b081f53d-6461-4243-bd6e-c785d89042c0","deepnote_cell_type":"code","deepnote_cell_height":441,"execution":{"iopub.status.busy":"2022-05-10T10:08:49.840609Z","iopub.execute_input":"2022-05-10T10:08:49.840956Z","iopub.status.idle":"2022-05-10T10:08:49.850686Z","shell.execute_reply.started":"2022-05-10T10:08:49.840898Z","shell.execute_reply":"2022-05-10T10:08:49.849309Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!mkdir output\n!mkdir output/models\n!mkdir output/performances","metadata":{"execution":{"iopub.status.busy":"2022-05-10T08:49:47.074512Z","iopub.execute_input":"2022-05-10T08:49:47.075261Z","iopub.status.idle":"2022-05-10T08:49:50.271975Z","shell.execute_reply.started":"2022-05-10T08:49:47.075211Z","shell.execute_reply":"2022-05-10T08:49:50.270672Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.utils import class_weight\nclass_weights = class_weight.compute_class_weight(class_weight='balanced', classes=np.unique(y), y=y)\nclass_weights = torch.tensor(class_weights, dtype=torch.float)","metadata":{"execution":{"iopub.status.busy":"2022-05-10T09:49:04.433698Z","iopub.execute_input":"2022-05-10T09:49:04.434033Z","iopub.status.idle":"2022-05-10T09:49:04.444394Z","shell.execute_reply.started":"2022-05-10T09:49:04.433997Z","shell.execute_reply":"2022-05-10T09:49:04.44338Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(class_weights)","metadata":{"execution":{"iopub.status.busy":"2022-05-10T09:49:06.155389Z","iopub.execute_input":"2022-05-10T09:49:06.156139Z","iopub.status.idle":"2022-05-10T09:49:06.169658Z","shell.execute_reply.started":"2022-05-10T09:49:06.156093Z","shell.execute_reply":"2022-05-10T09:49:06.168708Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"EPOCHS = 1000\nSAVE_EVERY_N_EPOCHS = 50\nLEARNING_RATE = 1e-6\n\nloss_fn = nn.BCELoss()\n# optimizer = torch.optim.SGD(model.parameters(), lr=learning_rate)\noptimizer = torch.optim.Adam(model.parameters(), lr=LEARNING_RATE)\n\nepoch_accuracies = []\nepoch_losses = []\nepoch_precisions = []\nepoch_recalls = []\nepoch_f1s = []\nfor t in range(EPOCHS):\n    print(f\"Epoch {t+1}\\n-------------------------------\")\n    train_loop(train_dataloader, model, loss_fn, optimizer)\n    epoch_acc, epoch_avg_loss, epoch_precision, epoch_recall, epoch_f1 = test_loop(test_dataloader, model, loss_fn)\n    epoch_accuracies.append(epoch_acc)\n    epoch_losses.append(epoch_avg_loss)\n    epoch_precisions.append(epoch_precision)\n    epoch_recalls.append(epoch_recall)\n    epoch_f1s.append(epoch_f1)\n    \n    if (t+1)%SAVE_EVERY_N_EPOCHS==0:\n        save_performance(t+1)\n\nprint(\"Done!\")","metadata":{"tags":[],"deepnote_to_be_reexecuted":false,"source_hash":"8c74223","execution_start":1652136088698,"execution_millis":175520,"cell_id":"00017-d829a1c7-87c2-448d-88a6-6a4e18259484","deepnote_cell_type":"code","deepnote_cell_height":1169,"deepnote_output_heights":[null,328.3125],"execution":{"iopub.status.busy":"2022-05-10T10:09:22.094635Z","iopub.execute_input":"2022-05-10T10:09:22.09496Z","iopub.status.idle":"2022-05-10T10:09:49.2102Z","shell.execute_reply.started":"2022-05-10T10:09:22.094902Z","shell.execute_reply":"2022-05-10T10:09:49.208517Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Save Model + Performance","metadata":{"tags":[],"cell_id":"00018-06cafd8f-1ad5-4eff-9560-c52323a1fba2","deepnote_cell_type":"markdown","deepnote_cell_height":70}},{"cell_type":"code","source":"# model.eval()","metadata":{"cell_id":"38c3947866f64e49807d36350a089ec1","tags":[],"deepnote_to_be_reexecuted":false,"source_hash":"3bbccb1f","execution_start":1652136264223,"execution_millis":3824375,"deepnote_cell_type":"code","deepnote_cell_height":81,"deepnote_output_heights":[78.78125],"execution":{"iopub.status.busy":"2022-05-10T08:50:01.130512Z","iopub.status.idle":"2022-05-10T08:50:01.130949Z","shell.execute_reply.started":"2022-05-10T08:50:01.130687Z","shell.execute_reply":"2022-05-10T08:50:01.130718Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#P","metadata":{"cell_id":"3b8ece9ce28a403fab82edb4dd14a6f3","tags":[],"deepnote_to_be_reexecuted":false,"source_hash":"969bd9f4","execution_start":1652136264234,"execution_millis":3824383,"deepnote_cell_type":"code","deepnote_cell_height":81,"deepnote_output_heights":[21.1875],"execution":{"iopub.status.busy":"2022-05-10T08:50:01.134091Z","iopub.status.idle":"2022-05-10T08:50:01.134898Z","shell.execute_reply.started":"2022-05-10T08:50:01.134548Z","shell.execute_reply":"2022-05-10T08:50:01.134586Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"save_performance('FINAL')","metadata":{"cell_id":"96441f8534db46f0a3a6aee254b60cf1","tags":[],"deepnote_to_be_reexecuted":false,"source_hash":"20a748f3","execution_start":1652136264238,"execution_millis":30,"deepnote_cell_type":"code","deepnote_cell_height":81,"execution":{"iopub.status.busy":"2022-05-10T08:50:01.136494Z","iopub.status.idle":"2022-05-10T08:50:01.137354Z","shell.execute_reply.started":"2022-05-10T08:50:01.13699Z","shell.execute_reply":"2022-05-10T08:50:01.137029Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# next(iter(train_dataloader))[0].shape","metadata":{"tags":[],"deepnote_to_be_reexecuted":false,"source_hash":"dbab5091","execution_start":1652136264320,"execution_millis":3824432,"cell_id":"00019-803d628a-de38-4c83-840b-e72338d5dec0","deepnote_cell_type":"code","deepnote_cell_height":81,"deepnote_output_heights":[21.1875],"execution":{"iopub.status.busy":"2022-05-10T08:50:01.139004Z","iopub.status.idle":"2022-05-10T08:50:01.139785Z","shell.execute_reply.started":"2022-05-10T08:50:01.139443Z","shell.execute_reply":"2022-05-10T08:50:01.13948Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# clf.score(X_test, y_test)\n# X.shape[1]","metadata":{"tags":[],"deepnote_to_be_reexecuted":false,"source_hash":"1c43715c","execution_start":1652136264322,"execution_millis":3824473,"cell_id":"00020-223fbdcb-f0e4-47ab-8beb-5d4b071c4aac","deepnote_cell_type":"code","deepnote_cell_height":99,"execution":{"iopub.status.busy":"2022-05-10T08:50:01.141443Z","iopub.status.idle":"2022-05-10T08:50:01.142273Z","shell.execute_reply.started":"2022-05-10T08:50:01.14193Z","shell.execute_reply":"2022-05-10T08:50:01.14197Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{"tags":[],"cell_id":"00021-49a90ffa-6d73-48b9-ba14-832ece3f1e11","deepnote_cell_type":"markdown","deepnote_cell_height":46}},{"cell_type":"markdown","source":"<a style='text-decoration:none;line-height:16px;display:flex;color:#5B5B62;padding:10px;justify-content:end;' href='https://deepnote.com?utm_source=created-in-deepnote-cell&projectId=af2d1be0-78d1-4a96-889c-2df983a097e8' target=\"_blank\">\n<img alt='Created in deepnote.com' style='display:inline;max-height:16px;margin:0px;margin-right:7.5px;' src='data:image/svg+xml;base64,PD94bWwgdmVyc2lvbj0iMS4wIiBlbmNvZGluZz0iVVRGLTgiPz4KPHN2ZyB3aWR0aD0iODBweCIgaGVpZ2h0PSI4MHB4IiB2aWV3Qm94PSIwIDAgODAgODAiIHZlcnNpb249IjEuMSIgeG1sbnM9Imh0dHA6Ly93d3cudzMub3JnLzIwMDAvc3ZnIiB4bWxuczp4bGluaz0iaHR0cDovL3d3dy53My5vcmcvMTk5OS94bGluayI+CiAgICA8IS0tIEdlbmVyYXRvcjogU2tldGNoIDU0LjEgKDc2NDkwKSAtIGh0dHBzOi8vc2tldGNoYXBwLmNvbSAtLT4KICAgIDx0aXRsZT5Hcm91cCAzPC90aXRsZT4KICAgIDxkZXNjPkNyZWF0ZWQgd2l0aCBTa2V0Y2guPC9kZXNjPgogICAgPGcgaWQ9IkxhbmRpbmciIHN0cm9rZT0ibm9uZSIgc3Ryb2tlLXdpZHRoPSIxIiBmaWxsPSJub25lIiBmaWxsLXJ1bGU9ImV2ZW5vZGQiPgogICAgICAgIDxnIGlkPSJBcnRib2FyZCIgdHJhbnNmb3JtPSJ0cmFuc2xhdGUoLTEyMzUuMDAwMDAwLCAtNzkuMDAwMDAwKSI+CiAgICAgICAgICAgIDxnIGlkPSJHcm91cC0zIiB0cmFuc2Zvcm09InRyYW5zbGF0ZSgxMjM1LjAwMDAwMCwgNzkuMDAwMDAwKSI+CiAgICAgICAgICAgICAgICA8cG9seWdvbiBpZD0iUGF0aC0yMCIgZmlsbD0iIzAyNjVCNCIgcG9pbnRzPSIyLjM3NjIzNzYyIDgwIDM4LjA0NzY2NjcgODAgNTcuODIxNzgyMiA3My44MDU3NTkyIDU3LjgyMTc4MjIgMzIuNzU5MjczOSAzOS4xNDAyMjc4IDMxLjY4MzE2ODMiPjwvcG9seWdvbj4KICAgICAgICAgICAgICAgIDxwYXRoIGQ9Ik0zNS4wMDc3MTgsODAgQzQyLjkwNjIwMDcsNzYuNDU0OTM1OCA0Ny41NjQ5MTY3LDcxLjU0MjI2NzEgNDguOTgzODY2LDY1LjI2MTk5MzkgQzUxLjExMjI4OTksNTUuODQxNTg0MiA0MS42NzcxNzk1LDQ5LjIxMjIyODQgMjUuNjIzOTg0Niw0OS4yMTIyMjg0IEMyNS40ODQ5Mjg5LDQ5LjEyNjg0NDggMjkuODI2MTI5Niw0My4yODM4MjQ4IDM4LjY0NzU4NjksMzEuNjgzMTY4MyBMNzIuODcxMjg3MSwzMi41NTQ0MjUgTDY1LjI4MDk3Myw2Ny42NzYzNDIxIEw1MS4xMTIyODk5LDc3LjM3NjE0NCBMMzUuMDA3NzE4LDgwIFoiIGlkPSJQYXRoLTIyIiBmaWxsPSIjMDAyODY4Ij48L3BhdGg+CiAgICAgICAgICAgICAgICA8cGF0aCBkPSJNMCwzNy43MzA0NDA1IEwyNy4xMTQ1MzcsMC4yNTcxMTE0MzYgQzYyLjM3MTUxMjMsLTEuOTkwNzE3MDEgODAsMTAuNTAwMzkyNyA4MCwzNy43MzA0NDA1IEM4MCw2NC45NjA0ODgyIDY0Ljc3NjUwMzgsNzkuMDUwMzQxNCAzNC4zMjk1MTEzLDgwIEM0Ny4wNTUzNDg5LDc3LjU2NzA4MDggNTMuNDE4MjY3Nyw3MC4zMTM2MTAzIDUzLjQxODI2NzcsNTguMjM5NTg4NSBDNTMuNDE4MjY3Nyw0MC4xMjg1NTU3IDM2LjMwMzk1NDQsMzcuNzMwNDQwNSAyNS4yMjc0MTcsMzcuNzMwNDQwNSBDMTcuODQzMDU4NiwzNy43MzA0NDA1IDkuNDMzOTE5NjYsMzcuNzMwNDQwNSAwLDM3LjczMDQ0MDUgWiIgaWQ9IlBhdGgtMTkiIGZpbGw9IiMzNzkzRUYiPjwvcGF0aD4KICAgICAgICAgICAgPC9nPgogICAgICAgIDwvZz4KICAgIDwvZz4KPC9zdmc+' > </img>\nCreated in <span style='font-weight:600;margin-left:4px;'>Deepnote</span></a>","metadata":{"tags":[],"created_in_deepnote_cell":true,"deepnote_cell_type":"markdown"}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}