{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Part 1: Exploratory Data Analysis","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\n%matplotlib inline\n\n\nplt.style.use(\"seaborn\")\npd.set_option(\"display.max_columns\", None)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-02-28T22:02:44.118808Z","iopub.execute_input":"2022-02-28T22:02:44.119224Z","iopub.status.idle":"2022-02-28T22:02:44.133838Z","shell.execute_reply.started":"2022-02-28T22:02:44.119125Z","shell.execute_reply":"2022-02-28T22:02:44.132975Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.read_csv(\"../input/predict-closed-questions-on-stack-overflow/train-sample.csv\")\ndf.head()","metadata":{"execution":{"iopub.status.busy":"2022-02-28T22:02:44.135941Z","iopub.execute_input":"2022-02-28T22:02:44.136268Z","iopub.status.idle":"2022-02-28T22:02:46.145100Z","shell.execute_reply.started":"2022-02-28T22:02:44.136226Z","shell.execute_reply":"2022-02-28T22:02:46.144375Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Total rows:\", df.shape[0])\nprint(\"Total columns:\", df.shape[1])","metadata":{"execution":{"iopub.status.busy":"2022-02-28T22:02:46.146356Z","iopub.execute_input":"2022-02-28T22:02:46.146722Z","iopub.status.idle":"2022-02-28T22:02:46.153183Z","shell.execute_reply.started":"2022-02-28T22:02:46.146679Z","shell.execute_reply":"2022-02-28T22:02:46.151868Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The **PostId** and **OwnerUserId** columns are ID columns and do not have any predictive power so they can be dropped.","metadata":{}},{"cell_type":"code","source":"df.drop(columns=[\"PostId\", \"OwnerUserId\"], inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-02-28T22:02:46.155599Z","iopub.execute_input":"2022-02-28T22:02:46.155920Z","iopub.status.idle":"2022-02-28T22:02:46.183750Z","shell.execute_reply.started":"2022-02-28T22:02:46.155846Z","shell.execute_reply":"2022-02-28T22:02:46.183043Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Analysing the number of unique values in various columns","metadata":{}},{"cell_type":"code","source":"df.nunique()","metadata":{"execution":{"iopub.status.busy":"2022-02-28T22:02:46.185224Z","iopub.execute_input":"2022-02-28T22:02:46.185511Z","iopub.status.idle":"2022-02-28T22:02:46.711878Z","shell.execute_reply.started":"2022-02-28T22:02:46.185475Z","shell.execute_reply":"2022-02-28T22:02:46.711163Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Analysing the percentage of missing values in various columns:","metadata":{}},{"cell_type":"code","source":"df.isna().sum()*100/df.shape[0]","metadata":{"execution":{"iopub.status.busy":"2022-02-28T22:02:46.713309Z","iopub.execute_input":"2022-02-28T22:02:46.713592Z","iopub.status.idle":"2022-02-28T22:02:46.855972Z","shell.execute_reply.started":"2022-02-28T22:02:46.713555Z","shell.execute_reply":"2022-02-28T22:02:46.855295Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"There are so many missing values in the columns **Tag2** to **Tag5** and **PostClosedDate** but only a small fraction of data points (about 160) in the **Tag1** column is missing. This tells that each question has atleast one tag associated with it. Moreover, some questions do not have any **BodyMarkdown** thus the content of such questions must be contained in the **Title**.","metadata":{}},{"cell_type":"code","source":"df.loc[df[\"Tag1\"].isna(), \"Tag1\"] = df[\"Tag1\"].mode()\n\ndrop_cols = [\"Tag2\", \"Tag3\", \"Tag4\", \"Tag5\", \"PostClosedDate\"]\ndf.drop(drop_cols, axis=1, inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-02-28T22:02:46.857459Z","iopub.execute_input":"2022-02-28T22:02:46.857728Z","iopub.status.idle":"2022-02-28T22:02:46.919349Z","shell.execute_reply.started":"2022-02-28T22:02:46.857692Z","shell.execute_reply":"2022-02-28T22:02:46.918615Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Renaming some columns with very long names","metadata":{}},{"cell_type":"code","source":"name_map = {\n    \"PostCreationDate\": \"PostDate\",\n    \"OwnerCreationDate\": \"OwnrDate\",\n    \"ReputationAtPostCreation\": \"OwnrRep\",\n    \"OwnerUndeletedAnswerCountAtPostTime\": \"AnsCount\"\n}\n\ndf.rename(columns=name_map, inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-02-28T22:02:46.922484Z","iopub.execute_input":"2022-02-28T22:02:46.922699Z","iopub.status.idle":"2022-02-28T22:02:46.929582Z","shell.execute_reply.started":"2022-02-28T22:02:46.922672Z","shell.execute_reply":"2022-02-28T22:02:46.928869Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Part 2: Feature Engineering","metadata":{}},{"cell_type":"markdown","source":"We can extract several date and time features from the columns - **PostCreationDate** and **OwnerCreationDate** - and binary encoding them. Further, we can create a new feature signifying how old the owner account is by finding the difference between these two datetime columns in seconds.","metadata":{}},{"cell_type":"code","source":"import category_encoders as ce\n\n\ndef datetime_features(df):\n    df[\"PostDate\"] = pd.to_datetime(df[\"PostDate\"])\n    df[\"OwnrDate\"] = pd.to_datetime(df[\"OwnrDate\"])\n\n    df[\"PostDay\"] = df[\"PostDate\"].dt.dayofweek\n    df[\"PostMonth\"] = df[\"PostDate\"].dt.month\n    df[\"PostYear\"] = df[\"PostDate\"].dt.year\n    df[\"PostHour\"] = df[\"PostDate\"].dt.hour\n    df[\"PostMin\"] = df[\"PostDate\"].dt.minute\n\n    df[\"OwnrDay\"] = df[\"OwnrDate\"].dt.weekday\n    df[\"OwnrMonth\"] = df[\"OwnrDate\"].dt.month\n    df[\"OwnrYear\"] = df[\"OwnrDate\"].dt.year\n    df[\"OwnrHour\"] = df[\"OwnrDate\"].dt.hour\n    df[\"OwnrMin\"] = df[\"OwnrDate\"].dt.minute\n\n    df[\"AccAge\"] = (df[\"PostDate\"] - df[\"OwnrDate\"])/np.timedelta64(1, 's')\n\n    del df[\"PostDate\"]\n    del df[\"OwnrDate\"]\n\n    cols = [\n        \"PostDay\",\n        \"PostMonth\",\n        \"PostYear\",\n        \"PostHour\",\n        \"PostMin\",\n        \"OwnrDay\",\n        \"OwnrMonth\",\n        \"OwnrYear\",\n        \"OwnrHour\",\n        \"OwnrMin\"\n    ]\n\n    encoder = ce.binary.BinaryEncoder(cols=cols)\n    sub_cols = encoder.fit_transform(df[cols])\n\n    df = pd.concat([df, sub_cols], axis=1)\n    df.drop(cols, axis=1, inplace=True)\n\n    return df\n\n\ndf = datetime_features(df)","metadata":{"execution":{"iopub.status.busy":"2022-02-28T22:02:46.932113Z","iopub.execute_input":"2022-02-28T22:02:46.932359Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Concatenating **Title** and **BodyMarkdown** to form a new column **QuestionText**.","metadata":{}},{"cell_type":"code","source":"df[\"BodyMarkdown\"] = df[\"Title\"] + df[\"BodyMarkdown\"]\ndf.drop(columns=[\"Title\"], axis=1, inplace=True)\ndf.rename(columns={\"BodyMarkdown\":\"QuestionText\"}, inplace=True)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Since, **Tag1** has many classes, one-hot encoding won't be a good idea. Performing binary encoding to encode all the categorical features. Also label encoding the target label since they are strings.","metadata":{}},{"cell_type":"code","source":"import category_encoders as ce\n\n\nencoder = ce.binary.BinaryEncoder()\ntag1_bits = encoder.fit_transform(df[\"Tag1\"])\n\ndf = pd.concat([df, tag1_bits], axis=1)\ndf.drop([\"Tag1\"], axis=1, inplace=True)\n\ntarget_map = {\n    \"not a real question\": 0,\n    \"not constructive\": 1,\n    \"off topic\": 2,\n    \"open\": 3,\n    \"too localized\": 4\n}\n\ndf[\"OpenStatus\"] = df[\"OpenStatus\"].map(target_map)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Part 3: Model building","metadata":{}},{"cell_type":"markdown","source":"Splitting the dataset into training and validation sets and then scaling with sklearn RobustScaler.","metadata":{}},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\nfrom sklearn.preprocessing import RobustScaler\n\n\ntrain_df, val_df = train_test_split(df, test_size=0.2, shuffle=True, random_state=42, stratify=df[\"OpenStatus\"])\ndel df\n\ntrain_text = train_df[[\"QuestionText\"]]\ntrain_meta = train_df.drop([\"OpenStatus\", \"QuestionText\"], axis=1)\ntrain_target = train_df[\"OpenStatus\"]\ndel train_df\n\nval_text = val_df[[\"QuestionText\"]]\nval_meta = val_df.drop([\"OpenStatus\", \"QuestionText\"], axis=1)\nval_target = val_df[\"OpenStatus\"]\ndel val_df\n\nscaler = RobustScaler()\ntrain_meta = scaler.fit_transform(train_meta)\nval_meta = scaler.transform(val_meta)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Since the dateset contains both text and meta data, we will require a multi-input neural network to process different kinds of input data type. The text data will pass through stacked LSTMs and then join with meta data where they will further pass through dense layers.","metadata":{}},{"cell_type":"code","source":"from tensorflow.keras.layers import TextVectorization\n\n\ntext2vec = TextVectorization()\ntext2vec.adapt(train_text)\n\nprint(\"Total tokens in training data:\", text2vec.vocabulary_size())\nprint(\"Largest length of any sequence:\", text2vec(train_text).shape[1])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import tensorflow as tf\nfrom tensorflow.keras import layers\nfrom tensorflow.keras import Model\n\n\nMAX_TOKENS = 10000\nMAX_LEN = 150\nEMBED_DIM = 50\nNUM_CLASSES = train_target.nunique()\n\ntext2vec = TextVectorization(max_tokens=MAX_TOKENS, output_sequence_length=MAX_LEN, name=\"text2vec\")\ntext2vec.adapt(train_text)\n\ntext_input = layers.Input(shape=(1,), dtype=tf.string, name=\"text_input\")\nx = text2vec(text_input)\nx = layers.Embedding(input_dim=MAX_TOKENS, output_dim=EMBED_DIM, input_length=MAX_LEN, name=\"embedding\")(x)\nx = layers.LSTM(units=128, name=\"hidden_lstm\")(x)\n\nmeta_input = layers.Input(shape=train_meta.shape[1:], name=\"meta_input\")\ny = layers.Concatenate()([x, meta_input])\ny = layers.Dense(units=256, activation=\"selu\", kernel_initializer=\"lecun_normal\", name=\"hidden_dense_1\")(y)\ny = layers.BatchNormalization()(y)\ny = layers.Dense(units=64, activation=\"selu\", kernel_initializer=\"lecun_normal\", name=\"hidden_dense_2\")(y)\ny = layers.Dense(units=NUM_CLASSES, activation=\"softmax\", name=\"softmax_output\")(y)\n\nmodel = Model(inputs=[text_input, meta_input], outputs=y, name=\"NLP_Model\")\n\nmodel.compile(\n    optimizer=\"adam\",\n    loss=\"sparse_categorical_crossentropy\",\n    metrics=[\"accuracy\"]\n)\n\nmodel.summary()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tensorflow.keras.utils import plot_model\n\n\nplot_model(\n    model=model,\n    to_file=\"model.jpeg\",\n    show_shapes=True\n)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tensorflow.keras.callbacks import ReduceLROnPlateau, EarlyStopping\n\n\nreduce_lr = ReduceLROnPlateau(\n    monitor=\"val_loss\",\n    factor=0.5,\n    patience=5,\n    verbose=True\n)\n\nearly_stop = EarlyStopping(\n    monitor=\"val_loss\",\n    patience=20,\n    restore_best_weights=True,\n    verbose=True\n)\n\ncallbacks = [reduce_lr, early_stop]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history = model.fit(\n    x=[train_text, train_meta],\n    y=train_target,\n    batch_size=256,\n    epochs=100,\n    verbose=1,\n    callbacks=callbacks,\n    validation_data=([val_text, val_meta], val_target),\n    shuffle=True\n)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.evaluate([val_text, val_meta], val_target)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}