{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport seaborn as sb\n%matplotlib inline\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-10-19T20:18:01.734492Z","iopub.execute_input":"2022-10-19T20:18:01.734885Z","iopub.status.idle":"2022-10-19T20:18:01.748575Z","shell.execute_reply.started":"2022-10-19T20:18:01.73484Z","shell.execute_reply":"2022-10-19T20:18:01.747359Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train=pd.read_csv('../input/predict-closed-questions-on-stack-overflow/train-sample.csv')\ndf_train.head()","metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","execution":{"iopub.status.busy":"2022-10-19T20:18:23.200974Z","iopub.execute_input":"2022-10-19T20:18:23.20133Z","iopub.status.idle":"2022-10-19T20:18:25.128952Z","shell.execute_reply.started":"2022-10-19T20:18:23.2013Z","shell.execute_reply":"2022-10-19T20:18:25.127912Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\ntrain_df,test_df=train_test_split(df_train,test_size=0.2)\nprint('Training data shape: {}'.format(train_df.shape))\nprint('Testing data shape: {}'.format(test_df.shape))\n","metadata":{"execution":{"iopub.status.busy":"2022-10-19T20:18:30.179097Z","iopub.execute_input":"2022-10-19T20:18:30.179487Z","iopub.status.idle":"2022-10-19T20:18:30.285678Z","shell.execute_reply.started":"2022-10-19T20:18:30.179449Z","shell.execute_reply":"2022-10-19T20:18:30.284595Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Exploratory Data Analysis","metadata":{}},{"cell_type":"code","source":"print(train_df.columns)\ncorr = train_df.corr()\nsb.heatmap(corr, cmap=\"Blues\", annot=True)","metadata":{"execution":{"iopub.status.busy":"2022-10-19T20:18:34.90828Z","iopub.execute_input":"2022-10-19T20:18:34.908684Z","iopub.status.idle":"2022-10-19T20:18:35.193497Z","shell.execute_reply.started":"2022-10-19T20:18:34.908628Z","shell.execute_reply":"2022-10-19T20:18:35.192446Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## OpenStatus","metadata":{}},{"cell_type":"code","source":"train_df['OpenStatus'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-10-19T20:14:21.3514Z","iopub.execute_input":"2022-10-19T20:14:21.3518Z","iopub.status.idle":"2022-10-19T20:14:21.373528Z","shell.execute_reply.started":"2022-10-19T20:14:21.351766Z","shell.execute_reply":"2022-10-19T20:14:21.372515Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## PostID","metadata":{}},{"cell_type":"markdown","source":"This feature is the id of the query posted by the user. This feauture does not have any importance in making the prediction but it will be used later for submission we will save it.","metadata":{}},{"cell_type":"code","source":"train_post_id=train_df.PostId\ntest_post_id=test_df.PostId","metadata":{"execution":{"iopub.status.busy":"2022-10-19T20:06:14.39671Z","iopub.execute_input":"2022-10-19T20:06:14.397097Z","iopub.status.idle":"2022-10-19T20:06:14.404298Z","shell.execute_reply.started":"2022-10-19T20:06:14.397064Z","shell.execute_reply":"2022-10-19T20:06:14.402865Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## PostCreationDate\n\nThe post creation date is the date and time on which the query was posted by the user let us see if we could make something out of this.","metadata":{}},{"cell_type":"code","source":"train_df.query('OpenStatus=open').shape","metadata":{"execution":{"iopub.status.busy":"2022-10-19T20:12:00.37827Z","iopub.status.idle":"2022-10-19T20:12:00.378803Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"As we have observed that it doesn't matter on the time the question is posted on its label thus we can just remove this column both from our train and test.","metadata":{}},{"cell_type":"code","source":"train_df.drop(['PostCreationDate'],axis=1,inplace=True)\ntest_df.drop(['PostCreationDate'],axis=1,inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-10-19T20:06:14.409278Z","iopub.execute_input":"2022-10-19T20:06:14.410329Z","iopub.status.idle":"2022-10-19T20:06:14.472283Z","shell.execute_reply.started":"2022-10-19T20:06:14.410273Z","shell.execute_reply":"2022-10-19T20:06:14.470975Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.head(2)","metadata":{"execution":{"iopub.status.busy":"2022-10-19T20:06:14.474344Z","iopub.execute_input":"2022-10-19T20:06:14.474861Z","iopub.status.idle":"2022-10-19T20:06:14.495151Z","shell.execute_reply.started":"2022-10-19T20:06:14.47478Z","shell.execute_reply":"2022-10-19T20:06:14.494195Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## OwnerUserId","metadata":{}},{"cell_type":"markdown","source":"This is the user id who posted the query. Let us check it","metadata":{}},{"cell_type":"code","source":"train_df['PostCount']=[1]*len(train_df['PostId'])","metadata":{"execution":{"iopub.status.busy":"2022-10-19T20:06:14.496309Z","iopub.execute_input":"2022-10-19T20:06:14.496799Z","iopub.status.idle":"2022-10-19T20:06:14.534967Z","shell.execute_reply.started":"2022-10-19T20:06:14.496764Z","shell.execute_reply":"2022-10-19T20:06:14.533952Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.pivot_table(train_df,index='OwnerUserId',columns='OpenStatus',values='PostCount',aggfunc='sum')","metadata":{"execution":{"iopub.status.busy":"2022-10-19T20:06:14.536726Z","iopub.execute_input":"2022-10-19T20:06:14.53714Z","iopub.status.idle":"2022-10-19T20:06:14.722782Z","shell.execute_reply.started":"2022-10-19T20:06:14.537084Z","shell.execute_reply":"2022-10-19T20:06:14.72157Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.loc[train_df['OpenStatus']=='open','OwnerUserId'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-10-19T20:06:14.724169Z","iopub.execute_input":"2022-10-19T20:06:14.724652Z","iopub.status.idle":"2022-10-19T20:06:14.752339Z","shell.execute_reply.started":"2022-10-19T20:06:14.724616Z","shell.execute_reply":"2022-10-19T20:06:14.750259Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We see that it does not matter on the OwnerUserId whether his question is going to be open or not as there are many user whose only one question remain open \nSo, we would be dropping this column","metadata":{}},{"cell_type":"code","source":"train_df.drop(['OwnerUserId'],axis=1,inplace=True)\ntest_df.drop(['OwnerUserId'],axis=1,inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-10-19T20:06:14.755086Z","iopub.execute_input":"2022-10-19T20:06:14.756008Z","iopub.status.idle":"2022-10-19T20:06:14.796989Z","shell.execute_reply.started":"2022-10-19T20:06:14.755948Z","shell.execute_reply":"2022-10-19T20:06:14.795607Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Reputation at Post Created\n\nThis can be a important factor. Let's take a look at it","metadata":{}},{"cell_type":"code","source":"df_train['ReputationAtPostCreation'].min()","metadata":{"execution":{"iopub.status.busy":"2022-10-19T20:06:14.79921Z","iopub.execute_input":"2022-10-19T20:06:14.800068Z","iopub.status.idle":"2022-10-19T20:06:14.807591Z","shell.execute_reply.started":"2022-10-19T20:06:14.799997Z","shell.execute_reply":"2022-10-19T20:06:14.806535Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train['ReputationAtPostCreation'].max()","metadata":{"execution":{"iopub.status.busy":"2022-10-19T20:06:14.809716Z","iopub.execute_input":"2022-10-19T20:06:14.810213Z","iopub.status.idle":"2022-10-19T20:06:14.821392Z","shell.execute_reply.started":"2022-10-19T20:06:14.810166Z","shell.execute_reply":"2022-10-19T20:06:14.82027Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Since the data range is such high let us scale it using MinMaxScaler","metadata":{}},{"cell_type":"code","source":"minimum=df_train['ReputationAtPostCreation'].min()\nmaximum=df_train['ReputationAtPostCreation'].max()","metadata":{"execution":{"iopub.status.busy":"2022-10-19T20:06:14.822748Z","iopub.execute_input":"2022-10-19T20:06:14.823074Z","iopub.status.idle":"2022-10-19T20:06:14.83563Z","shell.execute_reply.started":"2022-10-19T20:06:14.823045Z","shell.execute_reply":"2022-10-19T20:06:14.834428Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['ReputationAtPostCreation']=(train_df['ReputationAtPostCreation']-minimum)/(maximum-minimum)","metadata":{"execution":{"iopub.status.busy":"2022-10-19T20:06:14.837185Z","iopub.execute_input":"2022-10-19T20:06:14.837991Z","iopub.status.idle":"2022-10-19T20:06:14.851995Z","shell.execute_reply.started":"2022-10-19T20:06:14.837945Z","shell.execute_reply":"2022-10-19T20:06:14.850632Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df['ReputationAtPostCreation']=(test_df['ReputationAtPostCreation']-minimum)/(maximum-minimum)","metadata":{"execution":{"iopub.status.busy":"2022-10-19T20:06:14.856533Z","iopub.execute_input":"2022-10-19T20:06:14.856904Z","iopub.status.idle":"2022-10-19T20:06:14.866124Z","shell.execute_reply.started":"2022-10-19T20:06:14.856874Z","shell.execute_reply":"2022-10-19T20:06:14.864806Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## OwnerUndeletedAnswerCountAtPostTime ","metadata":{}},{"cell_type":"code","source":"train_df.OwnerUndeletedAnswerCountAtPostTime.value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-10-19T20:06:14.868347Z","iopub.execute_input":"2022-10-19T20:06:14.86872Z","iopub.status.idle":"2022-10-19T20:06:14.883482Z","shell.execute_reply.started":"2022-10-19T20:06:14.868686Z","shell.execute_reply":"2022-10-19T20:06:14.882267Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"This feature doesnot seem to be doing anything, we will be dropping it for now.","metadata":{}},{"cell_type":"code","source":"train_df.drop(['OwnerUndeletedAnswerCountAtPostTime'],axis=1,inplace=True)\ntest_df.drop(['OwnerUndeletedAnswerCountAtPostTime'],axis=1,inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-10-19T20:06:14.885138Z","iopub.execute_input":"2022-10-19T20:06:14.886116Z","iopub.status.idle":"2022-10-19T20:06:14.938657Z","shell.execute_reply.started":"2022-10-19T20:06:14.886067Z","shell.execute_reply":"2022-10-19T20:06:14.93757Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## PostClosedDate","metadata":{}},{"cell_type":"markdown","source":"Dropping this column as post closed date does not seem to be doing anything, so we would be dropping it ","metadata":{}},{"cell_type":"code","source":"train_df.drop(['PostClosedDate'],axis=1,inplace=True)\ntest_df.drop(['PostClosedDate'],axis=1,inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-10-19T20:06:14.940206Z","iopub.execute_input":"2022-10-19T20:06:14.940829Z","iopub.status.idle":"2022-10-19T20:06:14.98541Z","shell.execute_reply.started":"2022-10-19T20:06:14.940775Z","shell.execute_reply":"2022-10-19T20:06:14.984472Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Tag1","metadata":{}},{"cell_type":"code","source":"train_df['Tag1'].isnull().sum()","metadata":{"execution":{"iopub.status.busy":"2022-10-19T20:06:14.987372Z","iopub.execute_input":"2022-10-19T20:06:14.987772Z","iopub.status.idle":"2022-10-19T20:06:15.002303Z","shell.execute_reply.started":"2022-10-19T20:06:14.987735Z","shell.execute_reply":"2022-10-19T20:06:15.001348Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df['Tag1'].isnull().sum()","metadata":{"execution":{"iopub.status.busy":"2022-10-19T20:06:15.003393Z","iopub.execute_input":"2022-10-19T20:06:15.003866Z","iopub.status.idle":"2022-10-19T20:06:15.017609Z","shell.execute_reply.started":"2022-10-19T20:06:15.003806Z","shell.execute_reply":"2022-10-19T20:06:15.016734Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We can combine all the tags column into one this would help us in comparing the words which are used as tags and the words used in Title and Body.","metadata":{}},{"cell_type":"code","source":"train_df['Tag1']=train_df['Tag1'].replace(np.nan,' ')","metadata":{"execution":{"iopub.status.busy":"2022-10-19T20:06:15.018741Z","iopub.execute_input":"2022-10-19T20:06:15.019249Z","iopub.status.idle":"2022-10-19T20:06:15.044663Z","shell.execute_reply.started":"2022-10-19T20:06:15.019215Z","shell.execute_reply":"2022-10-19T20:06:15.042883Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['Tag2']=train_df['Tag2'].replace(np.nan,' ')\ntrain_df['Tag3']=train_df['Tag3'].replace(np.nan,' ')\ntrain_df['Tag4']=train_df['Tag4'].replace(np.nan,' ')\ntrain_df['Tag5']=train_df['Tag5'].replace(np.nan,' ')","metadata":{"execution":{"iopub.status.busy":"2022-10-19T20:06:15.046461Z","iopub.execute_input":"2022-10-19T20:06:15.047061Z","iopub.status.idle":"2022-10-19T20:06:15.110163Z","shell.execute_reply.started":"2022-10-19T20:06:15.047016Z","shell.execute_reply":"2022-10-19T20:06:15.108683Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df['Tag1']=test_df['Tag1'].replace(np.nan,' ')\ntest_df['Tag2']=test_df['Tag2'].replace(np.nan,' ')\ntest_df['Tag3']=test_df['Tag3'].replace(np.nan,' ')\ntest_df['Tag4']=test_df['Tag4'].replace(np.nan,' ')\ntest_df['Tag5']=test_df['Tag5'].replace(np.nan,' ')","metadata":{"execution":{"iopub.status.busy":"2022-10-19T20:06:15.111605Z","iopub.execute_input":"2022-10-19T20:06:15.111952Z","iopub.status.idle":"2022-10-19T20:06:15.146314Z","shell.execute_reply.started":"2022-10-19T20:06:15.111919Z","shell.execute_reply":"2022-10-19T20:06:15.144035Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['Tag1']","metadata":{"execution":{"iopub.status.busy":"2022-10-19T20:06:15.147806Z","iopub.execute_input":"2022-10-19T20:06:15.148143Z","iopub.status.idle":"2022-10-19T20:06:15.157174Z","shell.execute_reply.started":"2022-10-19T20:06:15.148114Z","shell.execute_reply":"2022-10-19T20:06:15.155759Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['Tags']=train_df['Tag1']+' '+train_df['Tag2']+' '+train_df['Tag3']+' '+train_df['Tag4']+' '+train_df['Tag5']","metadata":{"execution":{"iopub.status.busy":"2022-10-19T20:06:15.159427Z","iopub.execute_input":"2022-10-19T20:06:15.159995Z","iopub.status.idle":"2022-10-19T20:06:15.324365Z","shell.execute_reply.started":"2022-10-19T20:06:15.159945Z","shell.execute_reply":"2022-10-19T20:06:15.322287Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df['Tags']=test_df['Tag1']+' '+test_df['Tag2']+' '+test_df['Tag3']+' '+test_df['Tag4']+' '+test_df['Tag5']","metadata":{"execution":{"iopub.status.busy":"2022-10-19T20:06:15.325788Z","iopub.execute_input":"2022-10-19T20:06:15.326168Z","iopub.status.idle":"2022-10-19T20:06:15.37505Z","shell.execute_reply.started":"2022-10-19T20:06:15.326111Z","shell.execute_reply":"2022-10-19T20:06:15.374039Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['Tags']=train_df['Tags'].str.lower()","metadata":{"execution":{"iopub.status.busy":"2022-10-19T20:06:15.376289Z","iopub.execute_input":"2022-10-19T20:06:15.376616Z","iopub.status.idle":"2022-10-19T20:06:15.444838Z","shell.execute_reply.started":"2022-10-19T20:06:15.376586Z","shell.execute_reply":"2022-10-19T20:06:15.44375Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df['Tags']=test_df['Tags'].str.lower()","metadata":{"execution":{"iopub.status.busy":"2022-10-19T20:06:15.446235Z","iopub.execute_input":"2022-10-19T20:06:15.446567Z","iopub.status.idle":"2022-10-19T20:06:15.471082Z","shell.execute_reply.started":"2022-10-19T20:06:15.446525Z","shell.execute_reply":"2022-10-19T20:06:15.470005Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df['Tags']","metadata":{"execution":{"iopub.status.busy":"2022-10-19T20:06:15.472768Z","iopub.execute_input":"2022-10-19T20:06:15.473126Z","iopub.status.idle":"2022-10-19T20:06:15.482383Z","shell.execute_reply.started":"2022-10-19T20:06:15.473095Z","shell.execute_reply":"2022-10-19T20:06:15.481129Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['Tags']=train_df['Tags'].apply(lambda x:x.lstrip())\ntrain_df['Tags']=train_df['Tags'].apply(lambda x:x.rstrip())\n","metadata":{"execution":{"iopub.status.busy":"2022-10-19T20:06:15.484234Z","iopub.execute_input":"2022-10-19T20:06:15.484638Z","iopub.status.idle":"2022-10-19T20:06:15.577129Z","shell.execute_reply.started":"2022-10-19T20:06:15.484602Z","shell.execute_reply":"2022-10-19T20:06:15.576143Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['Tags']","metadata":{"execution":{"iopub.status.busy":"2022-10-19T20:06:15.583751Z","iopub.execute_input":"2022-10-19T20:06:15.584182Z","iopub.status.idle":"2022-10-19T20:06:15.595196Z","shell.execute_reply.started":"2022-10-19T20:06:15.584147Z","shell.execute_reply":"2022-10-19T20:06:15.593731Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df['Tags']=test_df['Tags'].apply(lambda x:x.lstrip())\ntest_df['Tags']=test_df['Tags'].apply(lambda x:x.rstrip())","metadata":{"execution":{"iopub.status.busy":"2022-10-19T20:06:15.596942Z","iopub.execute_input":"2022-10-19T20:06:15.597262Z","iopub.status.idle":"2022-10-19T20:06:15.633557Z","shell.execute_reply.started":"2022-10-19T20:06:15.597233Z","shell.execute_reply":"2022-10-19T20:06:15.631598Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.head(1)","metadata":{"execution":{"iopub.status.busy":"2022-10-19T20:06:15.635475Z","iopub.execute_input":"2022-10-19T20:06:15.635985Z","iopub.status.idle":"2022-10-19T20:06:15.657687Z","shell.execute_reply.started":"2022-10-19T20:06:15.635937Z","shell.execute_reply":"2022-10-19T20:06:15.655805Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Dropping excess columns \ntrain_df.drop(['PostId','OwnerCreationDate','Tag1','Tag2','Tag3','Tag4','Tag5','PostCount'],axis=1,inplace=True)\ntest_df.drop(['PostId','OwnerCreationDate','Tag1','Tag2','Tag3','Tag4','Tag5'],axis=1,inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-10-19T20:06:15.660107Z","iopub.execute_input":"2022-10-19T20:06:15.660719Z","iopub.status.idle":"2022-10-19T20:06:15.767204Z","shell.execute_reply.started":"2022-10-19T20:06:15.660667Z","shell.execute_reply":"2022-10-19T20:06:15.766064Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-10-19T20:06:15.768972Z","iopub.execute_input":"2022-10-19T20:06:15.76943Z","iopub.status.idle":"2022-10-19T20:06:15.785307Z","shell.execute_reply.started":"2022-10-19T20:06:15.769393Z","shell.execute_reply":"2022-10-19T20:06:15.783783Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-10-19T20:06:15.787137Z","iopub.execute_input":"2022-10-19T20:06:15.787582Z","iopub.status.idle":"2022-10-19T20:06:15.808445Z","shell.execute_reply.started":"2022-10-19T20:06:15.787538Z","shell.execute_reply":"2022-10-19T20:06:15.807184Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_train=train_df['OpenStatus']\ny_test=test_df['OpenStatus']","metadata":{"execution":{"iopub.status.busy":"2022-10-19T20:06:15.809687Z","iopub.execute_input":"2022-10-19T20:06:15.810083Z","iopub.status.idle":"2022-10-19T20:06:15.821637Z","shell.execute_reply.started":"2022-10-19T20:06:15.810021Z","shell.execute_reply":"2022-10-19T20:06:15.819553Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.drop(['ReputationAtPostCreation','OpenStatus'],axis=1,inplace=True)\ntest_df.drop(['ReputationAtPostCreation','OpenStatus'],axis=1,inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-10-19T20:06:15.823158Z","iopub.execute_input":"2022-10-19T20:06:15.823573Z","iopub.status.idle":"2022-10-19T20:06:15.854295Z","shell.execute_reply.started":"2022-10-19T20:06:15.823527Z","shell.execute_reply":"2022-10-19T20:06:15.853055Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_train=y_train.map({'not a real question':0,\n  'not constructive':1,\n  'off topic':2,\n  'open':3,\n  'too localized':4})","metadata":{"execution":{"iopub.status.busy":"2022-10-19T20:06:15.8561Z","iopub.execute_input":"2022-10-19T20:06:15.856606Z","iopub.status.idle":"2022-10-19T20:06:15.876946Z","shell.execute_reply.started":"2022-10-19T20:06:15.856546Z","shell.execute_reply":"2022-10-19T20:06:15.875581Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_test=y_test.map({'not a real question':0,\n  'not constructive':1,\n  'off topic':2,\n  'open':3,\n  'too localized':4})","metadata":{"execution":{"iopub.status.busy":"2022-10-19T20:06:15.878454Z","iopub.execute_input":"2022-10-19T20:06:15.878878Z","iopub.status.idle":"2022-10-19T20:06:15.893875Z","shell.execute_reply.started":"2022-10-19T20:06:15.878837Z","shell.execute_reply":"2022-10-19T20:06:15.892492Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-10-19T20:06:15.895419Z","iopub.execute_input":"2022-10-19T20:06:15.895866Z","iopub.status.idle":"2022-10-19T20:06:15.916406Z","shell.execute_reply.started":"2022-10-19T20:06:15.895793Z","shell.execute_reply":"2022-10-19T20:06:15.914142Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-10-19T20:06:15.919578Z","iopub.execute_input":"2022-10-19T20:06:15.919972Z","iopub.status.idle":"2022-10-19T20:06:15.933498Z","shell.execute_reply.started":"2022-10-19T20:06:15.919937Z","shell.execute_reply":"2022-10-19T20:06:15.932121Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['Text']=train_df['Title']+' '+train_df['BodyMarkdown']+' '+train_df['Tags']\ntest_df['Text']=test_df['Title']+' '+test_df['BodyMarkdown']+' '+test_df['Tags']","metadata":{"execution":{"iopub.status.busy":"2022-10-19T20:06:15.935742Z","iopub.execute_input":"2022-10-19T20:06:15.936611Z","iopub.status.idle":"2022-10-19T20:06:16.392309Z","shell.execute_reply.started":"2022-10-19T20:06:15.936557Z","shell.execute_reply":"2022-10-19T20:06:16.391441Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.drop(['Title','BodyMarkdown','Tags'],axis=1,inplace=True)\ntest_df.drop(['Title','BodyMarkdown','Tags'],axis=1,inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-10-19T20:06:16.393312Z","iopub.execute_input":"2022-10-19T20:06:16.393736Z","iopub.status.idle":"2022-10-19T20:06:16.448837Z","shell.execute_reply.started":"2022-10-19T20:06:16.393706Z","shell.execute_reply":"2022-10-19T20:06:16.447788Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.reset_index(inplace=True)\ntest_df.reset_index(inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-10-19T20:06:16.450337Z","iopub.execute_input":"2022-10-19T20:06:16.450686Z","iopub.status.idle":"2022-10-19T20:06:16.457651Z","shell.execute_reply.started":"2022-10-19T20:06:16.450644Z","shell.execute_reply":"2022-10-19T20:06:16.45619Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.drop(['index'],inplace=True,axis=1)","metadata":{"execution":{"iopub.status.busy":"2022-10-19T20:06:16.459251Z","iopub.execute_input":"2022-10-19T20:06:16.459558Z","iopub.status.idle":"2022-10-19T20:06:16.477246Z","shell.execute_reply.started":"2022-10-19T20:06:16.45953Z","shell.execute_reply":"2022-10-19T20:06:16.475705Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df.drop(['index'],axis=1,inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-10-19T20:06:16.478938Z","iopub.execute_input":"2022-10-19T20:06:16.479299Z","iopub.status.idle":"2022-10-19T20:06:16.493946Z","shell.execute_reply.started":"2022-10-19T20:06:16.479267Z","shell.execute_reply":"2022-10-19T20:06:16.492918Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Building the tensorflow model\n","metadata":{}},{"cell_type":"code","source":"from tensorflow.keras.preprocessing.text import Tokenizer\nmax_words=10000\ntokenizer=Tokenizer(max_words)\ntokenizer.fit_on_texts(train_df['Text'])\nsequence_train=tokenizer.texts_to_sequences(train_df['Text'])\nsequence_test=tokenizer.texts_to_sequences(test_df['Text'])","metadata":{"execution":{"iopub.status.busy":"2022-10-19T20:06:16.495363Z","iopub.execute_input":"2022-10-19T20:06:16.496258Z","iopub.status.idle":"2022-10-19T20:06:54.863254Z","shell.execute_reply.started":"2022-10-19T20:06:16.496207Z","shell.execute_reply":"2022-10-19T20:06:54.862329Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"word_2_vec=tokenizer.word_index\nV=len(word_2_vec)\nprint('Dataset has {} number of independent tokens'.format(V))","metadata":{"execution":{"iopub.status.busy":"2022-10-19T20:06:54.864724Z","iopub.execute_input":"2022-10-19T20:06:54.865365Z","iopub.status.idle":"2022-10-19T20:06:54.870735Z","shell.execute_reply.started":"2022-10-19T20:06:54.865329Z","shell.execute_reply":"2022-10-19T20:06:54.869905Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tensorflow.keras.preprocessing.sequence import pad_sequences\ndata_train=pad_sequences(sequence_train)\ndata_train.shape","metadata":{"execution":{"iopub.status.busy":"2022-10-19T20:06:54.871785Z","iopub.execute_input":"2022-10-19T20:06:54.872219Z","iopub.status.idle":"2022-10-19T20:06:58.67401Z","shell.execute_reply.started":"2022-10-19T20:06:54.872188Z","shell.execute_reply":"2022-10-19T20:06:58.673207Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"T=data_train.shape[1]\ndata_test=pad_sequences(sequence_test,maxlen=T)\ndata_test.shape\n","metadata":{"execution":{"iopub.status.busy":"2022-10-19T20:06:58.675051Z","iopub.execute_input":"2022-10-19T20:06:58.675457Z","iopub.status.idle":"2022-10-19T20:06:59.567341Z","shell.execute_reply.started":"2022-10-19T20:06:58.675425Z","shell.execute_reply":"2022-10-19T20:06:59.566309Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tensorflow.keras.layers import Input,Conv1D,MaxPooling1D,Dense,GlobalMaxPooling1D,Embedding\nfrom tensorflow.keras.models import Model\nD=20\ni=Input((T,))\nx=Embedding(V+1,D)(i)\nx=Conv1D(32,3,activation='relu')(x)\nx=MaxPooling1D(3)(x)\nx=Conv1D(64,3,activation='relu')(x)\nx=MaxPooling1D(3)(x)\nx=Conv1D(128,3,activation='relu')(x)\nx=GlobalMaxPooling1D()(x)\nx=Dense(5,activation='softmax')(x)\nmodel=Model(i,x)\nmodel.summary()","metadata":{"execution":{"iopub.status.busy":"2022-10-19T20:06:59.56873Z","iopub.execute_input":"2022-10-19T20:06:59.569071Z","iopub.status.idle":"2022-10-19T20:06:59.831933Z","shell.execute_reply.started":"2022-10-19T20:06:59.56904Z","shell.execute_reply":"2022-10-19T20:06:59.831006Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.compile(loss='sparse_categorical_crossentropy',optimizer='adam',metrics=['accuracy'])\ncnn_senti=model.fit(data_train,y_train,validation_data=(data_test,y_test),batch_size=100,epochs=5)","metadata":{"execution":{"iopub.status.busy":"2022-10-19T20:06:59.83343Z","iopub.execute_input":"2022-10-19T20:06:59.833746Z","iopub.status.idle":"2022-10-19T20:12:00.352969Z","shell.execute_reply.started":"2022-10-19T20:06:59.833715Z","shell.execute_reply":"2022-10-19T20:12:00.350943Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.predict(data_test)","metadata":{"execution":{"iopub.status.busy":"2022-10-19T20:12:00.354065Z","iopub.status.idle":"2022-10-19T20:12:00.354727Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_pred=model.predict(data_test)","metadata":{"execution":{"iopub.status.busy":"2022-10-19T20:12:00.355834Z","iopub.status.idle":"2022-10-19T20:12:00.356509Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_pred_final=np.argmax(y_pred,axis=1)\ny_pred_final","metadata":{"execution":{"iopub.status.busy":"2022-10-19T20:12:00.357557Z","iopub.status.idle":"2022-10-19T20:12:00.358145Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Scoring","metadata":{}},{"cell_type":"code","source":"from sklearn.metrics import confusion_matrix,classification_report\nimport seaborn as sns","metadata":{"execution":{"iopub.status.busy":"2022-10-19T20:12:00.359007Z","iopub.status.idle":"2022-10-19T20:12:00.359553Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cm=confusion_matrix(y_test,y_pred_final)\nax=sns.heatmap(cm,cmap='Blues',annot=True,fmt=' ')\nax.set_title('Confusion Matrix')\nax.set_xlabel('Y Test')\nax.set_ylabel('Y Pred')","metadata":{"execution":{"iopub.status.busy":"2022-10-19T20:12:00.360393Z","iopub.status.idle":"2022-10-19T20:12:00.360949Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(classification_report(y_test,y_pred_final))","metadata":{"execution":{"iopub.status.busy":"2022-10-19T20:12:00.361768Z","iopub.status.idle":"2022-10-19T20:12:00.362337Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_submission=pd.DataFrame(test_post_id,columns=['PostId'])\ndf_submission.head()","metadata":{"execution":{"iopub.status.busy":"2022-10-19T20:12:00.363157Z","iopub.status.idle":"2022-10-19T20:12:00.363681Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_post_id.reset_index(drop=True)","metadata":{"execution":{"iopub.status.busy":"2022-10-19T20:12:00.364506Z","iopub.status.idle":"2022-10-19T20:12:00.365052Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_submission_1=pd.DataFrame(y_pred,columns=[0,1,2,3,4])","metadata":{"execution":{"iopub.status.busy":"2022-10-19T20:12:00.365905Z","iopub.status.idle":"2022-10-19T20:12:00.366438Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_submission_1","metadata":{"execution":{"iopub.status.busy":"2022-10-19T20:12:00.367262Z","iopub.status.idle":"2022-10-19T20:12:00.367794Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_submission_1['PostId']=list(test_post_id)","metadata":{"execution":{"iopub.status.busy":"2022-10-19T20:12:00.368613Z","iopub.status.idle":"2022-10-19T20:12:00.36916Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"columns=['PostId',0,1,2,3,4]\ndf_submission_1=df_submission_1[columns]\ndf_submission_1","metadata":{"execution":{"iopub.status.busy":"2022-10-19T20:12:00.370013Z","iopub.status.idle":"2022-10-19T20:12:00.370554Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_submission_1.columns","metadata":{"execution":{"iopub.status.busy":"2022-10-19T20:12:00.371377Z","iopub.status.idle":"2022-10-19T20:12:00.371937Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_submission_1['Sum']=df_submission_1[0]+df_submission_1[1]+df_submission_1[2]+df_submission_1[3]+df_submission_1[4]\n","metadata":{"execution":{"iopub.status.busy":"2022-10-19T20:12:00.372746Z","iopub.status.idle":"2022-10-19T20:12:00.373285Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_submission_1","metadata":{"execution":{"iopub.status.busy":"2022-10-19T20:12:00.374138Z","iopub.status.idle":"2022-10-19T20:12:00.374678Z"},"trusted":true},"execution_count":null,"outputs":[]}]}