{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-07-26T18:28:25.308165Z","iopub.execute_input":"2021-07-26T18:28:25.308824Z","iopub.status.idle":"2021-07-26T18:28:25.325705Z","shell.execute_reply.started":"2021-07-26T18:28:25.30878Z","shell.execute_reply":"2021-07-26T18:28:25.324463Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train=pd.read_csv('../input/predict-closed-questions-on-stack-overflow/train-sample.csv')\ndf_train.head()","metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","execution":{"iopub.status.busy":"2021-07-26T18:28:34.456602Z","iopub.execute_input":"2021-07-26T18:28:34.457219Z","iopub.status.idle":"2021-07-26T18:28:39.289819Z","shell.execute_reply.started":"2021-07-26T18:28:34.457172Z","shell.execute_reply":"2021-07-26T18:28:39.288623Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.shape","metadata":{"execution":{"iopub.status.busy":"2021-07-26T18:28:50.70873Z","iopub.execute_input":"2021-07-26T18:28:50.709184Z","iopub.status.idle":"2021-07-26T18:28:50.717091Z","shell.execute_reply.started":"2021-07-26T18:28:50.709142Z","shell.execute_reply":"2021-07-26T18:28:50.715498Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\ntrain_df,test_df=train_test_split(df_train,test_size=0.2)\nprint('Training data shape: {}'.format(train_df.shape))\nprint('Testing data shape: {}'.format(test_df.shape))\n","metadata":{"execution":{"iopub.status.busy":"2021-07-26T18:28:53.864487Z","iopub.execute_input":"2021-07-26T18:28:53.864872Z","iopub.status.idle":"2021-07-26T18:28:55.174344Z","shell.execute_reply.started":"2021-07-26T18:28:53.864838Z","shell.execute_reply":"2021-07-26T18:28:55.172436Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Exploratory Data Analysis","metadata":{}},{"cell_type":"markdown","source":"## PostID","metadata":{}},{"cell_type":"markdown","source":"This feature is the id of the query posted by the user. This feauture does not have any importance in making the prediction but it will be used later for submission we will save it.","metadata":{}},{"cell_type":"code","source":"train_post_id=train_df.PostId\ntest_post_id=test_df.PostId","metadata":{"execution":{"iopub.status.busy":"2021-07-26T18:29:12.371262Z","iopub.execute_input":"2021-07-26T18:29:12.371694Z","iopub.status.idle":"2021-07-26T18:29:12.377297Z","shell.execute_reply.started":"2021-07-26T18:29:12.37166Z","shell.execute_reply":"2021-07-26T18:29:12.376189Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## PostCreationDate\n\nThe post creation date is the date and time on which the query was posted by the user let us see if we could make something out of this.","metadata":{}},{"cell_type":"markdown","source":"As we have observed that it doesn't matter on the time the question is posted on its label thus we can just remove this column both from our train and test.","metadata":{}},{"cell_type":"code","source":"train_df.drop(['PostCreationDate'],axis=1,inplace=True)\ntest_df.drop(['PostCreationDate'],axis=1,inplace=True)","metadata":{"execution":{"iopub.status.busy":"2021-07-26T18:29:17.980189Z","iopub.execute_input":"2021-07-26T18:29:17.981075Z","iopub.status.idle":"2021-07-26T18:29:18.011689Z","shell.execute_reply.started":"2021-07-26T18:29:17.980998Z","shell.execute_reply":"2021-07-26T18:29:18.009977Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.head(2)","metadata":{"execution":{"iopub.status.busy":"2021-07-26T18:30:01.441218Z","iopub.execute_input":"2021-07-26T18:30:01.441859Z","iopub.status.idle":"2021-07-26T18:30:01.461028Z","shell.execute_reply.started":"2021-07-26T18:30:01.441803Z","shell.execute_reply":"2021-07-26T18:30:01.459924Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## OwnerUserId","metadata":{}},{"cell_type":"markdown","source":"This is the user id who posted the query. Let us check it","metadata":{}},{"cell_type":"code","source":"train_df['PostCount']=[1]*len(train_df['PostId'])","metadata":{"execution":{"iopub.status.busy":"2021-07-26T18:30:04.853008Z","iopub.execute_input":"2021-07-26T18:30:04.853677Z","iopub.status.idle":"2021-07-26T18:30:04.917445Z","shell.execute_reply.started":"2021-07-26T18:30:04.853624Z","shell.execute_reply":"2021-07-26T18:30:04.916618Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.pivot_table(train_df,index='OwnerUserId',columns='OpenStatus',values='PostCount',aggfunc='sum')","metadata":{"execution":{"iopub.status.busy":"2021-07-26T18:24:45.180679Z","iopub.execute_input":"2021-07-26T18:24:45.181086Z","iopub.status.idle":"2021-07-26T18:24:45.331363Z","shell.execute_reply.started":"2021-07-26T18:24:45.181042Z","shell.execute_reply":"2021-07-26T18:24:45.330701Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.loc[train_df['OpenStatus']=='open','OwnerUserId'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2021-07-26T18:24:45.332286Z","iopub.execute_input":"2021-07-26T18:24:45.332643Z","iopub.status.idle":"2021-07-26T18:24:45.356771Z","shell.execute_reply.started":"2021-07-26T18:24:45.332616Z","shell.execute_reply":"2021-07-26T18:24:45.356032Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We see that it does not matter on the OwnerUserId whether his question is going to be open or not as there are many user whose only one question remain open \nSo, we would be dropping this column","metadata":{}},{"cell_type":"code","source":"train_df.drop(['OwnerUserId'],axis=1,inplace=True)\ntest_df.drop(['OwnerUserId'],axis=1,inplace=True)","metadata":{"execution":{"iopub.status.busy":"2021-07-26T18:24:45.357683Z","iopub.execute_input":"2021-07-26T18:24:45.358053Z","iopub.status.idle":"2021-07-26T18:24:45.39373Z","shell.execute_reply.started":"2021-07-26T18:24:45.358027Z","shell.execute_reply":"2021-07-26T18:24:45.392704Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Reputation at Post Created\n\nThis can be a important factor. Let's take a look at it","metadata":{}},{"cell_type":"code","source":"df_train['ReputationAtPostCreation'].min()","metadata":{"execution":{"iopub.status.busy":"2021-07-26T18:24:45.395345Z","iopub.execute_input":"2021-07-26T18:24:45.395755Z","iopub.status.idle":"2021-07-26T18:24:45.402513Z","shell.execute_reply.started":"2021-07-26T18:24:45.395712Z","shell.execute_reply":"2021-07-26T18:24:45.4016Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train['ReputationAtPostCreation'].max()","metadata":{"execution":{"iopub.status.busy":"2021-07-26T18:24:45.40416Z","iopub.execute_input":"2021-07-26T18:24:45.404564Z","iopub.status.idle":"2021-07-26T18:24:45.413386Z","shell.execute_reply.started":"2021-07-26T18:24:45.404522Z","shell.execute_reply":"2021-07-26T18:24:45.412705Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Since the data range is such high let us scale it using MinMaxScaler","metadata":{}},{"cell_type":"code","source":"minimum=df_train['ReputationAtPostCreation'].min()\nmaximum=df_train['ReputationAtPostCreation'].max()","metadata":{"execution":{"iopub.status.busy":"2021-07-26T18:24:45.414535Z","iopub.execute_input":"2021-07-26T18:24:45.414974Z","iopub.status.idle":"2021-07-26T18:24:45.424192Z","shell.execute_reply.started":"2021-07-26T18:24:45.414937Z","shell.execute_reply":"2021-07-26T18:24:45.423435Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['ReputationAtPostCreation']=(train_df['ReputationAtPostCreation']-minimum)/(maximum-minimum)","metadata":{"execution":{"iopub.status.busy":"2021-07-26T18:24:45.425313Z","iopub.execute_input":"2021-07-26T18:24:45.425868Z","iopub.status.idle":"2021-07-26T18:24:45.438232Z","shell.execute_reply.started":"2021-07-26T18:24:45.425829Z","shell.execute_reply":"2021-07-26T18:24:45.4374Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df['ReputationAtPostCreation']=(test_df['ReputationAtPostCreation']-minimum)/(maximum-minimum)","metadata":{"execution":{"iopub.status.busy":"2021-07-26T18:24:45.439908Z","iopub.execute_input":"2021-07-26T18:24:45.440174Z","iopub.status.idle":"2021-07-26T18:24:45.451213Z","shell.execute_reply.started":"2021-07-26T18:24:45.440148Z","shell.execute_reply":"2021-07-26T18:24:45.450055Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## OwnerUndeletedAnswerCountAtPostTime ","metadata":{}},{"cell_type":"code","source":"train_df.OwnerUndeletedAnswerCountAtPostTime.value_counts()","metadata":{"execution":{"iopub.status.busy":"2021-07-26T18:24:45.452671Z","iopub.execute_input":"2021-07-26T18:24:45.453051Z","iopub.status.idle":"2021-07-26T18:24:45.463838Z","shell.execute_reply.started":"2021-07-26T18:24:45.45301Z","shell.execute_reply":"2021-07-26T18:24:45.46299Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"This feature doesnot seem to be doing anything, we will be dropping it for now.","metadata":{}},{"cell_type":"code","source":"train_df.drop(['OwnerUndeletedAnswerCountAtPostTime'],axis=1,inplace=True)\ntest_df.drop(['OwnerUndeletedAnswerCountAtPostTime'],axis=1,inplace=True)","metadata":{"execution":{"iopub.status.busy":"2021-07-26T18:24:45.465164Z","iopub.execute_input":"2021-07-26T18:24:45.465449Z","iopub.status.idle":"2021-07-26T18:24:45.512237Z","shell.execute_reply.started":"2021-07-26T18:24:45.465423Z","shell.execute_reply":"2021-07-26T18:24:45.511075Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## PostClosedDate","metadata":{}},{"cell_type":"markdown","source":"Dropping this column as post closed date does not seem to be doing anything, so we would be dropping it ","metadata":{}},{"cell_type":"code","source":"train_df.drop(['PostClosedDate'],axis=1,inplace=True)\ntest_df.drop(['PostClosedDate'],axis=1,inplace=True)","metadata":{"execution":{"iopub.status.busy":"2021-07-26T18:24:45.513607Z","iopub.execute_input":"2021-07-26T18:24:45.513903Z","iopub.status.idle":"2021-07-26T18:24:45.553281Z","shell.execute_reply.started":"2021-07-26T18:24:45.513875Z","shell.execute_reply":"2021-07-26T18:24:45.552144Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Tag1","metadata":{}},{"cell_type":"code","source":"train_df['Tag1'].isnull().sum()","metadata":{"execution":{"iopub.status.busy":"2021-07-26T18:24:45.554324Z","iopub.execute_input":"2021-07-26T18:24:45.554577Z","iopub.status.idle":"2021-07-26T18:24:45.567366Z","shell.execute_reply.started":"2021-07-26T18:24:45.554552Z","shell.execute_reply":"2021-07-26T18:24:45.566336Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df['Tag1'].isnull().sum()","metadata":{"execution":{"iopub.status.busy":"2021-07-26T18:24:45.568864Z","iopub.execute_input":"2021-07-26T18:24:45.569243Z","iopub.status.idle":"2021-07-26T18:24:45.582371Z","shell.execute_reply.started":"2021-07-26T18:24:45.569203Z","shell.execute_reply":"2021-07-26T18:24:45.581303Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We can combine all the tags column into one this would help us in comparing the words which are used as tags and the words used in Title and Body.","metadata":{}},{"cell_type":"code","source":"train_df['Tag1']=train_df['Tag1'].replace(np.nan,' ')","metadata":{"execution":{"iopub.status.busy":"2021-07-26T18:24:45.583598Z","iopub.execute_input":"2021-07-26T18:24:45.583862Z","iopub.status.idle":"2021-07-26T18:24:45.60371Z","shell.execute_reply.started":"2021-07-26T18:24:45.583837Z","shell.execute_reply":"2021-07-26T18:24:45.602651Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['Tag2']=train_df['Tag2'].replace(np.nan,' ')\ntrain_df['Tag3']=train_df['Tag3'].replace(np.nan,' ')\ntrain_df['Tag4']=train_df['Tag4'].replace(np.nan,' ')\ntrain_df['Tag5']=train_df['Tag5'].replace(np.nan,' ')","metadata":{"execution":{"iopub.status.busy":"2021-07-26T18:24:45.605103Z","iopub.execute_input":"2021-07-26T18:24:45.605383Z","iopub.status.idle":"2021-07-26T18:24:45.650452Z","shell.execute_reply.started":"2021-07-26T18:24:45.605358Z","shell.execute_reply":"2021-07-26T18:24:45.649758Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df['Tag1']=test_df['Tag1'].replace(np.nan,' ')\ntest_df['Tag2']=test_df['Tag2'].replace(np.nan,' ')\ntest_df['Tag3']=test_df['Tag3'].replace(np.nan,' ')\ntest_df['Tag4']=test_df['Tag4'].replace(np.nan,' ')\ntest_df['Tag5']=test_df['Tag5'].replace(np.nan,' ')","metadata":{"execution":{"iopub.status.busy":"2021-07-26T18:24:45.651478Z","iopub.execute_input":"2021-07-26T18:24:45.651904Z","iopub.status.idle":"2021-07-26T18:24:45.67438Z","shell.execute_reply.started":"2021-07-26T18:24:45.651874Z","shell.execute_reply":"2021-07-26T18:24:45.673691Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['Tag1']","metadata":{"execution":{"iopub.status.busy":"2021-07-26T18:24:45.675494Z","iopub.execute_input":"2021-07-26T18:24:45.675785Z","iopub.status.idle":"2021-07-26T18:24:45.682543Z","shell.execute_reply.started":"2021-07-26T18:24:45.675757Z","shell.execute_reply":"2021-07-26T18:24:45.681633Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['Tags']=train_df['Tag1']+' '+train_df['Tag2']+' '+train_df['Tag3']+' '+train_df['Tag4']+' '+train_df['Tag5']","metadata":{"execution":{"iopub.status.busy":"2021-07-26T18:24:45.684037Z","iopub.execute_input":"2021-07-26T18:24:45.684421Z","iopub.status.idle":"2021-07-26T18:24:45.816164Z","shell.execute_reply.started":"2021-07-26T18:24:45.684381Z","shell.execute_reply":"2021-07-26T18:24:45.815084Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df['Tags']=test_df['Tag1']+' '+test_df['Tag2']+' '+test_df['Tag3']+' '+test_df['Tag4']+' '+test_df['Tag5']","metadata":{"execution":{"iopub.status.busy":"2021-07-26T18:24:45.817651Z","iopub.execute_input":"2021-07-26T18:24:45.818017Z","iopub.status.idle":"2021-07-26T18:24:45.852776Z","shell.execute_reply.started":"2021-07-26T18:24:45.817985Z","shell.execute_reply":"2021-07-26T18:24:45.851614Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['Tags']=train_df['Tags'].str.lower()","metadata":{"execution":{"iopub.status.busy":"2021-07-26T18:24:45.854303Z","iopub.execute_input":"2021-07-26T18:24:45.85471Z","iopub.status.idle":"2021-07-26T18:24:45.908961Z","shell.execute_reply.started":"2021-07-26T18:24:45.854667Z","shell.execute_reply":"2021-07-26T18:24:45.907875Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df['Tags']=test_df['Tags'].str.lower()","metadata":{"execution":{"iopub.status.busy":"2021-07-26T18:24:45.910302Z","iopub.execute_input":"2021-07-26T18:24:45.910581Z","iopub.status.idle":"2021-07-26T18:24:45.927236Z","shell.execute_reply.started":"2021-07-26T18:24:45.910554Z","shell.execute_reply":"2021-07-26T18:24:45.926068Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df['Tags']","metadata":{"execution":{"iopub.status.busy":"2021-07-26T18:24:45.928353Z","iopub.execute_input":"2021-07-26T18:24:45.928684Z","iopub.status.idle":"2021-07-26T18:24:45.939062Z","shell.execute_reply.started":"2021-07-26T18:24:45.928656Z","shell.execute_reply":"2021-07-26T18:24:45.937946Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['Tags']=train_df['Tags'].apply(lambda x:x.lstrip())\ntrain_df['Tags']=train_df['Tags'].apply(lambda x:x.rstrip())\n","metadata":{"execution":{"iopub.status.busy":"2021-07-26T18:24:45.940601Z","iopub.execute_input":"2021-07-26T18:24:45.9409Z","iopub.status.idle":"2021-07-26T18:24:46.007128Z","shell.execute_reply.started":"2021-07-26T18:24:45.940874Z","shell.execute_reply":"2021-07-26T18:24:46.006043Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['Tags']","metadata":{"execution":{"iopub.status.busy":"2021-07-26T18:24:46.008302Z","iopub.execute_input":"2021-07-26T18:24:46.008565Z","iopub.status.idle":"2021-07-26T18:24:46.016117Z","shell.execute_reply.started":"2021-07-26T18:24:46.008531Z","shell.execute_reply":"2021-07-26T18:24:46.015262Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df['Tags']=test_df['Tags'].apply(lambda x:x.lstrip())\ntest_df['Tags']=test_df['Tags'].apply(lambda x:x.rstrip())","metadata":{"execution":{"iopub.status.busy":"2021-07-26T18:24:46.023063Z","iopub.execute_input":"2021-07-26T18:24:46.023353Z","iopub.status.idle":"2021-07-26T18:24:46.044176Z","shell.execute_reply.started":"2021-07-26T18:24:46.023326Z","shell.execute_reply":"2021-07-26T18:24:46.042984Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.head(1)","metadata":{"execution":{"iopub.status.busy":"2021-07-26T18:24:46.045873Z","iopub.execute_input":"2021-07-26T18:24:46.046281Z","iopub.status.idle":"2021-07-26T18:24:46.062801Z","shell.execute_reply.started":"2021-07-26T18:24:46.046241Z","shell.execute_reply":"2021-07-26T18:24:46.061638Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Dropping excess columns \ntrain_df.drop(['PostId','OwnerCreationDate','Tag1','Tag2','Tag3','Tag4','Tag5','PostCount'],axis=1,inplace=True)\ntest_df.drop(['PostId','OwnerCreationDate','Tag1','Tag2','Tag3','Tag4','Tag5'],axis=1,inplace=True)","metadata":{"execution":{"iopub.status.busy":"2021-07-26T18:24:46.064465Z","iopub.execute_input":"2021-07-26T18:24:46.064877Z","iopub.status.idle":"2021-07-26T18:24:46.145312Z","shell.execute_reply.started":"2021-07-26T18:24:46.064835Z","shell.execute_reply":"2021-07-26T18:24:46.144373Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-07-26T18:24:46.14676Z","iopub.execute_input":"2021-07-26T18:24:46.147149Z","iopub.status.idle":"2021-07-26T18:24:46.160407Z","shell.execute_reply.started":"2021-07-26T18:24:46.147108Z","shell.execute_reply":"2021-07-26T18:24:46.159481Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-07-26T18:24:46.162266Z","iopub.execute_input":"2021-07-26T18:24:46.162689Z","iopub.status.idle":"2021-07-26T18:24:46.180821Z","shell.execute_reply.started":"2021-07-26T18:24:46.162647Z","shell.execute_reply":"2021-07-26T18:24:46.17981Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_train=train_df['OpenStatus']\ny_test=test_df['OpenStatus']","metadata":{"execution":{"iopub.status.busy":"2021-07-26T18:24:46.183912Z","iopub.execute_input":"2021-07-26T18:24:46.184187Z","iopub.status.idle":"2021-07-26T18:24:46.191975Z","shell.execute_reply.started":"2021-07-26T18:24:46.184161Z","shell.execute_reply":"2021-07-26T18:24:46.191066Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.drop(['ReputationAtPostCreation','OpenStatus'],axis=1,inplace=True)\ntest_df.drop(['ReputationAtPostCreation','OpenStatus'],axis=1,inplace=True)","metadata":{"execution":{"iopub.status.busy":"2021-07-26T18:24:46.192939Z","iopub.execute_input":"2021-07-26T18:24:46.193342Z","iopub.status.idle":"2021-07-26T18:24:46.219022Z","shell.execute_reply.started":"2021-07-26T18:24:46.193315Z","shell.execute_reply":"2021-07-26T18:24:46.218351Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_train=y_train.map({'not a real question':0,\n  'not constructive':1,\n  'off topic':2,\n  'open':3,\n  'too localized':4})","metadata":{"execution":{"iopub.status.busy":"2021-07-26T18:24:46.219902Z","iopub.execute_input":"2021-07-26T18:24:46.220257Z","iopub.status.idle":"2021-07-26T18:24:46.235708Z","shell.execute_reply.started":"2021-07-26T18:24:46.220231Z","shell.execute_reply":"2021-07-26T18:24:46.234865Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_test=y_test.map({'not a real question':0,\n  'not constructive':1,\n  'off topic':2,\n  'open':3,\n  'too localized':4})","metadata":{"execution":{"iopub.status.busy":"2021-07-26T18:24:46.236848Z","iopub.execute_input":"2021-07-26T18:24:46.237294Z","iopub.status.idle":"2021-07-26T18:24:46.24873Z","shell.execute_reply.started":"2021-07-26T18:24:46.237267Z","shell.execute_reply":"2021-07-26T18:24:46.247939Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-07-26T18:24:46.249828Z","iopub.execute_input":"2021-07-26T18:24:46.250269Z","iopub.status.idle":"2021-07-26T18:24:46.266083Z","shell.execute_reply.started":"2021-07-26T18:24:46.250241Z","shell.execute_reply":"2021-07-26T18:24:46.265205Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-07-26T18:24:46.267521Z","iopub.execute_input":"2021-07-26T18:24:46.267783Z","iopub.status.idle":"2021-07-26T18:24:46.27863Z","shell.execute_reply.started":"2021-07-26T18:24:46.267758Z","shell.execute_reply":"2021-07-26T18:24:46.277616Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['Text']=train_df['Title']+' '+train_df['BodyMarkdown']+' '+train_df['Tags']\ntest_df['Text']=test_df['Title']+' '+test_df['BodyMarkdown']+' '+test_df['Tags']","metadata":{"execution":{"iopub.status.busy":"2021-07-26T18:24:46.279623Z","iopub.execute_input":"2021-07-26T18:24:46.279916Z","iopub.status.idle":"2021-07-26T18:24:46.677799Z","shell.execute_reply.started":"2021-07-26T18:24:46.279888Z","shell.execute_reply":"2021-07-26T18:24:46.677065Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.drop(['Title','BodyMarkdown','Tags'],axis=1,inplace=True)\ntest_df.drop(['Title','BodyMarkdown','Tags'],axis=1,inplace=True)","metadata":{"execution":{"iopub.status.busy":"2021-07-26T18:24:46.678811Z","iopub.execute_input":"2021-07-26T18:24:46.679222Z","iopub.status.idle":"2021-07-26T18:24:46.728242Z","shell.execute_reply.started":"2021-07-26T18:24:46.679187Z","shell.execute_reply":"2021-07-26T18:24:46.727574Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.reset_index(inplace=True)\ntest_df.reset_index(inplace=True)","metadata":{"execution":{"iopub.status.busy":"2021-07-26T18:24:46.729226Z","iopub.execute_input":"2021-07-26T18:24:46.729651Z","iopub.status.idle":"2021-07-26T18:24:46.735531Z","shell.execute_reply.started":"2021-07-26T18:24:46.729609Z","shell.execute_reply":"2021-07-26T18:24:46.7346Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.drop(['index'],inplace=True,axis=1)","metadata":{"execution":{"iopub.status.busy":"2021-07-26T18:24:46.736815Z","iopub.execute_input":"2021-07-26T18:24:46.737101Z","iopub.status.idle":"2021-07-26T18:24:46.752478Z","shell.execute_reply.started":"2021-07-26T18:24:46.737072Z","shell.execute_reply":"2021-07-26T18:24:46.751517Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df.drop(['index'],axis=1,inplace=True)","metadata":{"execution":{"iopub.status.busy":"2021-07-26T18:24:46.755204Z","iopub.execute_input":"2021-07-26T18:24:46.755522Z","iopub.status.idle":"2021-07-26T18:24:46.764669Z","shell.execute_reply.started":"2021-07-26T18:24:46.755467Z","shell.execute_reply":"2021-07-26T18:24:46.763863Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Building the tensorflow model\n","metadata":{}},{"cell_type":"code","source":"from tensorflow.keras.preprocessing.text import Tokenizer\nmax_words=10000\ntokenizer=Tokenizer(max_words)\ntokenizer.fit_on_texts(train_df['Text'])\nsequence_train=tokenizer.texts_to_sequences(train_df['Text'])\nsequence_test=tokenizer.texts_to_sequences(test_df['Text'])","metadata":{"execution":{"iopub.status.busy":"2021-07-26T18:24:46.765939Z","iopub.execute_input":"2021-07-26T18:24:46.766351Z","iopub.status.idle":"2021-07-26T18:25:16.295134Z","shell.execute_reply.started":"2021-07-26T18:24:46.766308Z","shell.execute_reply":"2021-07-26T18:25:16.294128Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"word_2_vec=tokenizer.word_index\nV=len(word_2_vec)\nprint('Dataset has {} number of independent tokens'.format(V))","metadata":{"execution":{"iopub.status.busy":"2021-07-26T18:25:16.296344Z","iopub.execute_input":"2021-07-26T18:25:16.296623Z","iopub.status.idle":"2021-07-26T18:25:16.302521Z","shell.execute_reply.started":"2021-07-26T18:25:16.296596Z","shell.execute_reply":"2021-07-26T18:25:16.301397Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tensorflow.keras.preprocessing.sequence import pad_sequences\ndata_train=pad_sequences(sequence_train)\ndata_train.shape","metadata":{"execution":{"iopub.status.busy":"2021-07-26T18:25:16.303598Z","iopub.execute_input":"2021-07-26T18:25:16.303869Z","iopub.status.idle":"2021-07-26T18:25:19.558535Z","shell.execute_reply.started":"2021-07-26T18:25:16.303842Z","shell.execute_reply":"2021-07-26T18:25:19.557475Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"T=data_train.shape[1]\ndata_test=pad_sequences(sequence_test,maxlen=T)\ndata_test.shape\n","metadata":{"execution":{"iopub.status.busy":"2021-07-26T18:25:19.559624Z","iopub.execute_input":"2021-07-26T18:25:19.559892Z","iopub.status.idle":"2021-07-26T18:25:20.364788Z","shell.execute_reply.started":"2021-07-26T18:25:19.559865Z","shell.execute_reply":"2021-07-26T18:25:20.363829Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tensorflow.keras.layers import Input,Conv1D,MaxPooling1D,Dense,GlobalMaxPooling1D,Embedding\nfrom tensorflow.keras.models import Model\nD=20\ni=Input((T,))\nx=Embedding(V+1,D)(i)\nx=Conv1D(32,3,activation='relu')(x)\nx=MaxPooling1D(3)(x)\nx=Conv1D(64,3,activation='relu')(x)\nx=MaxPooling1D(3)(x)\nx=Conv1D(128,3,activation='relu')(x)\nx=GlobalMaxPooling1D()(x)\nx=Dense(5,activation='softmax')(x)\nmodel=Model(i,x)\nmodel.summary()","metadata":{"execution":{"iopub.status.busy":"2021-07-26T18:25:20.366046Z","iopub.execute_input":"2021-07-26T18:25:20.366324Z","iopub.status.idle":"2021-07-26T18:25:20.586806Z","shell.execute_reply.started":"2021-07-26T18:25:20.366297Z","shell.execute_reply":"2021-07-26T18:25:20.585744Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.compile(loss='sparse_categorical_crossentropy',optimizer='adam',metrics=['accuracy'])\ncnn_senti=model.fit(data_train,y_train,validation_data=(data_test,y_test),batch_size=100,epochs=5)","metadata":{"execution":{"iopub.status.busy":"2021-07-26T18:25:20.588208Z","iopub.execute_input":"2021-07-26T18:25:20.588479Z","iopub.status.idle":"2021-07-26T18:25:34.492103Z","shell.execute_reply.started":"2021-07-26T18:25:20.588451Z","shell.execute_reply":"2021-07-26T18:25:34.489801Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.predict(data_test)","metadata":{"execution":{"iopub.status.busy":"2021-07-26T18:25:34.493209Z","iopub.status.idle":"2021-07-26T18:25:34.493621Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_pred=model.predict(data_test)","metadata":{"execution":{"iopub.status.busy":"2021-07-26T18:25:34.494861Z","iopub.status.idle":"2021-07-26T18:25:34.495491Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_pred_final=np.argmax(y_pred,axis=1)\ny_pred_final","metadata":{"execution":{"iopub.status.busy":"2021-07-26T18:25:34.496653Z","iopub.status.idle":"2021-07-26T18:25:34.497284Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Scoring","metadata":{}},{"cell_type":"code","source":"from sklearn.metrics import confusion_matrix,classification_report\nimport seaborn as sns","metadata":{"execution":{"iopub.status.busy":"2021-07-26T18:25:34.49847Z","iopub.status.idle":"2021-07-26T18:25:34.499122Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cm=confusion_matrix(y_test,y_pred_final)\nax=sns.heatmap(cm,cmap='Blues',annot=True,fmt=' ')\nax.set_title('Confusion Matrix')\nax.set_xlabel('Y Test')\nax.set_ylabel('Y Pred')","metadata":{"execution":{"iopub.status.busy":"2021-07-26T18:25:34.500249Z","iopub.status.idle":"2021-07-26T18:25:34.501007Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(classification_report(y_test,y_pred_final))","metadata":{"execution":{"iopub.status.busy":"2021-07-26T18:25:34.50226Z","iopub.status.idle":"2021-07-26T18:25:34.502908Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_submission=pd.DataFrame(test_post_id,columns=['PostId'])\ndf_submission.head()","metadata":{"execution":{"iopub.status.busy":"2021-07-26T18:25:34.504064Z","iopub.status.idle":"2021-07-26T18:25:34.50473Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_post_id.reset_index(drop=True)","metadata":{"execution":{"iopub.status.busy":"2021-07-26T18:25:34.505899Z","iopub.status.idle":"2021-07-26T18:25:34.506564Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_submission_1=pd.DataFrame(y_pred,columns=[0,1,2,3,4])","metadata":{"execution":{"iopub.status.busy":"2021-07-26T18:25:34.507686Z","iopub.status.idle":"2021-07-26T18:25:34.508315Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_submission_1","metadata":{"execution":{"iopub.status.busy":"2021-07-26T18:25:34.509461Z","iopub.status.idle":"2021-07-26T18:25:34.510102Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_submission_1['PostId']=list(test_post_id)","metadata":{"execution":{"iopub.status.busy":"2021-07-26T18:25:34.511309Z","iopub.status.idle":"2021-07-26T18:25:34.511973Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"columns=['PostId',0,1,2,3,4]\ndf_submission_1=df_submission_1[columns]\ndf_submission_1","metadata":{"execution":{"iopub.status.busy":"2021-07-26T18:25:34.513183Z","iopub.status.idle":"2021-07-26T18:25:34.513854Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_submission_1.columns","metadata":{"execution":{"iopub.status.busy":"2021-07-26T18:25:34.515144Z","iopub.status.idle":"2021-07-26T18:25:34.515801Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_submission_1['Sum']=df_submission_1[0]+df_submission_1[1]+df_submission_1[2]+df_submission_1[3]+df_submission_1[4]\n","metadata":{"execution":{"iopub.status.busy":"2021-07-26T18:25:34.516941Z","iopub.status.idle":"2021-07-26T18:25:34.517593Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_submission_1","metadata":{"execution":{"iopub.status.busy":"2021-07-26T18:25:34.518729Z","iopub.status.idle":"2021-07-26T18:25:34.519362Z"},"trusted":true},"execution_count":null,"outputs":[]}]}