{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\n#import \nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport numpy as np\nfrom scipy.stats import norm\n#from sklearn.preprocessing import StandardScaler\nfrom scipy import stats\nimport warnings\nwarnings.filterwarnings('ignore')\n#%matplotlib inline\n\n\n#load data\ndf= pd.read_csv('D:/Kaggle/House Price/all/train.csv', index_col=False)\n#pandas.read_csv(filepath_or_buffer, yahan file ka location ayega\n#sep=', ', default value=','\n#header='infer', header mein jo column name rakhne hain wo row no. likhenge \n#names=None, yahan colummn k naam ki list pass karna hai \n#index_col=None, row labels ki list\n#usecols=None, columns names ki list\n#prefix=None, prefix for col names/nos\n#mangle_dupe_cols=True, duplicate cols will be named distinctly\n#dtype=None, datatype\n#engine=None, parser engine\n#converters=None, functions for converting certain values\n\n#check the columns\ndf.columns\nlen(df.columns)\n#object ka attribute\n\ndf.describe()\n#percentiles- returns 25, 50, 75th percentile\n#include- all the datatypes to be included\n#exclude - blacklist for datatypes to be excluded\n\n#descriptive statistics summary\ndf['SalePrice'].describe()\n\n#missing data\ntotal = df.isnull().sum().sort_values(ascending=False)\npercent = (df.isnull().sum()/df.isnull().count()).sort_values(ascending=False)*100\nmissing_data = pd.concat([total, percent], axis=1, keys=['Total', 'Percent'])\nmissing_data.head(20)\n\n#data cleanning\n\n#remove columns with missing values>80% \ndf=df.drop(['PoolQC', 'MiscFeature', 'Alley', 'Fence' ], axis=1)\n\n\n# check nulls\nprint('Columns With Nulls')\n\n#missing data\ntotal2 = df.isnull().sum().sort_values(ascending=False)\npercent2 = (df.isnull().sum()/df.isnull().count()).sort_values(ascending=False)*100\nmissing_data2 = pd.concat([total, percent], axis=1, keys=['Total', 'Percent'])\nmissing_data2.head(20)\n\n#data imputation\n#separate numerical and categorical values\n#impute numberical nulls with mean\n#impute categorical nulls with mode\n\ncolNames=df.columns\n\n\n#NAs as string to be removed still\n#df['LotFrontage']=np.where(df['LotFrontage'].isnan(), '',df['LotFrontage'] )\ndf=df.fillna(df.mode())\n \nfor c in colNames:\n    if(df[c].dtype=='object'):\n        df[c] = pd.Categorical(df[c]) # convert string / categoric to numeric\n        df[c] = df[c].cat.codes\n    else:\n        df=df.fillna(df.mean())\n\n\n#correlation matrix\ncorrmat = pd.DataFrame(df.corr())\ncorrmat.head()\ncorrmat.shape\n\n\n# DataFrame.corr(method='pearson', min_periods=1)[source]\n#method: can be person, kendall, spearman\n#min_periods: min. no . obs. to be passed\n\nfor i in range (0,76):\n    for j in range(0,76):\n        if (corrmat.iat[i,j] > 0.8 and corrmat.iat[i,j]!=1):\n            print('Row: ', i, 'Col: ', j, 'Value: ', corrmat.iat[i,j])\nfor i in range(0,76):\n    for j in range(0,76):\n        if (corrmat.iat[i,j] < -0.8 and corrmat.iat[i,j]!=1):\n            print('Row: ', i, 'Col: ', j, 'Value: ', corrmat.iat[i,j])\n\n#remove columns with high correlations\ndf=df.drop(columns=['Exterior1st', 'TotalBsmtSF', 'GrLivArea', 'Fireplaces', 'GarageCars', 'GarageQual']) \n\nfor i in range (0,70):\n    for j in range(0,70):\n        if (corrmat.iat[i,j] > 0.8 and corrmat.iat[i,j]!=1):\n            print('Row: ', i, 'Col: ', j, 'Value: ', corrmat.iat[i,j])\n           \n\n#subplots- nrows, ncols- for grid\n#sharex, sharey - shared properties for x and y\nf, ax = plt.subplots(figsize=(12, 12))\nsns.heatmap(corrmat, vmax=.8, square=True);\n\n\n#histogram\nsns.distplot(df['SalePrice']);\n#a : Series, jo data ka distribution chaiye\n#c=color\n#vertical(True)= y-axis\n\n#scatter plot grlivarea/saleprice\nplt.scatter(x=df['GrLivArea'], y=df['SalePrice']);\n#x= x axis ka data\n#y= y axis ka data\n#c= color\n#marker= marker style\n\n\n#scatter plot totalbsmtsf/saleprice\nplt.scatter(x=df['TotalBsmtSF'], y=df['SalePrice']);\n\n\n\nplt.bar(df['OverallQual'], df['SalePrice'], align='center')\n#x= x co-ordinates\n#align= alignment of bars\n#width= width of bars, default is 0.8\n#height= height of bars\n\n\nplt.scatter(x=df['YearBuilt'], y=df['SalePrice'], c=df['OverallQual']);\n#plt.bar(df['YearBuilt'], df['SalePrice'], align='center')\n#plt.rcParams[\"figure.figsize\"] = [16,9]\n#rc parameters: allows you to manage figure parameters\n\n#scatterplot\nsns.set()\ncols = ['SalePrice', 'OverallQual', 'GrLivArea', 'GarageCars', 'TotalBsmtSF', 'FullBath', 'YearBuilt']\nsns.pairplot(df[cols], size = 2.5)\n#hue : string (variable name), optional\n#markers : marker code\nplt.show();\n\n#bivariate analysis saleprice/grlivarea\nvar = 'GrLivArea'\ndata = pd.concat([df['SalePrice'], df[var]], axis=1)\ndata.plot.scatter(x=var, y='SalePrice', ylim=(0,800000));\n\n#bivariate analysis saleprice/grlivarea\nvar = 'TotalBsmtSF'\ndata = pd.concat([df['SalePrice'], df[var]], axis=1)\ndata.plot.scatter(x=var, y='SalePrice', ylim=(0,800000));\n\n\n#create column for new variable (one is enough because it's a binary categorical feature)\n#if area>0 it gets 1, for area==0 it gets 0\ndf['HasBsmt'] = pd.Series(len(df['TotalBsmtSF']), index=df.index)\ndf['HasBsmt'] = 0 \ndf.loc[df['TotalBsmtSF']>0,'HasBsmt'] = 1\n\n#basement area/total area\ndf['basementRatio']=df['TotalBsmtSF']/df['GrLivArea']\n\nplt.scatter(x=df['basementRatio'], y=df['SalePrice']);\n\n\n#Cluster analysis\ndf.groupby(['MSZoning'])['SalePrice'].mean()\ndf.groupby(['HeatingQC'])['SalePrice'].mean()\ndf.groupby(['CentralAir'])['SalePrice'].mean()\ndf.groupby(['FullBath'])['SalePrice'].mean()\ndf.groupby(['GarageType'])['SalePrice'].mean()\n\n#Cluster analysis\n#DataFrame.groupby(by=None, axis=0, level=None, squeeze=False, observed=False)\n#Attributes\n# By:  : mapping, function, label, or list of labels. Used to determine the groups for the groupby\n#axis : {0 or ‘index’, 1 or ‘columns’}, default is 0. Split along rows (0) or columns (1).\n#level : int, level name, or sequence of such, default None. If the axis is hierarchical, group by particular levels/level.\n#observed : bool, default False. This only applies if any of the groupers are Categoricals. If True: only show observed values for categorical groupers. If False: show all values for categorical groupers.\n#squeeze : bool, default False. Reduce the dimensionality of the return type if possible, otherwise return a consistent type.\n\n\ndf.groupby(['Utilities'])['SalePrice'].mean()\nplt.scatter(df['Utilities']==0, df['SalePrice'], color='#7f6d5f',  label='No Utiities', linewidths=1)\nplt.scatter(df['Utilities']==1, df['SalePrice'], color='#557f2d', label='Utilities', linewidths=1)\nplt.legend()\nplt.show()\n\n# sk linear\nfrom sklearn.linear_model import LinearRegression\n# stats model\nimport statsmodels.api as sm\n\n#X,y\ncolNames=df.columns.tolist()\ny = df['SalePrice'].values\ncolNames.remove('SalePrice')\nX = df[colNames].values\n\nmodel = LinearRegression()\nmodel.fit(X,y)\n\n# regression summary\nXconst = sm.add_constant(df[colNames])\nOconst = sm.OLS(y, Xconst)\nMconst = Oconst.fit()\nprint(Mconst.summary())\n\n#Significant columns- MSSubClass, LotFrontage, LotArea, Street, LandContour,  Neighborhood, Condition2, OverallQual, OverallCond, YearBuilt, RoofMatl, MasVnrType,  MasVnrArea, ExterQual, BsmtQual, BsmtCond, BsmtExposure, BsmtFinSF1, BsmtFinType2, BsmtFinSF2, 1stFlrSF, 2ndFlrSF, BsmtFullBath, BedroomAbvGr, KitchenAbvGr, KitchenQual, TotRmsAbvGrd, Functional, GarageArea, WoodDeckSF,  ScreenPorch, SaleCondition\nkeep_cols=['MSSubClass', 'LotFrontage', 'LotArea', 'Street',  'Neighborhood', 'Condition2', 'OverallQual', 'OverallCond', 'YearBuilt', 'RoofMatl', 'MasVnrType',  'MasVnrArea', 'ExterQual', 'BsmtQual', 'BsmtCond', 'BsmtExposure', 'BsmtFinSF1', 'BsmtFinType2', 'BsmtFinSF2', '1stFlrSF', '2ndFlrSF', 'BsmtFullBath', 'BedroomAbvGr', 'KitchenAbvGr', 'KitchenQual', 'TotRmsAbvGrd', 'Functional', 'GarageArea', 'WoodDeckSF',  'ScreenPorch', 'SaleCondition']\ndf=df.loc[:, keep_cols]\n\n#check regression summary again\n#X,y\ncolNames=df.columns.tolist()\ny = df['SalePrice']\ncolNames=colNames.remove('SalePrice')\nX = df[colNames].values\n\nmodel = LinearRegression()\nmodel.fit(X,y)\n\n# regression summary\nXconst = sm.add_constant(df[colNames])\nOconst = sm.OLS(y, Xconst)\nMconst = Oconst.fit()\nprint(Mconst.summary())\n\n\n#read test data set\nXnew= pd.read_csv('D:/Kaggle/House Price/all/test.csv')\n\n\ntest_colNames=Xnew.columns.tolist()\n\n\nXnew=Xnew.fillna(Xnew.mode())\n\nfor c in test_colNames:\n    if(Xnew[c].dtype=='object'or Xnew[c].dtype=='O'):\n        Xnew[c] = pd.Categorical(Xnew[c]) \n        Xnew[c] = Xnew[c].cat.codes\n    else:\n        Xnew[c]=Xnew[c].fillna(Xnew[c].mean())\n        \n#train_colNames=X.columns\n#test_colNames=Xnew.columns\n\n#subset test for similar columns in train dataset\nXnew = Xnew.loc[:,keep_cols]\n\n\n#prediction\n\n\nyNew = model.predict(Xnew.as_matrix())\n\n\nimport numpy as np\ny=np.delete(y, len(y)-1)\ny=np.float32(y)\n\n\n","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","collapsed":true,"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":false},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}