{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"#### All the Lifecycle In A Data Science Projects\n1. Data analisys\n2. Feature Engineering \n3. Feature Selection\n4. Model Biulding\n5. Model Deployment","metadata":{}},{"cell_type":"code","source":"## Data Analisis Phaze","metadata":{"execution":{"iopub.status.busy":"2022-08-01T08:20:57.381866Z","iopub.execute_input":"2022-08-01T08:20:57.382342Z","iopub.status.idle":"2022-08-01T08:20:57.387163Z","shell.execute_reply.started":"2022-08-01T08:20:57.382300Z","shell.execute_reply":"2022-08-01T08:20:57.386213Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## Import Libraries \n\nimport pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt \n%matplotlib inline \nimport seaborn as sns \nfrom pandas.api.types import is_numeric_dtype","metadata":{"execution":{"iopub.status.busy":"2022-08-01T08:20:57.420450Z","iopub.execute_input":"2022-08-01T08:20:57.421232Z","iopub.status.idle":"2022-08-01T08:20:58.098274Z","shell.execute_reply.started":"2022-08-01T08:20:57.421194Z","shell.execute_reply":"2022-08-01T08:20:58.097041Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## Display all the columns of the dataframe\n\npd.pandas.set_option('display.max_columns', None)\ndataset = pd.read_csv('../input/house-prices-advanced-regression-techniques/train.csv')","metadata":{"execution":{"iopub.status.busy":"2022-08-01T08:20:58.100591Z","iopub.execute_input":"2022-08-01T08:20:58.100939Z","iopub.status.idle":"2022-08-01T08:20:58.150654Z","shell.execute_reply.started":"2022-08-01T08:20:58.100907Z","shell.execute_reply":"2022-08-01T08:20:58.149533Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## Print shape of dataset with rows and columns \n\nprint(dataset.shape)","metadata":{"execution":{"iopub.status.busy":"2022-08-01T08:20:58.152453Z","iopub.execute_input":"2022-08-01T08:20:58.153140Z","iopub.status.idle":"2022-08-01T08:20:58.160153Z","shell.execute_reply.started":"2022-08-01T08:20:58.153076Z","shell.execute_reply":"2022-08-01T08:20:58.158564Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## Print the top5 records \n\ndataset.head()","metadata":{"execution":{"iopub.status.busy":"2022-08-01T08:20:58.164237Z","iopub.execute_input":"2022-08-01T08:20:58.165418Z","iopub.status.idle":"2022-08-01T08:20:58.239786Z","shell.execute_reply.started":"2022-08-01T08:20:58.165371Z","shell.execute_reply":"2022-08-01T08:20:58.238799Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### In Data Analysis We Will Analyze To Find Out The Below Stuff \n1. Missing values\n2. All the Numerical Variables\n3. Distribution of the Numerical Variables \n4. Categorical Variables\n5. Cardibality of Categorical Variables \n6. Outliers \n7. Relations between independent and dependent feature(SalePrice)","metadata":{}},{"cell_type":"markdown","source":"#### **Missing Values**","metadata":{}},{"cell_type":"code","source":"## Here we will chack the percentage of non values present in each feature \n\n## 1 step - make the list of features which has missing values: \nfeatures_with_non = [features for features in dataset.columns if dataset[features].isnull().sum() > 1]\n\n## 2 step - print the feature name and the percentage of missinf values:\nfor feature in features_with_non:\n    print(feature, np.round(dataset[feature].isnull().mean(), 4), ' % missing values')","metadata":{"execution":{"iopub.status.busy":"2022-08-01T08:20:58.241113Z","iopub.execute_input":"2022-08-01T08:20:58.241821Z","iopub.status.idle":"2022-08-01T08:20:58.293744Z","shell.execute_reply.started":"2022-08-01T08:20:58.241775Z","shell.execute_reply":"2022-08-01T08:20:58.292253Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Since they are many missing values, we need to find the relationship between missing values and Sale Price ","metadata":{}},{"cell_type":"code","source":"## Plot a diagram for this relationship \n\nfor feature in features_with_non:\n    data = dataset.copy()\n    \n    data[feature] = np.where(data[feature].isnull(), 1, 0)\n    data.groupby(feature)['SalePrice'].median().plot.bar()\n    plt.title(feature)\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2022-08-01T08:20:58.295417Z","iopub.execute_input":"2022-08-01T08:20:58.296388Z","iopub.status.idle":"2022-08-01T08:21:01.692511Z","shell.execute_reply.started":"2022-08-01T08:20:58.296337Z","shell.execute_reply":"2022-08-01T08:21:01.691192Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Here with the relation between the missing values and the dependent variable is clearly visible. We need to replace these non values with something meaningful which we will do in the Feature Engineering Section ","metadata":{}},{"cell_type":"markdown","source":"Dataset some of the features like id is not required ","metadata":{}},{"cell_type":"code","source":"print(f\"Id of Houses {len(dataset.Id)}\")","metadata":{"execution":{"iopub.status.busy":"2022-08-01T08:21:01.693803Z","iopub.execute_input":"2022-08-01T08:21:01.694659Z","iopub.status.idle":"2022-08-01T08:21:01.701339Z","shell.execute_reply.started":"2022-08-01T08:21:01.694619Z","shell.execute_reply":"2022-08-01T08:21:01.700281Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## **Numerical Variables**","metadata":{}},{"cell_type":"code","source":"#List of numerical variables\nnumerical_features = [feature for feature in dataset.select_dtypes(include=np.number).columns]\nprint(f\"Number of numerical variables {len(numerical_features)}\")\ndataset[numerical_features].head()","metadata":{"execution":{"iopub.status.busy":"2022-08-01T08:21:01.702840Z","iopub.execute_input":"2022-08-01T08:21:01.703838Z","iopub.status.idle":"2022-08-01T08:21:01.739720Z","shell.execute_reply.started":"2022-08-01T08:21:01.703803Z","shell.execute_reply":"2022-08-01T08:21:01.738578Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Temporal Variables(Eg: Datetime Variables)\nFrom the Dataset we have 4 year variables. We have the extra information from the datetime variables like no of years or no of days. One example in this specific scenario can be difference in years between the year the house was built and the year the hous was sold. We will be performing this analysis in the Feature Engineering.  ","metadata":{}},{"cell_type":"code","source":"# List of variables that contain the year informations \nyear_feature = [feature for feature in numerical_features if \"Yr\" in feature or \"Year\" in feature]\nyear_feature","metadata":{"execution":{"iopub.status.busy":"2022-08-01T08:21:01.741508Z","iopub.execute_input":"2022-08-01T08:21:01.742233Z","iopub.status.idle":"2022-08-01T08:21:01.752054Z","shell.execute_reply.started":"2022-08-01T08:21:01.742188Z","shell.execute_reply":"2022-08-01T08:21:01.750993Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Explore the content of these year variables \nfor feature in year_feature:\n    print(feature, dataset[feature].unique())","metadata":{"execution":{"iopub.status.busy":"2022-08-01T08:21:01.757835Z","iopub.execute_input":"2022-08-01T08:21:01.758675Z","iopub.status.idle":"2022-08-01T08:21:01.770285Z","shell.execute_reply.started":"2022-08-01T08:21:01.758630Z","shell.execute_reply":"2022-08-01T08:21:01.769133Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## Temporal Datetime Variables\n## Check a relation between year the hous is sold and\n\ndataset.groupby('YrSold')['SalePrice'].median().plot()\nplt.xlabel('Year Sold')\nplt.ylabel('Median House Price')\nplt.title('House Price vs Year Sold ')","metadata":{"execution":{"iopub.status.busy":"2022-08-01T08:21:01.771543Z","iopub.execute_input":"2022-08-01T08:21:01.771863Z","iopub.status.idle":"2022-08-01T08:21:01.995552Z","shell.execute_reply.started":"2022-08-01T08:21:01.771832Z","shell.execute_reply":"2022-08-01T08:21:01.994608Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## Compare the difference between All years feature with Sale Price\n\nfor feature in year_feature:\n    if feature != 'YrSold':\n        data = dataset.copy()\n        \n        ## Copture the difference between the year variable and year the house \n        data[feature] = data['YrSold'] - data[feature]\n        \n        plt.scatter(data[feature], data['SalePrice'])\n        plt.xlabel(feature)\n        plt.ylabel('SalePrice')\n        plt.show()","metadata":{"execution":{"iopub.status.busy":"2022-08-01T08:21:01.996784Z","iopub.execute_input":"2022-08-01T08:21:01.997122Z","iopub.status.idle":"2022-08-01T08:21:02.618813Z","shell.execute_reply.started":"2022-08-01T08:21:01.997080Z","shell.execute_reply":"2022-08-01T08:21:02.617731Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Numerical variables are usually of two types: Continous and Discrete Variables","metadata":{}},{"cell_type":"markdown","source":"#### **Discrete Variables**","metadata":{}},{"cell_type":"code","source":"discrete_feature = [feature for feature in numerical_features if len(dataset[feature].unique()) < 25 and feature not in year_feature + ['Id']]\nprint(f\"Number of discrete variables: {len(discrete_feature)}\")","metadata":{"execution":{"iopub.status.busy":"2022-08-01T08:21:02.620274Z","iopub.execute_input":"2022-08-01T08:21:02.620629Z","iopub.status.idle":"2022-08-01T08:21:02.637363Z","shell.execute_reply.started":"2022-08-01T08:21:02.620595Z","shell.execute_reply":"2022-08-01T08:21:02.636142Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"discrete_feature","metadata":{"execution":{"iopub.status.busy":"2022-08-01T08:21:02.639302Z","iopub.execute_input":"2022-08-01T08:21:02.639737Z","iopub.status.idle":"2022-08-01T08:21:02.647307Z","shell.execute_reply.started":"2022-08-01T08:21:02.639692Z","shell.execute_reply":"2022-08-01T08:21:02.646202Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset[discrete_feature].head()","metadata":{"execution":{"iopub.status.busy":"2022-08-01T08:21:02.649288Z","iopub.execute_input":"2022-08-01T08:21:02.650022Z","iopub.status.idle":"2022-08-01T08:21:02.670784Z","shell.execute_reply.started":"2022-08-01T08:21:02.649977Z","shell.execute_reply":"2022-08-01T08:21:02.669352Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## Find the realtionship between them and SalePrice\n\nfor feature in discrete_feature:\n    data = dataset.copy()\n    data.groupby(feature)['SalePrice'].median().plot.bar()\n    plt.xlabel(feature)\n    plt.ylabel('SalePrice')\n    plt.title(feature)\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2022-08-01T08:21:02.672098Z","iopub.execute_input":"2022-08-01T08:21:02.672462Z","iopub.status.idle":"2022-08-01T08:21:06.517963Z","shell.execute_reply.started":"2022-08-01T08:21:02.672429Z","shell.execute_reply":"2022-08-01T08:21:06.516550Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### **Continuous Variables**","metadata":{}},{"cell_type":"code","source":"continuous_feature = [feature for feature in numerical_features if feature not in discrete_feature+year_feature+['Id']]\nprint(f\"Number of Continuous Variables: {len(continuous_feature)}\")","metadata":{"execution":{"iopub.status.busy":"2022-08-01T08:21:06.519736Z","iopub.execute_input":"2022-08-01T08:21:06.520590Z","iopub.status.idle":"2022-08-01T08:21:06.528210Z","shell.execute_reply.started":"2022-08-01T08:21:06.520542Z","shell.execute_reply":"2022-08-01T08:21:06.527112Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## Analyse the continuous values by creating histograms to understand the \n\nfor feature in continuous_feature:\n    data = dataset.copy()\n    data[feature].hist(bins=25)\n    plt.xlabel(feature)\n    plt.ylabel(\"Count\")\n    plt.title(feature)\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2022-08-01T08:21:06.529762Z","iopub.execute_input":"2022-08-01T08:21:06.530130Z","iopub.status.idle":"2022-08-01T08:21:10.497231Z","shell.execute_reply.started":"2022-08-01T08:21:06.530072Z","shell.execute_reply":"2022-08-01T08:21:10.496113Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## Logarithmic transformation \n\nfor feature in continuous_feature:\n    data = dataset.copy()\n    if 0 in data[feature].unique():\n        pass\n    else:\n        data[feature] = np.log(data[feature])\n        data['SalePrice'] = np.log(data['SalePrice'])\n        plt.scatter(data[feature], data['SalePrice'])\n        plt.xlabel(feature)\n        plt.ylabel('SalePrice')\n        plt.title(feature)\n        plt.show()\n","metadata":{"execution":{"iopub.status.busy":"2022-08-01T08:21:10.498736Z","iopub.execute_input":"2022-08-01T08:21:10.499085Z","iopub.status.idle":"2022-08-01T08:21:11.513141Z","shell.execute_reply.started":"2022-08-01T08:21:10.499054Z","shell.execute_reply":"2022-08-01T08:21:11.511965Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### **Outliers**","metadata":{}},{"cell_type":"code","source":"for feature in continuous_feature:\n    data = dataset.copy()\n    if 0 in data[feature].unique():\n        pass\n    else:\n        data[feature] = np.log(data[feature])\n        data.boxplot(column=feature)\n        plt.ylabel(feature)\n        plt.title(feature)\n        plt.show()","metadata":{"execution":{"iopub.status.busy":"2022-08-01T08:21:11.514858Z","iopub.execute_input":"2022-08-01T08:21:11.515239Z","iopub.status.idle":"2022-08-01T08:21:12.376878Z","shell.execute_reply.started":"2022-08-01T08:21:11.515204Z","shell.execute_reply":"2022-08-01T08:21:12.375816Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### **Categorical Variables**","metadata":{}},{"cell_type":"code","source":"categorical_features = [feature for feature in dataset.columns if not is_numeric_dtype(data[feature])]\ncategorical_features","metadata":{"execution":{"iopub.status.busy":"2022-08-01T08:21:12.378601Z","iopub.execute_input":"2022-08-01T08:21:12.378937Z","iopub.status.idle":"2022-08-01T08:21:12.391778Z","shell.execute_reply.started":"2022-08-01T08:21:12.378897Z","shell.execute_reply":"2022-08-01T08:21:12.390634Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset[categorical_features].head()","metadata":{"execution":{"iopub.status.busy":"2022-08-01T08:21:12.393236Z","iopub.execute_input":"2022-08-01T08:21:12.394054Z","iopub.status.idle":"2022-08-01T08:21:12.439395Z","shell.execute_reply.started":"2022-08-01T08:21:12.394019Z","shell.execute_reply":"2022-08-01T08:21:12.438506Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for feature in categorical_features:\n    print(f\"The feature is {feature} and number of categories are {len(dataset[feature].unique())}\")","metadata":{"execution":{"iopub.status.busy":"2022-08-01T08:22:45.455362Z","iopub.execute_input":"2022-08-01T08:22:45.455807Z","iopub.status.idle":"2022-08-01T08:22:45.471367Z","shell.execute_reply.started":"2022-08-01T08:22:45.455772Z","shell.execute_reply":"2022-08-01T08:22:45.470153Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## Realtionship between categorical variable and dependent feature \n\nfor feature in categorical_features:\n    data = dataset.copy()\n    data.groupby(feature)['SalePrice'].median().plot.bar()\n    plt.xlabel(feature)\n    plt.ylabel('SalePrice')\n    plt.title(feature)\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2022-08-01T08:29:08.613053Z","iopub.execute_input":"2022-08-01T08:29:08.613953Z","iopub.status.idle":"2022-08-01T08:29:17.295468Z","shell.execute_reply.started":"2022-08-01T08:29:08.613787Z","shell.execute_reply":"2022-08-01T08:29:17.294422Z"},"trusted":true},"execution_count":null,"outputs":[]}]}