{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\n#Ignore warnings\nimport warnings\nwarnings.filterwarnings('ignore')\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-08-01T02:35:29.542384Z","iopub.execute_input":"2022-08-01T02:35:29.542904Z","iopub.status.idle":"2022-08-01T02:35:30.136800Z","shell.execute_reply.started":"2022-08-01T02:35:29.542775Z","shell.execute_reply":"2022-08-01T02:35:30.135058Z"},"_kg_hide-input":true,"_kg_hide-output":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"![](https://slidetodoc.com/presentation_image_h/86d31c89aa1784898ce5f48db8336aea/image-15.jpg)SlideToDoc.com","metadata":{}},{"cell_type":"markdown","source":"#Spoiler alert: No ROC and its Area under the curve on this Kaggle Notebook.","metadata":{}},{"cell_type":"code","source":"train = pd.read_csv('../input/tabular-playground-series-aug-2022/train.csv', encoding='utf8')\ntrain.head()","metadata":{"execution":{"iopub.status.busy":"2022-08-01T02:35:35.557387Z","iopub.execute_input":"2022-08-01T02:35:35.557886Z","iopub.status.idle":"2022-08-01T02:35:35.746639Z","shell.execute_reply.started":"2022-08-01T02:35:35.557833Z","shell.execute_reply":"2022-08-01T02:35:35.745257Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub = pd.read_csv('../input/tabular-playground-series-aug-2022/sample_submission.csv', encoding='utf8')\nsub.head()","metadata":{"execution":{"iopub.status.busy":"2022-08-01T02:36:13.746569Z","iopub.execute_input":"2022-08-01T02:36:13.747588Z","iopub.status.idle":"2022-08-01T02:36:13.779116Z","shell.execute_reply.started":"2022-08-01T02:36:13.747540Z","shell.execute_reply":"2022-08-01T02:36:13.777200Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install dataprep","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2022-08-01T02:36:30.066472Z","iopub.execute_input":"2022-08-01T02:36:30.067383Z","iopub.status.idle":"2022-08-01T02:37:00.169531Z","shell.execute_reply.started":"2022-08-01T02:36:30.067335Z","shell.execute_reply":"2022-08-01T02:37:00.167052Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"! python -m pip install \"dask[dataframe]\" --upgrade  # or python -m pip install","metadata":{"execution":{"iopub.status.busy":"2022-08-01T02:37:05.401177Z","iopub.execute_input":"2022-08-01T02:37:05.401679Z","iopub.status.idle":"2022-08-01T02:37:21.709187Z","shell.execute_reply.started":"2022-08-01T02:37:05.401635Z","shell.execute_reply":"2022-08-01T02:37:21.708058Z"},"_kg_hide-output":true,"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from dataprep.eda import plot, plot_correlation, create_report, plot_missing","metadata":{"execution":{"iopub.status.busy":"2022-08-01T02:37:38.260139Z","iopub.execute_input":"2022-08-01T02:37:38.260652Z","iopub.status.idle":"2022-08-01T02:37:41.625023Z","shell.execute_reply.started":"2022-08-01T02:37:38.260604Z","shell.execute_reply":"2022-08-01T02:37:41.623166Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#Correlations","metadata":{}},{"cell_type":"code","source":"#API Correlation\nplot_correlation(train)","metadata":{"execution":{"iopub.status.busy":"2022-08-01T02:38:03.351872Z","iopub.execute_input":"2022-08-01T02:38:03.352312Z","iopub.status.idle":"2022-08-01T02:38:06.609913Z","shell.execute_reply.started":"2022-08-01T02:38:03.352278Z","shell.execute_reply":"2022-08-01T02:38:06.608186Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#When x and y are both categorical columns, it plots a nested bar chart, stacked bar chart and heat map.","metadata":{}},{"cell_type":"code","source":"plot(train, \"attribute_0\",\"attribute_1\")","metadata":{"execution":{"iopub.status.busy":"2022-08-01T02:39:25.480216Z","iopub.execute_input":"2022-08-01T02:39:25.480684Z","iopub.status.idle":"2022-08-01T02:39:26.002339Z","shell.execute_reply.started":"2022-08-01T02:39:25.480646Z","shell.execute_reply":"2022-08-01T02:39:26.001321Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#When x is a numerical column, it computes column statistics, generates a Bar chart, Pie chart, WordCloud \n\n#It used to be a histogram, kde plot, box plot and qq-normal plot.","metadata":{}},{"cell_type":"code","source":"#When x is a numerical column, it computes column statistics, and generates a Bar chart, Pie chart, WordCloud\n\n#It used to be a histogram, kde plot, box plot and qq-normal plot. Not anymore.\n\nplot(train, \"failure\")","metadata":{"execution":{"iopub.status.busy":"2022-08-01T02:58:55.403869Z","iopub.execute_input":"2022-08-01T02:58:55.404407Z","iopub.status.idle":"2022-08-01T02:58:56.298714Z","shell.execute_reply.started":"2022-08-01T02:58:55.404366Z","shell.execute_reply":"2022-08-01T02:58:56.296838Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#When x is a numerical column, it computes column statistics, and generates a Bar chart, Pie chart, WordCloud \n\n#It used to be a histogram, a kde plot, a box plot and a qq-normal plot\n\nplot(train, \"attribute_2\")","metadata":{"execution":{"iopub.status.busy":"2022-08-01T02:40:49.842426Z","iopub.execute_input":"2022-08-01T02:40:49.842955Z","iopub.status.idle":"2022-08-01T02:40:50.827131Z","shell.execute_reply.started":"2022-08-01T02:40:49.842903Z","shell.execute_reply":"2022-08-01T02:40:50.825883Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#Categorical column computes column statistics, plots a bar chart, pie chart. It has WordCloud, WorldMap Word Length WordFrequency","metadata":{}},{"cell_type":"code","source":"plot(train, \"product_code\")","metadata":{"execution":{"iopub.status.busy":"2022-08-01T02:42:15.049382Z","iopub.execute_input":"2022-08-01T02:42:15.049921Z","iopub.status.idle":"2022-08-01T02:42:16.038967Z","shell.execute_reply.started":"2022-08-01T02:42:15.049878Z","shell.execute_reply":"2022-08-01T02:42:16.037705Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#When x and y are both numerical columns, it generates a Line Chart and Box plot:\n\nIt used to be a scatter plot, hexbin and boxplot. Not any more?","metadata":{}},{"cell_type":"code","source":"plot(train, \"measurement_17\",\"failure\")","metadata":{"execution":{"iopub.status.busy":"2022-08-01T02:54:00.328183Z","iopub.execute_input":"2022-08-01T02:54:00.329243Z","iopub.status.idle":"2022-08-01T02:54:00.899105Z","shell.execute_reply.started":"2022-08-01T02:54:00.329169Z","shell.execute_reply":"2022-08-01T02:54:00.897465Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot(train, \"failure\",\"measurement_14\")","metadata":{"execution":{"iopub.status.busy":"2022-08-01T02:55:40.342132Z","iopub.execute_input":"2022-08-01T02:55:40.342630Z","iopub.status.idle":"2022-08-01T02:55:40.945526Z","shell.execute_reply.started":"2022-08-01T02:55:40.342587Z","shell.execute_reply":"2022-08-01T02:55:40.943651Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot(train, \"attribute_3\",\"measurement_13\")","metadata":{"execution":{"iopub.status.busy":"2022-08-01T02:51:52.422274Z","iopub.execute_input":"2022-08-01T02:51:52.423645Z","iopub.status.idle":"2022-08-01T02:51:53.320483Z","shell.execute_reply.started":"2022-08-01T02:51:52.423565Z","shell.execute_reply":"2022-08-01T02:51:53.318792Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#Missing values ","metadata":{}},{"cell_type":"code","source":"plot_missing(train)","metadata":{"execution":{"iopub.status.busy":"2022-08-01T02:43:08.879996Z","iopub.execute_input":"2022-08-01T02:43:08.880425Z","iopub.status.idle":"2022-08-01T02:43:10.308334Z","shell.execute_reply.started":"2022-08-01T02:43:08.880391Z","shell.execute_reply":"2022-08-01T02:43:10.306731Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Code by Fabbasso   https://www.kaggle.com/code/fabrizio78/forbes-dataset \n\nimport missingno as msno\n\nfig, ax = plt.subplots(2,2,figsize=(12,7))\naxs = np.ravel(ax)\nmsno.matrix(train,  fontsize=9, color=(0.25,0,0.5),ax=axs[0]);\nmsno.bar(train, fontsize=8, color=(0.25,0,0.5), ax=axs[1]);\nmsno.heatmap(train,fontsize=8,ax=axs[2]);\nmsno.dendrogram(train,fontsize=8,ax=axs[3], orientation='top')\n\nfig.suptitle('Missing Values Analysis', y=1.01, fontsize=15);\n#plt.tight_layout()","metadata":{"execution":{"iopub.status.busy":"2022-08-01T02:44:20.144490Z","iopub.execute_input":"2022-08-01T02:44:20.145023Z","iopub.status.idle":"2022-08-01T02:44:25.063063Z","shell.execute_reply.started":"2022-08-01T02:44:20.144987Z","shell.execute_reply":"2022-08-01T02:44:25.061591Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#Another Cool Correlation Chart","metadata":{}},{"cell_type":"code","source":"#Code by Sadegh Jalalian https://www.kaggle.com/code/sadeghjalalian/prediction-of-stock-index-using-xgboost-rf-and-svm\n\nplt.figure(figsize=(12,8))\nsns.heatmap(train.describe()[1:].transpose(),\n            annot=True,linecolor=\"w\",\n            linewidth=2,cmap=sns.color_palette(\"Set2\"))\nplt.title(\"Data summary\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-08-01T02:45:10.463731Z","iopub.execute_input":"2022-08-01T02:45:10.464192Z","iopub.status.idle":"2022-08-01T02:45:11.578956Z","shell.execute_reply.started":"2022-08-01T02:45:10.464156Z","shell.execute_reply":"2022-08-01T02:45:11.577912Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#Acknowledgements:\n\nDataprep The easiest way to prepare data in Python\n\nDataPrep is free, open-source software released under the MIT license. Anyone can reuse DataPrep code for any purpose.\n\nDataPrep is built using Pandas/Dask DataFrame and can be seamlessly integrated with other Python libraries.\n\nDataPrep is designed for computational notebooks, the most popular environment among data scientists.\n\nJinglin Peng, Weiyuan Wu, Brandon Lockhart, Song Bian, Jing Nathan Yan, Linghao Xu, Zhixuan Chi, Jeffrey M. Rzeszotarski, and Jiannan Wang. DataPrep.EDA: Task-Centric Exploratory Data Analysis for Statistical Modeling in Python. SIGMOD 2021.\n\nSFU - Simon Fraser University\n\nhttps://dataprep.ai/\n\n\nSadegh Jalalian https://www.kaggle.com/code/sadeghjalalian/prediction-of-stock-index-using-xgboost-rf-and-svm\n\nFabbasso   https://www.kaggle.com/code/fabrizio78/forbes-dataset ","metadata":{}},{"cell_type":"markdown","source":"#The ROC curve and its Area, maybe I'll make the next time :)","metadata":{}}]}