{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nplt.style.use(\"ggplot\")\nfrom sklearn.model_selection import train_test_split","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-08-10T09:40:11.364298Z","iopub.execute_input":"2022-08-10T09:40:11.365314Z","iopub.status.idle":"2022-08-10T09:40:12.676594Z","shell.execute_reply.started":"2022-08-10T09:40:11.365210Z","shell.execute_reply":"2022-08-10T09:40:12.675614Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_set = pd.read_csv(\"../input/tabular-playground-series-aug-2022/train.csv\")\ntest_set = pd.read_csv(\"../input/tabular-playground-series-aug-2022/test.csv\")\nsample_submission = pd.read_csv(\"../input/tabular-playground-series-aug-2022/sample_submission.csv\")\n\ntrain = train_set.copy()\ntest = test_set.copy()","metadata":{"execution":{"iopub.status.busy":"2022-08-10T09:40:12.678337Z","iopub.execute_input":"2022-08-10T09:40:12.678878Z","iopub.status.idle":"2022-08-10T09:40:12.956858Z","shell.execute_reply.started":"2022-08-10T09:40:12.678848Z","shell.execute_reply":"2022-08-10T09:40:12.956020Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train Summary","metadata":{}},{"cell_type":"code","source":"summary = pd.DataFrame(index=train.columns.values)\nsummary[\"NULL_COUNT\"] = [train[column].isnull().sum() for column in train.columns.values]\nsummary[\"PCT\"] = [f\"{100*train[column].isnull().sum()/len(train):.2f} %\" for column in train.columns.values]\nsummary[\"DTYPE\"] = [train[column].dtype for column in train.columns.values]\nsummary[\"UNIQUE_COUNTS\"] = [len(train[column].unique()) for column in train.columns.values]\nsummary[\"MAX\"] = [train[column].max() if train[column].dtype != \"object\" else \"-\" for column in train.columns.values]\nsummary[\"MIN\"] = [train[column].min() if train[column].dtype != \"object\" else \"-\" for column in train.columns.values]\nsummary[\"MEDIAN\"] = [train[column].median() if train[column].dtype != \"object\" else \"-\" for column in train.columns.values]\nsummary[\"STD\"] = [train[column].std() if train[column].dtype != \"object\" else \"-\" for column in train.columns.values]\nsummary","metadata":{"execution":{"iopub.status.busy":"2022-08-10T09:40:12.958342Z","iopub.execute_input":"2022-08-10T09:40:12.959644Z","iopub.status.idle":"2022-08-10T09:40:13.076283Z","shell.execute_reply.started":"2022-08-10T09:40:12.959597Z","shell.execute_reply":"2022-08-10T09:40:13.075203Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Test Summary","metadata":{}},{"cell_type":"code","source":"summary = pd.DataFrame(index=test.columns.values)\nsummary[\"NULL_COUNT\"] = [test[column].isnull().sum() for column in test.columns.values]\nsummary[\"PCT\"] = [f\"{100*test[column].isnull().sum()/len(test):.2f} %\" for column in test.columns.values]\nsummary[\"DTYPE\"] = [test[column].dtype for column in test.columns.values]\nsummary[\"UNIQUE_COUNTS\"] = [len(test[column].unique()) for column in test.columns.values]\nsummary[\"MAX\"] = [test[column].max() if test[column].dtype != \"object\" else \"-\" for column in test.columns.values]\nsummary[\"MIN\"] = [test[column].min() if test[column].dtype != \"object\" else \"-\" for column in test.columns.values]\nsummary[\"MEDIAN\"] = [test[column].median() if test[column].dtype != \"object\" else \"-\" for column in test.columns.values]\nsummary[\"STD\"] = [test[column].std() if test[column].dtype != \"object\" else \"-\" for column in test.columns.values]\nsummary","metadata":{"execution":{"iopub.status.busy":"2022-08-10T09:40:13.077941Z","iopub.execute_input":"2022-08-10T09:40:13.078241Z","iopub.status.idle":"2022-08-10T09:40:13.169894Z","shell.execute_reply.started":"2022-08-10T09:40:13.078214Z","shell.execute_reply":"2022-08-10T09:40:13.168787Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.head()","metadata":{"execution":{"iopub.status.busy":"2022-08-10T09:42:04.100983Z","iopub.execute_input":"2022-08-10T09:42:04.101578Z","iopub.status.idle":"2022-08-10T09:42:04.134560Z","shell.execute_reply.started":"2022-08-10T09:42:04.101525Z","shell.execute_reply":"2022-08-10T09:42:04.133409Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test.head()","metadata":{"execution":{"iopub.status.busy":"2022-08-10T09:42:11.891329Z","iopub.execute_input":"2022-08-10T09:42:11.891757Z","iopub.status.idle":"2022-08-10T09:42:11.921622Z","shell.execute_reply.started":"2022-08-10T09:42:11.891717Z","shell.execute_reply":"2022-08-10T09:42:11.920554Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ones = len(train[train[\"failure\"] == 1])\nzeros = len(train[train[\"failure\"] == 0])\n\nplt.figure(figsize=(8,8))\nplt.pie([zeros, ones], labels=[\"0\",\"1\"], colors=[\"lavender\", \"powderblue\"], wedgeprops={'linewidth':3.5, 'edgecolor':'white'}, autopct='%1.1f%%')\nmy_circle = plt.Circle((0,0), 0.65, color='white')\np = plt.gcf()\np.gca().add_artist(my_circle)\nplt.title(\"TARGET DISTRIBUTIONS IN TRAIN SET\")\nplt.legend()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-08-08T14:08:35.437038Z","iopub.execute_input":"2022-08-08T14:08:35.437549Z","iopub.status.idle":"2022-08-08T14:08:35.720749Z","shell.execute_reply.started":"2022-08-08T14:08:35.437512Z","shell.execute_reply":"2022-08-08T14:08:35.719601Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for column in test.columns:\n    print(f\"\\33[1m\\33[34m{column}\\33[0m\\n\\33[34mTRAIN: \\33[0m{len(train[column].unique())} unique values\\n\\33[34mTEST: \\33[0m{len(test[column].unique())} unique values\\n\\33[34m\\n\")\n    print(f\"\\33[34mTRAIN DIFFERENCE FROM TEST: \\33[0m{len(set(train[column].unique()) - set(test[column].unique()))}\\n\\33[34mTEST DIFFERENCE FROM TRAIN: \\33[0m{len(set(test[column].unique()) - set(train[column].unique()))}\\n\")","metadata":{"execution":{"iopub.status.busy":"2022-08-10T09:43:06.888685Z","iopub.execute_input":"2022-08-10T09:43:06.889093Z","iopub.status.idle":"2022-08-10T09:43:07.107815Z","shell.execute_reply.started":"2022-08-10T09:43:06.889060Z","shell.execute_reply":"2022-08-10T09:43:07.106507Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"numeric_columns = test.select_dtypes(include=\"number\").columns\ni = 1\nplt.figure(figsize=(18,12))\nfor column in numeric_columns[1:]:\n        plt.subplot(7,3,i)\n        sns.histplot(data=train, x=column, hue=\"failure\")\n        i = i+1\nplt.subplots_adjust(hspace=0.8)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-08-10T10:30:40.533818Z","iopub.execute_input":"2022-08-10T10:30:40.534224Z","iopub.status.idle":"2022-08-10T10:30:51.434643Z","shell.execute_reply.started":"2022-08-10T10:30:40.534191Z","shell.execute_reply":"2022-08-10T10:30:51.433759Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# product_code","metadata":{}},{"cell_type":"code","source":"def product_code(df):\n    df[\"product_code\"] = df[\"product_code\"].str.split(\"_\",1,expand=True)\n    return df[\"product_code\"]\ntrain[\"product_code\"] = product_code(train)\ntest[\"product_code\"] = product_code(test)","metadata":{"execution":{"iopub.status.busy":"2022-08-10T10:33:10.939859Z","iopub.execute_input":"2022-08-10T10:33:10.940697Z","iopub.status.idle":"2022-08-10T10:33:11.600195Z","shell.execute_reply.started":"2022-08-10T10:33:10.940629Z","shell.execute_reply":"2022-08-10T10:33:11.598636Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(15,5))\nplt.subplot(1,2,1)\nsns.histplot(data=train, x=\"product_code\", color=\"deepskyblue\", edgecolor=\"white\")\nplt.title(\"Train set 'product_code'\")\nplt.subplot(1,2,2)\nsns.histplot(data=test, x=\"product_code\", color=\"deepskyblue\", edgecolor=\"white\")\nplt.title(\"Test set 'product_code'\")\nplt.subplots_adjust(wspace=0.4)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-08-10T10:46:04.061541Z","iopub.execute_input":"2022-08-10T10:46:04.061912Z","iopub.status.idle":"2022-08-10T10:46:04.435039Z","shell.execute_reply.started":"2022-08-10T10:46:04.061882Z","shell.execute_reply":"2022-08-10T10:46:04.433929Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}