{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport seaborn as sns\nimport missingno as msno\nimport matplotlib.pyplot as plt\nimport missingno as msno\nimport numpy as np\nfrom pathlib import Path\nimport path\nimport os\nimport warnings\nwarnings.filterwarnings(\"ignore\")","metadata":{"execution":{"iopub.status.busy":"2022-06-08T12:14:31.857648Z","iopub.execute_input":"2022-06-08T12:14:31.859734Z","iopub.status.idle":"2022-06-08T12:14:33.157764Z","shell.execute_reply.started":"2022-06-08T12:14:31.859609Z","shell.execute_reply":"2022-06-08T12:14:33.157011Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The dataset of this contest is of considerable size. If you're reading raw CSV files, the data barely fits in memory.That's why we read the data from @munumbutt's AMEX-Feather-Dataset. In this Feather file, floating point precision has been reduced from 64 bits to 16 bits and reading a Feather file is faster than reading a csv file because the Feather file format is binary.","metadata":{}},{"cell_type":"code","source":"train = pd.read_feather('../input/amexfeather/train_data.ftr')\ntest = pd.read_feather('../input/amexfeather/test_data.ftr')","metadata":{"execution":{"iopub.status.busy":"2022-06-08T12:14:33.159615Z","iopub.execute_input":"2022-06-08T12:14:33.160747Z","iopub.status.idle":"2022-06-08T12:15:37.321730Z","shell.execute_reply.started":"2022-06-08T12:14:33.160690Z","shell.execute_reply":"2022-06-08T12:15:37.319216Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(list(train.columns))","metadata":{"execution":{"iopub.status.busy":"2022-06-08T12:15:37.325395Z","iopub.execute_input":"2022-06-08T12:15:37.326105Z","iopub.status.idle":"2022-06-08T12:15:37.334717Z","shell.execute_reply.started":"2022-06-08T12:15:37.326040Z","shell.execute_reply":"2022-06-08T12:15:37.333659Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"markdown","source":"# Features similar to binary","metadata":{}},{"cell_type":"markdown","source":"### In this analysis, I decided to see how the features look like binary.\n#### Such features can be seen on the charts below.","metadata":{}},{"cell_type":"code","source":"fig = plt.figure(figsize=(20,5))\nplt.subplot(1, 3, 1)\nsns.histplot(data=train, x=\"B_31\", hue=\"target\", bins=400)\nplt.subplot(1, 3, 2)\nsns.histplot(data=train, x=\"B_32\", hue=\"target\", bins=400)\nplt.subplot(1, 3, 3)\nsns.histplot(data=train, x=\"B_33\", hue=\"target\", bins=400)\nplt.show()","metadata":{"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2022-06-08T12:15:37.336315Z","iopub.execute_input":"2022-06-08T12:15:37.336909Z","iopub.status.idle":"2022-06-08T12:15:52.263066Z","shell.execute_reply.started":"2022-06-08T12:15:37.336865Z","shell.execute_reply":"2022-06-08T12:15:52.261499Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_noise(train,col, treshold, bins = 400):\n    fig = plt.figure(figsize=(20,5))\n    plt.subplot(1, 3, 1)\n    sns.histplot(data=train, x=col, hue=\"target\", bins=bins)\n    plt.subplot(1, 3, 2)\n    sns.histplot(data=train[train[col] < treshold], x=col, hue=\"target\", bins=bins)\n    plt.subplot(1, 3, 3)\n    sns.histplot(data=train[train[col] > treshold], x=col, hue=\"target\", bins=bins)\n    plt.show() ","metadata":{"execution":{"iopub.status.busy":"2022-06-08T12:15:52.267167Z","iopub.execute_input":"2022-06-08T12:15:52.267747Z","iopub.status.idle":"2022-06-08T12:15:52.277258Z","shell.execute_reply.started":"2022-06-08T12:15:52.267694Z","shell.execute_reply":"2022-06-08T12:15:52.275912Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_noise(train= train, col = \"B_31\", treshold = 0.2 )","metadata":{"execution":{"iopub.status.busy":"2022-06-08T12:15:52.282946Z","iopub.execute_input":"2022-06-08T12:15:52.284542Z","iopub.status.idle":"2022-06-08T12:16:13.417084Z","shell.execute_reply.started":"2022-06-08T12:15:52.284497Z","shell.execute_reply":"2022-06-08T12:16:13.416047Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_noise(train= train, col = \"B_32\", treshold = 0.2 )","metadata":{"execution":{"iopub.status.busy":"2022-06-08T12:16:13.418459Z","iopub.execute_input":"2022-06-08T12:16:13.418836Z","iopub.status.idle":"2022-06-08T12:16:34.341968Z","shell.execute_reply.started":"2022-06-08T12:16:13.418793Z","shell.execute_reply":"2022-06-08T12:16:34.340872Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_noise(train= train, col = \"B_33\", treshold = 0.2 )","metadata":{"execution":{"iopub.status.busy":"2022-06-08T12:16:34.343236Z","iopub.execute_input":"2022-06-08T12:16:34.343589Z","iopub.status.idle":"2022-06-08T12:16:55.798072Z","shell.execute_reply.started":"2022-06-08T12:16:34.343555Z","shell.execute_reply":"2022-06-08T12:16:55.796967Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_noise(train= train, col = \"B_36\", treshold = 0.2 )","metadata":{"execution":{"iopub.status.busy":"2022-06-08T12:16:55.799341Z","iopub.execute_input":"2022-06-08T12:16:55.799656Z","iopub.status.idle":"2022-06-08T12:17:16.952884Z","shell.execute_reply.started":"2022-06-08T12:16:55.799628Z","shell.execute_reply":"2022-06-08T12:17:16.952059Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_noise(train= train, col = \"B_8\", treshold = 0.2 )","metadata":{"execution":{"iopub.status.busy":"2022-06-08T12:17:16.954010Z","iopub.execute_input":"2022-06-08T12:17:16.954825Z","iopub.status.idle":"2022-06-08T12:17:38.217078Z","shell.execute_reply.started":"2022-06-08T12:17:16.954783Z","shell.execute_reply":"2022-06-08T12:17:38.216013Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_noise(train= train, col = \"D_103\", treshold = 0.2 )","metadata":{"execution":{"iopub.status.busy":"2022-06-08T12:17:38.218343Z","iopub.execute_input":"2022-06-08T12:17:38.218673Z","iopub.status.idle":"2022-06-08T12:17:59.085929Z","shell.execute_reply.started":"2022-06-08T12:17:38.218643Z","shell.execute_reply":"2022-06-08T12:17:59.084717Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_noise(train= train, col = \"D_104\", treshold = 0.2 )","metadata":{"execution":{"iopub.status.busy":"2022-06-08T12:17:59.087527Z","iopub.execute_input":"2022-06-08T12:17:59.088043Z","iopub.status.idle":"2022-06-08T12:18:20.360040Z","shell.execute_reply.started":"2022-06-08T12:17:59.087991Z","shell.execute_reply":"2022-06-08T12:18:20.358350Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_noise(train= train, col = \"D_127\", treshold = 0.2 )","metadata":{"execution":{"iopub.status.busy":"2022-06-08T12:18:20.365371Z","iopub.execute_input":"2022-06-08T12:18:20.365827Z","iopub.status.idle":"2022-06-08T12:18:41.592446Z","shell.execute_reply.started":"2022-06-08T12:18:20.365790Z","shell.execute_reply":"2022-06-08T12:18:41.591226Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_noise(train= train, col = \"D_128\", treshold = 0.2 )","metadata":{"execution":{"iopub.status.busy":"2022-06-08T12:18:41.594081Z","iopub.execute_input":"2022-06-08T12:18:41.595121Z","iopub.status.idle":"2022-06-08T12:19:02.011187Z","shell.execute_reply.started":"2022-06-08T12:18:41.595069Z","shell.execute_reply":"2022-06-08T12:19:02.010036Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_noise(train= train, col = \"D_129\", treshold = 0.2 )","metadata":{"execution":{"iopub.status.busy":"2022-06-08T12:19:02.012749Z","iopub.execute_input":"2022-06-08T12:19:02.013261Z","iopub.status.idle":"2022-06-08T12:19:22.790098Z","shell.execute_reply.started":"2022-06-08T12:19:02.013212Z","shell.execute_reply":"2022-06-08T12:19:22.789019Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_noise(train= train, col = \"D_130\", treshold = 0.2 )","metadata":{"execution":{"iopub.status.busy":"2022-06-08T12:19:22.791592Z","iopub.execute_input":"2022-06-08T12:19:22.792095Z","iopub.status.idle":"2022-06-08T12:19:43.470415Z","shell.execute_reply.started":"2022-06-08T12:19:22.792045Z","shell.execute_reply":"2022-06-08T12:19:43.469288Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_noise(train= train, col = \"D_135\", treshold = 0.2 )","metadata":{"execution":{"iopub.status.busy":"2022-06-08T12:19:43.471972Z","iopub.execute_input":"2022-06-08T12:19:43.472357Z","iopub.status.idle":"2022-06-08T12:19:52.396730Z","shell.execute_reply.started":"2022-06-08T12:19:43.472321Z","shell.execute_reply":"2022-06-08T12:19:52.396006Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_noise(train= train, col = \"D_137\", treshold = 0.2 )","metadata":{"execution":{"iopub.status.busy":"2022-06-08T12:19:52.397718Z","iopub.execute_input":"2022-06-08T12:19:52.398641Z","iopub.status.idle":"2022-06-08T12:20:00.992390Z","shell.execute_reply.started":"2022-06-08T12:19:52.398587Z","shell.execute_reply":"2022-06-08T12:20:00.990584Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_noise(train= train, col = \"D_139\", treshold = 0.2 )","metadata":{"execution":{"iopub.status.busy":"2022-06-08T12:20:00.994043Z","iopub.execute_input":"2022-06-08T12:20:00.994414Z","iopub.status.idle":"2022-06-08T12:20:21.404589Z","shell.execute_reply.started":"2022-06-08T12:20:00.994381Z","shell.execute_reply":"2022-06-08T12:20:21.403629Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_noise(train= train, col = \"D_140\", treshold = 0.2 )","metadata":{"execution":{"iopub.status.busy":"2022-06-08T12:20:21.405895Z","iopub.execute_input":"2022-06-08T12:20:21.406299Z","iopub.status.idle":"2022-06-08T12:20:42.655120Z","shell.execute_reply.started":"2022-06-08T12:20:21.406264Z","shell.execute_reply":"2022-06-08T12:20:42.654010Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_noise(train= train, col = \"D_143\", treshold = 0.2 )","metadata":{"execution":{"iopub.status.busy":"2022-06-08T12:20:42.656614Z","iopub.execute_input":"2022-06-08T12:20:42.657092Z","iopub.status.idle":"2022-06-08T12:21:03.557585Z","shell.execute_reply.started":"2022-06-08T12:20:42.657046Z","shell.execute_reply":"2022-06-08T12:21:03.556014Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_noise(train= train, col = \"D_86\", treshold = 0.2 )","metadata":{"execution":{"iopub.status.busy":"2022-06-08T12:21:03.558822Z","iopub.execute_input":"2022-06-08T12:21:03.559216Z","iopub.status.idle":"2022-06-08T12:21:24.628177Z","shell.execute_reply.started":"2022-06-08T12:21:03.559183Z","shell.execute_reply":"2022-06-08T12:21:24.627020Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_noise(train= train, col = \"D_89\", treshold = 0.2 )","metadata":{"execution":{"iopub.status.busy":"2022-06-08T12:21:24.629393Z","iopub.execute_input":"2022-06-08T12:21:24.629751Z","iopub.status.idle":"2022-06-08T12:21:48.110141Z","shell.execute_reply.started":"2022-06-08T12:21:24.629707Z","shell.execute_reply":"2022-06-08T12:21:48.108711Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_noise(train= train, col = \"D_93\", treshold = 0.2 )","metadata":{"execution":{"iopub.status.busy":"2022-06-08T12:21:48.111777Z","iopub.execute_input":"2022-06-08T12:21:48.112299Z","iopub.status.idle":"2022-06-08T12:22:09.998540Z","shell.execute_reply.started":"2022-06-08T12:21:48.112245Z","shell.execute_reply":"2022-06-08T12:22:09.996839Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_noise(train= train, col = \"D_94\", treshold = 0.2 )\n","metadata":{"execution":{"iopub.status.busy":"2022-06-08T12:22:10.000178Z","iopub.execute_input":"2022-06-08T12:22:10.000628Z","iopub.status.idle":"2022-06-08T12:22:30.909669Z","shell.execute_reply.started":"2022-06-08T12:22:10.000595Z","shell.execute_reply":"2022-06-08T12:22:30.908554Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_noise(train= train, col = \"D_96\", treshold = 0.2 )\n","metadata":{"execution":{"iopub.status.busy":"2022-06-08T12:22:30.911149Z","iopub.execute_input":"2022-06-08T12:22:30.911963Z","iopub.status.idle":"2022-06-08T12:22:51.926608Z","shell.execute_reply.started":"2022-06-08T12:22:30.911908Z","shell.execute_reply":"2022-06-08T12:22:51.925597Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_noise(train= train, col = \"R_15\", treshold = 0.2 )\n","metadata":{"execution":{"iopub.status.busy":"2022-06-08T12:22:51.928208Z","iopub.execute_input":"2022-06-08T12:22:51.929228Z","iopub.status.idle":"2022-06-08T12:23:13.013156Z","shell.execute_reply.started":"2022-06-08T12:22:51.929185Z","shell.execute_reply":"2022-06-08T12:23:13.011876Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_noise(train= train, col = \"R_17\", treshold = 0.2,bins= 400 )\n","metadata":{"execution":{"iopub.status.busy":"2022-06-08T12:23:13.015031Z","iopub.execute_input":"2022-06-08T12:23:13.015558Z","iopub.status.idle":"2022-06-08T12:23:33.662726Z","shell.execute_reply.started":"2022-06-08T12:23:13.015505Z","shell.execute_reply":"2022-06-08T12:23:33.661692Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_noise(train= train, col = \"R_18\", treshold = 0.5,bins= 400 )\n","metadata":{"execution":{"iopub.status.busy":"2022-06-08T12:23:33.664381Z","iopub.execute_input":"2022-06-08T12:23:33.664754Z","iopub.status.idle":"2022-06-08T12:23:53.958784Z","shell.execute_reply.started":"2022-06-08T12:23:33.664719Z","shell.execute_reply":"2022-06-08T12:23:53.957725Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_noise(train= train, col = \"R_19\", treshold = 0.5,bins= 400 )\n","metadata":{"execution":{"iopub.status.busy":"2022-06-08T12:23:53.960273Z","iopub.execute_input":"2022-06-08T12:23:53.960789Z","iopub.status.idle":"2022-06-08T12:24:15.396150Z","shell.execute_reply.started":"2022-06-08T12:23:53.960713Z","shell.execute_reply":"2022-06-08T12:24:15.394593Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_noise(train= train, col = \"R_2\", treshold = 0.5,bins= 400 )\n","metadata":{"execution":{"iopub.status.busy":"2022-06-08T12:24:15.398141Z","iopub.execute_input":"2022-06-08T12:24:15.398603Z","iopub.status.idle":"2022-06-08T12:24:37.265079Z","shell.execute_reply.started":"2022-06-08T12:24:15.398566Z","shell.execute_reply":"2022-06-08T12:24:37.263297Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_noise(train= train, col = \"R_21\", treshold = 0.5,bins= 400 )\n","metadata":{"execution":{"iopub.status.busy":"2022-06-08T12:24:37.266670Z","iopub.execute_input":"2022-06-08T12:24:37.267163Z","iopub.status.idle":"2022-06-08T12:24:59.027576Z","shell.execute_reply.started":"2022-06-08T12:24:37.267120Z","shell.execute_reply":"2022-06-08T12:24:59.026553Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_noise(train= train, col = \"R_23\", treshold = 0.5,bins= 400 )\n","metadata":{"execution":{"iopub.status.busy":"2022-06-08T12:24:59.028804Z","iopub.execute_input":"2022-06-08T12:24:59.029188Z","iopub.status.idle":"2022-06-08T12:25:19.719072Z","shell.execute_reply.started":"2022-06-08T12:24:59.029154Z","shell.execute_reply":"2022-06-08T12:25:19.718248Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_noise(train= train, col = \"R_22\", treshold = 0.5,bins= 400 )\n","metadata":{"execution":{"iopub.status.busy":"2022-06-08T12:25:19.720188Z","iopub.execute_input":"2022-06-08T12:25:19.721315Z","iopub.status.idle":"2022-06-08T12:25:47.276328Z","shell.execute_reply.started":"2022-06-08T12:25:19.721241Z","shell.execute_reply":"2022-06-08T12:25:47.275068Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_noise(train= train, col = \"R_24\", treshold = 0.5,bins= 400 )\n","metadata":{"execution":{"iopub.status.busy":"2022-06-08T12:25:47.278156Z","iopub.execute_input":"2022-06-08T12:25:47.279504Z","iopub.status.idle":"2022-06-08T12:26:08.897182Z","shell.execute_reply.started":"2022-06-08T12:25:47.279455Z","shell.execute_reply":"2022-06-08T12:26:08.896034Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_noise(train= train, col = \"R_25\", treshold = 0.5,bins= 400 )\n","metadata":{"execution":{"iopub.status.busy":"2022-06-08T12:26:08.898658Z","iopub.execute_input":"2022-06-08T12:26:08.899159Z","iopub.status.idle":"2022-06-08T12:26:30.779807Z","shell.execute_reply.started":"2022-06-08T12:26:08.899109Z","shell.execute_reply":"2022-06-08T12:26:30.778721Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_noise(train= train, col = \"R_27\", treshold = 0.5,bins= 400 )\n","metadata":{"execution":{"iopub.status.busy":"2022-06-08T12:26:30.781131Z","iopub.execute_input":"2022-06-08T12:26:30.781495Z","iopub.status.idle":"2022-06-08T12:26:53.230861Z","shell.execute_reply.started":"2022-06-08T12:26:30.781463Z","shell.execute_reply":"2022-06-08T12:26:53.229373Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_noise(train= train, col = \"R_28\", treshold = 0.5,bins= 400 )\n","metadata":{"execution":{"iopub.status.busy":"2022-06-08T12:26:53.232213Z","iopub.execute_input":"2022-06-08T12:26:53.232581Z","iopub.status.idle":"2022-06-08T12:27:15.676811Z","shell.execute_reply.started":"2022-06-08T12:26:53.232548Z","shell.execute_reply":"2022-06-08T12:27:15.676022Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_noise(train= train, col = \"R_4\", treshold = 0.5,bins= 400 )\n","metadata":{"execution":{"iopub.status.busy":"2022-06-08T12:27:15.678027Z","iopub.execute_input":"2022-06-08T12:27:15.678775Z","iopub.status.idle":"2022-06-08T12:27:37.630692Z","shell.execute_reply.started":"2022-06-08T12:27:15.678733Z","shell.execute_reply":"2022-06-08T12:27:37.629434Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_noise(train= train, col = \"S_18\", treshold = 0.5,bins= 400 )\n","metadata":{"execution":{"iopub.status.busy":"2022-06-08T12:27:37.637902Z","iopub.execute_input":"2022-06-08T12:27:37.638338Z","iopub.status.idle":"2022-06-08T12:27:59.597253Z","shell.execute_reply.started":"2022-06-08T12:27:37.638302Z","shell.execute_reply":"2022-06-08T12:27:59.596235Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_noise(train= train, col = \"S_19\", treshold = 0.5,bins= 400 )\n","metadata":{"execution":{"iopub.status.busy":"2022-06-08T12:27:59.598868Z","iopub.execute_input":"2022-06-08T12:27:59.599238Z","iopub.status.idle":"2022-06-08T12:28:19.938239Z","shell.execute_reply.started":"2022-06-08T12:27:59.599206Z","shell.execute_reply":"2022-06-08T12:28:19.936632Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_noise(train= train, col = \"S_20\", treshold = 0.5,bins= 400 )\n","metadata":{"execution":{"iopub.status.busy":"2022-06-08T12:28:19.939890Z","iopub.execute_input":"2022-06-08T12:28:19.940476Z","iopub.status.idle":"2022-06-08T12:28:41.061076Z","shell.execute_reply.started":"2022-06-08T12:28:19.940421Z","shell.execute_reply":"2022-06-08T12:28:41.059978Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_noise(train= train, col = \"S_6\", treshold = 0.5,bins= 400 )\n","metadata":{"execution":{"iopub.status.busy":"2022-06-08T12:28:41.062250Z","iopub.execute_input":"2022-06-08T12:28:41.062599Z","iopub.status.idle":"2022-06-08T12:29:01.984825Z","shell.execute_reply.started":"2022-06-08T12:28:41.062568Z","shell.execute_reply":"2022-06-08T12:29:01.983684Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Observations\n### My hypotheses about features\nDistributions are divided into several groups:\n1. Uniform noise around 0 and 1 (B32 B33 B8 D103 D127 D128 D129 D130 D135 D137 D140 D143 \n   D86 D93 D94 D96 R15 R19 R2 R21 R23 R22 R24 R25 R28 R4 S18 S20 S6)\n2. 0 and 1 B31 S19\n3. Uniform and normal noise D104 ?R27?\n5. Distributions similar to noisy categorical variables D89 R17","metadata":{}},{"cell_type":"markdown","source":"# Features similar to categorial","metadata":{}},{"cell_type":"markdown","source":"### I assume that some features may be coded and noisy features \nI want to study such signs in more detail and try to sort such features into groups.","metadata":{}},{"cell_type":"code","source":"cols = [\"B_16\",  'B_17', \"B_18\",'B_19',\"B_20\",'B_22', 'D_107','D_108','D_111','D_122','D_125','D_131','D_136','D_138',\n        'D_139', 'D_141','D_145','D_44','D_51','D_70','D_74', 'D_75','D_78', 'D_79', 'D_80',\n        'D_91', 'D_92','P_4', 'R_1','R_10', 'R_11','R_4','R_9','S_11','S_13', 'S_15','S_18','S_6','S_8']\n\nfor col in cols:\n    plot_noise(train= train, col = col, treshold = 0, bins = 400 )\n    print(train[col].nunique())\n    print(train[col].value_counts())","metadata":{"execution":{"iopub.status.busy":"2022-06-08T12:29:01.986206Z","iopub.execute_input":"2022-06-08T12:29:01.987201Z","iopub.status.idle":"2022-06-08T12:40:17.271558Z","shell.execute_reply.started":"2022-06-08T12:29:01.987159Z","shell.execute_reply":"2022-06-08T12:40:17.270519Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Observations\n### My hypotheses about features\nDistributions are divided into several groups:\n1. [\"B_16\",\"B_18\",'B_19', \"D_111\", \"D_134\", \"D_139\", \"S_13\", \"S_18\", \"S_6\"]\n2. [\"B_22\", \"D_107\", \"D_108\", \"D_125\", \"D_138\", \"D_145\", \"D_44\", \"D_51\", \"D_70\", \"D_78\", \"D_79\", \"D_91\", 'R_1','R_10', 'R_11','R_4']\n3. [\"D_122\", \"D_136\", \"D_138\", \"D_74\", \"D_75\", \"D_80\", \"R_9\", \"S_11\", \"S_13\", \"S_15\", \"S_8\"]","metadata":{}},{"cell_type":"markdown","source":"#### To make sure that this is possible, just look at how clearly the noise in the S_11 feature is visible.","metadata":{}},{"cell_type":"markdown","source":"## Vote for this notebook if its content was useful or interesting to you ","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# To be continued...","metadata":{}}]}