{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Final Shakeup\n\nPretty late in the game to make any big changes. But recently it hit me that after all the time spent on the competition, the final shakeup could lay everything to waste (DUHHHH). I'm a little slow on the uptake. There have been several topics on LB shakeup and the differences in distribution.\n\nI spent the time I had learning to implement new algorithms and doing feature engineering. Comes down to lack of prioritization and clear direction in my case. But it's great to learn! I look forward to reading the brilliant solutions at the end of the competition\n\n![title](https://i0.kym-cdn.com/photos/images/original/000/549/564/f82.gif)\n\n\n\nHere is some quick analysis I did to figure out how screwed I am\n\nCredits:\n\n- https://www.kaggle.com/code/zakopur0/adversarial-validation-private-vs-public\n- https://www.kaggle.com/datasets/raddar/amex-data-integer-dtypes-parquet-format","metadata":{}},{"cell_type":"markdown","source":"# Load libraries","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\n\nimport gc","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-08-17T14:47:16.570326Z","iopub.execute_input":"2022-08-17T14:47:16.572121Z","iopub.status.idle":"2022-08-17T14:47:16.602416Z","shell.execute_reply.started":"2022-08-17T14:47:16.572003Z","shell.execute_reply":"2022-08-17T14:47:16.601222Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Prepare test data","metadata":{}},{"cell_type":"code","source":"path = \"../input/amex-data-integer-dtypes-parquet-format/test.parquet\"\ntest = pd.read_parquet(path)\nprint(f\"Test data shape: {test.shape}\")","metadata":{"execution":{"iopub.status.busy":"2022-08-17T14:47:16.604697Z","iopub.execute_input":"2022-08-17T14:47:16.605379Z","iopub.status.idle":"2022-08-17T14:47:59.885999Z","shell.execute_reply.started":"2022-08-17T14:47:16.605341Z","shell.execute_reply":"2022-08-17T14:47:59.884631Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"temp = test.groupby('customer_ID')['S_2'].max().reset_index()\ntemp['S_2'] = pd.to_datetime(temp['S_2'])\ntemp['S_2_month'] = temp['S_2'].dt.month\n\ntemp['S_2_month'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-08-17T14:47:59.888053Z","iopub.execute_input":"2022-08-17T14:47:59.888944Z","iopub.status.idle":"2022-08-17T14:49:26.628405Z","shell.execute_reply.started":"2022-08-17T14:47:59.888893Z","shell.execute_reply":"2022-08-17T14:49:26.627106Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"private_custLst = temp[temp['S_2_month']==10]['customer_ID'].tolist()\n\ndel temp\n_ = gc.collect()\n\nprint(f\"Number of customers in private LB: {len(private_custLst)}\")","metadata":{"execution":{"iopub.status.busy":"2022-08-17T14:49:26.630328Z","iopub.execute_input":"2022-08-17T14:49:26.630976Z","iopub.status.idle":"2022-08-17T14:49:26.818232Z","shell.execute_reply.started":"2022-08-17T14:49:26.630935Z","shell.execute_reply":"2022-08-17T14:49:26.816387Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_private = test[test['customer_ID'].isin(private_custLst)].copy()\n\ndel test\n_ = gc.collect()\n\nprint(f\"Shape of private test set: {test_private.shape}\")","metadata":{"execution":{"iopub.status.busy":"2022-08-17T14:49:26.822183Z","iopub.execute_input":"2022-08-17T14:49:26.823043Z","iopub.status.idle":"2022-08-17T14:49:35.310786Z","shell.execute_reply.started":"2022-08-17T14:49:26.822999Z","shell.execute_reply":"2022-08-17T14:49:35.309418Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Read in train data","metadata":{}},{"cell_type":"code","source":"path = \"../input/amex-data-integer-dtypes-parquet-format/train.parquet\"\ntrain = pd.read_parquet(path)\nprint(f\"Train data shape: {train.shape}\")","metadata":{"execution":{"iopub.status.busy":"2022-08-17T14:49:35.312567Z","iopub.execute_input":"2022-08-17T14:49:35.313038Z","iopub.status.idle":"2022-08-17T14:49:59.516377Z","shell.execute_reply.started":"2022-08-17T14:49:35.312995Z","shell.execute_reply":"2022-08-17T14:49:59.514788Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Compare nulls","metadata":{}},{"cell_type":"code","source":"nulltrain_df = ((train.isnull().sum())/train.shape[0]).to_frame()\nnulltrain_df.columns = ['train']\n\nnullpriv_df = ((test_private.isnull().sum())/test_private.shape[0]).to_frame()\nnullpriv_df.columns = ['test_priv']\n\nnull_df = pd.concat([nulltrain_df, nullpriv_df], axis=1)\n\nnull_df['null_count'] = null_df['train']+null_df['test_priv']\n\nnull_df['train_priv_diff'] = np.abs(null_df['train'] - null_df['test_priv'])\nnull_df['train_priv_diff_rank'] = null_df['train_priv_diff'].rank(method='min').values \n\nnull_df.shape","metadata":{"execution":{"iopub.status.busy":"2022-08-17T14:49:59.520218Z","iopub.execute_input":"2022-08-17T14:49:59.520606Z","iopub.status.idle":"2022-08-17T14:50:04.802472Z","shell.execute_reply.started":"2022-08-17T14:49:59.520572Z","shell.execute_reply":"2022-08-17T14:50:04.801124Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"null_df.sort_values(by=['train_priv_diff_rank'], ascending=False)","metadata":{"execution":{"iopub.status.busy":"2022-08-17T14:50:04.804055Z","iopub.execute_input":"2022-08-17T14:50:04.804631Z","iopub.status.idle":"2022-08-17T14:50:04.837045Z","shell.execute_reply.started":"2022-08-17T14:50:04.804585Z","shell.execute_reply":"2022-08-17T14:50:04.835873Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"_ = plt.hist(null_df['train_priv_diff'])","metadata":{"execution":{"iopub.status.busy":"2022-08-17T14:50:04.839402Z","iopub.execute_input":"2022-08-17T14:50:04.839918Z","iopub.status.idle":"2022-08-17T14:50:05.086635Z","shell.execute_reply.started":"2022-08-17T14:50:04.839869Z","shell.execute_reply":"2022-08-17T14:50:05.085464Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Looks like there is a small difference in % of nulls for most columns between the train set and the private LB\n\nAnd two columns with massive difference \n\n- B_29 with a 37% difference (this has been shown in several other notebooks and discussions)\n- S_9 with a 25% difference\n\nIdeally, we could build more sophisticated imputation strategies for these columns. Especially B_29 where there is more data available in the test set than the train set. \n\nHowever, my last minute bandaid approach is probably going to be to remove these columns from my modeling datasets","metadata":{}},{"cell_type":"code","source":"del null_df, nulltrain_df, nullpriv_df\n_ = gc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-08-17T14:50:05.088158Z","iopub.execute_input":"2022-08-17T14:50:05.089051Z","iopub.status.idle":"2022-08-17T14:50:05.216981Z","shell.execute_reply.started":"2022-08-17T14:50:05.089013Z","shell.execute_reply":"2022-08-17T14:50:05.215159Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Compare mean","metadata":{}},{"cell_type":"code","source":"removeCols = ['customer_ID','S_2']\ncatCols = [\"B_30\",\"B_38\",\"D_114\",\"D_116\",\"D_117\",\"D_120\",\"D_126\",\"D_63\",\"D_64\",\"D_66\",\"D_68\"]\n\ntrain.drop(columns = removeCols+catCols, inplace=True)\ntest_private.drop(columns = removeCols+catCols, inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-08-17T14:50:05.219180Z","iopub.execute_input":"2022-08-17T14:50:05.220091Z","iopub.status.idle":"2022-08-17T14:50:09.960043Z","shell.execute_reply.started":"2022-08-17T14:50:05.220040Z","shell.execute_reply":"2022-08-17T14:50:09.958667Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"meantrain_df = np.mean(train).to_frame()\nmeantrain_df.columns = ['train']\n\nmeanpriv_df = np.mean(test_private).to_frame()\nmeanpriv_df.columns = ['test_priv']\n\nrangetrain_df = (train.max() - train.min()).to_frame()\nrangetrain_df.columns = ['range']\n\nmean_df = pd.concat([meantrain_df, meanpriv_df, rangetrain_df], axis=1)\n\nmean_df['train_priv_diff'] = np.abs(mean_df['train'] - mean_df['test_priv']) / mean_df['range']\nmean_df['train_priv_diff_rank'] = mean_df['train_priv_diff'].rank(method='min').values \n\nmean_df.shape","metadata":{"execution":{"iopub.status.busy":"2022-08-17T14:50:09.961771Z","iopub.execute_input":"2022-08-17T14:50:09.962526Z","iopub.status.idle":"2022-08-17T14:50:21.421545Z","shell.execute_reply.started":"2022-08-17T14:50:09.962475Z","shell.execute_reply":"2022-08-17T14:50:21.420353Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mean_df.sort_values(by=['train_priv_diff_rank'], ascending=False)","metadata":{"execution":{"iopub.status.busy":"2022-08-17T14:50:21.423462Z","iopub.execute_input":"2022-08-17T14:50:21.424345Z","iopub.status.idle":"2022-08-17T14:50:21.446591Z","shell.execute_reply.started":"2022-08-17T14:50:21.424295Z","shell.execute_reply":"2022-08-17T14:50:21.445225Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"_ = plt.hist(mean_df['train_priv_diff'])","metadata":{"execution":{"iopub.status.busy":"2022-08-17T14:50:21.451347Z","iopub.execute_input":"2022-08-17T14:50:21.452116Z","iopub.status.idle":"2022-08-17T14:50:21.662061Z","shell.execute_reply.started":"2022-08-17T14:50:21.452059Z","shell.execute_reply":"2022-08-17T14:50:21.660296Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"There are just a handful of columns with a shift of the mean of ~5% of the range of the column \n\nThe highest being B_39 with ~6% shift in the mean","metadata":{}},{"cell_type":"code","source":"del mean_df, meantrain_df, meanpriv_df, rangetrain_df\n_ = gc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-08-17T14:50:21.663501Z","iopub.execute_input":"2022-08-17T14:50:21.663893Z","iopub.status.idle":"2022-08-17T14:50:21.808386Z","shell.execute_reply.started":"2022-08-17T14:50:21.663858Z","shell.execute_reply":"2022-08-17T14:50:21.806725Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Compare relative standard deviation","metadata":{}},{"cell_type":"code","source":"stdtrain_df = np.std(train).to_frame()\nstdtrain_df.columns = ['train']\n\nstdpriv_df = np.std(test_private).to_frame()\nstdpriv_df.columns = ['test_priv']\n\nmeantrain_df = np.mean(train).to_frame()\nmeantrain_df.columns = ['train_mean']\n\nmeantest_df = np.mean(test_private).to_frame()\nmeantest_df.columns = ['test_mean']\n\nstd_df = pd.concat([stdtrain_df, stdpriv_df, meantrain_df, meantest_df], axis=1)\n\nstd_df['train'] = std_df['train'] / std_df['train_mean']\nstd_df['test_priv'] = std_df['test_priv'] / std_df['test_mean']\n\ndel std_df['train_mean'], std_df['test_mean']\n\nstd_df['train_priv_diff'] = np.abs(std_df['train'] - std_df['test_priv'])\nstd_df['train_priv_diff_rank'] = std_df['train_priv_diff'].rank(method='min').values \n\nstd_df.shape","metadata":{"execution":{"iopub.status.busy":"2022-08-17T14:50:21.809905Z","iopub.execute_input":"2022-08-17T14:50:21.810266Z","iopub.status.idle":"2022-08-17T14:50:57.859820Z","shell.execute_reply.started":"2022-08-17T14:50:21.810232Z","shell.execute_reply":"2022-08-17T14:50:57.858082Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"std_df.sort_values(by=['train_priv_diff_rank'], ascending=False)","metadata":{"execution":{"iopub.status.busy":"2022-08-17T14:50:57.861526Z","iopub.execute_input":"2022-08-17T14:50:57.861939Z","iopub.status.idle":"2022-08-17T14:50:57.882570Z","shell.execute_reply.started":"2022-08-17T14:50:57.861899Z","shell.execute_reply":"2022-08-17T14:50:57.880947Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"_ = plt.hist(std_df['train_priv_diff'])","metadata":{"execution":{"iopub.status.busy":"2022-08-17T14:50:57.884444Z","iopub.execute_input":"2022-08-17T14:50:57.884875Z","iopub.status.idle":"2022-08-17T14:50:58.082606Z","shell.execute_reply.started":"2022-08-17T14:50:57.884827Z","shell.execute_reply":"2022-08-17T14:50:58.081307Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"It looks like some variables do have a higher rate of volatility compared to their distribution in the train set. D_83 being the standout, with D_69 and R_23 also showing significant difference","metadata":{}}]}