{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# About this notebook\n\nThis is a Prediction competition. So differently from Code competitions, we have access to all test data and it is important to analyse and spot drifts in variables behaviours and distributions in public and private sets.\n\nThis has been discussed on many topics in the forum and here I'd like to provide a simple code snippet to plot these two subsets.\n\nMany thanks to [@raddar](https://www.kaggle.com/raddar) for publishing such a great [dataset](https://www.kaggle.com/datasets/raddar/amex-data-integer-dtypes-parquet-format).","metadata":{}},{"cell_type":"markdown","source":"# Imports/Read","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom tqdm import tqdm","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-06-15T20:45:16.397904Z","iopub.execute_input":"2022-06-15T20:45:16.398341Z","iopub.status.idle":"2022-06-15T20:45:17.047175Z","shell.execute_reply.started":"2022-06-15T20:45:16.39825Z","shell.execute_reply":"2022-06-15T20:45:17.045979Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.read_parquet('../input/amex-data-integer-dtypes-parquet-format/test.parquet')\ndf['S_2'] = pd.to_datetime(df['S_2']).astype('datetime64[ns]')\n\nprint(df.shape)","metadata":{"execution":{"iopub.status.busy":"2022-06-15T20:45:17.048918Z","iopub.execute_input":"2022-06-15T20:45:17.049938Z","iopub.status.idle":"2022-06-15T20:46:06.799865Z","shell.execute_reply.started":"2022-06-15T20:45:17.049899Z","shell.execute_reply":"2022-06-15T20:46:06.798217Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = df.groupby('customer_ID').tail(1).reset_index(drop=True)\nprint(df.shape)\ndisplay(df.head())","metadata":{"execution":{"iopub.status.busy":"2022-06-15T20:46:06.801887Z","iopub.execute_input":"2022-06-15T20:46:06.80244Z","iopub.status.idle":"2022-06-15T20:46:11.706399Z","shell.execute_reply.started":"2022-06-15T20:46:06.802388Z","shell.execute_reply":"2022-06-15T20:46:11.705173Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Dates","metadata":{}},{"cell_type":"code","source":"df.S_2.hist(figsize=(12,6));","metadata":{"execution":{"iopub.status.busy":"2022-06-15T20:46:11.709462Z","iopub.execute_input":"2022-06-15T20:46:11.709921Z","iopub.status.idle":"2022-06-15T20:46:12.4664Z","shell.execute_reply.started":"2022-06-15T20:46:11.709877Z","shell.execute_reply":"2022-06-15T20:46:12.46549Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Split Public x Private","metadata":{}},{"cell_type":"code","source":"public = df[df['S_2'] < '2019-07-01'].reset_index()\npublic = public.drop('S_2', axis=1)\nprint(public.shape)\n\nprivate = df[df['S_2'] > '2019-07-01'].reset_index()\nprivate = private.drop('S_2', axis=1)\nprint(private.shape)","metadata":{"execution":{"iopub.status.busy":"2022-06-15T20:46:12.467587Z","iopub.execute_input":"2022-06-15T20:46:12.468679Z","iopub.status.idle":"2022-06-15T20:46:13.374116Z","shell.execute_reply.started":"2022-06-15T20:46:12.468625Z","shell.execute_reply":"2022-06-15T20:46:13.373158Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Histograms","metadata":{}},{"cell_type":"code","source":"cols = [col for col in df.columns if col not in ['customer_ID', 'S_2']]\nlen(cols)","metadata":{"execution":{"iopub.status.busy":"2022-06-15T20:46:13.375399Z","iopub.execute_input":"2022-06-15T20:46:13.375723Z","iopub.status.idle":"2022-06-15T20:46:13.383383Z","shell.execute_reply.started":"2022-06-15T20:46:13.375686Z","shell.execute_reply":"2022-06-15T20:46:13.382455Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"begin = 100\nend = 120\n\nn_vars_to_plot = len(cols[begin:end])\nrow = 0\n\nfig, axes = plt.subplots(n_vars_to_plot, 2, figsize = (14, n_vars_to_plot * 3))\n\nfor var in tqdm(cols[begin:end]):\n\n  axes[row, 0].set_title(f'Public: {var}', color='blue')\n  sns.histplot(data=public, x=var, ax=axes[row, 0], bins = 20)\n\n  axes[row, 1].set_title(f'Private: {var}', color='red')\n  sns.histplot(data=private, x=var, ax=axes[row, 1], bins = 20)\n\n  row = row + 1\n\nplt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-06-15T20:46:13.384918Z","iopub.execute_input":"2022-06-15T20:46:13.385357Z","iopub.status.idle":"2022-06-15T20:51:15.271629Z","shell.execute_reply.started":"2022-06-15T20:46:13.385317Z","shell.execute_reply":"2022-06-15T20:51:15.270273Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Categorical features","metadata":{}},{"cell_type":"code","source":"cat_cols = ['B_30', 'B_38', 'D_114', 'D_116', 'D_117', 'D_120', 'D_126', 'D_63', 'D_64', 'D_66', 'D_68']\nlen(cat_cols)","metadata":{"execution":{"iopub.status.busy":"2022-06-15T20:51:15.273769Z","iopub.execute_input":"2022-06-15T20:51:15.274444Z","iopub.status.idle":"2022-06-15T20:51:15.282701Z","shell.execute_reply.started":"2022-06-15T20:51:15.274401Z","shell.execute_reply":"2022-06-15T20:51:15.281783Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"begin = 0\nend = len(cat_cols)\n\nn_vars_to_plot = len(cat_cols[begin:end])\nrow = 0\n\nfig, axes = plt.subplots(n_vars_to_plot, 2, figsize = (14, n_vars_to_plot * 3))\n\nfor var in tqdm(cat_cols[begin:end]):\n\n  axes[row, 0].set_title(f'Public: {var}', color='blue')\n  sns.histplot(data=public, x=var, ax=axes[row, 0], bins = 20)\n\n  axes[row, 1].set_title(f'Private: {var}', color='red')\n  sns.histplot(data=private, x=var, ax=axes[row, 1], bins = 20)\n\n  row = row + 1\n\nplt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-06-15T20:51:15.285741Z","iopub.execute_input":"2022-06-15T20:51:15.286353Z","iopub.status.idle":"2022-06-15T20:51:28.856647Z","shell.execute_reply.started":"2022-06-15T20:51:15.286309Z","shell.execute_reply":"2022-06-15T20:51:28.855822Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# NaN","metadata":{}},{"cell_type":"code","source":"na = pd.DataFrame({'features' : cols})\n\nfor i in range(len(na.features)):\n    var_name = na.loc[i, 'features']\n    na.loc[i, 'public'] = 100 * public[var_name].isna().sum()/len(public)\n    na.loc[i, 'private'] = 100 * private[var_name].isna().sum()/len(private)\n    \nna = na.melt(id_vars = ['features'])\nna    ","metadata":{"execution":{"iopub.status.busy":"2022-06-15T20:51:28.857655Z","iopub.execute_input":"2022-06-15T20:51:28.858522Z","iopub.status.idle":"2022-06-15T20:51:29.307278Z","shell.execute_reply.started":"2022-06-15T20:51:28.858483Z","shell.execute_reply":"2022-06-15T20:51:29.30613Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"begin = 100\nend = 120\n\nn_vars_to_plot = len(cols[begin:end])\n\nfig, axes = plt.subplots(n_vars_to_plot, 1, figsize = (14, n_vars_to_plot * 3))\n\nrow = 0\n\nfor i in tqdm(cols[begin:end]):\n    \n  s = na[na.features == i]\n    \n  axes[row].set_title(i, color='blue')\n  sns.pointplot(data = s, x='variable', y='value', ax=axes[row])\n  axes[row].set_xlabel('')\n  axes[row].set(ylim = (-1,101))\n  axes[row].set_ylabel('Percentage of NaN')\n\n  row = row + 1\n\n\nplt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-06-15T20:51:29.308907Z","iopub.execute_input":"2022-06-15T20:51:29.309395Z","iopub.status.idle":"2022-06-15T20:51:31.635154Z","shell.execute_reply.started":"2022-06-15T20:51:29.309359Z","shell.execute_reply":"2022-06-15T20:51:31.634075Z"},"trusted":true},"execution_count":null,"outputs":[]}]}