{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":50160,"databundleVersionId":7921029,"sourceType":"competition"}],"dockerImageVersionId":30673,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom sklearn.metrics import accuracy_score,auc,roc_curve\nimport scipy.stats as ss\nimport gc\nimport warnings\nwarnings.filterwarnings(\"ignore\")","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-04-04T05:55:11.925190Z","iopub.execute_input":"2024-04-04T05:55:11.927359Z","iopub.status.idle":"2024-04-04T05:55:11.933772Z","shell.execute_reply.started":"2024-04-04T05:55:11.927316Z","shell.execute_reply":"2024-04-04T05:55:11.932759Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# working with static data : EMI Payment related details\n\n# please go though https://www.kaggle.com/code/kunduruanil/credit-risk-data-understanding for more information.","metadata":{}},{"cell_type":"markdown","source":"# https://www.kaggle.com/competitions/home-credit-credit-risk-model-stability/discussion/476449\n# https://www.kaggle.com/competitions/home-credit-credit-risk-model-stability/discussion/482474","metadata":{}},{"cell_type":"code","source":"train_base = pd.read_csv('/kaggle/input/home-credit-credit-risk-model-stability/csv_files/train/train_base.csv')\nprint(\"start date in training data : \",train_base[\"date_decision\"].min(),\"end date in training data : \",train_base[\"date_decision\"].max(),end='\\n\\n')\nprint(train_base.shape)\nprint(\"Sample of Training Base Table below : \")\ntrain_base.head()","metadata":{"execution":{"iopub.status.busy":"2024-04-04T05:55:15.891619Z","iopub.execute_input":"2024-04-04T05:55:15.892088Z","iopub.status.idle":"2024-04-04T05:55:17.403939Z","shell.execute_reply.started":"2024-04-04T05:55:15.892055Z","shell.execute_reply":"2024-04-04T05:55:17.402656Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Static Training data\nts = pd.DataFrame()\nfor i in range(2):\n    ts = pd.concat([ts,pd.read_parquet(f'/kaggle/input/home-credit-credit-risk-model-stability/parquet_files/train/train_static_0_{i}.parquet')],ignore_index=True)\nprint(ts.shape)\nts.head(3)","metadata":{"execution":{"iopub.status.busy":"2024-04-02T12:34:51.851542Z","iopub.execute_input":"2024-04-02T12:34:51.852042Z","iopub.status.idle":"2024-04-02T12:35:00.980551Z","shell.execute_reply.started":"2024-04-02T12:34:51.852002Z","shell.execute_reply":"2024-04-02T12:35:00.979609Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# filling null values","metadata":{}},{"cell_type":"code","source":"selected_columns = ts.select_dtypes(include=\"object\").columns\nts[selected_columns] = ts[selected_columns].fillna(' ')\nselected_columns = ts.select_dtypes(exclude=\"object\").columns\nts[selected_columns] = ts[selected_columns].fillna(0)","metadata":{"execution":{"iopub.status.busy":"2024-04-02T12:35:22.800245Z","iopub.execute_input":"2024-04-02T12:35:22.800654Z","iopub.status.idle":"2024-04-02T12:35:42.323357Z","shell.execute_reply.started":"2024-04-02T12:35:22.800618Z","shell.execute_reply":"2024-04-02T12:35:42.321621Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Anomaly detection :\n\n# 1. Z score.\n# 2. modified z score.\n# 3. extreme value analysis. - block maxima\n# 4. extreme value analysis. - peak over thresold.\n","metadata":{}},{"cell_type":"markdown","source":"# Anomaly detection using z score to check outlayers","metadata":{"execution":{"iopub.status.busy":"2024-03-31T01:28:14.677731Z","iopub.execute_input":"2024-03-31T01:28:14.678253Z","iopub.status.idle":"2024-03-31T01:28:14.683888Z","shell.execute_reply.started":"2024-03-31T01:28:14.678193Z","shell.execute_reply":"2024-03-31T01:28:14.682633Z"}}},{"cell_type":"code","source":"%%time\n# Calculate z-scores for each column\ndef get_z_scores(df):\n    return df.transform(lambda x: (x - x.mean()) / x.std())\nprint(len(selected_columns))\nts_zscores = get_z_scores(ts[selected_columns].iloc[:,1:].copy())  # Apply to a copy to avoid modifying the original DataFrame\nts_zscores.head()","metadata":{"execution":{"iopub.status.busy":"2024-04-01T10:34:16.921474Z","iopub.execute_input":"2024-04-01T10:34:16.922321Z","iopub.status.idle":"2024-04-01T10:34:24.299217Z","shell.execute_reply.started":"2024-04-01T10:34:16.922279Z","shell.execute_reply":"2024-04-01T10:34:24.298094Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ts_zscores[\"case_id\"] = ts['case_id'].copy()\nts_zscores = ts_zscores.merge(train_base[[\"case_id\",\"target\"]],on='case_id')","metadata":{"execution":{"iopub.status.busy":"2024-04-01T10:34:24.300739Z","iopub.execute_input":"2024-04-01T10:34:24.301394Z","iopub.status.idle":"2024-04-01T10:34:25.188152Z","shell.execute_reply.started":"2024-04-01T10:34:24.301353Z","shell.execute_reply":"2024-04-01T10:34:25.186664Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"auc_score = []\nfor col in ts_zscores.columns:\n    if col != 'case_id':\n        thesold = 2\n        fpr,tpr,thresolds = roc_curve(ts_zscores[\"target\"],((ts_zscores[col]>thesold) | (ts_zscores[col]<-thesold)))\n        auc_score.append(auc(fpr,tpr))\nnp.mean(auc_score)","metadata":{"execution":{"iopub.status.busy":"2024-04-01T10:34:25.189627Z","iopub.execute_input":"2024-04-01T10:34:25.189996Z","iopub.status.idle":"2024-04-01T10:34:39.037210Z","shell.execute_reply.started":"2024-04-01T10:34:25.189957Z","shell.execute_reply":"2024-04-01T10:34:39.036213Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.Series(dict(zip(set(ts_zscores.columns)- {\"case_id\"},auc_score))).idxmax()","metadata":{"execution":{"iopub.status.busy":"2024-04-01T10:34:39.038987Z","iopub.execute_input":"2024-04-01T10:34:39.039468Z","iopub.status.idle":"2024-04-01T10:34:39.048622Z","shell.execute_reply.started":"2024-04-01T10:34:39.039431Z","shell.execute_reply":"2024-04-01T10:34:39.047502Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Anomaly detection using modified z score to check outlayers","metadata":{}},{"cell_type":"code","source":"%%time\n# Calculate z-scores for each column\n\ndef modified_zscore(data, consistency_correction=1.4826):\n    \"\"\"\n    Returns the modified z score and Median Absolute Deviation (MAD) from the scores in data.\n    The consistency_correction factor converts the MAD to the standard deviation for a given\n    distribution. The default value (1.4826) is the conversion factor if the underlying data\n    is normally distributed\n    \"\"\"\n    median = np.median(data)\n    \n    deviation_from_med = np.array(data) - median\n\n    mad = np.median(np.abs(deviation_from_med))\n    mod_zscore = deviation_from_med/(consistency_correction*mad)\n    return mod_zscore\n\ndef get_modified_z_scores(df):\n    return df.transform(lambda x: modified_zscore(x))\n\nprint(len(selected_columns))\n\nts_mzscores = get_modified_z_scores(ts[selected_columns].iloc[:,1:].copy())  # Apply to a copy to avoid modifying the original DataFrame\nts_mzscores.head()","metadata":{"execution":{"iopub.status.busy":"2024-04-01T10:46:52.920168Z","iopub.execute_input":"2024-04-01T10:46:52.920597Z","iopub.status.idle":"2024-04-01T10:47:02.903485Z","shell.execute_reply.started":"2024-04-01T10:46:52.920563Z","shell.execute_reply":"2024-04-01T10:47:02.902344Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ts_mzscores[\"case_id\"] = ts['case_id'].copy()\nts_mzscores = ts_mzscores.merge(train_base[[\"case_id\",\"target\"]],on='case_id')","metadata":{"execution":{"iopub.status.busy":"2024-04-01T10:47:06.342692Z","iopub.execute_input":"2024-04-01T10:47:06.343103Z","iopub.status.idle":"2024-04-01T10:47:07.214335Z","shell.execute_reply.started":"2024-04-01T10:47:06.343073Z","shell.execute_reply":"2024-04-01T10:47:07.213255Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mauc_score = []\nfor col in ts_mzscores.columns:\n    if col != 'case_id':\n        thesold = 2\n        fpr,tpr,thresolds = roc_curve(ts_mzscores[\"target\"],((ts_mzscores[col]>thesold) | (ts_mzscores[col]<-thesold)))\n        mauc_score.append(auc(fpr,tpr))\nnp.mean(mauc_score)","metadata":{"execution":{"iopub.status.busy":"2024-04-01T10:47:09.901088Z","iopub.execute_input":"2024-04-01T10:47:09.901508Z","iopub.status.idle":"2024-04-01T10:47:23.825379Z","shell.execute_reply.started":"2024-04-01T10:47:09.901476Z","shell.execute_reply":"2024-04-01T10:47:23.823827Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Anomaly detection using extreme value analysis - Block maxima with 90 % extreme values threshold","metadata":{}},{"cell_type":"code","source":"def block_maxima_theshold(data, block_size):\n    \"\"\"Divides data into blocks and extracts the maximum value from each block.\n\n    Args:\n    data: A NumPy array of data.\n    block_size: The size of each block.\n\n    Returns:\n    A NumPy array of the block maxima.\n    \"\"\"\n\n    # Split the data into blocks.\n    blocks = np.array_split(data, len(data) // block_size)\n\n    # Extract the maximum value from each block.\n    block_maxima = [np.max(block) for block in blocks]\n    \n    # get thresold\n    thershold = ss.genextreme.ppf(0.9,*ss.genextreme.fit(block_maxima))\n    return data>=thershold","metadata":{"execution":{"iopub.status.busy":"2024-04-02T07:32:44.533454Z","iopub.execute_input":"2024-04-02T07:32:44.533896Z","iopub.status.idle":"2024-04-02T07:32:44.542861Z","shell.execute_reply.started":"2024-04-02T07:32:44.533865Z","shell.execute_reply":"2024-04-02T07:32:44.540958Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_block_maxima_thresold(df):\n    return df.transform(lambda x: block_maxima_theshold(x,block_size=100000))\n\nprint(len(selected_columns))\n\nts_bmscores = get_block_maxima_thresold(ts[selected_columns].iloc[:,1:].copy())  # Apply to a copy to avoid modifying the original DataFrame\nts_bmscores.head()","metadata":{"execution":{"iopub.status.busy":"2024-04-02T07:33:21.868511Z","iopub.execute_input":"2024-04-02T07:33:21.868998Z","iopub.status.idle":"2024-04-02T07:33:48.802541Z","shell.execute_reply.started":"2024-04-02T07:33:21.868964Z","shell.execute_reply":"2024-04-02T07:33:48.801353Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ts_bmscores[\"case_id\"] = ts['case_id'].copy()\nts_bmscores = ts_bmscores.merge(train_base[[\"case_id\",\"target\"]],on='case_id')","metadata":{"execution":{"iopub.status.busy":"2024-04-02T07:39:59.982446Z","iopub.execute_input":"2024-04-02T07:39:59.982893Z","iopub.status.idle":"2024-04-02T07:40:00.343385Z","shell.execute_reply.started":"2024-04-02T07:39:59.982859Z","shell.execute_reply":"2024-04-02T07:40:00.341965Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mauc_score = []\nfor col in ts_bmscores.columns:\n    if col != 'case_id':\n        fpr,tpr,thresolds = roc_curve(ts_bmscores[\"target\"],ts_bmscores[col])\n        mauc_score.append(auc(fpr,tpr))\nnp.mean(mauc_score)","metadata":{"execution":{"iopub.status.busy":"2024-04-02T07:40:01.859384Z","iopub.execute_input":"2024-04-02T07:40:01.859849Z","iopub.status.idle":"2024-04-02T07:40:14.708525Z","shell.execute_reply.started":"2024-04-02T07:40:01.859816Z","shell.execute_reply":"2024-04-02T07:40:14.707165Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Anomaly detection using extreme value analysis - Peak over threshold ","metadata":{}},{"cell_type":"code","source":"def block_pot_theshold(data, block_size):\n    \"\"\"Divides data into blocks and extracts the maximum value from each block.\n\n    Args:\n    data: A NumPy array of data.\n    block_size: The size of each block.\n\n    Returns:\n    A NumPy array of the block maxima.\n    \"\"\"\n\n    # Split the data into blocks.\n    blocks = np.array_split(data, len(data) // block_size)\n\n    # Extract the maximum value from each block.\n    block_maxima = [np.max(block) for block in blocks]\n    \n    # get thresold\n    thershold = ss.genextreme.ppf(0.6,*ss.genextreme.fit(block_maxima))\n    thershold = float(thershold)\n    \n    # Peak over threshold\n    if data[data>=thershold].shape[0]>0:\n        fit = ss.expon.fit(data[data>=thershold].astype(\"float\"))\n        thersholdpot = ss.expon.ppf(0.9,*fit)\n        return data>=thersholdpot\n    else:\n        return data>=thershold","metadata":{"execution":{"iopub.status.busy":"2024-04-02T12:41:11.815230Z","iopub.execute_input":"2024-04-02T12:41:11.815775Z","iopub.status.idle":"2024-04-02T12:41:11.830734Z","shell.execute_reply.started":"2024-04-02T12:41:11.815738Z","shell.execute_reply":"2024-04-02T12:41:11.828773Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_block_pot_thresold(df):\n    return df.transform(lambda x: block_pot_theshold(x,block_size=100000))\n\nprint(len(selected_columns))\n\nts_potscores = get_block_pot_thresold(ts[selected_columns].iloc[:,1:].copy())  # Apply to a copy to avoid modifying the original DataFrame\nts_potscores.head()","metadata":{"execution":{"iopub.status.busy":"2024-04-02T12:41:12.831847Z","iopub.execute_input":"2024-04-02T12:41:12.832425Z","iopub.status.idle":"2024-04-02T12:41:40.433413Z","shell.execute_reply.started":"2024-04-02T12:41:12.832387Z","shell.execute_reply":"2024-04-02T12:41:40.432522Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ts_potscores[\"case_id\"] = ts['case_id'].copy()\nts_potscores = ts_potscores.merge(train_base[[\"case_id\",\"target\"]],on='case_id')","metadata":{"execution":{"iopub.status.busy":"2024-04-02T12:44:42.132183Z","iopub.execute_input":"2024-04-02T12:44:42.133320Z","iopub.status.idle":"2024-04-02T12:44:42.704422Z","shell.execute_reply.started":"2024-04-02T12:44:42.133266Z","shell.execute_reply":"2024-04-02T12:44:42.703116Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mauc_score = []\nfor col in ts_potscores.columns:\n    if col != 'case_id':\n        fpr,tpr,thresolds = roc_curve(ts_potscores[\"target\"],ts_potscores[col])\n        mauc_score.append(auc(fpr,tpr))\nnp.mean(mauc_score)","metadata":{"execution":{"iopub.status.busy":"2024-04-02T12:44:44.889986Z","iopub.execute_input":"2024-04-02T12:44:44.890492Z","iopub.status.idle":"2024-04-02T12:44:57.817344Z","shell.execute_reply.started":"2024-04-02T12:44:44.890455Z","shell.execute_reply":"2024-04-02T12:44:57.816009Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Angle based Anomaly Detection \n\n# 1. Choose a point in the data\n# 2.  Calculate all angles that this point makes with other pairs of points in the data\n# 3. Calculate the variance of these angles\n# 4. Anomalies have low variance; normal points have high variance","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}