{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":50160,"databundleVersionId":7602123,"sourceType":"competition"}],"dockerImageVersionId":30648,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"### It's created for;\n* Finding identical features and combine them\n* Finding periodic features and perform feature engineering (new_x = 10_days_x / 30_days_x etc.)","metadata":{}},{"cell_type":"code","source":"!pip install -U -q sentence-transformers\n\nimport pandas as pd\nfrom sentence_transformers import SentenceTransformer\nfrom sklearn.cluster import DBSCAN\nfrom IPython.display import display\n\npd.set_option('display.max_colwidth', None)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-02-09T15:42:50.063425Z","iopub.execute_input":"2024-02-09T15:42:50.064117Z","iopub.status.idle":"2024-02-09T15:42:50.068957Z","shell.execute_reply.started":"2024-02-09T15:42:50.064083Z","shell.execute_reply":"2024-02-09T15:42:50.068011Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Description Table","metadata":{}},{"cell_type":"code","source":"df_info = pd.read_csv(\"/kaggle/input/home-credit-credit-risk-model-stability/feature_definitions.csv\")\n\ndf_info.head()","metadata":{"execution":{"iopub.status.busy":"2024-02-09T15:34:00.858589Z","iopub.execute_input":"2024-02-09T15:34:00.859046Z","iopub.status.idle":"2024-02-09T15:34:00.887026Z","shell.execute_reply.started":"2024-02-09T15:34:00.859019Z","shell.execute_reply":"2024-02-09T15:34:00.885981Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Sentence Transformer Embeddings","metadata":{}},{"cell_type":"code","source":"model = SentenceTransformer('paraphrase-MiniLM-L6-v2')\n\nsentences = df_info[\"Description\"].tolist()\n\nembedding = model.encode(sentences)","metadata":{"execution":{"iopub.status.busy":"2024-02-09T15:34:06.494282Z","iopub.execute_input":"2024-02-09T15:34:06.494983Z","iopub.status.idle":"2024-02-09T15:34:12.429695Z","shell.execute_reply.started":"2024-02-09T15:34:06.494946Z","shell.execute_reply":"2024-02-09T15:34:12.428701Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Sentence count: \", len(sentences))\nprint(\"Embedding dim: \", embedding.shape)","metadata":{"execution":{"iopub.status.busy":"2024-02-09T15:34:57.440225Z","iopub.execute_input":"2024-02-09T15:34:57.440767Z","iopub.status.idle":"2024-02-09T15:34:57.446346Z","shell.execute_reply.started":"2024-02-09T15:34:57.440731Z","shell.execute_reply":"2024-02-09T15:34:57.445420Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Clustering","metadata":{}},{"cell_type":"code","source":"clustering = DBSCAN(eps=3, min_samples=1)\nclustering.fit(embedding)\n\ndf_info[\"Category\"] = clustering.labels_","metadata":{"execution":{"iopub.status.busy":"2024-02-09T15:35:52.367893Z","iopub.execute_input":"2024-02-09T15:35:52.368735Z","iopub.status.idle":"2024-02-09T15:35:52.488296Z","shell.execute_reply.started":"2024-02-09T15:35:52.368693Z","shell.execute_reply":"2024-02-09T15:35:52.486852Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Result","metadata":{}},{"cell_type":"code","source":"for category in df_info[\"Category\"].unique():\n    display(df_info[df_info[\"Category\"].eq(category)])","metadata":{"execution":{"iopub.status.busy":"2024-02-09T15:43:31.024539Z","iopub.execute_input":"2024-02-09T15:43:31.024891Z","iopub.status.idle":"2024-02-09T15:43:32.485019Z","shell.execute_reply.started":"2024-02-09T15:43:31.024864Z","shell.execute_reply":"2024-02-09T15:43:32.484179Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}