{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport os\n\nimport matplotlib.pyplot as plt \nimport seaborn as sns\n\npd.options.display.max_columns = 1000\npd.options.display.max_rows = 1000\n\n\npath_submissions = '/'\ntarget_name = 'target'\nscores_folds = {}\n\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n        \nQuickRun = True #this is meant for running experminets before the final run","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-08-08T09:59:14.837820Z","iopub.execute_input":"2022-08-08T09:59:14.838643Z","iopub.status.idle":"2022-08-08T09:59:15.971944Z","shell.execute_reply.started":"2022-08-08T09:59:14.838543Z","shell.execute_reply":"2022-08-08T09:59:15.970721Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### The dataset is used for this competition is synthetic, but based on a real dataset and generated using a CTGAN. The original dataset deals with predicting identifying spam emails via various extracted features from the email. Although the features are anonymized, they have properties relating to real-world features.\n\nThe goal of these competitions is to provide a fun, and approachable for anyone, tabular dataset. These competitions will be great for people looking for something in between the Titanic Getting Started competition and a Featured competition. If you're an established competitions master or grandmaster, these probably won't be much of a challenge for you. We encourage you to avoid saturating the leaderboard.","metadata":{}},{"cell_type":"code","source":"nrows = 10000 if QuickRun else None\n\ntrain=pd.read_csv(\"/kaggle/input/tabular-playground-series-nov-2021/train.csv\", nrows = nrows)\ntest=pd.read_csv(\"/kaggle/input/tabular-playground-series-nov-2021/test.csv\", nrows = nrows)","metadata":{"execution":{"iopub.status.busy":"2022-08-08T09:59:15.974296Z","iopub.execute_input":"2022-08-08T09:59:15.974664Z","iopub.status.idle":"2022-08-08T09:59:16.581078Z","shell.execute_reply.started":"2022-08-08T09:59:15.974631Z","shell.execute_reply":"2022-08-08T09:59:16.580229Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.describe()","metadata":{"execution":{"iopub.status.busy":"2022-08-08T09:59:16.582260Z","iopub.execute_input":"2022-08-08T09:59:16.582744Z","iopub.status.idle":"2022-08-08T09:59:16.935371Z","shell.execute_reply.started":"2022-08-08T09:59:16.582712Z","shell.execute_reply":"2022-08-08T09:59:16.934161Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.isnull().sum().sum()","metadata":{"execution":{"iopub.status.busy":"2022-08-08T09:59:16.938332Z","iopub.execute_input":"2022-08-08T09:59:16.939035Z","iopub.status.idle":"2022-08-08T09:59:16.949819Z","shell.execute_reply.started":"2022-08-08T09:59:16.938989Z","shell.execute_reply":"2022-08-08T09:59:16.949043Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"colNames=[col for col in test.columns if col not in 'id']","metadata":{"execution":{"iopub.status.busy":"2022-08-08T09:59:16.951097Z","iopub.execute_input":"2022-08-08T09:59:16.951420Z","iopub.status.idle":"2022-08-08T09:59:16.956516Z","shell.execute_reply.started":"2022-08-08T09:59:16.951391Z","shell.execute_reply":"2022-08-08T09:59:16.955496Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install hdbscan","metadata":{"execution":{"iopub.status.busy":"2022-08-08T09:59:16.957722Z","iopub.execute_input":"2022-08-08T09:59:16.958003Z","iopub.status.idle":"2022-08-08T10:01:01.412933Z","shell.execute_reply.started":"2022-08-08T09:59:16.957975Z","shell.execute_reply":"2022-08-08T10:01:01.411517Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\nfrom sklearn.preprocessing import StandardScaler\nimport hdbscan\nimport umap\n\nscaler = StandardScaler()\n\nX = scaler.fit_transform(train[colNames])\n\nreducer = umap.UMAP(random_state=42, n_components=2)\nembedding = reducer.fit_transform(X)\n\nclusterer = hdbscan.HDBSCAN(prediction_data=True, min_cluster_size = 250).fit(embedding)\n\nu, counts = np.unique(clusterer.labels_, return_counts=True)\n\nprint(u)\nprint(counts)","metadata":{"execution":{"iopub.status.busy":"2022-08-08T10:01:01.414977Z","iopub.execute_input":"2022-08-08T10:01:01.415495Z","iopub.status.idle":"2022-08-08T10:02:04.505870Z","shell.execute_reply.started":"2022-08-08T10:01:01.415446Z","shell.execute_reply":"2022-08-08T10:02:04.504570Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n\nplt.figure(figsize=(10, 8))\nplt.scatter(embedding[:, 0], embedding[:, 1], s=5, c=clusterer.labels_, edgecolors='none', cmap='jet');\n\n","metadata":{"execution":{"iopub.status.busy":"2022-08-08T10:02:04.509579Z","iopub.execute_input":"2022-08-08T10:02:04.510362Z","iopub.status.idle":"2022-08-08T10:02:04.798349Z","shell.execute_reply.started":"2022-08-08T10:02:04.510313Z","shell.execute_reply":"2022-08-08T10:02:04.797185Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n\nplt.figure(figsize=(10, 8))\nplt.scatter(embedding[:, 0], embedding[:, 1], s=5, c=train.target, edgecolors='none', cmap='jet');\n\n","metadata":{"execution":{"iopub.status.busy":"2022-08-08T10:02:04.800162Z","iopub.execute_input":"2022-08-08T10:02:04.800816Z","iopub.status.idle":"2022-08-08T10:02:05.063222Z","shell.execute_reply.started":"2022-08-08T10:02:04.800771Z","shell.execute_reply":"2022-08-08T10:02:05.062117Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.preprocessing import RobustScaler\n\nscaler = RobustScaler()\n\nX = scaler.fit_transform(train[colNames])\n\nreducer = umap.UMAP(random_state=42, n_components=2)\nembedding = reducer.fit_transform(X)\nclusterer = hdbscan.HDBSCAN(prediction_data=True, min_cluster_size = 10).fit(embedding)\n\nu, counts = np.unique(clusterer.labels_, return_counts=True)\n\nprint(u)\nprint(counts)","metadata":{"execution":{"iopub.status.busy":"2022-08-08T10:02:05.067308Z","iopub.execute_input":"2022-08-08T10:02:05.067715Z","iopub.status.idle":"2022-08-08T10:02:27.682437Z","shell.execute_reply.started":"2022-08-08T10:02:05.067681Z","shell.execute_reply":"2022-08-08T10:02:27.681157Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(10, 8))\nplt.scatter(embedding[:, 0], embedding[:, 1], s=5, c=clusterer.labels_, edgecolors='none', cmap='jet');","metadata":{"execution":{"iopub.status.busy":"2022-08-08T10:02:27.683925Z","iopub.execute_input":"2022-08-08T10:02:27.684403Z","iopub.status.idle":"2022-08-08T10:02:27.944759Z","shell.execute_reply.started":"2022-08-08T10:02:27.684369Z","shell.execute_reply":"2022-08-08T10:02:27.943528Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n\nplt.figure(figsize=(10, 8))\nplt.scatter(embedding[:, 0], embedding[:, 1], s=5, c=train.target, edgecolors='none', cmap='jet');\n\n","metadata":{"execution":{"iopub.status.busy":"2022-08-08T10:02:27.946752Z","iopub.execute_input":"2022-08-08T10:02:27.947242Z","iopub.status.idle":"2022-08-08T10:02:28.210302Z","shell.execute_reply.started":"2022-08-08T10:02:27.947187Z","shell.execute_reply":"2022-08-08T10:02:28.209083Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"table_target = pd.crosstab(clusterer.labels_,train.target)","metadata":{"execution":{"iopub.status.busy":"2022-08-08T10:02:28.212053Z","iopub.execute_input":"2022-08-08T10:02:28.212401Z","iopub.status.idle":"2022-08-08T10:02:28.236354Z","shell.execute_reply.started":"2022-08-08T10:02:28.212370Z","shell.execute_reply":"2022-08-08T10:02:28.235503Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"proba = table_target.iloc[:,1] / table_target.sum(axis=1)","metadata":{"execution":{"iopub.status.busy":"2022-08-08T10:02:28.237557Z","iopub.execute_input":"2022-08-08T10:02:28.238369Z","iopub.status.idle":"2022-08-08T10:02:28.244146Z","shell.execute_reply.started":"2022-08-08T10:02:28.238315Z","shell.execute_reply":"2022-08-08T10:02:28.243297Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(10, 8))\nplt.scatter(embedding[:, 0], embedding[:, 1], s=5, c=[proba[i] for i in clusterer.labels_], edgecolors='none', cmap='jet',vmin=0.25,vmax=0.75);","metadata":{"execution":{"iopub.status.busy":"2022-08-08T10:02:28.245185Z","iopub.execute_input":"2022-08-08T10:02:28.246535Z","iopub.status.idle":"2022-08-08T10:02:28.569333Z","shell.execute_reply.started":"2022-08-08T10:02:28.246499Z","shell.execute_reply":"2022-08-08T10:02:28.568210Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n\nXtest = scaler.transform(test[colNames])\nembedding_test = reducer.transform(Xtest)\ntest_labels, strengths = hdbscan.approximate_predict(clusterer, embedding_test)\ntest_proba = [proba[i] for i in test_labels]\n\n","metadata":{"execution":{"iopub.status.busy":"2022-08-08T10:02:28.570804Z","iopub.execute_input":"2022-08-08T10:02:28.571219Z","iopub.status.idle":"2022-08-08T10:03:03.553808Z","shell.execute_reply.started":"2022-08-08T10:02:28.571178Z","shell.execute_reply":"2022-08-08T10:03:03.552360Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n\nplt.figure(figsize=(10, 8))\nplt.scatter(embedding_test[:, 0], embedding_test[:, 1], s=5, c=test_labels, edgecolors='none', cmap='jet');\n\n","metadata":{"execution":{"iopub.status.busy":"2022-08-08T10:03:03.555543Z","iopub.execute_input":"2022-08-08T10:03:03.555891Z","iopub.status.idle":"2022-08-08T10:03:03.814564Z","shell.execute_reply.started":"2022-08-08T10:03:03.555856Z","shell.execute_reply":"2022-08-08T10:03:03.813384Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n\nsub=pd.read_csv(\"../input/tabular-playground-series-nov-2021/sample_submission.csv\",nrows = nrows)\nsub['target']=test_proba\nsub.to_csv(\"submission.csv\",index=False)\n\n","metadata":{"execution":{"iopub.status.busy":"2022-08-08T10:03:03.815723Z","iopub.execute_input":"2022-08-08T10:03:03.816030Z","iopub.status.idle":"2022-08-08T10:03:03.861070Z","shell.execute_reply.started":"2022-08-08T10:03:03.816002Z","shell.execute_reply":"2022-08-08T10:03:03.860078Z"},"trusted":true},"execution_count":null,"outputs":[]}]}