{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import tqdm\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\n%matplotlib inline\nimport seaborn as sns\nsns.set()\nfrom sklearn.impute import SimpleImputer","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"033ec4aa101657d247412e8f4b4ee279a1a6490e"},"cell_type":"markdown","source":"### Loading datasets"},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"train = pd.read_csv(\"../input/training_set_metadata.csv\")\ntest = pd.read_csv(\"../input/test_set_metadata.csv\")\nsub_sample = pd.read_csv(\"../input/sample_submission.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"de9697115449193af05b395dbb2bebb49b27a506"},"cell_type":"code","source":"print(train.shape)\ntrain.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4c6f9e40dee6daffd7233e07a38c7b60ec352aac"},"cell_type":"code","source":"print(test.shape)\ntest.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"29f9cb342ae3485f999e9b6d2e34286e78e6c9d0"},"cell_type":"code","source":"sub_sample.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"f7efe7f0bce578e011fb4850fda89a16c026a550"},"cell_type":"markdown","source":"__It seems, submission format requires One Hot Encoded `target` class with `object_id`__."},{"metadata":{"trusted":true,"_uuid":"defdd2ce0e7bc17eb62ea5d7800e683210d77769"},"cell_type":"code","source":"# but class 99 missing here\nsorted(train.target.unique())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"31fcd70c88c799caa66d71ceeb0fbd784b8dff2a"},"cell_type":"code","source":"# checking null/missing values in train dataset\ntrain.isnull().sum()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"49619093dc7b320d22267b7281e79874feb711a9"},"cell_type":"markdown","source":"__`distmod` feature contains `NaN` values__."},{"metadata":{"trusted":true,"_uuid":"f178806fe84c34dc85b97b0f890dfa7b60d82fa2"},"cell_type":"code","source":"# impute missing values by replacing them with mean values of that feature\nimp = SimpleImputer(missing_values=np.nan, strategy='mean')\ntrain[\"distmod\"] = imp.fit_transform(np.array(train[\"distmod\"]).reshape(-1,1))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d489dcabcc662bea7b0345fc58af5e8798ad2c25"},"cell_type":"code","source":"train.isnull().sum().any()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"575c9b989967b79403e1b999d5078308e4072186"},"cell_type":"code","source":"# checking null values in test dataset\ntest.isnull().sum()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"87246f924c50e8b587228306cdb34f81e7cc5afa"},"cell_type":"markdown","source":"__`hostgal_specz` and `distmod`  feature contains `NaN` values__."},{"metadata":{"trusted":true,"_uuid":"d7ae99d06b5ab6d34ee1edc991638421dbc9565b"},"cell_type":"code","source":"# different approach to deal with null values\ntest.distmod.fillna(test[\"distmod\"].mean(), inplace=True)\ntest.hostgal_specz.fillna(test.hostgal_specz.mean(), inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"642302ffdc1089655de3cd7fe6ed64382df83a35"},"cell_type":"code","source":"test.isnull().sum().any()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b79722b1a5d0869ee127263573cf49b24644d0e4"},"cell_type":"code","source":"# checking correlation between features\nplt.figure(figsize=(12,9))\nsns.heatmap(train.corr(), annot=True, fmt=\".1f\", cmap=\"RdYlBu\")","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"113e43298dcd9e8952c25ba422fba68704a7bb4f"},"cell_type":"markdown","source":"__No highly correlated feature present in the train dataset__."},{"metadata":{"_uuid":"055306f4de19b335e70fb363c140db066bdbb928"},"cell_type":"markdown","source":"### Data Visualization"},{"metadata":{"trusted":true,"_uuid":"da86500783ceed656f1bd6c52b08f30a9b5f2a57"},"cell_type":"code","source":"# lets look at the target distribution visually\nplt.figure(figsize=(15,6))\ntrain.target.value_counts().sort_index().plot.bar()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a4993ac4643f7f5a2481e25623618f40671b18b9"},"cell_type":"code","source":"colors = np.random.rand(train.shape[0])\narea = (25 * np.random.rand(train.shape[0]))**2\n        \nplt.subplots(figsize=(15,6))\nplt.scatter(train.distmod, train.hostgal_specz, s = area, c = colors, alpha = 0.5)\nplt.xlabel(\"distmod\")\nplt.ylabel(\"host galaxy spectroscopic redshift\")\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bfa480bdaecef05fbe1f8f283e298e0f70957015"},"cell_type":"code","source":"colors = np.random.rand(train.shape[0])\narea = (25 * np.random.rand(train.shape[0]))**2\n        \nplt.subplots(figsize=(15,6))\nplt.scatter(train.distmod, train.hostgal_photoz, s = area, c = colors, alpha = 0.5)\nplt.xlabel(\"distmod\")\nplt.ylabel(\"host galaxy photometric redshift\")\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cc677ff29d1c750015b74fa4111d5ea9ea5d903e"},"cell_type":"code","source":"colors = np.random.rand(train.shape[0])\narea = (25 * np.random.rand(train.shape[0]))**2\n        \nplt.subplots(figsize=(15,6))\nplt.scatter(train.distmod, train.hostgal_photoz_err, s = area, c = colors, alpha = 0.5)\nplt.xlabel(\"distmod\")\nplt.ylabel(\"host galaxy photometric redshift error\")\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ed035fcd391b8407f7529f80e6957cc1716626f0"},"cell_type":"code","source":"fig, ax = plt.subplots(figsize=(18,8))\n\nfor target_class in train.target.unique():\n    used_class = train[train.target == target_class]\n    \n    colors = np.random.rand(len(used_class))\n    area = (25 * np.random.rand(len(used_class)))**2\n    \n    ax.scatter(x = used_class.gal_l, y = used_class.gal_b, alpha = 0.5, s = area, c = colors)\n\nplt.xlabel(\"galactical longitude\")\nplt.ylabel(\"galactical latitude\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4a14594a9ff5601f7d88836526b92f3796ffcdd5"},"cell_type":"code","source":"# adding 99 target class manually\ntargets = np.hstack([np.unique(train['target']), [99]])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9b207a0e525e0fd1f1658a2039ed369604cd667c"},"cell_type":"code","source":"targets","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d5528f215a5f4d99f418c47ac64037e765d5a967"},"cell_type":"code","source":"target_map = {j : i for i, j in enumerate(targets)}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8575ab01d025d59fde0a09ae07e41affca4f9209"},"cell_type":"code","source":"target_map","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6f94f68c5e652b3b8fa65245cca1673839aa5c5a"},"cell_type":"code","source":"target_ids = [target_map[i] for i in train['target']]\ntrain[\"target_id\"] = target_ids # adding a new feature to the train dataset","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"878e67f163e7749ce57dce375b9fb55d31300371"},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a2b5fd5148e4352a4fafc74c41a474e5280dc42e"},"cell_type":"markdown","source":"__Below code taken from this [kernel](https://www.kaggle.com/kyleboone/naive-benchmark-galactic-vs-extragalactic)__."},{"metadata":{"trusted":true,"_uuid":"d4d13b03780c457773f45f263986420ecbb04c9a"},"cell_type":"code","source":"# Build the flat probability arrays for both the galactic and extragalactic groups\ngalactic_cut = train['hostgal_specz'] == 0","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"870ab553361c89215fe88829aed44f530888716d"},"cell_type":"code","source":"print(galactic_cut[:5])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"56a55b2af5b66ed3aa5ad1aa501368d66a7e1497"},"cell_type":"code","source":"galactic_data = train[galactic_cut]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ea6fb8d7b3938821fa961ab260d5f414e38e091c"},"cell_type":"code","source":"galactic_data.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cfae91d101b98b0c890dd531f3d17eaa7aa3f0bc"},"cell_type":"code","source":"extragalactic_data = train[~galactic_cut]\ngalactic_classes = np.unique(galactic_data['target_id'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0a5e59729ae407fbc9cfda7edd8a30753a2726b8"},"cell_type":"code","source":"extragalactic_data.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3c327ab752c4569b7259665384d2fef435f80897"},"cell_type":"code","source":"galactic_classes","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"334a2891ddef9f0a5f111906eaf1399cd62c1488"},"cell_type":"code","source":"extragalactic_classes = np.unique(extragalactic_data['target_id'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b149bcb3dfde5c82512398f817dc127411803cc7"},"cell_type":"code","source":"extragalactic_classes","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"593da33bfc52be41d9d85dfe9a073ce2851fbad2"},"cell_type":"code","source":"# Add class 99 (id=14) to both groups.\ngalactic_classes = np.append(galactic_classes, 14)\nextragalactic_classes = np.append(extragalactic_classes, 14)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5253f7f53fc4ca76c181feef6ae23467499ae6e4"},"cell_type":"code","source":"galactic_probabilities = np.zeros(15)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2969352b88c85455f163eb8de149ad41692971c1"},"cell_type":"code","source":"galactic_probabilities","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b05e4e0cccba5007637ab1b92543f6e6516624d3"},"cell_type":"code","source":"galactic_probabilities[galactic_classes] = 1. / len(galactic_classes)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d4819b275f57f09b3ec54da51238af3ca0a7699e"},"cell_type":"code","source":"galactic_probabilities[galactic_classes]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d9196ecf286f3bc9aa561ab497758621dc82a8d7"},"cell_type":"code","source":"extragalactic_probabilities = np.zeros(15)\nextragalactic_probabilities[extragalactic_classes] = 1. / len(extragalactic_classes)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ed393ad41c226602d39aedad708658e232f8a43b"},"cell_type":"code","source":"extragalactic_probabilities[extragalactic_classes]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0f42f7cba4efd8e6f8da5f2cff40537456a5964b"},"cell_type":"code","source":"# Apply this prediction to a table\ndef do_prediction(table):\n    probs = []\n    for index, row in tqdm.tqdm(table.iterrows(), total=len(table)):\n        if row['hostgal_photoz'] == 0:\n            prob = galactic_probabilities\n        else:\n            prob = extragalactic_probabilities\n        probs.append(prob)\n    return np.array(probs)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f0a6b2541b1c7282beaaa3fcd9b721ec0d793229"},"cell_type":"code","source":"pred = do_prediction(train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0a6abc856952783e58718194b729d084b93aa67f"},"cell_type":"code","source":"pred","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"59eceac856e8cb70f3e1b5de5f2d17714e0f257f"},"cell_type":"code","source":"test_pred = do_prediction(test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c098e707c5ebe614be1bc34ab676f9b1f53c16c7"},"cell_type":"code","source":"col_names = ['class_%d' % i for i in targets]\nsubmission_df = pd.DataFrame(data = test_pred, columns = col_names)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fe63cc959bae67900faadd8d49902db1fb50433f"},"cell_type":"code","source":"submission_df.insert(0, \"object_id\", test[\"object_id\"].values)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0abdacc502e1553b52e34a510132b91f4c91104f"},"cell_type":"code","source":"submission_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6a3cc5a311b443a37ea2870727e9fe7542fc90aa"},"cell_type":"code","source":"submission_df.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"dc4bde5276a42d3e409b68b4839bec87c69349e2"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}