{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"markdown","source":"**TSNE analisys**"},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","collapsed":true,"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":false},"cell_type":"markdown","source":"1. import train and test df with calculated features. You can try your data\n2. preprocess data\n3. calculate TSNE (30-40 mins)\n4. visualize outliers and targets\n\nlater:\nsmart imputer (not mean but train from dataset)\ntsne import pretrained 2D,3D\ncalculate knn for outliers\nmerchants.csv additional features\nmultitask - classification+regression in one model"},{"metadata":{"trusted":true,"_uuid":"57a1a219f1f1889e2b7464eb80cd1496f35f8b30"},"cell_type":"code","source":"!pip install MulticoreTSNE","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"651ee69a94d709ab349cacddbb416eaabafd320a"},"cell_type":"code","source":"from MulticoreTSNE import MulticoreTSNE as TSNE\nfrom matplotlib import pyplot as plt\nplt.style.use(\"fivethirtyeight\")\n%matplotlib inline\nimport pandas as pd\nimport numpy as np","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0e21ca0ac9ecad6162a6304ad14c20a5a4184a99"},"cell_type":"code","source":"import os\nos.listdir('../input/')\npath='../input/features-for-elo-merchants-competition/'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"48b3796f9f626e59c02c3f66ad73caafea6ef673"},"cell_type":"code","source":"#import data:\ntrain_df = pd.read_csv(path+'full_train_df.csv', index_col='card_id')\ntest_df = pd.read_csv(path+'full_test_df.csv', index_col='card_id')\ndf = train_df.append(test_df)\ntrain_len = train_df.shape[0]\n\nFEATS_EXCLUDED = ['first_active_month', 'target', 'card_id', 'outliers',\n                  'hist_purchase_date_max', 'hist_purchase_date_min', 'hist_card_id_size',\n                  'new_purchase_date_max', 'new_purchase_date_min', 'new_card_id_size',\n                  'OOF_PRED', 'month_0']\ncols = [f for f in train_df.columns if f not in FEATS_EXCLUDED]\n\ndata = df[cols]\noutliers = train_df.outliers\ntarget = train_df.target\n\n#nan mean imputer:\nfrom sklearn.preprocessing import Imputer\n#from sklearn.impute import SimpleImputer\nimp = Imputer(missing_values=np.nan, strategy='mean')\ndata = imp.fit_transform(data)\n#scale:\nfrom sklearn.preprocessing import StandardScaler\nscaler = StandardScaler()\ndata=scaler.fit_transform(data)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"da425d140f37285ee93094e14a62aecba59a0223"},"cell_type":"code","source":"print(train_df.shape, test_df.shape, data.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5a50831fd6ca50005621d9668710fa273b441f35"},"cell_type":"code","source":"#TSNE train\ntsne_model = TSNE(n_jobs=-1, verbose=1, random_state=42)#, n_iter=100) #default n_iter=1000\nembeddings=tsne_model.fit_transform(data)\n#only train Iteration 1000: error is 4.964520\n#full Iteration 1000: error is 5.587587","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6285216eb8745202afa04cfda233364cc5cc1693"},"cell_type":"code","source":"tsne_model.kl_divergence_","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"0eca4cc4a9f012720c7760149671787b4c6467ed"},"cell_type":"markdown","source":"if you want to export:\npd.DataFrame(embeddings).to_csv('full_tsne_2d_embeddings.csv', index=True)"},{"metadata":{"trusted":true,"_uuid":"4bf20c9bf7952bf927becca16723b4538f567ef1"},"cell_type":"code","source":"#TSNE visualization\n\nc = np.array([10 if i>=train_len else outliers[i]*18 for i in range(df.shape[0])])\nprint('color map:', c.shape)\nprint('counts:', pd.value_counts(c))\nprint('embeddings shape:',embeddings.shape)\nvis_x = embeddings[:, 0]\nvis_y = embeddings[:, 1]\n\nplt.figure(1, figsize=(20, 10))\nplt.scatter(vis_x, vis_y, c=c,cmap=plt.cm.get_cmap(\"jet\", 20), marker='.', alpha=0.5)\nplt.colorbar(ticks=range(20))\nplt.clim(1, 20)\nplt.title('full dataset TSNE-2D 18 - outliers, 0 - train, 10 - test')\nplt.tight_layout()\nplt.savefig('full_tsne_2d_embeddings_outliers.png')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"370737d47f10675ec75481719b7aaed1a028c756"},"cell_type":"code","source":"plt.figure(1, figsize=(20, 10))\n\n#c2 = np.array([10 if i>=train_len else target[i]+10 for i in range(df.shape[0])])\n#plt.scatter(vis_x, vis_y, c=target+10,cmap=plt.cm.get_cmap(\"jet\", 20), marker='.')\n\nc2 = np.array([target[i]+8 if target[i]<30 else 18 for i in range(train_len)])\nplt.scatter(vis_x[:train_len], vis_y[:train_len], c=c2[:train_len],cmap=plt.cm.get_cmap(\"jet\", 20), marker='.', alpha=0.5)\nplt.colorbar(ticks=range(20))\nplt.clim(1, 20)\nplt.title('full dataset TSNE-2D target (19 - train)')\nplt.tight_layout()\nplt.savefig('full_tsne_2d_embeddings_targets.png')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"692f831ea5a08e974f492c7f04a282b6bfa8b882"},"cell_type":"markdown","source":""}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}