{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d73640f1810b98c8e686aa75d51f4de19053769b"},"cell_type":"markdown","source":"*CONTENTS*\n\n1. Reducing the memory usage\n2. Dealing with unbalanced data, using relevant evaluated measures\n3. Select the optimal classifier using classifiers' competition"},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport pandas as pa\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n%matplotlib inline","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"9ff867a8595c2ca9fe1e88d897409ff63b08a918"},"cell_type":"markdown","source":"*1. Reducing Memory Usage*\nFirst of all, since we are dealing with almost big data, for convenience we must reduce the memory usage using the code from the link:\nhttps://www.kaggle.com/gemartin/load-data-reduce-memory-usage\n"},{"metadata":{"trusted":true,"_uuid":"d9f64d9bbf8d6c88cf1778911168c533a966c9b5"},"cell_type":"code","source":"def reduce_mem_usage(df):\n    \"\"\" iterate through all the columns of a dataframe and modify the data type\n        to reduce memory usage.        \n    \"\"\"\n    start_mem = df.memory_usage().sum() / 1024**2\n    print('Memory usage of dataframe is {:.2f} MB'.format(start_mem))\n    \n    for col in df.columns:\n        col_type = df[col].dtype\n        \n        if col_type != object:\n            c_min = df[col].min()\n            c_max = df[col].max()\n            if str(col_type)[:3] == 'int':\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    df[col] = df[col].astype(np.int64)  \n            else:\n                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                    df[col] = df[col].astype(np.float16)\n                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    df[col] = df[col].astype(np.float32)\n                else:\n                    df[col] = df[col].astype(np.float64)\n        else:\n            df[col] = df[col].astype('category')\n\n    end_mem = df.memory_usage().sum() / 1024**2\n    print('Memory usage after optimization is: {:.2f} MB'.format(end_mem))\n    print('Decreased by {:.1f}%'.format(100 * (start_mem - end_mem) / start_mem))\n    \n    return df\n\n\ndef import_data(file):\n    \"\"\"create a dataframe and optimize its memory usage\"\"\"\n    df = pd.read_csv(file, parse_dates=True, keep_date_col=True)\n    df = reduce_mem_usage(df)\n    return df\n\nimport matplotlib.pyplot as plt\nimport itertools\ndef plot_confusion_matrix(cm, classes,normalize=True,\n                          title='Confusion matrix',\n                          cmap=plt.cm.Blues):\n    \"\"\"\n    This function prints and plots the confusion matrix.\n    Normalization can be applied by setting `normalize=True`.\n    \"\"\"\n    if normalize:\n        cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]\n        print(\"Normalized confusion matrix\")\n    else:\n        print('Confusion matrix, without normalization')\n\n    print(cm)\n\n    plt.imshow(cm, interpolation='nearest', cmap=cmap)\n    plt.title(title)\n    plt.colorbar()\n    tick_marks = np.arange(len(classes))\n    plt.xticks(tick_marks, classes, rotation=45)\n    plt.yticks(tick_marks, classes)\n\n    fmt = '.2f' if normalize else 'd'\n    thresh = cm.max() / 2.\n    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):\n        plt.text(j, i, format(cm[i, j], fmt),\n                 horizontalalignment=\"center\",\n                 color=\"white\" if cm[i, j] > thresh else \"black\")\n\n    plt.ylabel('True label')\n    plt.xlabel('Predicted label')\n    plt.tight_layout()\n    \n\ntrain = import_data(\"../input/train.csv\")\ntest = import_data(\"../input/test.csv\")\ntrain.head()\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"1b8d472fbca8667643923fc77f7cae56745e6c59"},"cell_type":"markdown","source":"*2. Dealing With Unbalanced Data*\nNow, we want to see how training data divided into classes (A, B, C, D) using pie plot:"},{"metadata":{"trusted":true,"_uuid":"45239a6007610a3bf26ae4a60e60ce16ee077ae5"},"cell_type":"code","source":"\nA = train.loc[train['event'] == 'A']\nd_A = len(A)\nB = train.loc[train['event'] == 'B']\nd_B = len(B)\nC = train.loc[train['event'] == 'C']\nd_C = len(C)\nD = train.loc[train['event'] == 'D']\nd_D = len(D)\nqunt = str(d_A + d_B + d_C + d_D)\nlabels = ['A', 'B', 'C', 'D']\nsizes = [d_A, d_B, d_C, d_D]\ncolors = ['gold', 'yellowgreen', 'lightcoral', 'lightskyblue']\nexplode = (0.1, 0, 0, 0)  \n# Plot\nplt.pie(sizes, explode=explode, labels=labels, colors=colors,\n        autopct='%1.1f%%', shadow=True, startangle=140)\nplt.title('Pie (' + qunt + ')')\n\nplt.axis('equal')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"1a4ffece98bb7850a265a6e5ed885ccaa92e0a79"},"cell_type":"markdown","source":"As we can see, from the pie figure above, the training data is unbalanced.\nTo deal with unbalanced data we must use two evaluated measure:\n1. Macro recall, see link: https://datascience.stackexchange.com/questions/15989/micro-average-vs-macro-average-performance-in-a-multiclass-classification-settin\n\n2. Log loss (using predict probablity)"},{"metadata":{"trusted":true,"_uuid":"dddf70304ddafae97fd34bbf35c77b859df2b6f3"},"cell_type":"code","source":"train.describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e865846a58e8f22bc37806e3d33753adfab6cdc4"},"cell_type":"code","source":"\ntest_id = test['id']\ntest.drop(['id'], axis=1, inplace=True)\n\nimport lightgbm as lgb\ndic = {'A': 0, 'B': 1, 'C': 2, 'D': 3}\ndic1 = {'CA':0,'DA':1,'SS':3,'LOFT':4}\ntrain[\"event\"] = train[\"event\"].apply(lambda x: dic[x])\ntrain[\"event\"] = train[\"event\"].astype('int8')\ntrain['experiment'] = train['experiment'].apply(lambda x: dic1[x])\ntest['experiment'] = test['experiment'].apply(lambda x: dic1[x])\n\ntrain['experiment'] = train['experiment'].astype('int8')\ntest['experiment'] = test['experiment'].astype('int8')\n\n#train.info()\ny = train['event']\ntrain.drop(['event'], axis=1, inplace=True)\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c40f4d9d542ab00cfcb7e8b60c253d2a5a2dd3ca"},"cell_type":"markdown","source":"*3. classifiers' competition*"},{"metadata":{"trusted":true,"_uuid":"d0cba88909a83fafb5d1f4280c00f44af4689f96"},"cell_type":"code","source":"from sklearn.model_selection import cross_validate\nfrom sklearn.neighbors import KNeighborsClassifier\nfrom sklearn.tree import DecisionTreeClassifier\nfrom sklearn.ensemble import RandomForestClassifier\n\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8dfdd4dc0d694846b2095e0f63dcc8d8c408501f"},"cell_type":"markdown","source":"You can choose some classified (even all), of course, the more classified amount will be the larger the waiting time will increase accordingly, in this kernel I just present the idea"},{"metadata":{"trusted":true,"_uuid":"3491bea17971093c38d32230645351425ba8d2fc"},"cell_type":"code","source":"classifiers = [\n    KNeighborsClassifier(3),\n    DecisionTreeClassifier(),\n    RandomForestClassifier()]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8459c43328db874dba03c2d3d308fe6e599dc6f2"},"cell_type":"code","source":"scoring = ['recall_macro']\nlog_cols = [\"Classifier\", \"macro average Recall\", \"Log Loss\"]\nlog = pa.DataFrame(columns=log_cols)\n\nfor clf in classifiers:\n    recall = cross_validate(clf, train, y, scoring=scoring,\n                              cv=5, return_train_score=False)\n    name = clf.__class__.__name__\n\n    print(\"=\" * 30)\n    print(name)\n    print('****Results****')\n    Recall = np.mean(recall['test_recall_macro'])\n    print(\"Macro Recall: {:.4%}\".format(Recall))\n\n    ll = 1 - Recall\n\n    print(\"Log Loss: {}\".format(ll))\n\n    log_entry = pa.DataFrame([[name, Recall * 100, ll]], columns=log_cols)\n    log = log.append(log_entry)\n\n\nprint(\"=\" * 30)\n\nsns.set_color_codes(\"muted\")\nsns.barplot(x='macro average Recall', y='Classifier', data=log, color=\"b\")\n\nplt.xlabel('Macro Average Recall %')\nplt.xticks(np.arange(0, 100, step=10))\nplt.title('recall scores -sensitivity')\nplt.show()\n\n\nsns.set_color_codes(\"muted\")\nsns.barplot(x='Log Loss', y='Classifier', data=log, color=\"g\")\n\nplt.xlabel('Log Loss')\n# plt.xticks(np.arange(0, 10, step=0.5))\nplt.title('Classifiers Log Loss using max-min rule')\nplt.show()\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bb10e5cd4a8d361b2c42bff14d0be0d9a5e27e57"},"cell_type":"markdown","source":"good luck!"}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}