{"cells":[{"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","collapsed":true,"trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)","execution_count":1,"outputs":[]},{"metadata":{"_cell_guid":"d64b5856-6b18-46b7-9d24-5e6ae537ba30","_uuid":"c212e55a597ce99352a53262a3bff39e5d82ba6a"},"cell_type":"markdown","source":"# Get a sense from the training samples\n---\n1. The target variable is extremely imbalanced\n2. The most downloaded app often have very low probability to be downloaded, lower than 0.1%.\n3. By looking at precision of recall of each single feature, we may use the product of recall and precision as the encoding method to represent each categorical value."},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","collapsed":true,"trusted":true},"cell_type":"code","source":"df_train_sample = pd.read_csv(\"../input/train_sample.csv\")","execution_count":3,"outputs":[]},{"metadata":{"_cell_guid":"5b2076cb-d425-4258-8bc3-a905fcd9151a","_uuid":"45e7abefde62b9265a16bf15ece41934f838e8a9","trusted":true},"cell_type":"code","source":"print(\"Training Samples contains {} rows {} columns\".format(*df_train_sample.shape))","execution_count":4,"outputs":[]},{"metadata":{"_cell_guid":"98313c5c-eb6c-427f-ac3d-d5ec67c9e01d","_uuid":"2341d445a3e4d08ca2990039afbf698b762c8507","trusted":true},"cell_type":"code","source":"df_train_sample.head()","execution_count":5,"outputs":[]},{"metadata":{"_cell_guid":"50805c2d-a508-489c-b179-82e39ac88fa4","_uuid":"471ad3759f2462a19615154c0c00f981e7080cc9","trusted":true},"cell_type":"code","source":"# It's extremely imbalanced\ndf_train_sample.groupby(\"is_attributed\").agg({\"ip\": \"count\"})","execution_count":6,"outputs":[]},{"metadata":{"_cell_guid":"63c53970-92ad-4111-af9a-60f968c7eee3","_uuid":"b6d945cef3d263f36becd9d1e9e6e361c7bd755d","collapsed":true,"trusted":true},"cell_type":"code","source":"def get_precision(x):\n    return x.sum() / x.shape[0]\ndef get_precision_recall_by_single_feature(col, df):\n    df_precision_recall = pd.DataFrame(columns=[\"Count\", \"Precision\", \"Recall\"], index=df[col].unique())\n    for c, f in [(\"Precision\", get_precision), (\"Count\", \"count\"), (\"Recall\", \"sum\")]:\n        _df = df.groupby(col).agg({\"is_attributed\": f})\n        df_precision_recall.loc[_df.index, c] = _df[\"is_attributed\"]\n    df_precision_recall[\"Recall\"] = df_precision_recall[\"Recall\"] / df[\"is_attributed\"].sum()\n    return df_precision_recall","execution_count":7,"outputs":[]},{"metadata":{"_cell_guid":"8d6886bd-8dac-4dfc-9be7-d13aa982574b","_uuid":"deb260c5e746858763237bd78dbd47eda8e3ccd2"},"cell_type":"markdown","source":"## Is APP a good feature"},{"metadata":{"_cell_guid":"420b54c7-d102-4a3f-8bdc-b4b4646ed117","_uuid":"839966224c0bd75f6ee1d88498ca31894b554561","trusted":true},"cell_type":"code","source":"col = \"app\"\ndf_app_precision_recall = get_precision_recall_by_single_feature(col, df_train_sample)\n# Get top 5 most indicative app\ndisplay(df_app_precision_recall.sort_values(\"Precision\", ascending=False).head())\n# Get top 5 most download app\ndisplay(df_app_precision_recall.sort_values(\"Recall\", ascending=False).head())","execution_count":8,"outputs":[]},{"metadata":{"_cell_guid":"f2a9f003-40df-48ae-a433-b1972db246d2","_uuid":"20172dea18eb75598d7b4b65db8db53c5e422c4e"},"cell_type":"markdown","source":"## Is DEVICE a good feature"},{"metadata":{"_cell_guid":"54fd3c8b-18ca-43d3-912a-2e114f3ecb6a","_uuid":"c0552c9828905006e3892f152fb3ed3b2669c155","trusted":true},"cell_type":"code","source":"col = \"device\"\ndf_app_precision_recall = get_precision_recall_by_single_feature(col, df_train_sample)\n# Get top 5 most indicative app\ndisplay(df_app_precision_recall.sort_values(\"Precision\", ascending=False).head())\n# Get top 5 most download app\ndisplay(df_app_precision_recall.sort_values(\"Recall\", ascending=False).head())","execution_count":9,"outputs":[]},{"metadata":{"_uuid":"ce3a4a811d9144f62d22102ac753e97df2f8ae80"},"cell_type":"markdown","source":"## Is OS a good feature"},{"metadata":{"_cell_guid":"43210026-222c-4ea9-8d8c-7237f8d9ab25","_uuid":"602da3b66ff38f806cd05ccca18f0009d8786e0c","trusted":true},"cell_type":"code","source":"col = \"os\"\ndf_app_precision_recall = get_precision_recall_by_single_feature(col, df_train_sample)\n# Get top 5 most indicative app\ndisplay(df_app_precision_recall.sort_values(\"Precision\", ascending=False).head())\n# Get top 5 most download app\ndisplay(df_app_precision_recall.sort_values(\"Recall\", ascending=False).head())","execution_count":10,"outputs":[]},{"metadata":{"_uuid":"c4c75ea372bbbba5db31920444fc5845916febed"},"cell_type":"markdown","source":"## Is CHANNEL a good feature"},{"metadata":{"_cell_guid":"bb29db8c-5220-46d7-8aea-8bac2869f8bb","_uuid":"28ca42f57b59a06b73abc0a204f968682cc3420b","trusted":true},"cell_type":"code","source":"col = \"channel\"\ndf_app_precision_recall = get_precision_recall_by_single_feature(col, df_train_sample)\n# Get top 5 most indicative app\ndisplay(df_app_precision_recall.sort_values(\"Precision\", ascending=False).head())\n# Get top 5 most download app\ndisplay(df_app_precision_recall.sort_values(\"Recall\", ascending=False).head())","execution_count":11,"outputs":[]},{"metadata":{"_cell_guid":"26857357-4e16-4061-9099-26ef88783eba","_uuid":"e750645d2f06ebcf26a72ddd3f6deec4c861d0fe","collapsed":true,"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.6.4"}},"nbformat":4,"nbformat_minor":1}