{"cells":[{"metadata":{"_uuid":"1c693a91606c1d1a5e3ed19246ea4f30679b9f1f"},"cell_type":"markdown","source":"# decision tree method lerning"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport os\nimport pandas as pd\nimport numpy as np\nimport sklearn\nfrom sklearn.model_selection import KFold\nfrom sklearn.tree import DecisionTreeClassifier\nimport matplotlib.pyplot as plt\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nprint(os.listdir(\"../input/prepare-data-for-decision-trees-algorithms\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"2384435cc09f8cb5a3147b8615d224e78bf71fb7"},"cell_type":"markdown","source":"# Load the dataset\nLoad the dataset from a pre-prepered dataset - See https://www.kaggle.com/itamargr/prepare-data-for-decision-trees-algorithms)"},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"# Now load the training set\nprint('Loading the training set:')\ndtypes = {\n    'MachineIdentifier': 'category',\n    'AVProductsInstalled': 'float32',\n    'CountryIdentifier': 'float32',\n    'OrganizationIdentifier': 'float32',\n    'GeoNameIdentifier': 'float32',\n    'LocaleEnglishNameIdentifier': 'float32',\n    'OsBuild': 'int16',\n    'OsSuite': 'float32',\n    'OsPlatformSubRelease': 'float32',\n    'SkuEdition': 'float32',\n    'IeVerIdentifier': 'float32',\n    'SmartScreen': 'float32',\n    'Census_MDC2FormFactor': 'float32',\n    'Census_ProcessorCoreCount': 'float16',\n    'Census_ProcessorManufacturerIdentifier': 'float32',\n    'Census_PrimaryDiskTotalCapacity': 'float32',\n    'Census_PrimaryDiskTypeName': 'float32',\n    'Census_SystemVolumeTotalCapacity': 'float32',\n    'Census_TotalPhysicalRAM': 'float32',\n    'Census_ChassisTypeName': 'float32',\n    'Census_InternalPrimaryDiagonalDisplaySizeInInches': 'float16',\n    'Census_InternalPrimaryDisplayResolutionHorizontal': 'float16',\n    'Census_InternalPrimaryDisplayResolutionVertical': 'float16',\n    'Census_PowerPlatformRoleName': 'float32',\n    'Census_InternalBatteryType': 'float32',\n    'Census_InternalBatteryNumberOfCharges': 'float32',\n    'Census_OSBranch': 'float32',\n    'Census_OSBuildNumber': 'int16',\n    'Census_OSBuildRevision': 'int32',\n    'Census_OSEdition': 'float32',\n    'Census_OSSkuName': 'float32',\n    'Census_OSInstallTypeName': 'float32',\n    'Census_OSInstallLanguageIdentifier': 'float32',\n    'Census_OSUILocaleIdentifier': 'float32',\n    'Census_OSWUAutoUpdateOptionsName': 'float32',\n    'Census_GenuineStateName': 'float32',\n    'Census_ActivationChannel': 'float32',\n    'Census_IsFlightingInternal': 'float32',\n    'Census_ThresholdOptIn': 'float16',\n    'Census_IsSecureBootEnabled': 'int8',\n    'Census_IsWIMBootEnabled': 'float32',\n    'Census_IsTouchEnabled': 'int8',\n    'Wdft_IsGamer': 'float32',\n    'Wdft_RegionIdentifier': 'float32',\n    'HasDetections': 'int8',\n    'EngineVersion_0': 'float32',\n    'EngineVersion_1': 'float32',\n    'EngineVersion_2': 'float32',\n    'EngineVersion_3': 'float32',\n    'AppVersion_0': 'float32',\n    'AppVersion_1': 'float32',\n    'AppVersion_2': 'float32',\n    'AppVersion_3': 'float32',\n    'AvSigVersion_0': 'float32',\n    'AvSigVersion_1': 'float32',\n    'AvSigVersion_2': 'float32',\n    'AvSigVersion_3': 'float32',\n    'Census_OSVersion_0': 'float32',\n    'Census_OSVersion_1': 'float32',\n    'Census_OSVersion_2': 'float32',\n    'Census_OSVersion_3': 'float32'\n}\ntraining_set = pd.read_csv('../input/prepare-data-for-decision-trees-algorithms/training_decisionTrees.csv', dtype=dtypes)\nprint('Training set loaded')\nprint(training_set.shape)\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"54915a67f6507f0d7e9046a3893f119b3e7be549"},"cell_type":"markdown","source":"# Perpare the data"},{"metadata":{"_uuid":"ff23ff3b45f7ee6e7ef7f0dc7b3d6f23e1463c5f"},"cell_type":"markdown","source":"First:\nscikit learn decision tree can have NaN in the data. So whenever I have NaN, I convert it to the average of the column.\nI will try other methos later."},{"metadata":{"trusted":true,"_uuid":"d936219039cee2ac9b3796c42c07cf199c769953"},"cell_type":"code","source":"# Handle Nans\ndef create_nan_dict(data):\n    ret = dict()\n    for col in data:\n        if col != 'HasDetections' and col != 'MachineIdentifier':\n            ret[col] = data[col].astype('float32').mean()\n    return ret\n\n\ndef remove_nans(data, nan_dict):\n    for col in data:\n        if col != 'HasDetections' and col != 'MachineIdentifier':\n            data[col] = data[col].fillna(nan_dict[col])\n\nprint('Handling NaN in training set')\nnan_dict = create_nan_dict(training_set)\nremove_nans(training_set, nan_dict)\nprint('Done')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"2548f9f0264140cea718ce0d170bc11ce39a29e0"},"cell_type":"markdown","source":"Now we remove colums that have only one value. Those colums are not relevant for Decision Tree and can be removed for run time"},{"metadata":{"trusted":true,"_uuid":"35f4ce6ed21d67c0d67c7d6db0013dd29ed7ffd9"},"cell_type":"code","source":"columns_to_remove = []\nfor col_name in training_set.columns.values:\n    if col_name == 'HasDetections' or col_name == 'MachineIdentifier':\n        continue\n    unique_values = training_set[col_name].value_counts(dropna=False)\n    msg = 'column ' + col_name + ' have ' + str(len(unique_values)) + ' unique values. The bigger category has ' + str(100 * unique_values.values[0] / training_set.shape[0]) + ' percent of the data'\n    if len(unique_values)==1:\n        msg = msg + \" - removed\"\n        del training_set[col_name]  \n        columns_to_remove.append(col_name)\n    print(msg)\n\nprint('')\nprint('Untill now ' + str(len(columns_to_remove)) + ' colums removed')\nprint(training_set.shape)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"fffd7b78b5e0e1885e231153a3739e403e7ed27a"},"cell_type":"markdown","source":"# Learning"},{"metadata":{"_uuid":"81b51d59f5a70e8fc7570e5c2b1da169827cc95a"},"cell_type":"markdown","source":"This part is a fine tuning. For now, the only parameter that need to be fine turned is min_samples_leaf\nI try some min_samples_leaf values.\nFor each value, I take 4/5 of the trainnig set as train and 1/5 as test (KFold). I train and check the ROC results. I take the min_samples_leaf value that gives me the best result."},{"metadata":{"trusted":true,"_uuid":"e6f97052affbd3a874640ea6ed1b85df6cfcc910"},"cell_type":"code","source":"def fine_tune_decision_tree(training_set, k_fold):\n    results = dict()\n    avg_grade = dict()\n    std_grade = dict()\n    min_grade = dict()\n    max_grade = dict()\n    best_sample_leaf = 0\n    best_grade = 0.5\n    for min_samples_leaf in [200, 400, 500, 600, 800, 1000]:\n        features = [c for c in training_set.columns if c not in ['MachineIdentifier', 'HasDetections']]\n        dt = DecisionTreeClassifier(min_samples_leaf=min_samples_leaf)\n        results[min_samples_leaf] = []\n\n        # Train and tests the data on k_fold splits and store the results\n        for train_indices, test_indices in k_fold.split(training_set):\n            print('Start fitting')\n            dt.fit(training_set[features].iloc[train_indices], training_set['HasDetections'].iloc[train_indices])\n            print('End fitting - Start predicting')\n            prob = dt.predict_proba(training_set[features].iloc[test_indices])\n            fpr, tpr, thresholds = sklearn.metrics.roc_curve(training_set['HasDetections'].iloc[test_indices], prob[:, 1])\n            results[min_samples_leaf].append(sklearn.metrics.auc(fpr, tpr))\n            print('End predicting')\n\n        grade = np.mean(results[min_samples_leaf])\n        if grade > best_grade:\n            best_grade = grade\n            best_sample_leaf = min_samples_leaf\n        avg_grade[min_samples_leaf] = grade\n        std_grade[min_samples_leaf] = np.std(results[min_samples_leaf])\n        min_grade[min_samples_leaf] = np.min(results[min_samples_leaf])\n        max_grade[min_samples_leaf] = np.max(results[min_samples_leaf])\n\n    # Now plot the result.\n    n_leafs = avg_grade.keys()\n    avgs = [avg_grade[l] for l in n_leafs]\n    stds = [std_grade[l] for l in n_leafs]\n    plt.figure()\n    plt.errorbar(n_leafs, avgs, stds)\n    plt.title('decision tree classifier k fold results')\n    plt.xlabel('number of minimum sample in a leaf')\n    plt.ylabel('ROC curve area')\n    plt.show()\n\n    return DecisionTreeClassifier(min_samples_leaf=best_sample_leaf)\n\n# Define decision tree predictor and fine tune its variables\nk_fold = KFold(n_splits=5, shuffle=True)\nclassifier = fine_tune_decision_tree(training_set, k_fold)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"add9f9e86fb49b642a24c66103e52bdddbac5674"},"cell_type":"markdown","source":"Now train with the hole set"},{"metadata":{"trusted":true,"_uuid":"71a80fda3b9d4a9c6df6f709c0005c5f8d4b7f24"},"cell_type":"code","source":"features = [c for c in training_set.columns if c not in ['MachineIdentifier', 'HasDetections']]\nX = training_set[features]\ny = training_set['HasDetections']\ndel training_set\nclassifier.fit(X, y)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c69d95354c60fd2e303d44273c123a1a3da1aec3"},"cell_type":"markdown","source":"# Calculting prediction on the test set and saving the results"},{"metadata":{"trusted":true,"_uuid":"f61aede176f77cf1574d27f1ca47c26df07f8039"},"cell_type":"code","source":"del X\ndel y\ntest = pd.read_csv('../input/prepare-data-for-decision-trees-algorithms/test_decisionTrees.csv', dtype=dtypes)\nremove_nans(test, nan_dict)\n\nX = test[features]\ny_pred = classifier.predict_proba(X)[:, 1]\nto_submit = pd.DataFrame(test['MachineIdentifier'])\nto_submit['HasDetections'] = y_pred\nto_submit.to_csv('decisionTreeClassifierRes.csv', index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}