{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# **Microsoft Malware Prediction**\n##### Predict if a machine will soon be hit with malware","metadata":{}},{"cell_type":"markdown","source":"### **Setup**\n\nImporting necessary modules and packages.","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport seaborn as sns\nimport lightgbm as lgb\nimport xgboost as xgb\nimport time, datetime, gc\n\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.model_selection import StratifiedKFold, KFold, TimeSeriesSplit\nfrom sklearn.metrics import mean_squared_error, roc_auc_score\nfrom sklearn.linear_model import LogisticRegression, LogisticRegressionCV\n\nfrom catboost import CatBoostClassifier\nfrom tqdm import tqdm_notebook\nimport plotly.graph_objs as go\nimport plotly.tools as tls\n\nimport plotly.offline as py\npy.init_notebook_mode(connected=True)\n\nimport matplotlib.pyplot as plt\n%matplotlib inline\nplt.style.use('ggplot')\n\n# ignore unnecessary warnings\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n\nimport logging\nlogging.basicConfig(filename='log.txt', level=logging.DEBUG, format='%(asctime)s %(message)s')\n\nimport pandas as pd\npd.set_option('max_colwidth', 500)\npd.set_option('max_columns', 500)\npd.set_option('max_rows', 100)\n\n#display contents of dataset directory\nimport os\nprint(os.listdir(\"../input/microsoft-malware-prediction\"))","metadata":{"execution":{"iopub.status.busy":"2022-08-05T09:38:05.937758Z","iopub.execute_input":"2022-08-05T09:38:05.938465Z","iopub.status.idle":"2022-08-05T09:38:09.110058Z","shell.execute_reply.started":"2022-08-05T09:38:05.938370Z","shell.execute_reply":"2022-08-05T09:38:09.108781Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### **Loading Data**","metadata":{}},{"cell_type":"code","source":"dtypes = {\n 'MachineIdentifier': 'category',\n 'ProductName': 'category',\n 'EngineVersion': 'category',\n 'AppVersion': 'category',\n 'AvSigVersion': 'category',\n 'IsBeta': 'int8',\n 'RtpStateBitfield': 'float16',\n 'IsSxsPassiveMode': 'int8',\n 'DefaultBrowsersIdentifier': 'float16',\n 'AVProductStatesIdentifier': 'float32',\n 'AVProductsInstalled': 'float16',\n 'AVProductsEnabled': 'float16',\n 'HasTpm': 'int8',\n 'CountryIdentifier': 'int16',\n 'CityIdentifier': 'float32',\n 'OrganizationIdentifier': 'float16',\n 'GeoNameIdentifier': 'float16',\n 'LocaleEnglishNameIdentifier': 'int8',\n 'Platform': 'category',\n 'Processor': 'category',\n 'OsVer': 'category',\n 'OsBuild': 'int16',\n 'OsSuite': 'int16',\n 'OsPlatformSubRelease': 'category',\n 'OsBuildLab': 'category',\n 'SkuEdition': 'category',\n 'IsProtected': 'float16',\n 'AutoSampleOptIn': 'int8',\n 'PuaMode': 'category',\n 'SMode': 'float16',\n 'IeVerIdentifier': 'float16',\n 'SmartScreen': 'category',\n 'Firewall': 'float16',\n 'UacLuaenable': 'float32',\n 'Census_MDC2FormFactor': 'category',\n 'Census_DeviceFamily': 'category',\n 'Census_OEMNameIdentifier': 'float16',\n 'Census_OEMModelIdentifier': 'float32',\n 'Census_ProcessorCoreCount': 'float16',\n 'Census_ProcessorManufacturerIdentifier': 'float16',\n 'Census_ProcessorModelIdentifier': 'float16',\n 'Census_ProcessorClass': 'category',\n 'Census_PrimaryDiskTotalCapacity': 'float32',\n 'Census_PrimaryDiskTypeName': 'category',\n 'Census_SystemVolumeTotalCapacity': 'float32',\n 'Census_HasOpticalDiskDrive': 'int8',\n 'Census_TotalPhysicalRAM': 'float32',\n 'Census_ChassisTypeName': 'category',\n 'Census_InternalPrimaryDiagonalDisplaySizeInInches': 'float16',\n 'Census_InternalPrimaryDisplayResolutionHorizontal': 'float16',\n 'Census_InternalPrimaryDisplayResolutionVertical': 'float16',\n 'Census_PowerPlatformRoleName': 'category',\n 'Census_InternalBatteryType': 'category',\n 'Census_InternalBatteryNumberOfCharges': 'float32',\n 'Census_OSVersion': 'category',\n 'Census_OSArchitecture': 'category',\n 'Census_OSBranch': 'category',\n 'Census_OSBuildNumber': 'int16',\n 'Census_OSBuildRevision': 'int32',\n 'Census_OSEdition': 'category',\n 'Census_OSSkuName': 'category',\n 'Census_OSInstallTypeName': 'category',\n 'Census_OSInstallLanguageIdentifier': 'float16',\n 'Census_OSUILocaleIdentifier': 'int16',\n 'Census_OSWUAutoUpdateOptionsName': 'category',\n 'Census_IsPortableOperatingSystem': 'int8',\n 'Census_GenuineStateName': 'category',\n 'Census_ActivationChannel': 'category',\n 'Census_IsFlightingInternal': 'float16',\n 'Census_IsFlightsDisabled': 'float16',\n 'Census_FlightRing': 'category',\n 'Census_ThresholdOptIn': 'float16',\n 'Census_FirmwareManufacturerIdentifier': 'float16',\n 'Census_FirmwareVersionIdentifier': 'float32',\n 'Census_IsSecureBootEnabled': 'int8',\n 'Census_IsWIMBootEnabled': 'float16',\n 'Census_IsVirtualDevice': 'float16',\n 'Census_IsTouchEnabled': 'int8',\n 'Census_IsPenCapable': 'int8',\n 'Census_IsAlwaysOnAlwaysConnectedCapable': 'float16',\n 'Wdft_IsGamer': 'float16',\n 'Wdft_RegionIdentifier': 'float16',\n 'HasDetections': 'int8'\n }\n\ndef reduce_mem_usage(df, verbose=True):\n numerics = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']\n start_mem = df.memory_usage(deep=True).sum() / 1024**2 \n for col in df.columns:\n col_type = df[col].dtypes\n if col_type in numerics:\n c_min = df[col].min()\n c_max = df[col].max()\n if str(col_type)[:3] == 'int':\n if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n df[col] = df[col].astype(np.int8)\n elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n df[col] = df[col].astype(np.int16)\n elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n df[col] = df[col].astype(np.int32)\n elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n df[col] = df[col].astype(np.int64) \n else:\n if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n df[col] = df[col].astype(np.float16)\n elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n df[col] = df[col].astype(np.float32)\n else:\n df[col] = df[col].astype(np.float64) \n end_mem = df.memory_usage(deep=True).sum() / 1024**2\n if verbose: \n print('Mem. usage decreased to {:5.2f} Mb ({:.1f}% reduction)'.format(end_mem, 100 * (start_mem - end_mem) / start_mem))\n return df","metadata":{"execution":{"iopub.status.busy":"2022-08-05T09:38:09.112432Z","iopub.execute_input":"2022-08-05T09:38:09.113189Z","iopub.status.idle":"2022-08-05T09:38:09.138538Z","shell.execute_reply.started":"2022-08-05T09:38:09.113141Z","shell.execute_reply":"2022-08-05T09:38:09.137326Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"numerics = ['int8', 'int16', 'int32', 'int64', 'float16', 'float32', 'float64']\nnumerical_columns = [c for c, v in dtypes.items() if v in numerics]\ncategorical_columns = [c for c, v in dtypes.items() if v not in numerics]","metadata":{"execution":{"iopub.status.busy":"2022-08-05T09:38:09.139991Z","iopub.execute_input":"2022-08-05T09:38:09.140867Z","iopub.status.idle":"2022-08-05T09:38:09.158336Z","shell.execute_reply.started":"2022-08-05T09:38:09.140827Z","shell.execute_reply":"2022-08-05T09:38:09.157383Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%time\ntrain = pd.read_csv('../input/microsoft-malware-prediction/train.csv', dtype=dtypes)","metadata":{"execution":{"iopub.status.busy":"2022-08-05T09:38:09.161385Z","iopub.execute_input":"2022-08-05T09:38:09.161980Z","iopub.status.idle":"2022-08-05T09:41:12.915618Z","shell.execute_reply.started":"2022-08-05T09:38:09.161943Z","shell.execute_reply":"2022-08-05T09:41:12.914658Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = reduce_mem_usage(train)","metadata":{"execution":{"iopub.status.busy":"2022-08-05T09:41:12.917496Z","iopub.execute_input":"2022-08-05T09:41:12.918152Z","iopub.status.idle":"2022-08-05T09:41:28.032650Z","shell.execute_reply.started":"2022-08-05T09:41:12.918079Z","shell.execute_reply":"2022-08-05T09:41:28.031275Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"stats = []\nfor col in train.columns:\n stats.append((col, train[col].nunique(), train[col].isnull().sum() * 100 / train.shape[0], train[col].value_counts(normalize=True, dropna=False).values[0] * 100, train[col].dtype))\n \nstats_df = pd.DataFrame(stats, columns=['Feature', 'Unique_values', 'Percentage of missing values', 'Percentage of values in the biggest category', 'type'])\nstats_df.sort_values('Percentage of missing values', ascending=False)","metadata":{"execution":{"iopub.status.busy":"2022-08-05T09:41:28.034465Z","iopub.execute_input":"2022-08-05T09:41:28.034855Z","iopub.status.idle":"2022-08-05T09:41:48.070752Z","shell.execute_reply.started":"2022-08-05T09:41:28.034823Z","shell.execute_reply":"2022-08-05T09:41:48.069490Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### **Observation**\n\n- ```PuaMode``` & ```Census_ProcessorClass``` have 99.5+% missing values. This implied that these columns are useless & should be dropped.\n\n- In ```DefaultBrowsersIdentifier``` column ~95% values belong to one category, so this columns also seems useless.\n\n- ```Census_IsFlightingInternal``` needs to be analyzed carefully.\n\n- There are 26 columns in total in which one category contains 90% values. These imbalanced columns should be removed from the dataset.\n\n- Except ```Census_SystemVolumeTotalCapacity``` all columns are categorical. \n\n- There are 3 columns, where most of the values are missing which can be dropped.","metadata":{}},{"cell_type":"code","source":"good_cols = list(train.columns)\nfor col in train.columns:\n rate = train[col].value_counts(normalize=True, dropna=False).values[0]\n if rate > 0.9:\n good_cols.remove(col)","metadata":{"execution":{"iopub.status.busy":"2022-08-05T09:41:48.072370Z","iopub.execute_input":"2022-08-05T09:41:48.072834Z","iopub.status.idle":"2022-08-05T09:41:58.516019Z","shell.execute_reply.started":"2022-08-05T09:41:48.072790Z","shell.execute_reply":"2022-08-05T09:41:58.514707Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = train[good_cols]","metadata":{"execution":{"iopub.status.busy":"2022-08-05T09:41:58.517559Z","iopub.execute_input":"2022-08-05T09:41:58.518688Z","iopub.status.idle":"2022-08-05T09:41:58.899918Z","shell.execute_reply.started":"2022-08-05T09:41:58.518638Z","shell.execute_reply":"2022-08-05T09:41:58.898622Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Now test data is readable","metadata":{}},{"cell_type":"code","source":"test_dtypes = {k: v for k, v in dtypes.items() if k in good_cols}\ntest = pd.read_csv('../input/microsoft-malware-prediction/test.csv', dtype=test_dtypes, usecols=good_cols[:-1])\ntest.loc[6529507, 'OsBuildLab'] = '17134.1.amd64fre.rs4_release.180410-1804'\ntest['OsBuildLab'] = test['OsBuildLab'].fillna('17134.1.amd64fre.rs4_release.180410-1804')\ntest = reduce_mem_usage(test)","metadata":{"execution":{"iopub.status.busy":"2022-08-05T09:41:58.901403Z","iopub.execute_input":"2022-08-05T09:41:58.902268Z","iopub.status.idle":"2022-08-05T09:44:44.321695Z","shell.execute_reply.started":"2022-08-05T09:41:58.902229Z","shell.execute_reply":"2022-08-05T09:44:44.320173Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### **Data Exploration**","metadata":{}},{"cell_type":"code","source":"# return top 5 rows of dataframe \ntrain.head()","metadata":{"execution":{"iopub.status.busy":"2022-08-05T09:44:44.327676Z","iopub.execute_input":"2022-08-05T09:44:44.328085Z","iopub.status.idle":"2022-08-05T09:44:44.392013Z","shell.execute_reply.started":"2022-08-05T09:44:44.328050Z","shell.execute_reply":"2022-08-05T09:44:44.390455Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# function to plot data\ndef plot_categorical_feature(col, only_bars=False, top_n=10, by_touch=False):\n top_n = top_n if train[col].nunique() > top_n else train[col].nunique()\n print(f\"{col} has {train[col].nunique()} unique values and type: {train[col].dtype}.\")\n print(train[col].value_counts(normalize=True, dropna=False).head())\n if not by_touch:\n if not only_bars:\n df = train.groupby([col]).agg({'HasDetections': ['count', 'mean']})\n df = df.sort_values(('HasDetections', 'count'), ascending=False).head(top_n).sort_index()\n data = [go.Bar(x=df.index, y=df['HasDetections']['count'].values, name='counts'),\n go.Scatter(x=df.index, y=df['HasDetections']['mean'], name='Detections rate', yaxis='y2')]\n\n layout = go.Layout(dict(title = f\"Counts of {col} by top-{top_n} categories and mean target value\",\n xaxis = dict(title = f'{col}',\n showgrid=False,\n zeroline=False,\n showline=False,),\n yaxis = dict(title = 'Counts',\n showgrid=False,\n zeroline=False,\n showline=False,),\n yaxis2=dict(title='Detections rate', overlaying='y', side='right')),\n legend=dict(orientation=\"v\"))\n\n else:\n top_cat = list(train[col].value_counts(dropna=False).index[:top_n])\n df0 = train.loc[(train[col].isin(top_cat)) & (train['HasDetections'] == 1), col].value_counts().head(10).sort_index()\n df1 = train.loc[(train[col].isin(top_cat)) & (train['HasDetections'] == 0), col].value_counts().head(10).sort_index()\n data = [go.Bar(x=df0.index, y=df0.values, name='Has Detections'),\n go.Bar(x=df1.index, y=df1.values, name='No Detections')]\n\n layout = go.Layout(dict(title = f\"Counts of {col} by top-{top_n} categories\",\n xaxis = dict(title = f'{col}',\n showgrid=False,\n zeroline=False,\n showline=False,),\n yaxis = dict(title = 'Counts',\n showgrid=False,\n zeroline=False,\n showline=False,),\n ),\n legend=dict(orientation=\"v\"), barmode='group')\n \n py.iplot(dict(data=data, layout=layout))\n \n else:\n top_n = 10\n top_cat = list(train[col].value_counts(dropna=False).index[:top_n])\n df = train.loc[train[col].isin(top_cat)]\n\n df1 = train.loc[train['Census_IsTouchEnabled'] == 1]\n df0 = train.loc[train['Census_IsTouchEnabled'] == 0]\n\n df0_ = df0.groupby([col]).agg({'HasDetections': ['count', 'mean']})\n df0_ = df0_.sort_values(('HasDetections', 'count'), ascending=False).head(top_n).sort_index()\n df1_ = df1.groupby([col]).agg({'HasDetections': ['count', 'mean']})\n df1_ = df1_.sort_values(('HasDetections', 'count'), ascending=False).head(top_n).sort_index()\n data1 = [go.Bar(x=df0_.index, y=df0_['HasDetections']['count'].values, name='Nontouch device counts'),\n go.Scatter(x=df0_.index, y=df0_['HasDetections']['mean'], name='Detections rate for nontouch devices', yaxis='y2')]\n data2 = [go.Bar(x=df1_.index, y=df1_['HasDetections']['count'].values, name='Touch device counts'),\n go.Scatter(x=df1_.index, y=df1_['HasDetections']['mean'], name='Detections rate for touch devices', yaxis='y2')]\n\n layout = go.Layout(dict(title = f\"Counts of {col} by top-{top_n} categories for nontouch devices\",\n xaxis = dict(title = f'{col}',\n showgrid=False,\n zeroline=False,\n showline=False,\n type='category'),\n yaxis = dict(title = 'Counts',\n showgrid=False,\n zeroline=False,\n showline=False,),\n yaxis2=dict(title='Detections rate', overlaying='y', side='right'),\n ),\n legend=dict(orientation=\"v\"), barmode='group')\n\n py.iplot(dict(data=data1, layout=layout))\n layout['title'] = f\"Counts of {col} by top-{top_n} categories for touch devices\"\n py.iplot(dict(data=data2, layout=layout))","metadata":{"execution":{"iopub.status.busy":"2022-08-05T09:44:44.394073Z","iopub.execute_input":"2022-08-05T09:44:44.394554Z","iopub.status.idle":"2022-08-05T09:44:44.422799Z","shell.execute_reply.started":"2022-08-05T09:44:44.394507Z","shell.execute_reply":"2022-08-05T09:44:44.421549Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### **Target**","metadata":{}},{"cell_type":"code","source":"train['HasDetections'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-08-05T09:44:44.424435Z","iopub.execute_input":"2022-08-05T09:44:44.425529Z","iopub.status.idle":"2022-08-05T09:44:44.508802Z","shell.execute_reply.started":"2022-08-05T09:44:44.425492Z","shell.execute_reply":"2022-08-05T09:44:44.507453Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Target seems balanced.","metadata":{}},{"cell_type":"markdown","source":"**Census_IsTouchEnabled**","metadata":{}},{"cell_type":"code","source":"plot_categorical_feature('Census_IsTouchEnabled', True)","metadata":{"execution":{"iopub.status.busy":"2022-08-05T09:44:44.510973Z","iopub.execute_input":"2022-08-05T09:44:44.511688Z","iopub.status.idle":"2022-08-05T09:44:46.205320Z","shell.execute_reply.started":"2022-08-05T09:44:44.511654Z","shell.execute_reply":"2022-08-05T09:44:46.204166Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Since Microsoft has much more computers than touch devices, rate of infections is lower for touch devices, but not by much.","metadata":{}},{"cell_type":"markdown","source":"First look at variables with lot of categories, then move onto variables with limited categories.","metadata":{}},{"cell_type":"markdown","source":"**EngineVersion**","metadata":{}},{"cell_type":"code","source":"plot_categorical_feature('EngineVersion', by_touch=True)","metadata":{"execution":{"iopub.status.busy":"2022-08-05T09:44:46.207596Z","iopub.execute_input":"2022-08-05T09:44:46.208919Z","iopub.status.idle":"2022-08-05T09:44:56.353518Z","shell.execute_reply.started":"2022-08-05T09:44:46.208836Z","shell.execute_reply":"2022-08-05T09:44:56.352137Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"2 categories take 84% of all values & thus, the difference in detection rates is noticeable. Other categories have different detecttion rates, but that is due to low number of samples in them. Patterns for touch & non-touch devices are quite similar.","metadata":{}},{"cell_type":"markdown","source":"**AppVersion**","metadata":{}},{"cell_type":"code","source":"plot_categorical_feature('AppVersion')","metadata":{"execution":{"iopub.status.busy":"2022-08-05T09:44:56.355451Z","iopub.execute_input":"2022-08-05T09:44:56.355921Z","iopub.status.idle":"2022-08-05T09:44:56.752446Z","shell.execute_reply.started":"2022-08-05T09:44:56.355888Z","shell.execute_reply":"2022-08-05T09:44:56.751201Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"There is one main category with 0.53 detection rate, rest are much smaller.","metadata":{}},{"cell_type":"markdown","source":"**AvSigVersion**","metadata":{}},{"cell_type":"code","source":"plot_categorical_feature('AvSigVersion')","metadata":{"execution":{"iopub.status.busy":"2022-08-05T09:44:56.754018Z","iopub.execute_input":"2022-08-05T09:44:56.754421Z","iopub.status.idle":"2022-08-05T09:44:57.184653Z","shell.execute_reply.started":"2022-08-05T09:44:56.754386Z","shell.execute_reply":"2022-08-05T09:44:57.183422Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"It seems that this is a version of an often updated software because it has huge amount of categories.","metadata":{}},{"cell_type":"markdown","source":"**AVProductStatesIdentifier**","metadata":{}},{"cell_type":"code","source":"plot_categorical_feature('AVProductStatesIdentifier', True, 10)","metadata":{"execution":{"iopub.status.busy":"2022-08-05T09:44:57.186502Z","iopub.execute_input":"2022-08-05T09:44:57.186904Z","iopub.status.idle":"2022-08-05T09:44:58.117468Z","shell.execute_reply.started":"2022-08-05T09:44:57.186867Z","shell.execute_reply":"2022-08-05T09:44:58.116177Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"This is a categorical variable.","metadata":{}},{"cell_type":"code","source":"train['AVProductStatesIdentifier'] = train['AVProductStatesIdentifier'].astype('category')\ntest['AVProductStatesIdentifier'] = test['AVProductStatesIdentifier'].astype('category')","metadata":{"execution":{"iopub.status.busy":"2022-08-05T09:44:58.119860Z","iopub.execute_input":"2022-08-05T09:44:58.120350Z","iopub.status.idle":"2022-08-05T09:44:58.734357Z","shell.execute_reply.started":"2022-08-05T09:44:58.120308Z","shell.execute_reply":"2022-08-05T09:44:58.733158Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**AVProductsInstalled**","metadata":{}},{"cell_type":"code","source":"plot_categorical_feature('AVProductsInstalled', True)","metadata":{"execution":{"iopub.status.busy":"2022-08-05T09:44:58.736013Z","iopub.execute_input":"2022-08-05T09:44:58.736405Z","iopub.status.idle":"2022-08-05T09:45:01.413437Z","shell.execute_reply.started":"2022-08-05T09:44:58.736371Z","shell.execute_reply":"2022-08-05T09:45:01.411653Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"A computer is less likely to be infected if it has an Antivirus. *BUT*, having 2 antiviruses has an opposite effect.\n\nOther categories have really low samples, so we'll combine them.","metadata":{}},{"cell_type":"code","source":"train['AVProductsInstalled'] = train['AVProductsInstalled'].astype('category')\ntest['AVProductsInstalled'] = test['AVProductsInstalled'].astype('category')","metadata":{"execution":{"iopub.status.busy":"2022-08-05T09:45:01.415563Z","iopub.execute_input":"2022-08-05T09:45:01.416002Z","iopub.status.idle":"2022-08-05T09:45:02.127334Z","shell.execute_reply.started":"2022-08-05T09:45:01.415955Z","shell.execute_reply":"2022-08-05T09:45:02.124907Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_categorical_feature('AVProductsInstalled', True, by_touch=True)","metadata":{"execution":{"iopub.status.busy":"2022-08-05T09:45:02.129123Z","iopub.execute_input":"2022-08-05T09:45:02.130395Z","iopub.status.idle":"2022-08-05T09:45:09.668881Z","shell.execute_reply.started":"2022-08-05T09:45:02.130349Z","shell.execute_reply":"2022-08-05T09:45:09.667738Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"It seems that even on touch devices people sometimes install 2 antiviruses.","metadata":{}},{"cell_type":"markdown","source":"**CountryIdentifier**","metadata":{}},{"cell_type":"code","source":"plot_categorical_feature('CountryIdentifier', True, 20)","metadata":{"execution":{"iopub.status.busy":"2022-08-05T09:45:09.670698Z","iopub.execute_input":"2022-08-05T09:45:09.671055Z","iopub.status.idle":"2022-08-05T09:45:10.278001Z","shell.execute_reply.started":"2022-08-05T09:45:09.671022Z","shell.execute_reply":"2022-08-05T09:45:10.276771Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"This is a categorical column defined as numerical.\n\nWhile most countries have rate of detections ~50%, there are some countries where infected devices are more in number. ","metadata":{}},{"cell_type":"code","source":"train['CountryIdentifier'] = train['CountryIdentifier'].astype('category')\ntest['CountryIdentifier'] = test['CountryIdentifier'].astype('category')","metadata":{"execution":{"iopub.status.busy":"2022-08-05T09:45:10.279740Z","iopub.execute_input":"2022-08-05T09:45:10.280262Z","iopub.status.idle":"2022-08-05T09:45:10.515995Z","shell.execute_reply.started":"2022-08-05T09:45:10.280216Z","shell.execute_reply":"2022-08-05T09:45:10.514928Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**CityIdentifier**","metadata":{}},{"cell_type":"code","source":"plot_categorical_feature('CityIdentifier', True, 20)","metadata":{"execution":{"iopub.status.busy":"2022-08-05T09:45:10.517544Z","iopub.execute_input":"2022-08-05T09:45:10.517861Z","iopub.status.idle":"2022-08-05T09:45:12.353122Z","shell.execute_reply.started":"2022-08-05T09:45:10.517833Z","shell.execute_reply":"2022-08-05T09:45:12.351781Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Same as ```CountryIdentifier```","metadata":{}},{"cell_type":"code","source":"train['CityIdentifier'] = train['CityIdentifier'].astype('category')\ntest['CityIdentifier'] = test['CityIdentifier'].astype('category')","metadata":{"execution":{"iopub.status.busy":"2022-08-05T09:45:12.354867Z","iopub.execute_input":"2022-08-05T09:45:12.355997Z","iopub.status.idle":"2022-08-05T09:45:13.062968Z","shell.execute_reply.started":"2022-08-05T09:45:12.355950Z","shell.execute_reply":"2022-08-05T09:45:13.061968Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**OrganizationIdentifier**","metadata":{}},{"cell_type":"code","source":"plot_categorical_feature('OrganizationIdentifier', True)","metadata":{"execution":{"iopub.status.busy":"2022-08-05T09:45:13.064605Z","iopub.execute_input":"2022-08-05T09:45:13.065242Z","iopub.status.idle":"2022-08-05T09:45:16.803044Z","shell.execute_reply.started":"2022-08-05T09:45:13.065206Z","shell.execute_reply":"2022-08-05T09:45:16.801793Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Only 2 organizations cover ~66% of all computers, while unknown organizations have 30% more. \n\nLet's combine values as these might be some specific industries.","metadata":{}},{"cell_type":"code","source":"train['OrganizationIdentifier'] = train['OrganizationIdentifier'].astype('category')\ntest['OrganizationIdentifier'] = test['OrganizationIdentifier'].astype('category')","metadata":{"execution":{"iopub.status.busy":"2022-08-05T09:45:16.805016Z","iopub.execute_input":"2022-08-05T09:45:16.806084Z","iopub.status.idle":"2022-08-05T09:45:17.567654Z","shell.execute_reply.started":"2022-08-05T09:45:16.806040Z","shell.execute_reply":"2022-08-05T09:45:17.566712Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_categorical_feature('OrganizationIdentifier', True, by_touch=True)","metadata":{"execution":{"iopub.status.busy":"2022-08-05T09:45:17.576795Z","iopub.execute_input":"2022-08-05T09:45:17.577826Z","iopub.status.idle":"2022-08-05T09:45:24.863171Z","shell.execute_reply.started":"2022-08-05T09:45:17.577780Z","shell.execute_reply":"2022-08-05T09:45:24.862129Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**GeoNameIdentifier**","metadata":{}},{"cell_type":"code","source":"plot_categorical_feature('GeoNameIdentifier', True)","metadata":{"execution":{"iopub.status.busy":"2022-08-05T09:45:24.864517Z","iopub.execute_input":"2022-08-05T09:45:24.865579Z","iopub.status.idle":"2022-08-05T09:45:27.499908Z","shell.execute_reply.started":"2022-08-05T09:45:24.865541Z","shell.execute_reply":"2022-08-05T09:45:27.498698Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['GeoNameIdentifier'] = train['GeoNameIdentifier'].astype('category')\ntest['GeoNameIdentifier'] = test['GeoNameIdentifier'].astype('category')","metadata":{"execution":{"iopub.status.busy":"2022-08-05T09:45:27.501572Z","iopub.execute_input":"2022-08-05T09:45:27.502023Z","iopub.status.idle":"2022-08-05T09:45:28.099784Z","shell.execute_reply.started":"2022-08-05T09:45:27.501971Z","shell.execute_reply":"2022-08-05T09:45:28.098330Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**LocaleEnglishNameIdentifier**","metadata":{}},{"cell_type":"code","source":"plot_categorical_feature('LocaleEnglishNameIdentifier', True)","metadata":{"execution":{"iopub.status.busy":"2022-08-05T09:45:28.101283Z","iopub.execute_input":"2022-08-05T09:45:28.101681Z","iopub.status.idle":"2022-08-05T09:45:28.663315Z","shell.execute_reply.started":"2022-08-05T09:45:28.101647Z","shell.execute_reply":"2022-08-05T09:45:28.662176Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['LocaleEnglishNameIdentifier'] = train['LocaleEnglishNameIdentifier'].astype('category')\ntest['LocaleEnglishNameIdentifier'] = test['LocaleEnglishNameIdentifier'].astype('category')","metadata":{"execution":{"iopub.status.busy":"2022-08-05T09:45:28.664987Z","iopub.execute_input":"2022-08-05T09:45:28.665460Z","iopub.status.idle":"2022-08-05T09:45:28.962589Z","shell.execute_reply.started":"2022-08-05T09:45:28.665415Z","shell.execute_reply":"2022-08-05T09:45:28.961410Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**OsPlatformSubRelease**","metadata":{}},{"cell_type":"code","source":"plot_categorical_feature('OsPlatformSubRelease', True, by_touch=True)","metadata":{"execution":{"iopub.status.busy":"2022-08-05T09:45:28.964045Z","iopub.execute_input":"2022-08-05T09:45:28.964446Z","iopub.status.idle":"2022-08-05T09:45:36.335104Z","shell.execute_reply.started":"2022-08-05T09:45:28.964413Z","shell.execute_reply":"2022-08-05T09:45:36.333566Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**OsBuildLab**","metadata":{}},{"cell_type":"code","source":"plot_categorical_feature('OsBuildLab', True)","metadata":{"execution":{"iopub.status.busy":"2022-08-05T09:45:36.337517Z","iopub.execute_input":"2022-08-05T09:45:36.337924Z","iopub.status.idle":"2022-08-05T09:45:36.997626Z","shell.execute_reply.started":"2022-08-05T09:45:36.337889Z","shell.execute_reply":"2022-08-05T09:45:36.996166Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**IeVerIdentifier**","metadata":{}},{"cell_type":"code","source":"plot_categorical_feature('IeVerIdentifier', True)","metadata":{"execution":{"iopub.status.busy":"2022-08-05T09:45:36.999236Z","iopub.execute_input":"2022-08-05T09:45:36.999633Z","iopub.status.idle":"2022-08-05T09:45:39.888962Z","shell.execute_reply.started":"2022-08-05T09:45:36.999600Z","shell.execute_reply":"2022-08-05T09:45:39.887814Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['IeVerIdentifier'] = train['IeVerIdentifier'].astype('category')\ntest['IeVerIdentifier'] = test['IeVerIdentifier'].astype('category')","metadata":{"execution":{"iopub.status.busy":"2022-08-05T09:45:39.890786Z","iopub.execute_input":"2022-08-05T09:45:39.891255Z","iopub.status.idle":"2022-08-05T09:45:40.470682Z","shell.execute_reply.started":"2022-08-05T09:45:39.891202Z","shell.execute_reply":"2022-08-05T09:45:40.469560Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Census_OEMNameIdentifier**","metadata":{}},{"cell_type":"code","source":"plot_categorical_feature('Census_OEMNameIdentifier', True)","metadata":{"execution":{"iopub.status.busy":"2022-08-05T09:45:40.471938Z","iopub.execute_input":"2022-08-05T09:45:40.472269Z","iopub.status.idle":"2022-08-05T09:45:43.408173Z","shell.execute_reply.started":"2022-08-05T09:45:40.472240Z","shell.execute_reply":"2022-08-05T09:45:43.407012Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['Census_OEMNameIdentifier'] = train['Census_OEMNameIdentifier'].astype('category')\ntest['Census_OEMNameIdentifier'] = test['Census_OEMNameIdentifier'].astype('category')","metadata":{"execution":{"iopub.status.busy":"2022-08-05T09:45:43.409817Z","iopub.execute_input":"2022-08-05T09:45:43.410148Z","iopub.status.idle":"2022-08-05T09:45:43.992741Z","shell.execute_reply.started":"2022-08-05T09:45:43.410120Z","shell.execute_reply":"2022-08-05T09:45:43.991418Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Census_OEMModelIdentifier**","metadata":{}},{"cell_type":"code","source":"plot_categorical_feature('Census_OEMModelIdentifier', True)","metadata":{"execution":{"iopub.status.busy":"2022-08-05T09:45:43.994487Z","iopub.execute_input":"2022-08-05T09:45:43.995426Z","iopub.status.idle":"2022-08-05T09:45:45.776679Z","shell.execute_reply.started":"2022-08-05T09:45:43.995386Z","shell.execute_reply":"2022-08-05T09:45:45.775351Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['Census_OEMModelIdentifier'] = train['Census_OEMModelIdentifier'].astype('category')\ntest['Census_OEMModelIdentifier'] = test['Census_OEMModelIdentifier'].astype('category')","metadata":{"execution":{"iopub.status.busy":"2022-08-05T09:45:45.778425Z","iopub.execute_input":"2022-08-05T09:45:45.778823Z","iopub.status.idle":"2022-08-05T09:45:46.715746Z","shell.execute_reply.started":"2022-08-05T09:45:45.778789Z","shell.execute_reply":"2022-08-05T09:45:46.714599Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Census_ProcessorCoreCount**","metadata":{}},{"cell_type":"code","source":"plot_categorical_feature('Census_ProcessorCoreCount', True, by_touch=True)","metadata":{"execution":{"iopub.status.busy":"2022-08-05T09:45:46.717433Z","iopub.execute_input":"2022-08-05T09:45:46.718516Z","iopub.status.idle":"2022-08-05T09:45:55.368898Z","shell.execute_reply.started":"2022-08-05T09:45:46.718463Z","shell.execute_reply":"2022-08-05T09:45:55.367408Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Most computers have 2, 4 or 8 cores. For touch devices 4 cores are much more common than other configurations. And these 3 variants cover 95% of all samples.","metadata":{}},{"cell_type":"markdown","source":"**Census_ProcessorModelIdentifier**","metadata":{}},{"cell_type":"code","source":"plot_categorical_feature('Census_ProcessorModelIdentifier', True)","metadata":{"execution":{"iopub.status.busy":"2022-08-05T09:45:55.370822Z","iopub.execute_input":"2022-08-05T09:45:55.372513Z","iopub.status.idle":"2022-08-05T09:45:57.926034Z","shell.execute_reply.started":"2022-08-05T09:45:55.372449Z","shell.execute_reply":"2022-08-05T09:45:57.924549Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['Census_ProcessorModelIdentifier'] = train['Census_ProcessorModelIdentifier'].astype('category')\ntest['Census_ProcessorModelIdentifier'] = test['Census_ProcessorModelIdentifier'].astype('category')","metadata":{"execution":{"iopub.status.busy":"2022-08-05T09:45:57.927749Z","iopub.execute_input":"2022-08-05T09:45:57.928135Z","iopub.status.idle":"2022-08-05T09:45:58.476277Z","shell.execute_reply.started":"2022-08-05T09:45:57.928078Z","shell.execute_reply":"2022-08-05T09:45:58.475169Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Census_PrimaryDiskTotalCapacity**","metadata":{}},{"cell_type":"code","source":"plot_categorical_feature('Census_PrimaryDiskTotalCapacity', True)","metadata":{"execution":{"iopub.status.busy":"2022-08-05T09:45:58.477881Z","iopub.execute_input":"2022-08-05T09:45:58.478699Z","iopub.status.idle":"2022-08-05T09:45:59.407513Z","shell.execute_reply.started":"2022-08-05T09:45:58.478652Z","shell.execute_reply":"2022-08-05T09:45:59.406263Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Census_SystemVolumeTotalCapacity**","metadata":{}},{"cell_type":"code","source":"plot_categorical_feature('Census_SystemVolumeTotalCapacity', True)","metadata":{"execution":{"iopub.status.busy":"2022-08-05T09:45:59.409089Z","iopub.execute_input":"2022-08-05T09:45:59.410307Z","iopub.status.idle":"2022-08-05T09:46:01.739507Z","shell.execute_reply.started":"2022-08-05T09:45:59.410254Z","shell.execute_reply":"2022-08-05T09:46:01.738089Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"It is a numerical variable, ```Census_PrimaryDiskTotalCapacity``` could be numerical but it has too little unique values & every category should be considered as a seperate disk model.","metadata":{}},{"cell_type":"markdown","source":"**Census_TotalPhysicalRAM**","metadata":{}},{"cell_type":"code","source":"plot_categorical_feature('Census_TotalPhysicalRAM', True, by_touch=True)","metadata":{"execution":{"iopub.status.busy":"2022-08-05T09:46:01.741201Z","iopub.execute_input":"2022-08-05T09:46:01.741571Z","iopub.status.idle":"2022-08-05T09:46:09.158019Z","shell.execute_reply.started":"2022-08-05T09:46:01.741539Z","shell.execute_reply":"2022-08-05T09:46:09.156603Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Most computers have <=8 Gb RAM","metadata":{}},{"cell_type":"markdown","source":"**Census_InternalPrimaryDiagonalDisplaySizeInInches**","metadata":{}},{"cell_type":"code","source":"plot_categorical_feature('Census_InternalPrimaryDiagonalDisplaySizeInInches', True, by_touch=True)","metadata":{"execution":{"iopub.status.busy":"2022-08-05T09:46:09.159823Z","iopub.execute_input":"2022-08-05T09:46:09.160187Z","iopub.status.idle":"2022-08-05T09:46:16.570977Z","shell.execute_reply.started":"2022-08-05T09:46:09.160156Z","shell.execute_reply":"2022-08-05T09:46:16.569737Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Most computers have 15 inch screens. And this is a rare situation, when for some categories detection rate on PC is higher than on touch devices.","metadata":{}},{"cell_type":"markdown","source":"**Census_InternalPrimaryDisplayResolutionHorizontal**","metadata":{}},{"cell_type":"code","source":"plot_categorical_feature('Census_InternalPrimaryDisplayResolutionHorizontal', True, by_touch=True)","metadata":{"execution":{"iopub.status.busy":"2022-08-05T09:46:16.572863Z","iopub.execute_input":"2022-08-05T09:46:16.573268Z","iopub.status.idle":"2022-08-05T09:46:24.951027Z","shell.execute_reply.started":"2022-08-05T09:46:16.573234Z","shell.execute_reply":"2022-08-05T09:46:24.949594Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Census_InternalPrimaryDisplayResolutionVertical**","metadata":{}},{"cell_type":"code","source":"plot_categorical_feature('Census_InternalPrimaryDisplayResolutionVertical', True, by_touch=True)","metadata":{"execution":{"iopub.status.busy":"2022-08-05T09:46:24.952819Z","iopub.execute_input":"2022-08-05T09:46:24.953278Z","iopub.status.idle":"2022-08-05T09:46:33.302663Z","shell.execute_reply.started":"2022-08-05T09:46:24.953240Z","shell.execute_reply":"2022-08-05T09:46:33.301121Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Census_InternalBatteryNumberOfCharges**","metadata":{}},{"cell_type":"code","source":"plot_categorical_feature('Census_InternalBatteryNumberOfCharges', True, by_touch=True)","metadata":{"execution":{"iopub.status.busy":"2022-08-05T09:46:33.305071Z","iopub.execute_input":"2022-08-05T09:46:33.305797Z","iopub.status.idle":"2022-08-05T09:46:40.583615Z","shell.execute_reply.started":"2022-08-05T09:46:33.305758Z","shell.execute_reply":"2022-08-05T09:46:40.582172Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"This is a categorical variable. It is expected that most PCs have 0 charges. 4.294967e+09 seems to be some kind of technical value. But having several charges for PC is strange.","metadata":{}},{"cell_type":"code","source":"train['Census_InternalBatteryNumberOfCharges'] = train['Census_InternalBatteryNumberOfCharges'].astype('category')\ntest['Census_InternalBatteryNumberOfCharges'] = test['Census_InternalBatteryNumberOfCharges'].astype('category')","metadata":{"execution":{"iopub.status.busy":"2022-08-05T09:46:40.585671Z","iopub.execute_input":"2022-08-05T09:46:40.586166Z","iopub.status.idle":"2022-08-05T09:46:41.120031Z","shell.execute_reply.started":"2022-08-05T09:46:40.586120Z","shell.execute_reply":"2022-08-05T09:46:41.118677Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Census_OSVersion**","metadata":{}},{"cell_type":"code","source":"plot_categorical_feature('Census_OSVersion', True)","metadata":{"execution":{"iopub.status.busy":"2022-08-05T09:46:41.121574Z","iopub.execute_input":"2022-08-05T09:46:41.121918Z","iopub.status.idle":"2022-08-05T09:46:41.657779Z","shell.execute_reply.started":"2022-08-05T09:46:41.121887Z","shell.execute_reply":"2022-08-05T09:46:41.656242Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Census_OSBranch**","metadata":{}},{"cell_type":"code","source":"plot_categorical_feature('Census_OSBranch', True)","metadata":{"execution":{"iopub.status.busy":"2022-08-05T09:46:41.659215Z","iopub.execute_input":"2022-08-05T09:46:41.659570Z","iopub.status.idle":"2022-08-05T09:46:42.240779Z","shell.execute_reply.started":"2022-08-05T09:46:41.659540Z","shell.execute_reply":"2022-08-05T09:46:42.239275Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Census_OSBuildNumber**","metadata":{}},{"cell_type":"code","source":"plot_categorical_feature('Census_OSBuildNumber', True)","metadata":{"execution":{"iopub.status.busy":"2022-08-05T09:46:42.242771Z","iopub.execute_input":"2022-08-05T09:46:42.243306Z","iopub.status.idle":"2022-08-05T09:46:42.874460Z","shell.execute_reply.started":"2022-08-05T09:46:42.243251Z","shell.execute_reply":"2022-08-05T09:46:42.873186Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['Census_OSBuildNumber'] = train['Census_OSBuildNumber'].astype('category')\ntest['Census_OSBuildNumber'] = test['Census_OSBuildNumber'].astype('category')","metadata":{"execution":{"iopub.status.busy":"2022-08-05T09:46:42.875857Z","iopub.execute_input":"2022-08-05T09:46:42.876239Z","iopub.status.idle":"2022-08-05T09:46:43.138046Z","shell.execute_reply.started":"2022-08-05T09:46:42.876206Z","shell.execute_reply":"2022-08-05T09:46:43.136884Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Census_OSBuildRevision**","metadata":{}},{"cell_type":"code","source":"plot_categorical_feature('Census_OSBuildRevision', True)","metadata":{"execution":{"iopub.status.busy":"2022-08-05T09:46:43.139472Z","iopub.execute_input":"2022-08-05T09:46:43.139796Z","iopub.status.idle":"2022-08-05T09:46:43.692033Z","shell.execute_reply.started":"2022-08-05T09:46:43.139766Z","shell.execute_reply":"2022-08-05T09:46:43.690858Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['Census_OSBuildRevision'] = train['Census_OSBuildRevision'].astype('category')\ntest['Census_OSBuildRevision'] = test['Census_OSBuildRevision'].astype('category')","metadata":{"execution":{"iopub.status.busy":"2022-08-05T09:46:43.693723Z","iopub.execute_input":"2022-08-05T09:46:43.694086Z","iopub.status.idle":"2022-08-05T09:46:43.975261Z","shell.execute_reply.started":"2022-08-05T09:46:43.694051Z","shell.execute_reply":"2022-08-05T09:46:43.973915Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Census_FirmwareManufacturerIdentifier**","metadata":{}},{"cell_type":"code","source":"plot_categorical_feature('Census_FirmwareManufacturerIdentifier', True)","metadata":{"execution":{"iopub.status.busy":"2022-08-05T09:46:43.977178Z","iopub.execute_input":"2022-08-05T09:46:43.978334Z","iopub.status.idle":"2022-08-05T09:46:47.017541Z","shell.execute_reply.started":"2022-08-05T09:46:43.978290Z","shell.execute_reply":"2022-08-05T09:46:47.016342Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['Census_FirmwareManufacturerIdentifier'] = train['Census_FirmwareManufacturerIdentifier'].astype('category')\ntest['Census_FirmwareManufacturerIdentifier'] = test['Census_FirmwareManufacturerIdentifier'].astype('category')","metadata":{"execution":{"iopub.status.busy":"2022-08-05T09:46:47.019306Z","iopub.execute_input":"2022-08-05T09:46:47.019671Z","iopub.status.idle":"2022-08-05T09:46:47.570079Z","shell.execute_reply.started":"2022-08-05T09:46:47.019637Z","shell.execute_reply":"2022-08-05T09:46:47.568427Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Census_FirmwareVersionIdentifier**","metadata":{}},{"cell_type":"code","source":"plot_categorical_feature('Census_FirmwareVersionIdentifier', True)","metadata":{"execution":{"iopub.status.busy":"2022-08-05T09:46:47.571647Z","iopub.execute_input":"2022-08-05T09:46:47.572560Z","iopub.status.idle":"2022-08-05T09:46:49.071801Z","shell.execute_reply.started":"2022-08-05T09:46:47.572523Z","shell.execute_reply":"2022-08-05T09:46:49.070518Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['Census_FirmwareVersionIdentifier'] = train['Census_FirmwareVersionIdentifier'].astype('category')\ntest['Census_FirmwareVersionIdentifier'] = test['Census_FirmwareVersionIdentifier'].astype('category')","metadata":{"execution":{"iopub.status.busy":"2022-08-05T09:46:49.073657Z","iopub.execute_input":"2022-08-05T09:46:49.074146Z","iopub.status.idle":"2022-08-05T09:46:49.828003Z","shell.execute_reply.started":"2022-08-05T09:46:49.074081Z","shell.execute_reply":"2022-08-05T09:46:49.826676Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**OsBuild**","metadata":{}},{"cell_type":"code","source":"plot_categorical_feature('OsBuild', True)","metadata":{"execution":{"iopub.status.busy":"2022-08-05T09:46:49.829391Z","iopub.execute_input":"2022-08-05T09:46:49.829730Z","iopub.status.idle":"2022-08-05T09:46:50.493417Z","shell.execute_reply.started":"2022-08-05T09:46:49.829700Z","shell.execute_reply":"2022-08-05T09:46:50.492115Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['OsBuild'] = train['OsBuild'].astype('category')\ntest['OsBuild'] = test['OsBuild'].astype('category')","metadata":{"execution":{"iopub.status.busy":"2022-08-05T09:46:50.494863Z","iopub.execute_input":"2022-08-05T09:46:50.495253Z","iopub.status.idle":"2022-08-05T09:46:50.796249Z","shell.execute_reply.started":"2022-08-05T09:46:50.495221Z","shell.execute_reply":"2022-08-05T09:46:50.795009Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Census_ChassisTypeName**","metadata":{}},{"cell_type":"code","source":"plot_categorical_feature('Census_ChassisTypeName', True, by_touch=True)","metadata":{"execution":{"iopub.status.busy":"2022-08-05T09:46:50.797532Z","iopub.execute_input":"2022-08-05T09:46:50.797877Z","iopub.status.idle":"2022-08-05T09:46:57.907952Z","shell.execute_reply.started":"2022-08-05T09:46:50.797846Z","shell.execute_reply":"2022-08-05T09:46:57.906611Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Census_InternalBatteryType**","metadata":{}},{"cell_type":"code","source":"def group_battery(x):\n x = x.lower()\n if 'li' in x:\n return 1\n else:\n return 0\n \ntrain['Census_InternalBatteryType'] = train['Census_InternalBatteryType'].apply(group_battery)\ntest['Census_InternalBatteryType'] = test['Census_InternalBatteryType'].apply(group_battery)","metadata":{"execution":{"iopub.status.busy":"2022-08-05T09:46:57.911729Z","iopub.execute_input":"2022-08-05T09:46:57.912233Z","iopub.status.idle":"2022-08-05T09:46:58.211960Z","shell.execute_reply.started":"2022-08-05T09:46:57.912195Z","shell.execute_reply":"2022-08-05T09:46:58.210419Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_categorical_feature('Census_InternalBatteryType', True)","metadata":{"execution":{"iopub.status.busy":"2022-08-05T09:46:58.214270Z","iopub.execute_input":"2022-08-05T09:46:58.214668Z","iopub.status.idle":"2022-08-05T09:46:59.129519Z","shell.execute_reply.started":"2022-08-05T09:46:58.214635Z","shell.execute_reply":"2022-08-05T09:46:59.128245Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Census_OSEdition**\n\nCombining similar versions into one.","metadata":{}},{"cell_type":"code","source":"def rename_edition(x):\n x = x.lower()\n if 'core' in x:\n return 'Core'\n elif 'pro' in x:\n return 'pro'\n elif 'enterprise' in x:\n return 'Enterprise'\n elif 'server' in x:\n return 'Server'\n elif 'home' in x:\n return 'Home'\n elif 'education' in x:\n return 'Education'\n elif 'cloud' in x:\n return 'Cloud'\n else:\n return x","metadata":{"execution":{"iopub.status.busy":"2022-08-05T09:46:59.131274Z","iopub.execute_input":"2022-08-05T09:46:59.131632Z","iopub.status.idle":"2022-08-05T09:46:59.139576Z","shell.execute_reply.started":"2022-08-05T09:46:59.131601Z","shell.execute_reply":"2022-08-05T09:46:59.138180Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['Census_OSEdition'] = train['Census_OSEdition'].astype(str)\ntest['Census_OSEdition'] = test['Census_OSEdition'].astype(str)\ntrain['Census_OSEdition'] = train['Census_OSEdition'].apply(rename_edition)\ntest['Census_OSEdition'] = test['Census_OSEdition'].apply(rename_edition)\ntrain['Census_OSEdition'] = train['Census_OSEdition'].astype('category')\ntest['Census_OSEdition'] = test['Census_OSEdition'].astype('category')","metadata":{"execution":{"iopub.status.busy":"2022-08-05T09:46:59.141535Z","iopub.execute_input":"2022-08-05T09:46:59.141929Z","iopub.status.idle":"2022-08-05T09:47:12.391673Z","shell.execute_reply.started":"2022-08-05T09:46:59.141883Z","shell.execute_reply":"2022-08-05T09:47:12.390343Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Census_OSEdition**","metadata":{}},{"cell_type":"code","source":"plot_categorical_feature('Census_OSEdition', True, by_touch=True)","metadata":{"execution":{"iopub.status.busy":"2022-08-05T09:47:12.393184Z","iopub.execute_input":"2022-08-05T09:47:12.393563Z","iopub.status.idle":"2022-08-05T09:47:19.835277Z","shell.execute_reply.started":"2022-08-05T09:47:12.393529Z","shell.execute_reply":"2022-08-05T09:47:19.834118Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Census_OSSkuName**","metadata":{}},{"cell_type":"code","source":"train['Census_OSSkuName'] = train['Census_OSSkuName'].astype(str)\ntest['Census_OSSkuName'] = test['Census_OSSkuName'].astype(str)\ntrain['Census_OSSkuName'] = train['Census_OSSkuName'].apply(rename_edition)\ntest['Census_OSSkuName'] = test['Census_OSSkuName'].apply(rename_edition)\ntrain['Census_OSSkuName'] = train['Census_OSSkuName'].astype('category')\ntest['Census_OSSkuName'] = test['Census_OSSkuName'].astype('category')","metadata":{"execution":{"iopub.status.busy":"2022-08-05T09:47:19.837028Z","iopub.execute_input":"2022-08-05T09:47:19.837401Z","iopub.status.idle":"2022-08-05T09:47:36.214549Z","shell.execute_reply.started":"2022-08-05T09:47:19.837370Z","shell.execute_reply":"2022-08-05T09:47:36.213144Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Census_OSSkuName**","metadata":{}},{"cell_type":"code","source":"plot_categorical_feature('Census_OSSkuName', True, by_touch=True)","metadata":{"execution":{"iopub.status.busy":"2022-08-05T09:47:36.216313Z","iopub.execute_input":"2022-08-05T09:47:36.216683Z","iopub.status.idle":"2022-08-05T09:47:43.372298Z","shell.execute_reply.started":"2022-08-05T09:47:36.216650Z","shell.execute_reply":"2022-08-05T09:47:43.370988Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Census_OSInstallLanguageIdentifier**","metadata":{}},{"cell_type":"code","source":"plot_categorical_feature('Census_OSInstallLanguageIdentifier', True)","metadata":{"execution":{"iopub.status.busy":"2022-08-05T09:47:43.374219Z","iopub.execute_input":"2022-08-05T09:47:43.374914Z","iopub.status.idle":"2022-08-05T09:47:46.175965Z","shell.execute_reply.started":"2022-08-05T09:47:43.374866Z","shell.execute_reply":"2022-08-05T09:47:46.174800Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['Census_OSInstallLanguageIdentifier'] = train['Census_OSInstallLanguageIdentifier'].astype('category')\ntest['Census_OSInstallLanguageIdentifier'] = test['Census_OSInstallLanguageIdentifier'].astype('category')","metadata":{"execution":{"iopub.status.busy":"2022-08-05T09:47:46.177785Z","iopub.execute_input":"2022-08-05T09:47:46.178515Z","iopub.status.idle":"2022-08-05T09:47:46.712996Z","shell.execute_reply.started":"2022-08-05T09:47:46.178474Z","shell.execute_reply":"2022-08-05T09:47:46.711868Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Census_OSUILocaleIdentifier**","metadata":{}},{"cell_type":"code","source":"plot_categorical_feature('Census_OSUILocaleIdentifier', True)","metadata":{"execution":{"iopub.status.busy":"2022-08-05T09:47:46.714306Z","iopub.execute_input":"2022-08-05T09:47:46.715286Z","iopub.status.idle":"2022-08-05T09:47:47.313943Z","shell.execute_reply.started":"2022-08-05T09:47:46.715249Z","shell.execute_reply":"2022-08-05T09:47:47.312310Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['Census_OSUILocaleIdentifier'] = train['Census_OSUILocaleIdentifier'].astype('category')\ntest['Census_OSUILocaleIdentifier'] = test['Census_OSUILocaleIdentifier'].astype('category')","metadata":{"execution":{"iopub.status.busy":"2022-08-05T09:47:47.315560Z","iopub.execute_input":"2022-08-05T09:47:47.316430Z","iopub.status.idle":"2022-08-05T09:47:47.608123Z","shell.execute_reply.started":"2022-08-05T09:47:47.316390Z","shell.execute_reply":"2022-08-05T09:47:47.606935Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_categorical_feature('Census_OSUILocaleIdentifier', True)","metadata":{"execution":{"iopub.status.busy":"2022-08-05T09:47:47.609797Z","iopub.execute_input":"2022-08-05T09:47:47.610275Z","iopub.status.idle":"2022-08-05T09:47:48.231734Z","shell.execute_reply.started":"2022-08-05T09:47:47.610227Z","shell.execute_reply":"2022-08-05T09:47:48.230263Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**OsSuite**","metadata":{}},{"cell_type":"code","source":"plot_categorical_feature('OsSuite', True)","metadata":{"execution":{"iopub.status.busy":"2022-08-05T09:47:48.233077Z","iopub.execute_input":"2022-08-05T09:47:48.233550Z","iopub.status.idle":"2022-08-05T09:47:48.888831Z","shell.execute_reply.started":"2022-08-05T09:47:48.233516Z","shell.execute_reply":"2022-08-05T09:47:48.887492Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['OsSuite'] = train['OsSuite'].astype('category')\ntest['OsSuite'] = test['OsSuite'].astype('category')","metadata":{"execution":{"iopub.status.busy":"2022-08-05T09:47:48.890321Z","iopub.execute_input":"2022-08-05T09:47:48.890657Z","iopub.status.idle":"2022-08-05T09:47:49.166965Z","shell.execute_reply.started":"2022-08-05T09:47:48.890627Z","shell.execute_reply":"2022-08-05T09:47:49.165635Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Wdft_RegionIdentifier**","metadata":{}},{"cell_type":"code","source":"plot_categorical_feature('Wdft_RegionIdentifier', True)","metadata":{"execution":{"iopub.status.busy":"2022-08-05T09:47:49.168888Z","iopub.execute_input":"2022-08-05T09:47:49.169374Z","iopub.status.idle":"2022-08-05T09:47:52.236441Z","shell.execute_reply.started":"2022-08-05T09:47:49.169327Z","shell.execute_reply":"2022-08-05T09:47:52.235319Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['Wdft_RegionIdentifier'] = train['Wdft_RegionIdentifier'].astype('category')\ntest['Wdft_RegionIdentifier'] = test['Wdft_RegionIdentifier'].astype('category')","metadata":{"execution":{"iopub.status.busy":"2022-08-05T09:47:52.238008Z","iopub.execute_input":"2022-08-05T09:47:52.238382Z","iopub.status.idle":"2022-08-05T09:47:52.761577Z","shell.execute_reply.started":"2022-08-05T09:47:52.238351Z","shell.execute_reply":"2022-08-05T09:47:52.760297Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**SkuEdition**","metadata":{}},{"cell_type":"code","source":"train['SkuEdition'].value_counts(dropna=False, normalize=True)","metadata":{"execution":{"iopub.status.busy":"2022-08-05T09:47:52.763110Z","iopub.execute_input":"2022-08-05T09:47:52.763556Z","iopub.status.idle":"2022-08-05T09:47:52.834200Z","shell.execute_reply.started":"2022-08-05T09:47:52.763525Z","shell.execute_reply":"2022-08-05T09:47:52.832850Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Home and Pro editions together give 97.9%+ of all values. Condidering that other categories are for Enterprise mostly, combining them with Pro.","metadata":{}},{"cell_type":"code","source":"pd.crosstab(train['SkuEdition'], train['Census_OSEdition'], normalize='columns')","metadata":{"execution":{"iopub.status.busy":"2022-08-05T09:47:52.846621Z","iopub.execute_input":"2022-08-05T09:47:52.847012Z","iopub.status.idle":"2022-08-05T09:47:53.631655Z","shell.execute_reply.started":"2022-08-05T09:47:52.846982Z","shell.execute_reply":"2022-08-05T09:47:53.630487Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"It seems Home Sku edition corresponds to Core OS Edition & Pro Sku edition corresponds to all other OS editions.","metadata":{}},{"cell_type":"markdown","source":"**SmartScreen**","metadata":{}},{"cell_type":"code","source":"train['SmartScreen'].value_counts(dropna=False, normalize=True).cumsum()","metadata":{"execution":{"iopub.status.busy":"2022-08-05T09:47:53.633250Z","iopub.execute_input":"2022-08-05T09:47:53.634460Z","iopub.status.idle":"2022-08-05T09:47:53.747644Z","shell.execute_reply.started":"2022-08-05T09:47:53.634419Z","shell.execute_reply":"2022-08-05T09:47:53.746239Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.set(rc={'figure.figsize':(15, 8)})\nsns.countplot(x=\"SmartScreen\", hue=\"HasDetections\", palette=\"PRGn\", data=train)\nplt.title(\"SmartScreen counts\")\nplt.xticks(rotation='vertical')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-08-05T09:47:53.749207Z","iopub.execute_input":"2022-08-05T09:47:53.749588Z","iopub.status.idle":"2022-08-05T09:47:55.096927Z","shell.execute_reply.started":"2022-08-05T09:47:53.749556Z","shell.execute_reply":"2022-08-05T09:47:55.095571Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Census_MDC2FormFactor**","metadata":{}},{"cell_type":"code","source":"train['Census_MDC2FormFactor'].value_counts(dropna=False, normalize=True).cumsum()","metadata":{"execution":{"iopub.status.busy":"2022-08-05T09:47:55.098630Z","iopub.execute_input":"2022-08-05T09:47:55.099012Z","iopub.status.idle":"2022-08-05T09:47:55.171913Z","shell.execute_reply.started":"2022-08-05T09:47:55.098979Z","shell.execute_reply":"2022-08-05T09:47:55.170599Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_categorical_feature('Census_MDC2FormFactor', True)","metadata":{"execution":{"iopub.status.busy":"2022-08-05T09:47:55.173663Z","iopub.execute_input":"2022-08-05T09:47:55.174205Z","iopub.status.idle":"2022-08-05T09:47:55.794838Z","shell.execute_reply.started":"2022-08-05T09:47:55.174152Z","shell.execute_reply":"2022-08-05T09:47:55.793572Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.catplot(x=\"Census_PrimaryDiskTypeName\", hue=\"HasDetections\", col=\"Census_MDC2FormFactor\",\n data=train, kind=\"count\",col_wrap=3)","metadata":{"execution":{"iopub.status.busy":"2022-08-05T09:47:55.796740Z","iopub.execute_input":"2022-08-05T09:47:55.797429Z","iopub.status.idle":"2022-08-05T09:48:41.151480Z","shell.execute_reply.started":"2022-08-05T09:47:55.797391Z","shell.execute_reply":"2022-08-05T09:48:41.150320Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['Census_PrimaryDiskTypeName'].value_counts(dropna=False, normalize=True).cumsum()","metadata":{"execution":{"iopub.status.busy":"2022-08-05T09:48:41.153088Z","iopub.execute_input":"2022-08-05T09:48:41.153686Z","iopub.status.idle":"2022-08-05T09:48:41.238372Z","shell.execute_reply.started":"2022-08-05T09:48:41.153649Z","shell.execute_reply":"2022-08-05T09:48:41.236536Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Census_ProcessorManufacturerIdentifier**","metadata":{}},{"cell_type":"code","source":"train['Census_ProcessorManufacturerIdentifier'].value_counts(dropna=False, normalize=True).cumsum()","metadata":{"execution":{"iopub.status.busy":"2022-08-05T09:48:41.240841Z","iopub.execute_input":"2022-08-05T09:48:41.241291Z","iopub.status.idle":"2022-08-05T09:48:41.433641Z","shell.execute_reply.started":"2022-08-05T09:48:41.241254Z","shell.execute_reply":"2022-08-05T09:48:41.432247Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_categorical_feature('Census_ProcessorManufacturerIdentifier', True)","metadata":{"execution":{"iopub.status.busy":"2022-08-05T09:48:41.435429Z","iopub.execute_input":"2022-08-05T09:48:41.436154Z","iopub.status.idle":"2022-08-05T09:48:43.912587Z","shell.execute_reply.started":"2022-08-05T09:48:41.436086Z","shell.execute_reply":"2022-08-05T09:48:43.911410Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Census_PowerPlatformRoleName**","metadata":{}},{"cell_type":"code","source":"train['Census_PowerPlatformRoleName'].value_counts(dropna=False, normalize=True).cumsum()","metadata":{"execution":{"iopub.status.busy":"2022-08-05T09:48:43.914612Z","iopub.execute_input":"2022-08-05T09:48:43.915115Z","iopub.status.idle":"2022-08-05T09:48:43.995641Z","shell.execute_reply.started":"2022-08-05T09:48:43.915052Z","shell.execute_reply":"2022-08-05T09:48:43.994418Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_categorical_feature('Census_PowerPlatformRoleName', True)","metadata":{"execution":{"iopub.status.busy":"2022-08-05T09:48:43.999280Z","iopub.execute_input":"2022-08-05T09:48:43.999640Z","iopub.status.idle":"2022-08-05T09:48:44.712300Z","shell.execute_reply.started":"2022-08-05T09:48:43.999610Z","shell.execute_reply":"2022-08-05T09:48:44.711160Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Census_OSInstallTypeName**","metadata":{}},{"cell_type":"code","source":"plot_categorical_feature('Census_OSInstallTypeName', True)","metadata":{"execution":{"iopub.status.busy":"2022-08-05T09:48:44.713647Z","iopub.execute_input":"2022-08-05T09:48:44.713979Z","iopub.status.idle":"2022-08-05T09:48:45.395835Z","shell.execute_reply.started":"2022-08-05T09:48:44.713948Z","shell.execute_reply":"2022-08-05T09:48:45.394497Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### **Feature engineering and transformation**","metadata":{}},{"cell_type":"code","source":"train.head()","metadata":{"execution":{"iopub.status.busy":"2022-08-05T09:48:45.397460Z","iopub.execute_input":"2022-08-05T09:48:45.397808Z","iopub.status.idle":"2022-08-05T09:48:45.473324Z","shell.execute_reply.started":"2022-08-05T09:48:45.397778Z","shell.execute_reply":"2022-08-05T09:48:45.472169Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['OsBuildLab'] = train['OsBuildLab'].cat.add_categories(['0.0.0.0.0-0'])\ntrain['OsBuildLab'] = train['OsBuildLab'].fillna('0.0.0.0.0-0')\ntest['OsBuildLab'] = test['OsBuildLab'].cat.add_categories(['0.0.0.0.0-0'])\ntest['OsBuildLab'] = test['OsBuildLab'].fillna('0.0.0.0.0-0')","metadata":{"execution":{"iopub.status.busy":"2022-08-05T09:48:45.474766Z","iopub.execute_input":"2022-08-05T09:48:45.475177Z","iopub.status.idle":"2022-08-05T09:48:45.541144Z","shell.execute_reply.started":"2022-08-05T09:48:45.475145Z","shell.execute_reply":"2022-08-05T09:48:45.539904Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def fe(df):\n df['EngineVersion_2'] = df['EngineVersion'].apply(lambda x: x.split('.')[2]).astype('category')\n df['EngineVersion_3'] = df['EngineVersion'].apply(lambda x: x.split('.')[3]).astype('category')\n\n df['AppVersion_1'] = df['AppVersion'].apply(lambda x: x.split('.')[1]).astype('category')\n df['AppVersion_2'] = df['AppVersion'].apply(lambda x: x.split('.')[2]).astype('category')\n df['AppVersion_3'] = df['AppVersion'].apply(lambda x: x.split('.')[3]).astype('category')\n\n df['AvSigVersion_0'] = df['AvSigVersion'].apply(lambda x: x.split('.')[0]).astype('category')\n df['AvSigVersion_1'] = df['AvSigVersion'].apply(lambda x: x.split('.')[1]).astype('category')\n df['AvSigVersion_2'] = df['AvSigVersion'].apply(lambda x: x.split('.')[2]).astype('category')\n\n df['OsBuildLab_0'] = df['OsBuildLab'].apply(lambda x: x.split('.')[0]).astype('category')\n df['OsBuildLab_1'] = df['OsBuildLab'].apply(lambda x: x.split('.')[1]).astype('category')\n df['OsBuildLab_2'] = df['OsBuildLab'].apply(lambda x: x.split('.')[2]).astype('category')\n df['OsBuildLab_3'] = df['OsBuildLab'].apply(lambda x: x.split('.')[3]).astype('category')\n\n df['Census_OSVersion_0'] = df['Census_OSVersion'].apply(lambda x: x.split('.')[0]).astype('category')\n df['Census_OSVersion_1'] = df['Census_OSVersion'].apply(lambda x: x.split('.')[1]).astype('category')\n df['Census_OSVersion_2'] = df['Census_OSVersion'].apply(lambda x: x.split('.')[2]).astype('category')\n df['Census_OSVersion_3'] = df['Census_OSVersion'].apply(lambda x: x.split('.')[3]).astype('category')\n\n # https://www.kaggle.com/adityaecdrid/simple-feature-engineering-xd\n df['primary_drive_c_ratio'] = df['Census_SystemVolumeTotalCapacity']/ df['Census_PrimaryDiskTotalCapacity']\n df['non_primary_drive_MB'] = df['Census_PrimaryDiskTotalCapacity'] - df['Census_SystemVolumeTotalCapacity']\n\n df['aspect_ratio'] = df['Census_InternalPrimaryDisplayResolutionHorizontal']/ df['Census_InternalPrimaryDisplayResolutionVertical']\n\n df['monitor_dims'] = df['Census_InternalPrimaryDisplayResolutionHorizontal'].astype(str) + '*' + df['Census_InternalPrimaryDisplayResolutionVertical'].astype('str')\n df['monitor_dims'] = df['monitor_dims'].astype('category')\n\n df['dpi'] = ((df['Census_InternalPrimaryDisplayResolutionHorizontal']**2 + df['Census_InternalPrimaryDisplayResolutionVertical']**2)**.5)/(df['Census_InternalPrimaryDiagonalDisplaySizeInInches'])\n df['dpi_square'] = df['dpi'] ** 2\n df['MegaPixels'] = (df['Census_InternalPrimaryDisplayResolutionHorizontal'] * df['Census_InternalPrimaryDisplayResolutionVertical'])/1e6\n df['Screen_Area'] = (df['aspect_ratio']* (df['Census_InternalPrimaryDiagonalDisplaySizeInInches']**2))/(df['aspect_ratio']**2 + 1)\n df['ram_per_processor'] = df['Census_TotalPhysicalRAM']/ df['Census_ProcessorCoreCount']\n df['new_num_0'] = df['Census_InternalPrimaryDiagonalDisplaySizeInInches'] / df['Census_ProcessorCoreCount']\n df['new_num_1'] = df['Census_ProcessorCoreCount'] * df['Census_InternalPrimaryDiagonalDisplaySizeInInches']\n\n df['Census_IsFlightingInternal'] = df['Census_IsFlightingInternal'].fillna(1)\n df['Census_ThresholdOptIn'] = df['Census_ThresholdOptIn'].fillna(1)\n df['Census_IsWIMBootEnabled'] = df['Census_IsWIMBootEnabled'].fillna(1)\n df['Wdft_IsGamer'] = df['Wdft_IsGamer'].fillna(0)\n \n return df","metadata":{"execution":{"iopub.status.busy":"2022-08-05T09:48:45.543149Z","iopub.execute_input":"2022-08-05T09:48:45.543519Z","iopub.status.idle":"2022-08-05T09:48:45.565231Z","shell.execute_reply.started":"2022-08-05T09:48:45.543486Z","shell.execute_reply":"2022-08-05T09:48:45.564079Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = fe(train)\ntest = fe(test)","metadata":{"execution":{"iopub.status.busy":"2022-08-05T09:48:45.566271Z","iopub.execute_input":"2022-08-05T09:48:45.566638Z","iopub.status.idle":"2022-08-05T09:49:58.669548Z","shell.execute_reply.started":"2022-08-05T09:48:45.566605Z","shell.execute_reply":"2022-08-05T09:49:58.668175Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cat_cols = [col for col in train.columns if col not in ['MachineIdentifier', 'Census_SystemVolumeTotalCapacity', 'HasDetections'] and str(train[col].dtype) == 'category']\nlen(cat_cols)","metadata":{"execution":{"iopub.status.busy":"2022-08-05T09:49:58.671071Z","iopub.execute_input":"2022-08-05T09:49:58.671481Z","iopub.status.idle":"2022-08-05T09:49:58.684699Z","shell.execute_reply.started":"2022-08-05T09:49:58.671442Z","shell.execute_reply":"2022-08-05T09:49:58.683244Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"more_cat_cols = []\nadd_cat_feats = [\n 'Census_OSBuildRevision',\n 'OsBuildLab',\n 'SmartScreen',\n'AVProductsInstalled']\nfor col1 in add_cat_feats:\n for col2 in add_cat_feats:\n if col1 != col2:\n train[col1 + '__' + col2] = train[col1].astype(str) + train[col2].astype(str)\n train[col1 + '__' + col2] = train[col1 + '__' + col2].astype('category')\n \n test[col1 + '__' + col2] = test[col1].astype(str) + test[col2].astype(str)\n test[col1 + '__' + col2] = test[col1 + '__' + col2].astype('category')\n more_cat_cols.append(col1 + '__' + col2)\n \ncat_cols = cat_cols + more_cat_cols","metadata":{"execution":{"iopub.status.busy":"2022-08-05T09:49:58.686465Z","iopub.execute_input":"2022-08-05T09:49:58.687234Z","iopub.status.idle":"2022-08-05T09:52:55.002987Z","shell.execute_reply.started":"2022-08-05T09:49:58.687185Z","shell.execute_reply":"2022-08-05T09:52:55.001828Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"to_encode = []\nfor col in cat_cols:\n if train[col].nunique() > 1000:\n print(col, train[col].nunique())\n to_encode.append(col)","metadata":{"execution":{"iopub.status.busy":"2022-08-05T09:52:55.004947Z","iopub.execute_input":"2022-08-05T09:52:55.005821Z","iopub.status.idle":"2022-08-05T09:53:00.497723Z","shell.execute_reply.started":"2022-08-05T09:52:55.005771Z","shell.execute_reply":"2022-08-05T09:53:00.496384Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### **Frequency encoding**\n\nTo get more correct values.","metadata":{}},{"cell_type":"code","source":"train = reduce_mem_usage(train)\ntest = reduce_mem_usage(test)\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-08-05T09:53:00.499397Z","iopub.execute_input":"2022-08-05T09:53:00.499859Z","iopub.status.idle":"2022-08-05T09:53:19.987849Z","shell.execute_reply.started":"2022-08-05T09:53:00.499826Z","shell.execute_reply":"2022-08-05T09:53:19.986548Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def frequency_encoding(variable):\n t = pd.concat([train[variable], test[variable]]).value_counts().reset_index()\n t = t.reset_index()\n t.loc[t[variable] == 1, 'level_0'] = np.nan\n t.set_index('index', inplace=True)\n max_label = t['level_0'].max() + 1\n t.fillna(max_label, inplace=True)\n return t.to_dict()['level_0']","metadata":{"execution":{"iopub.status.busy":"2022-08-05T09:53:19.989112Z","iopub.execute_input":"2022-08-05T09:53:19.989484Z","iopub.status.idle":"2022-08-05T09:53:19.996431Z","shell.execute_reply.started":"2022-08-05T09:53:19.989454Z","shell.execute_reply":"2022-08-05T09:53:19.995477Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for col in tqdm_notebook(to_encode):\n freq_enc_dict = frequency_encoding(col)\n train[col] = train[col].map(lambda x: freq_enc_dict.get(x, np.nan))\n test[col] = test[col].map(lambda x: freq_enc_dict.get(x, np.nan))\n cat_cols.remove(col)","metadata":{"execution":{"iopub.status.busy":"2022-08-05T09:53:19.997753Z","iopub.execute_input":"2022-08-05T09:53:19.999417Z","iopub.status.idle":"2022-08-05T09:53:57.677677Z","shell.execute_reply.started":"2022-08-05T09:53:19.999368Z","shell.execute_reply":"2022-08-05T09:53:57.676270Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### **Label Encoding**","metadata":{}},{"cell_type":"code","source":"%%time\nindexer = {}\nfor col in cat_cols:\n # print(col)\n _, indexer[col] = pd.factorize(train[col].astype(str), sort=True)\n \nfor col in tqdm_notebook(cat_cols):\n # print(col)\n train[col] = indexer[col].get_indexer(train[col].astype(str))\n test[col] = indexer[col].get_indexer(test[col].astype(str))\n \n train = reduce_mem_usage(train, verbose=False)\n test = reduce_mem_usage(test, verbose=False)","metadata":{"execution":{"iopub.status.busy":"2022-08-05T09:53:57.679627Z","iopub.execute_input":"2022-08-05T09:53:57.680116Z","iopub.status.idle":"2022-08-05T10:25:13.746301Z","shell.execute_reply.started":"2022-08-05T09:53:57.680048Z","shell.execute_reply":"2022-08-05T10:25:13.744967Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del indexer","metadata":{"execution":{"iopub.status.busy":"2022-08-05T10:25:13.748936Z","iopub.execute_input":"2022-08-05T10:25:13.749458Z","iopub.status.idle":"2022-08-05T10:25:13.767484Z","shell.execute_reply.started":"2022-08-05T10:25:13.749396Z","shell.execute_reply":"2022-08-05T10:25:13.765933Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.head()","metadata":{"execution":{"iopub.status.busy":"2022-08-05T10:25:13.769406Z","iopub.execute_input":"2022-08-05T10:25:13.770386Z","iopub.status.idle":"2022-08-05T10:25:13.857166Z","shell.execute_reply.started":"2022-08-05T10:25:13.770346Z","shell.execute_reply":"2022-08-05T10:25:13.855760Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### **Training on all data**\n\nTraining model on three subsets of train data seperately & blending predictions.","metadata":{}},{"cell_type":"code","source":"y = train['HasDetections']\ntrain = train.drop(['HasDetections', 'MachineIdentifier'], axis=1)\ntest = test.drop(['MachineIdentifier'], axis=1)\ngc.collect()\ntrain.sort_values('AvSigVersion')\ntrain1 = train[:4000000]\ntrain = train[4000000:8000000]\n\ny1 = y[:4000000]\ny = y[4000000:8000000]","metadata":{"execution":{"iopub.status.busy":"2022-08-05T10:25:13.858977Z","iopub.execute_input":"2022-08-05T10:25:13.862492Z","iopub.status.idle":"2022-08-05T10:25:46.293224Z","shell.execute_reply.started":"2022-08-05T10:25:13.862445Z","shell.execute_reply":"2022-08-05T10:25:46.291722Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### **Modelling**","metadata":{}},{"cell_type":"code","source":"n_fold = 5\nfolds = StratifiedKFold(n_splits=n_fold, shuffle=True, random_state=15)","metadata":{"execution":{"iopub.status.busy":"2022-08-05T10:25:46.295274Z","iopub.execute_input":"2022-08-05T10:25:46.295651Z","iopub.status.idle":"2022-08-05T10:25:46.301727Z","shell.execute_reply.started":"2022-08-05T10:25:46.295620Z","shell.execute_reply":"2022-08-05T10:25:46.300185Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from numba import jit\n@jit\ndef fast_auc(y_true, y_prob):\n y_true = np.asarray(y_true)\n y_true = y_true[np.argsort(y_prob)]\n nfalse = 0\n auc = 0\n n = len(y_true)\n for i in range(n):\n y_i = y_true[i]\n nfalse += (1 - y_i)\n auc += y_i * nfalse\n auc /= (nfalse * (n - nfalse))\n return auc\n\ndef eval_auc(preds, dtrain):\n labels = dtrain.get_label()\n return 'auc', fast_auc(labels, preds), True\n\ndef predict_chunk(model, test):\n initial_idx = 0\n chunk_size = 1000000\n current_pred = np.zeros(len(test))\n while initial_idx < test.shape[0]:\n final_idx = min(initial_idx + chunk_size, test.shape[0])\n idx = range(initial_idx, final_idx)\n current_pred[idx] = model.predict(test.iloc[idx], num_iteration=model.best_iteration)\n initial_idx = final_idx\n return current_pred\n\n\ndef train_model(X=train, X_test=test, y=y, params=None, folds=folds, model_type='lgb', plot_feature_importance=False, averaging='usual', make_oof=False):\n result_dict = {}\n if make_oof:\n oof = np.zeros(len(X))\n prediction = np.zeros(len(X_test))\n scores = []\n feature_importance = pd.DataFrame()\n for fold_n, (train_index, valid_index) in enumerate(folds.split(X, y)):\n gc.collect()\n print('Fold', fold_n + 1, 'started at', time.ctime())\n X_train, X_valid = X.iloc[train_index], X.iloc[valid_index]\n y_train, y_valid = y.iloc[train_index], y.iloc[valid_index]\n \n \n if model_type == 'lgb':\n train_data = lgb.Dataset(X_train, label=y_train, categorical_feature = cat_cols)\n valid_data = lgb.Dataset(X_valid, label=y_valid, categorical_feature = cat_cols)\n \n model = lgb.train(params,\n train_data,\n num_boost_round=2000,\n valid_sets = [train_data, valid_data],\n verbose_eval=500,\n early_stopping_rounds = 200,\n feval=eval_auc)\n\n del train_data, valid_data\n \n y_pred_valid = model.predict(X_valid, num_iteration=model.best_iteration)\n del X_valid\n gc.collect()\n y_pred = predict_chunk(model, X_test)\n \n if model_type == 'xgb':\n train_data = xgb.DMatrix(data=X_train, label=y_train)\n valid_data = xgb.DMatrix(data=X_valid, label=y_valid)\n\n watchlist = [(train_data, 'train'), (valid_data, 'valid_data')]\n model = xgb.train(dtrain=train_data, num_boost_round=20000, evals=watchlist, early_stopping_rounds=200, verbose_eval=500, params=params)\n y_pred_valid = model.predict(xgb.DMatrix(X_valid), ntree_limit=model.best_ntree_limit)\n y_pred = predict_chunk(model, xgb.DMatrix(X_test))\n \n if model_type == 'lcv':\n model = LogisticRegressionCV(scoring='roc_auc', cv=3)\n model.fit(X_train, y_train)\n\n y_pred_valid = model.predict(X_valid)\n y_pred = predict_chunk(model, X_test)\n \n if model_type == 'cat':\n model = CatBoostRegressor(iterations=20000, eval_metric='AUC', **params)\n model.fit(X_train, y_train, eval_set=(X_valid, y_valid), cat_features=[], use_best_model=True, verbose=False)\n\n y_pred_valid = model.predict(X_valid)\n y_pred = predict_chunk(model, X_test)\n \n if make_oof:\n oof[valid_index] = y_pred_valid.reshape(-1,)\n \n scores.append(fast_auc(y_valid, y_pred_valid))\n print('Fold roc_auc:', roc_auc_score(y_valid, y_pred_valid))\n print('')\n \n if averaging == 'usual':\n prediction += y_pred\n elif averaging == 'rank':\n prediction += pd.Series(y_pred).rank().values\n \n if model_type == 'lgb':\n fold_importance = pd.DataFrame()\n fold_importance[\"feature\"] = X.columns\n fold_importance[\"importance\"] = model.feature_importance()\n fold_importance[\"fold\"] = fold_n + 1\n feature_importance = pd.concat([feature_importance, fold_importance], axis=0)\n\n prediction /= n_fold\n \n print('CV mean score: {0:.4f}, std: {1:.4f}.'.format(np.mean(scores), np.std(scores)))\n \n if model_type == 'lgb':\n \n if plot_feature_importance:\n feature_importance[\"importance\"] /= n_fold\n cols = feature_importance[[\"feature\", \"importance\"]].groupby(\"feature\").mean().sort_values(\n by=\"importance\", ascending=False)[:50].index\n\n best_features = feature_importance.loc[feature_importance.feature.isin(cols)]\n logging.info('Top features')\n for f in best_features.sort_values(by=\"importance\", ascending=False)['feature'].values:\n logging.info(f)\n\n plt.figure(figsize=(16, 12));\n sns.barplot(x=\"importance\", y=\"feature\", data=best_features.sort_values(by=\"importance\", ascending=False));\n plt.title('LGB Features (avg over folds)');\n \n result_dict['feature_importance'] = feature_importance\n \n result_dict['prediction'] = prediction\n if make_oof:\n result_dict['oof'] = oof\n \n return result_dict","metadata":{"execution":{"iopub.status.busy":"2022-08-05T10:25:46.304158Z","iopub.execute_input":"2022-08-05T10:25:46.304925Z","iopub.status.idle":"2022-08-05T10:25:47.376629Z","shell.execute_reply.started":"2022-08-05T10:25:46.304877Z","shell.execute_reply":"2022-08-05T10:25:47.375123Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"params = {'num_leaves': 256,\n 'min_data_in_leaf': 42,\n 'objective': 'binary',\n 'max_depth': 5,\n 'learning_rate': 0.05,\n \"boosting\": \"gbdt\",\n \"feature_fraction\": 0.8,\n \"bagging_freq\": 5,\n \"bagging_fraction\": 0.8,\n \"bagging_seed\": 11,\n \"lambda_l1\": 0.15,\n \"lambda_l2\": 0.15,\n \"random_state\": 42, \n \"verbosity\": -1}","metadata":{"execution":{"iopub.status.busy":"2022-08-05T10:25:47.378868Z","iopub.execute_input":"2022-08-05T10:25:47.379669Z","iopub.status.idle":"2022-08-05T10:25:47.387706Z","shell.execute_reply.started":"2022-08-05T10:25:47.379618Z","shell.execute_reply":"2022-08-05T10:25:47.386120Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del stats_df, freq_enc_dict","metadata":{"execution":{"iopub.status.busy":"2022-08-05T10:25:47.389773Z","iopub.execute_input":"2022-08-05T10:25:47.390213Z","iopub.status.idle":"2022-08-05T10:25:47.437044Z","shell.execute_reply.started":"2022-08-05T10:25:47.390176Z","shell.execute_reply":"2022-08-05T10:25:47.435857Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"result_dict1 = train_model(X=train1, X_test=test, y=y1, params=params, model_type='lgb', plot_feature_importance=True, averaging='rank')","metadata":{"execution":{"iopub.status.busy":"2022-08-05T10:25:47.438455Z","iopub.execute_input":"2022-08-05T10:25:47.439458Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del train1, y1","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = pd.read_csv('../input/microsoft-malware-prediction/sample_submission.csv')\nsubmission","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}