{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# Microsoft Malware Classification Challenge (BIG 2015)\n# Classify malware into families based on file content and characteristics\n# This notebook will achieve 0.00670 Logloss in the private set and 0.00809 on the public set\n# which is a near-perfect classification using a mix of features from both binaries and source code\n\n# Credits:\n# This notebook uses both https://www.kaggle.com/datasets/muhammad4hmed/malwaremicrosoftbig and \n# and https://www.kaggle.com/datasets/songwonmin/malware-only-byte\n\n# Alejandro Mosquera (http://amsqr.com)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-08-04T18:30:02.826035Z","iopub.execute_input":"2022-08-04T18:30:02.826599Z","iopub.status.idle":"2022-08-04T18:30:09.486171Z","shell.execute_reply.started":"2022-08-04T18:30:02.826487Z","shell.execute_reply":"2022-08-04T18:30:09.484856Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport glob\nimport os\nimport re\nimport codecs\nimport operator\n\n\ndf_train_large = pd.read_csv(\"../input/malwaremicrosoftbig/Dataset/Dataset/train/LargeTrain.csv\")\ndf_test_large = pd.read_csv(\"../input/malwaremicrosoftbig/Dataset/Dataset/test/LargeTest.csv\")\ntrainLabels = pd.read_csv(\"../input/malwaremicrosoftbig/Dataset/Dataset/trainLabels.csv\").sort_values(by=\"Id\",inplace=False)\ntestIds = pd.read_csv(\"../input/malwaremicrosoftbig/Dataset/Dataset/sorted_test_id.csv\").sort_values(by=\"Id\",inplace=False)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"trainLabels[\"Class\"] = trainLabels.Class.apply(lambda x:x-1)\ndf_test_large[\"Id\"] = testIds[\"Id\"].values\ndf_train_large[\"Id\"] = trainLabels[\"Id\"].values\ndf_train_large[\"Class\"] = trainLabels[\"Class\"].values\ndf_train_large = df_train_large.sort_values(by=\"Id\", inplace=False).reset_index(drop=True)\ndf_test_large = df_test_large.sort_values(by=\"Id\",inplace=False).reset_index(drop=True)","metadata":{"execution":{"iopub.status.busy":"2022-08-04T18:30:09.488493Z","iopub.execute_input":"2022-08-04T18:30:09.488886Z","iopub.status.idle":"2022-08-04T18:30:09.856783Z","shell.execute_reply.started":"2022-08-04T18:30:09.488850Z","shell.execute_reply":"2022-08-04T18:30:09.855388Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train_large[[ \"Class\", \"Id\"]]","metadata":{"execution":{"iopub.status.busy":"2022-08-04T06:33:34.127383Z","iopub.execute_input":"2022-08-04T06:33:34.127751Z","iopub.status.idle":"2022-08-04T06:33:34.151664Z","shell.execute_reply.started":"2022-08-04T06:33:34.127716Z","shell.execute_reply":"2022-08-04T06:33:34.150751Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"trainLabels","metadata":{"execution":{"iopub.status.busy":"2022-08-04T06:33:34.154385Z","iopub.execute_input":"2022-08-04T06:33:34.154969Z","iopub.status.idle":"2022-08-04T06:33:34.167615Z","shell.execute_reply.started":"2022-08-04T06:33:34.154932Z","shell.execute_reply":"2022-08-04T06:33:34.166327Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train_large.Class.value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-08-04T06:33:34.168961Z","iopub.execute_input":"2022-08-04T06:33:34.170063Z","iopub.status.idle":"2022-08-04T06:33:34.184449Z","shell.execute_reply.started":"2022-08-04T06:33:34.169979Z","shell.execute_reply":"2022-08-04T06:33:34.183051Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train_heurbyte = pd.read_csv(\"../input/microsoft-malware-challenge-2015-features/trainheurbyte.csv\").sort_values(\"fname\").reset_index(drop=True)\ndf_test_heurbyte = pd.read_csv(\"../input/microsoft-malware-challenge-2015-features/testheurbyte.csv\").sort_values(\"fname\").reset_index(drop=True)\ndf_train_heurbyte\n\nfor k in [\"filesize\",\"byteint\",\"byte10\",\"byte11\",\"byteCC\",\"byte00\",\"byteC3\",\"byte558BEC\",\"contdll\",\"sizenocavities\",\"top2gram1\",\"top2gram2\",\"top2gram3\",\"top4gram1\",\"top4gram2\",\"top4gram3\"]:\n    df_train_large[k] = df_train_heurbyte[k]\n    df_test_large[k] = df_test_heurbyte[k]","metadata":{"execution":{"iopub.status.busy":"2022-08-04T18:30:22.470902Z","iopub.execute_input":"2022-08-04T18:30:22.471334Z","iopub.status.idle":"2022-08-04T18:30:22.598136Z","shell.execute_reply.started":"2022-08-04T18:30:22.471275Z","shell.execute_reply":"2022-08-04T18:30:22.596784Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#feats = [\"filesize\",\"byteint\",\"byte10\",\"byte11\",\"byteCC\",\"byte00\",\"byteC3\",\"byte558BEC\",\"contdll\",\"sizenocavities\",\"top2gram1\",\"top2gram2\",\"top2gram3\",\"top4gram1\",\"top4gram2\",\"top4gram3\"]\nfeats = [k for k in df_test_large.columns.values if (k not in [\"Id\",\"Class\"] and k in df_train_large.columns.values)]","metadata":{"execution":{"iopub.status.busy":"2022-08-04T18:30:56.738851Z","iopub.execute_input":"2022-08-04T18:30:56.739260Z","iopub.status.idle":"2022-08-04T18:30:56.821777Z","shell.execute_reply.started":"2022-08-04T18:30:56.739225Z","shell.execute_reply":"2022-08-04T18:30:56.820780Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.ensemble import RandomForestClassifier\nfrom sklearn.metrics import log_loss\nfrom sklearn.model_selection import KFold,StratifiedKFold\nfrom sklearn.calibration import CalibratedClassifierCV\nfrom sklearn.metrics import precision_recall_fscore_support\nimport lightgbm as lgb\n\nkf = StratifiedKFold(n_splits=2,  shuffle=True, random_state=1337)\noof =  np.zeros([df_train_large.shape[0],9])\n\ndef cls2arr(x):\n    salida=[]\n    for r in x:\n        k = [0,0,0,0,0,0,0,0,0]\n        r=r-1\n        k[r]=1\n        salida.append(k)\n    return salida\n    \ndef calibrate(x):\n    salida=[]\n    for r in x:\n        k = [0,0,0,0,0,0,0,0,0]\n        pos = np.argmax(np.array(r))\n        k[pos] = 1.0\n        salida.append(k)\n    return salida\n\ndef cls2pred(x):\n    salida=[]\n    for r in x:\n        pos = np.argmax(np.array(r))\n        salida.append(pos)\n    return salida\n\nniter = 500\nfor tr, te in kf.split(df_train_large,df_train_large.Class):\n    #lgb_train = lgb.Dataset(df_train_large.loc[tr][feats].values, df_train_large.Class.values[tr])\n    #lgb_eval = lgb.Dataset(df_train_large.loc[te][feats].values, df_train_large.Class.values[te], reference=lgb_train)\n    #params = {\n    #      'boosting_type': 'gbdt',\n    #      'objective': 'multiclass', \n    #      'num_class':9, \n    #      'metric': 'multi_logloss',\n    #      'max_depth':10, 'num_leaves':10, \n    #      'learning_rate': 0.01,\n    #      'verbose': 0}\n    #gbm = lgb.train(params, lgb_train, num_boost_round=500, valid_sets=lgb_eval)\n    gbm = lgb.LGBMClassifier(num_boost_round=niter,boosting_type='gbdt',objective= 'multiclass',num_class=9,  metric='multi_logloss',  max_depth=10, num_leaves=10, learning_rate= 0.01,verbose=0)\n    gbm.fit(df_train_large.loc[tr][feats].values, df_train_large.Class.values[tr])\n    #cal = CalibratedClassifierCV(gbm, cv='prefit')\n    #cal.fit(df_train_large.loc[tr][feats].values, df_train_large.Class.values[tr])\n   \n    res_prob = gbm.predict_proba(df_train_large.loc[te][feats].values)\n    #res_prob = cal.predict_proba(df_train_large.loc[te][feats].values)\n    oof[te] = res_prob\n    res = cls2pred(res_prob)\n    print(log_loss(df_train_large.loc[te].Class.values,res_prob), precision_recall_fscore_support(df_train_large.loc[te].Class.values, res, average='micro'))","metadata":{"execution":{"iopub.status.busy":"2022-08-04T18:31:12.153453Z","iopub.execute_input":"2022-08-04T18:31:12.153871Z","iopub.status.idle":"2022-08-04T18:36:21.238561Z","shell.execute_reply.started":"2022-08-04T18:31:12.153835Z","shell.execute_reply":"2022-08-04T18:36:21.237361Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(log_loss(df_train_large.Class,oof))","metadata":{"execution":{"iopub.status.busy":"2022-08-04T18:36:21.240863Z","iopub.execute_input":"2022-08-04T18:36:21.241398Z","iopub.status.idle":"2022-08-04T18:36:21.254975Z","shell.execute_reply.started":"2022-08-04T18:36:21.241348Z","shell.execute_reply":"2022-08-04T18:36:21.253586Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gbm = lgb.LGBMClassifier(num_boost_round=niter,boosting_type='gbdt',objective= 'multiclass',num_class=9,  metric='multi_logloss',  max_depth=10, num_leaves=10, learning_rate= 0.01,verbose=0)\ngbm.fit(df_train_large[feats].values, df_train_large.Class.values)","metadata":{"execution":{"iopub.status.busy":"2022-08-04T18:36:21.257006Z","iopub.execute_input":"2022-08-04T18:36:21.257370Z","iopub.status.idle":"2022-08-04T18:39:37.373229Z","shell.execute_reply.started":"2022-08-04T18:36:21.257333Z","shell.execute_reply":"2022-08-04T18:39:37.372085Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_lgbm_varimp(model, train_columns, max_vars=50):\n    if \"basic.Booster\" in str(model.__class__):\n        # lightgbm.basic.Booster was trained directly, so using feature_importance() function \n        cv_varimp_df = pd.DataFrame([train_columns, model.feature_importance()]).T\n    else:\n        # Scikit-learn API LGBMClassifier or LGBMRegressor was fitted, \n        # so using feature_importances_ property\n        cv_varimp_df = pd.DataFrame([train_columns, model.feature_importances_]).T\n    cv_varimp_df.columns = ['feature_name', 'varimp']\n    cv_varimp_df.sort_values(by='varimp', ascending=False, inplace=True)\n    cv_varimp_df = cv_varimp_df.iloc[0:max_vars]   \n    return cv_varimp_df\n\nget_lgbm_varimp(gbm,feats)","metadata":{"execution":{"iopub.status.busy":"2022-08-04T18:52:30.571091Z","iopub.execute_input":"2022-08-04T18:52:30.571542Z","iopub.status.idle":"2022-08-04T18:52:30.656043Z","shell.execute_reply.started":"2022-08-04T18:52:30.571501Z","shell.execute_reply":"2022-08-04T18:52:30.654763Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"res = gbm.predict_proba(df_test_large[feats].values)","metadata":{"execution":{"iopub.status.busy":"2022-08-04T19:19:56.087646Z","iopub.execute_input":"2022-08-04T19:19:56.088068Z","iopub.status.idle":"2022-08-04T19:19:57.281694Z","shell.execute_reply.started":"2022-08-04T19:19:56.088030Z","shell.execute_reply":"2022-08-04T19:19:57.280469Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test_large[\"Prediction1\"] = res[:,0]\ndf_test_large[\"Prediction2\"] = res[:,1]\ndf_test_large[\"Prediction3\"] = res[:,2]\ndf_test_large[\"Prediction4\"] = res[:,3]\ndf_test_large[\"Prediction5\"] = res[:,4]\ndf_test_large[\"Prediction6\"] = res[:,5]\ndf_test_large[\"Prediction7\"] = res[:,6]\ndf_test_large[\"Prediction8\"] = res[:,7]\ndf_test_large[\"Prediction9\"] = res[:,8]\ndf_test_large[[\"Id\",\"Prediction1\",\"Prediction2\",\"Prediction3\",\"Prediction4\",\"Prediction5\",\"Prediction6\",\"Prediction7\",\"Prediction8\",\"Prediction9\"]].to_csv(\"submission.csv\",index=False)","metadata":{"execution":{"iopub.status.busy":"2022-08-04T19:20:01.650404Z","iopub.execute_input":"2022-08-04T19:20:01.651321Z","iopub.status.idle":"2022-08-04T19:20:01.732715Z","shell.execute_reply.started":"2022-08-04T19:20:01.651258Z","shell.execute_reply":"2022-08-04T19:20:01.731495Z"},"trusted":true},"execution_count":null,"outputs":[]}]}