{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# Load Dependencies\n\nfrom fastai.tabular.all import *\nfrom sklearn.datasets import make_multilabel_classification\nimport pandas as pd\nimport numpy as np\nfrom tqdm import tqdm\nimport os\nimport warnings\nwarnings.filterwarnings(\"ignore\")\nfrom sklearn.model_selection import StratifiedKFold\nfold = StratifiedKFold(n_splits=5, shuffle = True, random_state=62)\n\nfrom lightgbm import LGBMClassifier\nfrom xgboost import XGBClassifier\nfrom catboost import CatBoostClassifier","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-10-09T15:27:02.153585Z","iopub.execute_input":"2022-10-09T15:27:02.154035Z","iopub.status.idle":"2022-10-09T15:27:07.692482Z","shell.execute_reply.started":"2022-10-09T15:27:02.153996Z","shell.execute_reply":"2022-10-09T15:27:07.691408Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Load Data","metadata":{}},{"cell_type":"code","source":"train_labels=pd.read_csv(\"../input/malware-classification/trainLabels.csv\")\nsample = pd.read_csv(\"../input/malware-classification/sampleSubmission.csv\",index_col=\"Id\")\na = '\"0\",\"1\",\"2\",\"3\",\"4\",\"5\",\"6\",\"7\",\"8\",\"9\",\"0a\",\"0b\",\"0c\",\"0d\",\"0e\",\"0f\",\"10\",\"11\",\"12\",\"13\",\"14\",\"15\",\"16\",\"17\",\"18\",\"19\",\"1a\",\"1b\",\"1c\",\"1d\",\"1e\",\"1f\",\"20\",\"21\",\"22\",\"23\",\"24\",\"25\",\"26\",\"27\",\"28\",\"29\",\"2a\",\"2b\",\"2c\",\"2d\",\"2e\",\"2f\",\"30\",\"31\",\"32\",\"33\",\"34\",\"35\",\"36\",\"37\",\"38\",\"39\",\"3a\",\"3b\",\"3c\",\"3d\",\"3e\",\"3f\",\"40\",\"41\",\"42\",\"43\",\"44\",\"45\",\"46\",\"47\",\"48\",\"49\",\"4a\",\"4b\",\"4c\",\"4d\",\"4e\",\"4f\",\"50\",\"51\",\"52\",\"53\",\"54\",\"55\",\"56\",\"57\",\"58\",\"59\",\"5a\",\"5b\",\"5c\",\"5d\",\"5e\",\"5f\",\"60\",\"61\",\"62\",\"63\",\"64\",\"65\",\"66\",\"67\",\"68\",\"69\",\"6a\",\"6b\",\"6c\",\"6d\",\"6e\",\"6f\",\"70\",\"71\",\"72\",\"73\",\"74\",\"75\",\"76\",\"77\",\"78\",\"79\",\"7a\",\"7b\",\"7c\",\"7d\",\"7e\",\"7f\",\"80\",\"81\",\"82\",\"83\",\"84\",\"85\",\"86\",\"87\",\"88\",\"89\",\"8a\",\"8b\",\"8c\",\"8d\",\"8e\",\"8f\",\"90\",\"91\",\"92\",\"93\",\"94\",\"95\",\"96\",\"97\",\"98\",\"99\",\"9a\",\"9b\",\"9c\",\"9d\",\"9e\",\"9f\",\"a0\",\"a1\",\"a2\",\"a3\",\"a4\",\"a5\",\"a6\",\"a7\",\"a8\",\"a9\",\"aa\",\"ab\",\"ac\",\"ad\",\"ae\",\"af\",\"b0\",\"b1\",\"b2\",\"b3\",\"b4\",\"b5\",\"b6\",\"b7\",\"b8\",\"b9\",\"ba\",\"bb\",\"bc\",\"bd\",\"be\",\"bf\",\"c0\",\"c1\",\"c2\",\"c3\",\"c4\",\"c5\",\"c6\",\"c7\",\"c8\",\"c9\",\"ca\",\"cb\",\"cc\",\"cd\",\"ce\",\"cf\",\"d0\",\"d1\",\"d2\",\"d3\",\"d4\",\"d5\",\"d6\",\"d7\",\"d8\",\"d9\",\"da\",\"db\",\"dc\",\"dd\",\"de\",\"df\",\"e0\",\"e1\",\"e2\",\"e3\",\"e4\",\"e5\",\"e6\",\"e7\",\"e8\",\"e9\",\"ea\",\"eb\",\"ec\",\"ed\",\"ee\",\"ef\",\"f0\",\"f1\",\"f2\",\"f3\",\"f4\",\"f5\",\"f6\",\"f7\",\"f8\",\"f9\",\"fa\",\"fb\",\"fc\",\"fd\",\"fe\",\"ff\",\"??\",\"size\",\"Class\"'","metadata":{"execution":{"iopub.status.busy":"2022-10-09T15:27:07.697784Z","iopub.execute_input":"2022-10-09T15:27:07.700134Z","iopub.status.idle":"2022-10-09T15:27:07.803143Z","shell.execute_reply.started":"2022-10-09T15:27:07.700093Z","shell.execute_reply":"2022-10-09T15:27:07.802021Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#https://openbase.com/python/py7zr\n# Helpful for install\n!pip install py7zr\nimport py7zr\nimport re","metadata":{"execution":{"iopub.status.busy":"2022-10-09T15:27:07.808289Z","iopub.execute_input":"2022-10-09T15:27:07.810877Z","iopub.status.idle":"2022-10-09T15:27:29.161659Z","shell.execute_reply.started":"2022-10-09T15:27:07.810837Z","shell.execute_reply":"2022-10-09T15:27:29.160522Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = !py7zr l \"../input/malware-classification/train.7z\"","metadata":{"execution":{"iopub.status.busy":"2022-10-09T15:27:29.167809Z","iopub.execute_input":"2022-10-09T15:27:29.170188Z","iopub.status.idle":"2022-10-09T15:27:31.656980Z","shell.execute_reply.started":"2022-10-09T15:27:29.170130Z","shell.execute_reply":"2022-10-09T15:27:31.655404Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train[:5], len(train)","metadata":{"execution":{"iopub.status.busy":"2022-10-09T15:27:31.662475Z","iopub.execute_input":"2022-10-09T15:27:31.664961Z","iopub.status.idle":"2022-10-09T15:27:31.677255Z","shell.execute_reply.started":"2022-10-09T15:27:31.664919Z","shell.execute_reply":"2022-10-09T15:27:31.676207Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Feature Extraction","metadata":{}},{"cell_type":"code","source":"# Make this into a function\ndef fileToPanda(df,top, bottom):\n    df = pd.DataFrame(data=df)[0].str.split(',',expand=True)\n    df = df.iloc[top:]\n    df = df.iloc[:-1*bottom]\n    df=df[0].str.split(expand=True)\n    df = df.rename({0: 'Date', 1: 'Time', 2: 'Attr',3: 'Size',4: 'Compressed',5: 'Name'}, axis=1)\n    return df\n\n# Splitter for IDName\ndef makeNameColumns(df):\n    df['IDName']='TBD'\n    df['Filetype']='TBD'\n    df.loc[df['Name'].isna(), 'Name'] = df['Compressed']\n    #train_df.loc[train_df['Name'].isna(), 'Name'] = train_df['Compressed']\n\n    for index, row in df.iterrows():\n        try:\n            strings= str(row['Name'])#\"test/Ig2DB5tSiEy1cJvV0zdw.asm\"\n            strings=strings.split('/')\n            strings=strings[1].split('.')\n            row['IDName']=strings[0]\n            row['Filetype']=strings[1]\n        except:\n            print (str(row['Name']))\n    return df\n                   \n#Merge columns afterwards\ndef Merge(df):\n    # Make Panda for byte\n    byte_df = df[df['Filetype'] == 'bytes']\n    byte_df=byte_df.rename(columns={\"Size\": \"SizeByteFile\"})\n    # Make Panda for asm\n    asm_df = df[df['Filetype'] == 'asm']\n    asm_df=asm_df.rename(columns={\"Size\": \"SizeAsmFile\"})\n    # Merge\n    df=pd.merge(byte_df, asm_df, on='IDName')\n    \n    return df","metadata":{"execution":{"iopub.status.busy":"2022-10-09T15:27:31.681884Z","iopub.execute_input":"2022-10-09T15:27:31.684596Z","iopub.status.idle":"2022-10-09T15:27:31.704845Z","shell.execute_reply.started":"2022-10-09T15:27:31.684559Z","shell.execute_reply":"2022-10-09T15:27:31.703837Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df=fileToPanda(train,3,1)\ntrain_df=makeNameColumns(train_df)\ntrain_df=Merge(train_df)\ntrain_df[\"Class\"]=11\ntrain_df[:5]","metadata":{"execution":{"iopub.status.busy":"2022-10-09T15:28:02.933342Z","iopub.execute_input":"2022-10-09T15:28:02.933785Z","iopub.status.idle":"2022-10-09T15:28:05.356265Z","shell.execute_reply.started":"2022-10-09T15:28:02.933747Z","shell.execute_reply":"2022-10-09T15:28:05.355298Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"trainlabels = pd.read_csv('../input/malware-classification/trainLabels.csv')","metadata":{"execution":{"iopub.status.busy":"2022-10-09T15:28:25.618468Z","iopub.execute_input":"2022-10-09T15:28:25.618943Z","iopub.status.idle":"2022-10-09T15:28:25.644997Z","shell.execute_reply.started":"2022-10-09T15:28:25.618904Z","shell.execute_reply":"2022-10-09T15:28:25.644064Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Merge Together training data\nfor index, row in trainlabels.iterrows():\n    train_df.loc[train_df['IDName'] == row['Id'], 'Class'] = row['Class']\n\ntrain_df[:5]","metadata":{"execution":{"iopub.status.busy":"2022-10-09T15:29:53.681900Z","iopub.execute_input":"2022-10-09T15:29:53.684181Z","iopub.status.idle":"2022-10-09T15:30:14.983835Z","shell.execute_reply.started":"2022-10-09T15:29:53.684129Z","shell.execute_reply":"2022-10-09T15:30:14.982763Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['Class'].value_counts(sort=True, ascending=True)","metadata":{"execution":{"iopub.status.busy":"2022-10-09T15:30:14.988988Z","iopub.execute_input":"2022-10-09T15:30:14.991304Z","iopub.status.idle":"2022-10-09T15:30:15.004444Z","shell.execute_reply.started":"2022-10-09T15:30:14.991265Z","shell.execute_reply":"2022-10-09T15:30:15.003368Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"droppers = train_df[train_df['Class'] == 11].index\ntrain_df.drop(droppers , inplace=True)\ntrain_df['Class'].value_counts(sort=True, ascending=True)\ntrain_df","metadata":{"execution":{"iopub.status.busy":"2022-10-09T15:30:22.965357Z","iopub.execute_input":"2022-10-09T15:30:22.965804Z","iopub.status.idle":"2022-10-09T15:30:23.013400Z","shell.execute_reply.started":"2022-10-09T15:30:22.965766Z","shell.execute_reply":"2022-10-09T15:30:23.012400Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Build out more Classes 1-hot encoding\n# A poor decisioin to one-hot encode classes\n#for i in range(1,10):\n#    train_df[\"Class_\"+str(i)] = 0\n#    train_df.loc[train_df[\"Class\"] == i, \"Class_\"+str(i)] = 1","metadata":{"execution":{"iopub.status.busy":"2022-10-09T15:30:34.870475Z","iopub.execute_input":"2022-10-09T15:30:34.870922Z","iopub.status.idle":"2022-10-09T15:30:34.878678Z","shell.execute_reply.started":"2022-10-09T15:30:34.870884Z","shell.execute_reply":"2022-10-09T15:30:34.877424Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# ModelTime\nhttps://gist.github.com/ElisonSherton/cd21b00e19806054485a656549bf9265","metadata":{}},{"cell_type":"code","source":"#https://gist.github.com/ElisonSherton/cd21b00e19806054485a656549bf9265\nfrom fastai.tabular.all import *\nfrom sklearn.datasets import make_multilabel_classification","metadata":{"execution":{"iopub.status.busy":"2022-10-09T15:30:37.893887Z","iopub.execute_input":"2022-10-09T15:30:37.894381Z","iopub.status.idle":"2022-10-09T15:30:37.900598Z","shell.execute_reply.started":"2022-10-09T15:30:37.894339Z","shell.execute_reply":"2022-10-09T15:30:37.899637Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#y_names = ['Class_1','Class_2','Class_3','Class_4','Class_5','Class_6','Class_7','Class_8','Class_9']\ntrain=train_df","metadata":{"execution":{"iopub.status.busy":"2022-10-09T15:30:46.765833Z","iopub.execute_input":"2022-10-09T15:30:46.766487Z","iopub.status.idle":"2022-10-09T15:30:46.774806Z","shell.execute_reply.started":"2022-10-09T15:30:46.766441Z","shell.execute_reply":"2022-10-09T15:30:46.773803Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.SizeByteFile=train.SizeByteFile.apply(int)\ntrain.SizeAsmFile=train.SizeAsmFile.apply(int)","metadata":{"execution":{"iopub.status.busy":"2022-10-09T15:30:52.856223Z","iopub.execute_input":"2022-10-09T15:30:52.856814Z","iopub.status.idle":"2022-10-09T15:30:52.890852Z","shell.execute_reply.started":"2022-10-09T15:30:52.856770Z","shell.execute_reply":"2022-10-09T15:30:52.889903Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"num_cols = [\"SizeAsmFile\",\"SizeByteFile\"]\nnum_cols","metadata":{"execution":{"iopub.status.busy":"2022-10-09T15:30:56.262277Z","iopub.execute_input":"2022-10-09T15:30:56.262893Z","iopub.status.idle":"2022-10-09T15:30:56.272019Z","shell.execute_reply.started":"2022-10-09T15:30:56.262827Z","shell.execute_reply":"2022-10-09T15:30:56.270875Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"splits = RandomSplitter(valid_pct=0.2)(range_of(train))\nsplits","metadata":{"execution":{"iopub.status.busy":"2022-10-09T15:30:58.420491Z","iopub.execute_input":"2022-10-09T15:30:58.420932Z","iopub.status.idle":"2022-10-09T15:30:58.434006Z","shell.execute_reply.started":"2022-10-09T15:30:58.420896Z","shell.execute_reply":"2022-10-09T15:30:58.432864Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"to = TabularPandas(train, procs=[Categorify,Normalize],\n                   cat_names = None, # List of all categorical\n                   cont_names = num_cols, # List of all Continous\n                   y_block=CategoryBlock, # We are making a category block\n                   y_names = 'Class', # This is the malware Class from the model\n                   splits=splits) # Splits the data","metadata":{"execution":{"iopub.status.busy":"2022-10-09T15:32:12.994422Z","iopub.execute_input":"2022-10-09T15:32:12.994910Z","iopub.status.idle":"2022-10-09T15:32:13.118198Z","shell.execute_reply.started":"2022-10-09T15:32:12.994873Z","shell.execute_reply":"2022-10-09T15:32:13.117202Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dls = to.dataloaders(bs=16).to(\"cuda\") ","metadata":{"execution":{"iopub.status.busy":"2022-10-09T15:37:42.005252Z","iopub.execute_input":"2022-10-09T15:37:42.005717Z","iopub.status.idle":"2022-10-09T15:37:42.023668Z","shell.execute_reply.started":"2022-10-09T15:37:42.005678Z","shell.execute_reply":"2022-10-09T15:37:42.022719Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"xb_cat, xb_cont, yb = dls.one_batch()\nxb_cat.shape, xb_cont.shape, yb.shape","metadata":{"execution":{"iopub.status.busy":"2022-10-09T15:37:42.372987Z","iopub.execute_input":"2022-10-09T15:37:42.373354Z","iopub.status.idle":"2022-10-09T15:37:42.397420Z","shell.execute_reply.started":"2022-10-09T15:37:42.373319Z","shell.execute_reply":"2022-10-09T15:37:42.396448Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#learn = tabular_learner(dls, metrics=[accuracy_multi], loss_func = BCEWithLogitsLossFlat())\n#learn = tabular_learner(dls, metrics=[accuracy_multi], loss_func = CrossEntropyLossFlat())\nlearn = tabular_learner(dls, metrics=accuracy)\nlearn.lr_find()","metadata":{"execution":{"iopub.status.busy":"2022-10-09T15:37:42.668782Z","iopub.execute_input":"2022-10-09T15:37:42.669264Z","iopub.status.idle":"2022-10-09T15:37:44.692537Z","shell.execute_reply.started":"2022-10-09T15:37:42.669226Z","shell.execute_reply":"2022-10-09T15:37:44.691474Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Train the model\nlearn.fit_one_cycle(10, 1e-3)\nlearn.unfreeze()\nlearn.fit_one_cycle(10, 1e-4)","metadata":{"execution":{"iopub.status.busy":"2022-10-09T15:37:44.694704Z","iopub.execute_input":"2022-10-09T15:37:44.695374Z","iopub.status.idle":"2022-10-09T15:38:00.548517Z","shell.execute_reply.started":"2022-10-09T15:37:44.695335Z","shell.execute_reply":"2022-10-09T15:38:00.543873Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"learn.lr_find()","metadata":{"execution":{"iopub.status.busy":"2022-10-09T15:38:00.551614Z","iopub.status.idle":"2022-10-09T15:38:00.554238Z","shell.execute_reply.started":"2022-10-09T15:38:00.553905Z","shell.execute_reply":"2022-10-09T15:38:00.553936Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"learn.fit_one_cycle(10, 1e-4)","metadata":{"execution":{"iopub.status.busy":"2022-10-09T15:38:00.557781Z","iopub.status.idle":"2022-10-09T15:38:00.558643Z","shell.execute_reply.started":"2022-10-09T15:38:00.558355Z","shell.execute_reply":"2022-10-09T15:38:00.558382Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Submit","metadata":{"execution":{"iopub.status.busy":"2022-09-21T15:02:11.240083Z","iopub.execute_input":"2022-09-21T15:02:11.240621Z","iopub.status.idle":"2022-09-21T15:02:11.247960Z","shell.execute_reply.started":"2022-09-21T15:02:11.240576Z","shell.execute_reply":"2022-09-21T15:02:11.246588Z"}}},{"cell_type":"code","source":"# Get Test data ready\ntest = !py7zr l \"../input/malware-classification/test.7z\"\ntest[:5]","metadata":{"execution":{"iopub.status.busy":"2022-10-09T15:38:07.338366Z","iopub.execute_input":"2022-10-09T15:38:07.338813Z","iopub.status.idle":"2022-10-09T15:38:09.907248Z","shell.execute_reply.started":"2022-10-09T15:38:07.338776Z","shell.execute_reply":"2022-10-09T15:38:09.905924Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df=fileToPanda(test,3,2)\ntest_df=makeNameColumns(test_df)\ntest_df=Merge(test_df)\n# Creates a new column based on sisze\ntest_df.SizeByteFile=test_df.SizeByteFile.apply(int)\ntest_df.SizeAsmFile=test_df.SizeAsmFile.apply(int)\ntest_df[:5]","metadata":{"execution":{"iopub.status.busy":"2022-10-09T15:38:09.912547Z","iopub.execute_input":"2022-10-09T15:38:09.914905Z","iopub.status.idle":"2022-10-09T15:38:12.342045Z","shell.execute_reply.started":"2022-10-09T15:38:09.914863Z","shell.execute_reply":"2022-10-09T15:38:12.341053Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df[:5]","metadata":{"execution":{"iopub.status.busy":"2022-10-09T15:38:12.347135Z","iopub.execute_input":"2022-10-09T15:38:12.349399Z","iopub.status.idle":"2022-10-09T15:38:12.374954Z","shell.execute_reply.started":"2022-10-09T15:38:12.349361Z","shell.execute_reply":"2022-10-09T15:38:12.374073Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Makes a simpler IDName\n\ntest_df['IDName']='TBD'\nfor index, row in test_df.iterrows():\n    try:\n        strings= str(row['Name'])#\"test/Ig2DB5tSiEy1cJvV0zdw.asm\"\n        strings=strings.split('/')\n        strings=strings[1].split('.')\n        row['IDName']=strings[0]\n    except:\n        print (str(row['Name']))\n","metadata":{}},{"cell_type":"code","source":"dl = learn.dls.test_dl(test_df)","metadata":{"execution":{"iopub.status.busy":"2022-10-09T15:38:16.078577Z","iopub.execute_input":"2022-10-09T15:38:16.079319Z","iopub.status.idle":"2022-10-09T15:38:16.103423Z","shell.execute_reply.started":"2022-10-09T15:38:16.079274Z","shell.execute_reply":"2022-10-09T15:38:16.102446Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds=learn.get_preds(dl=dl)\npreds[0].shape","metadata":{"execution":{"iopub.status.busy":"2022-10-09T15:38:17.552033Z","iopub.execute_input":"2022-10-09T15:38:17.552539Z","iopub.status.idle":"2022-10-09T15:38:20.558013Z","shell.execute_reply.started":"2022-10-09T15:38:17.552497Z","shell.execute_reply":"2022-10-09T15:38:20.557082Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Create Panda\n#px = pd.DataFrame(x.numpy())\npx = pd.DataFrame(preds[0])\npx[\"Id\"]=test_df[\"IDName\"]\npx","metadata":{"execution":{"iopub.status.busy":"2022-10-09T15:38:21.507859Z","iopub.execute_input":"2022-10-09T15:38:21.508319Z","iopub.status.idle":"2022-10-09T15:38:21.543127Z","shell.execute_reply.started":"2022-10-09T15:38:21.508280Z","shell.execute_reply":"2022-10-09T15:38:21.542217Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"px.columns = ['Prediction1', 'Prediction2', 'Prediction3', 'Prediction4', 'Prediction5', 'Prediction6', 'Prediction7', 'Prediction8', 'Prediction9', 'Id']\npx","metadata":{"execution":{"iopub.status.busy":"2022-10-09T15:41:04.933786Z","iopub.execute_input":"2022-10-09T15:41:04.934306Z","iopub.status.idle":"2022-10-09T15:41:04.964239Z","shell.execute_reply.started":"2022-10-09T15:41:04.934264Z","shell.execute_reply":"2022-10-09T15:41:04.963160Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Only read certain files; Useful for later\nfilter_pattern = re.compile(r'<your/target/file_and_directories/regex/expression>')\nwith SevenZipFile('archive.7z', 'r') as archive:\n    allfiles = archive.getnames()\n    selective_files = [f for f in allfiles if filter_pattern.match(f)]\n    archive.extract(targets=selective_files)","metadata":{"execution":{"iopub.status.busy":"2022-09-20T15:50:22.054594Z","iopub.execute_input":"2022-09-20T15:50:22.055076Z","iopub.status.idle":"2022-09-20T15:50:22.292913Z","shell.execute_reply.started":"2022-09-20T15:50:22.055019Z","shell.execute_reply":"2022-09-20T15:50:22.291553Z"}}},{"cell_type":"code","source":"px","metadata":{"execution":{"iopub.status.busy":"2022-10-09T15:41:05.265822Z","iopub.execute_input":"2022-10-09T15:41:05.266217Z","iopub.status.idle":"2022-10-09T15:41:05.293615Z","shell.execute_reply.started":"2022-10-09T15:41:05.266178Z","shell.execute_reply":"2022-10-09T15:41:05.292584Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final=pd.merge(sample, px, on='Id')\nfinal[:5]","metadata":{"execution":{"iopub.status.busy":"2022-10-09T15:41:05.441367Z","iopub.execute_input":"2022-10-09T15:41:05.441720Z","iopub.status.idle":"2022-10-09T15:41:05.486150Z","shell.execute_reply.started":"2022-10-09T15:41:05.441685Z","shell.execute_reply":"2022-10-09T15:41:05.485116Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final=pd.merge(sample, px, on='Id')\n#Prediction1_x\tPrediction2_x\tPrediction3_x\tPrediction4_x\tPrediction5_x\tPrediction6_x\tPrediction7_x\tPrediction8_x\tPrediction9_x\n#final=final.set_index('Id')\nfinal=final.drop(columns=[\"Prediction1_x\",\"Prediction2_x\",\"Prediction3_x\",\"Prediction4_x\",\"Prediction5_x\",\"Prediction6_x\",\"Prediction7_x\",\"Prediction8_x\",\"Prediction9_x\"])\nfinal.columns = ['Id','Prediction1', 'Prediction2', 'Prediction3', 'Prediction4', 'Prediction5', 'Prediction6', 'Prediction7', 'Prediction8', 'Prediction9']\nfinal","metadata":{"execution":{"iopub.status.busy":"2022-10-09T15:41:05.602510Z","iopub.execute_input":"2022-10-09T15:41:05.602864Z","iopub.status.idle":"2022-10-09T15:41:05.643258Z","shell.execute_reply.started":"2022-10-09T15:41:05.602830Z","shell.execute_reply":"2022-10-09T15:41:05.642216Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission=final\n# Create \"submission.csv\" file\nsubmission.to_csv('submission.csv', index=False)\nsubmission.head()","metadata":{"execution":{"iopub.status.busy":"2022-10-09T15:41:05.767126Z","iopub.execute_input":"2022-10-09T15:41:05.767476Z","iopub.status.idle":"2022-10-09T15:41:05.942952Z","shell.execute_reply.started":"2022-10-09T15:41:05.767444Z","shell.execute_reply":"2022-10-09T15:41:05.941888Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Test is there is something goofy; It should Sum to 1\nsubmission['SUM']= final.iloc[:, :10].sum(axis=1)\nsubmission[:5]","metadata":{"execution":{"iopub.status.busy":"2022-10-09T15:41:05.944968Z","iopub.execute_input":"2022-10-09T15:41:05.945634Z","iopub.status.idle":"2022-10-09T15:41:05.982867Z","shell.execute_reply.started":"2022-10-09T15:41:05.945597Z","shell.execute_reply":"2022-10-09T15:41:05.981823Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# References\nhttps://forums.fast.ai/t/selecting-a-tabular-multi-label-loss-function/99249/3 ","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}