{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\n#import cupy, cudf\nimport torch\nimport torch.nn as nn\nfrom torch.utils.data import DataLoader\n\nimport torch.nn.functional as F\nimport torch.optim as optim\nfrom tqdm import tqdm\nimport math\nimport os\nimport gc\nfrom sklearn.model_selection import KFold, StratifiedKFold\nfrom sklearn.preprocessing import StandardScaler","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-08-01T16:41:27.156699Z","iopub.execute_input":"2022-08-01T16:41:27.157083Z","iopub.status.idle":"2022-08-01T16:41:27.163425Z","shell.execute_reply.started":"2022-08-01T16:41:27.157052Z","shell.execute_reply":"2022-08-01T16:41:27.162216Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('Using PyTorch version',torch.__version__)","metadata":{"execution":{"iopub.status.busy":"2022-08-01T16:41:27.257382Z","iopub.execute_input":"2022-08-01T16:41:27.257845Z","iopub.status.idle":"2022-08-01T16:41:27.263423Z","shell.execute_reply.started":"2022-08-01T16:41:27.257814Z","shell.execute_reply":"2022-08-01T16:41:27.262263Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''\nimport torch.backends.cudnn\ntorch.backends.cudnn.benchmark = False\ntorch.backends.cudnn.deterministic = True\n'''","metadata":{"execution":{"iopub.status.busy":"2022-08-01T16:41:27.318563Z","iopub.execute_input":"2022-08-01T16:41:27.319046Z","iopub.status.idle":"2022-08-01T16:41:27.325312Z","shell.execute_reply.started":"2022-08-01T16:41:27.319017Z","shell.execute_reply":"2022-08-01T16:41:27.324218Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"CUDA_LAUNCH_BLOCKING=1","metadata":{"execution":{"iopub.status.busy":"2022-08-01T16:41:27.397447Z","iopub.execute_input":"2022-08-01T16:41:27.397893Z","iopub.status.idle":"2022-08-01T16:41:27.405377Z","shell.execute_reply.started":"2022-08-01T16:41:27.397863Z","shell.execute_reply":"2022-08-01T16:41:27.404067Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"path = '../input/amex-default-prediction'\nfeather_path= '../input/parquet-files-amexdefault-prediction'","metadata":{"execution":{"iopub.status.busy":"2022-08-01T16:41:27.501314Z","iopub.execute_input":"2022-08-01T16:41:27.501595Z","iopub.status.idle":"2022-08-01T16:41:27.506374Z","shell.execute_reply.started":"2022-08-01T16:41:27.501569Z","shell.execute_reply":"2022-08-01T16:41:27.505090Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Config:\n    EPOCHS = 1\n    DROP_OUT = 0.1\n    DEVICE = 'cuda:0'\n    BATCH_SIZE = 64\n    learning_rate = 0.01","metadata":{"execution":{"iopub.status.busy":"2022-08-01T16:41:27.602618Z","iopub.execute_input":"2022-08-01T16:41:27.604783Z","iopub.status.idle":"2022-08-01T16:41:27.610016Z","shell.execute_reply.started":"2022-08-01T16:41:27.604751Z","shell.execute_reply":"2022-08-01T16:41:27.609009Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"torch.cuda.is_available()","metadata":{"execution":{"iopub.status.busy":"2022-08-01T16:41:27.648901Z","iopub.execute_input":"2022-08-01T16:41:27.649183Z","iopub.status.idle":"2022-08-01T16:41:27.657504Z","shell.execute_reply.started":"2022-08-01T16:41:27.649158Z","shell.execute_reply":"2022-08-01T16:41:27.656251Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ndf = pd.read_feather(os.path.join(feather_path, 'train_data.ftr'))","metadata":{"execution":{"iopub.status.busy":"2022-08-01T16:41:27.693363Z","iopub.execute_input":"2022-08-01T16:41:27.693651Z","iopub.status.idle":"2022-08-01T16:41:33.847604Z","shell.execute_reply.started":"2022-08-01T16:41:27.693616Z","shell.execute_reply":"2022-08-01T16:41:33.846404Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"_ = gc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-08-01T16:41:33.849596Z","iopub.execute_input":"2022-08-01T16:41:33.850206Z","iopub.status.idle":"2022-08-01T16:41:33.973192Z","shell.execute_reply.started":"2022-08-01T16:41:33.850164Z","shell.execute_reply":"2022-08-01T16:41:33.971942Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tmp = df.groupby(['customer_ID']).size().reset_index(name='count')\nmissing_cstmr = tmp.loc[tmp['count'] !=13, 'customer_ID']","metadata":{"execution":{"iopub.status.busy":"2022-08-01T16:41:33.975157Z","iopub.execute_input":"2022-08-01T16:41:33.975553Z","iopub.status.idle":"2022-08-01T16:41:35.501517Z","shell.execute_reply.started":"2022-08-01T16:41:33.975494Z","shell.execute_reply":"2022-08-01T16:41:35.500491Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = df.loc[~df['customer_ID'].isin(missing_cstmr)]","metadata":{"execution":{"iopub.status.busy":"2022-08-01T16:41:35.504461Z","iopub.execute_input":"2022-08-01T16:41:35.504861Z","iopub.status.idle":"2022-08-01T16:41:43.170617Z","shell.execute_reply.started":"2022-08-01T16:41:35.504822Z","shell.execute_reply":"2022-08-01T16:41:43.169578Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.shape","metadata":{"execution":{"iopub.status.busy":"2022-08-01T16:41:43.172357Z","iopub.execute_input":"2022-08-01T16:41:43.172765Z","iopub.status.idle":"2022-08-01T16:41:43.179496Z","shell.execute_reply.started":"2022-08-01T16:41:43.172725Z","shell.execute_reply":"2022-08-01T16:41:43.178465Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del df","metadata":{"execution":{"iopub.status.busy":"2022-08-01T16:41:43.181289Z","iopub.execute_input":"2022-08-01T16:41:43.182050Z","iopub.status.idle":"2022-08-01T16:41:43.225821Z","shell.execute_reply.started":"2022-08-01T16:41:43.181988Z","shell.execute_reply":"2022-08-01T16:41:43.224892Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"_ = gc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-08-01T16:41:43.227520Z","iopub.execute_input":"2022-08-01T16:41:43.227920Z","iopub.status.idle":"2022-08-01T16:41:43.341370Z","shell.execute_reply.started":"2022-08-01T16:41:43.227860Z","shell.execute_reply":"2022-08-01T16:41:43.340322Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del tmp","metadata":{"execution":{"iopub.status.busy":"2022-08-01T16:41:43.342587Z","iopub.execute_input":"2022-08-01T16:41:43.344247Z","iopub.status.idle":"2022-08-01T16:41:43.356560Z","shell.execute_reply.started":"2022-08-01T16:41:43.344207Z","shell.execute_reply":"2022-08-01T16:41:43.355550Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_label = pd.read_csv(os.path.join(path, 'train_labels.csv'))","metadata":{"execution":{"iopub.status.busy":"2022-08-01T16:41:43.357968Z","iopub.execute_input":"2022-08-01T16:41:43.358779Z","iopub.status.idle":"2022-08-01T16:41:44.356423Z","shell.execute_reply.started":"2022-08-01T16:41:43.358731Z","shell.execute_reply":"2022-08-01T16:41:44.355350Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_label = train_label.loc[~train_label['customer_ID'].isin(missing_cstmr)]","metadata":{"execution":{"iopub.status.busy":"2022-08-01T16:41:44.361971Z","iopub.execute_input":"2022-08-01T16:41:44.362305Z","iopub.status.idle":"2022-08-01T16:41:44.483571Z","shell.execute_reply.started":"2022-08-01T16:41:44.362274Z","shell.execute_reply":"2022-08-01T16:41:44.482557Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_label[\"idx\"] = np.arange(train_label.shape[0])","metadata":{"execution":{"iopub.status.busy":"2022-08-01T16:41:44.486165Z","iopub.execute_input":"2022-08-01T16:41:44.486634Z","iopub.status.idle":"2022-08-01T16:41:44.493339Z","shell.execute_reply.started":"2022-08-01T16:41:44.486589Z","shell.execute_reply":"2022-08-01T16:41:44.492201Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_label = train_label.reset_index(drop=True)","metadata":{"execution":{"iopub.status.busy":"2022-08-01T16:41:44.495067Z","iopub.execute_input":"2022-08-01T16:41:44.495449Z","iopub.status.idle":"2022-08-01T16:41:44.520928Z","shell.execute_reply.started":"2022-08-01T16:41:44.495410Z","shell.execute_reply":"2022-08-01T16:41:44.519986Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_label","metadata":{"execution":{"iopub.status.busy":"2022-08-01T16:41:44.522324Z","iopub.execute_input":"2022-08-01T16:41:44.522670Z","iopub.status.idle":"2022-08-01T16:41:44.541986Z","shell.execute_reply.started":"2022-08-01T16:41:44.522634Z","shell.execute_reply":"2022-08-01T16:41:44.540981Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#train_dict = {}","metadata":{"execution":{"iopub.status.busy":"2022-08-01T16:41:44.545018Z","iopub.execute_input":"2022-08-01T16:41:44.545298Z","iopub.status.idle":"2022-08-01T16:41:44.550671Z","shell.execute_reply.started":"2022-08-01T16:41:44.545271Z","shell.execute_reply":"2022-08-01T16:41:44.549547Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#train_label.loc[train_label['idx'] == 0, 'customer_ID'].values[0]","metadata":{"execution":{"iopub.status.busy":"2022-08-01T16:41:44.552165Z","iopub.execute_input":"2022-08-01T16:41:44.553381Z","iopub.status.idle":"2022-08-01T16:41:44.559001Z","shell.execute_reply.started":"2022-08-01T16:41:44.553340Z","shell.execute_reply":"2022-08-01T16:41:44.557586Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#train.loc[train['customer_ID'] == ''].values","metadata":{"execution":{"iopub.status.busy":"2022-08-01T16:41:44.560430Z","iopub.execute_input":"2022-08-01T16:41:44.561524Z","iopub.status.idle":"2022-08-01T16:41:44.567670Z","shell.execute_reply.started":"2022-08-01T16:41:44.561486Z","shell.execute_reply":"2022-08-01T16:41:44.566764Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ntrain = train.merge(train_label, on='customer_ID', how='left')","metadata":{"execution":{"iopub.status.busy":"2022-08-01T16:41:44.569997Z","iopub.execute_input":"2022-08-01T16:41:44.573057Z","iopub.status.idle":"2022-08-01T16:43:04.676848Z","shell.execute_reply.started":"2022-08-01T16:41:44.573027Z","shell.execute_reply":"2022-08-01T16:43:04.675653Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = train.sort_values(by='idx')","metadata":{"execution":{"iopub.status.busy":"2022-08-01T16:43:04.678313Z","iopub.execute_input":"2022-08-01T16:43:04.679007Z","iopub.status.idle":"2022-08-01T16:43:12.225111Z","shell.execute_reply.started":"2022-08-01T16:43:04.678965Z","shell.execute_reply":"2022-08-01T16:43:12.224042Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"_cstmr = train_label.loc[train_label['idx'] == 0,'customer_ID'][0]","metadata":{"execution":{"iopub.status.busy":"2022-08-01T16:43:12.226478Z","iopub.execute_input":"2022-08-01T16:43:12.226892Z","iopub.status.idle":"2022-08-01T16:43:12.234544Z","shell.execute_reply.started":"2022-08-01T16:43:12.226854Z","shell.execute_reply":"2022-08-01T16:43:12.233636Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"not_wanted = ['customer_ID', 'S_2', 'target', 'idx']","metadata":{"execution":{"iopub.status.busy":"2022-08-01T16:43:12.235970Z","iopub.execute_input":"2022-08-01T16:43:12.236950Z","iopub.status.idle":"2022-08-01T16:43:12.243772Z","shell.execute_reply.started":"2022-08-01T16:43:12.236910Z","shell.execute_reply":"2022-08-01T16:43:12.242885Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"req_cols = [i for i in train.columns if i not in not_wanted]","metadata":{"execution":{"iopub.status.busy":"2022-08-01T16:43:12.245214Z","iopub.execute_input":"2022-08-01T16:43:12.246401Z","iopub.status.idle":"2022-08-01T16:43:12.254045Z","shell.execute_reply.started":"2022-08-01T16:43:12.246363Z","shell.execute_reply":"2022-08-01T16:43:12.252904Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(req_cols)","metadata":{"execution":{"iopub.status.busy":"2022-08-01T16:43:12.255839Z","iopub.execute_input":"2022-08-01T16:43:12.256900Z","iopub.status.idle":"2022-08-01T16:43:12.266738Z","shell.execute_reply.started":"2022-08-01T16:43:12.256872Z","shell.execute_reply":"2022-08-01T16:43:12.265752Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#train.loc[train['customer_ID'] == _cstmr, req_cols]","metadata":{"execution":{"iopub.status.busy":"2022-08-01T16:43:12.269130Z","iopub.execute_input":"2022-08-01T16:43:12.271943Z","iopub.status.idle":"2022-08-01T16:43:12.276776Z","shell.execute_reply.started":"2022-08-01T16:43:12.271905Z","shell.execute_reply":"2022-08-01T16:43:12.275756Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"_= gc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-08-01T16:43:12.278466Z","iopub.execute_input":"2022-08-01T16:43:12.279361Z","iopub.status.idle":"2022-08-01T16:43:12.400757Z","shell.execute_reply.started":"2022-08-01T16:43:12.279322Z","shell.execute_reply":"2022-08-01T16:43:12.399751Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"CATS = ['B_30', 'B_38', 'D_114', 'D_116', 'D_117', 'D_120', 'D_126', 'D_66', 'D_68']","metadata":{"execution":{"iopub.status.busy":"2022-08-01T16:43:12.402437Z","iopub.execute_input":"2022-08-01T16:43:12.402853Z","iopub.status.idle":"2022-08-01T16:43:12.410629Z","shell.execute_reply.started":"2022-08-01T16:43:12.402815Z","shell.execute_reply":"2022-08-01T16:43:12.409674Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['D_63'] = train['D_63'].astype('str')\ntrain['D_64'] = train['D_64'].astype('str')","metadata":{"execution":{"iopub.status.busy":"2022-08-01T16:43:12.412006Z","iopub.execute_input":"2022-08-01T16:43:12.412633Z","iopub.status.idle":"2022-08-01T16:43:13.009457Z","shell.execute_reply.started":"2022-08-01T16:43:12.412582Z","shell.execute_reply":"2022-08-01T16:43:13.008398Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"d_63_map = {'CL':2, 'CO':3, 'CR':4, 'XL':5, 'XM':6, 'XZ':7}\ntrain['D_63'] = train.D_63.map(d_63_map).fillna(1).astype('int8')\n\nd_64_map = {'-1':2,'O':3, 'R':4, 'U':5}\ntrain['D_64'] = train.D_64.map(d_64_map).fillna(1).astype('int8')","metadata":{"execution":{"iopub.status.busy":"2022-08-01T16:43:13.011724Z","iopub.execute_input":"2022-08-01T16:43:13.012435Z","iopub.status.idle":"2022-08-01T16:43:14.123047Z","shell.execute_reply.started":"2022-08-01T16:43:13.012394Z","shell.execute_reply":"2022-08-01T16:43:14.121952Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in CATS:\n    train[i] = train[i].fillna(1).astype('int8')","metadata":{"execution":{"iopub.status.busy":"2022-08-01T16:43:14.129902Z","iopub.execute_input":"2022-08-01T16:43:14.130219Z","iopub.status.idle":"2022-08-01T16:43:14.348307Z","shell.execute_reply.started":"2022-08-01T16:43:14.130192Z","shell.execute_reply":"2022-08-01T16:43:14.347277Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in train.columns:\n    if i not in CATS:\n        #print(i)\n        train[i] = train[i].fillna(-999)","metadata":{"execution":{"iopub.status.busy":"2022-08-01T16:43:14.349913Z","iopub.execute_input":"2022-08-01T16:43:14.350295Z","iopub.status.idle":"2022-08-01T16:43:20.003662Z","shell.execute_reply.started":"2022-08-01T16:43:14.350257Z","shell.execute_reply":"2022-08-01T16:43:20.002585Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#F_matrix = train[req_cols].values.reshape(train.shape[0] // 13, 13, len(train[req_cols].columns))","metadata":{"execution":{"iopub.status.busy":"2022-08-01T16:43:20.005263Z","iopub.execute_input":"2022-08-01T16:43:20.005691Z","iopub.status.idle":"2022-08-01T16:43:20.011108Z","shell.execute_reply.started":"2022-08-01T16:43:20.005622Z","shell.execute_reply":"2022-08-01T16:43:20.009198Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#del train\n_= gc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-08-01T16:43:20.012895Z","iopub.execute_input":"2022-08-01T16:43:20.013299Z","iopub.status.idle":"2022-08-01T16:43:20.135287Z","shell.execute_reply.started":"2022-08-01T16:43:20.013260Z","shell.execute_reply":"2022-08-01T16:43:20.134127Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#.save('F_matrix.npy', F_matrix)","metadata":{"execution":{"iopub.status.busy":"2022-08-01T16:43:20.136844Z","iopub.execute_input":"2022-08-01T16:43:20.137230Z","iopub.status.idle":"2022-08-01T16:43:20.145572Z","shell.execute_reply.started":"2022-08-01T16:43:20.137168Z","shell.execute_reply":"2022-08-01T16:43:20.144431Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class AmexDataset:\n    def __init__(self, data, train_df= train,  is_test=False):\n        \n        self.train = train_df\n        self.data = data\n        self.label = train_label\n        \n        self.is_test = is_test\n    \n    def __len__(self):\n        return len(self.data)\n    \n    def __getitem__(self,idx):\n        #print(idx)\n        current_data = self.train.loc[self.train['idx'] == idx, req_cols].values\n        \n        if self.is_test:\n\n            return {\n            \"x\":torch.tensor(current_data, dtype=torch.float),\n            #\"targets\": None\n                   }\n        else:\n            current_target = self.label.loc[self.label.idx == idx,'target'].values\n            #print(current_data)\n            #print(current_target)\n            return {\n            \"x\":torch.tensor(current_data, dtype=torch.float32),\n            \"targets\":torch.tensor(current_target, dtype=torch.float32)\n        }","metadata":{"execution":{"iopub.status.busy":"2022-08-01T16:47:09.397446Z","iopub.execute_input":"2022-08-01T16:47:09.397882Z","iopub.status.idle":"2022-08-01T16:47:09.407257Z","shell.execute_reply.started":"2022-08-01T16:47:09.397846Z","shell.execute_reply":"2022-08-01T16:47:09.406255Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset = AmexDataset(train_label, is_test=False)","metadata":{"execution":{"iopub.status.busy":"2022-08-01T16:47:10.881890Z","iopub.execute_input":"2022-08-01T16:47:10.882923Z","iopub.status.idle":"2022-08-01T16:47:10.888394Z","shell.execute_reply.started":"2022-08-01T16:47:10.882872Z","shell.execute_reply":"2022-08-01T16:47:10.887163Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset[100]['x'].shape","metadata":{"execution":{"iopub.status.busy":"2022-08-01T16:47:20.899718Z","iopub.execute_input":"2022-08-01T16:47:20.900783Z","iopub.status.idle":"2022-08-01T16:47:20.920000Z","shell.execute_reply.started":"2022-08-01T16:47:20.900731Z","shell.execute_reply":"2022-08-01T16:47:20.919101Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def amex_metric_mod(y_true, y_pred):\n\n    labels     = np.transpose(np.array([y_true, y_pred]))\n    labels     = labels[labels[:, 1].argsort()[::-1]]\n    weights    = np.where(labels[:,0]==0, 20, 1)\n    cut_vals   = labels[np.cumsum(weights) <= int(0.04 * np.sum(weights))]\n    top_four   = np.sum(cut_vals[:,0]) / np.sum(labels[:,0])\n\n    gini = [0,0]\n    for i in [1,0]:\n        labels         = np.transpose(np.array([y_true, y_pred]))\n        labels         = labels[labels[:, i].argsort()[::-1]]\n        weight         = np.where(labels[:,0]==0, 20, 1)\n        weight_random  = np.cumsum(weight / np.sum(weight))\n        total_pos      = np.sum(labels[:, 0] *  weight)\n        cum_pos_found  = np.cumsum(labels[:, 0] * weight)\n        lorentz        = cum_pos_found / total_pos\n        gini[i]        = np.sum((lorentz - weight_random) * weight)\n\n    return 0.5 * (gini[1]/gini[0] + top_four)","metadata":{"execution":{"iopub.status.busy":"2022-08-01T16:44:43.826220Z","iopub.execute_input":"2022-08-01T16:44:43.826947Z","iopub.status.idle":"2022-08-01T16:44:43.839275Z","shell.execute_reply.started":"2022-08-01T16:44:43.826906Z","shell.execute_reply":"2022-08-01T16:44:43.838084Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def loss_fn(x, targets):\n    #print(targets)\n    #print(x)\n    loss = nn.BCEWithLogitsLoss()\n    return loss(x,targets)","metadata":{"execution":{"iopub.status.busy":"2022-08-01T17:34:33.343681Z","iopub.execute_input":"2022-08-01T17:34:33.344582Z","iopub.status.idle":"2022-08-01T17:34:33.352432Z","shell.execute_reply.started":"2022-08-01T17:34:33.344543Z","shell.execute_reply":"2022-08-01T17:34:33.351482Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class SpatialDropout(nn.Module):\n    def __init__(self, rate):\n        super(SpatialDropout, self).__init__()\n        self.drop = nn.Dropout(rate)\n        \n    def forward(self, x):\n        x = self.drop(x.unsqueeze(-1)).squeeze(-1)\n        return x\n\n\nclass AmexModel(nn.Module):\n    def __init__(self, in_features, out_features):\n        super(AmexModel, self).__init__()\n        \n        '''\n        self.in_feat = in_features\n        self.out_feat = out_features\n        self.lstm_1 = nn.LSTM(self.in_feat,500, num_layers=2, batch_first=True)\n        self.lstm_2 = nn.LSTM(500,128, num_layers=2, batch_first=True)\n        self.linear = nn.Linear(128,self.out_feat)\n        self.sigmoid = nn.Sigmoid()\n        '''\n        self.layers = nn.Sequential(nn.Conv1d(in_features, in_features - 1, kernel_size=1, stride=1), #bottleneck\n                                    nn.Conv1d(in_features - 1, 64, kernel_size=3, stride=1), nn.BatchNorm1d(64), nn.ReLU(), #21\n                                    nn.Conv1d(64, 128, kernel_size=3, stride=1), nn.BatchNorm1d(128), SpatialDropout(0.1), nn.ReLU()) #19\n        self.rnn = nn.LSTM(128, 128, num_layers=4, batch_first=True)\n        self.top = nn.Linear(128, 1).to(Config.DEVICE)\n    \n    \n    \n    def forward(self, x, targets=None):\n\n        \n\n        \n        \n        x = self.layers(x.transpose(2, 1))\n        x, _ = self.rnn(x.transpose(2, 1))\n        x = self.top(x[:, -1, :])\n        #x = nn.Sigmoid(x)\n\n        if targets is None:\n            #print('Im here')\n            return x\n        else:\n            loss = loss_fn(x,targets)\n            return x, loss","metadata":{"execution":{"iopub.status.busy":"2022-08-01T17:34:49.114749Z","iopub.execute_input":"2022-08-01T17:34:49.115776Z","iopub.status.idle":"2022-08-01T17:34:49.130097Z","shell.execute_reply.started":"2022-08-01T17:34:49.115719Z","shell.execute_reply":"2022-08-01T17:34:49.129090Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train_fn(model, data_loader, optimizer):\n    model.train()\n    fin_loss = 0\n    tk0 = tqdm(data_loader, total=len(data_loader))\n    for data in tk0:\n        #print('I am here')\n\n        for key, value in data.items():\n            #print(key)\n            #print(value)\n            data[key] = value.to(Config.DEVICE)\n        \n        #model.hidden_state = model.init_hidden(Config.BATCH_SIZE, Config.DEVICE)\n        \n        optimizer.zero_grad()\n        _, loss = model(**data)\n        \n        \n        \n        loss.backward()\n        optimizer.step()\n\t\t\n        #scheduler.step()\n        fin_loss += loss.item()\n    return fin_loss / len(data_loader)","metadata":{"execution":{"iopub.status.busy":"2022-08-01T17:41:32.470665Z","iopub.execute_input":"2022-08-01T17:41:32.471490Z","iopub.status.idle":"2022-08-01T17:41:32.479276Z","shell.execute_reply.started":"2022-08-01T17:41:32.471449Z","shell.execute_reply":"2022-08-01T17:41:32.478050Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def eval_fn(model, data_loader):\n    model.eval()\n    fin_loss = 0\n    fin_preds = []\n    tk0 = tqdm(data_loader, total=len(data_loader))\n    with torch.no_grad():\n        for data in tk0:\n            #a\n            for key, value in data.items():\n                #print(key)\n                data[key] = value.to(Config.DEVICE)\n            #model.hidden_state = model.init_hidden(Config.BATCH_SIZE, Config.DEVICE)\n            batch_preds, loss = model(**data)\n            fin_loss += loss.item()\n            fin_preds.append(batch_preds.cpu().detach().numpy())\n\n    \n    return fin_preds, fin_loss / len(data_loader)","metadata":{"execution":{"iopub.status.busy":"2022-08-01T16:47:59.503784Z","iopub.execute_input":"2022-08-01T16:47:59.504517Z","iopub.status.idle":"2022-08-01T16:47:59.512519Z","shell.execute_reply.started":"2022-08-01T16:47:59.504480Z","shell.execute_reply":"2022-08-01T16:47:59.511456Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def run_training(label):\n    x= label\n    y = label['target']\n    #x_test = test.values\n    \n    oof_predictions = np.zeros(x.shape[0])\n    #test_predictions = np.zeros(x_test.shape[0])\n    \n    kfold = StratifiedKFold(n_splits=5, random_state=42, shuffle=True)\n    for fold, (trn_ind, val_ind) in enumerate(kfold.split(x, y=y)):\n        print(f'Training fold {fold + 1}')\n        x_train, x_val = x.iloc[trn_ind], x.iloc[val_ind]\n        y_train, y_val = y.iloc[trn_ind], y.iloc[val_ind]\n        \n        #print(x_val[:20])\n        \n        train_ds = AmexDataset(data= x_train, train_df=train, is_test=False)\n\n        valid_ds = AmexDataset(data=x_val, train_df=train, is_test=False)\n        \n        #sc = StandardScaler()\n        \n        #sc.fit(train_ds['x'])\n        \n        #train_ds['x'] = sc.transform(train_ds['x'])\n        #valid_ds['x'] = sc.transform(valid_ds['x'])\n\n        #test_ds = TimeDataset(F_matrix= test_F_matrix,data=x_test, is_test=True)\n        train_sample_strategy = torch.utils.data.sampler.WeightedRandomSampler(np.ones(x_train.shape[0]),\n                                                                               num_samples=x_train.shape[0], replacement=False)\n        \n        train_loader = DataLoader(train_ds, batch_size=Config.BATCH_SIZE,  num_workers=2, sampler=train_sample_strategy,\n                                  drop_last=True, pin_memory=True)\n        \n        val_sample_strategy = torch.utils.data.sampler.WeightedRandomSampler(np.ones(x_val.shape[0]),\n                                                                             num_samples=x_val.shape[0], replacement=False)\n\n        val_loader = DataLoader(valid_ds, batch_size=Config.BATCH_SIZE, num_workers=2, sampler=val_sample_strategy,\n                               drop_last=False, pin_memory=True)\n\n        #test_loader = DataLoader(test_ds, batch_size=Config.BATCH_SIZE, shuffle=False, num_workers=2,\n        #                  pin_memory=False, drop_last=False)\n        \n        model = AmexModel(in_features=len(req_cols), out_features=1)\n        #model = gru_model(in_feats=len(req_cols))\n        print(torch.device(0))\n        #model.hidden_state = model.init_hidden(1, Config.DEVICE)\n        model.to(Config.DEVICE)\n        \n        optimizer = optim.Adam(model.parameters(),lr=Config.learning_rate, betas=(0.9, 0.999),\n                                 eps=1e-08)\n        \n        \n        loss = 1000\n        v_preds = []\n        #t_preds = []\n        \n        for epoch in range(Config.EPOCHS):\n            train_loss = train_fn(model, train_loader, optimizer)\n            valid_preds, valid_loss = eval_fn(model, val_loader)\n            #test_preds = pred_fn(model, test_loader)\n            \n            #amex_metric = amex_metric_mod(y_val, valid_preds)\n\n            print(f'train loss is {train_loss} and valid loss is {valid_loss}')\n            \n            if valid_loss < loss:\n                v_preds = valid_preds\n                #t_preds = test_preds\n                loss = valid_loss\n            \n\n        #print(np.concatenate(v_preds,axis=0))\n        a = np.concatenate(v_preds, axis=0)\n        b = np.concatenate(a, axis=0)\n        oof_predictions[val_ind] = b\n\n        #c = np.concatenate(t_preds, axis=0)\n        #d = np.concatenate(c, axis=0)\n\n\n        #print(d)\n        #test_predictions += np.clip(np.round(d),0,3)\n        \n    \n    \n    return oof_predictions","metadata":{"execution":{"iopub.status.busy":"2022-08-01T17:41:44.553927Z","iopub.execute_input":"2022-08-01T17:41:44.554367Z","iopub.status.idle":"2022-08-01T17:41:44.573091Z","shell.execute_reply.started":"2022-08-01T17:41:44.554336Z","shell.execute_reply":"2022-08-01T17:41:44.572013Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"_=gc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-08-01T16:53:04.373550Z","iopub.execute_input":"2022-08-01T16:53:04.374637Z","iopub.status.idle":"2022-08-01T16:53:04.517087Z","shell.execute_reply.started":"2022-08-01T16:53:04.374587Z","shell.execute_reply":"2022-08-01T16:53:04.516050Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nos.environ['CUDA_LAUNCH_BLOCKING'] = \"1\"","metadata":{"execution":{"iopub.status.busy":"2022-08-01T16:53:05.852211Z","iopub.execute_input":"2022-08-01T16:53:05.852984Z","iopub.status.idle":"2022-08-01T16:53:05.857891Z","shell.execute_reply.started":"2022-08-01T16:53:05.852940Z","shell.execute_reply":"2022-08-01T16:53:05.856848Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"oof_predictions = run_training(train_label[100:20000])","metadata":{"execution":{"iopub.status.busy":"2022-08-01T17:41:47.942546Z","iopub.execute_input":"2022-08-01T17:41:47.943300Z","iopub.status.idle":"2022-08-01T17:48:40.446610Z","shell.execute_reply.started":"2022-08-01T17:41:47.943262Z","shell.execute_reply":"2022-08-01T17:48:40.444962Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"amex_metric_mod(train_label['target'][100:20000], oof_predictions)","metadata":{"execution":{"iopub.status.busy":"2022-08-01T16:43:20.207439Z","iopub.status.idle":"2022-08-01T16:43:20.208576Z","shell.execute_reply.started":"2022-08-01T16:43:20.208307Z","shell.execute_reply":"2022-08-01T16:43:20.208333Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_label['target'][100:20000].value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-08-01T16:43:20.209873Z","iopub.status.idle":"2022-08-01T16:43:20.210749Z","shell.execute_reply.started":"2022-08-01T16:43:20.210462Z","shell.execute_reply":"2022-08-01T16:43:20.210486Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}