{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"#### 노트북 셀 가로 넓게 보기\nfrom IPython.core.display import display, HTML\n\nnotebook_config = \"\"\nnotebook_config += \"\"\nHTML(notebook_config)\n\n# GPU 모니터링\n# watch -n 0.1 nvidia-smi","metadata":{"executionInfo":{"elapsed":5,"status":"ok","timestamp":1655014774192,"user":{"displayName":"‍이성만(대학원생-AI빅데이터전공)","userId":"10915876580228061149"},"user_tz":-540},"id":"F1COLST8BB3_","outputId":"e994a270-c34c-45e1-f464-1a6cfa716732","execution":{"iopub.status.busy":"2022-08-10T14:22:25.459633Z","iopub.execute_input":"2022-08-10T14:22:25.460357Z","iopub.status.idle":"2022-08-10T14:22:25.499967Z","shell.execute_reply.started":"2022-08-10T14:22:25.460240Z","shell.execute_reply":"2022-08-10T14:22:25.499127Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!nvidia-smi","metadata":{"execution":{"iopub.status.busy":"2022-08-10T14:22:25.502522Z","iopub.execute_input":"2022-08-10T14:22:25.503075Z","iopub.status.idle":"2022-08-10T14:22:26.543195Z","shell.execute_reply.started":"2022-08-10T14:22:25.503041Z","shell.execute_reply":"2022-08-10T14:22:26.542188Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Set Common Config","metadata":{"id":"PpDQRHvVasIT"}},{"cell_type":"code","source":"USE_MADE_FEATURES = True\nIS_INFERENCE = True\nIS_KAGGLE = True\n\nSEED = 42\nN_FOLD = 4\nSUBMISSION_FOLD = [0, 1, 2, 3]\nPROGRAM_VERSION = \"01ALL_codebert\"\nGRADIENT_ACCUMULATION_STEPS = 4 # nbme 2\nEPOCHS = 5\nNUM_WORKERS = 8\nMD_MAX_LEN = 128\nTOTAL_MAX_LEN = 512\nBATCH_SIZE = 8\nPATIENCE = 3\nif IS_INFERENCE: BATCH_SIZE = 64\n\n\n\nMODEL_NAME = 'microsoft/codebert-base'\n# MODEL_NAME = 'microsoft/graphcodebert-base'\n# MODEL_NAME = 'microsoft/codebert-base-mlm'\n\n\nDATA_PATH = '/content/drive/MyDrive/kaggle/ai4code/data/'\nSUBMISSION_PATH = '/content/drive/MyDrive/kaggle/ai4code/sub/'\nMODEL_PATH = '/content/drive/MyDrive/kaggle/ai4code/model/'\nCLEANED_PATH = '/content/drive/MyDrive/kaggle/ai4code/clean_all/'\nCONFIG_PATH = MODEL_PATH + 'config.pth'\nTOKENIZER = None\nif IS_KAGGLE:\n DATA_PATH = '../input/AI4Code/'\n SUBMISSION_PATH = './'\n MODEL_PATH = './ai4code-model-codebert-base/' # 기본 모델 생성 디렉토리\n CLEANED_PATH = './ai4code-clean-codebert-base/'\n CONFIG_PATH = MODEL_PATH + 'config.pth'\n if IS_INFERENCE: # inference만 하는 경우\n MODEL_PATH = '../input/ai4code-model-codebert-base/'\n CLEANED_PATH = '../input/ai4code-clean-codebert-base/'\n CONFIG_PATH = MODEL_PATH + 'config.pth'\n NUM_WORKERS = 0\n","metadata":{"executionInfo":{"elapsed":6,"status":"ok","timestamp":1655014777583,"user":{"displayName":"‍이성만(대학원생-AI빅데이터전공)","userId":"10915876580228061149"},"user_tz":-540},"id":"lexJt5IVBB4A","execution":{"iopub.status.busy":"2022-08-10T14:22:26.545148Z","iopub.execute_input":"2022-08-10T14:22:26.545568Z","iopub.status.idle":"2022-08-10T14:22:26.553850Z","shell.execute_reply.started":"2022-08-10T14:22:26.545527Z","shell.execute_reply":"2022-08-10T14:22:26.553077Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# colab에서 실행하는 경우 google drive에 연결 및 colab 서버에 drive mount\nif not IS_KAGGLE:\n from google.colab import drive\n drive.mount('/content/drive')","metadata":{"executionInfo":{"elapsed":1965,"status":"ok","timestamp":1655014779543,"user":{"displayName":"‍이성만(대학원생-AI빅데이터전공)","userId":"10915876580228061149"},"user_tz":-540},"id":"RTkOq6TMBB4B","outputId":"354a4aba-458a-42e5-a2b3-44e7f4ee0108","execution":{"iopub.status.busy":"2022-08-10T14:22:26.555355Z","iopub.execute_input":"2022-08-10T14:22:26.556061Z","iopub.status.idle":"2022-08-10T14:22:26.563558Z","shell.execute_reply.started":"2022-08-10T14:22:26.556026Z","shell.execute_reply":"2022-08-10T14:22:26.562842Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if not IS_KAGGLE:\n !pip install transformers","metadata":{"execution":{"iopub.status.busy":"2022-08-10T14:22:26.566139Z","iopub.execute_input":"2022-08-10T14:22:26.566927Z","iopub.status.idle":"2022-08-10T14:22:26.572496Z","shell.execute_reply.started":"2022-08-10T14:22:26.566892Z","shell.execute_reply":"2022-08-10T14:22:26.571788Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Import Basic Modules","metadata":{"id":"OGkhvGWkksSH"}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\n%matplotlib inline\nimport matplotlib.pyplot as plt\nimport re\nimport platform\nfrom tqdm import tqdm\ntqdm.pandas()\nimport os\nos.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'\nimport random\nimport collections\nimport json\nimport gc\nfrom pathlib import Path\nfrom scipy import sparse\nfrom bisect import bisect\nimport sys\n\nimport warnings; warnings.filterwarnings(\"ignore\")\nfrom scipy.stats import gmean\n\nfrom transformers import AutoModel, AutoTokenizer, AutoConfig, AdamW, get_linear_schedule_with_warmup\n\nfrom torch.utils.data import DataLoader, Dataset\nimport torch.nn.functional as F\nimport torch.nn as nn\nimport torch\n\n\nfrom sklearn.model_selection import GroupShuffleSplit\nfrom sklearn.model_selection import GroupKFold, KFold, StratifiedKFold","metadata":{"executionInfo":{"elapsed":7291,"status":"ok","timestamp":1655014786832,"user":{"displayName":"‍이성만(대학원생-AI빅데이터전공)","userId":"10915876580228061149"},"user_tz":-540},"id":"94lEG53S4xm9","execution":{"iopub.status.busy":"2022-08-10T14:22:26.573813Z","iopub.execute_input":"2022-08-10T14:22:26.574419Z","iopub.status.idle":"2022-08-10T14:22:33.978959Z","shell.execute_reply.started":"2022-08-10T14:22:26.574385Z","shell.execute_reply":"2022-08-10T14:22:33.977988Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\nprint('device : ', device)","metadata":{"id":"U-focurQN_Wr","executionInfo":{"status":"ok","timestamp":1655014786833,"user_tz":-540,"elapsed":34,"user":{"displayName":"‍이성만(대학원생-AI빅데이터전공)","userId":"10915876580228061149"}},"outputId":"45ac4f79-1a90-4209-ed1b-8a30a493c318","execution":{"iopub.status.busy":"2022-08-10T14:22:33.980915Z","iopub.execute_input":"2022-08-10T14:22:33.981883Z","iopub.status.idle":"2022-08-10T14:22:34.049320Z","shell.execute_reply.started":"2022-08-10T14:22:33.981846Z","shell.execute_reply":"2022-08-10T14:22:34.048467Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Define functions","metadata":{"id":"YlkzsDxCmpHD"}},{"cell_type":"markdown","source":"- 재현성 구성함수","metadata":{"id":"Ipgetq6-5P7A"}},{"cell_type":"code","source":"def reset_seeds(seed, display=False):\n np.random.seed(seed)\n random.seed(seed)\n os.environ['PYTHONHASHSEED'] = str(seed) \n torch.manual_seed(seed)\n torch.cuda.manual_seed(seed)\n \n os.environ['CUDA_VISIBLE_DEVICES'] = '' # for GPU\n if display:\n print(\"RANDOM SEEDS RESET {}\".format(seed)) # optional\n\nreset_seeds(SEED, True)","metadata":{"executionInfo":{"elapsed":22,"status":"ok","timestamp":1655014786833,"user":{"displayName":"‍이성만(대학원생-AI빅데이터전공)","userId":"10915876580228061149"},"user_tz":-540},"id":"0Fh7Va0YBB4C","outputId":"7b0ab00c-4898-4f13-bbc5-f0e61594cbcb","execution":{"iopub.status.busy":"2022-08-10T14:22:34.051056Z","iopub.execute_input":"2022-08-10T14:22:34.051505Z","iopub.status.idle":"2022-08-10T14:22:34.062706Z","shell.execute_reply.started":"2022-08-10T14:22:34.051471Z","shell.execute_reply":"2022-08-10T14:22:34.061763Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"- 데이타 읽기","metadata":{"id":"_O4PPsBq5P7B"}},{"cell_type":"code","source":"def read_notebook(path):\n return (\n pd.read_json(\n path,\n dtype={'cell_type': 'category', 'source': 'str'})\n .assign(id=path.stem)\n .rename_axis('cell_id')\n )","metadata":{"executionInfo":{"elapsed":20,"status":"ok","timestamp":1655014786833,"user":{"displayName":"‍이성만(대학원생-AI빅데이터전공)","userId":"10915876580228061149"},"user_tz":-540},"id":"Kk0rUBFhGzGO","execution":{"iopub.status.busy":"2022-08-10T14:22:34.064246Z","iopub.execute_input":"2022-08-10T14:22:34.064640Z","iopub.status.idle":"2022-08-10T14:22:34.070309Z","shell.execute_reply.started":"2022-08-10T14:22:34.064606Z","shell.execute_reply":"2022-08-10T14:22:34.069397Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"- 순위 적용","metadata":{"id":"kHMQhd_o5P7C"}},{"cell_type":"code","source":"def get_ranks(base, derived):\n return [base.index(d) for d in derived]","metadata":{"id":"nwIOXqzs5P7C","executionInfo":{"status":"ok","timestamp":1655014786834,"user_tz":-540,"elapsed":21,"user":{"displayName":"‍이성만(대학원생-AI빅데이터전공)","userId":"10915876580228061149"}},"execution":{"iopub.status.busy":"2022-08-10T14:22:34.071953Z","iopub.execute_input":"2022-08-10T14:22:34.072692Z","iopub.status.idle":"2022-08-10T14:22:34.079260Z","shell.execute_reply.started":"2022-08-10T14:22:34.072654Z","shell.execute_reply":"2022-08-10T14:22:34.078151Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"- 데이타 분할","metadata":{"id":"NPKY9ht05P7C"}},{"cell_type":"code","source":"def cv_split(data, n_fold):\n if data is None:\n return data\n fold = GroupKFold(n_splits=n_fold)\n groups = data['ancestor_id'].values\n for n, (train_index, val_index) in enumerate(fold.split(data, data['cell_type'], groups)):\n data.loc[val_index, f'model_fold_num'] = int(n)\n data[f'model_fold_num'] = data[f'model_fold_num'].astype(int)\n print(data.groupby(f'model_fold_num').size())\n\n return data","metadata":{"id":"UtmHGXgd5P7C","executionInfo":{"status":"ok","timestamp":1655014786834,"user_tz":-540,"elapsed":21,"user":{"displayName":"‍이성만(대학원생-AI빅데이터전공)","userId":"10915876580228061149"}},"execution":{"iopub.status.busy":"2022-08-10T14:22:34.080619Z","iopub.execute_input":"2022-08-10T14:22:34.081172Z","iopub.status.idle":"2022-08-10T14:22:34.089733Z","shell.execute_reply.started":"2022-08-10T14:22:34.081137Z","shell.execute_reply":"2022-08-10T14:22:34.088606Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"- 모델 입력데이타 전처리","metadata":{"id":"HpD4QPNg5P7D"}},{"cell_type":"code","source":"def clean_code(cell):\n return str(cell).replace(\"\\\\n\", \"\\n\")","metadata":{"id":"9yGr-QA95P7D","executionInfo":{"status":"ok","timestamp":1655014786834,"user_tz":-540,"elapsed":20,"user":{"displayName":"‍이성만(대학원생-AI빅데이터전공)","userId":"10915876580228061149"}},"execution":{"iopub.status.busy":"2022-08-10T14:22:34.091344Z","iopub.execute_input":"2022-08-10T14:22:34.091817Z","iopub.status.idle":"2022-08-10T14:22:34.099611Z","shell.execute_reply.started":"2022-08-10T14:22:34.091768Z","shell.execute_reply":"2022-08-10T14:22:34.098740Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"- 모델 입력데이타 Sampling","metadata":{"id":"Wka_p7uV5P7D"}},{"cell_type":"code","source":"def sample_cells(cells, n):\n cells = [clean_code(cell) for cell in cells]\n if n >= len(cells):\n return [cell[:200] for cell in cells]\n else:\n results = []\n step = len(cells) / n\n idx = 0\n while int(np.round(idx)) < len(cells):\n results.append(cells[int(np.round(idx))])\n idx += step\n assert cells[0] in results\n if cells[-1] not in results:\n results[-1] = cells[-1]\n return results","metadata":{"id":"DM_rEaNy5P7D","executionInfo":{"status":"ok","timestamp":1655014786834,"user_tz":-540,"elapsed":20,"user":{"displayName":"‍이성만(대학원생-AI빅데이터전공)","userId":"10915876580228061149"}},"execution":{"iopub.status.busy":"2022-08-10T14:22:34.101064Z","iopub.execute_input":"2022-08-10T14:22:34.101455Z","iopub.status.idle":"2022-08-10T14:22:34.113124Z","shell.execute_reply.started":"2022-08-10T14:22:34.101395Z","shell.execute_reply":"2022-08-10T14:22:34.112358Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"- 모델 입력데이타 Feature Engineering","metadata":{"id":"Pf5PaYbs5P7D"}},{"cell_type":"code","source":"def get_features(df):\n features = dict()\n df = df.sort_values(\"rank\").reset_index(drop=True)\n for idx, sub_df in tqdm(df.groupby(\"id\")):\n features[idx] = dict()\n total_md = sub_df[sub_df.cell_type == \"markdown\"].shape[0]\n code_sub_df = sub_df[sub_df.cell_type == \"code\"]\n total_code = code_sub_df.shape[0]\n codes = sample_cells(code_sub_df.source.values, 20)\n features[idx][\"total_code\"] = total_code\n features[idx][\"total_md\"] = total_md\n features[idx][\"codes\"] = codes\n return features","metadata":{"id":"IttYIKkg5P7E","executionInfo":{"status":"ok","timestamp":1655014786835,"user_tz":-540,"elapsed":21,"user":{"displayName":"‍이성만(대학원생-AI빅데이터전공)","userId":"10915876580228061149"}},"execution":{"iopub.status.busy":"2022-08-10T14:22:34.117990Z","iopub.execute_input":"2022-08-10T14:22:34.118402Z","iopub.status.idle":"2022-08-10T14:22:34.126613Z","shell.execute_reply.started":"2022-08-10T14:22:34.118377Z","shell.execute_reply":"2022-08-10T14:22:34.125767Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"- Metric 함수","metadata":{"id":"znhgL1cv5P7E"}},{"cell_type":"code","source":"def count_inversions(a):\n inversions = 0\n sorted_so_far = []\n for i, u in enumerate(a):\n j = bisect(sorted_so_far, u)\n inversions += i - j\n sorted_so_far.insert(j, u)\n return inversions\n\n\ndef kendall_tau(ground_truth, predictions):\n total_inversions = 0\n total_2max = 0 # twice the maximum possible inversions across all instances\n for gt, pred in zip(ground_truth, predictions):\n ranks = [gt.index(x) for x in pred] # rank predicted order in terms of ground truth\n total_inversions += count_inversions(ranks)\n n = len(gt)\n total_2max += n * (n - 1)\n return 1 - 4 * total_inversions / total_2max","metadata":{"id":"imazZtnZ5P7E","executionInfo":{"status":"ok","timestamp":1655014786835,"user_tz":-540,"elapsed":21,"user":{"displayName":"‍이성만(대학원생-AI빅데이터전공)","userId":"10915876580228061149"}},"execution":{"iopub.status.busy":"2022-08-10T14:22:34.128227Z","iopub.execute_input":"2022-08-10T14:22:34.128597Z","iopub.status.idle":"2022-08-10T14:22:34.137065Z","shell.execute_reply.started":"2022-08-10T14:22:34.128566Z","shell.execute_reply":"2022-08-10T14:22:34.135987Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"- Read Data From Dataloader ","metadata":{"id":"jr-CD3Al5P7E"}},{"cell_type":"code","source":"def read_data(data):\n return tuple(d.cuda() for d in data[:-1]), data[-1].cuda()","metadata":{"id":"KCKT9WwF5P7E","executionInfo":{"status":"ok","timestamp":1655014786835,"user_tz":-540,"elapsed":20,"user":{"displayName":"‍이성만(대학원생-AI빅데이터전공)","userId":"10915876580228061149"}},"execution":{"iopub.status.busy":"2022-08-10T14:22:34.138554Z","iopub.execute_input":"2022-08-10T14:22:34.139635Z","iopub.status.idle":"2022-08-10T14:22:34.145627Z","shell.execute_reply.started":"2022-08-10T14:22:34.139600Z","shell.execute_reply":"2022-08-10T14:22:34.144782Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"- n번째 fold의 모델 파일이 있는 경우 삭제","metadata":{}},{"cell_type":"code","source":"def remove_file(n_fold):\n if IS_KAGGLE:\n model_list_path = f'{MODEL_PATH}'\n else:\n model_list_path = f'{MODEL_PATH}{PROGRAM_VERSION}'\n file_list = os.listdir(model_list_path)\n file_list = [file_name for file_name in file_list if PROGRAM_VERSION in file_name]\n model_filename = [file_name for file_name in file_list if 'WEIGHTS_' + str(n_fold) in file_name]\n for file in model_filename:\n if os.path.isfile(model_list_path + '/' + file):\n os.remove(model_list_path + '/' + file)","metadata":{"execution":{"iopub.status.busy":"2022-08-10T14:22:34.146873Z","iopub.execute_input":"2022-08-10T14:22:34.147423Z","iopub.status.idle":"2022-08-10T14:22:34.155448Z","shell.execute_reply.started":"2022-08-10T14:22:34.147389Z","shell.execute_reply":"2022-08-10T14:22:34.154513Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"- 사전학습된 모델 파일 확인","metadata":{"id":"gbVx6oiv7aDf"}},{"cell_type":"code","source":"# 인수(idx)로 받은 모델번호의 모델파일명을 불러오는 함수\ndef get_model_filename(idx):\n if IS_KAGGLE:\n model_list_path = f'{MODEL_PATH}'\n else:\n model_list_path = f'{MODEL_PATH}{PROGRAM_VERSION}'\n file_list = os.listdir(model_list_path)\n file_list = [file_name for file_name in file_list if PROGRAM_VERSION in file_name]\n model_filename = [file_name for file_name in file_list if 'WEIGHTS_' + str(idx) in file_name]\n if len(model_filename) == 0:\n return None\n return model_filename[0]\n# get_model_filename(0)","metadata":{"id":"nV9bJJ4-7ZoH","executionInfo":{"status":"ok","timestamp":1655014786835,"user_tz":-540,"elapsed":20,"user":{"displayName":"‍이성만(대학원생-AI빅데이터전공)","userId":"10915876580228061149"}},"execution":{"iopub.status.busy":"2022-08-10T14:22:34.156876Z","iopub.execute_input":"2022-08-10T14:22:34.157649Z","iopub.status.idle":"2022-08-10T14:22:34.167263Z","shell.execute_reply.started":"2022-08-10T14:22:34.157598Z","shell.execute_reply":"2022-08-10T14:22:34.166321Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"- 디렉토리 생성 함","metadata":{"id":"PIis9S1W77Ev"}},{"cell_type":"code","source":"# 디렉토리 생성 함수 (없는 경우만 신규로 생성)\ndef createFolder(directory):\n try:\n if not os.path.exists(directory):\n os.makedirs(directory)\n except OSError:\n print ('Error: Creating directory. ' + directory)\n\n\ncreateFolder(f'{MODEL_PATH}')\ncreateFolder(f'{CLEANED_PATH}')\ncreateFolder(f'{MODEL_PATH}{PROGRAM_VERSION}')","metadata":{"id":"1Gj90cdz79SK","executionInfo":{"status":"ok","timestamp":1655014786836,"user_tz":-540,"elapsed":21,"user":{"displayName":"‍이성만(대학원생-AI빅데이터전공)","userId":"10915876580228061149"}},"execution":{"iopub.status.busy":"2022-08-10T14:22:34.170733Z","iopub.execute_input":"2022-08-10T14:22:34.171038Z","iopub.status.idle":"2022-08-10T14:22:34.179624Z","shell.execute_reply.started":"2022-08-10T14:22:34.171013Z","shell.execute_reply":"2022-08-10T14:22:34.178754Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 데이타 전처리","metadata":{"id":"HeWaJlHkGAv7"}},{"cell_type":"code","source":"if not USE_MADE_FEATURES:\n paths_train = list((Path(f'{DATA_PATH}train')).glob('*.json'))\n notebooks_train = [\n read_notebook(path) for path in tqdm(paths_train, desc='Train NBs')\n ]","metadata":{"executionInfo":{"elapsed":21,"status":"ok","timestamp":1655014786836,"user":{"displayName":"‍이성만(대학원생-AI빅데이터전공)","userId":"10915876580228061149"},"user_tz":-540},"id":"ON-ZZYm5sX0p","execution":{"iopub.status.busy":"2022-08-10T14:22:34.181063Z","iopub.execute_input":"2022-08-10T14:22:34.181540Z","iopub.status.idle":"2022-08-10T14:22:34.187059Z","shell.execute_reply.started":"2022-08-10T14:22:34.181505Z","shell.execute_reply":"2022-08-10T14:22:34.185885Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if not USE_MADE_FEATURES:\n df = (\n pd.concat(notebooks_train)\n .set_index('id', append=True)\n .swaplevel()\n .sort_index(level='id', sort_remaining=False)\n )","metadata":{"executionInfo":{"elapsed":20,"status":"ok","timestamp":1655014786836,"user":{"displayName":"‍이성만(대학원생-AI빅데이터전공)","userId":"10915876580228061149"},"user_tz":-540},"id":"q6ciThNP5J1s","execution":{"iopub.status.busy":"2022-08-10T14:22:34.188988Z","iopub.execute_input":"2022-08-10T14:22:34.189366Z","iopub.status.idle":"2022-08-10T14:22:34.196682Z","shell.execute_reply.started":"2022-08-10T14:22:34.189332Z","shell.execute_reply":"2022-08-10T14:22:34.195653Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if not USE_MADE_FEATURES:\n df_orders = pd.read_csv(\n f'{DATA_PATH}train_orders.csv',\n index_col='id',\n squeeze=True,\n ).str.split()","metadata":{"executionInfo":{"elapsed":20,"status":"ok","timestamp":1655014786836,"user":{"displayName":"‍이성만(대학원생-AI빅데이터전공)","userId":"10915876580228061149"},"user_tz":-540},"id":"lGM4gXl4-Xm5","execution":{"iopub.status.busy":"2022-08-10T14:22:34.198465Z","iopub.execute_input":"2022-08-10T14:22:34.198813Z","iopub.status.idle":"2022-08-10T14:22:34.205887Z","shell.execute_reply.started":"2022-08-10T14:22:34.198771Z","shell.execute_reply":"2022-08-10T14:22:34.204986Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if not USE_MADE_FEATURES:\n df_orders_ = df_orders.to_frame().join(\n df.reset_index('cell_id').groupby('id')['cell_id'].apply(list),\n how='right',\n )","metadata":{"executionInfo":{"elapsed":21,"status":"ok","timestamp":1655014786837,"user":{"displayName":"‍이성만(대학원생-AI빅데이터전공)","userId":"10915876580228061149"},"user_tz":-540},"id":"L6oPZoqRN6Y0","execution":{"iopub.status.busy":"2022-08-10T14:22:34.207904Z","iopub.execute_input":"2022-08-10T14:22:34.209473Z","iopub.status.idle":"2022-08-10T14:22:34.217483Z","shell.execute_reply.started":"2022-08-10T14:22:34.209446Z","shell.execute_reply":"2022-08-10T14:22:34.216637Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if not USE_MADE_FEATURES:\n ranks = {}\n for id_, cell_order, cell_id in df_orders_.itertuples():\n ranks[id_] = {'cell_id': cell_id, 'rank': get_ranks(cell_order, cell_id)}\n df_ranks = (\n pd.DataFrame\n .from_dict(ranks, orient='index')\n .rename_axis('id')\n .apply(pd.Series.explode)\n .set_index('cell_id', append=True)\n )","metadata":{"id":"Lc3FA00a5P7G","executionInfo":{"status":"ok","timestamp":1655014786837,"user_tz":-540,"elapsed":21,"user":{"displayName":"‍이성만(대학원생-AI빅데이터전공)","userId":"10915876580228061149"}},"execution":{"iopub.status.busy":"2022-08-10T14:22:34.218488Z","iopub.execute_input":"2022-08-10T14:22:34.219518Z","iopub.status.idle":"2022-08-10T14:22:34.227360Z","shell.execute_reply.started":"2022-08-10T14:22:34.219480Z","shell.execute_reply":"2022-08-10T14:22:34.226636Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if not USE_MADE_FEATURES:\n df_ancestors = pd.read_csv(f'{DATA_PATH}train_ancestors.csv', index_col='id')\n df = df.reset_index().merge(df_ranks, on=[\"id\", \"cell_id\"]).merge(df_ancestors, on=[\"id\"])\n df[\"pct_rank\"] = df[\"rank\"] / df.groupby(\"id\")[\"cell_id\"].transform(\"count\")","metadata":{"id":"Zt3r7lZu5P7G","executionInfo":{"status":"ok","timestamp":1655014786838,"user_tz":-540,"elapsed":21,"user":{"displayName":"‍이성만(대학원생-AI빅데이터전공)","userId":"10915876580228061149"}},"execution":{"iopub.status.busy":"2022-08-10T14:22:34.228571Z","iopub.execute_input":"2022-08-10T14:22:34.229053Z","iopub.status.idle":"2022-08-10T14:22:34.237745Z","shell.execute_reply.started":"2022-08-10T14:22:34.229019Z","shell.execute_reply":"2022-08-10T14:22:34.236745Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"- 데이타 분할","metadata":{"id":"Jb51NAl65P7G"}},{"cell_type":"code","source":"if not USE_MADE_FEATURES:\n train_data = cv_split(df, N_FOLD)\n display(train_data)","metadata":{"id":"iNx0rP5I5P7G","executionInfo":{"status":"ok","timestamp":1655014786838,"user_tz":-540,"elapsed":21,"user":{"displayName":"‍이성만(대학원생-AI빅데이터전공)","userId":"10915876580228061149"}},"execution":{"iopub.status.busy":"2022-08-10T14:22:34.239028Z","iopub.execute_input":"2022-08-10T14:22:34.239578Z","iopub.status.idle":"2022-08-10T14:22:34.247049Z","shell.execute_reply.started":"2022-08-10T14:22:34.239544Z","shell.execute_reply":"2022-08-10T14:22:34.246351Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"- 기본 전처리 데이타 저장","metadata":{"id":"17rZ6Ypq7Z0Z"}},{"cell_type":"code","source":"if not USE_MADE_FEATURES:\n for i in range(N_FOLD):\n train_df = train_data[train_data.model_fold_num != i]\n val_df = train_data[train_data.model_fold_num == i]\n \n train_df_mark = train_df[train_df[\"cell_type\"] == \"markdown\"].reset_index(drop=True)\n val_df_mark = val_df[val_df[\"cell_type\"] == \"markdown\"].reset_index(drop=True)\n train_df_mark.to_csv(f\"{CLEANED_PATH}train_mark_{i}.csv\", index=False)\n val_df_mark.to_csv(f\"{CLEANED_PATH}val_mark_{i}.csv\", index=False)\n val_df.to_csv(f\"{CLEANED_PATH}val_{i}.csv\", index=False)\n train_df.to_csv(f\"{CLEANED_PATH}train_{i}.csv\", index=False)","metadata":{"executionInfo":{"elapsed":22,"status":"ok","timestamp":1655014786839,"user":{"displayName":"‍이성만(대학원생-AI빅데이터전공)","userId":"10915876580228061149"},"user_tz":-540},"id":"YZo1xciR43s8","execution":{"iopub.status.busy":"2022-08-10T14:22:34.248583Z","iopub.execute_input":"2022-08-10T14:22:34.249049Z","iopub.status.idle":"2022-08-10T14:22:34.258351Z","shell.execute_reply.started":"2022-08-10T14:22:34.249017Z","shell.execute_reply":"2022-08-10T14:22:34.257561Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"- 추가 전처리 데이타 저장","metadata":{"id":"F3Jkb_CA5P7H"}},{"cell_type":"code","source":"%%time\nif not USE_MADE_FEATURES:\n for i in range(N_FOLD):\n train_df = train_data[train_data.model_fold_num != i]\n val_df = train_data[train_data.model_fold_num == i]\n \n val_fts = get_features(val_df)\n json.dump(val_fts, open(f\"{CLEANED_PATH}val_fts_{i}.json\", \"wt\"))\n train_fts = get_features(train_df)\n json.dump(train_fts, open(f\"{CLEANED_PATH}train_fts_{i}.json\",\"wt\"))","metadata":{"executionInfo":{"elapsed":22,"status":"ok","timestamp":1655014786839,"user":{"displayName":"‍이성만(대학원생-AI빅데이터전공)","userId":"10915876580228061149"},"user_tz":-540},"id":"R2MivXXsOqGP","outputId":"baae28f4-cdcf-4ae3-f079-bb5610eedce2","execution":{"iopub.status.busy":"2022-08-10T14:22:34.259460Z","iopub.execute_input":"2022-08-10T14:22:34.259803Z","iopub.status.idle":"2022-08-10T14:22:34.269561Z","shell.execute_reply.started":"2022-08-10T14:22:34.259763Z","shell.execute_reply":"2022-08-10T14:22:34.268613Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Create Tokenizer","metadata":{}},{"cell_type":"code","source":"def make_tokenizer():\n if IS_INFERENCE:\n tokenizer = AutoTokenizer.from_pretrained(MODEL_PATH, normalization=True)\n print(f'local tokenizer object load......[{MODEL_PATH}]')\n else:\n tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME, normalization=True)\n tokenizer.save_pretrained(f'{MODEL_PATH}')\n print(f'train tokenizer object load......[{MODEL_NAME}]')\n\n example = \"Ms. Madden is a 20 yo female presenting\" # train.iloc[0].pn_history \n tokens = tokenizer(example) # example\n tokens_ = tokenizer.tokenize(example) # example\n print(tokens)\n print(tokens_)\n \n return tokenizer\n\nTOKENIZER = make_tokenizer()","metadata":{"execution":{"iopub.status.busy":"2022-08-10T14:22:34.270893Z","iopub.execute_input":"2022-08-10T14:22:34.271350Z","iopub.status.idle":"2022-08-10T14:22:34.427763Z","shell.execute_reply.started":"2022-08-10T14:22:34.271311Z","shell.execute_reply":"2022-08-10T14:22:34.426918Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Define Torch Dataset ","metadata":{"id":"Z9sk-XZm5P7H"}},{"cell_type":"code","source":"class MarkdownDataset(Dataset):\n def __init__(self, df, total_max_len, md_max_len, fts, tokenizer):\n super().__init__()\n self.df = df.reset_index(drop=True)\n self.md_max_len = md_max_len\n self.total_max_len = total_max_len # maxlen allowed by model config\n self.tokenizer = tokenizer\n self.fts = fts\n\n def __getitem__(self, index):\n row = self.df.iloc[index]\n\n inputs = self.tokenizer.encode_plus(\n row.source,\n None,\n add_special_tokens=True,\n max_length=self.md_max_len,\n padding=\"max_length\",\n return_token_type_ids=True,\n truncation=True\n )\n code_inputs = self.tokenizer.batch_encode_plus(\n [str(x) for x in self.fts[row.id][\"codes\"]],\n add_special_tokens=True,\n max_length=23,\n padding=\"max_length\",\n truncation=True\n )\n n_md = self.fts[row.id][\"total_md\"]\n n_code = self.fts[row.id][\"total_md\"] # TODO total_md -> total_code 좀 더 높음\n if n_md + n_code == 0:\n fts = torch.FloatTensor([0])\n else:\n fts = torch.FloatTensor([n_md / (n_md + n_code)])\n\n ids = inputs['input_ids']\n for x in code_inputs['input_ids']:\n ids.extend(x[:-1])\n ids = ids[:self.total_max_len]\n if len(ids) != self.total_max_len:\n ids = ids + [self.tokenizer.pad_token_id, ] * (self.total_max_len - len(ids))\n ids = torch.LongTensor(ids)\n\n mask = inputs['attention_mask']\n for x in code_inputs['attention_mask']:\n mask.extend(x[:-1])\n mask = mask[:self.total_max_len]\n if len(mask) != self.total_max_len:\n mask = mask + [self.tokenizer.pad_token_id, ] * (self.total_max_len - len(mask))\n mask = torch.LongTensor(mask)\n\n assert len(ids) == self.total_max_len\n\n return ids, mask, fts, torch.FloatTensor([row.pct_rank])\n\n def __len__(self):\n return self.df.shape[0]","metadata":{"id":"BZ9T3uNI5P7H","executionInfo":{"status":"ok","timestamp":1655014786839,"user_tz":-540,"elapsed":19,"user":{"displayName":"‍이성만(대학원생-AI빅데이터전공)","userId":"10915876580228061149"}},"execution":{"iopub.status.busy":"2022-08-10T14:22:34.428816Z","iopub.execute_input":"2022-08-10T14:22:34.429155Z","iopub.status.idle":"2022-08-10T14:22:34.442151Z","shell.execute_reply.started":"2022-08-10T14:22:34.429128Z","shell.execute_reply":"2022-08-10T14:22:34.441328Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Define Torch Model","metadata":{"id":"9FpC0r5O5P7H"}},{"cell_type":"code","source":"class MarkdownModel(nn.Module):\n def __init__(self, model_path, config_path=None, pretrained=False):\n super(MarkdownModel, self).__init__()\n # 모델 환경 설정 읽기\n if config_path is None:\n self.config = AutoConfig.from_pretrained(model_path, output_hidden_states=True)\n else:\n self.config = torch.load(config_path)\n # 모델 읽기\n if pretrained:\n self.model = AutoModel.from_pretrained(model_path, config=self.config)\n else:\n self.model = AutoModel.from_config(self.config)\n\n# self.top = nn.Linear(769, 1) # 8*768 -> 8*1\n self.top = nn.Linear(self.config.hidden_size + 1, 1) # 8*768 -> 8*1\n # self._init_weights(self.fc)\n\n def _init_weights(self, module):\n if isinstance(module, nn.Linear):\n module.weight.data.normal_(mean=0.0, std=self.config.initializer_range)\n if module.bias is not None:\n module.bias.data.zero_()\n elif isinstance(module, nn.Embedding):\n module.weight.data.normal_(mean=0.0, std=self.config.initializer_range)\n if module.padding_idx is not None:\n module.weight.data[module.padding_idx].zero_()\n elif isinstance(module, nn.LayerNorm):\n module.bias.data.zero_()\n module.weight.data.fill_(1.0)\n\n def forward(self, ids, mask, fts):\n x = self.model(ids, mask)[0]\n x = torch.cat((x[:, 0, :], fts), 1)\n x = self.top(x)\n return x","metadata":{"id":"ydyRPgsu5P7H","executionInfo":{"status":"ok","timestamp":1655014786839,"user_tz":-540,"elapsed":19,"user":{"displayName":"‍이성만(대학원생-AI빅데이터전공)","userId":"10915876580228061149"}},"execution":{"iopub.status.busy":"2022-08-10T14:22:34.443611Z","iopub.execute_input":"2022-08-10T14:22:34.444350Z","iopub.status.idle":"2022-08-10T14:22:34.456980Z","shell.execute_reply.started":"2022-08-10T14:22:34.444315Z","shell.execute_reply":"2022-08-10T14:22:34.456044Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Define Train","metadata":{"id":"lraWomwl5P7I"}},{"cell_type":"code","source":"# # 1개 미리보기\n# inputs = next(iter(train_loader))\n# len(inputs), inputs[0].shape\n\n# # (4, torch.Size([8, 512]))","metadata":{"id":"M4gRbaw2PSGN","executionInfo":{"status":"ok","timestamp":1655014786840,"user_tz":-540,"elapsed":18,"user":{"displayName":"‍이성만(대학원생-AI빅데이터전공)","userId":"10915876580228061149"}},"execution":{"iopub.status.busy":"2022-08-10T14:22:34.460362Z","iopub.execute_input":"2022-08-10T14:22:34.460706Z","iopub.status.idle":"2022-08-10T14:22:34.468870Z","shell.execute_reply.started":"2022-08-10T14:22:34.460681Z","shell.execute_reply":"2022-08-10T14:22:34.468007Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train(model, train_loader, val_loader, epochs, n_fold):\n # Creating optimizer and lr schedulers\n param_optimizer = list(model.named_parameters())\n no_decay = ['bias', 'LayerNorm.bias', 'LayerNorm.weight']\n optimizer_grouped_parameters = [\n {'params': [p for n, p in param_optimizer if not any(nd in n for nd in no_decay)], 'weight_decay': 0.01},\n {'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)], 'weight_decay': 0.0}\n ]\n\n num_train_optimization_steps = int(epochs * len(train_loader) / GRADIENT_ACCUMULATION_STEPS)\n # To reproduce BertAdam specific behavior set correct_bias=False\n optimizer = AdamW(optimizer_grouped_parameters, lr=3e-5, correct_bias=False) \n # PyTorch scheduler\n num_warmup_steps_size = 0.05 * num_train_optimization_steps\n scheduler = get_linear_schedule_with_warmup(optimizer, num_warmup_steps=num_warmup_steps_size, num_training_steps=num_train_optimization_steps)\n\n criterion = torch.nn.L1Loss()\n scaler = torch.cuda.amp.GradScaler()\n\n n_patience = 0\n best_score = 0. \n for epoch in range(epochs):\n start_time = time.time()\n\n # 초기 epoch인 경우 기존에 훈련한 모델이 있으면 가중치를 읽어 모델에 적용한 후 Score를 구한다.\n if epoch == 0:\n weight_file_name = get_model_filename(i)\n if weight_file_name is not None:\n if IS_KAGGLE:\n model_filename = MODEL_PATH + '/' + weight_file_name\n else:\n model_filename = MODEL_PATH + PROGRAM_VERSION + '/' + weight_file_name\n model.load_state_dict(torch.load(model_filename))\n y_val, y_pred = validate(model, val_loader, epoch, n_fold)\n val_df[\"pred\"] = val_df.groupby([\"id\", \"cell_type\"])[\"rank\"].rank(pct=True)\n val_df.loc[val_df[\"cell_type\"] == \"markdown\", \"pred\"] = y_pred\n y_dummy = val_df.sort_values(\"pred\").groupby('id')['cell_id'].apply(list)\n best_score = kendall_tau(df_orders.loc[y_dummy.index], y_dummy)\n print(f\"{weight_file_name}.....weights loaded.\")\n \n model.train()\n loss_list = []\n preds = []\n labels = []\n\n train_display = tqdm(train_loader, total=len(train_loader), leave=False)\n train_display.set_description(f'Model [{MODEL_NAME}] Fold [{n_fold+1}/{N_FOLD}] Epoch [{epoch+1}/{epochs}]')\n for idx, data in enumerate(train_display):\n inputs, target = read_data(data)\n with torch.cuda.amp.autocast():\n pred = model(*inputs)\n loss = criterion(pred, target)\n scaler.scale(loss).backward()\n if idx % GRADIENT_ACCUMULATION_STEPS == 0 or idx == len(train_display) - 1:\n scaler.step(optimizer)\n scaler.update()\n optimizer.zero_grad()\n scheduler.step()\n loss_list.append(loss.detach().cpu().item())\n preds.append(pred.detach().cpu().numpy().ravel())\n labels.append(target.detach().cpu().numpy().ravel())\n avg_loss = np.round(np.mean(loss_list), 4)\n train_display.set_postfix(Loss=avg_loss, lr=f'{scheduler.get_last_lr()[0]:.5f}')\n\n y_val, y_pred = validate(model, val_loader, epoch, n_fold)\n epoch_display = tqdm([0], leave=True)\n for _ in epoch_display: \n epoch_display.set_description(f'Model [{MODEL_NAME}] Fold [{n_fold+1}/{N_FOLD}] Epoch [{epoch+1}/{epochs}]')\n\n val_df[\"pred\"] = val_df.groupby([\"id\", \"cell_type\"])[\"rank\"].rank(pct=True)\n val_df.loc[val_df[\"cell_type\"] == \"markdown\", \"pred\"] = y_pred\n y_dummy = val_df.sort_values(\"pred\").groupby('id')['cell_id'].apply(list)\n score = kendall_tau(df_orders.loc[y_dummy.index], y_dummy)\n if best_score < score:\n best_score = score\n remove_file(n_fold) # 이전의 모델 파일은 삭제한다.\n model_filename = f'{MODEL_PATH}{PROGRAM_VERSION}/{PROGRAM_VERSION}_WEIGHTS_{n_fold}.{score:.6f}.h5'\n torch.save(model.state_dict(), model_filename)\n n_patience = 0\n else:\n n_patience += 1\n elapsed = time.time() - start_time\n\n epoch_display.set_postfix(Loss=avg_loss, lr=f'{scheduler.get_last_lr()[0]:.5f}', time=f'{elapsed/60:.0f}m', Score=f'{score:.4f}')\n\n # 정해진 epoch동안 성능 개선이 없으면 빠져나간다.\n if n_patience >= PATIENCE:\n break\n \n return model, y_pred","metadata":{"id":"rgak5jz45P7I","executionInfo":{"status":"ok","timestamp":1655014786840,"user_tz":-540,"elapsed":18,"user":{"displayName":"‍이성만(대학원생-AI빅데이터전공)","userId":"10915876580228061149"}},"execution":{"iopub.status.busy":"2022-08-10T14:22:34.470392Z","iopub.execute_input":"2022-08-10T14:22:34.470862Z","iopub.status.idle":"2022-08-10T14:22:34.491697Z","shell.execute_reply.started":"2022-08-10T14:22:34.470820Z","shell.execute_reply":"2022-08-10T14:22:34.490892Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Define Validation","metadata":{"id":"42A9xKQI5P7I"}},{"cell_type":"code","source":"def validate(model, val_loader, epoch, n_fold):\n valid_display = tqdm(val_loader, total=len(val_loader), leave=False)\n valid_display.set_description(f'Model [{MODEL_NAME}] Fold [{n_fold+1}/{N_FOLD}] Epoch [{epoch+1}/{EPOCHS}]')\n\n criterion = torch.nn.L1Loss() \n model.eval()\n preds = []\n labels = []\n loss_list = []\n with torch.no_grad():\n for idx, data in enumerate(valid_display):\n inputs, target = read_data(data)\n with torch.cuda.amp.autocast():\n pred = model(*inputs)\n loss = criterion(pred, target)\n loss_list.append(loss.detach().cpu().item())\n avg_loss = np.round(np.mean(loss_list), 4)\n preds.append(pred.detach().cpu().numpy().ravel())\n labels.append(target.detach().cpu().numpy().ravel())\n valid_display.set_postfix(Loss=avg_loss)\n\n return np.concatenate(labels), np.concatenate(preds)","metadata":{"id":"EXJBJZFE5P7I","executionInfo":{"status":"ok","timestamp":1655014786840,"user_tz":-540,"elapsed":17,"user":{"displayName":"‍이성만(대학원생-AI빅데이터전공)","userId":"10915876580228061149"}},"execution":{"iopub.status.busy":"2022-08-10T14:22:34.493769Z","iopub.execute_input":"2022-08-10T14:22:34.494467Z","iopub.status.idle":"2022-08-10T14:22:34.505643Z","shell.execute_reply.started":"2022-08-10T14:22:34.494414Z","shell.execute_reply":"2022-08-10T14:22:34.504708Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Define Prediction","metadata":{"id":"Em4TBgKl857v"}},{"cell_type":"code","source":"def predict(model, test_df, test_fts):\n model.eval()\n test_df[\"pct_rank\"] = 0\n test_df_mark = test_df[test_df[\"cell_type\"] == \"markdown\"].reset_index(drop=True)\n test_ds = MarkdownDataset(test_df_mark, md_max_len=MD_MAX_LEN, total_max_len=TOTAL_MAX_LEN, fts=test_fts, tokenizer=TOKENIZER)\n test_loader = DataLoader(test_ds, batch_size=BATCH_SIZE, shuffle=False, num_workers=NUM_WORKERS, pin_memory=False, drop_last=False)\n\n preds = []\n test_display = tqdm(test_loader, total=len(test_loader))\n with torch.no_grad():\n for idx, data in enumerate(test_display):\n inputs, target = read_data(data)\n with torch.cuda.amp.autocast():\n pred = model(*inputs)\n preds.append(pred.detach().cpu().numpy().ravel())\n\n y_test = np.concatenate(preds)\n \n return y_test","metadata":{"id":"fXs98_9A88l4","executionInfo":{"status":"ok","timestamp":1655014786840,"user_tz":-540,"elapsed":17,"user":{"displayName":"‍이성만(대학원생-AI빅데이터전공)","userId":"10915876580228061149"}},"execution":{"iopub.status.busy":"2022-08-10T14:22:34.507981Z","iopub.execute_input":"2022-08-10T14:22:34.508271Z","iopub.status.idle":"2022-08-10T14:22:34.517559Z","shell.execute_reply.started":"2022-08-10T14:22:34.508240Z","shell.execute_reply":"2022-08-10T14:22:34.516306Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training","metadata":{"executionInfo":{"elapsed":15,"status":"ok","timestamp":1654787734448,"user":{"displayName":"‍이성만(대학원생-AI빅데이터전공)","userId":"10915876580228061149"},"user_tz":-540},"id":"xZZVkvIs30gR"}},{"cell_type":"code","source":"%%time\nif not IS_INFERENCE:\n model_list = []\n score_list = []\n order_df = pd.read_csv(f\"{DATA_PATH}train_orders.csv\").set_index(\"id\")\n df_orders = pd.read_csv(f\"{DATA_PATH}train_orders.csv\", index_col='id', squeeze=True).str.split()\n for i in range(N_FOLD):\n# if i in [0, 1, 2, 3]: continue\n reset_seeds(SEED)\n train_df_mark = pd.read_csv(f\"{CLEANED_PATH}train_mark_{i}.csv\").drop(\"parent_id\", axis=1).dropna().reset_index(drop=True)\n train_fts = json.load(open(f\"{CLEANED_PATH}train_fts_{i}.json\"))\n val_df_mark = pd.read_csv(f\"{CLEANED_PATH}val_mark_{i}.csv\").drop(\"parent_id\", axis=1).dropna().reset_index(drop=True)\n val_fts = json.load(open(f\"{CLEANED_PATH}val_fts_{i}.json\"))\n val_df = pd.read_csv(f\"{CLEANED_PATH}val_{i}.csv\")\n\n train_ds = MarkdownDataset(train_df_mark, md_max_len=MD_MAX_LEN, total_max_len=TOTAL_MAX_LEN, fts=train_fts, tokenizer=TOKENIZER)\n val_ds = MarkdownDataset(val_df_mark, md_max_len=MD_MAX_LEN, total_max_len=TOTAL_MAX_LEN, fts=val_fts, tokenizer=TOKENIZER)\n train_loader = DataLoader(train_ds, batch_size=BATCH_SIZE, shuffle=True, num_workers=NUM_WORKERS, pin_memory=False, drop_last=True)\n val_loader = DataLoader(val_ds, batch_size=BATCH_SIZE, shuffle=False, num_workers=NUM_WORKERS, pin_memory=False, drop_last=False)\n \n model = MarkdownModel(MODEL_NAME, None, True)\n torch.save(model.config, CONFIG_PATH)\n model = model.cuda()\n model, y_pred = train(model, train_loader, val_loader, EPOCHS, i)\n","metadata":{"executionInfo":{"elapsed":18,"status":"ok","timestamp":1655014786841,"user":{"displayName":"‍이성만(대학원생-AI빅데이터전공)","userId":"10915876580228061149"},"user_tz":-540},"id":"VVHTJ05eI1Vp","outputId":"e5d81092-00ec-4191-81da-83db46b983eb","execution":{"iopub.status.busy":"2022-08-10T14:22:34.518743Z","iopub.execute_input":"2022-08-10T14:22:34.519224Z","iopub.status.idle":"2022-08-10T14:22:34.532786Z","shell.execute_reply.started":"2022-08-10T14:22:34.519174Z","shell.execute_reply":"2022-08-10T14:22:34.531840Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Load Model Weights & Validation","metadata":{}},{"cell_type":"code","source":"get_model_filename(2)","metadata":{"execution":{"iopub.status.busy":"2022-08-10T14:22:34.534286Z","iopub.execute_input":"2022-08-10T14:22:34.534714Z","iopub.status.idle":"2022-08-10T14:22:34.551289Z","shell.execute_reply.started":"2022-08-10T14:22:34.534681Z","shell.execute_reply":"2022-08-10T14:22:34.550290Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in range(N_FOLD):\n weight_file_name = get_model_filename(i)\n print(weight_file_name)","metadata":{"executionInfo":{"elapsed":16,"status":"ok","timestamp":1655014786841,"user":{"displayName":"‍이성만(대학원생-AI빅데이터전공)","userId":"10915876580228061149"},"user_tz":-540},"id":"-UpogqOlss-Z","outputId":"1e6a02bc-681e-4c69-f31c-08d06c955481","execution":{"iopub.status.busy":"2022-08-10T14:22:34.552814Z","iopub.execute_input":"2022-08-10T14:22:34.553471Z","iopub.status.idle":"2022-08-10T14:22:34.560123Z","shell.execute_reply.started":"2022-08-10T14:22:34.553433Z","shell.execute_reply":"2022-08-10T14:22:34.559380Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"- 학습된 모델에 대한 Validation 수행","metadata":{"id":"7i1QCeTp-xjd"}},{"cell_type":"code","source":"%%time\nmodel_list = []\nscore_list = [0.847952, 0.843534, 0.850097, 0.840545]\nfor i in range(N_FOLD):\n # model_path = \"../input/codebert-base/codebert-base/\"\n # ckpt_path = \"../input/ai4codemodelspublic/model.bin\"\n model_filename = get_model_filename(i)\n if model_filename is None: continue\n if IS_KAGGLE:\n model_filename = MODEL_PATH + model_filename\n else:\n model_filename = MODEL_PATH + PROGRAM_VERSION + model_filename\n\n # train_df_mark = pd.read_csv(f\"{CLEANED_PATH}train_mark_{i}.csv\").drop(\"parent_id\", axis=1).dropna().reset_index(drop=True)\n # train_fts = json.load(open(f\"{CLEANED_PATH}train_fts_{i}.json\"))\n# val_df_mark = pd.read_csv(f\"{CLEANED_PATH}val_mark_{i}.csv\").drop(\"parent_id\", axis=1).dropna().reset_index(drop=True)\n# val_fts = json.load(open(f\"{CLEANED_PATH}val_fts_{i}.json\"))\n# val_df = pd.read_csv(f\"{CLEANED_PATH}val_{i}.csv\")\n\n # order_df = pd.read_csv(f\"{DATA_PATH}train_orders.csv\").set_index(\"id\")\n# df_orders = pd.read_csv(f\"{DATA_PATH}train_orders.csv\", index_col='id', squeeze=True).str.split()\n\n # train_ds = MarkdownDataset(train_df_mark, md_max_len=MD_MAX_LEN, total_max_len=TOTAL_MAX_LEN, fts=train_fts, tokenizer=TOKENIZER)\n# val_ds = MarkdownDataset(val_df_mark, md_max_len=MD_MAX_LEN, total_max_len=TOTAL_MAX_LEN, fts=val_fts, tokenizer=TOKENIZER)\n # train_loader = DataLoader(train_ds, batch_size=BATCH_SIZE, shuffle=True, num_workers=NUM_WORKERS, pin_memory=False, drop_last=True)\n# val_loader = DataLoader(val_ds, batch_size=BATCH_SIZE, shuffle=False, num_workers=NUM_WORKERS, pin_memory=False, drop_last=False)\n \n model = MarkdownModel(MODEL_PATH, CONFIG_PATH, False)\n model.load_state_dict(torch.load(model_filename))\n model = model.cuda()\n# y_val, y_pred = validate(model, val_loader, 0, i)\n# val_df[\"pred\"] = val_df.groupby([\"id\", \"cell_type\"])[\"rank\"].rank(pct=True)\n# val_df.loc[val_df[\"cell_type\"] == \"markdown\", \"pred\"] = y_pred\n# y_dummy = val_df.sort_values(\"pred\").groupby('id')['cell_id'].apply(list)\n# score = kendall_tau(df_orders.loc[y_dummy.index], y_dummy)\n print(f'{model_filename} Evaluate...[{score_list[i]}]')\n\n model_list.append(model)\n","metadata":{"executionInfo":{"elapsed":616644,"status":"ok","timestamp":1655015403471,"user":{"displayName":"‍이성만(대학원생-AI빅데이터전공)","userId":"10915876580228061149"},"user_tz":-540},"id":"rPo_Y8IWoP-y","outputId":"8c383203-2dad-4cc0-dd1c-d050e90f7022","execution":{"iopub.status.busy":"2022-08-10T14:23:16.061025Z","iopub.execute_input":"2022-08-10T14:23:16.061708Z","iopub.status.idle":"2022-08-10T14:23:26.149065Z","shell.execute_reply.started":"2022-08-10T14:23:16.061671Z","shell.execute_reply":"2022-08-10T14:23:26.148241Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Validation 에서 얻은 점수들과 평균 점수 확인\nscore_list, np.mean(score_list, axis=0), np.mean(score_list, axis=0) * 2","metadata":{"executionInfo":{"elapsed":21,"status":"ok","timestamp":1655015403471,"user":{"displayName":"‍이성만(대학원생-AI빅데이터전공)","userId":"10915876580228061149"},"user_tz":-540},"id":"P8VQNAjU63cB","outputId":"f66cf5ba-0011-4dc0-e8a0-7aab31007eea","execution":{"iopub.status.busy":"2022-08-10T14:23:26.150502Z","iopub.execute_input":"2022-08-10T14:23:26.151131Z","iopub.status.idle":"2022-08-10T14:23:26.164061Z","shell.execute_reply.started":"2022-08-10T14:23:26.151094Z","shell.execute_reply":"2022-08-10T14:23:26.161337Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 제출용 데이타 준비","metadata":{"id":"gZPIMG3c_Olp"}},{"cell_type":"code","source":"paths_test = list((Path(f'{DATA_PATH}test')).glob('*.json'))\nnotebooks_test = [\n read_notebook(path) for path in tqdm(paths_test, desc='Test NBs')\n]\ntest_df = (\n pd.concat(notebooks_test)\n .set_index('id', append=True)\n .swaplevel()\n .sort_index(level='id', sort_remaining=False)\n).reset_index()\ntest_df[\"rank\"] = test_df.groupby([\"id\", \"cell_type\"]).cumcount()\ntest_df[\"pred\"] = test_df.groupby([\"id\", \"cell_type\"])[\"rank\"].rank(pct=True)","metadata":{"executionInfo":{"elapsed":8,"status":"ok","timestamp":1655015403471,"user":{"displayName":"‍이성만(대학원생-AI빅데이터전공)","userId":"10915876580228061149"},"user_tz":-540},"id":"wouxXcsN0JVH","outputId":"436c33e3-2406-4110-c38f-397efa9ad858","execution":{"iopub.status.busy":"2022-08-10T14:23:26.166832Z","iopub.execute_input":"2022-08-10T14:23:26.168890Z","iopub.status.idle":"2022-08-10T14:23:26.249992Z","shell.execute_reply.started":"2022-08-10T14:23:26.168851Z","shell.execute_reply":"2022-08-10T14:23:26.248852Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(test_df)","metadata":{"id":"v_7l3Ih6ANmf","executionInfo":{"status":"ok","timestamp":1655015403472,"user_tz":-540,"elapsed":7,"user":{"displayName":"‍이성만(대학원생-AI빅데이터전공)","userId":"10915876580228061149"}},"outputId":"57d147ac-d627-4e4e-ea64-b0c769dc6728","execution":{"iopub.status.busy":"2022-08-10T14:23:26.254426Z","iopub.execute_input":"2022-08-10T14:23:26.255271Z","iopub.status.idle":"2022-08-10T14:23:26.274345Z","shell.execute_reply.started":"2022-08-10T14:23:26.255182Z","shell.execute_reply":"2022-08-10T14:23:26.273548Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_fts = get_features(test_df)","metadata":{"id":"gzkNUK0xAQ4l","executionInfo":{"status":"ok","timestamp":1655015403472,"user_tz":-540,"elapsed":6,"user":{"displayName":"‍이성만(대학원생-AI빅데이터전공)","userId":"10915876580228061149"}},"outputId":"4763001e-e03a-40b2-c8f5-7378de37abd4","execution":{"iopub.status.busy":"2022-08-10T14:23:26.275508Z","iopub.execute_input":"2022-08-10T14:23:26.276414Z","iopub.status.idle":"2022-08-10T14:23:26.293159Z","shell.execute_reply.started":"2022-08-10T14:23:26.276349Z","shell.execute_reply":"2022-08-10T14:23:26.292348Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"- StratifiedKFold() 5개 모델 각각 예측 수행","metadata":{"id":"7PgWuSwt_RjS"}},{"cell_type":"code","source":"# 모델별 예측\ny_test_2_list = []\nfor idx, model in enumerate(model_list):\n if idx in SUBMISSION_FOLD:\n y_test_2 = predict(model, test_df, test_fts)\n y_test_2_list.append(y_test_2)\ny_test_2_list","metadata":{"id":"LblGGWKkJNui","executionInfo":{"status":"ok","timestamp":1655015412279,"user_tz":-540,"elapsed":8811,"user":{"displayName":"‍이성만(대학원생-AI빅데이터전공)","userId":"10915876580228061149"}},"outputId":"59fd8b93-545c-47cb-e79f-8452f3ac7747","execution":{"iopub.status.busy":"2022-08-10T14:23:26.294539Z","iopub.execute_input":"2022-08-10T14:23:26.301480Z","iopub.status.idle":"2022-08-10T14:23:30.731148Z","shell.execute_reply.started":"2022-08-10T14:23:26.295233Z","shell.execute_reply":"2022-08-10T14:23:30.730217Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# fold 평균 구하기","metadata":{"id":"3vfgtbBOAyLv","executionInfo":{"status":"ok","timestamp":1655015412279,"user_tz":-540,"elapsed":19,"user":{"displayName":"‍이성만(대학원생-AI빅데이터전공)","userId":"10915876580228061149"}},"execution":{"iopub.status.busy":"2022-08-10T14:23:30.732663Z","iopub.execute_input":"2022-08-10T14:23:30.733036Z","iopub.status.idle":"2022-08-10T14:23:30.737057Z","shell.execute_reply.started":"2022-08-10T14:23:30.732999Z","shell.execute_reply":"2022-08-10T14:23:30.736133Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_test = np.mean(y_test_2_list, axis=0)\ny_test.shape, y_test","metadata":{"id":"T0CW1zfHA1Gl","executionInfo":{"status":"ok","timestamp":1655015412280,"user_tz":-540,"elapsed":19,"user":{"displayName":"‍이성만(대학원생-AI빅데이터전공)","userId":"10915876580228061149"}},"outputId":"170cf2fa-c900-4d2f-a0de-271e1ed4f308","execution":{"iopub.status.busy":"2022-08-10T14:23:30.738500Z","iopub.execute_input":"2022-08-10T14:23:30.738852Z","iopub.status.idle":"2022-08-10T14:23:30.750337Z","shell.execute_reply.started":"2022-08-10T14:23:30.738813Z","shell.execute_reply":"2022-08-10T14:23:30.749018Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df.loc[test_df[\"cell_type\"] == \"markdown\", \"pred\"] = y_test\ntest_df","metadata":{"id":"2CkHNl-rmB-E","executionInfo":{"status":"ok","timestamp":1655015412280,"user_tz":-540,"elapsed":16,"user":{"displayName":"‍이성만(대학원생-AI빅데이터전공)","userId":"10915876580228061149"}},"outputId":"0b7a151a-181d-4050-ee49-e1224c527ee1","execution":{"iopub.status.busy":"2022-08-10T14:23:30.751741Z","iopub.execute_input":"2022-08-10T14:23:30.752626Z","iopub.status.idle":"2022-08-10T14:23:30.771841Z","shell.execute_reply.started":"2022-08-10T14:23:30.752591Z","shell.execute_reply":"2022-08-10T14:23:30.771128Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub_df = test_df.sort_values(\"pred\").groupby(\"id\")[\"cell_id\"].apply(lambda x: \" \".join(x)).reset_index()\nsub_df.rename(columns={\"cell_id\": \"cell_order\"}, inplace=True)\nsub_df.head()","metadata":{"id":"gXB3m6yMHa1j","executionInfo":{"status":"ok","timestamp":1655015412280,"user_tz":-540,"elapsed":7,"user":{"displayName":"‍이성만(대학원생-AI빅데이터전공)","userId":"10915876580228061149"}},"outputId":"1fd918dd-2c1e-4407-9292-765745e3abac","execution":{"iopub.status.busy":"2022-08-10T14:23:30.773886Z","iopub.execute_input":"2022-08-10T14:23:30.774302Z","iopub.status.idle":"2022-08-10T14:23:30.788595Z","shell.execute_reply.started":"2022-08-10T14:23:30.774268Z","shell.execute_reply":"2022-08-10T14:23:30.787860Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub_df.to_csv(\"submission.csv\", index=False)","metadata":{"id":"_ShCvXWZHcIL","executionInfo":{"status":"ok","timestamp":1655015412280,"user_tz":-540,"elapsed":7,"user":{"displayName":"‍이성만(대학원생-AI빅데이터전공)","userId":"10915876580228061149"}},"execution":{"iopub.status.busy":"2022-08-10T14:23:30.789825Z","iopub.execute_input":"2022-08-10T14:23:30.790168Z","iopub.status.idle":"2022-08-10T14:23:30.795673Z","shell.execute_reply.started":"2022-08-10T14:23:30.790135Z","shell.execute_reply":"2022-08-10T14:23:30.794790Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# End Of Program","metadata":{"id":"V8apgTwUH21Q"}}]}