{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-05-22T06:38:06.216572Z","iopub.execute_input":"2023-05-22T06:38:06.217212Z","iopub.status.idle":"2023-05-22T06:38:06.233842Z","shell.execute_reply.started":"2023-05-22T06:38:06.217176Z","shell.execute_reply":"2023-05-22T06:38:06.232884Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import re\nimport numpy as np\nimport pandas as pd\nimport torch \nimport json\n#%%\n### Load Labels\n### feature engineer\ncols = [\n    \"session_id\",\n    \"hover_duration\",\n    \"text\",\n    \"fqid\",\n    \"room_fqid\",\n    \"text_fqid\",\n    \"fullscreen\",\n    \"hq\",\n    \"music\",\n    \"level_group\",\n    \"elapsed_time\",\n    \"room_coor_x\",\n    \"room_coor_y\",\n    \"screen_coor_x\",\n    \"screen_coor_y\",\n]\n# 1.处理 session_id 函数\n## 将原始数据 22100221145014656 拆解为 年月日时分秒的数据方便做特征\ndef session_id_func(df):\n    return (\n        pd.DataFrame()\n        # session_id的 0,1 位为年\n        .assign(year=df[\"session_id\"].apply(lambda x: int(str(x)[:2])))\n        # session_id的 2,3 位为月 原始为0-11 +1后1-12\n        .assign(month=df[\"session_id\"].apply(lambda x: int(str(x)[2:4]) + 1))\n        # session_id的 4,5 位为日 原始为0-6  +1后1-7\n        .assign(day=df[\"session_id\"].apply(lambda x: int(str(x)[4:6]) + 1))\n        # session_id的 6,7 位为时\n        .assign(hour=df[\"session_id\"].apply(lambda x: int(str(x)[6:8])))\n        # session_id的 8,9 位为分\n        .assign(minute=df[\"session_id\"].apply(lambda x: int(str(x)[8:10])))\n        # session_id的 10,11 位为秒\n        .assign(second=df[\"session_id\"].apply(lambda x: int(str(x)[10:12])))\n    )\n# 2.处理 hover_duration 函数\n## 对session_id 进行分组 求 对session_id对应的hover_duration的均值\n## 然后通过merge方式 连表(为了补充空值数据)\n## 最后对数据进行标准化处理\ndef hover_duration_func(df):\n    # 以session_id为分组 对同一session_id 的hover_duration填充均值 对空值填充0\n    hover_duration_df = (\n        df.groupby(\"session_id\", as_index=False).hover_duration.mean().fillna(0)\n    )\n    # 对 原始df 进行更新\n    df = df.drop(\"hover_duration\", axis=1).merge(hover_duration_df, on=\"session_id\")\n    x = df.hover_duration\n    # 进行标准化处理\n    val = (x - x.mean()) / (x.std())\n    return val\n# 3.处理 text 函数\n## 说拿出文本所有的单词,并计算出出现的次数,构成字典\n## 根据上面生成的字典 对每行句子的单词的次数进行求和\n## 进行标准化处理\ndef txt_func(df):\n    lines = df[\"text\"].fillna(\"none\").to_list()\n    lines = [\n        re.sub(\"[^A-Za-z]+\", \" \", line).strip().lower() for line in lines\n    ]  # 清洗\n    tokens = [line.split(\" \") for line in lines]  # 分词\n    txt_dict = pd.Series(tokens).explode().value_counts().to_dict()  # 构建每个词的数量的字典\n    new_txt = []\n    for i in tokens:\n        for j in i:\n            result = 0\n            val = txt_dict.get(j, 0)\n            result += val\n        new_txt.append(result)\n    x = pd.Series(new_txt)\n    # 进行标准化处理\n    val = (x - x.mean()) / (x.std())\n    return val\n# 4.fqid one-hot\ndef fqid_func(df):\n    df = pd.get_dummies(df[\"fqid\"])\n    return df\n# 5.room_fqid one-hot\ndef room_fqid_func(df):\n    df = pd.get_dummies(df['room_fqid'])\n    return df\n# 6.text_fqid one-hot\ndef text_fqid_func(df):\n    df = pd.get_dummies(df[\"text_fqid\"])\n    return df\n# 7.level_group one-hot 这个应该不需要one-hot，可能需要和level对应处理？\ndef level_group_func(df):\n    df = pd.get_dummies(df[\"level_group\"])\n    return df\n# 8. elapsed tinme: 处理成相邻两行的差值\ndef elapsed_time_delta(df):\n    elapsed_time_shift1_df = (\n        df.groupby(\"session_id\", as_index=False).elapsed_time.shift().fillna(0)\n    )\n    time_df = df.loc[:, ['elapsed_time']] - elapsed_time_shift1_df\n    # df = pd.concat([df, time_df], axis=1)\n\n    return time_df\ndef coor(df):\n    df[\"room_coor_x\"].fillna(0, inplace=True)\n    df[\"room_coor_y\"].fillna(0, inplace=True)\n    df[\"screen_coor_x\"].fillna(0, inplace=True)\n    df[\"screen_coor_y\"].fillna(0, inplace=True)\n    return df.loc[:, [\"room_coor_x\", \"room_coor_y\", \"screen_coor_x\", \"screen_coor_y\"]]\n\n# 9. 将event_name和name合成一列\ndef event_name(df):\n    df['merged_event_name'] = df['event_name'] + '_' + df['name']\n    return pd.get_dummies(df['merged_event_name'])\n    # or\n    # return df['merged_event_name']\n# fullscreen hq music 无需处理\ndef feature_engineer(unprocessed_data):\n    train = unprocessed_data\n    df1 = session_id_func(train.loc[:, cols])\n    df2 = hover_duration_func(train.loc[:, cols])\n    df3 = txt_func(train.loc[:, cols])\n    df4 = fqid_func(train.loc[:, cols])\n    df5 = room_fqid_func(train.loc[:, cols])\n    df6 = text_fqid_func(train.loc[:, cols])\n    df7 = level_group_func(train.loc[:, cols])\n    df8 = elapsed_time_delta(train.loc[:, cols])\n    df9 = coor(train.loc[:, cols])\n    return (\n        train.drop([\"fqid\", \"room_fqid\", \"text_fqid\", \"level_group\", \"elapsed_time\", \"page\",\n                     \"room_coor_x\", \"room_coor_y\", \"screen_coor_x\", \"screen_coor_y\",], axis=1)  # 拼接\n        .assign(hover_duration=df2)\n        .assign(text=df3)\n        .assign(elapsed_time_delta=df8)\n        .merge(df1, left_index=True, right_index=True)\n        .merge(df4, left_index=True, right_index=True)\n        .merge(df5, left_index=True, right_index=True)\n        .merge(df6, left_index=True, right_index=True)\n        .merge(df7, left_index=True, right_index=True)\n        .merge(df9, left_index=True, right_index=True)\n    )\ndef get_mean(df, col_name): # 统计某一列的均值\n    # 以session_id为分组 对同一session_id 的hover_duration填充均值 对空值填充0\n    ret_df = (\n        df.groupby(\"session_id\", as_index=False)[col_name].mean().fillna(0)\n    )\n    return ret_df.rename(columns={col_name: col_name+\"_mean\"})\n\ndef get_std(df, col_name): # 统计某一列的均值\n    # 以session_id为分组 对同一session_id 的hover_duration填充均值 对空值填充0\n    ret_df = (\n        df.groupby(\"session_id\", as_index=False)[col_name].std().fillna(0)\n    )\n    return ret_df.rename(columns={col_name: col_name+\"_std\"})\n\n# TODO:\ndef get_statistics(unprocessed_data, processed_data):\n\n    statistics_table = (\n        get_mean(processed_data, \"elapsed_time_delta\").\n        merge(get_std(processed_data, \"elapsed_time_delta\")).\n        merge(get_mean(processed_data, \"room_coor_x\")).\n        merge(get_std(processed_data, \"room_coor_x\")).\n        merge(get_mean(processed_data, \"room_coor_y\")).\n        merge(get_std(processed_data, \"room_coor_y\")).\n        merge(get_mean(processed_data, \"screen_coor_x\")).\n        merge(get_std(processed_data, \"screen_coor_x\")).\n        merge(get_mean(processed_data, \"screen_coor_y\")).\n        merge(get_std(processed_data, \"screen_coor_y\")).\n        merge(get_mean(unprocessed_data, \"hover_duration\")).\n        merge(get_std(unprocessed_data, \"hover_duration\"))\n    )\n    return statistics_table\n#%%\ntest_csv_path = \"/kaggle/input/predict-student-performance-from-game-play/test.csv\"\ntrain_csv_path = \"/kaggle/input/predict-student-performance-and-game-play/train.csv\"\ntarget_labels_csv = \"/kaggle/input/predict-student-performance-and-game-play/train_labels.csv\"\n\n# train_csv_path = \"/Users/zhuoyi/assignment/大数据/bighw/data/train.csv\"\ntrain = pd.read_csv(test_csv_path, usecols=[0])\n#%%\npieces = 25\nN = len(train)\nchunks = int(np.ceil(len(train) / pieces))\nprint(\"total count: \", len(train))\nreads = []\nskips = [0]\n\nfor k in range(pieces):\n    a = k * chunks\n    b = (k + 1) * chunks\n\n    if b > N:\n        b = N\n\n    r = b - a\n    reads.append(r)\n    skips.append(skips[-1] + r)\ntrain = pd.read_csv(test_csv_path)\ncategorical_cols = [\n    \"event_name\",\n    \"fqid\",\n    \"room_fqid\",\n    \"text\",\n    \"text_fqid\",\n]\n\nnumerical_cols = [\n    \"elapsed_time_delta\",\n    \"level\",\n    \"page\",\n    \"room_coor_x\",\n    \"room_coor_y\",\n    \"screen_coor_x\",\n    \"screen_coor_y\",\n    \"hover_duration\",\n]\nevent_list = train[\"event_name\"].unique().tolist()\nname_list = train[\"text\"].unique().tolist()\nfqid_list = train[\"fqid\"].unique().tolist()\nroom_list = train[\"room_fqid\"].unique().tolist()\nprint(event_list.__len__())\nprint(name_list.__len__())\nprint(fqid_list.__len__())\nprint(room_list.__len__())\ngroupby_cols = [\"session_id\", \"level_group\"]\n\n#%%\nimport gc\ndef feature_engineer2(train_df):\n    dfs = []\n\n    agg_functions = {c: [\"mean\", \"std\", \"sum\", \"max\", \"min\"] for c in numerical_cols}\n    for c, funcs in agg_functions.items():\n        tmp = train_df.groupby(groupby_cols)[c].agg(funcs)\n        tmp.columns = [f\"{c}_{agg_name}\" for agg_name in funcs]\n        dfs.append(tmp)\n\n    for c in categorical_cols:\n        tmp = train_df.groupby(groupby_cols)[c].agg(\"nunique\")\n        tmp.name = f\"{tmp.name}_nunique\"\n        dfs.append(tmp)\n\n    for c in event_list:\n        train_df[c] = (train_df[\"event_name\"] == c).astype(np.int8)\n\n    for c in event_list:\n        tmp = train_df.groupby(groupby_cols).agg({c: \"sum\", \"elapsed_time\": \"sum\"})\n        tmp.rename(\n            columns={c: f\"{c}_sum\", \"elapsed_time\": f\"{c}_elapsed_time_sum\"},\n            inplace=True,\n        )\n        dfs.append(tmp)\n\n    for c in room_list:\n        train_df[c] = (train_df[\"room_fqid\"] == c).astype(np.int8)\n\n    for c in room_list:\n        tmp = train_df.groupby(groupby_cols)[c].agg(\"sum\")\n        tmp.name = f\"{tmp.name}_sum\"\n        dfs.append(tmp)\n\n    # Frequency encoding of fqid\n    fqid_counts = train_df['fqid'].value_counts()\n    train_df['fqid_freq_encoded'] = train_df['fqid'].map(fqid_counts)\n\n    tmp = train_df.groupby(groupby_cols)['fqid_freq_encoded'].agg([\"mean\", \"sum\", \"max\", \"min\"])\n    tmp.columns = [f\"fqid_freq_encoded_{agg_name}\" for agg_name in tmp.columns]\n    dfs.append(tmp)\n\n    # train_df.drop(columns=['fqid', 'fqid_freq_encoded'], inplace=True)\n\n    # Frequency encoding of text\n    text_counts = train_df['text'].value_counts()\n    train_df['text_freq_encoded'] = train_df['text'].map(text_counts)\n\n    tmp = train_df.groupby(groupby_cols)['text_freq_encoded'].agg([\"mean\", \"sum\", \"max\", \"min\"])\n    tmp.columns = [f\"text_freq_encoded_{agg_name}\" for agg_name in tmp.columns]\n    dfs.append(tmp)\n\n    # train_df.drop(columns=['text', 'text_freq_encoded'], inplace=True)\n\n    df = pd.concat(dfs, axis=1).fillna(-1)\n    df = df.reset_index().set_index(\"session_id\")\n\n    _ = gc.collect()\n    return df\ndef get_statistics_of_mean(df_in):\n    with open(\"/kaggle/input/mean-std/mean_std.json\", \"r\") as f:\n        mean_std = json.load(f)\n    df = df_in.__deepcopy__()\n    for c in df.columns:\n        if c == \"session_id\" or c == \"level_group\":\n            continue\n        mean = mean_std[c+\"_mean\"]\n        std = mean_std[c+\"_std\"]\n        df.fillna(mean)\n        df[c] = (df[c] - mean)/std \n    return df\n#%%\nfrom tqdm import tqdm","metadata":{"execution":{"iopub.status.busy":"2023-05-22T06:38:06.337654Z","iopub.execute_input":"2023-05-22T06:38:06.339019Z","iopub.status.idle":"2023-05-22T06:38:08.348086Z","shell.execute_reply.started":"2023-05-22T06:38:06.338649Z","shell.execute_reply":"2023-05-22T06:38:08.347159Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport numpy as np\nfrom sklearn.metrics import roc_auc_score\n\n\ndef print_hi(name):\n    # Use a breakpoint in the code line below to debug your script.\n    print(f'Hi, {name}')  # Press ⌘F8 to toggle the breakpoint.\n\nclass FeedForward(torch.nn.Module):\n    def __init__(self, dim, dim_out, mult=4, glu=False, dropout=0.):\n        super().__init__()\n        inner_dim = int(dim * mult)\n        project_in = nn.Sequential(\n            nn.Linear(dim, inner_dim),\n            nn.GELU()\n        )\n\n        self.net = nn.Sequential(\n            project_in,\n            nn.Dropout(dropout),\n            nn.Linear(inner_dim, dim_out)\n        )\n\n    def forward(self, x):\n        return self.net(x)","metadata":{"execution":{"iopub.status.busy":"2023-05-22T06:38:08.350565Z","iopub.execute_input":"2023-05-22T06:38:08.350923Z","iopub.status.idle":"2023-05-22T06:38:08.754528Z","shell.execute_reply.started":"2023-05-22T06:38:08.350891Z","shell.execute_reply":"2023-05-22T06:38:08.753298Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Create environment\nimport jo_wilder_310\nenv = jo_wilder_310.make_env()\niter_test = env.iter_test()\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-05-22T06:38:08.756484Z","iopub.execute_input":"2023-05-22T06:38:08.756911Z","iopub.status.idle":"2023-05-22T06:38:08.969120Z","shell.execute_reply.started":"2023-05-22T06:38:08.756864Z","shell.execute_reply":"2023-05-22T06:38:08.968054Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nimport torch.nn as nn\nclass FeedForward(torch.nn.Module):\n    def __init__(self, dim, dim_out, mult=4, glu=False, dropout=0.):\n        super().__init__()\n        inner_dim = int(dim * mult)\n        project_in = nn.Sequential(\n            nn.Linear(dim, inner_dim),\n            nn.GELU()\n        )\n\n        self.net = nn.Sequential(\n            project_in,\n            nn.Dropout(dropout),\n            nn.Linear(inner_dim, dim_out)\n        )\n\n    def forward(self, x):\n        return self.net(x)","metadata":{"execution":{"iopub.status.busy":"2023-05-22T06:38:08.970936Z","iopub.execute_input":"2023-05-22T06:38:08.971521Z","iopub.status.idle":"2023-05-22T06:38:08.978976Z","shell.execute_reply.started":"2023-05-22T06:38:08.971484Z","shell.execute_reply":"2023-05-22T06:38:08.977995Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ff_1 = FeedForward(94, 3)\nff_2 = FeedForward(188, 10)\nff_3 = FeedForward(282, 5)\npath_1 = \"/kaggle/input/nnckpt/ff_1.ckpt\"\npath_2 = \"/kaggle/input/nnckpt/ff_2.ckpt\"\npath_3 = \"/kaggle/input/nnckpt/ff_3.ckpt\"\nff_1.load_state_dict(torch.load(path_1))\nff_2.load_state_dict(torch.load(path_2))\nff_3.load_state_dict(torch.load(path_3))\nff_1.eval()\nff_2.eval()\nff_3.eval()","metadata":{"execution":{"iopub.status.busy":"2023-05-22T06:38:08.981323Z","iopub.execute_input":"2023-05-22T06:38:08.982401Z","iopub.status.idle":"2023-05-22T06:38:09.021880Z","shell.execute_reply.started":"2023-05-22T06:38:08.982365Z","shell.execute_reply":"2023-05-22T06:38:09.020969Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"limits = {\"0-4\": (1, 4), \"5-12\": (4, 14), \"13-22\": (14, 19)}\n\nnow_part = 0\nfor test, sample_submission in iter_test:\n    # FEATURE ENGINEER TEST DATA\n    if now_part != 0:\n        test = pd.concat([pre_test, test], ignore_index=True)\n    \n    test_df = test.__deepcopy__()\n    df_elasped_time_delta = elapsed_time_delta(test_df.loc[:, cols])\n    \n    test_df = test_df.assign(elapsed_time_delta=df_elasped_time_delta)\n    df2 = feature_engineer2(test_df)\n    \n    \n    df3 = get_statistics_of_mean(df2)\n    \n\n    df3 = df3.to_numpy()\n    df3 = df3[:, 1:].astype('float32')\n    if now_part==0:\n        test_pred = ff_1(torch.tensor(df3, dtype=torch.float32))\n        test_label = (test_pred > 0).long() \n        for i in range(1, 4):\n            mask = sample_submission.session_id.str.contains(f\"q{i}\")\n            sample_submission.loc[mask, \"correct\"] = int(test_label[0][i-1])\n        pass\n    elif now_part==1:\n        df3 = df3.reshape(df3.shape[0]//2, -1)\n        test_pred = ff_2(torch.tensor(df3, dtype=torch.float32))\n        test_label = (test_pred > 0).long() \n        for i in range(4, 14):\n            mask = sample_submission.session_id.str.contains(f\"q{i}\")\n            sample_submission.loc[mask, \"correct\"] = int(test_label[0][i-4])\n        pass\n    else:\n        df3 = df3.reshape(df3.shape[0]//3, -1)\n        test_pred = ff_3(torch.tensor(df3, dtype=torch.float32))\n        test_label = (test_pred > 0).long() \n        for i in range(14, 19):\n            mask = sample_submission.session_id.str.contains(f\"q{i}\")\n            sample_submission.loc[mask, \"correct\"] = int(test_label[0][i-14])\n        pass\n    \n    now_part = (now_part + 1) % 3\n    env.predict(sample_submission)\n    pre_test = test\n\n","metadata":{"execution":{"iopub.status.busy":"2023-05-22T06:38:09.026009Z","iopub.execute_input":"2023-05-22T06:38:09.028257Z","iopub.status.idle":"2023-05-22T06:38:12.853931Z","shell.execute_reply.started":"2023-05-22T06:38:09.028217Z","shell.execute_reply":"2023-05-22T06:38:12.853055Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.read_csv(\"submission.csv\")\ndf","metadata":{"execution":{"iopub.status.busy":"2023-05-22T06:38:12.855513Z","iopub.execute_input":"2023-05-22T06:38:12.856151Z","iopub.status.idle":"2023-05-22T06:38:12.872398Z","shell.execute_reply.started":"2023-05-22T06:38:12.856118Z","shell.execute_reply":"2023-05-22T06:38:12.871382Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# df.to_csv(\"2333.csv\")","metadata":{"execution":{"iopub.status.busy":"2023-05-22T06:38:12.873837Z","iopub.execute_input":"2023-05-22T06:38:12.874154Z","iopub.status.idle":"2023-05-22T06:38:12.880595Z","shell.execute_reply.started":"2023-05-22T06:38:12.874127Z","shell.execute_reply":"2023-05-22T06:38:12.879634Z"},"trusted":true},"execution_count":null,"outputs":[]}]}