{"cells":[{"metadata":{"papermill":{"duration":0.105531,"end_time":"2020-11-14T16:50:21.611734","exception":false,"start_time":"2020-11-14T16:50:21.506203","status":"completed"},"tags":[]},"cell_type":"markdown","source":"# 日本語(Japanese)EDAで、個人的な解釈です。\n## memo書きみたいなものもあるので、言い回しが丁寧ではありません・・・\n\n"},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","papermill":{"duration":0.104156,"end_time":"2020-11-14T16:50:21.821877","exception":false,"start_time":"2020-11-14T16:50:21.717721","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.109488,"end_time":"2020-11-14T16:50:22.036021","exception":false,"start_time":"2020-11-14T16:50:21.926533","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.115974,"end_time":"2020-11-14T16:50:22.263024","exception":false,"start_time":"2020-11-14T16:50:22.14705","status":"completed"},"tags":[]},"cell_type":"markdown","source":"# 0. inputファイルの確認 (confirming input files)"},{"metadata":{"execution":{"iopub.execute_input":"2020-11-14T16:50:22.490688Z","iopub.status.busy":"2020-11-14T16:50:22.489585Z","iopub.status.idle":"2020-11-14T16:50:22.496751Z","shell.execute_reply":"2020-11-14T16:50:22.495709Z"},"papermill":{"duration":0.124018,"end_time":"2020-11-14T16:50:22.496964","exception":false,"start_time":"2020-11-14T16:50:22.372946","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-11-14T16:50:22.724569Z","iopub.status.busy":"2020-11-14T16:50:22.723546Z","iopub.status.idle":"2020-11-14T16:50:24.041593Z","shell.execute_reply":"2020-11-14T16:50:24.04075Z"},"papermill":{"duration":1.435788,"end_time":"2020-11-14T16:50:24.041726","exception":false,"start_time":"2020-11-14T16:50:22.605938","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"\nimport plotly.express as px\nfrom plotly.subplots import make_subplots\nimport plotly.graph_objs as go\n\nimport matplotlib.pyplot as plt","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.134526,"end_time":"2020-11-14T16:50:24.28356","exception":false,"start_time":"2020-11-14T16:50:24.149034","status":"completed"},"tags":[]},"cell_type":"markdown","source":"# 1. train.csvの確認(confirming train.csv)\n## ※ 1億行あるので、100万行でまずは読む (I read only 1 million rows because this file is composed of over 10 million."},{"metadata":{"execution":{"iopub.execute_input":"2020-11-14T16:50:24.526167Z","iopub.status.busy":"2020-11-14T16:50:24.525227Z","iopub.status.idle":"2020-11-14T16:50:25.797603Z","shell.execute_reply":"2020-11-14T16:50:25.798332Z"},"papermill":{"duration":1.391367,"end_time":"2020-11-14T16:50:25.798489","exception":false,"start_time":"2020-11-14T16:50:24.407122","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"traindf = pd.read_csv(\"/kaggle/input/riiid-test-answer-prediction/train.csv\",nrows=10**6 )\ntraindf","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-11-14T16:50:26.022695Z","iopub.status.busy":"2020-11-14T16:50:26.021305Z","iopub.status.idle":"2020-11-14T16:50:26.173087Z","shell.execute_reply":"2020-11-14T16:50:26.172231Z"},"papermill":{"duration":0.266946,"end_time":"2020-11-14T16:50:26.173234","exception":false,"start_time":"2020-11-14T16:50:25.906288","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"traindf.info()","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.108016,"end_time":"2020-11-14T16:50:26.3908","exception":false,"start_time":"2020-11-14T16:50:26.282784","status":"completed"},"tags":[]},"cell_type":"markdown","source":"♯ メモリーを効率的に使うための型変換 (他コンペから抜粋)"},{"metadata":{"execution":{"iopub.execute_input":"2020-11-14T16:50:26.622063Z","iopub.status.busy":"2020-11-14T16:50:26.616866Z","iopub.status.idle":"2020-11-14T16:50:26.639252Z","shell.execute_reply":"2020-11-14T16:50:26.638453Z"},"papermill":{"duration":0.140248,"end_time":"2020-11-14T16:50:26.639401","exception":false,"start_time":"2020-11-14T16:50:26.499153","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"from pandas.api.types import is_datetime64_any_dtype as is_datetime\nfrom pandas.api.types import is_categorical_dtype\n\ndef reduce_mem_usage(df, use_float16=False):\n    \"\"\"\n    Iterate through all the columns of a dataframe and modify the data type to reduce memory usage.        \n    \"\"\"\n    \n    start_mem = df.memory_usage().sum() / 1024**2\n    print(\"Memory usage of dataframe is {:.2f} MB\".format(start_mem))\n    \n    for col in df.columns:\n        if is_datetime(df[col]) or is_categorical_dtype(df[col]):\n            continue\n        col_type = df[col].dtype\n        \n        if col_type != object:\n            c_min = df[col].min()\n            c_max = df[col].max()\n            if str(col_type)[:3] == \"int\":\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    df[col] = df[col].astype(np.int64)  \n            else:\n                if use_float16 and c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                    df[col] = df[col].astype(np.float16)\n                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    df[col] = df[col].astype(np.float32)\n                else:\n                    df[col] = df[col].astype(np.float64)\n        else:\n            df[col] = df[col].astype(\"category\")\n\n    end_mem = df.memory_usage().sum() / 1024**2\n    print(\"Memory usage after optimization is: {:.2f} MB\".format(end_mem))\n    print(\"Decreased by {:.1f}%\".format(100 * (start_mem - end_mem) / start_mem))\n    \n    return df","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-11-14T16:50:26.863381Z","iopub.status.busy":"2020-11-14T16:50:26.862567Z","iopub.status.idle":"2020-11-14T16:50:27.104345Z","shell.execute_reply":"2020-11-14T16:50:27.104989Z"},"papermill":{"duration":0.358415,"end_time":"2020-11-14T16:50:27.10515","exception":false,"start_time":"2020-11-14T16:50:26.746735","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"traindf = reduce_mem_usage(traindf)","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-11-14T16:50:27.329597Z","iopub.status.busy":"2020-11-14T16:50:27.328807Z","iopub.status.idle":"2020-11-14T16:50:27.373066Z","shell.execute_reply":"2020-11-14T16:50:27.372233Z"},"papermill":{"duration":0.15963,"end_time":"2020-11-14T16:50:27.373202","exception":false,"start_time":"2020-11-14T16:50:27.213572","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"traindf.info()","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-11-14T16:50:27.608484Z","iopub.status.busy":"2020-11-14T16:50:27.607288Z","iopub.status.idle":"2020-11-14T16:50:27.611916Z","shell.execute_reply":"2020-11-14T16:50:27.611237Z"},"papermill":{"duration":0.130507,"end_time":"2020-11-14T16:50:27.61204","exception":false,"start_time":"2020-11-14T16:50:27.481533","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"traindf.head(3)","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.110028,"end_time":"2020-11-14T16:50:27.830951","exception":false,"start_time":"2020-11-14T16:50:27.720923","status":"completed"},"tags":[]},"cell_type":"markdown","source":" # <span style=\"background-color: yellow; \">1.train.csv\n## siteに細かい説明あるが、わかりやすく解釈したもの (Below sentence is my interpretation in order to understand easily.\n\n1.  row_id : dataframeのid\n2.  timestamp : 生徒の直前のアクションからの経過時間(ミリ秒)\n3.  user id : 生徒のid\n4.  content_id : 問題のid\n5.  content_type_id : 質問(0)か講義(1)\n6.  task_container_id : \n わかりやすく捉えると、4.content idの章みたいな感じ (TOEICの問題をイメージすると、１つの長文がこれに相当して、それの問題1,2,3みたいのが、content idでそれぞれ振り分けられている)\n6.  user_answer : 4択 (0,1,2,3)でどれを答えたか\n7.  answered_correctly : それがあっていたかどうか\n8.  prior_question_elapsed_time\t : 1つ前の質問(3個質問がある場合がある)からの経過時間\n9.  prior_question_had_explanation : 1つ前の質問(3個質問がある場合がある)の後に、解説および回答を見る機会があったか\n\n## 最終的には、answered_correctly以外の情報から、問題を当てられるかどうかを0～1の確率で出すコンペティション\n## (testデータにはもう少し細工が入っている)\n"},{"metadata":{"execution":{"iopub.execute_input":"2020-11-14T16:50:28.057144Z","iopub.status.busy":"2020-11-14T16:50:28.056299Z","iopub.status.idle":"2020-11-14T16:50:28.222476Z","shell.execute_reply":"2020-11-14T16:50:28.221622Z"},"papermill":{"duration":0.282201,"end_time":"2020-11-14T16:50:28.222619","exception":false,"start_time":"2020-11-14T16:50:27.940418","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"for a in traindf.columns:\n    print(a + \":\" + str(len(traindf[a].unique())))","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.159206,"end_time":"2020-11-14T16:50:28.492104","exception":false,"start_time":"2020-11-14T16:50:28.332898","status":"completed"},"tags":[]},"cell_type":"markdown","source":"# 100万個データから以下のことがわかる\n* 生徒(user_id)が3824人\n* 問題(content_id)が13320個\n* 問題 or 講義で２個(content_type_id)\n* 問題の章(task_container_id)が7740個\n* 生徒の回答の選択肢が５個 (-1,0,1,2,3)で-1は講義\n![](http://)* 正解 or not で３個(-1,0,1) で-1は講義"},{"metadata":{"papermill":{"duration":0.10883,"end_time":"2020-11-14T16:50:28.711966","exception":false,"start_time":"2020-11-14T16:50:28.603136","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.1097,"end_time":"2020-11-14T16:50:28.93157","exception":false,"start_time":"2020-11-14T16:50:28.82187","status":"completed"},"tags":[]},"cell_type":"markdown","source":"#### わかりやすく理解するために、user id:115のみを見てみる"},{"metadata":{"execution":{"iopub.execute_input":"2020-11-14T16:50:29.159731Z","iopub.status.busy":"2020-11-14T16:50:29.158939Z","iopub.status.idle":"2020-11-14T16:50:29.214825Z","shell.execute_reply":"2020-11-14T16:50:29.215481Z"},"papermill":{"duration":0.172313,"end_time":"2020-11-14T16:50:29.215695","exception":false,"start_time":"2020-11-14T16:50:29.043382","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"traindf115 = traindf[traindf[\"user_id\"]==115]\ntraindf115","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.111246,"end_time":"2020-11-14T16:50:29.440289","exception":false,"start_time":"2020-11-14T16:50:29.329043","status":"completed"},"tags":[]},"cell_type":"markdown","source":"#### task container idは、1～41章あって、40と41はそれぞれ問題が3個ある\n#### prior_question_elapsed_timeは講義(問題後の解説とか)を抜かしたタイムのため、単純に引き算ではなさそう。"},{"metadata":{"papermill":{"duration":0.110788,"end_time":"2020-11-14T16:50:29.662857","exception":false,"start_time":"2020-11-14T16:50:29.552069","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-11-14T16:50:29.890519Z","iopub.status.busy":"2020-11-14T16:50:29.889545Z","iopub.status.idle":"2020-11-14T16:50:31.028966Z","shell.execute_reply":"2020-11-14T16:50:31.029562Z"},"papermill":{"duration":1.255847,"end_time":"2020-11-14T16:50:31.029736","exception":false,"start_time":"2020-11-14T16:50:29.773889","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"# timestampの調査\npx.line(traindf115,x=\"row_id\",y=\"timestamp\",width = 500)","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.114996,"end_time":"2020-11-14T16:50:31.258991","exception":false,"start_time":"2020-11-14T16:50:31.143995","status":"completed"},"tags":[]},"cell_type":"markdown","source":"#### row idが38→39問目になったときに急激に上がっている。TOEICで言うとlisteningとreadingで間が空くとかそんなイメージ ?\n#### ※　後ほどquestion.csvとマージするとpartが変わっている"},{"metadata":{"execution":{"iopub.execute_input":"2020-11-14T16:50:31.495558Z","iopub.status.busy":"2020-11-14T16:50:31.494719Z","iopub.status.idle":"2020-11-14T16:50:31.499305Z","shell.execute_reply":"2020-11-14T16:50:31.498632Z"},"papermill":{"duration":0.125789,"end_time":"2020-11-14T16:50:31.499453","exception":false,"start_time":"2020-11-14T16:50:31.373664","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"# 正答率\ntraindf115[\"answered_correctly\"].sum()/len(traindf115)","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.113468,"end_time":"2020-11-14T16:50:31.72859","exception":false,"start_time":"2020-11-14T16:50:31.615122","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.114525,"end_time":"2020-11-14T16:50:31.958441","exception":false,"start_time":"2020-11-14T16:50:31.843916","status":"completed"},"tags":[]},"cell_type":"markdown","source":"# 100万個のデータから生徒ごとの正答率を出してみる。"},{"metadata":{"execution":{"iopub.execute_input":"2020-11-14T16:50:32.220364Z","iopub.status.busy":"2020-11-14T16:50:32.21926Z","iopub.status.idle":"2020-11-14T16:50:32.224772Z","shell.execute_reply":"2020-11-14T16:50:32.224102Z"},"papermill":{"duration":0.151277,"end_time":"2020-11-14T16:50:32.2249","exception":false,"start_time":"2020-11-14T16:50:32.073623","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"traindf","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-11-14T16:50:32.477348Z","iopub.status.busy":"2020-11-14T16:50:32.476553Z","iopub.status.idle":"2020-11-14T16:50:32.688021Z","shell.execute_reply":"2020-11-14T16:50:32.687378Z"},"papermill":{"duration":0.348162,"end_time":"2020-11-14T16:50:32.688153","exception":false,"start_time":"2020-11-14T16:50:32.339991","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"traindf.describe()","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-11-14T16:50:32.923985Z","iopub.status.busy":"2020-11-14T16:50:32.923178Z","iopub.status.idle":"2020-11-14T16:50:32.926712Z","shell.execute_reply":"2020-11-14T16:50:32.925983Z"},"papermill":{"duration":0.123246,"end_time":"2020-11-14T16:50:32.926844","exception":false,"start_time":"2020-11-14T16:50:32.803598","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"# answered_correctlyにmin -1 が入っている　→　質問と講義が入っているため","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-11-14T16:50:33.165045Z","iopub.status.busy":"2020-11-14T16:50:33.164293Z","iopub.status.idle":"2020-11-14T16:50:33.167717Z","shell.execute_reply":"2020-11-14T16:50:33.166964Z"},"papermill":{"duration":0.124259,"end_time":"2020-11-14T16:50:33.167863","exception":false,"start_time":"2020-11-14T16:50:33.043604","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"# 質問と講義に分ける","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-11-14T16:50:33.405917Z","iopub.status.busy":"2020-11-14T16:50:33.40512Z","iopub.status.idle":"2020-11-14T16:50:33.465035Z","shell.execute_reply":"2020-11-14T16:50:33.46425Z"},"papermill":{"duration":0.181592,"end_time":"2020-11-14T16:50:33.465164","exception":false,"start_time":"2020-11-14T16:50:33.283572","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"qdf = traindf[traindf[\"content_type_id\"]==0]\nqdf","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-11-14T16:50:33.7231Z","iopub.status.busy":"2020-11-14T16:50:33.722293Z","iopub.status.idle":"2020-11-14T16:50:33.930198Z","shell.execute_reply":"2020-11-14T16:50:33.929598Z"},"papermill":{"duration":0.346343,"end_time":"2020-11-14T16:50:33.93035","exception":false,"start_time":"2020-11-14T16:50:33.584007","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"qdf.describe()","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.117669,"end_time":"2020-11-14T16:50:34.166838","exception":false,"start_time":"2020-11-14T16:50:34.049169","status":"completed"},"tags":[]},"cell_type":"markdown","source":"#### 分離できた"},{"metadata":{"papermill":{"duration":0.116774,"end_time":"2020-11-14T16:50:34.402655","exception":false,"start_time":"2020-11-14T16:50:34.285881","status":"completed"},"tags":[]},"cell_type":"markdown","source":"#### 生徒ごとの正答率は、愚直にやれば以下の感じ・・・でも、時間かかると思われる"},{"metadata":{"execution":{"iopub.execute_input":"2020-11-14T16:50:34.644493Z","iopub.status.busy":"2020-11-14T16:50:34.64371Z","iopub.status.idle":"2020-11-14T16:50:34.655707Z","shell.execute_reply":"2020-11-14T16:50:34.655028Z"},"papermill":{"duration":0.13588,"end_time":"2020-11-14T16:50:34.655832","exception":false,"start_time":"2020-11-14T16:50:34.519952","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"userlist = qdf[\"user_id\"].unique()\nuserlist","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.119699,"end_time":"2020-11-14T16:50:34.89393","exception":false,"start_time":"2020-11-14T16:50:34.774231","status":"completed"},"tags":[]},"cell_type":"markdown","source":"# 0だけをまずやる"},{"metadata":{"execution":{"iopub.execute_input":"2020-11-14T16:50:35.137459Z","iopub.status.busy":"2020-11-14T16:50:35.136409Z","iopub.status.idle":"2020-11-14T16:50:35.140494Z","shell.execute_reply":"2020-11-14T16:50:35.141031Z"},"papermill":{"duration":0.128296,"end_time":"2020-11-14T16:50:35.141193","exception":false,"start_time":"2020-11-14T16:50:35.012897","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"userlist[0]","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-11-14T16:50:35.406901Z","iopub.status.busy":"2020-11-14T16:50:35.406132Z","iopub.status.idle":"2020-11-14T16:50:35.423104Z","shell.execute_reply":"2020-11-14T16:50:35.42242Z"},"papermill":{"duration":0.145666,"end_time":"2020-11-14T16:50:35.423224","exception":false,"start_time":"2020-11-14T16:50:35.277558","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"tmpdf = qdf[qdf[\"user_id\"]==userlist[0]]\ntmpdf.head(3)","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-11-14T16:50:35.672831Z","iopub.status.busy":"2020-11-14T16:50:35.671526Z","iopub.status.idle":"2020-11-14T16:50:35.676176Z","shell.execute_reply":"2020-11-14T16:50:35.675472Z"},"papermill":{"duration":0.133454,"end_time":"2020-11-14T16:50:35.676294","exception":false,"start_time":"2020-11-14T16:50:35.54284","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"tmpdf[\"answered_correctly\"].sum()/len(tmpdf)","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.120774,"end_time":"2020-11-14T16:50:35.920013","exception":false,"start_time":"2020-11-14T16:50:35.799239","status":"completed"},"tags":[]},"cell_type":"markdown","source":"# まとめてfor文で回す"},{"metadata":{"execution":{"iopub.execute_input":"2020-11-14T16:50:36.171047Z","iopub.status.busy":"2020-11-14T16:50:36.170019Z","iopub.status.idle":"2020-11-14T16:50:44.822848Z","shell.execute_reply":"2020-11-14T16:50:44.821993Z"},"papermill":{"duration":8.781953,"end_time":"2020-11-14T16:50:44.822982","exception":false,"start_time":"2020-11-14T16:50:36.041029","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"ratelist= []\nfor a in userlist:\n    tmpdf = qdf[qdf[\"user_id\"]==a]\n    ratelist.append([a,tmpdf[\"answered_correctly\"].sum()/len(tmpdf)])","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-11-14T16:50:45.085941Z","iopub.status.busy":"2020-11-14T16:50:45.084822Z","iopub.status.idle":"2020-11-14T16:50:45.090141Z","shell.execute_reply":"2020-11-14T16:50:45.089475Z"},"papermill":{"duration":0.14515,"end_time":"2020-11-14T16:50:45.090272","exception":false,"start_time":"2020-11-14T16:50:44.945122","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"ratelistdf = pd.DataFrame(ratelist)\nratelistdf.columns = [\"user_id\",\"rate\"]\nratelistdf","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.120858,"end_time":"2020-11-14T16:50:45.334296","exception":false,"start_time":"2020-11-14T16:50:45.213438","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.122284,"end_time":"2020-11-14T16:50:45.57884","exception":false,"start_time":"2020-11-14T16:50:45.456556","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.120881,"end_time":"2020-11-14T16:50:45.821866","exception":false,"start_time":"2020-11-14T16:50:45.700985","status":"completed"},"tags":[]},"cell_type":"markdown","source":"# 玄人はgroupbyで一発でやる"},{"metadata":{"execution":{"iopub.execute_input":"2020-11-14T16:50:46.073151Z","iopub.status.busy":"2020-11-14T16:50:46.072219Z","iopub.status.idle":"2020-11-14T16:50:46.076874Z","shell.execute_reply":"2020-11-14T16:50:46.076231Z"},"papermill":{"duration":0.132921,"end_time":"2020-11-14T16:50:46.076993","exception":false,"start_time":"2020-11-14T16:50:45.944072","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"userdf = qdf.groupby('user_id')\nuserdf","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-11-14T16:50:46.355769Z","iopub.status.busy":"2020-11-14T16:50:46.354976Z","iopub.status.idle":"2020-11-14T16:50:46.506351Z","shell.execute_reply":"2020-11-14T16:50:46.505698Z"},"papermill":{"duration":0.299185,"end_time":"2020-11-14T16:50:46.506492","exception":false,"start_time":"2020-11-14T16:50:46.207307","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"userdf_calc = userdf.agg({'answered_correctly': ['mean',\"sum\", 'count', 'std']}).copy() # .aggでその中身をmeanやsumなど複数処理可能。\nuserdf_calc.columns = [\n    'mean', \n    \"sum\",\n    'count', \n    'std', \n ]\n\nuserdf_calc","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-11-14T16:50:46.764605Z","iopub.status.busy":"2020-11-14T16:50:46.763765Z","iopub.status.idle":"2020-11-14T16:50:46.767578Z","shell.execute_reply":"2020-11-14T16:50:46.766803Z"},"papermill":{"duration":0.133717,"end_time":"2020-11-14T16:50:46.767708","exception":false,"start_time":"2020-11-14T16:50:46.633991","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"userdf_calc[\"rate\"] = userdf_calc[\"sum\"]/userdf_calc[\"count\"]","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-11-14T16:50:47.02394Z","iopub.status.busy":"2020-11-14T16:50:47.02315Z","iopub.status.idle":"2020-11-14T16:50:47.041478Z","shell.execute_reply":"2020-11-14T16:50:47.040861Z"},"papermill":{"duration":0.148793,"end_time":"2020-11-14T16:50:47.041606","exception":false,"start_time":"2020-11-14T16:50:46.892813","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"userdf_calc","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-11-14T16:50:47.299598Z","iopub.status.busy":"2020-11-14T16:50:47.298558Z","iopub.status.idle":"2020-11-14T16:50:47.411759Z","shell.execute_reply":"2020-11-14T16:50:47.410949Z"},"papermill":{"duration":0.245164,"end_time":"2020-11-14T16:50:47.411898","exception":false,"start_time":"2020-11-14T16:50:47.166734","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"px.histogram(\n    userdf_calc, \n    x=\"rate\",\n    nbins=50,\n    width = 500\n)\n\n","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.12723,"end_time":"2020-11-14T16:50:47.665625","exception":false,"start_time":"2020-11-14T16:50:47.538395","status":"completed"},"tags":[]},"cell_type":"markdown","source":"#### 生徒ごとに、約60%くらいの正答率であることがわかる"},{"metadata":{"papermill":{"duration":0.125753,"end_time":"2020-11-14T16:50:47.918202","exception":false,"start_time":"2020-11-14T16:50:47.792449","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.125894,"end_time":"2020-11-14T16:50:48.170642","exception":false,"start_time":"2020-11-14T16:50:48.044748","status":"completed"},"tags":[]},"cell_type":"markdown","source":"# 関数化　↑のをコピペして作成しているので、userdfとなっているが気にしない。"},{"metadata":{"execution":{"iopub.execute_input":"2020-11-14T16:50:48.432378Z","iopub.status.busy":"2020-11-14T16:50:48.431374Z","iopub.status.idle":"2020-11-14T16:50:48.434689Z","shell.execute_reply":"2020-11-14T16:50:48.433957Z"},"papermill":{"duration":0.138367,"end_time":"2020-11-14T16:50:48.434812","exception":false,"start_time":"2020-11-14T16:50:48.296445","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"def makingrate(col):\n    \n\n    userdf = qdf.groupby(col)\n    userdf_calc = userdf.agg({'answered_correctly': ['mean',\"sum\", 'count', 'std']}).copy() # .aggでその中身をmeanやsumなど複数処理可能。\n    userdf_calc.columns = [\n        'mean', \n        \"sum\",\n        'count', \n        'std', \n     ]\n    userdf_calc[\"rate\"] = userdf_calc[\"sum\"]/userdf_calc[\"count\"]\n    fig = px.histogram(\n        userdf_calc, \n        x=\"rate\",\n        nbins=50,\n        width = 500\n    )\n    fig.show()\n    return userdf_calc\n","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-11-14T16:50:48.715912Z","iopub.status.busy":"2020-11-14T16:50:48.704876Z","iopub.status.idle":"2020-11-14T16:50:48.720346Z","shell.execute_reply":"2020-11-14T16:50:48.72094Z"},"papermill":{"duration":0.159529,"end_time":"2020-11-14T16:50:48.721098","exception":false,"start_time":"2020-11-14T16:50:48.561569","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"qdf","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-11-14T16:50:48.980833Z","iopub.status.busy":"2020-11-14T16:50:48.980024Z","iopub.status.idle":"2020-11-14T16:50:49.224175Z","shell.execute_reply":"2020-11-14T16:50:49.223359Z"},"papermill":{"duration":0.376179,"end_time":"2020-11-14T16:50:49.224308","exception":false,"start_time":"2020-11-14T16:50:48.848129","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"cdf = makingrate(\"content_id\")\ncdf","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.134317,"end_time":"2020-11-14T16:50:49.493106","exception":false,"start_time":"2020-11-14T16:50:49.358789","status":"completed"},"tags":[]},"cell_type":"markdown","source":"#### 問題ごとの正答率が完成。\n#### 0 →　むちゃくちゃ難しい問題 ?, 1→むちゃくちゃ簡単な問題 ? 、その他はきれいなヒストグラム　←　train.csvを全部読むときれいになります。0と1が多くなるとかがなくなります。"},{"metadata":{"papermill":{"duration":0.138416,"end_time":"2020-11-14T16:50:49.766674","exception":false,"start_time":"2020-11-14T16:50:49.628258","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-11-14T16:50:50.049737Z","iopub.status.busy":"2020-11-14T16:50:50.048527Z","iopub.status.idle":"2020-11-14T16:50:50.253135Z","shell.execute_reply":"2020-11-14T16:50:50.253771Z"},"papermill":{"duration":0.348694,"end_time":"2020-11-14T16:50:50.253928","exception":false,"start_time":"2020-11-14T16:50:49.905234","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"tdf = makingrate(\"task_container_id\")\ntdf","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.144011,"end_time":"2020-11-14T16:50:50.543486","exception":false,"start_time":"2020-11-14T16:50:50.399475","status":"completed"},"tags":[]},"cell_type":"markdown","source":"#### 章ごとの正答率が完成。\n#### countが１個しかない章とかが正答率1とかになっているのか。　←　train.csvを全部読むときれいになります。0と1が多くなるとかがなくなります。"},{"metadata":{"execution":{"iopub.execute_input":"2020-11-14T16:50:50.886882Z","iopub.status.busy":"2020-11-14T16:50:50.885749Z","iopub.status.idle":"2020-11-14T16:50:50.906052Z","shell.execute_reply":"2020-11-14T16:50:50.906714Z"},"papermill":{"duration":0.214037,"end_time":"2020-11-14T16:50:50.906877","exception":false,"start_time":"2020-11-14T16:50:50.69284","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"tdf[tdf[\"count\"]==1]","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.141855,"end_time":"2020-11-14T16:50:51.189953","exception":false,"start_time":"2020-11-14T16:50:51.048098","status":"completed"},"tags":[]},"cell_type":"markdown","source":"# 上記の0,1は1個しか問題がないケースが多そう。2040個もある"},{"metadata":{"execution":{"iopub.execute_input":"2020-11-14T16:50:51.520347Z","iopub.status.busy":"2020-11-14T16:50:51.519527Z","iopub.status.idle":"2020-11-14T16:50:51.728651Z","shell.execute_reply":"2020-11-14T16:50:51.727964Z"},"papermill":{"duration":0.396511,"end_time":"2020-11-14T16:50:51.72878","exception":false,"start_time":"2020-11-14T16:50:51.332269","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"px.line(tdf,x=tdf.index ,y=\"count\",width=500)","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.148704,"end_time":"2020-11-14T16:50:52.032631","exception":false,"start_time":"2020-11-14T16:50:51.883927","status":"completed"},"tags":[]},"cell_type":"markdown","source":"# 章はidが大きくなると数が少なくなっている"},{"metadata":{"execution":{"iopub.execute_input":"2020-11-14T16:50:52.38081Z","iopub.status.busy":"2020-11-14T16:50:52.378956Z","iopub.status.idle":"2020-11-14T16:50:52.475118Z","shell.execute_reply":"2020-11-14T16:50:52.475781Z"},"papermill":{"duration":0.286531,"end_time":"2020-11-14T16:50:52.475945","exception":false,"start_time":"2020-11-14T16:50:52.189414","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"px.line(tdf,x=tdf.index ,y=\"rate\",width=500)","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.149909,"end_time":"2020-11-14T16:50:52.774738","exception":false,"start_time":"2020-11-14T16:50:52.624829","status":"completed"},"tags":[]},"cell_type":"markdown","source":"#### 上記2つから、task_contaier_idが小さくなるほど正答率が収束している。1億個のビッグデータだとどうなるのか・・・"},{"metadata":{"execution":{"iopub.execute_input":"2020-11-14T16:50:53.098404Z","iopub.status.busy":"2020-11-14T16:50:53.097236Z","iopub.status.idle":"2020-11-14T16:50:53.102233Z","shell.execute_reply":"2020-11-14T16:50:53.101409Z"},"papermill":{"duration":0.181012,"end_time":"2020-11-14T16:50:53.102412","exception":false,"start_time":"2020-11-14T16:50:52.9214","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"qdf","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-11-14T16:50:53.408297Z","iopub.status.busy":"2020-11-14T16:50:53.406834Z","iopub.status.idle":"2020-11-14T16:50:53.576144Z","shell.execute_reply":"2020-11-14T16:50:53.576745Z"},"papermill":{"duration":0.321986,"end_time":"2020-11-14T16:50:53.576911","exception":false,"start_time":"2020-11-14T16:50:53.254925","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"ptime = makingrate(\"prior_question_elapsed_time\")\nptime","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-11-14T16:50:53.903052Z","iopub.status.busy":"2020-11-14T16:50:53.901951Z","iopub.status.idle":"2020-11-14T16:50:53.963302Z","shell.execute_reply":"2020-11-14T16:50:53.962551Z"},"papermill":{"duration":0.232534,"end_time":"2020-11-14T16:50:53.963447","exception":false,"start_time":"2020-11-14T16:50:53.730913","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"px.line(ptime,x=ptime.index,y=\"count\")","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-11-14T16:50:54.293068Z","iopub.status.busy":"2020-11-14T16:50:54.29208Z","iopub.status.idle":"2020-11-14T16:50:54.355772Z","shell.execute_reply":"2020-11-14T16:50:54.355145Z"},"papermill":{"duration":0.239916,"end_time":"2020-11-14T16:50:54.355913","exception":false,"start_time":"2020-11-14T16:50:54.115997","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"px.line(ptime,x=ptime.index,y=\"rate\")","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-11-14T16:50:54.669816Z","iopub.status.busy":"2020-11-14T16:50:54.668537Z","iopub.status.idle":"2020-11-14T16:50:54.801735Z","shell.execute_reply":"2020-11-14T16:50:54.800968Z"},"papermill":{"duration":0.293219,"end_time":"2020-11-14T16:50:54.801861","exception":false,"start_time":"2020-11-14T16:50:54.508642","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"explanation = makingrate(\"prior_question_had_explanation\")\nexplanation","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.15467,"end_time":"2020-11-14T16:50:55.110488","exception":false,"start_time":"2020-11-14T16:50:54.955818","status":"completed"},"tags":[]},"cell_type":"markdown","source":"#### 問題の前に解説などあると、正解率が高い"},{"metadata":{"papermill":{"duration":0.156716,"end_time":"2020-11-14T16:50:55.422024","exception":false,"start_time":"2020-11-14T16:50:55.265308","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.160185,"end_time":"2020-11-14T16:50:55.737749","exception":false,"start_time":"2020-11-14T16:50:55.577564","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.153312,"end_time":"2020-11-14T16:50:56.044002","exception":false,"start_time":"2020-11-14T16:50:55.89069","status":"completed"},"tags":[]},"cell_type":"markdown","source":"# <span style=\"background-color: yellow; \">2.question.csv"},{"metadata":{"execution":{"iopub.execute_input":"2020-11-14T16:50:56.358292Z","iopub.status.busy":"2020-11-14T16:50:56.357478Z","iopub.status.idle":"2020-11-14T16:50:56.379675Z","shell.execute_reply":"2020-11-14T16:50:56.379035Z"},"papermill":{"duration":0.182381,"end_time":"2020-11-14T16:50:56.379808","exception":false,"start_time":"2020-11-14T16:50:56.197427","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"questions = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/questions.csv')\nquestions.head()","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.15525,"end_time":"2020-11-14T16:50:56.691607","exception":false,"start_time":"2020-11-14T16:50:56.536357","status":"completed"},"tags":[]},"cell_type":"markdown","source":"* ** question_id **：traincsvのcontent id\n\n* ** bundle_id **：どの質問と同時に送られたか ?\n\n* ** correct_answer **：問題の答え\n\n* **パート**：TOEICテストの関連セクション。\n\n* **タグ**：質問の1つ以上の詳細なタグコード。タグの意味は提供されませんが、これらのコードは質問をまとめるのに十分です。"},{"metadata":{"papermill":{"duration":0.155402,"end_time":"2020-11-14T16:50:57.002829","exception":false,"start_time":"2020-11-14T16:50:56.847427","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"questions[\"tags\"] = questions[\"tags\"].astype(\"str\")\ntags = questions[\"tags\"].apply(lambda x: pd.Series(x.split())) # googleでdataframe　スペース区切りとかやると出てくる","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tags","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tags.columns = [\"tag0\",\"tag1\",\"tag2\",\"tag3\",\"tag4\",\"tag5\"]\ntags","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"questions = pd.concat([questions,tags],axis=1)\nquestions = questions.drop(\"tags\",axis=1)\nquestions","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-11-14T16:50:57.321578Z","iopub.status.busy":"2020-11-14T16:50:57.320409Z","iopub.status.idle":"2020-11-14T16:50:57.344152Z","shell.execute_reply":"2020-11-14T16:50:57.343287Z"},"papermill":{"duration":0.186704,"end_time":"2020-11-14T16:50:57.344292","exception":false,"start_time":"2020-11-14T16:50:57.157588","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"questions.describe()","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-11-14T16:50:57.692963Z","iopub.status.busy":"2020-11-14T16:50:57.692205Z","iopub.status.idle":"2020-11-14T16:50:57.93342Z","shell.execute_reply":"2020-11-14T16:50:57.932761Z"},"papermill":{"duration":0.430898,"end_time":"2020-11-14T16:50:57.933545","exception":false,"start_time":"2020-11-14T16:50:57.502647","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"qdf.describe()","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.156601,"end_time":"2020-11-14T16:50:58.247446","exception":false,"start_time":"2020-11-14T16:50:58.090845","status":"completed"},"tags":[]},"cell_type":"markdown","source":"# content idとquestion idが同じなので、mergeしてしまう"},{"metadata":{"execution":{"iopub.execute_input":"2020-11-14T16:50:58.568307Z","iopub.status.busy":"2020-11-14T16:50:58.567265Z","iopub.status.idle":"2020-11-14T16:50:58.572421Z","shell.execute_reply":"2020-11-14T16:50:58.571667Z"},"papermill":{"duration":0.169396,"end_time":"2020-11-14T16:50:58.572543","exception":false,"start_time":"2020-11-14T16:50:58.403147","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"questions.columns","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-11-14T16:50:58.898492Z","iopub.status.busy":"2020-11-14T16:50:58.893984Z","iopub.status.idle":"2020-11-14T16:50:58.907569Z","shell.execute_reply":"2020-11-14T16:50:58.908086Z"},"papermill":{"duration":0.17805,"end_time":"2020-11-14T16:50:58.908239","exception":false,"start_time":"2020-11-14T16:50:58.730189","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"questions.columns=['content_id', 'bundle_id', 'correct_answer', 'part', 'tag0', 'tag1',\n       'tag2', 'tag3', 'tag4', 'tag5']\nquestions","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-11-14T16:50:59.236397Z","iopub.status.busy":"2020-11-14T16:50:59.235136Z","iopub.status.idle":"2020-11-14T16:50:59.468066Z","shell.execute_reply":"2020-11-14T16:50:59.467299Z"},"papermill":{"duration":0.399292,"end_time":"2020-11-14T16:50:59.468189","exception":false,"start_time":"2020-11-14T16:50:59.068897","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"qdf2 = pd.merge(qdf,questions,on=\"content_id\",how=\"left\")\nqdf2","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-11-14T16:50:59.799213Z","iopub.status.busy":"2020-11-14T16:50:59.797457Z","iopub.status.idle":"2020-11-14T16:50:59.843165Z","shell.execute_reply":"2020-11-14T16:50:59.842422Z"},"papermill":{"duration":0.216256,"end_time":"2020-11-14T16:50:59.843286","exception":false,"start_time":"2020-11-14T16:50:59.62703","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"qdf115 = qdf2[qdf2[\"user_id\"]==115]","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-11-14T16:51:00.199139Z","iopub.status.busy":"2020-11-14T16:51:00.192475Z","iopub.status.idle":"2020-11-14T16:51:00.205567Z","shell.execute_reply":"2020-11-14T16:51:00.204795Z"},"papermill":{"duration":0.201924,"end_time":"2020-11-14T16:51:00.205694","exception":false,"start_time":"2020-11-14T16:51:00.00377","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"qdf115.tail(30)","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.161002,"end_time":"2020-11-14T16:51:00.527433","exception":false,"start_time":"2020-11-14T16:51:00.366431","status":"completed"},"tags":[]},"cell_type":"markdown","source":"#### bundle idは３つ質問があったら、それと紐づいている ex 3363-3365の問題は3363と一緒にされたもの　という意味。\n#### partの1,2,3,4で時間ロスが大きく変わっているので、問題の傾向が変わっている\n#### tagは問題の形式かな ? 何かが入っていると正答率が悪いとかあれば、面白い　meltしちゃって、並べても良いが、1億行にさらに増やしてもな・・・と迷いどころ\n#### ×6の6億個になってしまう"},{"metadata":{"papermill":{"duration":0.163601,"end_time":"2020-11-14T16:51:00.856015","exception":false,"start_time":"2020-11-14T16:51:00.692414","status":"completed"},"tags":[],"trusted":false},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.163853,"end_time":"2020-11-14T16:51:01.182465","exception":false,"start_time":"2020-11-14T16:51:01.018612","status":"completed"},"tags":[],"trusted":false},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.168524,"end_time":"2020-11-14T16:51:01.516758","exception":false,"start_time":"2020-11-14T16:51:01.348234","status":"completed"},"tags":[]},"cell_type":"markdown","source":"# partごとの正答率を出してみる"},{"metadata":{"execution":{"iopub.execute_input":"2020-11-14T16:51:01.852584Z","iopub.status.busy":"2020-11-14T16:51:01.851568Z","iopub.status.idle":"2020-11-14T16:51:01.854853Z","shell.execute_reply":"2020-11-14T16:51:01.854096Z"},"papermill":{"duration":0.175476,"end_time":"2020-11-14T16:51:01.854977","exception":false,"start_time":"2020-11-14T16:51:01.679501","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"# さきほどのmakingrate2のqdfをqdf2に変えただけ\ndef makingrate2(col):\n    \n\n    userdf = qdf2.groupby(col)\n    userdf_calc = userdf.agg({'answered_correctly': ['mean',\"sum\", 'count', 'std']}).copy() # .aggでその中身をmeanやsumなど複数処理可能。\n    userdf_calc.columns = [\n        'mean', \n        \"sum\",\n        'count', \n        'std', \n     ]\n    userdf_calc[\"rate\"] = userdf_calc[\"sum\"]/userdf_calc[\"count\"]\n    \n   \n    return userdf_calc\n","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-11-14T16:51:02.190614Z","iopub.status.busy":"2020-11-14T16:51:02.189816Z","iopub.status.idle":"2020-11-14T16:51:02.365838Z","shell.execute_reply":"2020-11-14T16:51:02.366406Z"},"papermill":{"duration":0.346896,"end_time":"2020-11-14T16:51:02.366568","exception":false,"start_time":"2020-11-14T16:51:02.019672","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"partdf = makingrate2(\"part\")\npartdf","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-11-14T16:51:02.720563Z","iopub.status.busy":"2020-11-14T16:51:02.719788Z","iopub.status.idle":"2020-11-14T16:51:02.787237Z","shell.execute_reply":"2020-11-14T16:51:02.787829Z"},"papermill":{"duration":0.25349,"end_time":"2020-11-14T16:51:02.788","exception":false,"start_time":"2020-11-14T16:51:02.53451","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"px.bar(partdf,x=partdf.index,y=\"rate\",width=500)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## tagごとの正答率を見る"},{"metadata":{"trusted":true},"cell_type":"code","source":"qdf2","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"qdf2_tag = qdf2[[\"row_id\",\"content_id\",\"answered_correctly\",\"tag0\",\"tag1\",\"tag2\",\"tag3\",\"tag4\",\"tag5\"]]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"qdf2_tag","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"qdf_melt = pd.melt(qdf2_tag,id_vars=[\"row_id\",\"content_id\",\"answered_correctly\"],var_name=\"tag\",value_name=\"tagvalue\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"qdf_melt","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# nanデータの削除の練習"},{"metadata":{"trusted":true},"cell_type":"code","source":"qdf_melt[\"tagvalue\"].unique()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# nan という文字列がある。これとNaNが別物"},{"metadata":{"trusted":true},"cell_type":"code","source":"qdf_melt[\"tagvalue\"] =  pd.to_numeric( qdf_melt['tagvalue'], errors='coerce') # nanという文字列をNaNに変えているのだと思います。","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"qdf_melt = qdf_melt.dropna(how=\"any\")\nqdf_melt","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# さきほどのmakingrate2のqdfをqdf2に変えただけ\ndef makingrate3(col):\n    \n\n    userdf = qdf_melt.groupby(col)\n    userdf_calc = userdf.agg({'answered_correctly': ['mean',\"sum\", 'count', 'std']}).copy() # .aggでその中身をmeanやsumなど複数処理可能。\n    userdf_calc.columns = [\n        'mean', \n        \"sum\",\n        'count', \n        'std', \n     ]\n    userdf_calc[\"rate\"] = userdf_calc[\"sum\"]/userdf_calc[\"count\"]\n    \n   \n    return userdf_calc\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tagdf = makingrate3(\"tagvalue\")\ntagdf","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"px.bar(tagdf,x=tagdf.index,y=\"mean\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# tagの順番にも何か影響がありそう。"},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tagdf2 = makingrate3(\"tag\")\ntagdf2","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"px.bar(tagdf2,x=tagdf2.index,y=\"mean\",width=500)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"px.line(tagdf2,x=tagdf2.index,y=\"count\",width=500)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 後ろのほうにかかれているtagのほうが正答率高いかも。データ少ないだけ ?"},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.174016,"end_time":"2020-11-14T16:51:04.024032","exception":false,"start_time":"2020-11-14T16:51:03.850016","status":"completed"},"tags":[],"trusted":false},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.170461,"end_time":"2020-11-14T16:51:04.366741","exception":false,"start_time":"2020-11-14T16:51:04.19628","status":"completed"},"tags":[],"trusted":false},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.169657,"end_time":"2020-11-14T16:51:04.70695","exception":false,"start_time":"2020-11-14T16:51:04.537293","status":"completed"},"tags":[]},"cell_type":"markdown","source":"# <span style=\"background-color: yellow; \">3.lecture.csv"},{"metadata":{"execution":{"iopub.execute_input":"2020-11-14T16:51:05.054292Z","iopub.status.busy":"2020-11-14T16:51:05.053253Z","iopub.status.idle":"2020-11-14T16:51:05.056675Z","shell.execute_reply":"2020-11-14T16:51:05.055926Z"},"papermill":{"duration":0.178061,"end_time":"2020-11-14T16:51:05.056807","exception":false,"start_time":"2020-11-14T16:51:04.878746","status":"completed"},"tags":[],"trusted":false},"cell_type":"code","source":"## traindfの中でもlectureだけを抜く","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-11-14T16:51:05.465664Z","iopub.status.busy":"2020-11-14T16:51:05.464796Z","iopub.status.idle":"2020-11-14T16:51:05.469588Z","shell.execute_reply":"2020-11-14T16:51:05.468984Z"},"papermill":{"duration":0.242191,"end_time":"2020-11-14T16:51:05.46972","exception":false,"start_time":"2020-11-14T16:51:05.227529","status":"completed"},"tags":[],"trusted":false},"cell_type":"code","source":"traindf","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-11-14T16:51:05.819347Z","iopub.status.busy":"2020-11-14T16:51:05.818557Z","iopub.status.idle":"2020-11-14T16:51:05.84792Z","shell.execute_reply":"2020-11-14T16:51:05.847227Z"},"papermill":{"duration":0.206462,"end_time":"2020-11-14T16:51:05.848058","exception":false,"start_time":"2020-11-14T16:51:05.641596","status":"completed"},"tags":[],"trusted":false},"cell_type":"code","source":"ldf = traindf[traindf[\"content_type_id\"]==1]\nldf","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-11-14T16:51:06.197103Z","iopub.status.busy":"2020-11-14T16:51:06.196244Z","iopub.status.idle":"2020-11-14T16:51:06.212786Z","shell.execute_reply":"2020-11-14T16:51:06.211994Z"},"papermill":{"duration":0.194662,"end_time":"2020-11-14T16:51:06.212915","exception":false,"start_time":"2020-11-14T16:51:06.018253","status":"completed"},"tags":[],"trusted":false},"cell_type":"code","source":"lectures = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/lectures.csv')\nlectures","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.172865,"end_time":"2020-11-14T16:51:06.557805","exception":false,"start_time":"2020-11-14T16:51:06.38494","status":"completed"},"tags":[]},"cell_type":"markdown","source":"* lecture id : traindfのcontent id\n* tag : 講義用の1つのタグコード。タグの意味は提供されませんが、これらのコードは講義をまとめるのに十分です。\n* part : 講義のトップレベルのカテゴリコード。\n* type of : 講義のタイプ\n"},{"metadata":{"execution":{"iopub.execute_input":"2020-11-14T16:51:06.907863Z","iopub.status.busy":"2020-11-14T16:51:06.906813Z","iopub.status.idle":"2020-11-14T16:51:06.909341Z","shell.execute_reply":"2020-11-14T16:51:06.909909Z"},"papermill":{"duration":0.180679,"end_time":"2020-11-14T16:51:06.910061","exception":false,"start_time":"2020-11-14T16:51:06.729382","status":"completed"},"tags":[],"trusted":false},"cell_type":"code","source":"# これもマージしてみる","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-11-14T16:51:07.263419Z","iopub.status.busy":"2020-11-14T16:51:07.262308Z","iopub.status.idle":"2020-11-14T16:51:07.266483Z","shell.execute_reply":"2020-11-14T16:51:07.267038Z"},"papermill":{"duration":0.184115,"end_time":"2020-11-14T16:51:07.267205","exception":false,"start_time":"2020-11-14T16:51:07.08309","status":"completed"},"tags":[],"trusted":false},"cell_type":"code","source":"lectures.columns","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-11-14T16:51:07.621264Z","iopub.status.busy":"2020-11-14T16:51:07.620496Z","iopub.status.idle":"2020-11-14T16:51:07.624219Z","shell.execute_reply":"2020-11-14T16:51:07.623434Z"},"papermill":{"duration":0.183569,"end_time":"2020-11-14T16:51:07.624387","exception":false,"start_time":"2020-11-14T16:51:07.440818","status":"completed"},"tags":[],"trusted":false},"cell_type":"code","source":"lectures.columns = ['content_id', 'tag', 'part', 'type_of']","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-11-14T16:51:07.983622Z","iopub.status.busy":"2020-11-14T16:51:07.982759Z","iopub.status.idle":"2020-11-14T16:51:08.022548Z","shell.execute_reply":"2020-11-14T16:51:08.02192Z"},"papermill":{"duration":0.220778,"end_time":"2020-11-14T16:51:08.022682","exception":false,"start_time":"2020-11-14T16:51:07.801904","status":"completed"},"tags":[],"trusted":false},"cell_type":"code","source":"ldf2 = pd.merge(ldf,lectures,on=\"content_id\",how=\"left\")\nldf2","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-11-14T16:51:08.41067Z","iopub.status.busy":"2020-11-14T16:51:08.409843Z","iopub.status.idle":"2020-11-14T16:51:08.432634Z","shell.execute_reply":"2020-11-14T16:51:08.431854Z"},"papermill":{"duration":0.22794,"end_time":"2020-11-14T16:51:08.43276","exception":false,"start_time":"2020-11-14T16:51:08.20482","status":"completed"},"tags":[],"trusted":false},"cell_type":"code","source":"ldf2[ldf2[\"user_id\"]==20938253]","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.175502,"end_time":"2020-11-14T16:51:08.787753","exception":false,"start_time":"2020-11-14T16:51:08.612251","status":"completed"},"tags":[]},"cell_type":"markdown","source":"# 講義は意味あるのだろうか・・・"},{"metadata":{"execution":{"iopub.execute_input":"2020-11-14T16:51:09.145983Z","iopub.status.busy":"2020-11-14T16:51:09.145194Z","iopub.status.idle":"2020-11-14T16:51:09.172407Z","shell.execute_reply":"2020-11-14T16:51:09.173039Z"},"papermill":{"duration":0.210178,"end_time":"2020-11-14T16:51:09.173196","exception":false,"start_time":"2020-11-14T16:51:08.963018","status":"completed"},"tags":[],"trusted":false},"cell_type":"code","source":"tdf20938253 = traindf[traindf[\"user_id\"]==20938253]\ntdf20938253","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-11-14T16:51:09.532609Z","iopub.status.busy":"2020-11-14T16:51:09.531772Z","iopub.status.idle":"2020-11-14T16:51:09.766221Z","shell.execute_reply":"2020-11-14T16:51:09.765467Z"},"papermill":{"duration":0.417494,"end_time":"2020-11-14T16:51:09.766374","exception":false,"start_time":"2020-11-14T16:51:09.34888","status":"completed"},"tags":[],"trusted":false},"cell_type":"code","source":"tdf20938253[\"content_type_id\"].plot()","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.177979,"end_time":"2020-11-14T16:51:10.126735","exception":false,"start_time":"2020-11-14T16:51:09.948756","status":"completed"},"tags":[]},"cell_type":"markdown","source":"# 講義見て、問題解いて・・・とかのケースがあるということだと推測。"},{"metadata":{"papermill":{"duration":0.18066,"end_time":"2020-11-14T16:51:10.484465","exception":false,"start_time":"2020-11-14T16:51:10.303805","status":"completed"},"tags":[],"trusted":false},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# いったんまとめ\n## -- 正答率に影響ありそうなパラメータ --\n* content_id : 問題\n* prior_question_had_explanation : 前の問題のあと説明や解説があったか。\n* part : TOEICのパート。リスニング、リーディングなど\n* task_contaier_id : 章\n* tag : 問題のタグ\n* tag no : 問題のタグが最初から何番目にかかれているか\n\n## --- まだわからないところ ---\n* lectureを受けた後、正答率が上がるか ?\n* user idはtestデータでも重複する可能性があるか ?"},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.181852,"end_time":"2020-11-14T16:51:10.848219","exception":false,"start_time":"2020-11-14T16:51:10.666367","status":"completed"},"tags":[]},"cell_type":"markdown","source":"# <span style=\"background-color: yellow; \">4. example_test.csv"},{"metadata":{"execution":{"iopub.execute_input":"2020-11-14T16:51:11.218438Z","iopub.status.busy":"2020-11-14T16:51:11.217555Z","iopub.status.idle":"2020-11-14T16:51:11.244311Z","shell.execute_reply":"2020-11-14T16:51:11.243522Z"},"papermill":{"duration":0.214349,"end_time":"2020-11-14T16:51:11.244463","exception":false,"start_time":"2020-11-14T16:51:11.030114","status":"completed"},"tags":[],"trusted":false},"cell_type":"code","source":"test_ex = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/example_test.csv')\ntest_ex","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-11-14T16:51:11.63208Z","iopub.status.busy":"2020-11-14T16:51:11.631226Z","iopub.status.idle":"2020-11-14T16:51:11.635115Z","shell.execute_reply":"2020-11-14T16:51:11.635697Z"},"papermill":{"duration":0.208015,"end_time":"2020-11-14T16:51:11.635866","exception":false,"start_time":"2020-11-14T16:51:11.427851","status":"completed"},"tags":[],"trusted":false},"cell_type":"code","source":"traindf.head(3)","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.18086,"end_time":"2020-11-14T16:51:11.998532","exception":false,"start_time":"2020-11-14T16:51:11.817672","status":"completed"},"tags":[],"trusted":false},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.204776,"end_time":"2020-11-14T16:51:12.386571","exception":false,"start_time":"2020-11-14T16:51:12.181795","status":"completed"},"tags":[]},"cell_type":"markdown","source":"## train.csvと違うところ\n* group numberがある\n* prior_groups_answers_correctがある\n* priot_groups_responsesがある\n* answered_correctlyがない。（これを推測)"},{"metadata":{"execution":{"iopub.execute_input":"2020-11-14T16:51:12.801105Z","iopub.status.busy":"2020-11-14T16:51:12.800223Z","iopub.status.idle":"2020-11-14T16:51:12.805971Z","shell.execute_reply":"2020-11-14T16:51:12.805202Z"},"papermill":{"duration":0.217527,"end_time":"2020-11-14T16:51:12.806098","exception":false,"start_time":"2020-11-14T16:51:12.588571","status":"completed"},"tags":[],"trusted":false},"cell_type":"code","source":"test_ex","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-11-14T16:51:13.202472Z","iopub.status.busy":"2020-11-14T16:51:13.20149Z","iopub.status.idle":"2020-11-14T16:51:13.206948Z","shell.execute_reply":"2020-11-14T16:51:13.206347Z"},"papermill":{"duration":0.214015,"end_time":"2020-11-14T16:51:13.207083","exception":false,"start_time":"2020-11-14T16:51:12.993068","status":"completed"},"tags":[],"trusted":false},"cell_type":"code","source":"test_ex[test_ex[\"user_id\"]==275030867]","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-11-14T16:51:13.606547Z","iopub.status.busy":"2020-11-14T16:51:13.605486Z","iopub.status.idle":"2020-11-14T16:51:13.612184Z","shell.execute_reply":"2020-11-14T16:51:13.611571Z"},"papermill":{"duration":0.216663,"end_time":"2020-11-14T16:51:13.612336","exception":false,"start_time":"2020-11-14T16:51:13.395673","status":"completed"},"tags":[],"trusted":false},"cell_type":"code","source":"test_ex[test_ex[\"user_id\"]==554169193]","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.192811,"end_time":"2020-11-14T16:51:13.993742","exception":false,"start_time":"2020-11-14T16:51:13.800931","status":"completed"},"tags":[],"trusted":false},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-11-14T16:51:14.388635Z","iopub.status.busy":"2020-11-14T16:51:14.387551Z","iopub.status.idle":"2020-11-14T16:51:14.393157Z","shell.execute_reply":"2020-11-14T16:51:14.392557Z"},"papermill":{"duration":0.215222,"end_time":"2020-11-14T16:51:14.393281","exception":false,"start_time":"2020-11-14T16:51:14.178059","status":"completed"},"tags":[],"trusted":false},"cell_type":"code","source":"g0 = test_ex[test_ex[\"group_num\"]==0]\ng0","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-11-14T16:51:14.79115Z","iopub.status.busy":"2020-11-14T16:51:14.79005Z","iopub.status.idle":"2020-11-14T16:51:14.79596Z","shell.execute_reply":"2020-11-14T16:51:14.795339Z"},"papermill":{"duration":0.219099,"end_time":"2020-11-14T16:51:14.796091","exception":false,"start_time":"2020-11-14T16:51:14.576992","status":"completed"},"tags":[],"trusted":false},"cell_type":"code","source":"g1 = test_ex[test_ex[\"group_num\"]==1]\ng1","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.186732,"end_time":"2020-11-14T16:51:15.170517","exception":false,"start_time":"2020-11-14T16:51:14.983785","status":"completed"},"tags":[]},"cell_type":"markdown","source":"#### 0行目に前のgroupが正解したかなどの情報がはいっている"},{"metadata":{"execution":{"iopub.execute_input":"2020-11-14T16:51:15.546408Z","iopub.status.busy":"2020-11-14T16:51:15.545411Z","iopub.status.idle":"2020-11-14T16:51:15.550382Z","shell.execute_reply":"2020-11-14T16:51:15.549602Z"},"papermill":{"duration":0.195688,"end_time":"2020-11-14T16:51:15.550519","exception":false,"start_time":"2020-11-14T16:51:15.354831","status":"completed"},"tags":[],"trusted":false},"cell_type":"code","source":"tmp = g1.iloc[0,-2]\ntmp","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-11-14T16:51:15.934062Z","iopub.status.busy":"2020-11-14T16:51:15.933093Z","iopub.status.idle":"2020-11-14T16:51:15.937427Z","shell.execute_reply":"2020-11-14T16:51:15.938019Z"},"papermill":{"duration":0.200415,"end_time":"2020-11-14T16:51:15.938197","exception":false,"start_time":"2020-11-14T16:51:15.737782","status":"completed"},"tags":[],"trusted":false},"cell_type":"code","source":"tmp = g1.iloc[0,-2]\ntmp = tmp.split(\"[\")[1]\ntmp = tmp.split(\"]\")[0]\ntmp = tmp.split(\",\")\ntmp","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-11-14T16:51:16.320094Z","iopub.status.busy":"2020-11-14T16:51:16.319091Z","iopub.status.idle":"2020-11-14T16:51:16.322258Z","shell.execute_reply":"2020-11-14T16:51:16.321587Z"},"papermill":{"duration":0.196633,"end_time":"2020-11-14T16:51:16.322405","exception":false,"start_time":"2020-11-14T16:51:16.125772","status":"completed"},"tags":[],"trusted":false},"cell_type":"code","source":"def mojihenkan(mojilist):\n    tmp = mojilist\n    tmp = tmp.split(\"[\")[1]\n    tmp = tmp.split(\"]\")[0]\n    tmp = tmp.split(\",\")\n    return tmp","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-11-14T16:51:16.712281Z","iopub.status.busy":"2020-11-14T16:51:16.711154Z","iopub.status.idle":"2020-11-14T16:51:16.715868Z","shell.execute_reply":"2020-11-14T16:51:16.714994Z"},"papermill":{"duration":0.205419,"end_time":"2020-11-14T16:51:16.716043","exception":false,"start_time":"2020-11-14T16:51:16.510624","status":"completed"},"tags":[],"trusted":false},"cell_type":"code","source":"g0[\"correct_ornot\"] = mojihenkan(g1.iloc[0,-2])\ng0[\"choice\"] = mojihenkan(g1.iloc[0,-1])","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-11-14T16:51:17.124715Z","iopub.status.busy":"2020-11-14T16:51:17.123056Z","iopub.status.idle":"2020-11-14T16:51:17.129907Z","shell.execute_reply":"2020-11-14T16:51:17.130887Z"},"papermill":{"duration":0.22352,"end_time":"2020-11-14T16:51:17.131081","exception":false,"start_time":"2020-11-14T16:51:16.907561","status":"completed"},"tags":[],"trusted":false},"cell_type":"code","source":"g0","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.189895,"end_time":"2020-11-14T16:51:17.512061","exception":false,"start_time":"2020-11-14T16:51:17.322166","status":"completed"},"tags":[],"trusted":false},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.190569,"end_time":"2020-11-14T16:51:17.893094","exception":false,"start_time":"2020-11-14T16:51:17.702525","status":"completed"},"tags":[]},"cell_type":"markdown","source":"# group(時間?)が進むと、前の回答があっていたかどうかの判定が出る。\n# 提出方法は ?"},{"metadata":{"papermill":{"duration":0.190825,"end_time":"2020-11-14T16:51:18.274352","exception":false,"start_time":"2020-11-14T16:51:18.083527","status":"completed"},"tags":[]},"cell_type":"markdown","source":"# <span style=\"background-color: yellow; \">5. about submission"},{"metadata":{"execution":{"iopub.execute_input":"2020-11-14T16:51:18.660431Z","iopub.status.busy":"2020-11-14T16:51:18.659372Z","iopub.status.idle":"2020-11-14T16:51:18.662215Z","shell.execute_reply":"2020-11-14T16:51:18.662775Z"},"papermill":{"duration":0.198998,"end_time":"2020-11-14T16:51:18.662956","exception":false,"start_time":"2020-11-14T16:51:18.463958","status":"completed"},"tags":[],"trusted":false},"cell_type":"code","source":"# このコンペティションの提出方法は特殊。以下、スターターnotebookから、解釈。\n# inputの中にriiideducationというpyファイルが入っているので、それをimport","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-11-14T16:51:19.051349Z","iopub.status.busy":"2020-11-14T16:51:19.050576Z","iopub.status.idle":"2020-11-14T16:51:19.065391Z","shell.execute_reply":"2020-11-14T16:51:19.064547Z"},"papermill":{"duration":0.211625,"end_time":"2020-11-14T16:51:19.065532","exception":false,"start_time":"2020-11-14T16:51:18.853907","status":"completed"},"tags":[],"trusted":false},"cell_type":"code","source":"import riiideducation\nenv = riiideducation.make_env()","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-11-14T16:51:19.471031Z","iopub.status.busy":"2020-11-14T16:51:19.469979Z","iopub.status.idle":"2020-11-14T16:51:19.472432Z","shell.execute_reply":"2020-11-14T16:51:19.473042Z"},"papermill":{"duration":0.208359,"end_time":"2020-11-14T16:51:19.473199","exception":false,"start_time":"2020-11-14T16:51:19.26484","status":"completed"},"tags":[],"trusted":false},"cell_type":"code","source":"iter_test = env.iter_test()","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-11-14T16:51:19.864298Z","iopub.status.busy":"2020-11-14T16:51:19.863492Z","iopub.status.idle":"2020-11-14T16:51:19.878529Z","shell.execute_reply":"2020-11-14T16:51:19.877357Z"},"papermill":{"duration":0.214315,"end_time":"2020-11-14T16:51:19.878707","exception":false,"start_time":"2020-11-14T16:51:19.664392","status":"completed"},"tags":[],"trusted":false},"cell_type":"code","source":"(test_df, sample_prediction_df) = next(iter_test) # 中身確認のみ本番は消す。\n","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-11-14T16:51:20.283373Z","iopub.status.busy":"2020-11-14T16:51:20.282243Z","iopub.status.idle":"2020-11-14T16:51:20.287582Z","shell.execute_reply":"2020-11-14T16:51:20.286942Z"},"papermill":{"duration":0.218429,"end_time":"2020-11-14T16:51:20.287716","exception":false,"start_time":"2020-11-14T16:51:20.069287","status":"completed"},"tags":[],"trusted":false},"cell_type":"code","source":"test_df","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-11-14T16:51:20.682961Z","iopub.status.busy":"2020-11-14T16:51:20.681894Z","iopub.status.idle":"2020-11-14T16:51:20.687188Z","shell.execute_reply":"2020-11-14T16:51:20.686512Z"},"papermill":{"duration":0.208728,"end_time":"2020-11-14T16:51:20.687337","exception":false,"start_time":"2020-11-14T16:51:20.478609","status":"completed"},"tags":[],"trusted":false},"cell_type":"code","source":"sample_prediction_df","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.191554,"end_time":"2020-11-14T16:51:21.072651","exception":false,"start_time":"2020-11-14T16:51:20.881097","status":"completed"},"tags":[],"trusted":false},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.190827,"end_time":"2020-11-14T16:51:21.455162","exception":false,"start_time":"2020-11-14T16:51:21.264335","status":"completed"},"tags":[]},"cell_type":"markdown","source":"#### Note that we'll get an error if we try to continue on to the next batch without making our predictions for the current batch.\n#### 現在のバッチの予測を作ることなしに次のバッチに行こうとするとエラーが出るよ。と書いてあるので、\n#### 1 groupずつ、predictionして、保存して、次に行くということをやるのだと推測。"},{"metadata":{"execution":{"iopub.execute_input":"2020-11-14T16:51:21.850803Z","iopub.status.busy":"2020-11-14T16:51:21.849726Z","iopub.status.idle":"2020-11-14T16:51:21.853307Z","shell.execute_reply":"2020-11-14T16:51:21.852573Z"},"papermill":{"duration":0.206739,"end_time":"2020-11-14T16:51:21.853474","exception":false,"start_time":"2020-11-14T16:51:21.646735","status":"completed"},"tags":[],"trusted":false},"cell_type":"code","source":"# 保存方法 # 中身確認のみ本番は消す。\nenv.predict(sample_prediction_df)","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-11-14T16:51:22.25244Z","iopub.status.busy":"2020-11-14T16:51:22.251643Z","iopub.status.idle":"2020-11-14T16:51:22.254556Z","shell.execute_reply":"2020-11-14T16:51:22.253937Z"},"papermill":{"duration":0.207202,"end_time":"2020-11-14T16:51:22.2547","exception":false,"start_time":"2020-11-14T16:51:22.047498","status":"completed"},"tags":[],"trusted":false},"cell_type":"code","source":"(test_df, sample_prediction_df) = next(iter_test) # 中身確認のみ本番は消す。","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-11-14T16:51:22.6673Z","iopub.status.busy":"2020-11-14T16:51:22.666207Z","iopub.status.idle":"2020-11-14T16:51:22.673028Z","shell.execute_reply":"2020-11-14T16:51:22.672167Z"},"papermill":{"duration":0.225578,"end_time":"2020-11-14T16:51:22.67317","exception":false,"start_time":"2020-11-14T16:51:22.447592","status":"completed"},"tags":[],"trusted":false},"cell_type":"code","source":"test_df","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-11-14T16:51:23.07004Z","iopub.status.busy":"2020-11-14T16:51:23.068968Z","iopub.status.idle":"2020-11-14T16:51:23.072449Z","shell.execute_reply":"2020-11-14T16:51:23.071679Z"},"papermill":{"duration":0.204526,"end_time":"2020-11-14T16:51:23.072578","exception":false,"start_time":"2020-11-14T16:51:22.868052","status":"completed"},"tags":[],"trusted":false},"cell_type":"code","source":"env.predict(sample_prediction_df)","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.191915,"end_time":"2020-11-14T16:51:23.458663","exception":false,"start_time":"2020-11-14T16:51:23.266748","status":"completed"},"tags":[]},"cell_type":"markdown","source":"# 注意点 :\n# 講義のところは提出しないとある\n# traincsvの中にない問題(content_id)があるとも記述あり\n# ですが、訂正されて、traincsvの中にない問題(content_id)はないそうです(コメントありがとうございます)\n\n* ---大本の記述---\n* https://www.kaggle.com/sohier/competition-api-detailed-introduction　にはそのように書いてありますが、\n\n* ---訂正のお知らせサイト---\n* https://www.kaggle.com/c/riiid-test-answer-prediction/discussion/191106\n\n* ただし、このnotebookの場合、100万行のtrainデータなので、足りません・・・。\n* 参考 100万行のcontent_id数 : 13320個。　trainデータ全部(約1億行)のcontent_id数 : 13782個"},{"metadata":{"execution":{"iopub.execute_input":"2020-11-14T16:51:23.85617Z","iopub.status.busy":"2020-11-14T16:51:23.855286Z","iopub.status.idle":"2020-11-14T16:51:23.860101Z","shell.execute_reply":"2020-11-14T16:51:23.859294Z"},"papermill":{"duration":0.20423,"end_time":"2020-11-14T16:51:23.860233","exception":false,"start_time":"2020-11-14T16:51:23.656003","status":"completed"},"tags":[],"trusted":false},"cell_type":"code","source":"\"\"\"\nbasic codeは以下だが、つまらないので、少し変更する\n\nfor (test_df, sample_prediction_df) in iter_test:\n    test_df['answered_correctly'] = 0.5\n    env.predict(test_df.loc[test_df['content_type_id'] == 0, ['row_id', 'answered_correctly']])\n\n\"\"\"\n\n","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-11-14T16:51:24.289191Z","iopub.status.busy":"2020-11-14T16:51:24.28785Z","iopub.status.idle":"2020-11-14T16:51:24.293917Z","shell.execute_reply":"2020-11-14T16:51:24.29292Z"},"papermill":{"duration":0.233917,"end_time":"2020-11-14T16:51:24.294052","exception":false,"start_time":"2020-11-14T16:51:24.060135","status":"completed"},"tags":[],"trusted":false},"cell_type":"code","source":"test_df","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-11-14T16:51:24.697759Z","iopub.status.busy":"2020-11-14T16:51:24.696669Z","iopub.status.idle":"2020-11-14T16:51:24.71995Z","shell.execute_reply":"2020-11-14T16:51:24.718899Z"},"papermill":{"duration":0.22649,"end_time":"2020-11-14T16:51:24.720096","exception":false,"start_time":"2020-11-14T16:51:24.493606","status":"completed"},"tags":[],"trusted":false},"cell_type":"code","source":"cdf=cdf.reset_index()\ncdf","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-11-14T16:51:25.134275Z","iopub.status.busy":"2020-11-14T16:51:25.133187Z","iopub.status.idle":"2020-11-14T16:51:25.136957Z","shell.execute_reply":"2020-11-14T16:51:25.137592Z"},"papermill":{"duration":0.212185,"end_time":"2020-11-14T16:51:25.137754","exception":false,"start_time":"2020-11-14T16:51:24.925569","status":"completed"},"tags":[],"trusted":false},"cell_type":"code","source":"cdf.rate.mean()","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.199401,"end_time":"2020-11-14T16:51:25.536117","exception":false,"start_time":"2020-11-14T16:51:25.336716","status":"completed"},"tags":[],"trusted":false},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-11-14T16:51:25.93859Z","iopub.status.busy":"2020-11-14T16:51:25.937788Z","iopub.status.idle":"2020-11-14T16:51:25.981356Z","shell.execute_reply":"2020-11-14T16:51:25.980515Z"},"papermill":{"duration":0.247492,"end_time":"2020-11-14T16:51:25.981491","exception":false,"start_time":"2020-11-14T16:51:25.733999","status":"completed"},"tags":[],"trusted":false},"cell_type":"code","source":"test_df2 = pd.merge(test_df,cdf,on=\"content_id\",how=\"left\")\ntest_df2","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-11-14T16:51:26.394441Z","iopub.status.busy":"2020-11-14T16:51:26.393592Z","iopub.status.idle":"2020-11-14T16:51:26.718098Z","shell.execute_reply":"2020-11-14T16:51:26.717236Z"},"papermill":{"duration":0.535882,"end_time":"2020-11-14T16:51:26.718234","exception":false,"start_time":"2020-11-14T16:51:26.182352","status":"completed"},"tags":[],"trusted":false},"cell_type":"code","source":"for (test_df, sample_prediction_df) in iter_test:\n    test_df = test_df[test_df[\"content_type_id\"]==0] # 質問only\n    test_df = pd.merge(test_df,cdf,on=\"content_id\",how=\"left\") # 問題ごとの正答率をマージ\n    test_df['answered_correctly'] = test_df[\"rate\"]\n    test_df = test_df[[\"content_type_id\",\"row_id\",\"answered_correctly\"]] # この後のfillnaをするときに、他の列があると、型が違くて無理というエラーが出ることがあるため\n    \n    test_df.fillna(value = 0.696, inplace = True) # 今回のnotebook(100万行データ)でない問題は、全部の問題の正答率の平均を使う。\n    \n    env.predict(test_df.loc[test_df['content_type_id'] == 0, ['row_id', 'answered_correctly']])\n","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.20048,"end_time":"2020-11-14T16:51:27.126585","exception":false,"start_time":"2020-11-14T16:51:26.926105","status":"completed"},"tags":[]},"cell_type":"markdown","source":"# 注意点 : \n1. internet = offでcommitしないと登録できないので、注意。\n1. make_env() が１回しかできないので、最初のグループと2個目のグループはこのnotebookだとtest_dfを解説用に表示したが、このままだと最初と2個目は、0.5に全部なるので、修正必要\n1. 100万個のデータであることに注意。全部train.csvを読み込んで(約1億行)、同様に問題ごとの平均値を入れた時のスコアは0.703(別のnotebookで試しました)"},{"metadata":{"papermill":{"duration":0.202896,"end_time":"2020-11-14T16:51:27.532125","exception":false,"start_time":"2020-11-14T16:51:27.329229","status":"completed"},"tags":[],"trusted":false},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}