{"cells":[{"metadata":{"papermill":{"duration":0.045334,"end_time":"2020-10-19T04:47:03.869876","exception":false,"start_time":"2020-10-19T04:47:03.824542","status":"completed"},"tags":[]},"cell_type":"markdown","source":"# Keras\n\nFeatures:\n* Feature engineering: boolean flags that indicate that user watched at least one lecture of each type\n* Feature engineering: features from question dataset\n* Feature engineering: fill 'prior_question_elapsed_time' empty values with mean value\n* Target encoding using all train data\n* Standard scaling of 'prior_question_elapsed_time'\n* Keras\n* Optuna to find optimal NN parameters"},{"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","execution":{"iopub.execute_input":"2020-10-19T04:47:03.962590Z","iopub.status.busy":"2020-10-19T04:47:03.961593Z","iopub.status.idle":"2020-10-19T04:47:03.968312Z","shell.execute_reply":"2020-10-19T04:47:03.967479Z"},"papermill":{"duration":0.05659,"end_time":"2020-10-19T04:47:03.968454","exception":false,"start_time":"2020-10-19T04:47:03.911864","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\n\nimport warnings\nwarnings.simplefilter('ignore')\n\nimport gc, sys\ngc.enable()\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.037968,"end_time":"2020-10-19T04:47:04.045442","exception":false,"start_time":"2020-10-19T04:47:04.007474","status":"completed"},"tags":[]},"cell_type":"markdown","source":"## Load data"},{"metadata":{"papermill":{"duration":0.036696,"end_time":"2020-10-19T04:47:04.119244","exception":false,"start_time":"2020-10-19T04:47:04.082548","status":"completed"},"tags":[]},"cell_type":"markdown","source":"Load more train data. Keep records where user last time answers a question."},{"metadata":{"execution":{"iopub.execute_input":"2020-10-19T04:47:04.207366Z","iopub.status.busy":"2020-10-19T04:47:04.206114Z","iopub.status.idle":"2020-10-19T04:47:04.209628Z","shell.execute_reply":"2020-10-19T04:47:04.210412Z"},"papermill":{"duration":0.0542,"end_time":"2020-10-19T04:47:04.210599","exception":false,"start_time":"2020-10-19T04:47:04.156399","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"chunksize = 10**6","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-10-19T04:47:04.307303Z","iopub.status.busy":"2020-10-19T04:47:04.306193Z","iopub.status.idle":"2020-10-19T04:49:52.581781Z","shell.execute_reply":"2020-10-19T04:49:52.582602Z"},"papermill":{"duration":168.327403,"end_time":"2020-10-19T04:49:52.582889","exception":false,"start_time":"2020-10-19T04:47:04.255486","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"train = None\n\nload_count = 0\n\nfor load_train in pd.read_csv('/kaggle/input/riiid-test-answer-prediction/train.csv', chunksize=chunksize, iterator=True):\n    \n    load_train = load_train[load_train['answered_correctly'] != -1]\n    \n    load_train['user_id__content_id'] = load_train['user_id'].astype(str) + '__' + load_train['content_id'].astype(str)\n    \n    load_train = load_train.drop_duplicates('user_id__content_id', keep = 'last')\n    \n    if train is None:\n        train = load_train\n    else:\n        train = pd.concat([train, load_train]).drop_duplicates('user_id__content_id', keep = 'last')\n    \n    load_count += chunksize\n    print('Rows processed:', load_count, 'Train set size:', train.shape[0])\n    \n    if load_count >= 15 * chunksize:\n        break","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-10-19T04:49:52.696792Z","iopub.status.busy":"2020-10-19T04:49:52.695621Z","iopub.status.idle":"2020-10-19T04:49:52.703277Z","shell.execute_reply":"2020-10-19T04:49:52.702599Z"},"papermill":{"duration":0.067206,"end_time":"2020-10-19T04:49:52.703405","exception":false,"start_time":"2020-10-19T04:49:52.636199","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"lectures = pd.read_csv(\"/kaggle/input/riiid-test-answer-prediction/lectures.csv\")","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-10-19T04:49:52.798037Z","iopub.status.busy":"2020-10-19T04:49:52.797114Z","iopub.status.idle":"2020-10-19T04:49:52.813135Z","shell.execute_reply":"2020-10-19T04:49:52.812334Z"},"papermill":{"duration":0.066054,"end_time":"2020-10-19T04:49:52.813270","exception":false,"start_time":"2020-10-19T04:49:52.747216","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"questions = pd.read_csv(\"/kaggle/input/riiid-test-answer-prediction/questions.csv\")","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.043069,"end_time":"2020-10-19T04:49:52.900180","exception":false,"start_time":"2020-10-19T04:49:52.857111","status":"completed"},"tags":[]},"cell_type":"markdown","source":"## Feature engineering"},{"metadata":{"execution":{"iopub.execute_input":"2020-10-19T04:49:52.993584Z","iopub.status.busy":"2020-10-19T04:49:52.992777Z","iopub.status.idle":"2020-10-19T04:49:52.996947Z","shell.execute_reply":"2020-10-19T04:49:52.996109Z"},"papermill":{"duration":0.05368,"end_time":"2020-10-19T04:49:52.997088","exception":false,"start_time":"2020-10-19T04:49:52.943408","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"question_columns = ['question_id', 'bundle_id', 'part']\n\n\ndef merge_question_columns(data):\n    return data.merge(questions[question_columns], right_on='question_id', left_on='content_id', how='left')","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-10-19T04:49:53.093734Z","iopub.status.busy":"2020-10-19T04:49:53.092972Z","iopub.status.idle":"2020-10-19T04:49:55.728508Z","shell.execute_reply":"2020-10-19T04:49:55.727843Z"},"papermill":{"duration":2.686014,"end_time":"2020-10-19T04:49:55.728647","exception":false,"start_time":"2020-10-19T04:49:53.042633","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"train = merge_question_columns(train)","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.043997,"end_time":"2020-10-19T04:49:55.817018","exception":false,"start_time":"2020-10-19T04:49:55.773021","status":"completed"},"tags":[]},"cell_type":"markdown","source":"### Per user lecture stats"},{"metadata":{"papermill":{"duration":0.043066,"end_time":"2020-10-19T04:49:55.903773","exception":false,"start_time":"2020-10-19T04:49:55.860707","status":"completed"},"tags":[]},"cell_type":"markdown","source":"Do OHE for some lectures features"},{"metadata":{"execution":{"iopub.execute_input":"2020-10-19T04:49:56.009915Z","iopub.status.busy":"2020-10-19T04:49:56.008532Z","iopub.status.idle":"2020-10-19T04:49:56.014004Z","shell.execute_reply":"2020-10-19T04:49:56.013321Z"},"papermill":{"duration":0.066957,"end_time":"2020-10-19T04:49:56.014149","exception":false,"start_time":"2020-10-19T04:49:55.947192","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"lectures['type_of'] = lectures['type_of'].replace('solving question', 'solving_question')\n\nlectures = pd.get_dummies(lectures, columns=['part', 'type_of'])\n\npart_lectures_columns = [column for column in lectures.columns if column.startswith('part')]\n\ntypes_of_lectures_columns = [column for column in lectures.columns if column.startswith('type_of_')]","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-10-19T04:49:56.115706Z","iopub.status.busy":"2020-10-19T04:49:56.108034Z","iopub.status.idle":"2020-10-19T04:49:56.131893Z","shell.execute_reply":"2020-10-19T04:49:56.131004Z"},"papermill":{"duration":0.073758,"end_time":"2020-10-19T04:49:56.132043","exception":false,"start_time":"2020-10-19T04:49:56.058285","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"lectures.head(10).T","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.046562,"end_time":"2020-10-19T04:49:56.225303","exception":false,"start_time":"2020-10-19T04:49:56.178741","status":"completed"},"tags":[]},"cell_type":"markdown","source":"Collect per user lecture stats"},{"metadata":{"execution":{"iopub.execute_input":"2020-10-19T04:49:56.329549Z","iopub.status.busy":"2020-10-19T04:49:56.328757Z","iopub.status.idle":"2020-10-19T04:49:56.332390Z","shell.execute_reply":"2020-10-19T04:49:56.331715Z"},"papermill":{"duration":0.06062,"end_time":"2020-10-19T04:49:56.332540","exception":false,"start_time":"2020-10-19T04:49:56.271920","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"user_lecture_stats = None\n\n\ndef collect_user_lecture_stats(train):\n    \n    # merge lecture features to train dataset\n    train_lectures = train[train['content_type_id'] == 1].merge(lectures, right_on='lecture_id', left_on='content_id', how='left')\n    \n    # collect per user stats\n    user_lecture_stats_part = train_lectures.groupby('user_id')[part_lectures_columns + types_of_lectures_columns].sum()\n    \n    # add boolean features\n    for column in user_lecture_stats_part.columns:\n        bool_column = column + '_boolean'\n        user_lecture_stats_part[bool_column] = (user_lecture_stats_part[column] > 0).astype(int)\n    \n    return user_lecture_stats_part\n\ndef update_user_lecture_stats(user_lecture_stats_part):\n    global user_lecture_stats\n    if user_lecture_stats is None:\n        user_lecture_stats = user_lecture_stats_part\n    else:\n        user_lecture_stats = user_lecture_stats.add(user_lecture_stats_part, fill_value=0.)\n\ndef merge_user_lecture_stats(data):\n    return data.merge(user_lecture_stats, left_on='user_id', right_index=True, how='left').fillna(0)","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.04558,"end_time":"2020-10-19T04:49:56.624759","exception":false,"start_time":"2020-10-19T04:49:56.579179","status":"completed"},"tags":[]},"cell_type":"markdown","source":"### Target encoding"},{"metadata":{"execution":{"iopub.execute_input":"2020-10-19T04:49:56.734891Z","iopub.status.busy":"2020-10-19T04:49:56.733625Z","iopub.status.idle":"2020-10-19T04:49:56.737992Z","shell.execute_reply":"2020-10-19T04:49:56.738579Z"},"papermill":{"duration":0.069458,"end_time":"2020-10-19T04:49:56.738741","exception":false,"start_time":"2020-10-19T04:49:56.669283","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"from collections import Counter\n\n\nper_value_counts = {}\nglobal_counts = Counter()\n\n\ncolumns_target_encode = ['user_id', 'content_id', 'task_container_id',\n                         'bundle_id', 'part']\n\n\ndef add_or_update(column, value, count, answered_correctly):\n    # column\n    column_data = per_value_counts.get(column, {})\n    per_value_counts[column] = column_data\n    # value\n    value_data = column_data.get(value, Counter())\n    column_data[value] = value_data\n    # counters\n    value_data += Counter({'count': count, 'answered_correctly': answered_correctly})\n\ndef update_counts(data, column):\n    agg = data.groupby(column)['answered_correctly'].agg(['count', 'mean'])\n    agg['answered_correctly'] = agg['count'] * agg['mean']\n    for idx,row in agg.iterrows():\n        add_or_update(column, idx, row['count'], row['answered_correctly'])\n\ndef update_global_counts(data):\n    global global_counts\n    count = data.shape[0]\n    clicks = data[data['answered_correctly'] == 1].shape[0]\n    global_counts += Counter({'count': count, 'answered_correctly': clicks})\n\ndef update_all_counts(data, columns):\n    for column in columns:\n        update_counts(data, column)\n    update_global_counts(data)\n\ndef target_encode_value(column, value):\n    counts = per_value_counts.get(column, {}).get(value, Counter())\n    if 'answered_correctly' in counts:\n        return counts['answered_correctly'] / counts['count']\n    else:\n        return global_counts['answered_correctly'] / global_counts['count']\n\ndef target_encode(data, columns):\n    out = pd.DataFrame(index=data.index)\n    for column in columns:\n        out[column] = data[column].apply(lambda value: target_encode_value(column, value))\n    return out","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.044435,"end_time":"2020-10-19T04:49:56.827907","exception":false,"start_time":"2020-10-19T04:49:56.783472","status":"completed"},"tags":[]},"cell_type":"markdown","source":"### Standard scaling"},{"metadata":{"execution":{"iopub.execute_input":"2020-10-19T04:49:56.927117Z","iopub.status.busy":"2020-10-19T04:49:56.926308Z","iopub.status.idle":"2020-10-19T04:49:58.029636Z","shell.execute_reply":"2020-10-19T04:49:58.028969Z"},"papermill":{"duration":1.155677,"end_time":"2020-10-19T04:49:58.029779","exception":false,"start_time":"2020-10-19T04:49:56.874102","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"from sklearn.preprocessing import StandardScaler\n\n\ncolumns_std = ['prior_question_elapsed_time']\n\n\nscaler = StandardScaler()","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.045504,"end_time":"2020-10-19T04:49:58.120508","exception":false,"start_time":"2020-10-19T04:49:58.075004","status":"completed"},"tags":[]},"cell_type":"markdown","source":"### Warmup using all train data"},{"metadata":{"execution":{"iopub.execute_input":"2020-10-19T04:49:58.221519Z","iopub.status.busy":"2020-10-19T04:49:58.220418Z","iopub.status.idle":"2020-10-19T04:59:33.361265Z","shell.execute_reply":"2020-10-19T04:59:33.360464Z"},"papermill":{"duration":575.196475,"end_time":"2020-10-19T04:59:33.361414","exception":false,"start_time":"2020-10-19T04:49:58.164939","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"warmup_count = 0\n\nfor warmup_train in pd.read_csv('/kaggle/input/riiid-test-answer-prediction/train.csv', chunksize=chunksize, iterator=True):\n    \n    update_user_lecture_stats(collect_user_lecture_stats(warmup_train))\n    \n    warmup_train = warmup_train[warmup_train['answered_correctly'] != -1]\n    \n    warmup_train = merge_question_columns(warmup_train)\n    \n    update_all_counts(warmup_train, columns_target_encode)\n    \n    scaler.partial_fit(warmup_train[columns_std])\n    \n    warmup_count += chunksize\n    # print('Rows processed:', warmup_count)","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-10-19T04:59:33.485124Z","iopub.status.busy":"2020-10-19T04:59:33.483969Z","iopub.status.idle":"2020-10-19T04:59:33.490161Z","shell.execute_reply":"2020-10-19T04:59:33.489506Z"},"papermill":{"duration":0.0837,"end_time":"2020-10-19T04:59:33.490298","exception":false,"start_time":"2020-10-19T04:59:33.406598","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"user_lecture_stats.head(10).T","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-10-19T04:59:33.593791Z","iopub.status.busy":"2020-10-19T04:59:33.592733Z","iopub.status.idle":"2020-10-19T04:59:33.597038Z","shell.execute_reply":"2020-10-19T04:59:33.596316Z"},"papermill":{"duration":0.056157,"end_time":"2020-10-19T04:59:33.597164","exception":false,"start_time":"2020-10-19T04:59:33.541007","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"user_lecture_stats_boolean_columns = [column for column in user_lecture_stats.columns if column.endswith('_boolean')]\n\nuser_lecture_stats_boolean_columns","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.046571,"end_time":"2020-10-19T04:59:33.690308","exception":false,"start_time":"2020-10-19T04:59:33.643737","status":"completed"},"tags":[]},"cell_type":"markdown","source":"## Prepare x, y"},{"metadata":{"execution":{"iopub.execute_input":"2020-10-19T04:59:33.794541Z","iopub.status.busy":"2020-10-19T04:59:33.793570Z","iopub.status.idle":"2020-10-19T04:59:33.796622Z","shell.execute_reply":"2020-10-19T04:59:33.795909Z"},"papermill":{"duration":0.060269,"end_time":"2020-10-19T04:59:33.796741","exception":false,"start_time":"2020-10-19T04:59:33.736472","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"columns_copy = ['prior_question_had_explanation']\n\n\ndef make_x(data):\n    \n    # copy without changes\n    x = data[columns_copy + ['user_id']].fillna(0)\n    \n    # convert Bool to Int\n    x['prior_question_had_explanation'] = x['prior_question_had_explanation'].astype(int)\n    \n    # merge per user lecture stats\n    x_lecture_stats = merge_user_lecture_stats(x)[user_lecture_stats_boolean_columns]\n    \n    x = x.drop('user_id', axis=1)\n    \n    # target encode\n    x_target_encode = target_encode(data, columns_target_encode)\n    \n    # std\n    x_std = pd.DataFrame(scaler.transform(data[columns_std]), index=data.index, columns=columns_std)\n    for i,column in enumerate(x_std.columns):\n        x_std[column].fillna(scaler.mean_[i], inplace=True)\n    \n    return pd.concat([x, x_lecture_stats, x_target_encode, x_std], axis=1)","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-10-19T04:59:33.896953Z","iopub.status.busy":"2020-10-19T04:59:33.895388Z","iopub.status.idle":"2020-10-19T05:04:53.585343Z","shell.execute_reply":"2020-10-19T05:04:53.586011Z"},"papermill":{"duration":319.743427,"end_time":"2020-10-19T05:04:53.586281","exception":false,"start_time":"2020-10-19T04:59:33.842854","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"x_train = make_x(train)\n\ny_train = train['answered_correctly']\n\ndel train\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-10-19T05:04:53.687433Z","iopub.status.busy":"2020-10-19T05:04:53.686310Z","iopub.status.idle":"2020-10-19T05:04:53.727147Z","shell.execute_reply":"2020-10-19T05:04:53.726440Z"},"papermill":{"duration":0.094154,"end_time":"2020-10-19T05:04:53.727280","exception":false,"start_time":"2020-10-19T05:04:53.633126","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"x_train.head(10).T","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.063909,"end_time":"2020-10-19T05:04:53.851047","exception":false,"start_time":"2020-10-19T05:04:53.787138","status":"completed"},"tags":[]},"cell_type":"markdown","source":"## Keras"},{"metadata":{"execution":{"iopub.execute_input":"2020-10-19T05:04:53.952309Z","iopub.status.busy":"2020-10-19T05:04:53.951454Z","iopub.status.idle":"2020-10-19T05:05:01.443449Z","shell.execute_reply":"2020-10-19T05:05:01.442573Z"},"papermill":{"duration":7.544679,"end_time":"2020-10-19T05:05:01.443597","exception":false,"start_time":"2020-10-19T05:04:53.898918","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"import tensorflow as tf\nimport tensorflow_addons as tfa\n\nimport tensorflow.keras.backend as K","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-10-19T05:05:01.552056Z","iopub.status.busy":"2020-10-19T05:05:01.550728Z","iopub.status.idle":"2020-10-19T05:05:01.562266Z","shell.execute_reply":"2020-10-19T05:05:01.562861Z"},"papermill":{"duration":0.070089,"end_time":"2020-10-19T05:05:01.563058","exception":false,"start_time":"2020-10-19T05:05:01.492969","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"def make_layer(x, units, dropout_rate):\n    t = tfa.layers.WeightNormalization(tf.keras.layers.Dense(units))(x)\n    t = tf.keras.layers.BatchNormalization()(t)\n    t = tf.keras.layers.Activation('relu')(t)\n    t = tf.keras.layers.Dropout(dropout_rate)(t)\n    return t\n\n\ndef make_model(columns, units, dropout_rates):\n    \n    inputs = tf.keras.layers.Input(shape=(columns,))\n    x = tf.keras.layers.BatchNormalization()(inputs)\n\n    for i in range(len(units)):\n        u = units[i]\n        d = dropout_rates[i]\n        x = make_layer(x, u, d)\n       \n    y = tf.keras.layers.Dense(1, activation='sigmoid', name='dense_output')(x)\n    \n    model = tf.keras.Model(inputs=inputs, outputs=y)\n    model.compile(loss='binary_crossentropy',\n                  optimizer=tfa.optimizers.AdamW(learning_rate=1e-3, weight_decay=1e-5),\n                  metrics=['accuracy'])\n    return model","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-10-19T05:05:01.677848Z","iopub.status.busy":"2020-10-19T05:05:01.675856Z","iopub.status.idle":"2020-10-19T05:05:01.765595Z","shell.execute_reply":"2020-10-19T05:05:01.764741Z"},"papermill":{"duration":0.152427,"end_time":"2020-10-19T05:05:01.765729","exception":false,"start_time":"2020-10-19T05:05:01.613302","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"from sklearn.model_selection import KFold\n\n\ndef fit_validate(n_splits, x_train, y_train, units, dropout_rates, epochs, verbose, random_state):\n\n    estimators = []\n    histories = []\n    \n    scores = []\n\n    cv = KFold(n_splits=n_splits, shuffle=True, random_state=random_state)\n    for train_idx, valid_idx in cv.split(x_train, y_train):\n\n        x_train_train = x_train.iloc[train_idx]\n        y_train_train = y_train.iloc[train_idx]\n        x_train_valid = x_train.iloc[valid_idx]\n        y_train_valid = y_train.iloc[valid_idx]\n\n        K.clear_session()\n        \n        es = tf.keras.callbacks.EarlyStopping(monitor='val_accuracy', min_delta=0.001, patience=10,\n                                              verbose=verbose, mode='max', restore_best_weights=True)\n\n        rl = tf.keras.callbacks.ReduceLROnPlateau(monitor='val_accuracy', factor=0.1, patience=3, min_lr=1e-5,\n                                                  mode='max', verbose=verbose)\n\n        estimator = make_model(x_train.shape[1], units, dropout_rates)\n\n        history = estimator.fit(x_train_train, y_train_train,\n                                batch_size=128, epochs=epochs, callbacks=[es, rl],\n                                validation_data=(x_train_valid, y_train_valid),\n                                verbose=verbose)\n        \n        estimators.append(estimator)\n        histories.append(history)\n        \n        scores.append(history.history['val_accuracy'][-1])\n    \n    score = np.mean(scores)\n    \n    return estimators, histories, score","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-10-19T05:05:01.876484Z","iopub.status.busy":"2020-10-19T05:05:01.873437Z","iopub.status.idle":"2020-10-19T05:05:02.947834Z","shell.execute_reply":"2020-10-19T05:05:02.946846Z"},"papermill":{"duration":1.132733,"end_time":"2020-10-19T05:05:02.947995","exception":false,"start_time":"2020-10-19T05:05:01.815262","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"import optuna\n\nfrom logging import CRITICAL\noptuna.logging.set_verbosity(CRITICAL)\n\n\ndef objective(trial):\n    \n    n_layers = trial.suggest_int('n_layers', 1, 5)\n    \n    units = []\n    dropout_rates = []\n    for i in range(n_layers):\n        u = trial.suggest_categorical('units_{}'.format(i+1), [16, 32, 64, 128])\n        units.append(u)\n        r = trial.suggest_loguniform('dropout_rate_{}'.format(i+1), 0.1, 0.5)\n        dropout_rates.append(r)\n    \n    print('Units:', units, \"Dropout rates:\", dropout_rates)\n    \n    _, _, score = fit_validate(3, x_train, y_train, units, dropout_rates, 10, 0, 42)\n    print('Score:', score)\n    \n    return score\n\n\n# study = optuna.create_study(direction='maximize')\n# study.optimize(objective, n_trials=100)","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-10-19T05:05:03.067040Z","iopub.status.busy":"2020-10-19T05:05:03.066186Z","iopub.status.idle":"2020-10-19T05:05:03.070479Z","shell.execute_reply":"2020-10-19T05:05:03.069631Z"},"papermill":{"duration":0.070766,"end_time":"2020-10-19T05:05:03.070623","exception":false,"start_time":"2020-10-19T05:05:02.999857","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"# params = study.best_trial.params\n# params","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-10-19T05:05:03.189294Z","iopub.status.busy":"2020-10-19T05:05:03.188078Z","iopub.status.idle":"2020-10-19T05:05:03.196076Z","shell.execute_reply":"2020-10-19T05:05:03.194922Z"},"papermill":{"duration":0.072935,"end_time":"2020-10-19T05:05:03.196248","exception":false,"start_time":"2020-10-19T05:05:03.123313","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"params = {\n    'n_layers': 2,\n    'units_1': 128,\n    'dropout_rate_1': 0.1576269633262961,\n    'units_2': 64,\n    'dropout_rate_2': 0.26394371768001645\n}\nparams","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-10-19T05:05:03.317415Z","iopub.status.busy":"2020-10-19T05:05:03.316514Z","iopub.status.idle":"2020-10-19T07:46:56.596142Z","shell.execute_reply":"2020-10-19T07:46:56.588102Z"},"papermill":{"duration":9713.341688,"end_time":"2020-10-19T07:46:56.596774","exception":false,"start_time":"2020-10-19T05:05:03.255086","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"K.clear_session()\n\n\nn_layers = params['n_layers']\nunits = []\ndropout_rates = []\nfor i in range(n_layers):\n    u = params['units_{}'.format(i+1)]\n    units.append(u)\n    d = params['dropout_rate_{}'.format(i+1)]\n    dropout_rates.append(d)\n\n\nestimators, histories, score = fit_validate(3, x_train, y_train, units, dropout_rates, 50, 2, 42)","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-10-19T07:46:56.807924Z","iopub.status.busy":"2020-10-19T07:46:56.806777Z","iopub.status.idle":"2020-10-19T07:46:56.814570Z","shell.execute_reply":"2020-10-19T07:46:56.813915Z"},"papermill":{"duration":0.117498,"end_time":"2020-10-19T07:46:56.814697","exception":false,"start_time":"2020-10-19T07:46:56.697199","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"print('Validation score:', score)","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-10-19T07:46:58.300853Z","iopub.status.busy":"2020-10-19T07:46:57.111208Z","iopub.status.idle":"2020-10-19T07:46:58.722070Z","shell.execute_reply":"2020-10-19T07:46:58.720557Z"},"papermill":{"duration":1.807869,"end_time":"2020-10-19T07:46:58.722219","exception":false,"start_time":"2020-10-19T07:46:56.914350","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"del x_train\ndel y_train\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.103686,"end_time":"2020-10-19T07:46:58.930423","exception":false,"start_time":"2020-10-19T07:46:58.826737","status":"completed"},"tags":[]},"cell_type":"markdown","source":"## Show graphs"},{"metadata":{"execution":{"iopub.execute_input":"2020-10-19T07:46:59.164239Z","iopub.status.busy":"2020-10-19T07:46:59.163386Z","iopub.status.idle":"2020-10-19T07:47:00.138564Z","shell.execute_reply":"2020-10-19T07:47:00.139188Z"},"papermill":{"duration":1.099701,"end_time":"2020-10-19T07:47:00.139350","exception":false,"start_time":"2020-10-19T07:46:59.039649","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"import matplotlib.pyplot as plt\n\n\nfig, axs = plt.subplots(2, 2, figsize=(18,18))\n\n# accuracy\nfor h in histories:\n    axs[0,0].plot(h.history['accuracy'], color='g')\naxs[0,0].set_title('Model accuracy - Train')\naxs[0,0].set_ylabel('Accuracy')\naxs[0,0].set_xlabel('Epoch')\n\nfor h in histories:\n    axs[0,1].plot(h.history['val_accuracy'], color='b')\naxs[0,1].set_title('Model accuracy - Test')\naxs[0,1].set_ylabel('Accuracy')\naxs[0,1].set_xlabel('Epoch')\n\n# loss\nfor h in histories:\n    axs[1,0].plot(h.history['loss'], color='g')\naxs[1,0].set_title('Model loss - Train')\naxs[1,0].set_ylabel('Loss')\naxs[1,0].set_xlabel('Epoch')\n\nfor h in histories:\n    axs[1,1].plot(h.history['val_loss'], color='b')\naxs[1,1].set_title('Model loss - Test')\naxs[1,1].set_ylabel('Loss')\naxs[1,1].set_xlabel('Epoch')\n\nfig.show()","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.106922,"end_time":"2020-10-19T07:47:00.355830","exception":false,"start_time":"2020-10-19T07:47:00.248908","status":"completed"},"tags":[]},"cell_type":"markdown","source":"## Predict and submit"},{"metadata":{"execution":{"iopub.execute_input":"2020-10-19T07:47:00.569720Z","iopub.status.busy":"2020-10-19T07:47:00.568788Z","iopub.status.idle":"2020-10-19T07:47:00.605923Z","shell.execute_reply":"2020-10-19T07:47:00.605132Z"},"papermill":{"duration":0.146067,"end_time":"2020-10-19T07:47:00.606060","exception":false,"start_time":"2020-10-19T07:47:00.459993","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"import riiideducation\n\nenv = riiideducation.make_env()\n\niter_test = env.iter_test()","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-10-19T07:47:00.829869Z","iopub.status.busy":"2020-10-19T07:47:00.828908Z","iopub.status.idle":"2020-10-19T07:47:03.331346Z","shell.execute_reply":"2020-10-19T07:47:03.332009Z"},"papermill":{"duration":2.617494,"end_time":"2020-10-19T07:47:03.332203","exception":false,"start_time":"2020-10-19T07:47:00.714709","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"for (test, sample_prediction) in iter_test:\n    \n    test = merge_question_columns(test)\n    \n    x_test = make_x(test)\n    \n    y_preds = []\n    for estimator in estimators:\n        y_pred = estimator.predict(x_test)\n        y_preds.append(y_pred)\n    \n    test['answered_correctly'] = np.mean(y_preds, axis=0)\n    \n    env.predict(test.loc[test['content_type_id'] == 0, ['row_id', 'answered_correctly']])","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}