{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"I used **word2vec** to train & predict using this [NOTEBOOK](https://www.kaggle.com/code/syurenuko/clrp-word2vec-lightgbm-baseline/notebook) as a reference.","metadata":{}},{"cell_type":"code","source":"import os\nimport pandas as pd\nimport numpy as np\n\nfrom sklearn.preprocessing import OneHotEncoder\nfrom sklearn.model_selection import StratifiedGroupKFold\nfrom sklearn.metrics import log_loss\n\nimport gensim\nfrom scipy import sparse\nimport lightgbm as lgb\n\nimport warnings\nwarnings.filterwarnings('ignore')","metadata":{"execution":{"iopub.status.busy":"2022-07-06T06:08:43.053415Z","iopub.execute_input":"2022-07-06T06:08:43.054954Z","iopub.status.idle":"2022-07-06T06:08:43.064115Z","shell.execute_reply.started":"2022-07-06T06:08:43.054874Z","shell.execute_reply":"2022-07-06T06:08:43.063220Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# config","metadata":{"id":"tcheO37n2ejd"}},{"cell_type":"code","source":"class CFG:\n    seed = 42\n    n_folds = 4","metadata":{"id":"THSaDZ500zA2","executionInfo":{"status":"ok","timestamp":1657075243345,"user_tz":-540,"elapsed":11,"user":{"displayName":"鈴木明作","userId":"03916733245168369171"}},"execution":{"iopub.status.busy":"2022-07-06T06:08:43.066101Z","iopub.execute_input":"2022-07-06T06:08:43.066498Z","iopub.status.idle":"2022-07-06T06:08:43.079389Z","shell.execute_reply.started":"2022-07-06T06:08:43.066461Z","shell.execute_reply":"2022-07-06T06:08:43.077924Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data Load","metadata":{"id":"aa9gRf-u2hOp"}},{"cell_type":"code","source":"INPUT_DIR = \"../input/feedback-prize-effectiveness/\"","metadata":{"id":"hU4IP1llzn8L","executionInfo":{"status":"ok","timestamp":1657075243346,"user_tz":-540,"elapsed":10,"user":{"displayName":"鈴木明作","userId":"03916733245168369171"}},"execution":{"iopub.status.busy":"2022-07-06T06:08:43.081867Z","iopub.execute_input":"2022-07-06T06:08:43.082861Z","iopub.status.idle":"2022-07-06T06:08:43.092476Z","shell.execute_reply.started":"2022-07-06T06:08:43.082807Z","shell.execute_reply":"2022-07-06T06:08:43.091266Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_train_essay(essay_id):\n    essay_path = os.path.join(INPUT_DIR,f'train/{essay_id}.txt')\n    essay_text = open(essay_path,'r').read()\n    return essay_text\n\ndef get_test_essay(essay_id):\n    essay_path = os.path.join(INPUT_DIR,f'test/{essay_id}.txt')\n    essay_text = open(essay_path,'r').read()\n    return essay_text","metadata":{"id":"2uJ4blLczoAn","executionInfo":{"status":"ok","timestamp":1657076660155,"user_tz":-540,"elapsed":189,"user":{"displayName":"鈴木明作","userId":"03916733245168369171"}},"execution":{"iopub.status.busy":"2022-07-06T06:08:43.095163Z","iopub.execute_input":"2022-07-06T06:08:43.096110Z","iopub.status.idle":"2022-07-06T06:08:43.108799Z","shell.execute_reply.started":"2022-07-06T06:08:43.096045Z","shell.execute_reply":"2022-07-06T06:08:43.107366Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_csv(INPUT_DIR+'train.csv')\ntest = pd.read_csv(INPUT_DIR+'test.csv')\ntrain['essay_text'] = train['essay_id'].apply(get_train_essay)\ntest['essay_text'] = test['essay_id'].apply(get_test_essay)\ndisplay(train)\ndisplay(test)","metadata":{"id":"A2jVwtWmzoFB","executionInfo":{"status":"ok","timestamp":1657076384117,"user_tz":-540,"elapsed":1140778,"user":{"displayName":"鈴木明作","userId":"03916733245168369171"}},"outputId":"98b6de45-3a98-4756-8eb1-2f001a9b4884","execution":{"iopub.status.busy":"2022-07-06T06:08:43.113157Z","iopub.execute_input":"2022-07-06T06:08:43.114335Z","iopub.status.idle":"2022-07-06T06:09:09.127840Z","shell.execute_reply.started":"2022-07-06T06:08:43.114258Z","shell.execute_reply":"2022-07-06T06:09:09.126399Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Seed","metadata":{"id":"ZnDGTc9z2jTV"}},{"cell_type":"code","source":"def set_seed(seed=42):\n    np.random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n\nset_seed(CFG.seed)","metadata":{"id":"LagTCJCb2YNW","executionInfo":{"status":"ok","timestamp":1657076384118,"user_tz":-540,"elapsed":12,"user":{"displayName":"鈴木明作","userId":"03916733245168369171"}},"execution":{"iopub.status.busy":"2022-07-06T06:09:09.129726Z","iopub.execute_input":"2022-07-06T06:09:09.130091Z","iopub.status.idle":"2022-07-06T06:09:09.136940Z","shell.execute_reply.started":"2022-07-06T06:09:09.130060Z","shell.execute_reply":"2022-07-06T06:09:09.135664Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# CV","metadata":{"id":"iYg4hycC25zR"}},{"cell_type":"code","source":"effectiveness_map = {'Ineffective':0, 'Adequate':1, 'Effective':2}\ntrain['target'] = train['discourse_effectiveness'].map(effectiveness_map)","metadata":{"id":"zek0OSpO7wlC","executionInfo":{"status":"ok","timestamp":1657076384118,"user_tz":-540,"elapsed":9,"user":{"displayName":"鈴木明作","userId":"03916733245168369171"}},"execution":{"iopub.status.busy":"2022-07-06T06:09:09.139246Z","iopub.execute_input":"2022-07-06T06:09:09.139665Z","iopub.status.idle":"2022-07-06T06:09:09.160632Z","shell.execute_reply.started":"2022-07-06T06:09:09.139630Z","shell.execute_reply":"2022-07-06T06:09:09.158794Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sgkf = StratifiedGroupKFold(n_splits=CFG.n_folds,shuffle=True,random_state=CFG.seed)\n\nfor fold, (_,val_idx) in enumerate(sgkf.split(X=train, y=train['target'], groups=train.essay_id)):\n    train.loc[val_idx,'kfold'] = fold\n\ntrain.head()","metadata":{"id":"tHyAiC6u2YQ8","executionInfo":{"status":"ok","timestamp":1657076385964,"user_tz":-540,"elapsed":1854,"user":{"displayName":"鈴木明作","userId":"03916733245168369171"}},"outputId":"cc9433ba-3db2-4899-ada3-ea4ae5f29e7a","execution":{"iopub.status.busy":"2022-07-06T06:09:09.165175Z","iopub.execute_input":"2022-07-06T06:09:09.166042Z","iopub.status.idle":"2022-07-06T06:09:10.849510Z","shell.execute_reply.started":"2022-07-06T06:09:09.165994Z","shell.execute_reply":"2022-07-06T06:09:10.848255Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.groupby('kfold')['discourse_effectiveness'].value_counts()","metadata":{"id":"bwb9YeQe2YVc","executionInfo":{"status":"ok","timestamp":1657076385965,"user_tz":-540,"elapsed":8,"user":{"displayName":"鈴木明作","userId":"03916733245168369171"}},"outputId":"a65ae166-d6ac-4d2a-a2d1-59b42b9005c3","execution":{"iopub.status.busy":"2022-07-06T06:09:10.851105Z","iopub.execute_input":"2022-07-06T06:09:10.851509Z","iopub.status.idle":"2022-07-06T06:09:10.874042Z","shell.execute_reply.started":"2022-07-06T06:09:10.851472Z","shell.execute_reply":"2022-07-06T06:09:10.873007Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Embedding by Word2vec\n\n> Word2vec represents words in 300 dimensions. By averaging the 300-dimensional vectors of the words in the sentence, the sentence was represented in 300 dimensions.\n\nhttps://www.kaggle.com/code/syurenuko/clrp-word2vec-lightgbm-baseline/notebook","metadata":{"id":"rg1OwDbk1cJv"}},{"cell_type":"code","source":"word2vec_model = gensim.models.KeyedVectors.load_word2vec_format('../input/googlenewsvectorsnegative300/GoogleNews-vectors-negative300.bin', binary=True)\nprint(word2vec_model.vectors.shape)","metadata":{"id":"oVbDMfUz1X7g","executionInfo":{"status":"ok","timestamp":1657076461963,"user_tz":-540,"elapsed":76002,"user":{"displayName":"鈴木明作","userId":"03916733245168369171"}},"outputId":"3923b5d2-80e0-4986-a17d-2faea1e53cf7","execution":{"iopub.status.busy":"2022-07-06T06:09:10.875504Z","iopub.execute_input":"2022-07-06T06:09:10.876081Z","iopub.status.idle":"2022-07-06T06:10:18.766556Z","shell.execute_reply.started":"2022-07-06T06:09:10.876022Z","shell.execute_reply":"2022-07-06T06:10:18.765065Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"word2vec_model","metadata":{"id":"6e-DjcSa3GtS","executionInfo":{"status":"ok","timestamp":1657076461964,"user_tz":-540,"elapsed":26,"user":{"displayName":"鈴木明作","userId":"03916733245168369171"}},"outputId":"6a5aaf8a-a773-4543-d2e3-d96ffb320f1c","execution":{"iopub.status.busy":"2022-07-06T06:10:18.770001Z","iopub.execute_input":"2022-07-06T06:10:18.770433Z","iopub.status.idle":"2022-07-06T06:10:18.778166Z","shell.execute_reply.started":"2022-07-06T06:10:18.770394Z","shell.execute_reply":"2022-07-06T06:10:18.776884Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def avg_feature_vector(sentence, model, num_features):\n    words = sentence.replace('\\n',\" \").replace(',',' ').replace('.',\" \").split()\n    feature_vec = np.zeros((num_features,),dtype=\"float32\")\n    i=0\n    for word in words:\n        try:\n            feature_vec = np.add(feature_vec, model[word])\n        except KeyError as error:\n            feature_vec \n            i = i + 1\n    if len(words) > 0:\n        feature_vec = np.divide(feature_vec, len(words)- i)\n    return feature_vec","metadata":{"id":"Voi3ExUE1X-W","executionInfo":{"status":"ok","timestamp":1657076461965,"user_tz":-540,"elapsed":24,"user":{"displayName":"鈴木明作","userId":"03916733245168369171"}},"execution":{"iopub.status.busy":"2022-07-06T06:10:18.780208Z","iopub.execute_input":"2022-07-06T06:10:18.781076Z","iopub.status.idle":"2022-07-06T06:10:18.795557Z","shell.execute_reply.started":"2022-07-06T06:10:18.781025Z","shell.execute_reply":"2022-07-06T06:10:18.793816Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# LightGBM\n","metadata":{"id":"RPZ21aI1Bir_"}},{"cell_type":"code","source":"#https://www.kaggle.com/code/dlaststark/fpe-no-fancy-stuff\n\nparams = {}\nparams[\"objective\"] = 'multiclass'\nparams['metric'] = 'multi_logloss'\nparams['boosting'] = 'gbdt'\nparams['num_class'] = 3\nparams['is_unbalance'] = True\nparams[\"learning_rate\"] = 0.05\nparams[\"lambda_l2\"] = 0.0256\nparams[\"num_leaves\"] = 52\nparams[\"max_depth\"] = 10\nparams[\"feature_fraction\"] = 0.503\nparams[\"bagging_fraction\"] = 0.741\nparams[\"bagging_freq\"] = 8\nparams[\"bagging_seed\"] = 10\nparams[\"min_data_in_leaf\"] = 10\nparams[\"verbosity\"] = -1\nparams[\"random_state\"] = 42\nnum_rounds = 5000","metadata":{"id":"7B3wAkwV8Mv0","executionInfo":{"status":"ok","timestamp":1657076469184,"user_tz":-540,"elapsed":8,"user":{"displayName":"鈴木明作","userId":"03916733245168369171"}},"execution":{"iopub.status.busy":"2022-07-06T06:10:18.797537Z","iopub.execute_input":"2022-07-06T06:10:18.798044Z","iopub.status.idle":"2022-07-06T06:10:18.809551Z","shell.execute_reply.started":"2022-07-06T06:10:18.798003Z","shell.execute_reply":"2022-07-06T06:10:18.808350Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#https://www.kaggle.com/code/kvsnoufal/logisticregression-tfidf-5-folds\n#https://www.kaggle.com/code/dlaststark/fpe-no-fancy-stuff\n\noof_score = 0\ny_test_pred = np.zeros((test.shape[0], 3))\n\nfor fold in range(CFG.n_folds):\n    print(f'=============fold:{fold}==================')\n    train_fold=train[train['kfold']!=fold].reset_index(drop=True)\n    valid_fold=train[train['kfold']==fold].reset_index(drop=True)\n\n    #word2vec\n\n    #discourse_text\n    word2vec_train_disc_text = np.zeros((len(train_fold.index),300),dtype=\"float32\")\n    word2vec_valid_disc_text = np.zeros((len(valid_fold.index),300),dtype=\"float32\")\n    word2vec_test_disc_text = np.zeros((len(test.index),300),dtype=\"float32\")\n    for i in range(len(train_fold.index)):\n        word2vec_train_disc_text[i] = avg_feature_vector(train_fold[\"discourse_text\"][i], word2vec_model, 300)\n    for i in range(len(valid_fold.index)):\n        word2vec_valid_disc_text[i] = avg_feature_vector(valid_fold[\"discourse_text\"][i], word2vec_model, 300)\n    for i in range(len(test.index)):\n        word2vec_test_disc_text[i] = avg_feature_vector(test[\"discourse_text\"][i], word2vec_model, 300)\n\n    #essay_text\n    word2vec_train_essay_text = np.zeros((len(train_fold.index),300),dtype=\"float32\")\n    word2vec_valid_essay_text = np.zeros((len(valid_fold.index),300),dtype=\"float32\")\n    word2vec_test_essay_text = np.zeros((len(test.index),300),dtype=\"float32\")\n    for i in range(len(train_fold.index)):\n        word2vec_train_essay_text[i] = avg_feature_vector(train_fold[\"essay_text\"][i], word2vec_model, 300)\n    for i in range(len(valid_fold.index)):\n        word2vec_valid_essay_text[i] = avg_feature_vector(valid_fold[\"essay_text\"][i], word2vec_model, 300)\n    for i in range(len(test.index)):\n        word2vec_test_essay_text[i] = avg_feature_vector(test[\"essay_text\"][i], word2vec_model, 300)\n\n    #OneHot\n    ohe = OneHotEncoder()\n    train_type_ohe=sparse.csr_matrix(ohe.fit_transform(train_fold['discourse_type'].values.reshape(-1,1)))\n    valid_type_ohe=sparse.csr_matrix(ohe.transform(valid_fold['discourse_type'].values.reshape(-1,1)))\n    test_type_ohe=sparse.csr_matrix(ohe.transform(test['discourse_type'].values.reshape(-1,1)))\n\n\n    #merge\n    Xtrain_word2vec = sparse.hstack((train_type_ohe,word2vec_train_disc_text,word2vec_train_essay_text))\n    Xvalid_word2vec = sparse.hstack((valid_type_ohe,word2vec_valid_disc_text,word2vec_valid_essay_text))\n    test_word2vec = sparse.hstack((test_type_ohe,word2vec_test_disc_text,word2vec_test_essay_text))\n\n    #lgbm\n    lgtrain = lgb.Dataset(Xtrain_word2vec, label=train_fold['target'].ravel())\n    lgvalidation = lgb.Dataset(Xvalid_word2vec, label=valid_fold['target'].ravel())\n\n    model = lgb.train(params, lgtrain, num_rounds, \n                    valid_sets=[lgtrain, lgvalidation], \n                    early_stopping_rounds=100, verbose_eval=100)\n\n    y_pred = model.predict(Xvalid_word2vec, num_iteration=model.best_iteration)\n    y_test_pred += model.predict(test_word2vec, num_iteration=model.best_iteration)\n\n    score = log_loss(valid_fold['target'], y_pred)\n    oof_score += score\n\n    print(f'Fold:{fold},valid score:{score}')\n    \ny_test_pred = y_test_pred / float(CFG.n_folds)\noof_score /= float(CFG.n_folds)\nprint(\"Aggregate OOF Score: {}\".format(oof_score))","metadata":{"id":"xSXW8V5a8MyY","outputId":"822442bf-79ab-4092-9f26-e9d90db2d9c5","executionInfo":{"status":"ok","timestamp":1657078049795,"user_tz":-540,"elapsed":1250464,"user":{"displayName":"鈴木明作","userId":"03916733245168369171"}},"execution":{"iopub.status.busy":"2022-07-06T06:10:18.811641Z","iopub.execute_input":"2022-07-06T06:10:18.812004Z","iopub.status.idle":"2022-07-06T06:20:09.375230Z","shell.execute_reply.started":"2022-07-06T06:10:18.811972Z","shell.execute_reply":"2022-07-06T06:20:09.374160Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# submit","metadata":{"id":"lubAwfFPCW2R"}},{"cell_type":"code","source":"submission = pd.read_csv(INPUT_DIR+\"sample_submission.csv\")\nsubmission['Ineffective'] = y_test_pred[:,0]\nsubmission['Adequate'] = y_test_pred[:,1]\nsubmission['Effective'] = y_test_pred[:,2]\nsubmission.to_csv(\"./submission.csv\", index=False)\nsubmission.head()","metadata":{"id":"G5QZ_LiwBoax","executionInfo":{"status":"ok","timestamp":1657078050287,"user_tz":-540,"elapsed":499,"user":{"displayName":"鈴木明作","userId":"03916733245168369171"}},"outputId":"6eec8773-9a0b-4ac9-eeff-9fc2da1b833e","execution":{"iopub.status.busy":"2022-07-06T06:20:09.376981Z","iopub.execute_input":"2022-07-06T06:20:09.377676Z","iopub.status.idle":"2022-07-06T06:20:09.408384Z","shell.execute_reply.started":"2022-07-06T06:20:09.377635Z","shell.execute_reply":"2022-07-06T06:20:09.407246Z"},"trusted":true},"execution_count":null,"outputs":[]}]}