{"cells":[{"metadata":{"_uuid":"533567e9291ec80f2d252df0544bc3a6ff597b77"},"cell_type":"markdown","source":"Links to orginal kernel https://www.kaggle.com/tunguz/annoying-ab-shreck-and-bluetooth\nhttps://www.kaggle.com/domcastro/let-s-annoy-abhishek\nhttps://www.kaggle.com/abhishek/maybe-something-interesting-here\n\nEAS - All I have done is change the parameters for the LGB. I am also abandoning the OptimizedRounder and have hand-tuned the coefficients to match the distribution of the test predictions to the distribution of the train targets. McCartney Taylor provided the tweaks to the learning rate and lambda.\n\n** Change Log **\n\n * v19 - added regularization lambda (0.2)\n * v21 - lowered lambda (0.05)\n * v22 - lowered lambda again (0.02)\n * v23 - lambda up to (0.075)\n * v24 - lambda back to 0.05, commenting out tfidf code that causes error\n * v25 - uncommenting commented out code\n * v26 - fixed bug that was throwing error for tf-idf code\n * v27 - examining code that had been throwing error, fixing the error lowers the score\n * v28 - passing list of categorical features\n * v29 - tuning params\n * v30 - using more tfidf features\n * v31, 32 - playing with tfidf params\n * v35 - back to 120 tfidf svd features\n * v36 - reverting to baseline and not using OptimizedRounder, hand-tuning coefs instead\n * v38 - Changing lambda and learning rate."},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import json\n\nimport scipy as sp\nimport pandas as pd\nimport numpy as np\n\nfrom functools import partial\nfrom math import sqrt\n\nfrom sklearn.metrics import cohen_kappa_score, mean_squared_error\nfrom sklearn.metrics import confusion_matrix as sk_cmatrix\nfrom sklearn.model_selection import StratifiedKFold\n\nfrom sklearn.feature_extraction.text import TfidfVectorizer\nfrom sklearn.decomposition import TruncatedSVD\n\nfrom collections import Counter\n\nimport lightgbm as lgb\nnp.random.seed(369)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"# The following 3 functions have been taken from Ben Hamner's github repository\n# https://github.com/benhamner/Metrics\ndef confusion_matrix(rater_a, rater_b, min_rating=None, max_rating=None):\n    \"\"\"\n    Returns the confusion matrix between rater's ratings\n    \"\"\"\n    assert(len(rater_a) == len(rater_b))\n    if min_rating is None:\n        min_rating = min(rater_a + rater_b)\n    if max_rating is None:\n        max_rating = max(rater_a + rater_b)\n    num_ratings = int(max_rating - min_rating + 1)\n    conf_mat = [[0 for i in range(num_ratings)]\n                for j in range(num_ratings)]\n    for a, b in zip(rater_a, rater_b):\n        conf_mat[a - min_rating][b - min_rating] += 1\n    return conf_mat\n\n\ndef histogram(ratings, min_rating=None, max_rating=None):\n    \"\"\"\n    Returns the counts of each type of rating that a rater made\n    \"\"\"\n    if min_rating is None:\n        min_rating = min(ratings)\n    if max_rating is None:\n        max_rating = max(ratings)\n    num_ratings = int(max_rating - min_rating + 1)\n    hist_ratings = [0 for x in range(num_ratings)]\n    for r in ratings:\n        hist_ratings[r - min_rating] += 1\n    return hist_ratings\n\n\ndef quadratic_weighted_kappa(y, y_pred):\n    \"\"\"\n    Calculates the quadratic weighted kappa\n    axquadratic_weighted_kappa calculates the quadratic weighted kappa\n    value, which is a measure of inter-rater agreement between two raters\n    that provide discrete numeric ratings.  Potential values range from -1\n    (representing complete disagreement) to 1 (representing complete\n    agreement).  A kappa value of 0 is expected if all agreement is due to\n    chance.\n    quadratic_weighted_kappa(rater_a, rater_b), where rater_a and rater_b\n    each correspond to a list of integer ratings.  These lists must have the\n    same length.\n    The ratings should be integers, and it is assumed that they contain\n    the complete range of possible ratings.\n    quadratic_weighted_kappa(X, min_rating, max_rating), where min_rating\n    is the minimum possible rating, and max_rating is the maximum possible\n    rating\n    \"\"\"\n    rater_a = y\n    rater_b = y_pred\n    min_rating=None\n    max_rating=None\n    rater_a = np.array(rater_a, dtype=int)\n    rater_b = np.array(rater_b, dtype=int)\n    assert(len(rater_a) == len(rater_b))\n    if min_rating is None:\n        min_rating = min(min(rater_a), min(rater_b))\n    if max_rating is None:\n        max_rating = max(max(rater_a), max(rater_b))\n    conf_mat = confusion_matrix(rater_a, rater_b,\n                                min_rating, max_rating)\n    num_ratings = len(conf_mat)\n    num_scored_items = float(len(rater_a))\n\n    hist_rater_a = histogram(rater_a, min_rating, max_rating)\n    hist_rater_b = histogram(rater_b, min_rating, max_rating)\n\n    numerator = 0.0\n    denominator = 0.0\n\n    for i in range(num_ratings):\n        for j in range(num_ratings):\n            expected_count = (hist_rater_a[i] * hist_rater_b[j]\n                              / num_scored_items)\n            d = pow(i - j, 2.0) / pow(num_ratings - 1, 2.0)\n            numerator += d * conf_mat[i][j] / num_scored_items\n            denominator += d * expected_count / num_scored_items\n\n    return (1.0 - numerator / denominator)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ef5d56c772b4442e32b2637e94141cb09ac70751"},"cell_type":"code","source":"class OptimizedRounder(object):\n    def __init__(self):\n        self.coef_ = 0\n\n    def _kappa_loss(self, coef, X, y):\n        X_p = np.copy(X)\n        for i, pred in enumerate(X_p):\n            if pred < coef[0]:\n                X_p[i] = 0\n            elif pred >= coef[0] and pred < coef[1]:\n                X_p[i] = 1\n            elif pred >= coef[1] and pred < coef[2]:\n                X_p[i] = 2\n            elif pred >= coef[2] and pred < coef[3]:\n                X_p[i] = 3\n            else:\n                X_p[i] = 4\n\n        ll = quadratic_weighted_kappa(y, X_p)\n        return -ll\n\n    def fit(self, X, y):\n        loss_partial = partial(self._kappa_loss, X=X, y=y)\n        initial_coef = [0.5, 1.5, 2.5, 3.5]\n        self.coef_ = sp.optimize.minimize(loss_partial, initial_coef, method='nelder-mead')\n\n    def predict(self, X, coef):\n        X_p = np.copy(X)\n        for i, pred in enumerate(X_p):\n            if pred < coef[0]:\n                X_p[i] = 0\n            elif pred >= coef[0] and pred < coef[1]:\n                X_p[i] = 1\n            elif pred >= coef[1] and pred < coef[2]:\n                X_p[i] = 2\n            elif pred >= coef[2] and pred < coef[3]:\n                X_p[i] = 3\n            else:\n                X_p[i] = 4\n        return X_p\n\n    def coefficients(self):\n        return self.coef_['x']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bc98a419d5875d48df84fa0fbb89ccef0eda6fe2"},"cell_type":"code","source":"def rmse(actual, predicted):\n    return sqrt(mean_squared_error(actual, predicted))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cc49ce567fd41bf67451f6e69eeb6999439cc4ce"},"cell_type":"code","source":"print('Train')\ntrain = pd.read_csv(\"../input/train/train.csv\")\nprint(train.shape)\n\nprint('Test')\ntest = pd.read_csv(\"../input/test/test.csv\")\nprint(test.shape)\n\nprint('Breeds')\nbreeds = pd.read_csv(\"../input/breed_labels.csv\")\nprint(breeds.shape)\n\nprint('Colors')\ncolors = pd.read_csv(\"../input/color_labels.csv\")\nprint(colors.shape)\n\nprint('States')\nstates = pd.read_csv(\"../input/state_labels.csv\")\nprint(states.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"06f3e354607a8172db78728918b757f300ca6e48"},"cell_type":"code","source":"target = train['AdoptionSpeed']\ntrain_id = train['PetID']\ntest_id = test['PetID']\ntrain.drop(['AdoptionSpeed', 'PetID'], axis=1, inplace=True)\ntest.drop(['PetID'], axis=1, inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"08afacd9661548e3c28d856f2833a073a023d19d"},"cell_type":"code","source":"doc_sent_mag = []\ndoc_sent_score = []\nnf_count = 0\nfor pet in train_id:\n    try:\n        with open('../input/train_sentiment/' + pet + '.json', 'r') as f:\n            sentiment = json.load(f)\n        doc_sent_mag.append(sentiment['documentSentiment']['magnitude'])\n        doc_sent_score.append(sentiment['documentSentiment']['score'])\n    except FileNotFoundError:\n        nf_count += 1\n        doc_sent_mag.append(-1)\n        doc_sent_score.append(-1)\n\ntrain.loc[:, 'doc_sent_mag'] = doc_sent_mag\ntrain.loc[:, 'doc_sent_score'] = doc_sent_score\n\ndoc_sent_mag = []\ndoc_sent_score = []\nnf_count = 0\nfor pet in test_id:\n    try:\n        with open('../input/test_sentiment/' + pet + '.json', 'r') as f:\n            sentiment = json.load(f)\n        doc_sent_mag.append(sentiment['documentSentiment']['magnitude'])\n        doc_sent_score.append(sentiment['documentSentiment']['score'])\n    except FileNotFoundError:\n        nf_count += 1\n        doc_sent_mag.append(-1)\n        doc_sent_score.append(-1)\n\ntest.loc[:, 'doc_sent_mag'] = doc_sent_mag\ntest.loc[:, 'doc_sent_score'] = doc_sent_score\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4cf9bb3d17a17bebab97b2f5fb3eae549eb7d3f5"},"cell_type":"code","source":"train_desc = train.Description.fillna(\"none\").values\ntest_desc = test.Description.fillna(\"none\").values\n\ntfv = TfidfVectorizer(min_df=2,  max_features=None,\n        strip_accents='unicode', analyzer='word', token_pattern=r'(?u)\\b\\w+\\b',\n        ngram_range=(1, 3), use_idf=1, smooth_idf=1, sublinear_tf=1,\n        )\n    \n# Fit TFIDF\ntfv.fit(list(train_desc))\nX =  tfv.transform(train_desc)\nX_test = tfv.transform(test_desc)\nprint(\"X (tfidf):\", X.shape)\n\nsvd = TruncatedSVD(n_components=200)\nsvd.fit(X)\n# print(svd.explained_variance_ratio_.sum())\n# print(svd.explained_variance_ratio_)\nX = svd.transform(X)\nprint(\"X (svd):\", X.shape)\n\n# X = pd.DataFrame(X, columns=['svd_{}'.format(i) for i in range(120)])\n# train = pd.concat((train, X), axis=1)\n# X_test = svd.transform(X_test)\n# X_test = pd.DataFrame(X_test, columns=['svd_{}'.format(i) for i in range(120)])\n# test = pd.concat((test, X_test), axis=1)\n\nprint(\"train:\", train.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b26a95057358f0ec18ee4266765a1634010d2777"},"cell_type":"code","source":"## WITHOUT ERROR FIXED\ntrain_desc = train.Description.fillna(\"none\").values\ntest_desc = test.Description.fillna(\"none\").values\n\ntfv = TfidfVectorizer(min_df=3,  max_features=10000,\n        strip_accents='unicode', analyzer='word', token_pattern=r'\\w{1,}',\n        ngram_range=(1, 3), use_idf=1, smooth_idf=1, sublinear_tf=1,\n        stop_words = 'english')\n    \n# Fit TFIDF\ntfv.fit(list(train_desc))\nX =  tfv.transform(train_desc)\nX_test = tfv.transform(test_desc)\nprint(\"X (tfidf):\", X.shape)\n\nsvd = TruncatedSVD(n_components=120)\nsvd.fit(X)\n# print(svd.explained_variance_ratio_.sum())\n# print(svd.explained_variance_ratio_)\nX = svd.transform(X)\nprint(\"X (svd):\", X.shape)\n\nX = pd.DataFrame(X, columns=['svd_{}'.format(i) for i in range(120)])\ntrain = pd.concat((train, X), axis=1)\nX_test = svd.transform(X_test)\nX_test = pd.DataFrame(X_test, columns=['svd_{}'.format(i) for i in range(120)])\ntest = pd.concat((test, X_test), axis=1)\n\nprint(\"train:\", train.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"81d0d3799c39b17fea45f5e94bab31e08c4eac7a"},"cell_type":"code","source":"vertex_xs = []\nvertex_ys = []\nbounding_confidences = []\nbounding_importance_fracs = []\ndominant_blues = []\ndominant_greens = []\ndominant_reds = []\ndominant_pixel_fracs = []\ndominant_scores = []\nlabel_descriptions = []\nlabel_scores = []\nnf_count = 0\nnl_count = 0\nfor pet in train_id:\n    try:\n        with open('../input/train_metadata/' + pet + '-1.json', 'r') as f:\n            data = json.load(f)\n        vertex_x = data['cropHintsAnnotation']['cropHints'][0]['boundingPoly']['vertices'][2]['x']\n        vertex_xs.append(vertex_x)\n        vertex_y = data['cropHintsAnnotation']['cropHints'][0]['boundingPoly']['vertices'][2]['y']\n        vertex_ys.append(vertex_y)\n        bounding_confidence = data['cropHintsAnnotation']['cropHints'][0]['confidence']\n        bounding_confidences.append(bounding_confidence)\n        bounding_importance_frac = data['cropHintsAnnotation']['cropHints'][0].get('importanceFraction', -1)\n        bounding_importance_fracs.append(bounding_importance_frac)\n        dominant_blue = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['color']['blue']\n        dominant_blues.append(dominant_blue)\n        dominant_green = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['color']['green']\n        dominant_greens.append(dominant_green)\n        dominant_red = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['color']['red']\n        dominant_reds.append(dominant_red)\n        dominant_pixel_frac = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['pixelFraction']\n        dominant_pixel_fracs.append(dominant_pixel_frac)\n        dominant_score = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['score']\n        dominant_scores.append(dominant_score)\n        if data.get('labelAnnotations'):\n            label_description = data['labelAnnotations'][0]['description']\n            label_descriptions.append(label_description)\n            label_score = data['labelAnnotations'][0]['score']\n            label_scores.append(label_score)\n        else:\n            nl_count += 1\n            label_descriptions.append('nothing')\n            label_scores.append(-1)\n    except FileNotFoundError:\n        nf_count += 1\n        vertex_xs.append(-1)\n        vertex_ys.append(-1)\n        bounding_confidences.append(-1)\n        bounding_importance_fracs.append(-1)\n        dominant_blues.append(-1)\n        dominant_greens.append(-1)\n        dominant_reds.append(-1)\n        dominant_pixel_fracs.append(-1)\n        dominant_scores.append(-1)\n        label_descriptions.append('nothing')\n        label_scores.append(-1)\n\nprint(nf_count)\nprint(nl_count)\ntrain.loc[:, 'vertex_x'] = vertex_xs\ntrain.loc[:, 'vertex_y'] = vertex_ys\ntrain.loc[:, 'bounding_confidence'] = bounding_confidences\ntrain.loc[:, 'bounding_importance'] = bounding_importance_fracs\ntrain.loc[:, 'dominant_blue'] = dominant_blues\ntrain.loc[:, 'dominant_green'] = dominant_greens\ntrain.loc[:, 'dominant_red'] = dominant_reds\ntrain.loc[:, 'dominant_pixel_frac'] = dominant_pixel_fracs\ntrain.loc[:, 'dominant_score'] = dominant_scores\ntrain.loc[:, 'label_description'] = label_descriptions\ntrain.loc[:, 'label_score'] = label_scores\n\n\nvertex_xs = []\nvertex_ys = []\nbounding_confidences = []\nbounding_importance_fracs = []\ndominant_blues = []\ndominant_greens = []\ndominant_reds = []\ndominant_pixel_fracs = []\ndominant_scores = []\nlabel_descriptions = []\nlabel_scores = []\nnf_count = 0\nnl_count = 0\nfor pet in test_id:\n    try:\n        with open('../input/test_metadata/' + pet + '-1.json', 'r') as f:\n            data = json.load(f)\n        vertex_x = data['cropHintsAnnotation']['cropHints'][0]['boundingPoly']['vertices'][2]['x']\n        vertex_xs.append(vertex_x)\n        vertex_y = data['cropHintsAnnotation']['cropHints'][0]['boundingPoly']['vertices'][2]['y']\n        vertex_ys.append(vertex_y)\n        bounding_confidence = data['cropHintsAnnotation']['cropHints'][0]['confidence']\n        bounding_confidences.append(bounding_confidence)\n        bounding_importance_frac = data['cropHintsAnnotation']['cropHints'][0].get('importanceFraction', -1)\n        bounding_importance_fracs.append(bounding_importance_frac)\n        dominant_blue = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['color']['blue']\n        dominant_blues.append(dominant_blue)\n        dominant_green = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['color']['green']\n        dominant_greens.append(dominant_green)\n        dominant_red = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['color']['red']\n        dominant_reds.append(dominant_red)\n        dominant_pixel_frac = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['pixelFraction']\n        dominant_pixel_fracs.append(dominant_pixel_frac)\n        dominant_score = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['score']\n        dominant_scores.append(dominant_score)\n        if data.get('labelAnnotations'):\n            label_description = data['labelAnnotations'][0]['description']\n            label_descriptions.append(label_description)\n            label_score = data['labelAnnotations'][0]['score']\n            label_scores.append(label_score)\n        else:\n            nl_count += 1\n            label_descriptions.append('nothing')\n            label_scores.append(-1)\n    except FileNotFoundError:\n        nf_count += 1\n        vertex_xs.append(-1)\n        vertex_ys.append(-1)\n        bounding_confidences.append(-1)\n        bounding_importance_fracs.append(-1)\n        dominant_blues.append(-1)\n        dominant_greens.append(-1)\n        dominant_reds.append(-1)\n        dominant_pixel_fracs.append(-1)\n        dominant_scores.append(-1)\n        label_descriptions.append('nothing')\n        label_scores.append(-1)\n\nprint(nf_count)\ntest.loc[:, 'vertex_x'] = vertex_xs\ntest.loc[:, 'vertex_y'] = vertex_ys\ntest.loc[:, 'bounding_confidence'] = bounding_confidences\ntest.loc[:, 'bounding_importance'] = bounding_importance_fracs\ntest.loc[:, 'dominant_blue'] = dominant_blues\ntest.loc[:, 'dominant_green'] = dominant_greens\ntest.loc[:, 'dominant_red'] = dominant_reds\ntest.loc[:, 'dominant_pixel_frac'] = dominant_pixel_fracs\ntest.loc[:, 'dominant_score'] = dominant_scores\ntest.loc[:, 'label_description'] = label_descriptions\ntest.loc[:, 'label_score'] = label_scores","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f508f56d4cab23c936752a1033028117e5a4b08c"},"cell_type":"code","source":"train.drop(['Name', 'RescuerID', 'Description'], axis=1, inplace=True)\ntest.drop(['Name', 'RescuerID', 'Description'], axis=1, inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"253e65605d45da4db28f84363edeb98fb54431fb"},"cell_type":"code","source":"numeric_cols = ['Age', 'Quantity', 'Fee', 'VideoAmt', 'PhotoAmt', 'AdoptionSpeed', 'doc_sent_mag', 'doc_sent_score', 'dominant_score', 'dominant_pixel_frac', 'dominant_red', 'dominant_green', 'dominant_blue', 'bounding_importance', 'bounding_confidence', 'vertex_x', 'vertex_y', 'label_score'] + ['svd_{}'.format(i) for i in range(120)]\ncat_cols = list(set(train.columns) - set(numeric_cols))\ntrain.loc[:, cat_cols] = train[cat_cols].astype('category')\ntest.loc[:, cat_cols] = test[cat_cols].astype('category')\nprint(train.shape)\nprint(test.shape)\ntrain.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"93bc91b0da1583142b7d54f4a1351e6021ca3c9c"},"cell_type":"code","source":"# get the categorical features\nfoo = train.dtypes\ncat_feature_names = foo[foo == \"category\"]\ncat_features = [train.columns.get_loc(c) for c in train.columns if c in cat_feature_names]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"759093088e557f75b31476a6d4ad45eee84fcc8f"},"cell_type":"code","source":"def run_cv_model(train, test, target, model_fn, params={}, eval_fn=None, label='model'):\n    kf = StratifiedKFold(n_splits=5, random_state=42, shuffle=True)\n    fold_splits = kf.split(train, target)\n    cv_scores = []\n    qwk_scores = []\n    pred_full_test = 0\n    pred_train = np.zeros((train.shape[0], 5))\n    all_coefficients = np.zeros((5, 4))\n    feature_importance_df = pd.DataFrame()\n    i = 1\n    for dev_index, val_index in fold_splits:\n        print('Started ' + label + ' fold ' + str(i) + '/5')\n        if isinstance(train, pd.DataFrame):\n            dev_X, val_X = train.iloc[dev_index], train.iloc[val_index]\n            dev_y, val_y = target[dev_index], target[val_index]\n        else:\n            dev_X, val_X = train[dev_index], train[val_index]\n            dev_y, val_y = target[dev_index], target[val_index]\n        params2 = params.copy()\n        pred_val_y, pred_test_y, importances, coefficients, qwk = model_fn(dev_X, dev_y, val_X, val_y, test, params2)\n        pred_full_test = pred_full_test + pred_test_y\n        pred_train[val_index] = pred_val_y\n        all_coefficients[i-1, :] = coefficients\n        if eval_fn is not None:\n            cv_score = eval_fn(val_y, pred_val_y)\n            cv_scores.append(cv_score)\n            qwk_scores.append(qwk)\n            print(label + ' cv score {}: RMSE {} QWK {}'.format(i, cv_score, qwk))\n        fold_importance_df = pd.DataFrame()\n        fold_importance_df['feature'] = train.columns.values\n        fold_importance_df['importance'] = importances\n        fold_importance_df['fold'] = i\n        feature_importance_df = pd.concat([feature_importance_df, fold_importance_df], axis=0)        \n        i += 1\n    print('{} cv RMSE scores : {}'.format(label, cv_scores))\n    print('{} cv mean RMSE score : {}'.format(label, np.mean(cv_scores)))\n    print('{} cv std RMSE score : {}'.format(label, np.mean(cv_scores)))\n    print('{} cv QWK scores : {}'.format(label, qwk_scores))\n    print('{} cv mean QWK score : {}'.format(label, np.mean(qwk_scores)))\n    print('{} cv std QWK score : {}'.format(label, np.std(qwk_scores)))\n    pred_full_test = pred_full_test / 5.0\n    results = {'label': label,\n               'train': pred_train, 'test': pred_full_test,\n                'cv': cv_scores, 'qwk': qwk_scores,\n               'importance': feature_importance_df,\n               'coefficients': all_coefficients}\n    return results\n\nparams = {'application': 'regression',\n          'boosting': 'gbdt',\n          'metric': 'rmse',\n          'num_leaves': 70,\n          'max_depth': 8,\n          'learning_rate': 0.01,\n          'bagging_fraction': 0.85,\n          'feature_fraction': 0.8,\n          'min_split_gain': 0.02,\n          'min_child_samples': 150,\n          'min_child_weight': 0.02,\n          'lambda_l2': 0.0475,\n          'verbosity': -1,\n          'data_random_seed': 17,\n          'early_stop': 100,\n          'verbose_eval': 100,\n          'num_rounds': 10000}\n\ndef runLGB(train_X, train_y, test_X, test_y, test_X2, params):\n    print('Prep LGB')\n    d_train = lgb.Dataset(train_X, label=train_y)\n    d_valid = lgb.Dataset(test_X, label=test_y)\n    watchlist = [d_train, d_valid]\n    print('Train LGB')\n    num_rounds = params.pop('num_rounds')\n    verbose_eval = params.pop('verbose_eval')\n    early_stop = None\n    if params.get('early_stop'):\n        early_stop = params.pop('early_stop')\n    model = lgb.train(params,\n                      train_set=d_train,\n                      num_boost_round=num_rounds,\n                      valid_sets=watchlist,\n                      verbose_eval=verbose_eval,\n                      categorical_feature=list(cat_features),\n                      early_stopping_rounds=early_stop)\n    \n    print('Predict 1/2')\n    pred_test_y = model.predict(test_X, num_iteration=model.best_iteration)\n    optR = OptimizedRounder()\n    optR.fit(pred_test_y, test_y)\n    coefficients = optR.coefficients()\n    pred_test_y_k = optR.predict(pred_test_y, coefficients)\n    print(\"Valid Counts = \", Counter(test_y))\n    print(\"Predicted Counts = \", Counter(pred_test_y_k))\n    print(\"Coefficients = \", coefficients)\n    qwk = quadratic_weighted_kappa(test_y, pred_test_y_k)\n    print(\"QWK = \", qwk)\n    print('Predict 2/2')\n    pred_test_y2 = model.predict(test_X2, num_iteration=model.best_iteration)\n    return pred_test_y.reshape(-1, 1), pred_test_y2.reshape(-1, 1), model.feature_importance(), coefficients, qwk\n\nresults = run_cv_model(train, test, target, runLGB, params, rmse, 'lgb')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"167b566191b143a63192937ac7ce74084393c0f0"},"cell_type":"code","source":"imports = results['importance'].groupby('feature')['feature', 'importance'].mean().reset_index()\nimports.sort_values('importance', ascending=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"70395ed3da4534cfb936c2888cac4c72ac2f52bf"},"cell_type":"code","source":"optR = OptimizedRounder()\ncoefficients_ = np.mean(results['coefficients'], axis=0)\nprint(coefficients_)\n# manually adjust coefs\ncoefficients_[0] = 1.64\ncoefficients_[1] = 2.11\ncoefficients_[3] = 2.85\ntrain_predictions = [r[0] for r in results['train']]\ntrain_predictions = optR.predict(train_predictions, coefficients_).astype(int)\nCounter(train_predictions)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c29fcea7bb80ebbcd1e4bc42e34d45909dc772c7"},"cell_type":"code","source":"optR = OptimizedRounder()\ncoefficients_ = np.mean(results['coefficients'], axis=0)\nprint(coefficients_)\n# manually adjust coefs\ncoefficients_[0] = 1.645\ncoefficients_[1] = 2.115\ncoefficients_[3] = 2.84\ntest_predictions = [r[0] for r in results['test']]\ntest_predictions = optR.predict(test_predictions, coefficients_).astype(int)\nCounter(test_predictions)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4dbac595f87c3e5dcb36a1c32402b5436d67335f"},"cell_type":"code","source":"print(\"True Distribution:\")\nprint(pd.value_counts(target, normalize=True).sort_index())\nprint(\"Test Predicted Distribution:\")\nprint(pd.value_counts(test_predictions, normalize=True).sort_index())\nprint(\"Train Predicted Distribution:\")\nprint(pd.value_counts(train_predictions, normalize=True).sort_index())\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c5f6278282b47aeae87af9da543edccac737c419"},"cell_type":"code","source":"pd.DataFrame(sk_cmatrix(target, train_predictions), index=list(range(5)), columns=list(range(5)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"36ee65ded47bdbde1d8033c30224e65abdc17f43"},"cell_type":"code","source":"quadratic_weighted_kappa(target, train_predictions)\nrmse(target, [r[0] for r in results['train']])\nsubmission = pd.DataFrame({'PetID': test_id, 'AdoptionSpeed': test_predictions})\nsubmission.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c2059154b328091cb23629cd28726e187f06f824"},"cell_type":"code","source":"submission.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}