{"cells":[{"metadata":{"_uuid":"7fe760d40092806c1826e188b0627a275cff6a5e"},"cell_type":"markdown","source":"## Change log\n|Version|Change|Score|Mean validation RMSE|Mean validation QWK|\n|-|-|-|-|-|\n|2|folding added||||\n|3|irrelevant parameter removed||||\n|4|regressor made verbose||||\n|5|submission index removed||||\n|6|predict method fixed|0.223|||\n|8|description length added as feature|0.228|||\n|9|language added as feature|0.229|||\n|10|average image w and h added as feature|0.241|||\n|11|one-hot encoding added|0.236|||\n|12|categorical features used as is, lightgbm only prediction|0.280|||\n|13|sentiment score and magnitude added|0.287|||\n|14|word 'adoption' used as 'no name' indicator|0.279|||\n|15|advanced rounding introduced. 'adoption' indicator removed||||\n|16|error in variable name fixed|0.304|||\n|17|RescuerID added as categorical feature|0.294|||\n|18|RescuerID removed. Metadata for 4 colors added. Sorted by score|0.269|||\n|19|Change log added. Metadata for 4 sorted by pixelFraction|0.307|1.0545||\n|20|Missing values set to -1|0.317|1.0546||\n|21|Text features added|0.246|1.0325||\n|22|Text features temporary disabled. Missing values set to np.NaN|0.252|1.0545|0.3156|\n|23|Parameters updated|0.301|1.0572|0.3189|\n|24|Color encoding removed|0.287|1.0586|0.3089|\n|26|Color encoding reverted. Labels annotations added|0.293|1.0561|0.3205|\n|27|Text features strike back||1.0373|0.3412|"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\nfrom os.path import join as jp\n\n# Any results you write to the current directory are saved as output.\ninput_dir = jp(os.pardir, 'input')","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"def read_csv(subfolder, filename):\n    _csv = pd.read_csv(jp(input_dir, subfolder, '{}.csv'.format(filename)))\n    return _csv\n\ntrain = read_csv('train', 'train')\ntrain.columns","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5478d20800b0fa2b5e1438f4a5d7ff20c5fc7f69"},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f40ef8bb1de53e181c9ddceb8eba732b2431f3f2"},"cell_type":"code","source":"import json\nfrom tqdm import tqdm\nMISSING_VALUE = np.NaN\n\ndef read_sentiments(subfolder, pet_ids):\n    sentiments = []\n    with tqdm(total=len(pet_ids), desc='Reading sentiment') as pbar:\n        for pet_id in pet_ids:\n            result = {'magnitude': MISSING_VALUE, 'score': MISSING_VALUE}\n            filepath = jp(input_dir, subfolder, '{}.json'.format(pet_id))\n            if os.path.isfile(filepath):\n                with open(filepath) as f:\n                    data = json.load(f)\n                    result['magnitude'] = data['documentSentiment']['magnitude']\n                    result['score'] = data['documentSentiment']['score']\n            sentiments.append(result)\n            pbar.update()\n    return sentiments","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6cdc04709e41c72c12450f361ee61292c5d69ca4"},"cell_type":"code","source":"def encode_color(r, g, b):\n    _max = 255 * 256 ** 2 + 255 * 256 + 255\n    value = r * 256 ** 2 + g * 256 + b\n    return value / _max\n\ndef read_colors_metadata(subfolder, pet_ids, num_colors=4, prioritize_by='score'):\n    \n    def get_color_value(color, channel):\n        c = color['color']\n        if channel in c.keys():\n            return c[channel]\n        else:\n            return 0.0\n    \n    metadata = []\n    with tqdm(total=len(pet_ids), desc='Reading colors metadata') as pbar:\n        for pet_id in pet_ids:\n            result = {}\n            filepath = jp(input_dir, subfolder, '{}-1.json'.format(pet_id))\n            if os.path.isfile(filepath):\n                with open(filepath) as f:\n                    data = json.load(f)\n                    colors = data['imagePropertiesAnnotation']['dominantColors']['colors']\n                    colors = sorted(colors, key=lambda x: x[prioritize_by], reverse=True)[:num_colors]\n                    for i, color in enumerate(colors):\n                        r = get_color_value(color, 'red')\n                        g = get_color_value(color, 'green')\n                        b = get_color_value(color, 'blue')\n                        result['color_{}'.format(str(i))] = encode_color(r, g, b)\n                        result['score_{}'.format(str(i))] = color['score']\n                        result['pixel_fraction_{}'.format(str(i))] = color['pixelFraction']\n            else:\n                for i in range(num_colors):\n                    result['color_{}'.format(str(i))] = MISSING_VALUE\n                    result['score_{}'.format(str(i))] = MISSING_VALUE\n                    result['pixel_fraction_{}'.format(str(i))] = MISSING_VALUE\n\n            metadata.append(result)\n            pbar.update()\n    return metadata","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2282b451b42f6b17acdcdddd28d1dc5b8164f863"},"cell_type":"code","source":"def read_image_metadata(subfolder, pet_ids):\n    metadata = []\n    with tqdm(total=len(pet_ids), desc='Reading images metadata') as pbar:\n        for pet_id in pet_ids:\n            result = {}\n            filepath = jp(input_dir, subfolder, '{}-1.json'.format(pet_id))\n            if os.path.isfile(filepath):\n                with open(filepath) as f:\n                    data = json.load(f)\n                    try:\n                        annotations = data['labelAnnotations']\n                        top_annotation = annotations[0]\n                        score = top_annotation['score']\n                        description = top_annotation['description']\n                        result['label_description'] = description\n                        result['label_score'] = score\n                    except:\n                        result['label_description'] = MISSING_VALUE\n                        result['label_score'] = MISSING_VALUE\n            else:\n                result['label_description'] = MISSING_VALUE\n                result['label_score'] = MISSING_VALUE\n\n            metadata.append(result)\n            pbar.update()\n    return metadata","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"afbce4a7fc4f21e25722a622917cb5cabde27da3"},"cell_type":"code","source":"import langid\n\ndef is_hex(value):\n    try:\n        int(value, 16)\n        return True\n    except:\n        return False\n\ndef is_empty(value):\n    return value == ''\n\ndef is_numeric(value):\n    return isinstance(value, int) or isinstance(value, float)\n    \ndef has_no_name(pbar):\n    def _has_no_name(value):\n        pbar.update()\n        if isinstance(value, float) and np.isnan(value):\n            return MISSING_VALUE\n        if is_numeric(value):\n            return True\n        is_no_name = 'no name' in value.lower()\n        is_none = 'none' in value.lower()\n        return is_no_name or is_none\n    return _has_no_name\n\ndef detect_language(pbar):\n    def _detect_language(description):\n        pbar.update()\n        if isinstance(description, float) and np.isnan(description):\n            return MISSING_VALUE\n        lang = langid.classify(description)[0]\n        if lang in ['fi', 'lb', 'pl', 'nb', 'eo', 'et', 'pt', 'lt',\n                    'no', 'de', 'tl', 'nl', 'da', 'ro', 'fr', 'it',\n                    'hr', 'la', 'sw', 'es', 'mg', 'mt', 'sl', 'eu',\n                    'sv', 'ca', 'cs', 'sk', 'xh', 'hu']:\n            lang = 'en'\n        if lang in ['af', 'ms', 'jv', 'ja', 'bs']:\n            lang = 'id'\n        return lang\n    return _detect_language\n\ndef calc_description_length(description):\n    if isinstance(description, float) and np.isnan(description):\n        return MISSING_VALUE\n    result = len(description)\n    if result == 0:\n        return MISSING_VALUE\n    return result","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a856463ddbdec7918d07c2cbafe615c5ee05faa9"},"cell_type":"code","source":"from PIL import Image\n\ndef get_image_sizes(df, subfolder):\n    sizes = []\n    with tqdm(total=len(df), desc='Image sizes calculation') as pbar:\n        for row in df.itertuples():\n            _id = row.PetID\n            _photos = int(row.PhotoAmt)\n            x = 0.0\n            y = 0.0\n            count = 0\n            for i in range(_photos):\n                image_name = '{}-{}.jpg'.format(_id, str(i + 1))\n                image_path = jp(input_dir, subfolder, image_name)\n                if os.path.isfile(image_path):\n                    size = Image.open(image_path).size\n                    x += size[0]\n                    y += size[1]\n                    count += 1\n            if count > 0:\n                x = x / _photos\n                y = y / _photos\n            else:\n                x = MISSING_VALUE\n                y = MISSING_VALUE\n            sizes.append({'PetID': _id, 'width': x, 'height': y})\n            pbar.update()\n        return sizes","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1425d21e624bcae6d0807efd31c6cf9c6b036443"},"cell_type":"code","source":"from sklearn.decomposition import TruncatedSVD\nfrom sklearn.feature_extraction.text import TfidfVectorizer\n\ndef generate_text_features(df):\n    desc = df.Description.values\n\n    tfv = TfidfVectorizer(\n        min_df=3,\n        max_features=10000,\n        strip_accents='unicode',\n        analyzer='word',\n        token_pattern=r'\\w{1,}',\n        ngram_range=(1, 3),\n        use_idf=True,\n        smooth_idf=True,\n        sublinear_tf=True,\n        stop_words='english')\n\n    # Fit TFIDF\n    tfv.fit(list(desc))\n    transformed = tfv.transform(desc)\n    print(\"tfidf:\", transformed.shape)\n\n    svd = TruncatedSVD(n_components=120)\n    svd.fit(transformed)\n    transformed = svd.transform(transformed)\n    print(\"svd:\", transformed.shape)\n    return transformed","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bcfbfa9299bc21c83cd7b7927b2db170fc58b5b6"},"cell_type":"code","source":"def preprocess(df, sizes, sentiments, colors_metadata, image_metadata):\n    preprocessed = df.copy()\n    preprocessed['DescLength'] = preprocessed.Description.apply(calc_description_length)\n    preprocessed['Description'].fillna('', inplace=True)\n    \n    with tqdm(total=len(df), desc='Replacing names') as pbar:\n        preprocessed['Name'] = np.where(preprocessed['Name'].apply(has_no_name(pbar)), 0, 1)\n\n    with tqdm(total=len(df), desc='Detecting languages') as pbar:\n        preprocessed['Lang'] = preprocessed.Description.apply(detect_language(pbar))\n\n    preprocessed = pd.concat([preprocessed, pd.DataFrame(sizes)[['width', 'height']]], sort=False, axis=1)\n    preprocessed = pd.concat([preprocessed, pd.DataFrame.from_dict(sentiments)], sort=False, axis=1)\n    preprocessed = pd.concat([preprocessed, pd.DataFrame(colors_metadata)], sort=False, axis=1)\n    preprocessed = pd.concat([preprocessed, pd.DataFrame(image_metadata)], sort=False, axis=1)\n    return preprocessed\n\nsizes = get_image_sizes(train, 'train_images')\nsentiments = read_sentiments('train_sentiment', train.PetID.values)\ncolors_metadata = read_colors_metadata('train_metadata', train.PetID, prioritize_by='pixelFraction')\nimage_metadata = read_image_metadata('train_metadata', train.PetID)\ntrain = preprocess(train, sizes, sentiments, colors_metadata, image_metadata)\n\ntrain_svd = generate_text_features(train)\ntrain_svd_df = pd.DataFrame(train_svd, columns=['svd_{}'.format(i) for i in range(train_svd.shape[1])])\ntrain = pd.concat([train, train_svd_df], axis=1, sort=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1cf20eac8e5ef522f55ffe315fa681ef6e79620f"},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"aad5e3c524d868ee5206e7a297711529968c7bd3"},"cell_type":"code","source":"categorical_columns = [\n    \"Type\", # 1 = Dog, 2 = Cat\n    \"Name\", # Has name or not\n    \"Breed1\", # Primary breed of pet (Refer to BreedLabels dictionary)\n    \"Breed2\", # Secondary, if mixed\n    \"Gender\", # 1 = Male, 2 = Female, 3 = Mixed, if profile represents group of pets\n    \"Color1\", # Color 1 of pet (Refer to ColorLabels dictionary)\n    \"Color2\",\n    \"Color3\", \n    \"MaturitySize\", # Size at maturity (1 = Small, 2 = Medium, 3 = Large, 4 = Extra Large, 0 = Not Specified)\n    \"FurLength\", # Fur length (1 = Short, 2 = Medium, 3 = Long, 0 = Not Specified)\n    \"Vaccinated\", # Pet has been vaccinated (1 = Yes, 2 = No, 3 = Not Sure)\n    \"Dewormed\", # Pet has been dewormed (1 = Yes, 2 = No, 3 = Not Sure)\n    \"Sterilized\", # Pet has been spayed / neutered (1 = Yes, 2 = No, 3 = Not Sure)\n    \"Health\", # Health Condition (1 = Healthy, 2 = Minor Injury, 3 = Serious Injury, 0 = Not Specified)\n    \"State\", # State location in Malaysia (Refer to StateLabels dictionary)\n    \"Lang\", # Language of the Description\n    \"label_description\" # Description from image label annotation\n]\n\ncategorical_indices = []\nfor cc in categorical_columns:\n    for i, column in enumerate(train.columns):\n        if column == cc:\n            categorical_indices.append(i)\n            break\n\ncategorical_indices","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6d8a4f57fb37b4fb1b332c3104cb443aad3cb4c9"},"cell_type":"code","source":"# Set type for categorical columns\ndef set_categorical_type(df):\n    for cc in categorical_columns:\n        df[cc] = df[cc].astype('category')\n    return df\n\ntrain = set_categorical_type(train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"77ed6952d91a32263c272fb210683a2126b3434d"},"cell_type":"code","source":"def fill_missing_categories(df):\n    preprocessed = df.copy()\n    for cc in ['Color1', 'Color2', 'Color3', 'MaturitySize', 'FurLength', 'Health']:\n        preprocessed[cc] = preprocessed[cc].apply(lambda x: MISSING_VALUE if x == 0 else x)\n    return preprocessed\n\ntrain = fill_missing_categories(train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"772ac88e449818d7e501710130a1b8a7b5c91dfb"},"cell_type":"code","source":"import lightgbm as lgb\n\ndef train_lgb(X_train, y_train, X_valid, y_valid, categorical_features=[]):\n    lgb_train = lgb.Dataset(X_train, y_train, categorical_feature=categorical_features)\n    lgb_valid = lgb.Dataset(X_valid, y_valid, reference=lgb_train, categorical_feature=categorical_features)\n    params = {\n        'application': 'regression',\n        'boosting': 'gbdt',\n        'metric': 'rmse',\n        'num_leaves': 70,\n        'max_depth': 8,\n        'learning_rate': 0.002,\n        'bagging_fraction': 0.85,\n        'feature_fraction': 0.8,\n        'min_split_gain': 0.02,\n        'min_child_samples': 150,\n        'min_child_weight': 0.02,\n        'lambda_l2': 0.05,\n        'verbosity': -1,\n        'data_random_seed': 17,\n        'early_stop': 100,\n        'verbose_eval': 100,\n        'num_rounds': 10000\n    }\n\n    gbm = lgb.train(params, lgb_train, num_boost_round=10000, valid_sets=lgb_valid, early_stopping_rounds=100)\n    return gbm","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0a44ca7039407efd353db55bb7fdd6361703797a"},"cell_type":"code","source":"irrelevant_columns = ['AdoptionSpeed', 'PetID', 'RescuerID', 'Description']\nrelevant_columns = [c for c in train.columns if c not in irrelevant_columns]\nrelevant_columns[:5]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"91c7d97ffd6225e8ada63109c51e6b76f8d22e85"},"cell_type":"code","source":"from functools import partial\nimport scipy as sp\n\ndef quadratic_weighted_kappa(y, y_pred):\n    \"\"\"\n    Calculates the quadratic weighted kappa\n    axquadratic_weighted_kappa calculates the quadratic weighted kappa\n    value, which is a measure of inter-rater agreement between two raters\n    that provide discrete numeric ratings.  Potential values range from -1\n    (representing complete disagreement) to 1 (representing complete\n    agreement). A kappa value of 0 is expected if all agreement is due to\n    chance.\n    quadratic_weighted_kappa(rater_a, rater_b), where rater_a and rater_b\n    each correspond to a list of integer ratings. These lists must have the\n    same length.\n    The ratings should be integers, and it is assumed that they contain\n    the complete range of possible ratings.\n    quadratic_weighted_kappa(X, min_rating, max_rating), where min_rating\n    is the minimum possible rating, and max_rating is the maximum possible\n    rating\n    \"\"\"\n    rater_a = y\n    rater_b = y_pred\n    min_rating=None\n    max_rating=None\n    rater_a = np.array(rater_a, dtype=int)\n    rater_b = np.array(rater_b, dtype=int)\n    assert(len(rater_a) == len(rater_b))\n    if min_rating is None:\n        min_rating = min(min(rater_a), min(rater_b))\n    if max_rating is None:\n        max_rating = max(max(rater_a), max(rater_b))\n    conf_mat = confusion_matrix(rater_a, rater_b,\n                                min_rating, max_rating)\n    num_ratings = len(conf_mat)\n    num_scored_items = float(len(rater_a))\n\n    hist_rater_a = histogram(rater_a, min_rating, max_rating)\n    hist_rater_b = histogram(rater_b, min_rating, max_rating)\n\n    numerator = 0.0\n    denominator = 0.0\n\n    for i in range(num_ratings):\n        for j in range(num_ratings):\n            expected_count = (hist_rater_a[i] * hist_rater_b[j]\n                              / num_scored_items)\n            d = pow(i - j, 2.0) / pow(num_ratings - 1, 2.0)\n            numerator += d * conf_mat[i][j] / num_scored_items\n            denominator += d * expected_count / num_scored_items\n\n    return (1.0 - numerator / denominator)\n\ndef confusion_matrix(rater_a, rater_b, min_rating=None, max_rating=None):\n    \"\"\"\n    Returns the confusion matrix between rater's ratings\n    \"\"\"\n    assert(len(rater_a) == len(rater_b))\n    if min_rating is None:\n        min_rating = min(rater_a + rater_b)\n    if max_rating is None:\n        max_rating = max(rater_a + rater_b)\n    num_ratings = int(max_rating - min_rating + 1)\n    conf_mat = [[0 for i in range(num_ratings)]\n                for j in range(num_ratings)]\n    for a, b in zip(rater_a, rater_b):\n        conf_mat[a - min_rating][b - min_rating] += 1\n    return conf_mat\n\ndef histogram(ratings, min_rating=None, max_rating=None):\n    \"\"\"\n    Returns the counts of each type of rating that a rater made\n    \"\"\"\n    if min_rating is None:\n        min_rating = min(ratings)\n    if max_rating is None:\n        max_rating = max(ratings)\n    num_ratings = int(max_rating - min_rating + 1)\n    hist_ratings = [0 for x in range(num_ratings)]\n    for r in ratings:\n        hist_ratings[r - min_rating] += 1\n    return hist_ratings\n\nclass OptimizedRounder(object):\n    def __init__(self):\n        self.coef_ = 0\n\n    def _kappa_loss(self, coef, X, y):\n        X_p = np.copy(X)\n        for i, pred in enumerate(X_p):\n            if pred < coef[0]:\n                X_p[i] = 0\n            elif pred >= coef[0] and pred < coef[1]:\n                X_p[i] = 1\n            elif pred >= coef[1] and pred < coef[2]:\n                X_p[i] = 2\n            elif pred >= coef[2] and pred < coef[3]:\n                X_p[i] = 3\n            else:\n                X_p[i] = 4\n\n        ll = quadratic_weighted_kappa(y, X_p)\n        return -ll\n\n    def fit(self, X, y):\n        loss_partial = partial(self._kappa_loss, X=X, y=y)\n        initial_coef = [0.5, 1.5, 2.5, 3.5]\n        self.coef_ = sp.optimize.minimize(loss_partial, initial_coef, method='nelder-mead')\n\n    def predict(self, X, coef):\n        X_p = np.copy(X)\n        for i, pred in enumerate(X_p):\n            if pred < coef[0]:\n                X_p[i] = 0\n            elif pred >= coef[0] and pred < coef[1]:\n                X_p[i] = 1\n            elif pred >= coef[1] and pred < coef[2]:\n                X_p[i] = 2\n            elif pred >= coef[2] and pred < coef[3]:\n                X_p[i] = 3\n            else:\n                X_p[i] = 4\n        return X_p\n\n    def coefficients(self):\n        return self.coef_['x']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"66b93fdf684cd7813e4e359b7fbd01c36ce7d597"},"cell_type":"code","source":"def predict_lgb(gbm, X_test):\n    y_pred = gbm.predict(X_test, num_iteration=gbm.best_iteration)\n    return y_pred\n\ndef predict_xgb(xgb, X_test):\n    dtest = xgboost.DMatrix(X_test)\n    y_pred = xgb.predict(dtest, ntree_limit=xgb.best_ntree_limit)\n    return y_pred","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1a5c978d1ede4682aef3446c8ce9432a896b8972"},"cell_type":"code","source":"from sklearn.model_selection import StratifiedKFold\n\npredictors = {}\nvalid_predictions = []\n\ny = list(train['AdoptionSpeed'])\nX = train['PetID']\nskf = StratifiedKFold(n_splits=5, random_state=42, shuffle=True)\nfor i, tpl in enumerate(skf.split(X, y)):\n    train_index, valid_index = tpl\n    train_df = train.iloc[train_index]\n    valid_df = train.iloc[valid_index]\n    \n    X_train = train_df[relevant_columns]\n    y_train = train_df[['AdoptionSpeed']]\n\n    X_valid = valid_df[relevant_columns]\n    y_valid = valid_df[['AdoptionSpeed']]\n    \n    gbm = train_lgb(X_train, y_train, X_valid, y_valid, categorical_features=categorical_indices)\n    predictors['gbm_{}'.format(str(i))] = gbm\n    valid_predictions.append(predict_lgb(gbm, X_valid))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"174815e8cbefa59274752a8f19b305b97fe7597e"},"cell_type":"code","source":"X_test = read_csv('test', 'test')\ntest_sizes = get_image_sizes(X_test, 'test_images')\ntest_sentiments = read_sentiments('test_sentiment', X_test.PetID.values)\ncolors_metadata = read_colors_metadata('test_metadata', X_test.PetID, prioritize_by='pixelFraction')\nimage_metadata = read_image_metadata('test_metadata', X_test.PetID)\n\nX_test = preprocess(X_test, test_sizes, test_sentiments, colors_metadata, image_metadata)\nX_test = set_categorical_type(X_test)\nX_test = fill_missing_categories(X_test)\n\ntest_svd = generate_text_features(X_test)\ntest_svd_df = pd.DataFrame(test_svd, columns=['svd_{}'.format(i) for i in range(test_svd.shape[1])])\nX_test = pd.concat([X_test, test_svd_df], axis=1, sort=False)\n\nX_test = X_test[relevant_columns]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4e683138bfbfc607d2e4cbd64cbad1b18c87329b"},"cell_type":"code","source":"X_test.head(10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1bd34892256c839f2befa7c732af53b6b5c3be29"},"cell_type":"code","source":"predictions = {}\nfor name, predictor in predictors.items():\n    if 'gbm' in name:\n        lgb_pred = predict_lgb(predictor, X_test)\n        predictions[name] = lgb_pred\n    elif 'xgb' in name:\n        xgb_pred = predict_xgb(predictor, X_test)\n        predictions[name] = xgb_pred","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"78671346d923e7dabba7f1ca9af7f32ce4f541d3"},"cell_type":"code","source":"lgb_preds = np.zeros(predictions['gbm_0'].shape)\n# xgb_preds = np.zeros(predictions['xgb_0'].shape)\nfor name, prediction in predictions.items():\n    if 'gbm' in name:\n        lgb_preds = lgb_preds + prediction\n    elif 'xgb' in name:\n        xgb_preds = xgb_preds + prediction","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"12cc3c4f49e5982a31a64378b522744d26827854"},"cell_type":"code","source":"avg_final_preds = lgb_preds / 5\navg_final_preds[:10]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"15e43730b720f829c987176446979ad4d89750a2"},"cell_type":"code","source":"train_predictions = np.array([item for sublist in valid_predictions for item in sublist])\noptimized_rounder = OptimizedRounder()\noptimized_rounder.fit(train_predictions, train.AdoptionSpeed.values)\ncoefficients = optimized_rounder.coefficients()\nrounded_final_preds = optimized_rounder.predict(avg_final_preds, coefficients).astype(np.int)\nrounded_final_preds","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"60fbdd20910eb1f42ce2d9811b56e3052dcddc8b"},"cell_type":"code","source":"submission = read_csv('test', 'sample_submission')\nsubmission['AdoptionSpeed'] = rounded_final_preds\nsubmission.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"164f4fd84eccfcb9050a8c83a77705ab2938a362"},"cell_type":"code","source":"submission.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6cda5a22c38954a590ef6c5f4b01ac795990bdcc"},"cell_type":"code","source":"from sklearn.metrics import mean_squared_error\n\ny = list(train['AdoptionSpeed'])\nX = train['PetID']\nskf = StratifiedKFold(n_splits=5, random_state=42, shuffle=True)\nerrors = []\nkappas = []\nfor i, tpl in enumerate(skf.split(X, y)):\n    _, valid_index = tpl\n    valid_df = train.iloc[valid_index]\n    y_pred = valid_predictions[i]\n    y_valid = valid_df[['AdoptionSpeed']]\n    rmse = mean_squared_error(y_valid.values, y_pred) ** 0.5\n    errors.append(rmse)\n    print('RMSE of prediction for {} fold is:'.format(str(i)), rmse)\n    y_pred_int = optimized_rounder.predict(y_pred, coefficients).astype(np.int)\n    kappa = quadratic_weighted_kappa(y_valid.values.T[0], y_pred_int)\n    kappas.append(kappa)\n    print('QWK of prediction for {} fold is:'.format(str(i)), kappa)\nprint('Mean RMSE', np.array(errors).mean())\nprint('Mean QWK', np.array(kappas).mean())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e6708af1a8d5419bcf872026d15e39baf3745e6f"},"cell_type":"code","source":"importances = {}\nfor j, tpl in enumerate(predictors.items()):\n    name, predictor = tpl\n    fis = predictor.feature_importance()\n    fold_importances = []\n    for i, fi in enumerate(fis):\n        name = X_train.columns[i]\n        fi_dict = {}\n        fi_dict['Name'] = name\n        fi_dict['Importance'] = fi\n        fold_importances.append(fi_dict)\n    importances[j] = fold_importances\n\npd.DataFrame(importances[2])\n        \n#     imports = results['importance'].groupby('feature')['feature', 'importance'].mean().reset_index()\n#     imports.sort_values('importance', ascending=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"887360807d2352e2f9a010e658ab5a4278e52d22"},"cell_type":"code","source":"import xgboost\n\ndef train_xgb(X_train, y_train, X_valid, y_valid):\n    params = {\n        'objective': 'reg:linear',\n        'eval_metric': 'rmse',\n        'eta': 0.001,\n        'max_depth': 10,\n        'subsample': 0.6,\n        'colsample_bytree': 0.6,\n        'alpha':0.001,\n        'random_state': 42,\n        'silent': False\n    }\n\n    tr_data = xgboost.DMatrix(X_train, y_train)\n    va_data = xgboost.DMatrix(X_valid, y_valid)\n\n    watchlist = [(tr_data, 'train'), (va_data, 'valid')]\n\n    xgb = xgboost.train(params, tr_data, 2000, watchlist, maximize=False, early_stopping_rounds = 100, verbose_eval=100)\n    return xgb","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c394d60a8c017ab9bed22c49126bef5b08fbbcd4"},"cell_type":"code","source":"# for i, tpl in enumerate(skf.split(X, y)):\n#     train_index, valid_index = tpl\n#     train_df = train.iloc[train_index]\n#     valid_df = train.iloc[valid_index]\n    \n#     X_train = train_df[relevant_columns]\n#     y_train = train_df[['AdoptionSpeed']]\n\n#     X_valid = valid_df[relevant_columns]\n#     y_valid = valid_df[['AdoptionSpeed']]\n    \n#     xgb = train_xgb(X_train, y_train, X_valid, y_valid)\n#     predictors['xgb_{}'.format(str(i))] = xgb","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9199f71907a4274dbb25bcaf872c9871c7ec08a8"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"}},"nbformat":4,"nbformat_minor":1}