{"cells":[{"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","trusted":false},"cell_type":"code","source":"from scipy.sparse import coo_matrix, hstack\nfrom catboost import CatBoostRegressor\nfrom matplotlib import pyplot\nimport lightgbm as lgb\nimport xgboost as xgb\nfrom PIL import Image\nfrom sklearn import *\nfrom glob import glob\nimport pandas as pd\nimport numpy as np\nimport json\n\ntrain = pd.read_csv('../input/train/train.csv').fillna(-99)\ntest = pd.read_csv('../input/test/test.csv').fillna(-99)\nsub = pd.read_csv('../input/test/sample_submission.csv')\nbreed_labels = pd.read_csv('../input/breed_labels.csv')\n#state_labels = pd.read_csv('../input/state_labels.csv')\n#color_labels = pd.read_csv('../input/color_labels.csv')\n\nmore_data = []\nfor path in ['train_images', 'train_metadata', 'train_sentiment', 'test_images', 'test_metadata', 'test_sentiment']:\n    more_data += list(glob('../input/'+path+'/**'))\nmore_data = pd.DataFrame(more_data, columns=['path'])\nmore_data['type1'] = more_data['path'].map(lambda x: x.split('/')[2].split('_')[0])\nmore_data['type2'] = more_data['path'].map(lambda x: x.split('/')[2].split('_')[1])\nmore_data['PetID'] = more_data['path'].map(lambda x: x.split('/')[3].split('-')[0].split('.')[0])\n\nprint(train.shape, test.shape, breed_labels.shape)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ff05dd964397324a633960dfb6ecf87d31e64413","trusted":false},"cell_type":"code","source":"def get_sentiment(path):\n    d = json.load(open(path))\n    return d['documentSentiment']['score']\n\ndef get_magnitude(path):\n    d = json.load(open(path))\n    return d['documentSentiment']['magnitude']\n\nsentiment = more_data[more_data['type2']=='sentiment'].copy()\nsentiment['Sentiment_score'] = sentiment['path'].map(lambda x: get_sentiment(x))\nsentiment['Magnitude_score'] = sentiment['path'].map(lambda x: get_magnitude(x))\ntrain = pd.merge(train, sentiment[['PetID', 'Sentiment_score', 'Magnitude_score']], how='left', on=['PetID']).fillna(-99).reset_index(drop=True)\ntest = pd.merge(test, sentiment[['PetID', 'Sentiment_score', 'Magnitude_score']], how='left', on=['PetID']).fillna(-99).reset_index(drop=True)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"43cd2613916f542b41ebfaabcd054636de6c8597","trusted":false},"cell_type":"code","source":"#https://www.kaggle.com/abhishek/maybe-something-interesting-here\ndef pet_image_info(PetIDArr, tt='train'):\n    df = []\n    for pet in PetIDArr:\n        try:\n            with open('../input/' + tt + '_metadata/' + pet + '-1.json', 'r') as f:\n                data = json.load(f)\n            vertex_xs = data['cropHintsAnnotation']['cropHints'][0]['boundingPoly']['vertices'][2]['x']\n            vertex_ys = data['cropHintsAnnotation']['cropHints'][0]['boundingPoly']['vertices'][2]['y']\n            bounding_confidences = data['cropHintsAnnotation']['cropHints'][0]['confidence']\n            bounding_importance_fracs = data['cropHintsAnnotation']['cropHints'][0].get('importanceFraction', -1)\n            dominant_blues = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['color']['blue']\n            dominant_greens = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['color']['green']\n            dominant_reds = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['color']['red']\n            dominant_pixel_fracs = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['pixelFraction']\n            dominant_scores = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['score']\n            if data.get('labelAnnotations'):\n                label_descriptions = data['labelAnnotations'][0]['description']\n                label_scores = data['labelAnnotations'][0]['score']\n            else:\n                label_descriptions = 'nothing'\n                label_scores = -1\n        except FileNotFoundError:\n            vertex_xs = -1\n            vertex_ys = -1\n            bounding_confidences = -1\n            bounding_importance_fracs = -1\n            dominant_blues = -1\n            dominant_greens = -1\n            dominant_reds = -1\n            dominant_pixel_fracs = -1\n            dominant_scores = -1\n            label_descriptions = -1\n            label_scores = -1\n            \n        df.append([pet, vertex_xs, vertex_ys, bounding_confidences, bounding_importance_fracs, dominant_blues, dominant_greens, dominant_reds, dominant_pixel_fracs, dominant_scores, label_descriptions, label_scores])\n    df = pd.DataFrame(df, columns=['PetID', 'vertex_xs', 'vertex_ys', 'bounding_confidences', 'bounding_importance_fracs', 'dominant_blues', 'dominant_greens', 'dominant_reds', 'dominant_pixel_fracs', 'dominant_scores', 'label_descriptions', 'label_scores'])\n    return df\n\ntrain = pd.merge(train, pet_image_info(train.PetID.values, 'train'), how='left', on=['PetID'])\ntest = pd.merge(test, pet_image_info(test.PetID.values, 'test'), how='left', on=['PetID'])\ntrain.shape, test.shape","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ccf9fc84783198816b479bc398c49893d71d7324","trusted":false},"cell_type":"code","source":"def description_features(df):\n    df['Description_len'] = df['Description'].map(lambda x: len(str(x)))\n    df['Description_wc'] = df['Description'].map(lambda x: len(str(x).split(' ')))\n    df['Description_wcu'] = df['Description'].map(lambda x: len(set(str(x).split(' '))))\n    df[\"Description_mwl\"] = df['Description'].map(lambda x: np.mean([len(w) for w in str(x).split()]))\n    df['Description_wcu%'] = df['Description_wcu'] / df['Description_wc']\n    return df\n\ntrain = description_features(train); print(train.shape)\ntest = description_features(test); print(test.shape)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"1b8d82853173e55f812b3891a4ffa62f1c026799","trusted":false},"cell_type":"code","source":"tfidf = feature_extraction.text.TfidfVectorizer(min_df=3,  max_features=10000, strip_accents='unicode', token_pattern=r'\\w{1,}', ngram_range=(1, 3), sublinear_tf=True, stop_words = 'english')\nsvd = decomposition.TruncatedSVD(n_components=120)\n\ntfidf.fit(pd.concat((train.apply(lambda r: ' '.join([str(r['Name']), str(r['Description']), str(r['label_descriptions'])]), axis=1), \n                     test.apply(lambda r: ' '.join([str(r['Name']), str(r['Description']), str(r['label_descriptions'])]), axis=1))))\n\n#trainf = hstack([coo_matrix(train[col]), tfidf.transform(train.apply(lambda r: ' '.join([str(r['Name']), str(r['Description']), str(r['label_descriptions'])]), axis=1).astype(str))]); print(trainf.shape)\n#testf = hstack([coo_matrix(test[col]), tfidf.transform(test.apply(lambda r: ' '.join([str(r['Name']), str(r['Description']), str(r['label_descriptions'])]), axis=1).astype(str))]); print(testf.shape)\n\ntrainf = tfidf.transform(train.apply(lambda r: ' '.join([str(r['Name']), str(r['Description']), str(r['label_descriptions'])]), axis=1).astype(str))\ntrainf = svd.fit_transform(trainf)\ntrainf = pd.DataFrame(trainf, columns=['SVD_' + str(i).zfill(3) for i in range(120)])\ntestf = tfidf.transform(test.apply(lambda r: ' '.join([str(r['Name']), str(r['Description']), str(r['label_descriptions'])]), axis=1).astype(str))\ntestf = svd.fit_transform(testf)\ntestf = pd.DataFrame(testf, columns=['SVD_' + str(i).zfill(3) for i in range(120)])\n\ntrain = pd.concat((train, trainf), axis=1); print(train.shape)\ntest = pd.concat((test, testf), axis=1); print(test.shape)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"5420592b7d3c22e8c704235bf85578136f628b6d","trusted":false},"cell_type":"code","source":"col = [c for c in train.columns if c not in ['PetID', 'AdoptionSpeed', 'Description', 'RescuerID', 'label_descriptions', 'Name']]","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-output":false,"_uuid":"b1b9f957b840dc1b96fe103296d1be708f926b26","trusted":false},"cell_type":"code","source":"x1, x2, y1, y2 = model_selection.train_test_split(train[col], train['AdoptionSpeed'], test_size=0.2, random_state=5)\nparams = {'eta': 0.02, 'objective': 'reg:linear', 'max_depth': 7, 'subsample': 0.9, 'colsample_bytree': 0.9,  'eval_metric': 'rmse', 'seed': 3, 'silent': True}\n\ndef ks_xgb(pred, y):\n    y = y.get_label()\n    pred = pred.round().astype(int).clip(0,4)\n    return 'kappa', metrics.cohen_kappa_score(y, pred, weights='quadratic')\n\nwatchlist = [(xgb.DMatrix(x1, y1), 'train'), (xgb.DMatrix(x2, y2), 'valid')]\nmodel = xgb.train(params, xgb.DMatrix(x1, y1), 2500,  watchlist, feval=ks_xgb, maximize=True, verbose_eval=100, early_stopping_rounds=200)\ntest['AdoptionSpeed'] = (model.predict(xgb.DMatrix(test[col]), ntree_limit=model.best_ntree_limit))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"9c58f52abded26b2fb4f8039e8a4b70ff2ee5b6b","trusted":false},"cell_type":"code","source":"xgb.plot_importance(model, importance_type='weight', max_num_features=20)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"90896bbbd61cf64b1e76d377740577cddf2b7621","trusted":false},"cell_type":"code","source":"#https://www.kaggle.com/tayyabali55/complete-pet-finder-analysis-with-lgbm-4-0\nx1, x2, y1, y2 = model_selection.train_test_split(train[col], train['AdoptionSpeed'], test_size=0.2, random_state=6)\nparams = {'learning_rate': 0.02,'max_depth': 9, 'num_leaves': 80, 'application': 'regression', 'boosting': 'gbdt', 'metric': 'rmse', 'seed': 3}\n\ndef ks_lgb(pred, dtrain):\n    y = list(dtrain.get_label())\n    pred = pred.round().astype(int).clip(0,4)\n    score = metrics.cohen_kappa_score(y, pred, weights='quadratic')\n    return 'kappa', score, True\n\nmodel = lgb.train(params, lgb.Dataset(x1, label=y1), 2500, lgb.Dataset(x2, label=y2), feval=ks_lgb, verbose_eval=100, early_stopping_rounds=200)\ntest['AdoptionSpeed'] += model.predict(test[col], num_iteration=model.best_iteration)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"1dc0af3913691c0ae305ec42e2bbf15afb5ab3ed","trusted":false},"cell_type":"code","source":"lgb.plot_importance(model, importance_type='split', max_num_features=20)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"793ea392beb693b30bb610f385dc70af7c21c117","trusted":false},"cell_type":"code","source":"#https://www.kaggle.com/skooch/petfinder-simple-catboost-baseline\n#x1, x2, y1, y2 = model_selection.train_test_split(trainf.toarray(), train['AdoptionSpeed'], test_size=0.1, random_state=4)\nx1, x2, y1, y2 = model_selection.train_test_split(train[col], train['AdoptionSpeed'], test_size=0.2, random_state=7)\nparams = {'depth': 9,'eta': 0.05, 'task_type' :'GPU', 'random_strength': 1.5, 'one_hot_max_size': 2,\n          'reg_lambda': 6,'od_type': 'Iter', 'fold_len_multiplier': 2, 'border_count': 128,\n          'bootstrap_type' : \"Bayesian\", 'bagging_temperature': 1,\n          'random_seed': 217, 'early_stopping_rounds':100, 'num_boost_round': 2500}\n\nmodel = CatBoostRegressor(**params)\nmodel.fit(x1, y1, eval_set=(x2,y2), verbose=100)\nprint(metrics.cohen_kappa_score(y2, model.predict(x2).round().astype(int).clip(0,4), weights='quadratic'))\n\ntest['AdoptionSpeed'] += model.predict(test[col]) #\ntest['AdoptionSpeed'] = (test['AdoptionSpeed'] / 3).round().astype(int).clip(0,4) \ntest[['PetID', 'AdoptionSpeed']].to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ce5f42b2a620ead729cd3ef8bf5e7db34fb347c3","trusted":false},"cell_type":"code","source":"img = Image.open('../input/test_images/0df5238d7-13.jpg')\npyplot.imshow(img)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"0c95b7d553edc47308d6df3585bcbbaeff6345ce","trusted":false},"cell_type":"code","source":"!rm -r catboost_info","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.6.7"}},"nbformat":4,"nbformat_minor":1}