{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"collapsed":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer\nfrom sklearn import preprocessing, model_selection, metrics\nfrom sklearn.decomposition import TruncatedSVD\n\ncolor = sns.color_palette()\n%matplotlib inline\n\npd.options.mode.chained_assignment = None\npd.options.display.max_columns = 999\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":72,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true,"collapsed":true},"cell_type":"code","source":"train_df = pd.read_csv(\"../input/train.csv\", parse_dates=[\"activation_date\"])\ntest_df = pd.read_csv(\"../input/test.csv\", parse_dates=[\"activation_date\"])\nprint(\"Train file rows and columns are : \", train_df.shape)\nprint(\"Test file rows and columns are : \", test_df.shape)","execution_count":73,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"df6265e103b8446208e63900b61c8d8f17a2f6da"},"cell_type":"code","source":"#Data Manipulation converting Russian to English\n\nfrom io import StringIO\n\ntemp_data = StringIO(\"\"\"\nregion,region_en\nСвердловская область, Sverdlovsk oblast\nСамарская область, Samara oblast\nРостовская область, Rostov oblast\nТатарстан, Tatarstan\nВолгоградская область, Volgograd oblast\nНижегородская область, Nizhny Novgorod oblast\nПермский край, Perm Krai\nОренбургская область, Orenburg oblast\nХанты-Мансийский АО, Khanty-Mansi Autonomous Okrug\nТюменская область, Tyumen oblast\nБашкортостан, Bashkortostan\nКраснодарский край, Krasnodar Krai\nНовосибирская область, Novosibirsk oblast\nОмская область, Omsk oblast\nБелгородская область, Belgorod oblast\nЧелябинская область, Chelyabinsk oblast\nВоронежская область, Voronezh oblast\nКемеровская область, Kemerovo oblast\nСаратовская область, Saratov oblast\nВладимирская область, Vladimir oblast\nКалининградская область, Kaliningrad oblast\nКрасноярский край, Krasnoyarsk Krai\nЯрославская область, Yaroslavl oblast\nУдмуртия, Udmurtia\nАлтайский край, Altai Krai\nИркутская область, Irkutsk oblast\nСтавропольский край, Stavropol Krai\nТульская область, Tula oblast\n\"\"\")\n\nregion_df = pd.read_csv(temp_data)\ntrain_df = pd.merge(train_df, region_df, how=\"left\", on=\"region\")\ntest_df = pd.merge(test_df, region_df, how=\"left\", on=\"region\")","execution_count":74,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"dcc1783bc6a52a3833d74525d9f76c5a83b9f20a","collapsed":true},"cell_type":"code","source":"temp_data = StringIO(\"\"\"\nparent_category_name,parent_category_name_en\nЛичные вещи,Personal belongings\nДля дома и дачи,For the home and garden\nБытовая электроника,Consumer electronics\nНедвижимость,Real estate\nХобби и отдых,Hobbies & leisure\nТранспорт,Transport\nУслуги,Services\nЖивотные,Animals\nДля бизнеса,For business\n\"\"\")\n\ntemp_df = pd.read_csv(temp_data)\ntrain_df = pd.merge(train_df, temp_df, on=\"parent_category_name\", how=\"left\")\ntest_df = pd.merge(test_df, temp_df, how=\"left\", on=\"parent_category_name\")\n\ntemp_data = StringIO(\"\"\"\ncategory_name,category_name_en\n\"Одежда, обувь, аксессуары\",\"Clothing, shoes, accessories\"\nДетская одежда и обувь,Children's clothing and shoes\nТовары для детей и игрушки,Children's products and toys\nКвартиры,Apartments\nТелефоны,Phones\nМебель и интерьер,Furniture and interior\nПредложение услуг,Offer services\nАвтомобили,Cars\nРемонт и строительство,Repair and construction\nБытовая техника,Appliances\nТовары для компьютера,Products for computer\n\"Дома, дачи, коттеджи\",\"Houses, villas, cottages\"\nКрасота и здоровье,Health and beauty\nАудио и видео,Audio and video\nСпорт и отдых,Sports and recreation\nКоллекционирование,Collecting\nОборудование для бизнеса,Equipment for business\nЗемельные участки,Land\nЧасы и украшения,Watches and jewelry\nКниги и журналы,Books and magazines\nСобаки,Dogs\n\"Игры, приставки и программы\",\"Games, consoles and software\"\nДругие животные,Other animals\nВелосипеды,Bikes\nНоутбуки,Laptops\nКошки,Cats\nГрузовики и спецтехника,Trucks and buses\nПосуда и товары для кухни,Tableware and goods for kitchen\nРастения,Plants\nПланшеты и электронные книги,Tablets and e-books\nТовары для животных,Pet products\nКомнаты,Room\nФототехника,Photo\nКоммерческая недвижимость,Commercial property\nГаражи и машиноместа,Garages and Parking spaces\nМузыкальные инструменты,Musical instruments\nОргтехника и расходники,Office equipment and consumables\nПтицы,Birds\nПродукты питания,Food\nМотоциклы и мототехника,Motorcycles and bikes\nНастольные компьютеры,Desktop computers\nАквариум,Aquarium\nОхота и рыбалка,Hunting and fishing\nБилеты и путешествия,Tickets and travel\nВодный транспорт,Water transport\nГотовый бизнес,Ready business\nНедвижимость за рубежом,Property abroad\n\"\"\")\n\ntemp_df = pd.read_csv(temp_data)\ntrain_df = pd.merge(train_df, temp_df, on=\"category_name\", how=\"left\")\ntest_df = pd.merge(test_df, temp_df, on=\"category_name\", how=\"left\")\n\ntrain_df[\"title_nwords\"] = train_df[\"title\"].apply(lambda x: len(x.split()))\ntest_df[\"title_nwords\"] = test_df[\"title\"].apply(lambda x: len(x.split()))","execution_count":75,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"237437cc2fd73afba01fc90396ee1643c26b6fb7","collapsed":true},"cell_type":"code","source":"train_df[\"description\"].fillna(\"NA\", inplace=True)\ntrain_df[\"desc_nwords\"] = train_df[\"description\"].apply(lambda x: len(x.split()))\n\ntest_df[\"description\"].fillna(\"NA\", inplace=True)\ntest_df[\"desc_nwords\"] = test_df[\"description\"].apply(lambda x: len(x.split()))","execution_count":76,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fee4ab1560b7fe653330f90fab6c95efcdba0215","collapsed":true},"cell_type":"code","source":"train_df.head()","execution_count":77,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"89082c16fd96cb7bf2ed68d74eaa022656724949","collapsed":true},"cell_type":"code","source":"### TFIDF Vectorizer ###\ntfidf_vec = TfidfVectorizer(ngram_range=(1,1))\n#ngram_range defines how you want to have words in your dictionary. (min,max) = (1,2) will mean you will have unigrams and bigrms in your vocabulary. \n#Example String: \"The old fox\"\n#Vocabulary: \"The\", \"old\", \"fox\", \"The old\", \"old fox\"\n\nfull_tfidf = tfidf_vec.fit_transform(train_df['title'].values.tolist() + test_df['title'].values.tolist())\n#train_df['title'].values.tolist() this converts all the values in the title column into a list. '+' appends two lists\n\ntrain_tfidf = tfidf_vec.transform(train_df['title'].values.tolist())\ntest_tfidf = tfidf_vec.transform(test_df['title'].values.tolist())","execution_count":78,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"71277def8861c0ad7d25c12eaaa8fe3965857db2","collapsed":true},"cell_type":"code","source":"### SVD Components ###\nn_comp = 5\nsvd_obj = TruncatedSVD(n_components=n_comp, algorithm='arpack')\nsvd_obj.fit(full_tfidf)\ntrain_svd = pd.DataFrame(svd_obj.transform(train_tfidf))\ntest_svd = pd.DataFrame(svd_obj.transform(test_tfidf))\ntrain_svd.columns = ['svd_title_'+str(i+1) for i in range(n_comp)]\ntest_svd.columns = ['svd_title_'+str(i+1) for i in range(n_comp)]\ntrain_df = pd.concat([train_df, train_svd], axis=1)\ntest_df = pd.concat([test_df, test_svd], axis=1)\ndel full_tfidf, train_tfidf, test_tfidf, train_svd, test_svd","execution_count":79,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b458b180ae81b26bb604830004925927cc1a2016","collapsed":true},"cell_type":"code","source":"tfidf_vec = TfidfVectorizer(ngram_range=(1,1), max_features=100000)\nfull_tfidf = tfidf_vec.fit_transform(train_df['description'].values.tolist() + test_df['description'].values.tolist())\ntrain_tfidf = tfidf_vec.transform(train_df['description'].values.tolist())\ntest_tfidf = tfidf_vec.transform(test_df['description'].values.tolist())\n\n### SVD Components ###\nn_comp = 5\nsvd_obj = TruncatedSVD(n_components=n_comp, algorithm='arpack')\nsvd_obj.fit(full_tfidf)\ntrain_svd = pd.DataFrame(svd_obj.transform(train_tfidf))\ntest_svd = pd.DataFrame(svd_obj.transform(test_tfidf))\ntrain_svd.columns = ['svd_desc_'+str(i+1) for i in range(n_comp)]\ntest_svd.columns = ['svd_desc_'+str(i+1) for i in range(n_comp)]\ntrain_df = pd.concat([train_df, train_svd], axis=1)\ntest_df = pd.concat([test_df, test_svd], axis=1)\ndel full_tfidf, train_tfidf, test_tfidf, train_svd, test_svd","execution_count":80,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"eb4363b4c715fd77fa60c3ca668fe33ab8e66862"},"cell_type":"code","source":"train_df['param123'] = train_df['param_1'].fillna('') + \" \" + train_df['param_2'].fillna('') + \" \" + train_df['param_3'].fillna('') \ntest_df['param123'] = test_df['param_1'].fillna('') + \" \" + test_df['param_2'].fillna('') + \" \" + test_df['param_3'].fillna('') ","execution_count":81,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"018bbff9eb2065c4406d96694af3addac6894180"},"cell_type":"code","source":"tfidf_vec = TfidfVectorizer(ngram_range=(1,1), max_features=100000)\nfull_tfidf = tfidf_vec.fit_transform(train_df['param123'].values.tolist() + test_df['param123'].values.tolist())\ntrain_tfidf = tfidf_vec.transform(train_df['param123'].values.tolist())\ntest_tfidf = tfidf_vec.transform(test_df['param123'].values.tolist())\n\n### SVD Components ###\nn_comp = 5\nsvd_obj = TruncatedSVD(n_components=n_comp, algorithm='arpack')\nsvd_obj.fit(full_tfidf)\ntrain_svd = pd.DataFrame(svd_obj.transform(train_tfidf))\ntest_svd = pd.DataFrame(svd_obj.transform(test_tfidf))\ntrain_svd.columns = ['svd_params_'+str(i+1) for i in range(n_comp)]\ntest_svd.columns = ['svd_params_'+str(i+1) for i in range(n_comp)]\ntrain_df = pd.concat([train_df, train_svd], axis=1)\ntest_df = pd.concat([test_df, test_svd], axis=1)\ndel full_tfidf, train_tfidf, test_tfidf, train_svd, test_svd","execution_count":82,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b54c02711ec2cd8c31ebee11d25e060060b70619","collapsed":true},"cell_type":"code","source":"# New variable on weekday #\ntrain_df[\"activation_weekday\"] = train_df[\"activation_date\"].dt.weekday\ntest_df[\"activation_weekday\"] = test_df[\"activation_date\"].dt.weekday\n\ntrain_df.head()","execution_count":83,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"44a4dfec9ac46ab8a366e77ad230b01e3e229927","collapsed":true},"cell_type":"code","source":"train_df[\"price_new\"] = train_df[\"price\"].values\ntrain_df[\"price_new\"].fillna(np.nanmedian(train_df[\"price\"].values), inplace=True)\n\ntest_df[\"price_new\"] = test_df[\"price\"].values\ntest_df[\"price_new\"].fillna(np.nanmedian(train_df[\"price\"].values), inplace=True)\n\n# Feature Scaling\n#from sklearn.preprocessing import StandardScaler\n#sc = StandardScaler()\n#train_df[\"price_new\"] = sc.fit_transform(train_df[\"price_new\"])\n#test_df[\"price\"] = sc.transform(test_df[\"price\"])\n\n# Label encode the categorical variables #\ncat_vars = [\"region\", \"city\", \"parent_category_name\", \"category_name\", \"user_type\"]\nfor col in cat_vars:\n    lbl = preprocessing.LabelEncoder()\n    lbl.fit(list(train_df[col].values.astype('str')) + list(test_df[col].values.astype('str')))\n    train_df[col] = lbl.transform(list(train_df[col].values.astype('str')))\n    test_df[col] = lbl.transform(list(test_df[col].values.astype('str')))\n\ncols_to_drop = [\"item_id\", \"user_id\", \"title\", \"description\", \"activation_date\", \"image\", \"param_1\", \"param_2\", \"param_3\", \"param123\", \"price_new\"]\ntrain_X = train_df.drop(cols_to_drop + [\"region_en\", \"parent_category_name_en\", \"category_name_en\", \"deal_probability\"], axis=1)\ntest_X = test_df.drop(cols_to_drop + [\"region_en\", \"parent_category_name_en\", \"category_name_en\"], axis=1)\n\ntrain_y = train_df[\"deal_probability\"].values\ntest_id = test_df[\"item_id\"].values","execution_count":84,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"eb705db030674f22e853c7d82ebe28ef9847bf0e","collapsed":true},"cell_type":"code","source":"train_X.head()","execution_count":85,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"acae2886037777952f6272682a73e9fbe7c446bd","collapsed":true},"cell_type":"code","source":"test_X.head()","execution_count":86,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f6d6cddeb77e5f3571a6bc156d5d7fd885903e6f","collapsed":true},"cell_type":"code","source":"#custom function to build the LightGBM model.\ndef run_lgb(train_X, train_y, val_X, val_y, test_X):\n    params = {\n        \"objective\" : \"regression\",\n        \"metric\" : \"rmse\",\n        \"num_leaves\" : 200,\n        \"learning_rate\" : 0.01,\n        \"bagging_fraction\" : 0.8,\n        \"feature_fraction\" : 0.8,\n        \"bagging_freq\" : 5,\n        \"bagging_seed\" : 2018,\n        \"verbosity\" : -1,\n        #'max_depth':-1,\n        \"min_child_samples\":20\n       # ,\"boosting\":\"rf\"\n    }\n    \n    lgtrain = lgb.Dataset(train_X, label=train_y)\n    lgval = lgb.Dataset(val_X, label=val_y)\n    evals_result = {}\n    model = lgb.train(params, lgtrain, 1000, valid_sets=[lgval], early_stopping_rounds=100, verbose_eval=20, evals_result=evals_result)\n    \n    #model = lgb.cv(params, lgtrain, 1000, early_stopping_rounds=20, verbose_eval=20, stratified=False )\n    \n    pred_test_y = model.predict(test_X, num_iteration=model.best_iteration)\n    return pred_test_y, model, evals_result","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9ef54aec037a6575559154e464b29226e1160d86","collapsed":true},"cell_type":"code","source":"#split the train into development and validation sample. Take the last 100K rows as validation sample.\n# Splitting the data for model training#\ndev_X = train_X.iloc[:-100000,:]\nval_X = train_X.iloc[-100000:,:]\ndev_y = train_y[:-100000]\nval_y = train_y[-100000:]\nprint(dev_X.shape, val_X.shape, test_X.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9b0862faf1afadc32ad6bb5d7d616b11ae32dc5d","scrolled":true,"collapsed":true},"cell_type":"code","source":"# Training the model #\nimport lightgbm as lgb\npred_test, model, evals_result = run_lgb(dev_X, dev_y, val_X, val_y, test_X)","execution_count":71,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a2751f8058a23655f15f081923db9b35307aecd3","collapsed":true},"cell_type":"code","source":"# Plot importance\nlgb.plot_importance(model, importance_type=\"split\", title=\"split\")\nplt.show()\n\nlgb.plot_importance(model, importance_type=\"gain\", title='gain')\nplt.show()\n\n# Importance values are also available in:\nprint(model.feature_importance(\"split\"))\nprint(model.feature_importance(\"gain\"))","execution_count":37,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6d94d8d9e870608e8b2f8fbb0cb1a304b89c3be0","collapsed":true},"cell_type":"code","source":"# Making a submission file #\npred_test[pred_test>1] = 1\npred_test[pred_test<0] = 0\nsub_df = pd.DataFrame({\"item_id\":test_id})\nsub_df[\"deal_probability\"] = pred_test\nsub_df.to_csv(\"baseline_lgb.csv\", index=False)","execution_count":27,"outputs":[]},{"metadata":{"_kg_hide-output":false,"_kg_hide-input":false,"trusted":true,"_uuid":"483dd45bc9560f728bc2e0f7a74759c04d3c9999","collapsed":true},"cell_type":"code","source":"#print(os.listdir(\"../working\"))","execution_count":27,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.5","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}