{"cells":[{"metadata":{"_uuid":"316621eccc298d43a192f150cf1600e100b45faa","_cell_guid":"34e40b6a-d8f9-419a-9be3-88e2c9b8e048","trusted":true},"cell_type":"code","source":"#It is just a start, i hope. I hope, I will have time for something more clever :-)\nimport os\nimport gc\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport matplotlib.pyplot as plt\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n%matplotlib inline\nfrom sklearn.model_selection import train_test_split\nimport lightgbm as lgb\n\nprint(os.getcwd())\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))","execution_count":1,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nNROWS1     = 3000000\nNROWS2     = 1000000\nRG         = 500000 #154903891\nSKIPROWS   = range(1,RG)\npath       = '../input/' \npath_train = path + 'train.csv'\npath_test  =  path + 'test.csv'\ntrain_cols = ['ip', 'app', 'device', 'os', 'channel', 'click_time', 'is_attributed']\ntest_cols  = ['ip', 'app', 'device', 'os', 'channel', 'click_time']\ndtypes = {\n        'ip'            : 'uint32',\n        'app'           : 'uint16',\n        'device'        : 'uint16',\n        'os'            : 'uint16',\n        'channel'       : 'uint16',\n        'is_attributed' : 'uint8',\n        'click_id'      : 'uint32'\n        }\n\nprint('Loading the training data...')\ntrain = pd.read_csv(path_train, skiprows=SKIPROWS, nrows=NROWS1, dtype=dtypes, header=0)\nprint('End loading train data...')\nprint('Loading the test data...')\ntest = pd.read_csv(path_test, dtype=dtypes, header=0)  #nrows=NROWS2,\nprint('End loading test data...')","execution_count":3,"outputs":[]},{"metadata":{"_uuid":"eaa3b3acbd97ba4b26544f56cfb987200696b0da","_cell_guid":"1612f6ef-ad35-4511-823e-c591c0343683","trusted":true},"cell_type":"code","source":"print(\"Train head\")\nprint(train.head(5))\nprint(\"Test head\")\nprint(test.head(5))","execution_count":4,"outputs":[]},{"metadata":{"_uuid":"de9ac17f4d1c1ac7f34d08b13248b400fce6fe2d","_cell_guid":"ee33a2db-287a-4d17-9b4d-a48546f8c5cf","trusted":true},"cell_type":"code","source":"print(\"Unique data train\")\nUnique_data_train = pd.to_datetime(train.click_time).dt.day.astype('uint8').value_counts().sort_index()\nprint(Unique_data_train)\nprint(\"Unique data test\")\nUnique_data_test =  pd.to_datetime(test.click_time).dt.day.astype('uint8').value_counts().sort_index()\nprint(Unique_data_test)","execution_count":6,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"len_train = len(train)\nprint('The initial size of the train set is', len_train)\ntrain = train.append(test)\nprint('Binding the training and test set together...')\ndel test","execution_count":7,"outputs":[]},{"metadata":{"_uuid":"d3bac701220fb39071a236d1afc1d20808c81797","_cell_guid":"39208f72-8f06-402f-851b-eacebe22d381","trusted":true},"cell_type":"code","source":"print(\"Train and test together\")\nprint(train.head(5))","execution_count":9,"outputs":[]},{"metadata":{"_uuid":"f81296be6170f71f256a2b5615e98c3e514ce990","_cell_guid":"e8d387b4-26c9-4573-a087-6eb6221f0933","trusted":true},"cell_type":"code","source":"#time\ntrain['hour']    = pd.to_datetime(train.click_time).dt.hour.astype('uint8')\ntrain['day']     = pd.to_datetime(train.click_time).dt.day.astype('uint8')\ntrain['wday']    = pd.to_datetime(train.click_time).dt.dayofweek.astype('uint8')\ntrain['minute']  = pd.to_datetime(train.click_time).dt.minute.astype('uint8')\ntrain['second']  = pd.to_datetime(train.click_time).dt.second.astype('uint8')\ntrain[\"doy\"]     = pd.to_datetime(train.click_time).dt.dayofyear.astype('uint8')\ntrain            = train.drop(['click_time', 'attributed_time'], axis=1)\nprint(train.dtypes) ","execution_count":11,"outputs":[]},{"metadata":{"_uuid":"04645d95cda644d015f4e822210857a35b38bbb0","_cell_guid":"220922ed-414e-474b-8656-d7bb6860c7ad","trusted":true},"cell_type":"code","source":"print(\"Frequent hours\")\nfrequent_hour = train.hour.value_counts().sort_index()\nprint(frequent_hour)\nprint(\"Frequent days\")\nfrequent_day = train.day.value_counts().sort_index()\nprint(frequent_day)\nprint(\"Frequent doy\")\nfrequent_doy = train.doy.value_counts().sort_index()\nprint(frequent_doy)\nprint(\"Frequent week days\")\nfrequent_wday = train.wday.value_counts().sort_index()\nprint(frequent_wday)\nprint(\"Frequent minutes\")\nfrequent_minute = train.minute.value_counts().sort_index()\nprint(frequent_minute)\nprint(\"Frequent seconds\")\nfrequent_second = train.second.value_counts().sort_index()\nprint(frequent_minute)","execution_count":12,"outputs":[]},{"metadata":{"_uuid":"56e53ee60de59138f5f5076c2b683d0ddeb5aeb8","_cell_guid":"88a899d8-2173-4a58-a6c7-e9ba568dc190","trusted":true},"cell_type":"code","source":"plt.figure(figsize=(15,20))\n\nplt.subplot(321)\nfrequent_hour.plot(kind='bar')\nplt.title(\"Frequent hours\")\nplt.xlabel(\"Hours\")\nplt.ylabel(\"Number\")\n\nplt.subplot(322)\nfrequent_day.plot(kind='bar')\nplt.title(\"Frequent days\")\nplt.xlabel(\"Days\")\nplt.ylabel(\"Number\")\n\nplt.subplot(323)\nfrequent_doy.plot(kind='bar')\nplt.title(\"Frequent day of year\")\nplt.xlabel(\"Frequent day of year\")\nplt.ylabel(\"Number\")\n\nplt.subplot(324)\nfrequent_wday.plot(kind='bar')\nplt.title(\"Frequent day of week\")\nplt.xlabel(\"Frequent day of week\")\nplt.ylabel(\"Number\")\n\nplt.subplot(325)\nfrequent_minute.plot(kind='bar')\nplt.xticks(np.arange(0, 69, step=10), (0,9,19,29,39,49,59))\nplt.title(\"Frequent  minutes\")\nplt.xlabel(\"Minutes\")\nplt.ylabel(\"Number\")\n\nplt.subplot(326)\nfrequent_second.plot(kind='bar')\nplt.xticks(np.arange(0, 69, step=10), (0,9,19,29,39,49,59))\nplt.title(\"Frequent seconds\")\nplt.xlabel(\"Seconds\")\nplt.ylabel(\"Number\")\n\ndel frequent_hour,frequent_day,frequent_doy,frequent_wday,frequent_minute,frequent_second\ngc.collect()","execution_count":13,"outputs":[]},{"metadata":{"_uuid":"378d7bf755f82eba71a946b8e1cdb9f3bb33a9c3","_cell_guid":"06e7b311-7bfb-4b5f-bbe0-29eaba36de34","trusted":true},"cell_type":"code","source":"BIN = 30\nplt.figure(figsize=(15,20))\nplt.subplot(321)\nplt.hist(train['hour'], bins=BIN)\nplt.title(\"Histogram of frequent hours\")\nplt.xlabel(\"Frequent hours\")\n\nplt.subplot(322)\nplt.hist(train['day'], bins=BIN)\nplt.title(\"Histogram of frequent days\")\nplt.xlabel(\"Frequent days\")\n\nplt.subplot(323)\nplt.hist(train['doy'], bins=BIN)\nplt.title(\"Histogram of frequent doys\")\nplt.xlabel(\"Frequent doys\")\n\nplt.subplot(324)\nplt.hist(train['wday'], bins=BIN)\nplt.title(\"Histogram of frequent week days\")\nplt.xlabel(\"Frequent week days\")\n\nplt.subplot(325)\nplt.hist(train['minute'], bins=BIN)\nplt.title(\"Histogram of frequent minutes\")\nplt.xlabel(\"Frequent minutes\")\n\nplt.subplot(326)\nplt.hist(train['second'], bins=BIN)\nplt.title(\"Histogram of frequent seconds\")\nplt.xlabel(\"Frequent seconds\")\nplt.show()","execution_count":14,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3611d604f579698fbb7285c561a6da762e43aa5b"},"cell_type":"code","source":"most_freq_hours_in_data    = [4, 5, 9, 10, 13, 14]\nmiddle1_freq_hours_in_data = [16, 17, 22]\nleast_freq_hours_in_data   = [6, 11, 15]\ntrain['in_hh'] = (   4 \n                     - 3*train['hour'].isin(  most_freq_hours_in_data ) \n                     - 2*train['hour'].isin(  middle1_freq_hours_in_data ) \n                     - 1*train['hour'].isin( least_freq_hours_in_data ) ).astype('uint8')\n\ngp    = train[['ip', 'day', 'in_hh', 'channel']].groupby(by=['ip', 'day', 'in_hh'])[['channel']].count().reset_index().rename(index=str, columns={'channel': 'nip_day_hh'})\ntrain = train.merge(gp, on=['ip','day','in_hh'], how='left')\ntrain.drop(['in_hh'], axis=1, inplace=True)\ntrain['nip_day_hh'] = train['nip_day_hh'].astype('uint32')\ndel gp\ngc.collect()","execution_count":15,"outputs":[]},{"metadata":{"_uuid":"c86e3418f4c873ba909474e7cfb05f5b0ab667e3","_cell_guid":"44c5df17-caf0-4dee-816c-bdc3a1a6659e","trusted":true},"cell_type":"code","source":"print(\"Train new time parameters\")\nprint(train.dtypes)","execution_count":16,"outputs":[]},{"metadata":{"_uuid":"831839c16a479c0756a4877f6342df31202aab61","_cell_guid":"f92ef6f1-34b9-4b63-8df7-0f79441f48ea","trusted":true},"cell_type":"code","source":"# Define all the groupby transformations\nGROUPBY_AGGREGATIONS = [\n    # Variance in day, for ip-app-channel\n    {'groupby': ['ip','app','channel'], 'select': 'day', 'agg': 'var', 'type': 'float32'},\n    # Variance in day, for ip-app-device\n    {'groupby': ['ip','app','device'], 'select': 'day', 'agg': 'var', 'type': 'float32'},\n    # Variance in day, for ip-app-os\n    {'groupby': ['ip','app','os'], 'select': 'day', 'agg': 'var', 'type': 'float32'},\n    \n    # Variance in hour, for ip-app-channel\n    #{'groupby': ['ip','app','channel'], 'select': 'hour', 'agg': 'var'},\n    # Variance in hour, for ip-app-device\n    #{'groupby': ['ip','app','device'], 'select': 'hour', 'agg': 'var'},\n    # Variance in hour, for ip-app-os\n    #{'groupby': ['ip','app','os'], 'select': 'hour', 'agg': 'var'},\n\n    # Count, for ip-day\n    #{'groupby': ['ip','day'], 'select': 'channel', 'agg': 'count'},\n    # Count, for ip-day\n    #{'groupby': ['ip','day'], 'select': 'device', 'agg': 'count'},\n    # Count, for ip-day\n    #{'groupby': ['ip','day'], 'select': 'os', 'agg': 'count'},\n    \n    # Count, for ip-hour\n   # {'groupby': ['ip','hour'], 'select': 'channel', 'agg': 'count'},\n    # Count, for ip-hour\n    #{'groupby': ['ip','hour'], 'select': 'device', 'agg': 'count'},\n    # Count, for ip-hour\n    #{'groupby': ['ip','hour'], 'select': 'os', 'agg': 'count'},\n\n    # Count, for ip-day-hour\n    {'groupby': ['ip','day','hour'], 'select': 'channel', 'agg': 'count', 'type': 'uint32'},\n    # Count, for ip-day-hour\n    #{'groupby': ['ip','day','hour'], 'select': 'device', 'agg': 'count', 'type': 'uint32'},\n    # Count, for ip-day-hour\n   # {'groupby': ['ip','day','hour'], 'select': 'os', 'agg': 'count', 'type': 'uint32'},\n    \n    # Count, for ip-app\n    {'groupby': ['ip', 'app'], 'select': 'channel', 'agg': 'count', 'type': 'uint32'},        \n    # Count, for ip-app-os\n    {'groupby': ['ip', 'app', 'os'], 'select': 'channel', 'agg': 'count', 'type': 'uint32'},\n    # Count, for ip-app-day-hour\n    {'groupby': ['ip','app','day','hour'], 'select': 'channel', 'agg': 'count', 'type': 'uint32'},\n    \n    # Mean hour, for ip-app-channel\n    {'groupby': ['ip','app','channel'], 'select': 'hour', 'agg': 'mean', 'type': 'float32', 'type': 'float32'}\n]\n# Apply all the groupby transformations\nfor spec in GROUPBY_AGGREGATIONS:\n    print(f\"Grouping by {spec['groupby']}, and aggregating {spec['select']} with {spec['agg']}\")\n    \n    # Unique list of features to select\n    all_features = list(set(spec['groupby'] + [spec['select']]))\n    # Name of new feature\n    new_feature = '{}_{}_{}'.format('_'.join(spec['groupby']), spec['agg'], spec['select'])\n     # Perform the groupby\n    gp = train[all_features]. \\\n        groupby(spec['groupby'])[spec['select']]. \\\n        agg(spec['agg']). \\\n        reset_index(). \\\n        rename(index=str, columns={spec['select']: new_feature}).astype(spec['type'])\n     # Merge back to X_train\n    train = train.merge(gp, on=spec['groupby'], how='left')\ndel gp\ngc.collect()\nprint(\"End\")","execution_count":17,"outputs":[]},{"metadata":{"_uuid":"03afc3202039c9b9a5e2ece184023b71fffdfad2","_cell_guid":"5c35d686-5e0b-48d6-9f54-619ac412d13d","trusted":true},"cell_type":"code","source":"print(train.dtypes)","execution_count":18,"outputs":[]},{"metadata":{"_uuid":"420c4513e542700b72a27bb193130b3592407dd2","_cell_guid":"2acd6fb6-e4f1-4549-8b07-e4ff46e8fd93","collapsed":true,"trusted":true},"cell_type":"code","source":"train['app']           = train['app'].astype('uint16')\ntrain['channel']       = train['channel'].astype('uint16')\ntrain['device']        = train['device'].astype('uint16')\ntrain['ip']            = train['ip'].astype('uint32')\ntrain['os']            = train['os'].astype('uint16')","execution_count":19,"outputs":[]},{"metadata":{"_uuid":"a85f43b4572094a8bda2eea9fdda5fc1fe06af9a","_cell_guid":"ea9a01cb-512f-4d6b-bee6-022fca3de36c","collapsed":true,"trusted":true},"cell_type":"code","source":"train_X  = train[:len_train].drop(['click_id', 'is_attributed'], axis=1)\ntrain_y  = train[:len_train]['is_attributed'].astype('uint8')\ntest_X   = train[len_train:].drop(['click_id', 'is_attributed'], axis=1)\ntest_id  = train[len_train:]['click_id'].astype('int')\ndel train","execution_count":20,"outputs":[]},{"metadata":{"_uuid":"a5326ed3707a9eb3654a58b82efea15420d618e2","_cell_guid":"d8ad02b9-5a20-4b4d-850f-dc2aa26f0623","trusted":true},"cell_type":"code","source":"print(train_X.dtypes)","execution_count":21,"outputs":[]},{"metadata":{"_uuid":"b322f5fa460d9ff0d3d528b14db025d533ce6535","_cell_guid":"44087ce7-3e33-4fe3-bbb0-837b0ff8c1db","collapsed":true,"trusted":true},"cell_type":"code","source":"#path_train_X = path_out + 'train_X.csv'\n#path_train_y = path_out + 'train_y.csv'\n#print('Loading the pre training data...')\n#train_X = pd.read_csv(path_train_X, header=0)\n#train_y = pd.read_csv(path_train_y, header=0)\n#print('End loading pre train data...')\npredictors  = ['app','device','os', 'channel', 'hour', 'day', 'doy', 'wday','minute','second',\n               'ip_app_channel_var_day',\n               'ip_app_device_var_day',\n               'ip_app_os_var_day',\n               'ip_day_hour_count_channel',\n               'ip_app_count_channel',\n               'ip_app_os_count_channel',\n               'ip_app_day_hour_count_channel',\n               'ip_app_channel_mean_hour',\n              'nip_day_hh']\ncategorical = ['app','device','os', 'channel', 'hour', 'day', 'doy', 'wday','minute','second']           ","execution_count":22,"outputs":[]},{"metadata":{"_uuid":"29f9ba503dce680d9333c4c1c99ea617e37fdbf9","_cell_guid":"9210023f-193b-46f7-889c-3e31d6585c1a","trusted":true},"cell_type":"code","source":"metrics = 'auc'\nlgb_params = {\n        'boosting_type': 'gbdt',\n        'objective': 'binary',\n        'metric':metrics,\n        'learning_rate': 0.05,\n        'num_leaves': 7,  # we should let it be smaller than 2^(max_depth)\n        'max_depth': 4,  # -1 means no limit\n        'min_child_samples': 100,  # Minimum number of data need in a child(min_data_in_leaf)\n        'max_bin': 100,  # Number of bucketed bin for feature values\n        'subsample': 0.7,  # Subsample ratio of the training instance.\n        'subsample_freq': 1,  # frequence of subsample, <=0 means no enable\n        'colsample_bytree': 0.7,  # Subsample ratio of columns when constructing each tree.\n        'min_child_weight': 0,  # Minimum sum of instance weight(hessian) needed in a child(leaf)\n        'min_split_gain': 0,  # lambda_l1, lambda_l2 and min_gain_to_split to regularization\n        'nthread': 8,\n        'verbose': 0,\n        'scale_pos_weight':99.7, # because training data is extremely unbalanced \n        'metric':metrics\n}\n \nearly_stopping_rounds = 100\nnum_boost_round       = 10000\n\nprint(\"Preparing validation datasets\")\ntrain_X, val_X = train_test_split(train_X, train_size=.95, shuffle=False )\ntrain_y, val_y = train_test_split(train_y, train_size=.95, shuffle=False )\nprint(\"End preparing validation datasets\")\n\nxgtrain = lgb.Dataset(train_X[predictors].values, label=train_y,feature_name=predictors,\n                       categorical_feature=categorical)\nxgvalid = lgb.Dataset(val_X[predictors].values, label=val_y,feature_name=predictors,\n                      categorical_feature=categorical)\nevals_results = {}\nmodel_lgb     = lgb.train(lgb_params,xgtrain,valid_sets=[xgtrain, xgvalid], \n                          valid_names=['train','valid'], \n                           evals_result=evals_results, \n                           num_boost_round=num_boost_round,\n                           early_stopping_rounds=early_stopping_rounds,\n                           verbose_eval=10, feval=None)   \n\n","execution_count":23,"outputs":[]},{"metadata":{"_uuid":"b85138dc2cf3fc4dc73adae4425b72f469432b62","_cell_guid":"3f9533f8-e463-4832-a242-fcfee60e7a23","trusted":true},"cell_type":"code","source":"print(\"Features importance...\")\ngain = model_lgb.feature_importance('gain')\nft = pd.DataFrame({'feature':model_lgb.feature_name(), \n                   'split':model_lgb.feature_importance('split'), \n                   'gain':100 * gain / gain.sum()}).sort_values('gain', ascending=False)\nprint(ft.head(50))\nft.to_csv('importance_lightgbm.csv',index=True)\nplt.figure()\nft = ft.sort_values('gain', ascending=True)\nft[['feature','gain']].head(50).plot(kind='barh', x='feature', y='gain', legend=False, figsize=(10, 10))\nplt.gcf().savefig('features_importance.png')","execution_count":24,"outputs":[]},{"metadata":{"_uuid":"b7b90c1f3d86277d8fa68fa0d9afe48d49e8417e","_cell_guid":"c1096e47-61f6-4650-a643-f7d6e25a1603","trusted":true},"cell_type":"code","source":"sub = pd.DataFrame()\nsub['click_id'] = test_id\nprint(\"Sub dimension \"    + str(sub.shape))\nprint(\"Test_X dimension \" + str(test_X.shape))","execution_count":25,"outputs":[]},{"metadata":{"_uuid":"cc9ed05089fa8ca009e9cc0ee0494c4025a8f89c","_cell_guid":"aa1081ce-2493-4f3d-9d68-55675feda962","trusted":true},"cell_type":"code","source":"print(\"Predicting...\")\nsub['is_attributed'] = model_lgb.predict(test_X[predictors])  #\nprint(\"Writing...\")\nsub.to_csv('sub_Yatsenko_01.csv',index=False)\nprint(\"Done...\")","execution_count":27,"outputs":[]},{"metadata":{"_uuid":"ab8ef77fa6fe6f2ae5fb2d3105a2507759451dd8","_cell_guid":"e860c771-50d7-4232-8966-de1fd64b1c97","collapsed":true,"trusted":false},"cell_type":"code","source":"#train[:len_train].drop(['click_time', 'click_id', 'is_attributed'], axis=1).to_csv('train_X.csv', index=False)\n#print('End saving train_X')\n#train[:len_train]['is_attributed'].astype('uint8').to_csv('train_y.csv', index=False)\n#print('End saving train_y')\n#train[len_train:].drop(['click_time', 'click_id', 'is_attributed'], axis=1).to_csv('test_X.csv', index=False)\n#print('End saving test_X')\n#train[len_train:]['click_id'].astype('int').to_csv('test_ids.csv', index=False)\n#print('End saving test_ids')\n#del train\n\n","execution_count":null,"outputs":[]}],"metadata":{"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"}},"nbformat":4,"nbformat_minor":1}