{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"train_data_df = pd.read_csv('../input/training_set.csv')\ntrain_metadata_df = pd.read_csv('../input/training_set_metadata.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"23d3593c5c0b001243ff3ce282b23653b34bbbf9"},"cell_type":"code","source":"train_data_df['flux_ratio_sq'] = np.power(train_data_df['flux'] / train_data_df['flux_err'], 2.0)\ntrain_data_df['flux_by_flux_ratio_sq'] = train_data_df['flux'] * train_data_df['flux_ratio_sq']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ea222492f509c790746d0035935c4b3716725e3a"},"cell_type":"code","source":"data_features = train_data_df.columns[1:]\nmetadata_features = train_metadata_df.columns[1:]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9ee1737cdf1bf098837f2029ca0152ec3d670928"},"cell_type":"code","source":"groupObjects = train_data_df.groupby('object_id')[data_features]\n\nprint(\"Add constant object features\")\nfeatures = train_metadata_df.drop(['target'], axis=1)\n\nprint(\"Add mean of mutable object features\")\nfeatures = pd.merge(features, groupObjects.agg('mean'), how='right', on='object_id', suffixes=['', '_mean'])\n\nprint(\"Add sum of mutable object features\")\nfeatures = pd.merge(features, groupObjects.agg('sum'), how='right', on='object_id', suffixes=['', '_sum'])\n\nprint(\"Add median of mutable features\")\nfeatures = pd.merge(features, groupObjects.agg('median'), how='right', on='object_id', suffixes=['', '_median'])\n\nprint(\"Add minimum of mutable features\")\nfeatures = pd.merge(features, groupObjects.agg('min'), how='right', on='object_id', suffixes=['', '_min'])\n\nprint(\"Add maximum of mutable features\")\nfeatures = pd.merge(features, groupObjects.agg('max'), how='right', on='object_id', suffixes=['', '_max'])\n\nprint(\"Add range of mutable features\")\nfeatures = pd.merge(features, groupObjects.agg(lambda x: max(x) - min(x)), how='right', on='object_id', suffixes=['', '_range'])\n\nprint(\"Add standard deviation of mutable features\")\nfeatures = pd.merge(features, groupObjects.agg('std'), how='right', on='object_id', suffixes=['', '_stddev'])\n\nprint(\"Add skew of mutable features\")\nfeatures = pd.merge(features, groupObjects.agg('skew'), how='right', on='object_id', suffixes=['', '_skew'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1b134970ee1752c684547024216f440a9f60c63d"},"cell_type":"code","source":"features = features.fillna(features.mean())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"495afe1e32b26b87ab71e47af658750b28f71e35"},"cell_type":"code","source":"features","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ce2a67abaf66f1daf0086161600d733044ecac10"},"cell_type":"code","source":"import tensorflow as tf","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c60e3857016c2d9cac1187d6ca14604c93913dac"},"cell_type":"code","source":"import keras\nfrom keras.utils import to_categorical","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3c9a93d095f0c395008db5b8f782fb9aeb58ce93"},"cell_type":"code","source":"train_metadata_df['target'] = train_metadata_df.target.map({6:0, 15:1, 16:2, 42:3, 52:4, 53:5, 62:6, 64:7, 65:8, 67:9, 88:10, 90:11, 92:12, 95:13})\ntargets = train_metadata_df['target']","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d82ec2ef4843b416792158fb76ce425da3014e00"},"cell_type":"markdown","source":"**Engineer new features using Genetic Programming with the gplearn library.**"},{"metadata":{"trusted":true,"_uuid":"79dd14ba913875a04caee4ce43291ee5afb09be1"},"cell_type":"code","source":"import gplearn\nfrom gplearn.genetic import SymbolicTransformer","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1798209c7c52541310a231aa0bd9dce26a132ae1"},"cell_type":"code","source":"function_set = ['add', 'sub', 'mul', 'div',\n                'sqrt', 'log', 'abs', 'neg', 'inv',\n                'max', 'min']\n\ngp = SymbolicTransformer(generations=100, population_size=2000,\n                         hall_of_fame=100, n_components=10,\n                         function_set=function_set,\n                         parsimony_coefficient=0.0005,\n                         max_samples=0.9, verbose=1,\n                         random_state=0, n_jobs=3)\n\ngp.fit(features.drop('object_id', axis=1).values, targets.values)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9b348a872027c137b31084c1366032bf4f6ab4ef"},"cell_type":"code","source":"engineered_features = gp._programs\n\nfor i in range(len(engineered_features)):\n    for engineered_feature in engineered_features[i]:\n        if engineered_feature != None:\n            print(engineered_feature)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7ca5b7695538d9fcf5416828d5b91badd3f7b62b"},"cell_type":"code","source":"new_features = pd.DataFrame(gp.transform(features.drop('object_id', axis=1).values))\nfeatures = pd.concat([features, new_features], axis=1, join_axes=[features.index])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"89ad81739e1830ab5c4d76b44dc03565a360dd97"},"cell_type":"code","source":"features","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cadb5ceb8c7e81578d3fdf1f788d695d85b4aec0"},"cell_type":"code","source":"targets = to_categorical(targets)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"40d4b5a8b990dac27ec8685b862896b5899dcbd1"},"cell_type":"code","source":"features = features.drop(['object_id'], axis=1).values","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"09a47a3a4cfed3b091fca8910b72d6b7a7c6a719"},"cell_type":"code","source":"import sklearn\nfrom sklearn.preprocessing import MinMaxScaler\n\nscaler = MinMaxScaler(feature_range=(-1, 1)).fit(features)\nfeatures = scaler.transform(features)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"dcda4b772104579c5893cd8e9ebccd0e6e3b9113"},"cell_type":"code","source":"train_features = features [:np.int32(0.8*len(features))]\ntrain_targets = targets [:np.int32(0.8*len(features))]\n\nval_features = features[np.int32(0.8*len(features)):]\nval_targets = targets[np.int32(0.8*len(features)):]","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"0b96496429519068b27bfcad67e97b31a3b260a6"},"cell_type":"markdown","source":"**Build a simple Deep Neural Network using Dense, Dropout and BatchNormalization layers in Keras**"},{"metadata":{"trusted":true,"_uuid":"543f1b31f48c29a9788d62e8914e48826269cdbd"},"cell_type":"code","source":"from keras.models import Sequential\nfrom keras.layers import Dense, Dropout, BatchNormalization\nfrom keras.regularizers import L1L2","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"91fa0d4cbb5c1f53931391df7098b535ab30aa41"},"cell_type":"code","source":"model = Sequential()\n\nmodel.add(Dense(30, activation='relu')) # kernel_regularizer=L1L2(l1=0.00, l2=0.01), bias_regularizer=L1L2(l1=0.00, l2=0.01)\nmodel.add(BatchNormalization())\nmodel.add(Dropout(0.2))\n\n\nmodel.add(Dense(40, activation='relu')) # kernel_regularizer=L1L2(l1=0.00, l2=0.01), bias_regularizer=L1L2(l1=0.00, l2=0.01)\nmodel.add(BatchNormalization())\nmodel.add(Dropout(0.2))\n\nmodel.add(Dense(50, activation='relu')) # kernel_regularizer=L1L2(l1=0.00, l2=0.01), bias_regularizer=L1L2(l1=0.00, l2=0.01)\nmodel.add(BatchNormalization())\nmodel.add(Dropout(0.2))\n\nmodel.add(Dense(10, activation='relu')) # kernel_regularizer=L1L2(l1=0.00, l2=0.01), bias_regularizer=L1L2(l1=0.00, l2=0.01)\nmodel.add(BatchNormalization())\nmodel.add(Dropout(0.2))\n\nmodel.add(Dense(targets.shape[1], activation='softmax')) # kernel_regularizer=L1L2(l1=0.00, l2=0.01), bias_regularizer=L1L2(l1=0.00, l2=0.01)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"48129a7f3a1deea8528bf5706f089a17a3e37385"},"cell_type":"markdown","source":"**Compile the model with Categorical Cross Entropy and RMSProp as the loss function and optimizer of the model respectively. **"},{"metadata":{"trusted":true,"_uuid":"32297385176c3305c11f9b4402796f51d79e5ae0"},"cell_type":"code","source":"model.compile(loss='categorical_crossentropy', optimizer='rmsprop')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"13f15cc7c5ad4949b6cef5b842f714119297d9be"},"cell_type":"markdown","source":"**Train the model and check performance on unseen data with a validation data set.**"},{"metadata":{"trusted":true,"_uuid":"33c36f090c33bc639b7475a10618c7189e9b647d"},"cell_type":"code","source":"model.fit(train_features, train_targets, validation_data=(val_features, val_targets), epochs=500)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"544931d886b979532a9795016d678e33883aaa35"},"cell_type":"markdown","source":"**Iterate over the test data in batches and make predictions on each batch using the trained model. **"},{"metadata":{"trusted":true,"_uuid":"4f783192d986cf117e2ff90da63e61a6194fa186"},"cell_type":"code","source":"import time\n\ntest_metadata_df = pd.read_csv('../input/test_set_metadata.csv')\n\n# print(\"Add constant object features\")\ntest_metadata_df = test_metadata_df.fillna(test_metadata_df.mean())\n\npredictions = []\nobject_ids = []\n\nchunks = 5000000\ntotal = 0\n\nfor i_c, test_data_df in enumerate(pd.read_csv('../input/test_set.csv', chunksize=chunks, iterator=True)):\n    startTime = time.time()\n    \n    test_data_df['flux_ratio_sq'] = np.power(test_data_df['flux'] / test_data_df['flux_err'], 2.0)\n    test_data_df['flux_by_flux_ratio_sq'] = test_data_df['flux'] * test_data_df['flux_ratio_sq']\n    \n    groupObjects = test_data_df.fillna(test_data_df.mean()).groupby('object_id')[data_features]\n\n    # print(\"Add mean of mutable object features\")\n    features = groupObjects.agg('mean')\n    \n    # print(\"Add sum of mutable object features\")\n    features = pd.merge(features, groupObjects.agg('sum'), how='right', on='object_id', suffixes=['', '_sum'])\n\n    # print(\"Add median of mutable features\")\n    features = pd.merge(features, groupObjects.agg('median'), how='right', on='object_id', suffixes=['', '_median'])\n\n    # print(\"Add minimum of mutable features\")\n    features = pd.merge(features, groupObjects.agg('min'), how='right', on='object_id', suffixes=['', '_min'])\n\n    # print(\"Add maximum of mutable features\")\n    features = pd.merge(features, groupObjects.agg('max'), how='right', on='object_id', suffixes=['', '_max'])\n\n    # print(\"Add range of mutable features\")\n    features = pd.merge(features, groupObjects.agg(lambda x: max(x) - min(x)), how='right', on='object_id', suffixes=['', '_range'])\n\n    # print(\"Add standard deviation of mutable features\")\n    features = pd.merge(features, groupObjects.agg('std'), how='right', on='object_id', suffixes=['', '_stddev'])\n    \n    # print(\"Add skew of mutable features\")\n    features = pd.merge(features, groupObjects.agg('skew'), how='right', on='object_id', suffixes=['', '_skew'])\n    \n    test_features = pd.merge(test_metadata_df, features, on='object_id')\n    \n    new_features = pd.DataFrame(gp.transform(test_features.drop('object_id', axis=1).values))\n    test_features = pd.concat([test_features, new_features], axis=1, join_axes=[test_features.index])\n    \n    object_ids.extend(list(test_features['object_id']))\n    test_features = test_features.drop(['object_id'], axis=1).values\n    \n    test_features = scaler.transform(test_features)\n    total = total + len(test_features)\n    \n    predictions.extend(model.predict(test_features))\n\n    endTime = time.time()\n    \n    print(\"Iteration : \" + str(i_c))\n    print(\"Time taken : \" + str(endTime - startTime) + \" s\")\n    print(\"Total objects predicted on : \" + str(total))\n    print(\"\")","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"702c0c5386ba93e1674cde9fa73b62b262ab63f0"},"cell_type":"markdown","source":"**Prepare predictions for submission.**"},{"metadata":{"trusted":true,"_uuid":"96713b8f285c8ad640cf09fee2780588df4340e5"},"cell_type":"code","source":"predictions = pd.DataFrame(predictions)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f79df137e1ffd357e3bff34815676064a73c355d"},"cell_type":"code","source":"predictions.columns = ['class_6', 'class_15', 'class_16', 'class_42', 'class_52', 'class_53', 'class_62', 'class_64', 'class_65', 'class_67', 'class_88', 'class_90', 'class_92', 'class_95']","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"085c1e0ffbb8e739e88a94a9f3a003abd027f5c2"},"cell_type":"markdown","source":"**The class-99 probability is calculated as 1 minus the maximum probability predicted by the model.**"},{"metadata":{"trusted":true,"_uuid":"d3c9fbc5adf4a89060920d0783fcbb9e46cd8ee8"},"cell_type":"code","source":"predictions['class_99'] = 1 - predictions.max(axis=1)\npredictions['object_id'] = object_ids","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d931f4dac029841f0e858743fe01932be176acbe"},"cell_type":"code","source":"predictions","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"95a24fb849d189c23d6623ccd3d66f30ae4f874a"},"cell_type":"code","source":"predictions.to_csv('plasticc_submission_file.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ea69d5368747eac52b6f69313d310bdbe30e3fe7"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}