import numpy as np
import pandas as pd
import pyarrow.parquet as pq
# import data
meta_train = pd.read_csv('../input/metadata_train.csv')
subset_train = pq.read_pandas('../input/train.parquet', columns=[str(i) for i in range(len(meta_train))]).to_pandas()

# Data preprovess
def get_imgs(df):
    imgs = []
    for i in range(0,df.shape[1]):  # subset_train.shape[1] = 8712
        a = Series.as_matrix(df.iloc[:,i]).reshape(800,1000)
        imgs.append(a)
    return np.array(imgs)

(row, col, color) = 800, 1000, 1
Xtrain_CNN = Xtrain_CNN.reshape(Xtrain_CNN.shape[0], row, col, color) #(8712, 800, 1000) -> (8712, 800, 1000, 1)

Ytrain_CNN = np.array(meta_train['target'])

# Model Training
from keras.models import Sequential
from keras.layers import Dense, Dropout, Flatten, Activation
from keras.layers import Conv2D, MaxPooling2D
from keras.callbacks import EarlyStopping, ModelCheckpoint, ReduceLROnPlateau
from keras.layers.normalization import BatchNormalization
from keras.optimizers import Adam
import keras

model = Sequential()
model.add(Conv2D(32, kernel_size=(3, 3), activation='relu', input_shape=(row, col, color) ))
model.add(Conv2D(64, (3, 3), activation='relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Dropout(0.25))
model.add(Conv2D(64, (3, 3), activation='relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Dropout(0.25))
model.add(Flatten())
model.add(Dense(128, activation='relu'))
model.add(Dropout(0.5))
model.add(Dense(1, activation="sigmoid"))
model.compile(loss='binary_crossentropy', optimizer=Adam(lr=0.001, decay=0.0), metrics=['accuracy'])

model.fit(Xtrain_CNN, Ytrain_CNN, batch_size=100, epochs=20, verbose=1, validation_split=0.2)

# Model Test
meta_test = pd.read_csv('../input/metadata_test.csv')
subset_test = pq.read_pandas('../input/test.parquet', columns=[str(i) for i in range(len(meta_test))]).to_pandas()
X_test = get_imgs(subset_test)
prediction = model.predict_classes(X_test)
df_preds = pd.DataFrame(predictions, columns=['signal_id', 'target'])
image_id = df_preds['signal_id']
y_pred = df_preds['target']
submission = pd.DataFrame({'signal_id':image_id, 
                           'target':y_pred, 
                          }).set_index('id')
submission.to_csv('sample_submission.csv', columns=['target'])