{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"Private score: **0.609391**, Public score: **0.716267**\n\nIn this competition only submission file was required while source data was quite massive. Thus, I was training my models on my PC (with GPU). \n\nThis notebook is quick *port* from my source files to share code with the community.","metadata":{}},{"cell_type":"code","source":"import tensorflow as tf\n\n@tf.function\ndef correlation_score(y_true, y_pred):\n    # corrsum = 0\n    # for i in range(len(y_true)):\n    #     corrsum += np.corrcoef(y_true[i], y_pred[i])[1, 0]\n    # return corrsum / len(y_true)\n\n    # from: https://stackoverflow.com/questions/43834830/tensorflow-equivalent-of-np-corrcoef-on-a-specific-axis\n    x = y_true\n    y = y_pred\n    dsize = 3\n    x_t = x #tf.constant(x)\n    y_t = y #tf.constant(y)\n    xy_t = tf.concat([x, y], axis=0)\n    mean_t = tf.reduce_mean(xy_t, axis=1, keepdims=True)\n    cov_t = ((xy_t-mean_t) @  tf.transpose(xy_t-mean_t))/(dsize-1)\n    cov2_t = tf.linalg.diag(1/tf.sqrt(tf.linalg.diag_part(cov_t)))\n    #cor =   tf.matmul(tf.matmul(cov2_t,  cov_t), cov2_t) # cov2_t @ cov_t @ cov2_t\n    cor = cov2_t @ cov_t @ cov2_t\n    return cor\n\n@tf.function\ndef correlation_loss(y_true, y_pred):\n    return -correlation_score(y_true, y_pred)\n\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**datasets.py**","metadata":{}},{"cell_type":"code","source":"import tensorflow as tf\nimport numpy as np\nimport h5py\nimport hdf5plugin\nimport random\nfrom custom_metrics import *\nimport os\n\n# axis0 -- columns\n# axis1 -- rows (cell_ids)\n\ntest_cite_inputs_fname = \"open-problems-multimodal/test_cite_inputs.h5\"\ntest_multi_inputs_fname = \"open-problems-multimodal/test_multi_inputs.h5\"\n\ncite_targets_filename = \"open-problems-multimodal/train_cite_targets.h5\"\nmulti_targets_filename = \"open-problems-multimodal/train_multi_targets.h5\"\n\ncite_inputs_filename = \"open-problems-multimodal/train_cite_inputs.h5\"\nmulti_inputs_filename = \"open-problems-multimodal/train_multi_inputs.h5\"\n\nclass data_generator:\n    def __init__(self, input_file_name, input_key):\n        self.input_file_name = input_file_name\n        self.input_key = input_key\n\n    def __call__(self):\n        with h5py.File(self.input_file_name, 'r') as h_input: \n            for i in h_input[self.input_key]['block0_values']:\n                    yield i\n\n\ndef load_input_dyn_datasets(input_file_name, input_key, save_file_name=None):\n\n    with h5py.File(input_file_name, \"r\") as f:\n        input_data_shape =            f[input_key]['block0_values'].shape\n        input_column_names =    np.array(f[input_key]['axis0'])\n        input_row_names =       np.array(f[input_key]['axis1'])\n\n    ds = tf.data.Dataset.from_generator(data_generator(input_file_name, input_key),  \n        output_signature=(\n            tf.TensorSpec(shape=(input_data_shape[1]), dtype=tf.float32)))\n\n    if save_file_name is not None:\n        #ds = tf.data.Dataset.load(\"multi_dataset_full\")\n        ds.save(save_file_name)\n\n    return ds, input_row_names, input_column_names, input_data_shape\n\ndef load_h5_array(input_file_name, input_key, table_name):\n\n    with h5py.File(input_file_name, \"r\") as f:\n        table =       np.array(f[input_key][table_name])\n\n    return table","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**train_cite_mode.py**","metadata":{}},{"cell_type":"code","source":"import tensorflow as tf\nimport numpy as np\nimport h5py\nimport hdf5plugin\nimport random\nfrom custom_metrics import *\n\n\n# axis0 -- columns\n# axis1 -- rows (cell_ids)\n\ntest_cite_inputs_fname = \"open-problems-multimodal/test_cite_inputs.h5\"\ntest_multi_inputs_fname = \"open-problems-multimodal/test_multi_inputs.h5\"\n\ncite_targets_filename = \"open-problems-multimodal/train_cite_targets.h5\"\nmulti_targets_filename = \"open-problems-multimodal/train_multi_targets.h5\"\n\ncite_inputs_filename = \"open-problems-multimodal/train_cite_inputs.h5\"\nmulti_inputs_filename = \"open-problems-multimodal/train_multi_inputs.h5\"\n\ndef load_cite_datasets(input_file_name, target_file_name, input_key, target_key, validation_size=None):\n    with h5py.File(input_file_name, \"r\") as f:\n        input_column_names =    np.array(f[input_key]['axis0'])\n        input_row_names =       np.array(f[input_key]['axis1'])\n        input_data =            np.array(f[input_key]['block0_values'])\n\n    with h5py.File(target_file_name, \"r\") as f:\n        target_column_names =    np.array(f[target_key]['axis0'])\n        target_row_names =       np.array(f[target_key]['axis1'])\n        target_data =            np.array(f[target_key]['block0_values'])\n\n    print(f\"{input_key}: {input_data.shape}\")\n    print(f\"{target_key}: {target_data.shape}\")\n\n    assert input_data.shape[0] == target_data.shape[0]\n\n    model_shape = (input_data.shape[1], target_data.shape[1])\n\n    if validation_size is not None:\n        indexes = range(input_data.shape[0])\n        validation_indexes = random.sample(indexes, int(input_data.shape[0] * validation_size))\n        train_indexes = list(set(indexes) - set(validation_indexes))\n        print(f\"Train size: {len(train_indexes)}, validation size: {len(validation_indexes)}\")\n\n        train_ds = tf.data.Dataset.from_tensor_slices((input_data[train_indexes, :], target_data[train_indexes, :]))\n        validation_ds = tf.data.Dataset.from_tensor_slices((input_data[validation_indexes, :], target_data[validation_indexes, :]))\n        return train_ds, validation_ds, model_shape\n\n    else:\n        return tf.data.Dataset.from_tensor_slices((input_data, target_data)), model_shape\n\n\ndef build_model(cite_model_shape, units=1024, n_stages=3, train_ds=None, dropout=0.1):\n\n    normalisation_layer = tf.keras.layers.Normalization(axis=-1, input_shape=(cite_model_shape[0],))\n    normalisation_layer.adapt(train_ds.map(lambda x, y: x).prefetch(tf.data.AUTOTUNE))\n\n    model = tf.keras.Sequential()\n    model.add(tf.keras.Input(shape=(cite_model_shape[0],)))\n\n    model.add(normalisation_layer)\n\n    for n in range(1, n_stages):\n        model.add(tf.keras.layers.Dense(units // 2**n, activation='leaky_relu'))\n        model.add(tf.keras.layers.Dropout(dropout))\n\n    for n in range(n_stages, 0, -1):\n        model.add(tf.keras.layers.Dense(units // 2**n, activation='leaky_relu'))\n        model.add(tf.keras.layers.Dropout(dropout))\n\n    model.add(tf.keras.layers.Dense(units=cite_model_shape[1]))    \n\n    return model\n\n\ndef run():\n    MAX_EPOCHS = 30\n    BATCH_SIZE = 64\n    LR = 0.000005\n    DECAY = 0.9\n\n    print(\"Loading cite datasets...\")\n    cite_train_ds, cite_valid_ds, cite_model_shape = load_cite_datasets(cite_inputs_filename, cite_targets_filename, 'train_cite_inputs', 'train_cite_targets', 0.1)\n    print(f\"Cite DS: Train:{len(cite_train_ds)}, Valid: {len(cite_valid_ds)}\")\n    print(f\"Cite model shape:{cite_model_shape}\")\n\n    print(\"Building model...\")\n    cite_model = build_model(cite_model_shape, 1024, n_stages=6, train_ds=cite_train_ds, dropout=0.1)\n\n    optimizer = tf.keras.optimizers.Adam(learning_rate = LR)\n\n    cite_model.compile(optimizer=optimizer, \n        metrics=['mae', 'mse', correlation_score],\n        loss = [correlation_loss]  # 'mae', 'huber'\n        )\n\n    print(\"\\nModel evaluation:\")\n    cite_model.evaluate(cite_valid_ds.batch(BATCH_SIZE))\n\n\n    early_stopping = tf.keras.callbacks.EarlyStopping(monitor='val_loss',\n                                                    patience=5,\n                                                    mode='min', restore_best_weights=True)\n\n    def lr_scheduler(start_e=0):\n        lr_schedule = tf.keras.callbacks.LearningRateScheduler(lambda epoch: LR * (DECAY ** (start_e + epoch)))\n        return lr_schedule\n\n    print(\"\\nFitting:\")\n    history = cite_model.fit(\n        cite_train_ds.shuffle(BATCH_SIZE * 10).batch(BATCH_SIZE).prefetch(tf.data.AUTOTUNE), \n        validation_data = cite_valid_ds.batch(BATCH_SIZE).prefetch(tf.data.AUTOTUNE),\n        epochs=MAX_EPOCHS,\n        callbacks=[early_stopping, lr_scheduler(0)])\n\n    print(\"Saving model: 'cite_model_eval'\")\n    cite_model.save(\"cite_model_eval\")\n\n    train_epochs = len(history.epoch)\n    print(f\"Took {train_epochs} epochs.\")\n\n    print(\"\\nFitting on full dataset:\")\n    cite_model.fit(\n        cite_train_ds.concatenate(cite_valid_ds).shuffle(BATCH_SIZE * 10).batch(BATCH_SIZE).prefetch(tf.data.AUTOTUNE), \n        validation_data = cite_valid_ds.batch(BATCH_SIZE).prefetch(tf.data.AUTOTUNE),\n        epochs=train_epochs // 2,\n        callbacks=[early_stopping, lr_scheduler(train_epochs)])\n\n    print(\"Saving model: 'cite_model_final'\")\n    cite_model.save(\"cite_model_final\")\n\nif __name__ == \"__main__\":\n    # physical_devices = tf.config.list_physical_devices('GPU')\n    # tf.config.experimental.set_memory_growth(physical_devices[0], True)\n    run()\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**train_multi_data.py**","metadata":{}},{"cell_type":"code","source":"import tensorflow as tf\nimport numpy as np\nimport h5py\nimport hdf5plugin\nimport random\nfrom custom_metrics import *\nimport os\n\n# axis0 -- columns\n# axis1 -- rows (cell_ids)\n\ntest_cite_inputs_fname = \"open-problems-multimodal/test_cite_inputs.h5\"\ntest_multi_inputs_fname = \"open-problems-multimodal/test_multi_inputs.h5\"\n\ncite_targets_filename = \"open-problems-multimodal/train_cite_targets.h5\"\nmulti_targets_filename = \"open-problems-multimodal/train_multi_targets.h5\"\n\ncite_inputs_filename = \"open-problems-multimodal/train_cite_inputs.h5\"\nmulti_inputs_filename = \"open-problems-multimodal/train_multi_inputs.h5\"\n\nclass data_generator:\n    def __init__(self, input_file_name, input_key, target_file_name, target_key, pca=None):\n        self.input_file_name = input_file_name\n        self.target_file_name = target_file_name\n        self.input_key = input_key\n        self.target_key = target_key\n        self.pca = pca\n\n    def __call__(self):\n        with h5py.File(self.input_file_name, 'r') as h_input: \n            with h5py.File(self.target_file_name, 'r') as h_target:\n  #              assert h_input[self.input_key]['block0_values'].shape[0] == h_target[self.target_key]['block0_values'].shape[0]\n\n                for i, t in zip(h_input[self.input_key]['block0_values'], h_target[self.target_key]['block0_values']):\n                    if self.pca is not None:\n                        ii = self.pca.transform(i.reshape(1,-1))\n                        yield (ii.reshape(-1), t)\n                    else:\n                        yield (i, t)\n\n\ndef load_multi_datasets(input_file_name, target_file_name, input_key, target_key, validation_size=None, pca=None):\n\n    with h5py.File(input_file_name, \"r\") as f:\n        input_data_shape =            f[input_key]['block0_values'].shape\n    with h5py.File(target_file_name, \"r\") as f:\n        target_data_shape =           f[target_key]['block0_values'].shape\n\n #   assert input_data_shape[0] == target_data_shape[0]\n    model_shape = (input_data_shape[1], target_data_shape[1])\n\n    if pca is not None:\n        model_shape = (pca.n_components, model_shape[1])\n\n    ds = tf.data.Dataset.from_generator(data_generator(input_file_name, input_key, target_file_name, target_key, pca=pca),  \n        output_signature=(\n            tf.TensorSpec(shape=(model_shape[0]), dtype=tf.float32),\n            tf.TensorSpec(shape=(model_shape[1]), dtype=tf.float32)))\n\n    os.makedirs(\"cache\", exist_ok=True)\n\n    #ds = tf.data.Dataset.load(\"multi_dataset_full\")\n    #ds.save(\"multi_dataset_full\")\n\n    validation_size = int(input_data_shape[0] * validation_size)\n    train_ds = ds.skip(validation_size)#.cache(\"cache/ds_t\")\n    valid_ds = ds.take(validation_size)#.cache(\"cache/ds_v\")\n\n    # test = next(iter(ds))\n    # print(test)\n\n    return train_ds, valid_ds, model_shape, input_data_shape[0]\n\n\ndef build_model(multi_model_shape, units=1024, n_stages=3, head_size=1024, train_ds=None, dropout=0.1, no_dec=False, with_normalisation=True):\n\n    #normalisation_layer = tf.keras.layers.Normalization(axis=-1, input_shape=(multi_model_shape[0],))\n    #normalisation_layer.adapt(train_ds.map(lambda x, y: x).prefetch(tf.data.AUTOTUNE))\n\n    #np.save(\"normalisation_multi_mean.npy\", normalisation_layer.adapt_mean.numpy())\n    #np.save(\"normalisation_multi_variance.npy\", normalisation_layer.adapt_variance.numpy())\n\n    normalisation_multi_mean = np.load(\"normalisation_multi_mean.npy\", allow_pickle=True)\n    normalisation_multi_variance = np.load(\"normalisation_multi_variance.npy\", allow_pickle=True)\n    normalisation_layer = tf.keras.layers.Normalization(axis=-1, \n        mean=normalisation_multi_mean,\n        variance=normalisation_multi_variance,\n        input_shape=(multi_model_shape[0],))\n\n    model = tf.keras.Sequential()\n    model.add(tf.keras.Input(shape=(multi_model_shape[0],)))\n\n    if with_normalisation:\n        model.add(normalisation_layer)\n\n    for n in range(0, n_stages):\n        model.add(tf.keras.layers.Dense(units // 2**n, activation='leaky_relu'))\n        #model.add(tf.keras.layers.LayerNormalization())\n        model.add(tf.keras.layers.BatchNormalization())\n        model.add(tf.keras.layers.Dropout(dropout))\n\n    if no_dec is False:\n        for n in range(n_stages, -1, -1):\n            model.add(tf.keras.layers.Dense(units // 2**n, activation='leaky_relu'))\n            #model.add(tf.keras.layers.LayerNormalization())\n            model.add(tf.keras.layers.Dropout(dropout))\n\n    if type(head_size) is list:\n        for h in head_size:\n            model.add(tf.keras.layers.Dense(h, activation='leaky_relu'))\n    else:\n        model.add(tf.keras.layers.Dense(head_size, activation='leaky_relu'))\n\n    model.add(tf.keras.layers.Dropout(dropout))\n    model.add(tf.keras.layers.Dense(units=multi_model_shape[1], activation='tanh'))\n    model.add(tf.keras.layers.Lambda(lambda x: x * 10.0))    \n\n    return model\n\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**train_multi_model.py**","metadata":{}},{"cell_type":"code","source":"import tensorflow as tf\nimport numpy as np\nimport random\nfrom custom_metrics import *\nimport os\nfrom train_multi_data import *\n\nimport gc\n\ndef m_report(history, *metrics, item=-1):\n    return \", \".join(f\"{x}: {history.history[x][item]:.4f}\" for x in metrics)\n\n\ndef run():\n    MAX_EPOCHS = 90\n    BATCH_SIZE = 512\n    DECAY = 0.9\n    EVAL_SIZE = 0.05\n\n    TEST_EPOCHS = 50\n    LR = 0.000005\n    STAGES = 5\n    LOSS = correlation_loss #[correlation_loss, 'mse' ]\n    UNITS = 64\n    DROPOUT = 0.0\n    NO_DEC=True\n    WITH_NORM=False\n    HEAD = [2048,2048]\n\n\n    print(\"Loading multi datasets...\")\n    multi_inputs_filename = \"pca_train_multi_inputs_28.h5\"\n    multi_train_ds, multi_valid_ds, multi_model_shape, row_count = load_multi_datasets(multi_inputs_filename, multi_targets_filename, 'train_multi_inputs', 'train_multi_targets', EVAL_SIZE)\n    ##print(f\"Multi DS: Train:{len(multi_train_ds)}, Valid: {len(multi_valid_ds)}\")\n    print(f\"Multi model shape:{multi_model_shape}\")\n\n    print(f\"Building model...\")\n\n    multi_model = build_model(multi_model_shape, UNITS, n_stages=STAGES, head_size=HEAD, train_ds=multi_train_ds, dropout=DROPOUT, no_dec=NO_DEC, with_normalisation=WITH_NORM)\n\n    multi_train_ds = multi_train_ds.shuffle(BATCH_SIZE * 5).batch(BATCH_SIZE).prefetch(tf.data.AUTOTUNE)\n    multi_valid_ds = multi_valid_ds.batch(BATCH_SIZE).cache().prefetch(tf.data.AUTOTUNE)\n\n    optimizer = tf.keras.optimizers.Adam(learning_rate = LR)\n\n    multi_model.compile(optimizer=optimizer, \n        metrics=['mae', 'mse', correlation_score],\n        loss = LOSS  # 'mae', 'huber', correlation_loss\n        )\n\n    print(\"\\nModel evaluation:\")\n    eval = multi_model.evaluate(multi_valid_ds)\n\n    early_stopping = tf.keras.callbacks.EarlyStopping(monitor='val_loss',\n                                                    patience=5,\n                                                    mode='min', restore_best_weights=True)\n\n    reduce_lr = tf.keras.callbacks.ReduceLROnPlateau(\n                monitor = \"val_loss\",\n                factor = 0.9, \n                patience = 3, \n                verbose = 1\n                )\n\n    def lr_scheduler(start_e=0):\n        lr_schedule = tf.keras.callbacks.LearningRateScheduler(lambda epoch: LR * (DECAY ** (start_e + epoch)))\n        return lr_schedule  \n\n\n    print(\"\\nFitting:\")\n    history = multi_model.fit(\n        multi_train_ds, \n        validation_data = multi_valid_ds,\n        epochs=MAX_EPOCHS,\n        callbacks=[early_stopping, \n            reduce_lr,\n            #lr_scheduler(0)\n            ])\n\n    print(\"Saving model: 'multi_model_eval'\")\n    multi_model.save(\"multi_model_eval\")\n    with open(\"training_results.txt\", \"a\") as log_file:\n        print(\"------------------------\", file=log_file)\n        print(\"  'multi_model_eval'\", file=log_file)\n        print(f\" F: {m_report(history, 'loss', 'mse', 'correlation_score', 'val_loss', 'val_mse', 'val_correlation_score')}\", file=log_file, flush=True)\n        print(f\"    {m_report(history, 'loss', 'mse', 'correlation_score', 'val_loss', 'val_mse', 'val_correlation_score', item=0)}\", file=log_file, flush=True)\n\n    train_epochs = len(history.epoch)\n    print(f\"Took {train_epochs} epochs.\")\n\n    print(\"\\nFitting on full dataset:\")\n    history = multi_model.fit(\n        multi_train_ds.concatenate(multi_valid_ds), \n        validation_data = multi_valid_ds,\n        epochs= 5,\n        callbacks=[early_stopping, lr_scheduler(train_epochs)])\n\n    print(\"Saving model: 'multi_model_final'\")\n    multi_model.save(\"multi_model_final\")\n\n    with open(\"training_results.txt\", \"a\") as log_file:\n        print(\"\", file=log_file)\n        print(\"  'multi_model_final'\", file=log_file)\n        print(f\" F: {m_report(history, 'loss', 'mse', 'correlation_score', 'val_loss', 'val_mse', 'val_correlation_score')}\", file=log_file, flush=True)\n        print(f\"    {m_report(history, 'loss', 'mse', 'correlation_score', 'val_loss', 'val_mse', 'val_correlation_score', item=0)}\", file=log_file, flush=True)\n\n\nif __name__ == \"__main__\":\n    # physical_devices = tf.config.list_physical_devices('GPU')\n    # tf.config.experimental.set_memory_growth(physical_devices[0], True)\n    run()\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"An attempt for PCA sequential transformation","metadata":{}},{"cell_type":"code","source":"\nimport numpy as np\nimport random\nfrom custom_metrics import *\nimport os\nfrom train_multi_data import *\n\nimport gc\n\n\nimport pickle\nimport time \nfrom datetime import timedelta\n\n\nfor version in [28]:\n    with open(f'pca_{version}.pkl', 'rb') as pickle_file:\n        pca = pickle.load(pickle_file)\n\n    t_batch_size=32\n    with h5py.File(multi_inputs_filename, 'r') as h_target:\n        with h5py.File(f'pca_train_multi_inputs_{version}.h5', 'w') as hf:\n            hf.create_dataset('train_multi_inputs/block0_values', dtype=np.float32, shape=(t_batch_size, pca.n_components), maxshape=(None, pca.n_components)) \n\n            shape = h_target['train_multi_inputs']['block0_values'].shape\n            n = t_batch_size\n            N = shape[0] // n + (shape[0] % n > 0)\n            for i in range(N):\n                start = time.time()\n                chunk = h_target['train_multi_inputs']['block0_values'][i*n:(i+1)*n,:]\n                ii = pca.transform(chunk)\n                print(ii.shape)\n                end = time.time()\n                print(f\"Took: {timedelta(seconds=end-start)}. ETA: {timedelta(seconds=(end-start)*(N-i))}\")\n\n                hf[\"train_multi_inputs/block0_values\"].resize((hf[\"train_multi_inputs/block0_values\"].shape[0] + ii.shape[0]), axis = 0)\n                hf[\"train_multi_inputs/block0_values\"][-ii.shape[0]:] = ii\n\nexit()\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#import tfx\nimport pickle\nimport tensorflow as tf\nimport numpy as np\nimport h5py\nimport hdf5plugin\nimport random\nfrom sklearn.decomposition import PCA, IncrementalPCA\nimport json\nfrom tqdm import tqdm\nimport gc\n\nmulti_inputs_filename = \"open-problems-multimodal/train_multi_inputs.h5\"\n\nclass data_gen:\n    def __init__(self, target_file_name = multi_inputs_filename, target_key=\"train_multi_inputs\", n_chunks = 100, dtype=None):\n        self.target_file_name = target_file_name\n        self.target_key = target_key\n        self.multi_inputs_filename = multi_inputs_filename\n        self.dtype = dtype\n\n        with h5py.File(target_file_name, \"r\") as f:\n            self.data_shape = f[target_key]['block0_values'].shape\n\n        self.chunk_size = self.data_shape[0] // n_chunks \n        self.n_chunks = self.data_shape[0] // self.chunk_size + (self.data_shape[0] % self.chunk_size > 0)\n\n    def __call__(self):\n        with h5py.File(self.target_file_name, 'r') as h_target:\n            for n in range(self.n_chunks):\n                data_ptr = h_target[self.target_key]['block0_values']\n                data_chunk = data_ptr[n * self.chunk_size : (n+1) * self.chunk_size, :]\n                yield np.array(data_chunk, dtype=np.float16)\n\ndef load_src_matrix(target_file_name = multi_inputs_filename, target_key=\"train_multi_inputs\", size = 0.1):\n    with h5py.File(target_file_name, \"r\") as f:\n        data_shape = f[target_key]['block0_values'].shape\n        rows = int(data_shape[0] * size)\n        data =           np.array((f[taget_key]['block0_values'])[0:rows, :], dtype=np.float32)\n\n    return data\n\n\nprint(\"loading data...\")\ndata_src = data_gen(n_chunks=50, dtype=None)\nprint(\"Data shape: \", data_src.data_shape)\n\nprint(\"Running pca...\")\nn_components = min (data_src.data_shape[1] // 2, data_src.chunk_size)\n\nprint(f\"Reducing components from: {data_src.data_shape[1]:,} to {n_components:,}\")\npca = IncrementalPCA(n_components=n_components, copy=False)\n\nfor n, chunk in tqdm(enumerate(data_src())):\n    pca.partial_fit(chunk)\n    with open(f'pca_{n}.pkl', 'wb') as pickle_file:\n            pickle.dump(pca, pickle_file)\n    print(\"explained_variance_ratio_\", pca.explained_variance_ratio_)\n    print(\"sum:\", pca.explained_variance_ratio_.sum())\n    gc.collect()\n\nprint(\"Done\")\n\n\n\n\n\n","metadata":{},"execution_count":null,"outputs":[]}]}