#!/bin/bash
# Bulk-download public notebook sources, then resume the competition archive.
# Per-notebook API pulls sit on HTTP 429 for about 15s each. The Meta Kaggle
# Code zip is one transfer of the same sources.
set -u
set -o pipefail
cd /www/wwwroot/dataset/kaggle
mkdir -p meta/meta-kaggle-code meta/code_sources download_state
LOG=download_state/metacode.log
{
  set -e
  echo "START $(date -Is)"
  kaggle datasets download -d kaggle/meta-kaggle-code -p meta/meta-kaggle-code &
  download_pid=$!
  python3 -u download_competitions.py --build-kernel-versions
  echo "VERSIONS_DONE $(date -Is)"
  if ! wait "$download_pid"; then
    echo "DOWNLOAD_FAILED $(date -Is)"
    exit 1
  fi
  echo "DOWNLOAD_DONE $(date -Is)"
  python3 -u extract_kernel_sources.py
  echo "EXTRACT_DONE $(date -Is)"
  python3 -u download_competitions.py --build-forum-db
  echo "FORUM_DONE $(date -Is)"
  python3 -u download_competitions.py --all
  echo "ALL_DONE $(date -Is)"
} 2>&1 | tee -a "$LOG"