#!/usr/bin/env bash
#
# -----------------------------------------------------------------------------
# Gaia CDN Bulk Downloader
#
# Description:
#   This script:
#     1. Generates a list of file URLs by parsing the files listed in the
#        "_MD5SUM.txt" file within a given Gaia data catalogue directory on:
#            https://cdn.gea.esac.esa.int/
#     2. Downloads all referenced files using curl.
#
# Usage:
#   ./bulk_download.sh <CDN_URL> [--parallel [N]]
#
# Examples:
#   ./bulk_download.sh "gdr3/gaia_source/"
#   ./bulk_download.sh "gdr3/gaia_source/" --parallel
#   ./bulk_download.sh "gdr3/gaia_source/" --parallel 16
#
# Notes:
#   This script also downloads the associated "_MD5SUM.txt".
#
# -----------------------------------------------------------------------------

# -----------------------
# Argument parsing
# -----------------------
PARALLEL=0
JOBS=8   # default

if [[ $# -lt 1 ]]; then
    echo "Error: No input URL provided." >&2
    echo "Usage: $0 <CDN_DIRECTORY> [--parallel [N]]" >&2
    echo "Example (serial download):               $0 \"gdr3/gaia_source/\"" >&2
    echo "Example (download with 8 parallel jobs): $0 \"gdr3/gaia_source/\" --parallel 8" >&2
    exit 1
fi

CDN_DIR="$1"
shift

# Parse optional flags
while [[ $# -gt 0 ]]; do
    case "$1" in
        -p|--parallel)
            PARALLEL=1
            # If next arg is a number -> treat it as job count
            if [[ "$2" =~ ^[0-9]+$ ]]; then
                JOBS="$2"
                shift
            fi
            ;;
        *)
            echo "Unknown option: $1" >&2
            exit 1
            ;;
    esac
    shift
done

# -----------------------
# URL setup
# -----------------------
URL="https://cdn.gea.esac.esa.int/?prefix=Gaia/${CDN_DIR}"
URL="${URL//?prefix=/}"

FILE_LIST="file_list.txt"
MD5FILE="_MD5SUM.txt"

# -----------------------
# Build file list
# -----------------------
curl -s "${URL}${MD5FILE}" | awk -v url="$URL" '{print url $2}' > "$FILE_LIST"
echo "${URL}${MD5FILE}" >> "$FILE_LIST"


# -----------------------
# Download mode (progress bar + timing per file)
# -----------------------
download_file() {
    local url="$1"
    echo "Downloading: $url"

    start_time=$(date +%s)

    curl --progress-bar -O -L "$url"

    end_time=$(date +%s)
    elapsed=$(( end_time - start_time ))

    echo "Finished: $url  (time: ${elapsed}s)"
    echo
}

export -f download_file

if [[ $PARALLEL -eq 1 ]]; then
    echo "Downloading in PARALLEL mode with $JOBS jobs..."
    xargs -n 1 -P "$JOBS" curl -O -L < "$FILE_LIST"
else
    echo "Downloading in SERIAL mode..."
    xargs -n 1 -I {} bash -c 'download_file "$@"' _ {} < "$FILE_LIST"
fi

echo "All downloads completed."