Mirror the Entire PyPI Repository with Bash

#!/bin/bash

# Create the mirror directory
mkdir -p ./pypi_mirror

# Log file to track last mirrored package
LOG_FILE="./pypi_mirror/index.log"

# Get the list of all package names (strip "/simple/")
packages=($(curl -s https://pypi.org/simple/ | awk -F '"' '/href="/ {print $2}' | sed 's|/simple/||g' | sed 's|/$||'))

# Get the total number of packages
total_packages=${#packages[@]}
start_time=$SECONDS

echo "Total packages to download: $total_packages"
echo ""

# Read last completed package from log
if [[ -f "$LOG_FILE" ]]; then
    last_package=$(tail -n 1 "$LOG_FILE")
    echo "Resuming from package: $last_package"
    skip=true
else
    last_package=""
    skip=false
fi

# Loop through each package and download all available versions
for i in "${!packages[@]}"; do
    package="${packages[$i]}"

    # Skip previously completed packages
    if [[ "$skip" == true ]]; then
        if [[ "$package" == "$last_package" ]]; then
            skip=false  # Found the last completed package, start from the next one
        fi
        continue
    fi

    # Update progress
    progress=$(( (i + 1) * 100 / total_packages ))
    elapsed_time=$(( SECONDS - start_time ))
    avg_time_per_pkg=$(( elapsed_time / (i + 1) ))
    remaining_pkgs=$(( total_packages - i - 1 ))
    eta=$(( avg_time_per_pkg * remaining_pkgs ))

    # Prevent negative ETA
    if [[ $eta -lt 0 ]]; then eta=0; fi

    # Progress bar settings
    bar_length=40
    filled_length=$(( bar_length * (i + 1) / total_packages ))

    # Ensure at least 1 character for cut
    if [[ $filled_length -lt 1 ]]; then filled_length=1; fi

    # Construct progress bar
    bar=$(printf "%-${bar_length}s" "█████████████████████████████████████████" | cut -c1-"$filled_length")
    empty_bar=$(printf "%-${bar_length}s" "")

    # Print progress dynamically
    tput sc
    echo -ne "Progress: [$bar$empty_bar] $progress% | Elapsed: ${elapsed_time}s | ETA: ${eta}s | Downloading: $package\r"
    tput rc

    # Create a directory for the package
    mkdir -p "./pypi_mirror/$package"

    # Get the list of package files from PyPI
    package_page=$(curl -s "https://pypi.org/simple/$package/")

    # Extract all file URLs
    urls=$(echo "$package_page" | awk -F '"' '/href="https/ {print $2}')

    if [[ -z "$urls" ]]; then
        continue  # Skip if no files found
    fi

    # Download each file (silent mode to keep terminal clean)
    for url in $urls; do
        cleaned_url="${url%%#*}"
        file_name="./pypi_mirror/$package/$(basename "$cleaned_url")"
        
        # Check if the file already exists and is not empty
        if [[ -f "$file_name" && -s "$file_name" ]]; then
            echo "Skipping already downloaded file: $file_name"
            continue
        fi
        
        wget -q -P "./pypi_mirror/$package/" "$url"
    done

    # Log the completed package
    echo "$package" >> "$LOG_FILE"
done

# Final message
echo -e "\n\n🎉 PyPI mirroring complete! All $total_packages packages downloaded."