How to Download Files via cURL With Battle Ready Examples (2024)

curl --version

mkdir ~/curl-downloadscd ~/curl-downloads

sudo apt install ca-certificates

curl -O https://example.com/file.zip

curl [options] [URL]

curl -O https://example.com/awesome-file.pdf

curl -o my-awesome-name.pdf https://example.com/boring-original-name.pdf

# Real-world example from our financial data scrapingcurl -o "company_report_$(date +%Y%m%d).pdf" https://example.com/report.pdf

# Create directory structure automaticallycurl -o "./downloads/reports/2024/Q1/report.pdf" https://example.com/report.pdf --create-dirs# Use content-disposition header (when available)curl -OJ https://example.com/mystery-file

# Follow redirects like a procurl -L -O https://example.com/file-that-moves-around# See where your file leads youcurl -IL https://example.com/mysterious-file

# Complete silence (no progress or error messages)curl -s -O https://example.com/quiet-file.zip# Silent but shows errors (my personal favorite)curl -sS -O https://example.com/important-file.zip# Silent with custom error redirectioncurl -s -O https://example.com/file.zip 2>errors.log

# Classic progress barcurl -# -O https://example.com/big-file.zip# Fancy progress meter with statscurl --progress-bar -O https://example.com/big-file.zip# Custom progress format (my favorite for scripting)curl -O https://example.com/big-file.zip \ --progress-bar \ --write-out "\nDownload completed!\nAverage Speed: %{speed_download}bytes/sec\nTime: %{time_total}s\n"

# Resume a partial downloadcurl -C - -O https://example.com/massive-dataset.zip# Check file size before resumingcurl -I https://example.com/massive-dataset.zip# Resume with retry logic (battle-tested version)curl -C - --retry 3 --retry-delay 5 -O https://example.com/massive-dataset.zip

#!/bin/bashdownload_with_resume() { local url=$1 local max_retries=3 local retry_count=0 while [ $retry_count -lt $max_retries ]; do curl -C - --retry 3 --retry-delay 5 -O "$url" if [ $? -eq 0 ]; then echo "Download completed successfully! 🎉" return 0 fi let retry_count++ echo "Retry $retry_count of $max_retries... 🔄" sleep 5 done return 1}# Usagedownload_with_resume "https://example.com/huge-file.zip"

curl -C - \ --retry 3 \ --retry-delay 5 \ -# \ -o "./downloads/$(date +%Y%m%d)_${filename}" \ --create-dirs \ -L \ "https://example.com/file.zip"

alias superdownload='function _dl() { curl -C - --retry 3 --retry-delay 5 -# -o "./downloads/$(date +%Y%m%d)_$2" --create-dirs -L "$1"; };_dl'

superdownload https://example.com/file.zip custom_name

# Basic authentication (the classic way)curl -u username:password -O https://secure-site.com/file.zip# Using netrc file (my preferred method for automation)echo "machine secure-site.com login myuser password mypass" >> ~/.netrcchmod 600 ~/.netrc # Important security step!curl -n -O https://secure-site.com/file.zip

# Create a secure credential handlercurl -u "$(security find-generic-password -a $USER -s "api-access" -w)"

# Save cookiescurl -c cookies.txt -O https://example.com/login-required-file.zip# Use saved cookiescurl -b cookies.txt -O https://example.com/another-file.zip# The power combo (save and use in one go)curl -b cookies.txt -c cookies.txt -O https://example.com/file.zip

#!/bin/bashSESSION_HANDLER() { local username="$1" local password="$2" local cookie_file=$(mktemp) local max_retries=3 local retry_delay=2  # Input validation if [[ -z "$username" || -z "$password" ]]; then echo "❌ Error: Username and password are required" rm -f "$cookie_file" return 1 }  echo "🔐 Initiating secure session..."  # First, login and save cookies with error handling if ! curl -s -c "$cookie_file" \ --connect-timeout 10 \ --max-time 30 \ --retry $max_retries \ --retry-delay $retry_delay \ --fail-with-body \ -d "username=${username}&password=${password}" \ "https://example.com/login" > /dev/null 2>&1; then echo "❌ Login failed!" rm -f "$cookie_file" return 1 fi  echo "✅ Login successful"  # Verify cookie file existence and content if [[ ! -s "$cookie_file" ]]; then echo "❌ No cookies were saved" rm -f "$cookie_file" return 1 fi  # Now download with session echo "📥 Downloading protected file..." if ! curl -sS -b "$cookie_file" \ -O \ --retry $max_retries \ --retry-delay $retry_delay \ --connect-timeout 10 \ --max-time 300 \ "https://example.com/protected-file.zip"; then echo "❌ Download failed" rm -f "$cookie_file" return 1 fi  echo "✅ Download completed successfully"  # Cleanup rm -f "$cookie_file" return 0}# Usage ExampleSESSION_HANDLER "myusername" "mypassword"

# Single headercurl -H "User-Agent: Mozilla/5.0" -O https://example.com/file.zip# Multiple headers (production-ready version)curl -H "User-Agent: Mozilla/5.0" \ -H "Accept: application/pdf" \ -H "Referer: https://example.com" \ -O https://example.com/file.pdf

#!/bin/bash# Define browser headersUSER_AGENTS=( "Mozilla/5.0 (Windows NT 10.0; Win64; x64) Chrome/120.0.0.0" "Mozilla/5.0 (Windows NT 10.0; Firefox/121.0)" "Mozilla/5.0 (Macintosh; Safari/605.1.15)")SMART_DOWNLOAD() { local url="$1" local output="$2"  # Select random User-Agent local agent=${USER_AGENTS[$RANDOM % ${#USER_AGENTS[@]}]}  # Execute download with common headers curl --fail \ -H "User-Agent: $agent" \ -H "Accept: text/html,application/xhtml+xml,*/*" \ -H "Accept-Language: en-US,en;q=0.9" \ -H "Connection: keep-alive" \ -o "$output" \ "$url"  # Be nice to servers sleep 1}# Usage ExampleSMART_DOWNLOAD "https://example.com/file.pdf" "downloaded.pdf"

# Basic proxy usagecurl -x proxy.example.com:8080 -O https://example.com/file.zip# Authenticated proxy (my preferred setup)curl -x "username:password@proxy.example.com:8080" \ -O https://example.com/file.zip# SOCKS5 proxy (for extra sneakiness)curl --socks5 proxy.example.com:1080 -O https://example.com/file.zip

#!/bin/bash# Proxy configurations with authenticationdeclare -A PROXY_CONFIGS=( ["proxy1"]="username1:password1@proxy1.example.com:8080" ["proxy2"]="username2:password2@proxy2.example.com:8080" ["proxy3"]="username3:password3@proxy3.example.com:8080")PROXY_DOWNLOAD() { local url="$1" local output="$2" local max_retries=3 local retry_delay=2 local timeout=30 local temp_log=$(mktemp)  # Input validation if [[ -z "$url" ]]; then echo "❌ Error: URL is required" rm -f "$temp_log" return 1 }  # Get proxy list keys local proxy_keys=("${!PROXY_CONFIGS[@]}")  for ((retry=0; retry<max_retries; retry++)); do # Select random proxy local selected_proxy=${proxy_keys[$RANDOM % ${#proxy_keys[@]}]} local proxy_auth="${PROXY_CONFIGS[$selected_proxy]}"  echo "🔄 Attempt $((retry + 1))/$max_retries using proxy: ${selected_proxy}"  # Test proxy before download if curl --connect-timeout 5 \ -x "$proxy_auth" \ -s "https://api.ipify.org" > /dev/null 2>&1; then echo "✅ Proxy connection successful"  # Attempt download with the working proxy if curl -x "$proxy_auth" \ --connect-timeout "$timeout" \ --max-time $((timeout * 2)) \ --retry 2 \ --retry-delay "$retry_delay" \ --fail \ --silent \ --show-error \ -o "$output" \ "$url" 2> "$temp_log"; then echo "✅ Download completed successfully using ${selected_proxy}" rm -f "$temp_log" return 0 else echo "⚠️ Download failed with ${selected_proxy}" cat "$temp_log" fi else echo "⚠️ Proxy ${selected_proxy} is not responding" fi  # Wait before trying next proxy if ((retry < max_retries - 1)); then local wait_time=$((retry_delay * (retry + 1))) echo "⏳ Waiting ${wait_time}s before next attempt..." sleep "$wait_time" fi done  echo "❌ All proxy attempts failed" rm -f "$temp_log" return 1}# Usage ExamplePROXY_DOWNLOAD "https://example.com/file.zip" "downloaded_file.zip"

#!/bin/bash# SOCKS5 specific download functionSOCKS5_DOWNLOAD() { local url="$1" local output="$2" local socks5_proxy="$3"  echo "🧦 Using SOCKS5 proxy: $socks5_proxy"  if curl --socks5 "$socks5_proxy" \ --connect-timeout 10 \ --max-time 60 \ --retry 3 \ --retry-delay 2 \ --fail \ --silent \ --show-error \ -o "$output" \ "$url"; then echo "✅ SOCKS5 download successful" return 0 else echo "❌ SOCKS5 download failed" return 1 fi}# Usage ExampleSOCKS5_DOWNLOAD "https://example.com/file.zip" "output.zip" "proxy.example.com:1080"

#!/bin/bash# Batch download with proxy rotationBATCH_PROXY_DOWNLOAD() { local -a urls=("$@") local success_count=0 local fail_count=0  echo "📦 Starting batch download with proxy rotation..."  for url in "${urls[@]}"; do local filename="${url##*/}" if PROXY_DOWNLOAD "$url" "$filename"; then ((success_count++)) else ((fail_count++)) echo "⚠️ Failed to download: $url" fi done  echo "📊 Download Summary:" echo "✅ Successful: $success_count" echo "❌ Failed: $fail_count"}# Usage ExampleBATCH_PROXY_DOWNLOAD "https://example.com/file1.zip" "https://example.com/file2.zip"

# Limit download speed (1M = 1MB/s)curl --limit-rate 1M -O https://example.com/huge-file.zip# Bandwidth control with retry logic (500KB/s limit)curl --limit-rate 500k \ --retry 3 \ --retry-delay 5 \ -C - \ -O https://example.com/huge-file.zip

#!/bin/bashadaptive_download() { local url="$1" local output="$2" local base_rate="500k" local retry_count=0 local max_retries=3 local backoff_delay=5  while [ $retry_count -lt $max_retries ]; do if curl --limit-rate $base_rate \ --connect-timeout 10 \ --max-time 3600 \ -o "$output" \ -C - \ "$url"; then echo "✅ Download successful!" return 0 else base_rate="250k" # Reduce speed on failure let retry_count++ echo "⚠️ Retry $retry_count with reduced speed: $base_rate" sleep $((backoff_delay * retry_count)) fi done return 1}# Usage Exampleadaptive_download "https://example.com/huge-file.zip" "my-download.zip"

exec 1> >(tee -a "${LOG_DIR}/download_$(date +%Y%m%d).log")exec 2>&1

curl -v --tlsv1.2 -O https://example.com/file.zip

curl -I --proxy-insecure https://example.com/file.zip

curl -v --proxy proxy.example.com:8080 https://example.com/file.zip >/dev/null

#!/bin/bashSMART_DOWNLOAD() { local url="$1" local output_name="$2" local proxy="${PROXY_LIST[RANDOM % ${#PROXY_LIST[@]}]}"  curl -x "$proxy" \ --limit-rate 1M \ --retry 3 \ --retry-delay 5 \ -C - \ -# \ -H "User-Agent: ${USER_AGENTS[RANDOM % ${#USER_AGENTS[@]}]}" \ -b cookies.txt \ -c cookies.txt \ --create-dirs \ -o "./downloads/$(date +%Y%m%d)_${output_name}" \ "$url"}# Usage Examples# Single file download with all the bells and whistlesSMART_DOWNLOAD "https://example.com/dataset.zip" "important_dataset.zip"# Multiple filesfor file in file1.zip file2.zip file3.zip; do SMART_DOWNLOAD "https://example.com/$file" "$file" sleep 2 # Be nice to serversdone

# From a file containing URLs (my most-used method)while read url; do curl -O "$url"done < urls.txt# Multiple files from same domain (clean and simple)# Downloads: file1.pdf, file2.pdf, file3.pdfcurl -O http://example.com/file[1-100].pdf

#!/bin/bashBATCH_DOWNLOAD() { local url="$1" local retries=3 local wait_time=2  echo "🎯 Downloading: $url"  for ((i=1; i<=retries; i++)); do if curl -sS --fail \ --retry-connrefused \ --connect-timeout 10 \ --max-time 300 \ -O "$url"; then echo "✅ Success: $url" return 0 else echo "⚠️ Attempt $i failed, waiting ${wait_time}s..." sleep $wait_time wait_time=$((wait_time * 2)) # Exponential backoff fi done  echo "❌ Failed after $retries attempts: $url" return 1}# Usage ExampleBATCH_DOWNLOAD "https://example.com/large-file.zip"# Or in a loop:while read url; do BATCH_DOWNLOAD "$url"done < urls.txt

#!/bin/bashPROCESS_URL_LIST() { trap 'echo "⚠️ Process interrupted"; exit 1' SIGINT SIGTERM local input_file="$1" local success_count=0 local total_urls=$(wc -l < "$input_file")  echo "🚀 Starting batch download of $total_urls files..."  while IFS='' read -r url || [[ -n "$url" ]]; do if [[ "$url" =~ ^#.*$ ]] || [[ -z "$url" ]]; then continue # Skip comments and empty lines fi  if BATCH_DOWNLOAD "$url"; then ((success_count++)) printf "Progress: [%d/%d] (%.2f%%)\n" \ $success_count $total_urls \ $((success_count * 100 / total_urls)) fi done < "$input_file"  echo "✨ Download complete! Success rate: $((success_count * 100 / total_urls))%"}

# Add to the script abovefailed_urls=()if ! BATCH_DOWNLOAD "$url"; then failed_urls+=("$url") echo "$url" >> failed_downloads.txtfi

# Basic recursive downloadcurl -r -O 'http://example.com/files/{file1,file2,file3}.pdf'# Advanced recursive download with pattern matching

#!/bin/bashPARALLEL_DOWNLOAD() { trap 'kill $(jobs -p) 2>/dev/null; exit 1' SIGINT SIGTERM  local max_parallel=5 # Adjust based on your needs local active_downloads=0  while read -r url; do # Check if we've hit our parallel limit while [ $active_downloads -ge $max_parallel ]; do wait -n # Wait for any child process to finish ((active_downloads--)) done  # Start new download in background ( if curl -sS --fail -O "$url"; then echo "✅ Success: $url" else echo "❌ Failed: $url" echo "$url" >> failed_urls.txt fi ) &  ((active_downloads++)) echo "🚀 Started download: $url (Active: $active_downloads)" done < urls.txt  # Wait for remaining downloads wait}# Usage Exampleecho "https://example.com/file1.ziphttps://example.com/file2.zip" > urls.txtPARALLEL_DOWNLOAD

# Add dynamic throttling based on failure rateSMART_PARALLEL_DOWNLOAD() { local fail_count=0 local total_count=0 local max_parallel=5  monitor_failures() { if [ $((total_count % 10)) -eq 0 ]; then local failure_rate=$((fail_count * 100 / total_count)) if [ $failure_rate -gt 20 ]; then ((max_parallel--)) echo "⚠️ High failure rate detected! Reducing parallel downloads to $max_parallel" fi fi } # ... rest of parallel download logic}

#!/bin/bashDOWNLOAD_WITH_ERROR_HANDLING() { trap 'rm -f "$temp_file"' EXIT local url="$1" local retry_count=0 local max_retries=3 local backoff_time=5 local temp_file=$(mktemp)  while [ $retry_count -lt $max_retries ]; do if curl -sS \ --fail \ --connect-timeout 15 \ --max-time 300 \ --retry 3 \ --retry-delay 5 \ -o "$temp_file" \ "$url"; then  # Verify file integrity if [ -s "$temp_file" ]; then mv "$temp_file" "$(basename "$url")" echo "✅ Download successful: $url" return 0 else echo "⚠️ Downloaded file is empty" fi fi  ((retry_count++)) echo "🔄 Retry $retry_count/$max_retries for $url" sleep $((backoff_time * retry_count)) # Exponential backoff done  rm -f "$temp_file" return 1}# Usage ExampleDOWNLOAD_WITH_ERROR_HANDLING "https://example.com/large-file.dat"

#!/bin/bashMASTER_BATCH_DOWNLOAD() { set -eo pipefail # Exit on error trap 'kill $(jobs -p) 2>/dev/null; echo "⚠️ Process interrupted"; exit 1' SIGINT SIGTERM local url_file="$1" local max_parallel=5 local success_count=0 local fail_count=0  # Setup logging local log_dir="logs/$(date +%Y%m%d_%H%M%S)" mkdir -p "$log_dir" exec 1> >(tee -a "${log_dir}/download.log") exec 2>&1  echo "🚀 Starting batch download at $(date)"  # Process URLs in parallel with smart throttling cat "$url_file" | while read -r url; do while [ $(jobs -p | wc -l) -ge $max_parallel ]; do wait -n done  ( if DOWNLOAD_WITH_ERROR_HANDLING "$url"; then echo "$url" >> "${log_dir}/success.txt" ((success_count++)) else echo "$url" >> "${log_dir}/failed.txt" ((fail_count++)) fi  # Progress update total=$((success_count + fail_count)) echo "Progress: $total files processed (Success: $success_count, Failed: $fail_count)" ) & done  wait # Wait for all downloads to complete  # Generate report echo " 📊 Download Summary ================== Total Files: $((success_count + fail_count)) Successful: $success_count Failed: $fail_count Success Rate: $((success_count * 100 / (success_count + fail_count)))%  Log Location: $log_dir ================== "}# Usage Exampleecho "https://example.com/file1.ziphttps://example.com/file2.ziphttps://example.com/file3.zip" > batch_urls.txt# Execute master downloadMASTER_BATCH_DOWNLOAD "batch_urls.txt"

#!/bin/bashIMAGE_BATCH_DOWNLOAD() { local url_list="$1" local output_dir="images/$(date +%Y%m%d)" local log_dir="logs/$(date +%Y%m%d)" local max_size=$((10*1024*1024)) # 10MB limit by default  # Setup directories mkdir -p "$output_dir" "$log_dir"  # Initialize counters declare -A stats=([success]=0 [failed]=0 [invalid]=0 [oversized]=0)  # Cleanup function cleanup() { local exit_code=$? rm -f "$temp_file" 2>/dev/null echo "🧹 Cleaning up temporary files..." exit $exit_code }  # Set trap for cleanup trap cleanup EXIT INT TERM  validate_image() { local file="$1" local mime_type=$(file -b --mime-type "$file") local file_size=$(stat -f%z "$file" 2>/dev/null || stat -c%s "$file")  # Check file size if [ "$file_size" -gt "$max_size" ]; then ((stats[oversized]++)) echo "⚠️ File too large: $file_size bytes (max: $max_size bytes)" return 1 }  case "$mime_type" in image/jpeg|image/png|image/gif|image/webp) return 0 ;; *) return 1 ;; esac }  download_image() { local url="$1" local filename=$(basename "$url") local temp_file=$(mktemp)  echo "🎯 Downloading: $url"  if curl -sS --fail \ --retry 3 \ --retry-delay 2 \ --max-time 30 \ -o "$temp_file" \ "$url"; then  if validate_image "$temp_file"; then mv "$temp_file" "$output_dir/$filename" ((stats[success]++)) echo "✅ Success: $filename" return 0 else rm "$temp_file" ((stats[invalid]++)) echo "⚠️ Invalid image type or size: $url" return 1 fi else rm "$temp_file" ((stats[failed]++)) echo "❌ Download failed: $url" return 1 fi }}# Usage Examples# Create a file with image URLsecho "https://example.com/image1.jpghttps://example.com/image2.png" > url_list.txt# Execute with default 10MB limitIMAGE_BATCH_DOWNLOAD "url_list.txt"# Or modify max_size variable for different limitmax_size=$((20*1024*1024)) IMAGE_BATCH_DOWNLOAD "url_list.txt" # 20MB limit

#!/bin/bashAPI_DATA_DOWNLOAD() { local api_base="https://api.example.com" local token="" local rate_limit=60 # requests per minute local last_token_refresh=0 local log_file="api_downloads.log" local max_retries=3  log_message() { local timestamp=$(date '+%Y-%m-%d %H:%M:%S') echo "[$timestamp] $1" | tee -a "$log_file" }  refresh_token() { local refresh_response refresh_response=$(curl -sS \ -X POST \ -H "Content-Type: application/json" \ --max-time 30 \ -d '{"key": "YOUR_API_KEY"}' \ "${api_base}/auth")  if [ $? -ne 0 ]; then log_message "❌ Token refresh failed: Network error" return 1 fi  token=$(echo "$refresh_response" | jq -r '.token') if [ -z "$token" ] || [ "$token" = "null" ]; then log_message "❌ Token refresh failed: Invalid response" return 1 }  last_token_refresh=$(date +%s) log_message "✅ Token refreshed successfully" return 0 }  calculate_backoff() { local retry_count=$1 echo $((2 ** (retry_count - 1) * 5)) # Exponential backoff: 5s, 10s, 20s... }  download_data() { local endpoint="$1" local output_file="$2" local retry_count=0 local success=false  # Check token age local current_time=$(date +%s) if [ $((current_time - last_token_refresh)) -gt 3600 ]; then log_message "🔄 Token expired, refreshing..." refresh_token || return 1 fi  # Rate limiting sleep $(( 60 / rate_limit ))  while [ $retry_count -lt $max_retries ] && [ "$success" = false ]; do local response response=$(curl -sS \ -H "Authorization: Bearer $token" \ -H "Accept: application/json" \ --max-time 30 \ "${api_base}${endpoint}")  if [ $? -eq 0 ] && [ "$(echo "$response" | jq -e 'type')" = "object" ]; then echo "$response" > "$output_file" log_message "✅ Successfully downloaded data to $output_file" success=true else retry_count=$((retry_count + 1)) if [ $retry_count -lt $max_retries ]; then local backoff_time=$(calculate_backoff $retry_count) log_message "⚠️ Attempt $retry_count failed. Retrying in ${backoff_time}s..." sleep $backoff_time else log_message "❌ Download failed after $max_retries attempts" return 1 fi fi done  return 0 }}# Usage Examples# Initialize the functionAPI_DATA_DOWNLOAD# Download data from specific endpointdownload_data "/v1/users" "users_data.json"# Download with custom rate limitrate_limit=30 download_data "/v1/transactions" "transactions.json"

#!/bin/bashFTP_BATCH_DOWNLOAD() { local host="$1" local user="$2" local pass="$3" local remote_dir="$4" local local_dir="downloads/ftp/$(date +%Y%m%d)" local log_file="$local_dir/ftp_transfer.log" local netrc_file local status=0  # Create secure temporary .netrc file netrc_file=$(mktemp)  log_message() { local timestamp=$(date '+%Y-%m-%d %H:%M:%S') echo "[$timestamp] $1" | tee -a "$log_file" }  cleanup() { local exit_code=$? if [ -f "$netrc_file" ]; then shred -u "$netrc_file" 2>/dev/null || rm -P "$netrc_file" 2>/dev/null || rm "$netrc_file" fi log_message "🧹 Cleanup completed" exit $exit_code }  validate_downloads() { local failed_files=0 log_message "🔍 Validating downloaded files..."  while IFS= read -r file; do if [ ! -s "$file" ]; then log_message "⚠️ Empty or invalid file: $file" ((failed_files++)) fi done < <(find "$local_dir" -type f -not -name "*.log")  return $failed_files }  # Set trap for cleanup trap cleanup EXIT INT TERM  # Create directories mkdir -p "$local_dir"  # Create .netrc with secure permissions umask 077 echo "machine $host login $user password $pass" > "$netrc_file"  log_message "🚀 Starting FTP download from $host..." log_message "📁 Remote directory: $remote_dir"  # Download with enhanced options curl --retry 3 \ --retry-delay 10 \ --retry-all-errors \ --ftp-create-dirs \ --create-dirs \ --connect-timeout 30 \ --max-time 3600 \ -C - \ --netrc-file "$netrc_file" \ --stderr - \ --progress-bar \ "ftp://$host/$remote_dir/*" \ --output "$local_dir/#1" 2>&1 | tee -a "$log_file"  status=$?  if [ $status -eq 0 ]; then # Validate downloads validate_downloads local validation_status=$?  if [ $validation_status -eq 0 ]; then log_message "✅ FTP download completed successfully!" else log_message "⚠️ Download completed but $validation_status files failed validation" status=1 fi else log_message "❌ FTP download failed with status: $status" fi  return $status}# Usage ExampleFTP_BATCH_DOWNLOAD "ftp.example.com" "username" "password" "remote/directory"# With error handlingif FTP_BATCH_DOWNLOAD "ftp.example.com" "username" "password" "remote/directory"; then echo "Transfer successful"else echo "Transfer failed"fi

#!/bin/bashLARGE_FILE_DOWNLOAD() { local url="$1" local filename="$2" local min_speed=1000 # 1KB/s minimum local timeout=300 # 5 minutes local chunk_size="10M"  echo "🎯 Starting large file download: $filename"  # Create temporary directory for chunks local temp_dir=$(mktemp -d) local final_file="downloads/large/$filename" mkdir -p "$(dirname "$final_file")"  download_chunk() { local start=$1 local end=$2 local chunk_file="$temp_dir/chunk_${start}-${end}"  curl -sS \ --range "$start-$end" \ --retry 3 \ --retry-delay 5 \ --speed-limit $min_speed \ --speed-time $timeout \ -o "$chunk_file" \ "$url"  return $? }  # Get file size local size=$(curl -sI "$url" | grep -i content-length | awk '{print $2}' | tr -d '\r') local chunks=$(( (size + (1024*1024*10) - 1) / (1024*1024*10) ))  echo "📦 File size: $(( size / 1024 / 1024 ))MB, Split into $chunks chunks"  # Download chunks in parallel for ((i=0; i<chunks; i++)); do local start=$((i * 1024 * 1024 * 10)) local end=$(( (i + 1) * 1024 * 1024 * 10 - 1 )) if [ $end -ge $size ]; then end=$((size - 1)) fi  (download_chunk $start $end) &  # Limit parallel downloads if [ $((i % 5)) -eq 0 ]; then wait fi done  wait # Wait for all chunks  # Combine chunks echo "🔄 Combining chunks..." cat "$temp_dir"/chunk_* > "$final_file"  # Verify file size local downloaded_size=$(stat --format=%s "$final_file" 2>/dev/null || stat -f %z "$final_file") if [ "$downloaded_size" -eq "$size" ]; then echo "✅ Download complete and verified: $filename" rm -rf "$temp_dir" return 0 else echo "❌ Size mismatch! Expected: $size, Got: $downloaded_size" rm -rf "$temp_dir" return 1 fi}# Usage ExampleLARGE_FILE_DOWNLOAD "https://example.com/huge.zip" "backup.zip"

#!/bin/bashmonitor_progress() { local file="$1" local total_size="$2" local timeout="${3:-3600}" # Default 1 hour timeout  local start_time=$(date +%s) local last_size=0 local last_check_time=$start_time  # Function to format sizes format_size() { local size=$1 if [ $size -ge $((1024*1024*1024)) ]; then printf "%.1fGB" $(echo "scale=1; $size/1024/1024/1024" | bc) elif [ $size -ge $((1024*1024)) ]; then printf "%.1fMB" $(echo "scale=1; $size/1024/1024" | bc) else printf "%.1fKB" $(echo "scale=1; $size/1024" | bc) fi }  # Function to draw progress bar draw_progress_bar() { local percentage=$1 local width=50 local completed=$((percentage * width / 100)) local remaining=$((width - completed))  printf "[" printf "%${completed}s" | tr " " "=" printf ">" printf "%${remaining}s" | tr " " " " printf "] " }  # Check if file exists if [ ! -f "$file" ]; then echo "❌ Error: File '$file' not found!" return 1 }  # Check if total size is valid if [ $total_size -le 0 ]; then echo "❌ Error: Invalid total size!" return 1 }  echo "🚀 Starting progress monitoring..."  # Loop to continuously check the file size while true; do # Get the current file size local current_size=$(stat --format=%s "$file" 2>/dev/null || stat -f %z "$file" 2>/dev/null || echo 0) local current_time=$(date +%s)  # Calculate time elapsed local elapsed=$((current_time - start_time))  # Check timeout if [ $elapsed -gt $timeout ]; then echo -e "\n⚠️ Monitoring timed out after $(($timeout/60)) minutes" return 1 }  # Calculate speed and ETA local time_diff=$((current_time - last_check_time)) local size_diff=$((current_size - last_size)) if [ $time_diff -gt 0 ]; then local speed=$((size_diff / time_diff)) local remaining_size=$((total_size - current_size)) local eta=$((remaining_size / speed)) fi  # Calculate percentage local percentage=0 if [ $total_size -gt 0 ]; then percentage=$(( (current_size * 100) / total_size )) fi  # Clear line and show progress echo -ne "\r\033[K"  # Draw progress bar draw_progress_bar $percentage  # Show detailed stats printf "%3d%% " $percentage printf "$(format_size $current_size)/$(format_size $total_size) "  if [ $speed ]; then printf "@ $(format_size $speed)/s " if [ $eta -gt 0 ]; then printf "ETA: %02d:%02d " $((eta/60)) $((eta%60)) fi fi  # Check if download is complete if [ "$current_size" -ge "$total_size" ]; then echo -e "\n✅ Download complete! Total time: $((elapsed/60))m $((elapsed%60))s" return 0 fi  # Update last check values last_size=$current_size last_check_time=$current_time  # Wait before next check sleep 1 done}# Usage Examples# Basic usage (monitoring a 100MB download)monitor_progress "downloading_file.zip" 104857600# With custom timeout (2 hours)monitor_progress "large_file.iso" 1073741824 7200# Integration with curlcurl -o "download.zip" "https://example.com/file.zip" &monitor_progress "download.zip" $(curl -sI "https://example.com/file.zip" | grep -i content-length | awk '{print $2}' | tr -d '\r')

# 1. Enable compression (huge performance boost!)curl --compressed -O https://example.com/file.zip# 2. Use keepalive connectionscurl --keepalive-time 60 -O https://example.com/file.zip# 3. Optimize DNS resolutioncurl --resolve example.com:443:1.2.3.4 -O https://example.com/file.zip

#!/bin/bashOPTIMIZED_DOWNLOAD() { local url="$1" local output="$2"  # Pre-resolve DNS local domain=$(echo "$url" | awk -F[/:] '{print $4}') local ip=$(dig +short "$domain" | head -n1)  # Performance optimization flags local opts=( --compressed --keepalive-time 60 --resolve "${domain}:443:${ip}" --connect-timeout 10 --max-time 300 --retry 3 --retry-delay 5 )  echo "🚀 Downloading with optimizations..." curl "${opts[@]}" -o "$output" "$url"}# Usage Examples# Basic optimized downloadOPTIMIZED_DOWNLOAD "https://example.com/large-file.zip" "downloaded-file.zip"# Multiple files (using xargs for parallel downloads)cat url-list.txt | xargs -P 4 -I {} sh -c 'OPTIMIZED_DOWNLOAD "{}" "$(basename {})"'# With custom namingOPTIMIZED_DOWNLOAD "https://example.com/data.zip" "backup-$(date +%Y%m%d).zip"

#!/bin/bashROBUST_DOWNLOAD() { local url="$1" local retries=3 local timeout=30 local success=false  # Trap cleanup on script exit trap 'cleanup' EXIT  cleanup() { rm -f "$temp_file" [ "$success" = true ] || echo "❌ Download failed for $url" }  verify_download() { local file="$1" local expected_size="$2"  # Basic existence check [ -f "$file" ] || return 1  # Size verification (must be larger than 0 bytes) [ -s "$file" ] || return 1  # If expected size is provided, verify it matches if [ -n "$expected_size" ]; then local actual_size=$(stat --format=%s "$file" 2>/dev/null || stat -f %z "$file" 2>/dev/null) [ "$actual_size" = "$expected_size" ] || return 1 fi  # Basic content verification head -c 512 "$file" | grep -q '[^\x00]' || return 1  return 0 }  # Main download logic with comprehensive error handling local temp_file=$(mktemp) local output_file=$(basename "$url")  for ((i=1; i<=retries; i++)); do echo "🔄 Attempt $i of $retries"  if curl -sS \ --fail \ --show-error \ --location \ --connect-timeout "$timeout" \ -o "$temp_file" \ "$url" 2>error.log; then  if verify_download "$temp_file"; then mv "$temp_file" "$output_file" success=true echo "✅ Download successful! Saved as $output_file" break else echo "⚠️ File verification failed, retrying..." sleep $((2 ** i)) # Exponential backoff fi else echo "⚠️ Error on attempt $i:" cat error.log sleep $((2 ** i)) fi done}# Usage Examples# Basic download with error handlingROBUST_DOWNLOAD "https://example.com/file.zip"# Integration with size verificationexpected_size=$(curl -sI "https://example.com/file.zip" | grep -i content-length | awk '{print $2}' | tr -d '\r')ROBUST_DOWNLOAD "https://example.com/file.zip" "$expected_size"# Multiple files with error handlingcat urls.txt | while read url; do ROBUST_DOWNLOAD "$url" sleep 2 # Polite delay between downloadsdone

#!/bin/bashSECURE_DOWNLOAD() { local url="$1" local output="$2" local expected_hash="$3"  # Security configuration local security_opts=(  --ssl-reqd --tlsv1.2 --ssl-no-revoke --cert-status --remote-header-name --proto '=https' )  # Hash verification function verify_checksum() { local file="$1" local expected_hash="$2"  if [ -z "$expected_hash" ]; then echo "⚠️ No checksum provided. Skipping verification." return 0 fi  local actual_hash actual_hash=$(sha256sum "$file" | cut -d' ' -f1)  if [[ "$actual_hash" == "$expected_hash" ]]; then return 0 else echo "❌ Checksum verification failed!" return 1 fi }  echo "🔒 Initiating secure download..."  # Create secure temporary directory local temp_dir=$(mktemp -d) chmod 700 "$temp_dir"  # Set temporary filename local temp_file="${temp_dir}/$(basename "$output")"  # Execute secure download if curl "${security_opts[@]}" \ -o "$temp_file" \ "$url"; then  # Verify file integrity if verify_checksum "$temp_file" "$expected_hash"; then mv "$temp_file" "$output" echo "✅ Secure download complete! Saved as $output" else echo "❌ Integrity check failed. File removed." rm -rf "$temp_dir" return 1 fi else echo "❌ Download failed for $url" rm -rf "$temp_dir" return 1 fi  # Cleanup rm -rf "$temp_dir"}# Usage ExampleSECURE_DOWNLOAD "https://example.com/file.zip" "downloaded_file.zip" "expected_sha256_hash"

# DON'T do thiscurl https://example.com/huge-file.zip > file.zip# DO this insteadSMART_MEMORY_DOWNLOAD() { local url="$1" local buffer_size="16k"  if [[ -z "$url" ]]; then echo "Error: URL required" >&2 return 1 }  curl --limit-rate 50M \ --max-filesize 10G \ --buffer-size "$buffer_size" \ --fail \ --silent \ --show-error \ -O "$url"  return $?}# UsageSMART_MEMORY_DOWNLOAD "https://example.com/large-dataset.zip"

# DON'T disable SSL verificationcurl -k https://example.com/file# DO handle certificates properlyCERT_AWARE_DOWNLOAD() { local url="$1" local ca_path="/etc/ssl/certs"  if [[ ! -d "$ca_path" ]]; then echo "Error: Certificate path not found" >&2 return 1 }  curl --cacert "$ca_path/ca-certificates.crt" \ --capath "$ca_path" \ --ssl-reqd \ --fail \ -O "$url"}# UsageCERT_AWARE_DOWNLOAD "https://api.example.com/secure-file.zip"

# Proper resource managementCLEAN_DOWNLOAD() { local url="$1" local temp_dir=$(mktemp -d) local temp_files=()  cleanup() { local exit_code=$? echo "Cleaning up temporary files..." for file in "${temp_files[@]}"; do rm -f "$file" done rmdir "$temp_dir" 2>/dev/null exit $exit_code }  trap cleanup EXIT INT TERM  # Your download logic here curl --fail "$url" -o "$temp_dir/download" temp_files+=("$temp_dir/download")}# UsageCLEAN_DOWNLOAD "https://example.com/file.tar.gz" 

...PRODUCTION_DOWNLOAD() { # Pre-download checks [[ -z "$1" ]] && { echo "❌ URL required"; return 1; } [[ -d "$(dirname "$2")" ]] || mkdir -p "$(dirname "$2")"  # Resource monitoring local start_time=$(date +%s) local disk_space=$(df -h . | awk 'NR==2 {print $4}')  echo "📊 Pre-download stats:" echo "Available disk space: $disk_space"  # The actual download with all best practices local result=$(SECURE_DOWNLOAD "$1" "$2" 2>&1) local status=$?  # Post-download verification if [[ $status -eq 0 ]]; then local end_time=$(date +%s) local duration=$((end_time - start_time))  echo "✨ Download Statistics:" echo "Duration: ${duration}s" echo "Final size: $(du -h "$2" | cut -f1)" echo "Success! 🎉" else echo "❌ Download failed: $result" fi  return $status}# UsagePRODUCTION_DOWNLOAD "https://example.com/file.zip" "/path/to/destination/file.zip"

CHECK_ENVIRONMENT() { local required_space=$((5 * 1024 * 1024)) # 5GB in KB local available_space=$(df . | awk 'NR==2 {print $4}') local curl_version=$(curl --version | head -n1)  # Check disk space [[ $available_space -lt $required_space ]] && { echo "⚠️ Insufficient disk space!" >&2 return 1 }  # Check curl installation command -v curl >/dev/null 2>&1 || { echo "⚠️ curl is not installed!" >&2 return 1 }  # Check SSL support curl --version | grep -q "SSL" || { echo "⚠️ curl lacks SSL support!" >&2 return 1 }  echo "✅ Environment check passed!" echo "🔍 curl version: $curl_version" echo "💾 Available space: $(numfmt --to=iec-i --suffix=B $available_space)"}# UsageCHECK_ENVIRONMENT || exit 1

#!/bin/bashDIAGNOSE_DOWNLOAD() { local url="$1" local error_log=$(mktemp)  echo "🔍 Starting download diagnosis..."  # Step 1: Basic connection test with status code check echo "Testing basic connectivity..." if ! curl -sS -w "\nHTTP Status: %{http_code}\n" --head "$url" > "$error_log" 2>&1; then echo "❌ Connection failed! Details:" cat "$error_log"  # Check for common issues if grep -q "Could not resolve host" "$error_log"; then echo "💡 DNS resolution failed. Checking DNS..." dig +short "$(echo "$url" | awk -F[/:] '{print $4}')" elif grep -q "Connection timed out" "$error_log"; then echo "💡 Connection timeout. Try:" echo "curl --connect-timeout 30 --max-time 300 \"$url\"" fi else echo "✅ Basic connectivity OK" # Check HTTP status code status_code=$(grep "HTTP Status:" "$error_log" | awk '{print $3}') if [[ $status_code -ge 400 ]]; then echo "⚠️ Server returned error status: $status_code" fi fi  # Step 2: SSL verification echo "Checking SSL/TLS..." if ! curl -vI --ssl-reqd "$url" > "$error_log" 2>&1; then echo "❌ SSL verification failed! Details:" grep "SSL" "$error_log" else echo "✅ SSL verification OK" fi # Cleanup rm -f "$error_log"}# Usage ExampleDIAGNOSE_DOWNLOAD "https://api.github.com/repos/curl/curl/releases/latest"

#!/bin/bashNETWORK_DIAGNOSIS() { local url="$1" local domain=$(echo "$url" | awk -F[/:] '{print $4}') local trace_log=$(mktemp)  echo "🌐 Starting network diagnosis..."  # DNS Resolution echo "Testing DNS resolution..." local dns_result=$(dig +short "$domain") if [[ -z "$dns_result" ]]; then echo "❌ DNS resolution failed!" echo "Try: nslookup $domain 8.8.8.8" rm -f "$trace_log" return 1 fi  # Trace Route with timeout echo "Checking network path..." if command -v timeout >/dev/null 2>&1; then timeout 30 traceroute -n "$domain" | tee "$trace_log" else traceroute -w 2 -n "$domain" | tee "$trace_log" # -w 2 sets 2-second timeout per hop fi  # Bandwidth Test echo "Testing download speed..." curl -s -w "\nSpeed: %{speed_download} bytes/sec\nTime to first byte: %{time_starttransfer} seconds\n" \ "$url" -o /dev/null # Cleanup rm -f "$trace_log"}# Usage ExampleNETWORK_DIAGNOSIS "https://downloads.example.com/large-file.zip"

#!/bin/bashPERMISSION_CHECK() { local target_path="$1"  # Check if target path was provided if [[ -z "$target_path" ]]; then echo "❌ Error: No target path provided" return 1 } local temp_dir=$(dirname "$target_path")  echo "🔍 Checking permissions..."  # Check if path exists if [[ ! -e "$temp_dir" ]]; then echo "❌ Directory does not exist: $temp_dir" return 1 } # File system type check local fs_type=$(df -PT "$temp_dir" | awk 'NR==2 {print $2}') echo "📂 File system type: $fs_type" case "$fs_type" in "nfs"|"cifs") echo "⚠️ Network file system detected - special permissions may apply" ;; "tmpfs") echo "⚠️ Temporary file system - data will not persist after reboot" ;; esac # Directory permissions check check_directory_access() { local dir="$1" if [[ ! -w "$dir" ]]; then echo "❌ No write permission in: $dir" echo "Current permissions: $(ls -ld "$dir")" echo "💡 Fix with: sudo chown $(whoami) \"$dir\"" return 1 fi return 0 }  # Disk space verification with reserved space check check_disk_space() { local required_mb="$1" local available_kb=$(df . | awk 'NR==2 {print $4}') local available_mb=$((available_kb / 1024)) local reserved_space_mb=100 # Reserve 100MB for safety  if [[ $((available_mb - reserved_space_mb)) -lt $required_mb ]]; then echo "❌ Insufficient disk space!" echo "Required: ${required_mb}MB" echo "Available: ${available_mb}MB (with ${reserved_space_mb}MB reserved)" return 1 fi return 0 }  # Full permission diagnostic DIAGNOSE_PERMISSIONS() { echo "📁 Directory structure:" namei -l "$target_path"  echo -e "\n💾 Disk usage:" df -h "$(dirname "$target_path")"  echo -e "\n👤 Current user context:" id  echo -e "\n🔒 SELinux status (if applicable):" if command -v getenforce >/dev/null 2>&1; then getenforce fi } # Run the diagnostics check_directory_access "$temp_dir" check_disk_space 500 # Assuming 500MB minimum required DIAGNOSE_PERMISSIONS}# Usage ExamplePERMISSION_CHECK "/var/www/downloads/target-file.zip"

#!/bin/bashSSL_DIAGNOSTIC() { local url="$1" local domain=$(echo "$url" | awk -F[/:] '{print $4}') local temp_cert=$(mktemp)  echo "🔐 Starting SSL diagnosis..."  # Input validation if [[ -z "$domain" ]]; then echo "❌ Invalid URL provided" rm -f "$temp_cert" return 1 }  # Certificate validation with timeout check_cert() { echo "Checking certificate for $domain..." if timeout 10 openssl s_client -connect "${domain}:443" 2> "$temp_cert" | \ openssl x509 -noout -dates -subject -issuer; then echo "✅ Certificate details retrieved successfully" else echo "❌ Certificate retrieval failed" cat "$temp_cert" fi }  # Certificate chain verification with SNI support verify_chain() { echo "Verifying certificate chain..." if timeout 10 openssl s_client -connect "${domain}:443" \ -servername "${domain}" \ -showcerts \ -verify 5 \ -verifyCAfile /etc/ssl/certs/ca-certificates.crt \ < /dev/null 2> "$temp_cert"; then echo "✅ Certificate chain verification successful" else echo "❌ Certificate chain verification failed" cat "$temp_cert" fi }  # Smart SSL handler with cipher suite check SMART_SSL_DOWNLOAD() { local url="$1" local output="$2" local retry_count=0 local max_retries=3  # Check supported ciphers first echo "Checking supported cipher suites..." openssl ciphers -v | grep TLSv1.2  # Try different SSL versions with retry mechanism for version in "tlsv1.3" "tlsv1.2" "tlsv1.1"; do while [[ $retry_count -lt $max_retries ]]; do echo "Attempting with $version (attempt $((retry_count + 1)))..." if curl --tlsv1.2 \ --tls-max "$version" \ --retry 3 \ --retry-delay 2 \ --connect-timeout 10 \ -o "$output" \ "$url"; then echo "✅ Success with $version!" rm -f "$temp_cert" return 0 fi retry_count=$((retry_count + 1)) sleep 2 done retry_count=0 done  echo "❌ All SSL versions failed" rm -f "$temp_cert" return 1 } # Run the checks check_cert verify_chain}# Usage ExamplesSSL_DIAGNOSTIC "https://api.github.com"SSL_DIAGNOSTIC "https://example.com" && SMART_SSL_DOWNLOAD "https://example.com/file.zip" "download.zip"

#!/bin/bashMASTER_TROUBLESHOOT() { local url="$1" local output="$2" local start_time=$(date +%s) local temp_log=$(mktemp)  echo "🔄 Starting comprehensive diagnosis..."  # Parameter validation if [[ -z "$url" || -z "$output" ]]; then echo "❌ Usage: MASTER_TROUBLESHOOT <url> <output_path>" rm -f "$temp_log" return 1 }  # Step 1: Quick connectivity check with timeout echo "Step 1: Basic connectivity" if ! timeout 10 curl -Is "$url" &> "$temp_log"; then echo "❌ Connectivity check failed" cat "$temp_log" NETWORK_DIAGNOSIS "$url" rm -f "$temp_log" return 1 fi  # Step 2: SSL verification with protocol detection echo "Step 2: SSL verification" if [[ "$url" =~ ^https:// ]]; then if ! curl -Iv "$url" 2>&1 | grep -q "SSL connection"; then echo "❌ SSL verification failed" SSL_DIAGNOSTIC "$url" rm -f "$temp_log" return 1 fi fi  # Step 3: Permission check with space verification echo "Step 3: Permission verification" local target_dir=$(dirname "$output") if ! PERMISSION_CHECK "$target_dir"; then # Check if we have at least 1GB free space if ! df -P "$target_dir" | awk 'NR==2 {exit($4<1048576)}'; then echo "⚠️ Warning: Less than 1GB free space available" fi rm -f "$temp_log" return 1 fi  # Step 4: Trial download with resume capability echo "Step 4: Test download" if ! curl -sS --fail \ --max-time 10 \ --retry 3 \ --retry-delay 2 \ -C - \ -r 0-1024 \ "$url" -o "$temp_log"; then echo "❌ Download failed! Full diagnosis needed..." DIAGNOSE_DOWNLOAD "$url" rm -f "$temp_log" return 1 fi  # Calculate total diagnostic time local end_time=$(date +%s) local duration=$((end_time - start_time))  echo "✅ All systems go! Ready for download." echo "📊 Diagnostic completed in $duration seconds"  # Cleanup rm -f "$temp_log" return 0}# Usage ExampleMASTER_TROUBLESHOOT "https://downloads.example.com/large-file.zip" "/path/to/output/file.zip"