#!/usr/bin/perl -w
#
# 955-999 1635-1699 1891-1899 2377-2399 3086-3099 3283-3299 4002-4099 4168-4199 4228-4299
# 4444-4499

# Fetch all the RFCs

use strict;

my $site = "http://www.ietf.org/rfc";
my $start = 1;
my $end   = 5500;
my $update = 0;

while ($ARGV[0]) {
    if     ($ARGV[0] eq "-s") {
	shift;
	$start = $ARGV[0] - 0;
    }
    elsif ($ARGV[0] eq "-e") {
	shift;
	$end = $ARGV[0] - 0;
    }
    elsif ($ARGV[0] eq "-u") { # update even if already there
	$update = 1;
    }
    else {
	die "Unknown option '$ARGV[0]'.\n";
    }
    shift;
}

warn "Fetching RFCs $start to $end into current dir ($ENV{PWD}).\n";

system "curl -O $site/_index.html";
for (my $n=$start; $n<=$end; $n+=100) {
    my $max = $n + 99;
    if ($max > $end) { $max = $end; }
    my $cmd = sprintf("curl -O '$site/rfc[%04d-%04d].txt'", $n, $max);
    warn "\n======= New batch, $n-$max...\n";
    system "$cmd";
    sleep 1;
}
warn "Done.\n";
exit;


###############################################################################
# curl http://example.com/files/foo[001-100].txt for iterative fetch.
#    -A append to output
#    -C offset    to continue at offset (-C - to figure it out)
#    -d data      send as if a form
#    -o file      output to here
#    -O           output to same name as remote file
#    -r s-e       just get bytes s to e (some servers won't do it)
#    -s           silent
#    -T file      upload
#    -u user:pw   
#    --url u      handy when using a config file
#