From e0a3162893bfd392f1f67223e342b1ede568158f Mon Sep 17 00:00:00 2001 From: Steven Schubiger Date: Fri, 21 Oct 2011 22:10:02 +0200 Subject: [PATCH 01/75] paramcheck: Use + quantifier and return copy. --- ChangeLog | 5 +++++ util/paramcheck.pl | 16 ++++++++-------- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/ChangeLog b/ChangeLog index 352af25f..82bcb863 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,8 @@ +2011-10-16 Steven Schubiger + + * util/paramcheck.pl: Match 1 or more times where applicable. + (extract_entries): Return a copy instead of reference. + 2011-09-04 Alan Hourihane (tiny change) * configure.ac: Check for libz when gnutls is used. diff --git a/util/paramcheck.pl b/util/paramcheck.pl index 832f5dc7..660cdb4f 100755 --- a/util/paramcheck.pl +++ b/util/paramcheck.pl @@ -33,11 +33,11 @@ my $tex_content = read_file($tex_file); my @args = ([ $main_content, - qr/static \s+? struct \s+? cmdline_option \s+? option_data\[\] \s+? = \s+? \{ (.*?) \}\;/sx, + qr/static \s+? struct \s+? cmdline_option \s+? option_data\[\] \s+? = \s+? \{ (.+?) \}\;/sx, [ qw(long_name short_name type data argtype) ], ], [ $init_content, - qr/commands\[\] \s+? = \s+? \{ (.*?) \}\;/sx, + qr/commands\[\] \s+? = \s+? \{ (.+?) \}\;/sx, [ qw(name place action) ], ]); @@ -78,18 +78,18 @@ sub extract_entries my (@entries, %index, $i); foreach my $chunk (@$chunks) { - my ($args) = $chunk =~ /\{ \s+? (.*?) \s+? \}/sx; + my ($args) = $chunk =~ /\{ \s+? (.+?) \s+? \}/sx; next unless defined $args; my @args = map { tr/'"//d; $_ } map { - /\((.*?)\)/ ? $1 : $_ + /\((.+?)\)/ ? $1 : $_ } split /\,\s+/, $args; my $entry = { map { $_ => shift @args } @$names }; - ($entry->{line}) = $chunk =~ /^ \s+? (\{.*)/mx; + ($entry->{line}) = $chunk =~ /^ \s+? (\{.+)/mx; if ($chunk =~ /deprecated/i) { $entries[-1]->{deprecated} = true; } @@ -103,9 +103,9 @@ sub extract_entries push @entries, $entry; } - push @entries, \%index; + push @entries, { %index }; - return \@entries; + return [ @entries ]; } sub output_results @@ -281,7 +281,7 @@ sub emit_undocumented_opts while ($tex =~ /^\@item\w*? \s+? --([-a-z0-9]+)/gmx) { $tex_items{$1} = true; } - my ($help) = $main =~ /\n print_help .*? \{\n (.*) \n\} \n/sx; + my ($help) = $main =~ /\n print_help .*? \{\n (.+) \n\} \n/sx; while ($help =~ /--([-a-z0-9]+)/g) { $main_items{$1} = true; } From 8c7bd588fe94bdc12b62b38e286027acfedde751 Mon Sep 17 00:00:00 2001 From: Steven Schweda Date: Sun, 23 Oct 2011 13:11:22 +0200 Subject: [PATCH 02/75] Fix some problems under VMS. --- src/ChangeLog | 25 +++++++++++++++++++++++++ src/connect.c | 9 +++++++-- src/ftp.c | 26 ++++++++++++++++++++------ src/init.c | 23 ++++++++++++++++++----- src/log.c | 6 +++--- src/main.c | 39 +++++++++++++++++++++++---------------- src/openssl.c | 4 +++- src/utils.c | 3 +-- 8 files changed, 100 insertions(+), 35 deletions(-) diff --git a/src/ChangeLog b/src/ChangeLog index c9c317ba..c2af118e 100644 --- a/src/ChangeLog +++ b/src/ChangeLog @@ -1,3 +1,28 @@ +2011-10-07 Steven Schweda + + * connect.c: Add HAVE_SYS_SELECT_H and HAVE_SYS_SOCKET_H conditions + on includes of and , respectively. + * ftp.c (getftp): Move BIN_TYPE_TRANSFER macro into VMS-specific + section. On VMS, use Stream_LF attributes for listing files. Pass + BIN_TYPE_FILE to fopen_excl() instead of constant-everywhere "true". + * ftp.c (ftp_retrieve_list): Restore lost test of opt.preserve_perm + (--preserve-permissions) on the chmod() operation. + * init.c, main.c: Remove "deprecated" from opt.preserve_perm + (--preserve-permissions). + * init.c (initialize): Use distinct messages for errors in C macro + SYSTEM_WGETRC and environment-variable SYSTEM_WGETRC. Avoid use of + C macro SYSTEM_WGETRC when it's not defined. + * log.c (log_close): Avoid closing logfp when it's stderr. + * main.c (print_help): Restore --preserve-permissions. + * main.c (main): Avoid using a negative value of longindex as a + subscript (for long_options[]) when searching for "--config". + * main.c (main): Exit the program using exit() instead of "return". + (VMS handles these differently, and exit() is better.) + * openssl.c (ssl_init): Add type cast (SSL_METHOD *) to newly "const" + "meth" argument to accommodate OpenSSL version 0.9.8, where that + argument is not "const" in the OpenSSL function (SSL_CTX_new). + * utils.c (fopen_excl): Comment typography. + 2011-10-02 Henrik Holst (tiny change) * http.c (gethttp): If 'contentonerror' is used then do not skip the http body on 4xx and 5xx errors. diff --git a/src/connect.c b/src/connect.c index e12c049a..6008c3c2 100644 --- a/src/connect.c +++ b/src/connect.c @@ -36,8 +36,13 @@ as that of the covered work. */ #include #include -#include -#include +#ifdef HAVE_SYS_SOCKET_H +# include +#endif /* def HAVE_SYS_SOCKET_H */ + +#ifdef HAVE_SYS_SELECT_H +# include +#endif /* def HAVE_SYS_SELECT_H */ #ifndef WINDOWS # ifdef __VMS diff --git a/src/ftp.c b/src/ftp.c index a586d849..f75397d0 100644 --- a/src/ftp.c +++ b/src/ftp.c @@ -1152,13 +1152,25 @@ Error in server response, closing control connection.\n")); Elsewhere, define a constant "binary" flag. Isn't it nice to have distinct text and binary file types? */ -# define BIN_TYPE_TRANSFER (type_char != 'A') +/* 2011-09-30 SMS. + Added listing files to the set of non-"binary" (text, Stream_LF) + files. (Wget works either way, but other programs, like, say, text + editors, work better on listing files which have text attributes.) + Now we use "binary" attributes for a binary ("IMAGE") transfer, + unless "--ftp-stmlf" was specified, and we always use non-"binary" + (text, Stream_LF) attributes for a listing file, or for an ASCII + transfer. + Tidied the VMS-specific BIN_TYPE_xxx macros, and changed the call to + fopen_excl() (restored?) to use BIN_TYPE_FILE instead of "true". +*/ #ifdef __VMS +# define BIN_TYPE_TRANSFER (type_char != 'A') +# define BIN_TYPE_FILE \ + ((!(cmd & DO_LIST)) && BIN_TYPE_TRANSFER && (opt.ftp_stmlf == 0)) # define FOPEN_OPT_ARGS "fop=sqo", "acc", acc_cb, &open_id # define FOPEN_OPT_ARGS_BIN "ctx=bin,stm", "rfm=fix", "mrs=512" FOPEN_OPT_ARGS -# define BIN_TYPE_FILE (BIN_TYPE_TRANSFER && (opt.ftp_stmlf == 0)) #else /* def __VMS */ -# define BIN_TYPE_FILE 1 +# define BIN_TYPE_FILE true #endif /* def __VMS [else] */ if (restval && !(con->cmd & DO_LIST)) @@ -1217,7 +1229,7 @@ Error in server response, closing control connection.\n")); } else { - fp = fopen_excl (con->target, true); + fp = fopen_excl (con->target, BIN_TYPE_FILE); if (!fp && errno == EEXIST) { /* We cannot just invent a new name and use it (which is @@ -1880,8 +1892,10 @@ Already have correct symlink %s -> %s\n\n"), set_local_file (&actual_target, con->target); - /* If downloading a plain file, set valid (non-zero) permissions. */ - if (dlthis && (actual_target != NULL) && (f->type == FT_PLAINFILE)) + /* If downloading a plain file, and the user requested it, then + set valid (non-zero) permissions. */ + if (dlthis && (actual_target != NULL) && + (f->type == FT_PLAINFILE) && opt.preserve_perm) { if (f->perms) chmod (actual_target, f->perms); diff --git a/src/init.c b/src/init.c index b40be8ad..eae35523 100644 --- a/src/init.c +++ b/src/init.c @@ -214,7 +214,7 @@ static const struct { { "postdata", &opt.post_data, cmd_string }, { "postfile", &opt.post_file_name, cmd_file }, { "preferfamily", NULL, cmd_spec_prefer_family }, - { "preservepermissions", &opt.preserve_perm, cmd_boolean },/* deprecated */ + { "preservepermissions", &opt.preserve_perm, cmd_boolean }, #ifdef HAVE_SSL { "privatekey", &opt.private_key, cmd_file }, { "privatekeytype", &opt.private_key_type, cmd_cert_type }, @@ -598,21 +598,34 @@ initialize (void) variable has been set. For internal testing purposes only! */ env_sysrc = getenv ("SYSTEM_WGETRC"); if (env_sysrc && file_exists_p (env_sysrc)) - ok &= run_wgetrc (env_sysrc); + { + ok &= run_wgetrc (env_sysrc); + /* If there are any problems parsing the system wgetrc file, tell + the user and exit */ + if (! ok) + { + fprintf (stderr, _("\ +Parsing system wgetrc file (env SYSTEM_WGETRC) failed. Please check\n\ +'%s',\n\ +or specify a different file using --config.\n"), env_sysrc); + exit (2); + } + } /* Otherwise, if SYSTEM_WGETRC is defined, use it. */ #ifdef SYSTEM_WGETRC else if (file_exists_p (SYSTEM_WGETRC)) ok &= run_wgetrc (SYSTEM_WGETRC); -#endif /* If there are any problems parsing the system wgetrc file, tell the user and exit */ if (! ok) { fprintf (stderr, _("\ -Parsing system wgetrc file failed, please check '%s'. \ -Or specify a different file using --config\n"), SYSTEM_WGETRC); +Parsing system wgetrc file failed. Please check\n\ +'%s',\n\ +or specify a different file using --config.\n"), SYSTEM_WGETRC); exit (2); } +#endif /* Override it with your own, if one exists. */ file = wgetrc_file_name (); if (!file) diff --git a/src/log.c b/src/log.c index 361b4537..e6875f6b 100644 --- a/src/log.c +++ b/src/log.c @@ -573,14 +573,14 @@ log_init (const char *file, bool appendp) } } -/* Close LOGFP, inhibit further logging and free the memory associated - with it. */ +/* Close LOGFP (only if we opened it, not if it's stderr), inhibit + further logging and free the memory associated with it. */ void log_close (void) { int i; - if (logfp) + if (logfp && (logfp != stderr)) fclose (logfp); logfp = NULL; inhibit_logging = true; diff --git a/src/main.c b/src/main.c index b80eef0a..05ad0e76 100644 --- a/src/main.c +++ b/src/main.c @@ -243,7 +243,7 @@ static struct cmdline_option option_data[] = { "post-data", 0, OPT_VALUE, "postdata", -1 }, { "post-file", 0, OPT_VALUE, "postfile", -1 }, { "prefer-family", 0, OPT_VALUE, "preferfamily", -1 }, - { "preserve-permissions", 0, OPT_BOOLEAN, "preservepermissions", -1 }, /* deprecated */ + { "preserve-permissions", 0, OPT_BOOLEAN, "preservepermissions", -1 }, { IF_SSL ("private-key"), 0, OPT_VALUE, "privatekey", -1 }, { IF_SSL ("private-key-type"), 0, OPT_VALUE, "privatekeytype", -1 }, { "progress", 0, OPT_VALUE, "progress", -1 }, @@ -646,6 +646,8 @@ FTP options:\n"), --no-glob turn off FTP file name globbing.\n"), N_("\ --no-passive-ftp disable the \"passive\" transfer mode.\n"), + N_("\ + --preserve-permissions preserve remote file permissions.\n"), N_("\ --retr-symlinks when recursing, get linked-to files (not dir).\n"), "\n", @@ -948,8 +950,8 @@ main (int argc, char **argv) init_switches (); - /* This seperate getopt_long is needed to find the user config - and parse it before the other user options. */ + /* This separate getopt_long is needed to find the user config file + option ("--config") and parse it before the other user options. */ longindex = -1; int retconf; bool use_userconfig = false; @@ -960,20 +962,25 @@ main (int argc, char **argv) int confval; bool userrc_ret = true; struct cmdline_option *config_opt; - confval = long_options[longindex].val; - config_opt = &option_data[confval & ~BOOLEAN_NEG_MARKER]; - if (strcmp (config_opt->long_name, "config") == 0) + + /* There is no short option for "--config". */ + if (longindex >= 0) { - userrc_ret &= run_wgetrc (optarg); - use_userconfig = true; + confval = long_options[longindex].val; + config_opt = &option_data[confval & ~BOOLEAN_NEG_MARKER]; + if (strcmp (config_opt->long_name, "config") == 0) + { + userrc_ret &= run_wgetrc (optarg); + use_userconfig = true; + } + if (!userrc_ret) + { + printf ("Exiting due to error in %s\n", optarg); + exit (2); + } + else + break; } - if (!userrc_ret) - { - printf ("Exiting due to error in %s\n", optarg); - exit (2); - } - else - break; } /* If the user did not specify a config, read the system wgetrc and ~/.wgetrc. */ @@ -1470,7 +1477,7 @@ outputting to a regular file.\n")); xfree (url[i]); cleanup (); - return get_exit_status (); + exit (get_exit_status ()); } #endif /* TESTING */ diff --git a/src/openssl.c b/src/openssl.c index 2e236690..bc374915 100644 --- a/src/openssl.c +++ b/src/openssl.c @@ -201,7 +201,9 @@ ssl_init () abort (); } - ssl_ctx = SSL_CTX_new (meth); + /* The type cast below accommodates older OpenSSL versions (0.9.8) + where SSL_CTX_new() is declared without a "const" argument. */ + ssl_ctx = SSL_CTX_new ((SSL_METHOD *)meth); if (!ssl_ctx) goto error; diff --git a/src/utils.c b/src/utils.c index 4950ab2e..509088b6 100644 --- a/src/utils.c +++ b/src/utils.c @@ -769,8 +769,7 @@ fopen_excl (const char *fname, int binary) open_id = 13; fd = open( fname, /* File name. */ flags, /* Flags. */ - 0777, /* Mode for default protection. -*/ + 0777, /* Mode for default protection. */ "rfm=stmlf", /* Stream_LF. */ OPEN_OPT_ARGS); /* Access callback. */ } From a5fdba0958eb77a0fdd8e1da8655567a150dce89 Mon Sep 17 00:00:00 2001 From: Giuseppe Scrivano Date: Sun, 23 Oct 2011 13:23:46 +0200 Subject: [PATCH 03/75] bootstrap.conf: Include module `vsnprintf'. --- ChangeLog | 4 ++++ bootstrap.conf | 1 + 2 files changed, 5 insertions(+) diff --git a/ChangeLog b/ChangeLog index 82bcb863..7691445e 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,7 @@ +2011-10-23 Giuseppe Scrivano + + * bootstrap.conf (gnulib_modules): Include module `vsnprintf'. + 2011-10-16 Steven Schubiger * util/paramcheck.pl: Match 1 or more times where applicable. diff --git a/bootstrap.conf b/bootstrap.conf index 9753d9dc..77230dbb 100644 --- a/bootstrap.conf +++ b/bootstrap.conf @@ -66,6 +66,7 @@ strerror_r-posix unlocked-io update-copyright vasprintf +vsnprintf write " From e3820953b25ec3ea6472649375df36745aeb5696 Mon Sep 17 00:00:00 2001 From: Gijs van Tulder Date: Fri, 4 Nov 2011 22:25:00 +0100 Subject: [PATCH 04/75] Add support for WARC files. --- bootstrap.conf | 3 + configure.ac | 12 + src/ChangeLog | 6 + src/Makefile.am | 4 +- src/ftp.c | 67 ++- src/http.c | 438 +++++++++++++--- src/init.c | 40 ++ src/log.c | 60 ++- src/log.h | 4 + src/main.c | 100 ++++ src/options.h | 9 + src/retr.c | 41 +- src/retr.h | 2 +- src/test.c | 2 + src/warc.c | 1332 +++++++++++++++++++++++++++++++++++++++++++++++ src/warc.h | 19 + src/wget.h | 4 +- 17 files changed, 2048 insertions(+), 95 deletions(-) create mode 100644 src/warc.c create mode 100644 src/warc.h diff --git a/bootstrap.conf b/bootstrap.conf index 77230dbb..6473cbba 100644 --- a/bootstrap.conf +++ b/bootstrap.conf @@ -28,6 +28,7 @@ gnulib_modules=" accept alloca announce-gen +base32 bind c-ctype clock-time @@ -49,6 +50,7 @@ maintainer-makefile mbtowc mkdir crypto/md5 +crypto/sha1 pipe quote quotearg @@ -63,6 +65,7 @@ socket stdbool strcasestr strerror_r-posix +tmpdir unlocked-io update-copyright vasprintf diff --git a/configure.ac b/configure.ac index 76c6fa28..360f6c91 100644 --- a/configure.ac +++ b/configure.ac @@ -511,7 +511,19 @@ if test "X$iri" != "Xno"; then fi fi +dnl +dnl Check for UUID +dnl +AC_CHECK_HEADER(uuid/uuid.h, + AC_CHECK_LIB(uuid, uuid_generate, + [LIBS="${LIBS} -luuid" + AC_DEFINE([HAVE_LIBUUID], 1, + [Define if libuuid is available.]) + ]) +) + + dnl Needed by src/Makefile.am AM_CONDITIONAL([IRI_IS_ENABLED], [test "X$iri" != "Xno"]) diff --git a/src/ChangeLog b/src/ChangeLog index c2af118e..65c48072 100644 --- a/src/ChangeLog +++ b/src/ChangeLog @@ -1,3 +1,6 @@ +2011-11-04 Giuseppe Scrivano + + 2011-10-07 Steven Schweda * connect.c: Add HAVE_SYS_SELECT_H and HAVE_SYS_SOCKET_H conditions @@ -21,7 +24,10 @@ * openssl.c (ssl_init): Add type cast (SSL_METHOD *) to newly "const" "meth" argument to accommodate OpenSSL version 0.9.8, where that argument is not "const" in the OpenSSL function (SSL_CTX_new). + * test.c: Declare "program_argstring". * utils.c (fopen_excl): Comment typography. + * warc.h: New file. + * warc.c: New file. 2011-10-02 Henrik Holst (tiny change) * http.c (gethttp): If 'contentonerror' is used then do not diff --git a/src/Makefile.am b/src/Makefile.am index 6b951988..8ef931a6 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -46,13 +46,13 @@ wget_SOURCES = cmpt.c connect.c convert.c cookies.c ftp.c \ css_.c css-url.c \ ftp-basic.c ftp-ls.c hash.c host.c html-parse.c html-url.c \ http.c init.c log.c main.c netrc.c progress.c ptimer.c \ - recur.c res.c retr.c spider.c url.c \ + recur.c res.c retr.c spider.c url.c warc.c \ utils.c exits.c build_info.c $(IRI_OBJ) \ css-url.h css-tokens.h connect.h convert.h cookies.h \ ftp.h hash.h host.h html-parse.h html-url.h \ http.h http-ntlm.h init.h log.h mswindows.h netrc.h \ options.h progress.h ptimer.h recur.h res.h retr.h \ - spider.h ssl.h sysdep.h url.h utils.h wget.h iri.h \ + spider.h ssl.h sysdep.h url.h warc.h utils.h wget.h iri.h \ exits.h gettext.h nodist_wget_SOURCES = version.c EXTRA_wget_SOURCES = iri.c diff --git a/src/ftp.c b/src/ftp.c index f75397d0..989a1dda 100644 --- a/src/ftp.c +++ b/src/ftp.c @@ -49,6 +49,7 @@ as that of the covered work. */ #include "netrc.h" #include "convert.h" /* for downloaded_file */ #include "recur.h" /* for INFINITE_RECURSION */ +#include "warc.h" #ifdef __VMS # include "vms.h" @@ -237,10 +238,11 @@ static uerr_t ftp_get_listing (struct url *, ccon *, struct fileinfo **); /* Retrieves a file with denoted parameters through opening an FTP connection to the server. It always closes the data connection, - and closes the control connection in case of error. */ + and closes the control connection in case of error. If warc_tmp + is non-NULL, the downloaded data will be written there as well. */ static uerr_t getftp (struct url *u, wgint passed_expected_bytes, wgint *qtyread, - wgint restval, ccon *con, int count) + wgint restval, ccon *con, int count, FILE *warc_tmp) { int csock, dtsock, local_sock, res; uerr_t err = RETROK; /* appease the compiler */ @@ -1155,7 +1157,7 @@ Error in server response, closing control connection.\n")); /* 2011-09-30 SMS. Added listing files to the set of non-"binary" (text, Stream_LF) files. (Wget works either way, but other programs, like, say, text - editors, work better on listing files which have text attributes.) + editors, work better on listing files which have text attributes.) Now we use "binary" attributes for a binary ("IMAGE") transfer, unless "--ftp-stmlf" was specified, and we always use non-"binary" (text, Stream_LF) attributes for a listing file, or for an ASCII @@ -1194,7 +1196,7 @@ Error in server response, closing control connection.\n")); } else if (opt.noclobber || opt.always_rest || opt.timestamping || opt.dirstruct || opt.output_document || count > 0) - { + { if (opt.unlink && file_exists_p (con->target)) { int res = unlink (con->target); @@ -1274,7 +1276,7 @@ Error in server response, closing control connection.\n")); rd_size = 0; res = fd_read_body (dtsock, fp, expected_bytes ? expected_bytes - restval : 0, - restval, &rd_size, qtyread, &con->dltime, flags); + restval, &rd_size, qtyread, &con->dltime, flags, warc_tmp); tms = datetime_str (time (NULL)); tmrate = retr_rate (rd_size, con->dltime); @@ -1285,15 +1287,18 @@ Error in server response, closing control connection.\n")); if (!output_stream || con->cmd & DO_LIST) fclose (fp); - /* If fd_read_body couldn't write to fp, bail out. */ - if (res == -2) + /* If fd_read_body couldn't write to fp or warc_tmp, bail out. */ + if (res == -2 || (warc_tmp != NULL && res == -3)) { logprintf (LOG_NOTQUIET, _("%s: %s, closing control connection.\n"), con->target, strerror (errno)); fd_close (csock); con->csock = -1; fd_close (dtsock); - return FWRITEERR; + if (res == -2) + return FWRITEERR; + else if (res == -3) + return WARC_TMP_FWRITEERR; } else if (res == -1) { @@ -1409,6 +1414,11 @@ ftp_loop_internal (struct url *u, struct fileinfo *f, ccon *con, char **local_fi uerr_t err; struct_stat st; + /* Declare WARC variables. */ + bool warc_enabled = (opt.warc_filename != NULL); + FILE *warc_tmp = NULL; + ip_address *warc_ip = NULL; + /* Get the target, and set the name for the message accordingly. */ if ((f == NULL) && (con->target)) { @@ -1445,6 +1455,21 @@ ftp_loop_internal (struct url *u, struct fileinfo *f, ccon *con, char **local_fi orig_lp = con->cmd & LEAVE_PENDING ? 1 : 0; + /* For file RETR requests, we can write a WARC record. + We record the file contents to a temporary file. */ + if (warc_enabled && (con->cmd & DO_RETR)) + { + warc_tmp = warc_tempfile (); + if (warc_tmp == NULL) + return WARC_TMP_FOPENERR; + + if (!con->proxy && con->csock != -1) + { + warc_ip = (ip_address *) alloca (sizeof (ip_address)); + socket_ip_address (con->csock, warc_ip, ENDPOINT_PEER); + } + } + /* THE loop. */ do { @@ -1509,7 +1534,10 @@ ftp_loop_internal (struct url *u, struct fileinfo *f, ccon *con, char **local_fi len = f->size; else len = 0; - err = getftp (u, len, &qtyread, restval, con, count); + + /* If we are working on a WARC record, getftp should also write + to the warc_tmp file. */ + err = getftp (u, len, &qtyread, restval, con, count, warc_tmp); if (con->csock == -1) con->st &= ~DONE_CWD; @@ -1520,8 +1548,10 @@ ftp_loop_internal (struct url *u, struct fileinfo *f, ccon *con, char **local_fi { case HOSTERR: case CONIMPOSSIBLE: case FWRITEERR: case FOPENERR: case FTPNSFOD: case FTPLOGINC: case FTPNOPASV: case CONTNOTSUPPORTED: - case UNLINKERR: + case UNLINKERR: case WARC_TMP_FWRITEERR: /* Fatal errors, give up. */ + if (warc_tmp != NULL) + fclose (warc_tmp); return err; case CONSOCKERR: case CONERROR: case FTPSRVERR: case FTPRERR: case WRITEFAILED: case FTPUNKNOWNTYPE: case FTPSYSERR: @@ -1589,6 +1619,19 @@ ftp_loop_internal (struct url *u, struct fileinfo *f, ccon *con, char **local_fi xfree (hurl); } + if (warc_enabled && (con->cmd & DO_RETR)) + { + /* Create and store a WARC resource record for the retrieved file. */ + bool warc_res; + + warc_res = warc_write_resource_record (NULL, u->url, NULL, NULL, + warc_ip, NULL, warc_tmp, -1); + if (! warc_res) + return WARC_ERR; + + /* warc_write_resource_record has also closed warc_tmp. */ + } + if ((con->cmd & DO_LIST)) /* This is a directory listing file. */ { @@ -1928,7 +1971,9 @@ Already have correct symlink %s -> %s\n\n"), xfree (ofile); /* Break on fatals. */ - if (err == QUOTEXC || err == HOSTERR || err == FWRITEERR) + if (err == QUOTEXC || err == HOSTERR || err == FWRITEERR + || err == WARC_ERR || err == WARC_TMP_FOPENERR + || err == WARC_TMP_FWRITEERR) break; con->cmd &= ~ (DO_CWD | DO_LOGIN); f = f->next; diff --git a/src/http.c b/src/http.c index 7eef453f..6a2ffe86 100644 --- a/src/http.c +++ b/src/http.c @@ -58,6 +58,7 @@ as that of the covered work. */ #include "md5.h" #include "convert.h" #include "spider.h" +#include "warc.h" #ifdef TESTING #include "test.h" @@ -320,10 +321,12 @@ request_remove_header (struct request *req, char *name) p += A_len; \ } while (0) -/* Construct the request and write it to FD using fd_write. */ +/* Construct the request and write it to FD using fd_write. + If warc_tmp is set to a file pointer, the request string will + also be written to that file. */ static int -request_send (const struct request *req, int fd) +request_send (const struct request *req, int fd, FILE *warc_tmp) { char *request_string, *p; int i, size, write_error; @@ -374,6 +377,13 @@ request_send (const struct request *req, int fd) if (write_error < 0) logprintf (LOG_VERBOSE, _("Failed writing HTTP request: %s.\n"), fd_errstr (fd)); + else if (warc_tmp != NULL) + { + /* Write a copy of the data to the WARC record. */ + int warc_tmp_written = fwrite (request_string, 1, size - 1, warc_tmp); + if (warc_tmp_written != size - 1) + return -2; + } return write_error; } @@ -444,10 +454,12 @@ register_basic_auth_host (const char *hostname) /* Send the contents of FILE_NAME to SOCK. Make sure that exactly PROMISED_SIZE bytes are sent over the wire -- if the file is - longer, read only that much; if the file is shorter, report an error. */ + longer, read only that much; if the file is shorter, report an error. + If warc_tmp is set to a file pointer, the post data will + also be written to that file. */ static int -post_file (int sock, const char *file_name, wgint promised_size) +post_file (int sock, const char *file_name, wgint promised_size, FILE *warc_tmp) { static char chunk[8192]; wgint written = 0; @@ -472,6 +484,16 @@ post_file (int sock, const char *file_name, wgint promised_size) fclose (fp); return -1; } + if (warc_tmp != NULL) + { + /* Write a copy of the data to the WARC record. */ + int warc_tmp_written = fwrite (chunk, 1, towrite, warc_tmp); + if (warc_tmp_written != towrite) + { + fclose (fp); + return -2; + } + } written += towrite; } fclose (fp); @@ -1462,6 +1484,135 @@ File %s already there; not retrieving.\n\n"), quote (filename)); *dt |= TEXTHTML; } +/* Download the response body from the socket and writes it to + an output file. The headers have already been read from the + socket. If WARC is enabled, the response body will also be + written to a WARC response record. + + hs, contlen, contrange, chunked_transfer_encoding and url are + parameters from the gethttp method. fp is a pointer to the + output file. + + url, warc_timestamp_str, warc_request_uuid, warc_ip, type + and statcode will be saved in the headers of the WARC record. + The head parameter contains the HTTP headers of the response. + + If fp is NULL and WARC is enabled, the response body will be + written only to the WARC file. If WARC is disabled and fp + is a file pointer, the data will be written to the file. + If fp is a file pointer and WARC is enabled, the body will + be written to both destinations. + + Returns the error code. */ +static int +read_response_body (struct http_stat *hs, int sock, FILE *fp, wgint contlen, + wgint contrange, bool chunked_transfer_encoding, + char *url, char *warc_timestamp_str, char *warc_request_uuid, + ip_address *warc_ip, char *type, int statcode, char *head) +{ + int warc_payload_offset = 0; + FILE *warc_tmp = NULL; + int warcerr = 0; + + if (opt.warc_filename != NULL) + { + /* Open a temporary file where we can write the response before we + add it to the WARC record. */ + warc_tmp = warc_tempfile (); + if (warc_tmp == NULL) + warcerr = WARC_TMP_FOPENERR; + + if (warcerr == 0) + { + /* We should keep the response headers for the WARC record. */ + int head_len = strlen (head); + int warc_tmp_written = fwrite (head, 1, head_len, warc_tmp); + if (warc_tmp_written != head_len) + warcerr = WARC_TMP_FWRITEERR; + warc_payload_offset = head_len; + } + + if (warcerr != 0) + { + if (warc_tmp != NULL) + fclose (warc_tmp); + return warcerr; + } + } + + if (fp != NULL) + { + /* This confuses the timestamping code that checks for file size. + #### The timestamping code should be smarter about file size. */ + if (opt.save_headers && hs->restval == 0) + fwrite (head, 1, strlen (head), fp); + } + + /* Read the response body. */ + int flags = 0; + if (contlen != -1) + /* If content-length is present, read that much; otherwise, read + until EOF. The HTTP spec doesn't require the server to + actually close the connection when it's done sending data. */ + flags |= rb_read_exactly; + if (fp != NULL && hs->restval > 0 && contrange == 0) + /* If the server ignored our range request, instruct fd_read_body + to skip the first RESTVAL bytes of body. */ + flags |= rb_skip_startpos; + if (chunked_transfer_encoding) + flags |= rb_chunked_transfer_encoding; + + hs->len = hs->restval; + hs->rd_size = 0; + /* Download the response body and write it to fp. + If we are working on a WARC file, we simultaneously write the + response body to warc_tmp. */ + hs->res = fd_read_body (sock, fp, contlen != -1 ? contlen : 0, + hs->restval, &hs->rd_size, &hs->len, &hs->dltime, + flags, warc_tmp); + if (hs->res >= 0) + { + if (warc_tmp != NULL) + { + /* Create a response record and write it to the WARC file. + Note: per the WARC standard, the request and response should share + the same date header. We re-use the timestamp of the request. + The response record should also refer to the uuid of the request. */ + bool r = warc_write_response_record (url, warc_timestamp_str, + warc_request_uuid, warc_ip, + warc_tmp, warc_payload_offset, + type, statcode, hs->newloc); + + /* warc_write_response_record has closed warc_tmp. */ + + if (! r) + return WARC_ERR; + } + + return RETRFINISHED; + } + + if (warc_tmp != NULL) + fclose (warc_tmp); + + if (hs->res == -2) + { + /* Error while writing to fd. */ + return FWRITEERR; + } + else if (hs->res == -3) + { + /* Error while writing to warc_tmp. */ + return WARC_TMP_FWRITEERR; + } + else + { + /* A read error! */ + hs->rderrmsg = xstrdup (fd_errstr (sock)); + return RETRFINISHED; + } +} + #define BEGINS_WITH(line, string_constant) \ (!strncasecmp (line, string_constant, sizeof (string_constant) - 1) \ && (c_isspace (line[sizeof (string_constant) - 1]) \ @@ -1519,9 +1670,9 @@ gethttp (struct url *u, struct http_stat *hs, int *dt, struct url *proxy, wgint contlen, contrange; struct url *conn; FILE *fp; + int err; int sock = -1; - int flags; /* Set to 1 when the authorization has already been sent and should not be tried again. */ @@ -1547,6 +1698,14 @@ gethttp (struct url *u, struct http_stat *hs, int *dt, struct url *proxy, char hdrval[256]; char *message; + /* Declare WARC variables. */ + bool warc_enabled = (opt.warc_filename != NULL); + FILE *warc_tmp = NULL; + char warc_timestamp_str [21]; + char warc_request_uuid [48]; + ip_address *warc_ip = NULL; + long int warc_payload_offset = -1; + /* Whether this connection will be kept alive after the HTTP request is done. */ bool keep_alive; @@ -1852,7 +2011,7 @@ gethttp (struct url *u, struct http_stat *hs, int *dt, struct url *proxy, that the contents of Host would be exactly the same as the contents of CONNECT. */ - write_error = request_send (connreq, sock); + write_error = request_send (connreq, sock, 0); request_free (connreq); if (write_error < 0) { @@ -1924,8 +2083,26 @@ gethttp (struct url *u, struct http_stat *hs, int *dt, struct url *proxy, #endif /* HAVE_SSL */ } + /* Open the temporary file where we will write the request. */ + if (warc_enabled) + { + warc_tmp = warc_tempfile (); + if (warc_tmp == NULL) + { + CLOSE_INVALIDATE (sock); + request_free (req); + return WARC_TMP_FOPENERR; + } + + if (! proxy) + { + warc_ip = (ip_address *) alloca (sizeof (ip_address)); + socket_ip_address (sock, warc_ip, ENDPOINT_PEER); + } + } + /* Send the request to server. */ - write_error = request_send (req, sock); + write_error = request_send (req, sock, warc_tmp); if (write_error >= 0) { @@ -1933,16 +2110,39 @@ gethttp (struct url *u, struct http_stat *hs, int *dt, struct url *proxy, { DEBUGP (("[POST data: %s]\n", opt.post_data)); write_error = fd_write (sock, opt.post_data, post_data_size, -1); + if (write_error >= 0 && warc_tmp != NULL) + { + /* Remember end of headers / start of payload. */ + warc_payload_offset = ftell (warc_tmp); + + /* Write a copy of the data to the WARC record. */ + int warc_tmp_written = fwrite (opt.post_data, 1, post_data_size, warc_tmp); + if (warc_tmp_written != post_data_size) + write_error = -2; + } } else if (opt.post_file_name && post_data_size != 0) - write_error = post_file (sock, opt.post_file_name, post_data_size); + { + if (warc_tmp != NULL) + /* Remember end of headers / start of payload. */ + warc_payload_offset = ftell (warc_tmp); + + write_error = post_file (sock, opt.post_file_name, post_data_size, warc_tmp); + } } if (write_error < 0) { CLOSE_INVALIDATE (sock); request_free (req); - return WRITEFAILED; + + if (warc_tmp != NULL) + fclose (warc_tmp); + + if (write_error == -2) + return WARC_TMP_FWRITEERR; + else + return WRITEFAILED; } logprintf (LOG_VERBOSE, _("%s request sent, awaiting response... "), proxy ? "Proxy" : "HTTP"); @@ -1950,6 +2150,29 @@ gethttp (struct url *u, struct http_stat *hs, int *dt, struct url *proxy, contrange = 0; *dt &= ~RETROKF; + + if (warc_enabled) + { + bool warc_result; + /* Generate a timestamp and uuid for this request. */ + warc_timestamp (warc_timestamp_str); + warc_uuid_str (warc_request_uuid); + + /* Create a request record and store it in the WARC file. */ + warc_result = warc_write_request_record (u->url, warc_timestamp_str, + warc_request_uuid, warc_ip, + warc_tmp, warc_payload_offset); + if (! warc_result) + { + CLOSE_INVALIDATE (sock); + request_free (req); + return WARC_ERR; + } + + /* warc_write_request_record has also closed warc_tmp. */ + } + + read_header: head = read_http_response_head (sock); if (!head) @@ -2073,11 +2296,42 @@ read_header: if (statcode == HTTP_STATUS_UNAUTHORIZED) { /* Authorization is required. */ - if (keep_alive && !head_only - && skip_short_body (sock, contlen, chunked_transfer_encoding)) - CLOSE_FINISH (sock); + + /* Normally we are not interested in the response body. + But if we are writing a WARC file we are: we like to keep everyting. */ + if (warc_enabled) + { + int err; + type = resp_header_strdup (resp, "Content-Type"); + err = read_response_body (hs, sock, NULL, contlen, 0, + chunked_transfer_encoding, + u->url, warc_timestamp_str, + warc_request_uuid, warc_ip, type, + statcode, head); + xfree_null (type); + + if (err != RETRFINISHED || hs->res < 0) + { + CLOSE_INVALIDATE (sock); + request_free (req); + xfree_null (message); + resp_free (resp); + xfree (head); + return err; + } + else + CLOSE_FINISH (sock); + } else - CLOSE_INVALIDATE (sock); + { + /* Since WARC is disabled, we are not interested in the response body. */ + if (keep_alive && !head_only + && skip_short_body (sock, contlen, chunked_transfer_encoding)) + CLOSE_FINISH (sock); + else + CLOSE_INVALIDATE (sock); + } + pconn.authorized = false; if (!auth_finished && (user && passwd)) { @@ -2325,11 +2579,42 @@ read_header: _("Location: %s%s\n"), hs->newloc ? escnonprint_uri (hs->newloc) : _("unspecified"), hs->newloc ? _(" [following]") : ""); - if (keep_alive && !head_only - && skip_short_body (sock, contlen, chunked_transfer_encoding)) - CLOSE_FINISH (sock); + + /* In case the caller cares to look... */ + hs->len = 0; + hs->res = 0; + hs->restval = 0; + + /* Normally we are not interested in the response body of a redirect. + But if we are writing a WARC file we are: we like to keep everyting. */ + if (warc_enabled) + { + int err = read_response_body (hs, sock, NULL, contlen, 0, + chunked_transfer_encoding, + u->url, warc_timestamp_str, + warc_request_uuid, warc_ip, type, + statcode, head); + + if (err != RETRFINISHED || hs->res < 0) + { + CLOSE_INVALIDATE (sock); + xfree_null (type); + xfree (head); + return err; + } + else + CLOSE_FINISH (sock); + } else - CLOSE_INVALIDATE (sock); + { + /* Since WARC is disabled, we are not interested in the response body. */ + if (keep_alive && !head_only + && skip_short_body (sock, contlen, chunked_transfer_encoding)) + CLOSE_FINISH (sock); + else + CLOSE_INVALIDATE (sock); + } + xfree_null (type); xfree (head); /* From RFC2616: The status codes 303 and 307 have @@ -2447,8 +2732,6 @@ read_header: logputs (LOG_VERBOSE, "\n"); } } - xfree_null (type); - type = NULL; /* We don't need it any more. */ /* Return if we have no intention of further downloading. */ if ((!(*dt & RETROKF) && !opt.content_on_error) || head_only) @@ -2456,21 +2739,48 @@ read_header: /* In case the caller cares to look... */ hs->len = 0; hs->res = 0; - xfree_null (type); - if (head_only) - /* Pre-1.10 Wget used CLOSE_INVALIDATE here. Now we trust the - servers not to send body in response to a HEAD request, and - those that do will likely be caught by test_socket_open. - If not, they can be worked around using - `--no-http-keep-alive'. */ - CLOSE_FINISH (sock); - else if (keep_alive - && skip_short_body (sock, contlen, chunked_transfer_encoding)) - /* Successfully skipped the body; also keep using the socket. */ - CLOSE_FINISH (sock); + hs->restval = 0; + + /* Normally we are not interested in the response body of a error responses. + But if we are writing a WARC file we are: we like to keep everyting. */ + if (warc_enabled) + { + int err = read_response_body (hs, sock, NULL, contlen, 0, + chunked_transfer_encoding, + u->url, warc_timestamp_str, + warc_request_uuid, warc_ip, type, + statcode, head); + + if (err != RETRFINISHED || hs->res < 0) + { + CLOSE_INVALIDATE (sock); + xfree (head); + xfree_null (type); + return err; + } + else + CLOSE_FINISH (sock); + } else - CLOSE_INVALIDATE (sock); + { + /* Since WARC is disabled, we are not interested in the response body. */ + if (head_only) + /* Pre-1.10 Wget used CLOSE_INVALIDATE here. Now we trust the + servers not to send body in response to a HEAD request, and + those that do will likely be caught by test_socket_open. + If not, they can be worked around using + `--no-http-keep-alive'. */ + CLOSE_FINISH (sock); + else if (keep_alive + && skip_short_body (sock, contlen, chunked_transfer_encoding)) + /* Successfully skipped the body; also keep using the socket. */ + CLOSE_FINISH (sock); + else + CLOSE_INVALIDATE (sock); + } + xfree (head); + xfree_null (type); return RETRFINISHED; } @@ -2512,6 +2822,7 @@ read_header: strerror (errno)); CLOSE_INVALIDATE (sock); xfree (head); + xfree_null (type); return UNLINKERR; } } @@ -2539,6 +2850,7 @@ read_header: hs->local_file); CLOSE_INVALIDATE (sock); xfree (head); + xfree_null (type); return FOPEN_EXCL_ERR; } } @@ -2547,6 +2859,7 @@ read_header: logprintf (LOG_NOTQUIET, "%s: %s\n", hs->local_file, strerror (errno)); CLOSE_INVALIDATE (sock); xfree (head); + xfree_null (type); return FOPENERR; } } @@ -2560,49 +2873,26 @@ read_header: HYPHENP (hs->local_file) ? quote ("STDOUT") : quote (hs->local_file)); } - /* This confuses the timestamping code that checks for file size. - #### The timestamping code should be smarter about file size. */ - if (opt.save_headers && hs->restval == 0) - fwrite (head, 1, strlen (head), fp); + + err = read_response_body (hs, sock, fp, contlen, contrange, + chunked_transfer_encoding, + u->url, warc_timestamp_str, + warc_request_uuid, warc_ip, type, + statcode, head); /* Now we no longer need to store the response header. */ xfree (head); - - /* Download the request body. */ - flags = 0; - if (contlen != -1) - /* If content-length is present, read that much; otherwise, read - until EOF. The HTTP spec doesn't require the server to - actually close the connection when it's done sending data. */ - flags |= rb_read_exactly; - if (hs->restval > 0 && contrange == 0) - /* If the server ignored our range request, instruct fd_read_body - to skip the first RESTVAL bytes of body. */ - flags |= rb_skip_startpos; - - if (chunked_transfer_encoding) - flags |= rb_chunked_transfer_encoding; - - hs->len = hs->restval; - hs->rd_size = 0; - hs->res = fd_read_body (sock, fp, contlen != -1 ? contlen : 0, - hs->restval, &hs->rd_size, &hs->len, &hs->dltime, - flags); + xfree_null (type); if (hs->res >= 0) CLOSE_FINISH (sock); else - { - if (hs->res < 0) - hs->rderrmsg = xstrdup (fd_errstr (sock)); - CLOSE_INVALIDATE (sock); - } + CLOSE_INVALIDATE (sock); if (!output_stream) fclose (fp); - if (hs->res == -2) - return FWRITEERR; - return RETRFINISHED; + + return err; } /* The genuine HTTP loop! This is the part where the retrieval is @@ -2626,6 +2916,12 @@ http_loop (struct url *u, struct url *original_url, char **newloc, char *file_name; bool force_full_retrieve = false; + + /* If we are writing to a WARC file: always retrieve the whole file. */ + if (opt.warc_filename != NULL) + force_full_retrieve = true; + + /* Assert that no value for *LOCAL_FILE was passed. */ assert (local_file == NULL || *local_file == NULL); @@ -2795,6 +3091,18 @@ Spider mode enabled. Check if remote file exists.\n")); /* Fatal errors just return from the function. */ ret = err; goto exit; + case WARC_ERR: + /* A fatal WARC error. */ + logputs (LOG_VERBOSE, "\n"); + logprintf (LOG_NOTQUIET, _("Cannot write to WARC file..\n")); + ret = err; + goto exit; + case WARC_TMP_FOPENERR: case WARC_TMP_FWRITEERR: + /* A fatal WARC error. */ + logputs (LOG_VERBOSE, "\n"); + logprintf (LOG_NOTQUIET, _("Cannot write to temporary WARC file.\n")); + ret = err; + goto exit; case CONSSLERR: /* Another fatal error. */ logprintf (LOG_NOTQUIET, _("Unable to establish SSL connection.\n")); diff --git a/src/init.c b/src/init.c index eae35523..47fdea06 100644 --- a/src/init.c +++ b/src/init.c @@ -88,6 +88,7 @@ CMD_DECLARE (cmd_vector); CMD_DECLARE (cmd_spec_dirstruct); CMD_DECLARE (cmd_spec_header); +CMD_DECLARE (cmd_spec_warc_header); CMD_DECLARE (cmd_spec_htmlify); CMD_DECLARE (cmd_spec_mirror); CMD_DECLARE (cmd_spec_prefer_family); @@ -264,6 +265,15 @@ static const struct { { "verbose", NULL, cmd_spec_verbose }, { "wait", &opt.wait, cmd_time }, { "waitretry", &opt.waitretry, cmd_time }, + { "warccdx", &opt.warc_cdx_enabled, cmd_boolean }, + { "warccdxdedup", &opt.warc_cdx_dedup_filename, cmd_file }, + { "warccompression", &opt.warc_compression_enabled, cmd_boolean }, + { "warcdigests", &opt.warc_digests_enabled, cmd_boolean }, + { "warcfile", &opt.warc_filename, cmd_file }, + { "warcheader", NULL, cmd_spec_warc_header }, + { "warckeeplog", &opt.warc_keep_log, cmd_boolean }, + { "warcmaxsize", &opt.warc_maxsize, cmd_bytes }, + { "warctempdir", &opt.warc_tempdir, cmd_directory }, #ifdef USE_WATT32 { "wdebug", &opt.wdebug, cmd_boolean }, #endif @@ -362,6 +372,14 @@ defaults (void) opt.useservertimestamps = true; opt.show_all_dns_entries = false; + + opt.warc_maxsize = 0; /* 1024 * 1024 * 1024; */ + opt.warc_compression_enabled = true; + opt.warc_digests_enabled = true; + opt.warc_cdx_enabled = false; + opt.warc_cdx_dedup_filename = NULL; + opt.warc_tempdir = NULL; + opt.warc_keep_log = true; } /* Return the user's home directory (strdup-ed), or NULL if none is @@ -1235,6 +1253,27 @@ cmd_spec_header (const char *com, const char *val, void *place_ignored) return true; } +static bool +cmd_spec_warc_header (const char *com, const char *val, void *place_ignored) +{ + /* Empty value means reset the list of headers. */ + if (*val == '\0') + { + free_vec (opt.warc_user_headers); + opt.warc_user_headers = NULL; + return true; + } + + if (!check_user_specified_header (val)) + { + fprintf (stderr, _("%s: %s: Invalid WARC header %s.\n"), + exec_name, com, quote (val)); + return false; + } + opt.warc_user_headers = vec_append (opt.warc_user_headers, val); + return true; +} + static bool cmd_spec_htmlify (const char *com, const char *val, void *place_ignored) { @@ -1639,6 +1678,7 @@ cleanup (void) xfree_null (opt.http_user); xfree_null (opt.http_passwd); free_vec (opt.user_headers); + free_vec (opt.warc_user_headers); # ifdef HAVE_SSL xfree_null (opt.cert_file); xfree_null (opt.private_key); diff --git a/src/log.c b/src/log.c index e6875f6b..0185df19 100644 --- a/src/log.c +++ b/src/log.c @@ -79,6 +79,10 @@ as that of the covered work. */ logging is inhibited, logfp is set back to NULL. */ static FILE *logfp; +/* A second file descriptor pointing to the temporary log file for the + WARC writer. If WARC writing is disabled, this is NULL. */ +static FILE *warclogfp; + /* If true, it means logging is inhibited, i.e. nothing is printed or stored. */ static bool inhibit_logging; @@ -304,6 +308,31 @@ get_log_fp (void) return logfp; return stderr; } + +/* Returns the file descriptor for the secondary log file. This is + WARCLOGFP, except if called before log_init, in which case it + returns stderr. This is useful in case someone calls a logging + function before log_init. + + If logging is inhibited, return NULL. */ + +static FILE * +get_warc_log_fp (void) +{ + if (inhibit_logging) + return NULL; + if (warclogfp) + return warclogfp; + return NULL; +} + +/* Sets the file descriptor for the secondary log file. */ + +void +log_set_warc_log_fp (FILE * fp) +{ + warclogfp = fp; +} /* Log a literal string S. The string is logged as-is, without a newline appended. */ @@ -312,13 +341,17 @@ void logputs (enum log_options o, const char *s) { FILE *fp; + FILE *warcfp; check_redirect_output (); if ((fp = get_log_fp ()) == NULL) return; + warcfp = get_warc_log_fp (); CHECK_VERBOSE (o); FPUTS (s, fp); + if (warcfp != NULL) + FPUTS (s, warcfp); if (save_context_p) saved_append (s); if (flush_log_p) @@ -356,8 +389,9 @@ log_vprintf_internal (struct logvprintf_state *state, const char *fmt, int available_size = sizeof (smallmsg); int numwritten; FILE *fp = get_log_fp (); + FILE *warcfp = get_warc_log_fp (); - if (!save_context_p) + if (!save_context_p && warcfp == NULL) { /* In the simple case just call vfprintf(), to avoid needless allocation and games with vsnprintf(). */ @@ -407,8 +441,11 @@ log_vprintf_internal (struct logvprintf_state *state, const char *fmt, } /* Writing succeeded. */ - saved_append (write_ptr); + if (save_context_p) + saved_append (write_ptr); FPUTS (write_ptr, fp); + if (warcfp != NULL) + FPUTS (write_ptr, warcfp); if (state->bigmsg) xfree (state->bigmsg); @@ -426,6 +463,7 @@ void logflush (void) { FILE *fp = get_log_fp (); + FILE *warcfp = get_warc_log_fp (); if (fp) { /* 2005-10-25 SMS. @@ -440,6 +478,10 @@ logflush (void) fflush (fp); #endif /* def __VMS [else] */ } + + if (warcfp != NULL) + fflush (warcfp); + needs_flushing = false; } @@ -598,6 +640,7 @@ log_dump_context (void) { int num = log_line_current; FILE *fp = get_log_fp (); + FILE *warcfp = get_warc_log_fp (); if (!fp) return; @@ -609,14 +652,23 @@ log_dump_context (void) { struct log_ln *ln = log_lines + num; if (ln->content) - FPUTS (ln->content, fp); + { + FPUTS (ln->content, fp); + if (warcfp != NULL) + FPUTS (ln->content, warcfp); + } ROT_ADVANCE (num); } while (num != log_line_current); if (trailing_line) if (log_lines[log_line_current].content) - FPUTS (log_lines[log_line_current].content, fp); + { + FPUTS (log_lines[log_line_current].content, fp); + if (warcfp != NULL) + FPUTS (log_lines[log_line_current].content, warcfp); + } fflush (fp); + fflush (warcfp); } /* String escape functions. */ diff --git a/src/log.h b/src/log.h index 48c2f1b1..d74ca53d 100644 --- a/src/log.h +++ b/src/log.h @@ -34,8 +34,12 @@ as that of the covered work. */ /* The log file to which Wget writes to after HUP. */ #define DEFAULT_LOGFILE "wget-log" +#include + enum log_options { LOG_VERBOSE, LOG_NOTQUIET, LOG_NONVERBOSE, LOG_ALWAYS }; +void log_set_warc_log_fp (FILE *); + void logprintf (enum log_options, const char *, ...) GCC_FORMAT_ATTR (2, 3); void debug_logprintf (const char *, ...) GCC_FORMAT_ATTR (1, 2); diff --git a/src/main.c b/src/main.c index 05ad0e76..28467359 100644 --- a/src/main.c +++ b/src/main.c @@ -55,6 +55,7 @@ as that of the covered work. */ #include "spider.h" #include "http.h" /* for save_cookies */ #include "ptimer.h" +#include "warc.h" #include #include @@ -287,6 +288,15 @@ static struct cmdline_option option_data[] = { "version", 'V', OPT_FUNCALL, (void *) print_version, no_argument }, { "wait", 'w', OPT_VALUE, "wait", -1 }, { "waitretry", 0, OPT_VALUE, "waitretry", -1 }, + { "warc-cdx", 0, OPT_BOOLEAN, "warccdx", -1 }, + { "warc-compression", 0, OPT_BOOLEAN, "warccompression", -1 }, + { "warc-dedup", 0, OPT_VALUE, "warccdxdedup", -1 }, + { "warc-digests", 0, OPT_BOOLEAN, "warcdigests", -1 }, + { "warc-file", 0, OPT_VALUE, "warcfile", -1 }, + { "warc-header", 0, OPT_VALUE, "warcheader", -1 }, + { "warc-keep-log", 0, OPT_BOOLEAN, "warckeeplog", -1 }, + { "warc-max-size", 0, OPT_VALUE, "warcmaxsize", -1 }, + { "warc-tempdir", 0, OPT_VALUE, "warctempdir", -1 }, #ifdef USE_WATT32 { "wdebug", 0, OPT_BOOLEAN, "wdebug", -1 }, #endif @@ -652,6 +662,29 @@ FTP options:\n"), --retr-symlinks when recursing, get linked-to files (not dir).\n"), "\n", + N_("\ +WARC options:\n"), + N_("\ + --warc-file=FILENAME save request/response data to a .warc.gz file.\n"), + N_("\ + --warc-header=STRING insert STRING into the warcinfo record.\n"), + N_("\ + --warc-max-size=NUMBER set maximum size of WARC files to NUMBER.\n"), + N_("\ + --warc-cdx write CDX index files.\n"), + N_("\ + --warc-dedup=FILENAME do not store records listed in this CDX file.\n"), + N_("\ + --no-warc-compression do not compress WARC files with GZIP.\n"), + N_("\ + --no-warc-digests do not calculate SHA1 digests.\n"), + N_("\ + --no-warc-keep-log do not store the log file in a WARC record.\n"), + N_("\ + --warc-tempdir=DIRECTORY location for temporary files created by the\n\ + WARC writer.\n"), + "\n", + N_("\ Recursive download:\n"), N_("\ @@ -910,6 +943,7 @@ There is NO WARRANTY, to the extent permitted by law.\n"), stdout) < 0) } char *program_name; /* Needed by lib/error.c. */ +char *program_argstring; /* Needed by wget_warc.c. */ int main (int argc, char **argv) @@ -945,6 +979,22 @@ main (int argc, char **argv) windows_main ((char **) &exec_name); #endif + /* Construct the arguments string. */ + int argstring_length = 1; + for (i = 1; i < argc; i++) + argstring_length += strlen (argv[i]) + 2 + 1; + char *p = program_argstring = malloc (argstring_length * sizeof (char)); + for (i = 1; i < argc; i++) + { + *p++ = '"'; + int arglen = strlen (argv[i]); + memcpy (p, argv[i], arglen); + p += arglen; + *p++ = '"'; + *p++ = ' '; + } + *p = '\0'; + /* Load the hard-coded defaults. */ defaults (); @@ -1194,6 +1244,47 @@ for details.\n\n")); } } + if (opt.warc_filename != 0) + { + if (opt.noclobber) + { + fprintf (stderr, + _("WARC output does not work with --no-clobber, " + "--no-clobber will be disabled.\n")); + opt.noclobber = false; + } + if (opt.timestamping) + { + fprintf (stderr, + _("WARC output does not work with timestamping, " + "timestamping will be disabled.\n")); + opt.timestamping = false; + } + if (opt.spider) + { + fprintf (stderr, + _("WARC output does not work with --spider.\n")); + exit (1); + } + if (opt.always_rest) + { + fprintf (stderr, + _("WARC output does not work with --continue, " + "--continue will be disabled.\n")); + opt.always_rest = false; + } + if (opt.warc_cdx_dedup_filename != 0 && !opt.warc_digests_enabled) + { + fprintf (stderr, + _("Digests are disabled; WARC deduplication will " + "not find duplicate records.\n")); + } + if (opt.warc_keep_log) + { + opt.progress_type = "dot"; + } + } + if (opt.ask_passwd && opt.passwd) { fprintf (stderr, @@ -1273,6 +1364,10 @@ for details.\n\n")); /* Initialize logging. */ log_init (opt.lfilename, append_to_log); + /* Open WARC file. */ + if (opt.warc_filename != 0) + warc_init (); + DEBUGP (("DEBUG output created by Wget %s on %s.\n\n", version_string, OS_TYPE)); @@ -1472,7 +1567,12 @@ outputting to a regular file.\n")); if (opt.convert_links && !opt.delete_after) convert_all_links (); + /* Close WARC file. */ + if (opt.warc_filename != 0) + warc_close (); + log_close (); + for (i = 0; i < nurl; i++) xfree (url[i]); cleanup (); diff --git a/src/options.h b/src/options.h index 5e7c1eb6..0be66814 100644 --- a/src/options.h +++ b/src/options.h @@ -87,6 +87,15 @@ struct options FTP. */ char *output_document; /* The output file to which the documents will be printed. */ + char *warc_filename; /* WARC output filename */ + char *warc_tempdir; /* WARC temp dir */ + char *warc_cdx_dedup_filename; /* CDX file to be used for deduplication. */ + wgint warc_maxsize; /* WARC max archive size */ + bool warc_compression_enabled; /* For GZIP compression. */ + bool warc_digests_enabled; /* For SHA1 digests. */ + bool warc_cdx_enabled; /* Create CDX files? */ + bool warc_keep_log; /* Store the log file in a WARC record. */ + char **warc_user_headers; /* User-defined WARC header(s). */ char *user; /* Generic username */ char *passwd; /* Generic password */ diff --git a/src/retr.c b/src/retr.c index 73947658..3df582b8 100644 --- a/src/retr.c +++ b/src/retr.c @@ -139,13 +139,16 @@ limit_bandwidth (wgint bytes, struct ptimer *timer) /* Write data in BUF to OUT. However, if *SKIP is non-zero, skip that amount of data and decrease SKIP. Increment *TOTAL by the amount - of data written. */ + of data written. If OUT2 is not NULL, also write BUF to OUT2. + In case of error writing to OUT, -1 is returned. In case of error + writing to OUT2, -2 is returned. In case of any other error, + 1 is returned. */ static int -write_data (FILE *out, const char *buf, int bufsize, wgint *skip, - wgint *written) +write_data (FILE *out, FILE *out2, const char *buf, int bufsize, + wgint *skip, wgint *written) { - if (!out) + if (out == NULL && out2 == NULL) return 1; if (*skip > bufsize) { @@ -161,7 +164,10 @@ write_data (FILE *out, const char *buf, int bufsize, wgint *skip, return 1; } - fwrite (buf, 1, bufsize, out); + if (out != NULL) + fwrite (buf, 1, bufsize, out); + if (out2 != NULL) + fwrite (buf, 1, bufsize, out2); *written += bufsize; /* Immediately flush the downloaded data. This should not hinder @@ -178,9 +184,17 @@ write_data (FILE *out, const char *buf, int bufsize, wgint *skip, actual justification. (Also, why 16K? Anyone test other values?) */ #ifndef __VMS - fflush (out); + if (out != NULL) + fflush (out); + if (out2 != NULL) + fflush (out2); #endif /* ndef __VMS */ - return !ferror (out); + if (out != NULL && ferror (out)) + return -1; + else if (out2 != NULL && ferror (out2)) + return -2; + else + return 0; } /* Read the contents of file descriptor FD until it the connection @@ -198,13 +212,17 @@ write_data (FILE *out, const char *buf, int bufsize, wgint *skip, the amount of data written to disk. The time it took to download the data is stored to ELAPSED. + If OUT2 is non-NULL, the contents is also written to OUT2. + The function exits and returns the amount of data read. In case of error while reading data, -1 is returned. In case of error while - writing data, -2 is returned. */ + writing data to OUT, -2 is returned. In case of error while writing + data to OUT2, -3 is returned. */ int fd_read_body (int fd, FILE *out, wgint toread, wgint startpos, - wgint *qtyread, wgint *qtywritten, double *elapsed, int flags) + wgint *qtyread, wgint *qtywritten, double *elapsed, int flags, + FILE *out2) { int ret = 0; #undef max @@ -343,9 +361,10 @@ fd_read_body (int fd, FILE *out, wgint toread, wgint startpos, if (ret > 0) { sum_read += ret; - if (!write_data (out, dlbuf, ret, &skip, &sum_written)) + int write_res = write_data (out, out2, dlbuf, ret, &skip, &sum_written); + if (write_res != 0) { - ret = -2; + ret = (write_res == -3) ? -3 : -2; goto out; } if (chunked) diff --git a/src/retr.h b/src/retr.h index 7329b037..22ab9ecd 100644 --- a/src/retr.h +++ b/src/retr.h @@ -50,7 +50,7 @@ enum { rb_chunked_transfer_encoding = 4 }; -int fd_read_body (int, FILE *, wgint, wgint, wgint *, wgint *, double *, int); +int fd_read_body (int, FILE *, wgint, wgint, wgint *, wgint *, double *, int, FILE *); typedef const char *(*hunk_terminator_t) (const char *, const char *, int); diff --git a/src/test.c b/src/test.c index e7ce54cf..80abafff 100644 --- a/src/test.c +++ b/src/test.c @@ -46,6 +46,8 @@ const char *test_append_uri_pathel(); const char *test_are_urls_equal(); const char *test_is_robots_txt_url(); +const char *program_argstring = "TEST"; + int tests_run; static const char * diff --git a/src/warc.c b/src/warc.c new file mode 100644 index 00000000..77ef3692 --- /dev/null +++ b/src/warc.c @@ -0,0 +1,1332 @@ +/* Utility functions for writing WARC files. */ +#define _GNU_SOURCE + +#include "wget.h" +#include "hash.h" +#include "utils.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#ifdef HAVE_LIBUUID +#include +#endif + +#include "warc.h" + +extern char *version_string; + +/* Set by main in main.c */ +extern char *program_argstring; + + +/* The log file (a temporary file that contains a copy + of the wget log). */ +static FILE *warc_log_fp; + +/* The manifest file (a temporary file that contains the + warcinfo uuid of every file in this crawl). */ +static FILE *warc_manifest_fp; + +/* The current WARC file (or NULL, if WARC is disabled). */ +static FILE *warc_current_file; + +/* The gzip stream for the current WARC file + (or NULL, if WARC or gzip is disabled). */ +static gzFile *warc_current_gzfile; + +/* The offset of the current gzip record in the WARC file. */ +static size_t warc_current_gzfile_offset; + +/* The uncompressed size (so far) of the current record. */ +static size_t warc_current_gzfile_uncompressed_size; + +/* This is true until a warc_write_* method fails. */ +static bool warc_write_ok; + +/* The current CDX file (or NULL, if CDX is disabled). */ +static FILE *warc_current_cdx_file; + +/* The record id of the warcinfo record of the current WARC file. */ +static char *warc_current_warcinfo_uuid_str; + +/* The file name of the current WARC file. */ +static char *warc_current_filename; + +/* The serial number of the current WARC file. This number is + incremented each time a new file is opened and is used in the + WARC file's filename. */ +static int warc_current_file_number; + +/* The table of CDX records, if deduplication is enabled. */ +struct hash_table * warc_cdx_dedup_table; + +static bool warc_start_new_file (bool meta); + + +struct warc_cdx_record +{ + char *url; + char *uuid; + char digest[SHA1_DIGEST_SIZE]; +}; + +static unsigned long +warc_hash_sha1_digest (const void *key) +{ + /* We just use some of the first bytes of the digest. */ + unsigned long v = 0; + memcpy (&v, key, sizeof (unsigned long)); + return v; +} + +static int +warc_cmp_sha1_digest (const void *digest1, const void *digest2) +{ + return !memcmp (digest1, digest2, SHA1_DIGEST_SIZE); +} + + + +/* Writes SIZE bytes from BUFFER to the current WARC file, + through gzwrite if compression is enabled. + Returns the number of uncompressed bytes written. */ +static size_t +warc_write_buffer (const char *buffer, size_t size) +{ + if (warc_current_gzfile) + { + warc_current_gzfile_uncompressed_size += size; + return gzwrite (warc_current_gzfile, buffer, size); + } + else + return fwrite (buffer, 1, size, warc_current_file); +} + +/* Writes STR to the current WARC file. + Returns false and set warc_write_ok to false if there + is an error. */ +static bool +warc_write_string (const char *str) +{ + if (!warc_write_ok) + return false; + + size_t n = strlen (str); + if (n != warc_write_buffer (str, n)) + warc_write_ok = false; + + return warc_write_ok; +} + + +#define EXTRA_GZIP_HEADER_SIZE 12 +#define GZIP_STATIC_HEADER_SIZE 10 +#define FLG_FEXTRA 0x04 +#define OFF_FLG 3 + +/* Starts a new WARC record. Writes the version header. + If opt.warc_maxsize is set and the current file is becoming + too large, this will open a new WARC file. + + If compression is enabled, this will start a new + gzip stream in the current WARC file. + + Returns false and set warc_write_ok to false if there + is an error. */ +static bool +warc_write_start_record () +{ + if (!warc_write_ok) + return false; + + fflush (warc_current_file); + if (opt.warc_maxsize > 0 && ftell (warc_current_file) >= opt.warc_maxsize) + warc_start_new_file (false); + + /* Start a GZIP stream, if required. */ + if (opt.warc_compression_enabled) + { + /* Record the starting offset of the new record. */ + warc_current_gzfile_offset = ftell (warc_current_file); + + /* Reserve space for the extra GZIP header field. + In warc_write_end_record we will fill this space + with information about the uncompressed and + compressed size of the record. */ + fprintf (warc_current_file, "XXXXXXXXXXXX"); + fflush (warc_current_file); + + /* Start a new GZIP stream. */ + warc_current_gzfile = gzdopen (dup (fileno (warc_current_file)), "wb+9"); + warc_current_gzfile_uncompressed_size = 0; + + if (warc_current_gzfile == NULL) + { + logprintf (LOG_NOTQUIET, _("Error opening GZIP stream to WARC file.\n")); + warc_write_ok = false; + return false; + } + } + + warc_write_string ("WARC/1.0\r\n"); + return warc_write_ok; +} + +/* Writes a WARC header to the current WARC record. + This method may be run after warc_write_start_record and + before warc_write_block_from_file. */ +static bool +warc_write_header (const char *name, const char *value) +{ + if (value) + { + warc_write_string (name); + warc_write_string (": "); + warc_write_string (value); + warc_write_string ("\r\n"); + } + return warc_write_ok; +} + +/* Copies the contents of DATA_IN to the WARC record. + Adds a Content-Length header to the WARC record. + Run this method after warc_write_header, + then run warc_write_end_record. */ +static bool +warc_write_block_from_file (FILE *data_in) +{ + /* Add the Content-Length header. */ + char *content_length; + fseek (data_in, 0L, SEEK_END); + if (! asprintf (&content_length, "%ld", ftell (data_in))) + { + warc_write_ok = false; + return false; + } + warc_write_header ("Content-Length", content_length); + free (content_length); + + /* End of the WARC header section. */ + warc_write_string ("\r\n"); + + if (fseek (data_in, 0L, SEEK_SET) != 0) + warc_write_ok = false; + + /* Copy the data in the file to the WARC record. */ + char buffer[BUFSIZ]; + size_t s; + while (warc_write_ok && (s = fread (buffer, 1, BUFSIZ, data_in)) > 0) + { + if (warc_write_buffer (buffer, s) < s) + warc_write_ok = false; + } + + return warc_write_ok; +} + +/* Run this method to close the current WARC record. + + If compression is enabled, this method closes the + current GZIP stream and fills the extra GZIP header + with the uncompressed and compressed length of the + record. */ +static bool +warc_write_end_record () +{ + warc_write_buffer ("\r\n\r\n", 4); + + /* We start a new gzip stream for each record. */ + if (warc_write_ok && warc_current_gzfile) + { + if (gzclose (warc_current_gzfile) != Z_OK) + { + warc_write_ok = false; + return false; + } + + fflush (warc_current_file); + fseek (warc_current_file, 0, SEEK_END); + + /* The WARC standard suggests that we add 'skip length' data in the + extra header field of the GZIP stream. + + In warc_write_start_record we reserved space for this extra header. + This extra space starts at warc_current_gzfile_offset and fills + EXTRA_GZIP_HEADER_SIZE bytes. The static GZIP header starts at + warc_current_gzfile_offset + EXTRA_GZIP_HEADER_SIZE. + + We need to do three things: + 1. Move the static GZIP header to warc_current_gzfile_offset; + 2. Set the FEXTRA flag in the GZIP header; + 3. Write the extra GZIP header after the static header, that is, + starting at warc_current_gzfile_offset + GZIP_STATIC_HEADER_SIZE. + */ + + /* Calculate the uncompressed and compressed sizes. */ + size_t current_offset = ftell (warc_current_file); + size_t uncompressed_size = current_offset - warc_current_gzfile_offset; + size_t compressed_size = warc_current_gzfile_uncompressed_size; + + /* Go back to the static GZIP header. */ + fseek (warc_current_file, warc_current_gzfile_offset + EXTRA_GZIP_HEADER_SIZE, SEEK_SET); + + /* Read the header. */ + char static_header[GZIP_STATIC_HEADER_SIZE]; + size_t result = fread (static_header, 1, GZIP_STATIC_HEADER_SIZE, warc_current_file); + if (result != GZIP_STATIC_HEADER_SIZE) + { + warc_write_ok = false; + return false; + } + + /* Set the FEXTRA flag in the flags byte of the header. */ + static_header[OFF_FLG] = static_header[OFF_FLG] | FLG_FEXTRA; + + /* Write the header back to the file, but starting at warc_current_gzfile_offset. */ + fseek (warc_current_file, warc_current_gzfile_offset, SEEK_SET); + fwrite (static_header, 1, GZIP_STATIC_HEADER_SIZE, warc_current_file); + + /* Prepare the extra GZIP header. */ + char extra_header[EXTRA_GZIP_HEADER_SIZE]; + /* XLEN, the length of the extra header fields. */ + extra_header[0] = ((EXTRA_GZIP_HEADER_SIZE - 2) & 255); + extra_header[1] = ((EXTRA_GZIP_HEADER_SIZE - 2) >> 8) & 255; + /* The extra header field identifier for the WARC skip length. */ + extra_header[2] = 's'; + extra_header[3] = 'l'; + /* The size of the uncompressed record. */ + extra_header[4] = (uncompressed_size & 255); + extra_header[5] = (uncompressed_size >> 8) & 255; + extra_header[6] = (uncompressed_size >> 16) & 255; + extra_header[7] = (uncompressed_size >> 24) & 255; + /* The size of the compressed record. */ + extra_header[8] = (compressed_size & 255); + extra_header[9] = (compressed_size >> 8) & 255; + extra_header[10] = (compressed_size >> 16) & 255; + extra_header[11] = (compressed_size >> 24) & 255; + + /* Write the extra header after the static header. */ + fseek (warc_current_file, warc_current_gzfile_offset + GZIP_STATIC_HEADER_SIZE, SEEK_SET); + fwrite (extra_header, 1, EXTRA_GZIP_HEADER_SIZE, warc_current_file); + + /* Done, move back to the end of the file. */ + fflush (warc_current_file); + fseek (warc_current_file, 0, SEEK_END); + } + + return warc_write_ok; +} + + +/* Writes the WARC-Date header for the given timestamp to + the current WARC record. + If timestamp is NULL, the current time will be used. */ +static bool +warc_write_date_header (char *timestamp) +{ + if (timestamp == NULL) + { + char current_timestamp[21]; + warc_timestamp (current_timestamp); + timestamp = current_timestamp; + } + return warc_write_header ("WARC-Date", timestamp); +} + +/* Writes the WARC-IP-Address header for the given IP to + the current WARC record. If IP is NULL, no header will + be written. */ +static bool +warc_write_ip_header (ip_address *ip) +{ + if (ip != NULL) + return warc_write_header ("WARC-IP-Address", print_address (ip)); + else + return warc_write_ok; +} + + +/* warc_sha1_stream_with_payload is a modified copy of sha1_stream + from gnulib/sha1.c. This version calculates two digests in one go. + + Compute SHA1 message digests for bytes read from STREAM. The + digest of the complete file will be written into the 16 bytes + beginning at RES_BLOCK. + + If payload_offset >= 0, a second digest will be calculated of the + portion of the file starting at payload_offset and continuing to + the end of the file. The digest number will be written into the + 16 bytes beginning ad RES_PAYLOAD. */ +static int +warc_sha1_stream_with_payload (FILE *stream, void *res_block, void *res_payload, long int payload_offset) +{ +#define BLOCKSIZE 32768 + + struct sha1_ctx ctx_block; + struct sha1_ctx ctx_payload; + long int pos; + size_t sum; + + char *buffer = malloc (BLOCKSIZE + 72); + if (!buffer) + return 1; + + /* Initialize the computation context. */ + sha1_init_ctx (&ctx_block); + if (payload_offset >= 0) + sha1_init_ctx (&ctx_payload); + + pos = 0; + + /* Iterate over full file contents. */ + while (1) + { + /* We read the file in blocks of BLOCKSIZE bytes. One call of the + computation function processes the whole buffer so that with the + next round of the loop another block can be read. */ + size_t n; + sum = 0; + + /* Read block. Take care for partial reads. */ + while (1) + { + n = fread (buffer + sum, 1, BLOCKSIZE - sum, stream); + + sum += n; + pos += n; + + if (sum == BLOCKSIZE) + break; + + if (n == 0) + { + /* Check for the error flag IFF N == 0, so that we don't + exit the loop after a partial read due to e.g., EAGAIN + or EWOULDBLOCK. */ + if (ferror (stream)) + { + free (buffer); + return 1; + } + goto process_partial_block; + } + + /* We've read at least one byte, so ignore errors. But always + check for EOF, since feof may be true even though N > 0. + Otherwise, we could end up calling fread after EOF. */ + if (feof (stream)) + goto process_partial_block; + } + + /* Process buffer with BLOCKSIZE bytes. Note that + BLOCKSIZE % 64 == 0 + */ + sha1_process_block (buffer, BLOCKSIZE, &ctx_block); + if (payload_offset >= 0 && payload_offset < pos) + { + /* At least part of the buffer contains data from payload. */ + int start_of_payload = payload_offset - (pos - BLOCKSIZE); + if (start_of_payload <= 0) + /* All bytes in the buffer belong to the payload. */ + start_of_payload = 0; + + /* Process the payload part of the buffer. + Note: we can't use sha1_process_block here even if we + process the complete buffer. Because the payload doesn't + have to start with a full block, there may still be some + bytes left from the previous buffer. Therefore, we need + to continue with sha1_process_bytes. */ + sha1_process_bytes (buffer + start_of_payload, BLOCKSIZE - start_of_payload, &ctx_payload); + } + } + + process_partial_block:; + + /* Process any remaining bytes. */ + if (sum > 0) + { + sha1_process_bytes (buffer, sum, &ctx_block); + if (payload_offset >= 0 && payload_offset < pos) + { + /* At least part of the buffer contains data from payload. */ + int start_of_payload = payload_offset - (pos - sum); + if (start_of_payload <= 0) + /* All bytes in the buffer belong to the payload. */ + start_of_payload = 0; + + /* Process the payload part of the buffer. */ + sha1_process_bytes (buffer + start_of_payload, sum - start_of_payload, &ctx_payload); + } + } + + /* Construct result in desired memory. */ + sha1_finish_ctx (&ctx_block, res_block); + if (payload_offset >= 0) + sha1_finish_ctx (&ctx_payload, res_payload); + free (buffer); + return 0; + +#undef BLOCKSIZE +} + +/* Converts the SHA1 digest to a base32-encoded string. + "sha1:DIGEST\0" (Allocates a new string for the response.) */ +static char * +warc_base32_sha1_digest (char *sha1_digest) +{ + // length: "sha1:" + digest + "\0" + char *sha1_base32 = malloc (BASE32_LENGTH(SHA1_DIGEST_SIZE) + 1 + 5 ); + base32_encode (sha1_digest, SHA1_DIGEST_SIZE, sha1_base32 + 5, BASE32_LENGTH(SHA1_DIGEST_SIZE) + 1); + memcpy (sha1_base32, "sha1:", 5); + sha1_base32[BASE32_LENGTH(SHA1_DIGEST_SIZE) + 5] = '\0'; + return sha1_base32; +} + + +/* Sets the digest headers of the record. + This method will calculate the block digest and, if payload_offset >= 0, + will also calculate the payload digest of the payload starting at the + provided offset. */ +static void +warc_write_digest_headers (FILE *file, long payload_offset) +{ + if (opt.warc_digests_enabled) + { + /* Calculate the block and payload digests. */ + char sha1_res_block[SHA1_DIGEST_SIZE]; + char sha1_res_payload[SHA1_DIGEST_SIZE]; + + rewind (file); + if (warc_sha1_stream_with_payload (file, sha1_res_block, sha1_res_payload, payload_offset) == 0) + { + char *digest; + + digest = warc_base32_sha1_digest (sha1_res_block); + warc_write_header ("WARC-Block-Digest", digest); + free (digest); + + if (payload_offset >= 0) + { + digest = warc_base32_sha1_digest (sha1_res_payload); + warc_write_header ("WARC-Payload-Digest", digest); + free (digest); + } + } + } +} + + +/* Fills timestamp with the current time and date. + The UTC time is formatted following ISO 8601, as required + for use in the WARC-Date header. + The timestamp will be 21 characters long. */ +void +warc_timestamp (char *timestamp) +{ + time_t rawtime; + struct tm * timeinfo; + time ( &rawtime ); + timeinfo = gmtime (&rawtime); + strftime (timestamp, 21, "%Y-%m-%dT%H:%M:%SZ", timeinfo); +} + +/* Fills uuid_str with a UUID based on random numbers. + (See RFC 4122, UUID version 4.) + + Note: this is a fallback method, it is much better to use the + methods provided by libuuid. + + The uuid_str will be 36 characters long. */ +static void +warc_uuid_random (char *uuid_str) +{ + // RFC 4122, a version 4 UUID with only random numbers + + unsigned char uuid_data[16]; + int i; + for (i=0; i<16; i++) + uuid_data[i] = random_number (255); + + // Set the four most significant bits (bits 12 through 15) of the + // time_hi_and_version field to the 4-bit version number + uuid_data[6] = (uuid_data[6] & 0x0F) | 0x40; + + // Set the two most significant bits (bits 6 and 7) of the + // clock_seq_hi_and_reserved to zero and one, respectively. + uuid_data[8] = (uuid_data[8] & 0xBF) | 0x80; + + sprintf (uuid_str, + "%02x%02x%02x%02x-%02x%02x-%02x%02x-%02x%02x-%02x%02x%02x%02x%02x%02x", + uuid_data[0], uuid_data[1], uuid_data[2], uuid_data[3], uuid_data[4], + uuid_data[5], uuid_data[6], uuid_data[7], uuid_data[8], uuid_data[9], + uuid_data[10], uuid_data[11], uuid_data[12], uuid_data[13], uuid_data[14], + uuid_data[15]); +} + +/* Fills urn_str with a UUID in the format required + for the WARC-Record-Id header. + The string will be 47 characters long. */ +void +warc_uuid_str (char *urn_str) +{ + char uuid_str[37]; + +# ifdef HAVE_LIBUUID + uuid_t record_id; + uuid_generate (record_id); + uuid_unparse (record_id, uuid_str); +# else + warc_uuid_random (uuid_str); +# endif + + sprintf (urn_str, "", uuid_str); +} + +/* Write a warcinfo record to the current file. + Updates warc_current_warcinfo_uuid_str. */ +bool +warc_write_warcinfo_record (char *filename) +{ + /* Write warc-info record as the first record of the file. */ + /* We add the record id of this info record to the other records in the file. */ + warc_current_warcinfo_uuid_str = (char *) malloc (48); + warc_uuid_str (warc_current_warcinfo_uuid_str); + + char timestamp[22]; + warc_timestamp (timestamp); + + char *filename_copy, *filename_basename; + filename_copy = strdup (filename); + filename_basename = basename (filename_copy); + + warc_write_start_record (); + warc_write_header ("WARC-Type", "warcinfo"); + warc_write_header ("Content-Type", "application/warc-fields"); + warc_write_header ("WARC-Date", timestamp); + warc_write_header ("WARC-Record-ID", warc_current_warcinfo_uuid_str); + warc_write_header ("WARC-Filename", filename_basename); + + /* Create content. */ + FILE *warc_tmp = warc_tempfile (); + if (warc_tmp == NULL) + { + free (filename_copy); + return false; + } + + fprintf (warc_tmp, "software: Wget/%s (%s)\r\n", version_string, OS_TYPE); + fprintf (warc_tmp, "format: WARC File Format 1.0\r\n"); + fprintf (warc_tmp, "conformsTo: http://bibnum.bnf.fr/WARC/WARC_ISO_28500_version1_latestdraft.pdf\r\n"); + fprintf (warc_tmp, "robots: %s\r\n", (opt.use_robots ? "classic" : "off")); + fprintf (warc_tmp, "wget-arguments: %s\r\n", program_argstring); + /* Add the user headers, if any. */ + if (opt.warc_user_headers) + { + int i; + for (i = 0; opt.warc_user_headers[i]; i++) + fprintf (warc_tmp, "%s\r\n", opt.warc_user_headers[i]); + } + fprintf(warc_tmp, "\r\n"); + + warc_write_digest_headers (warc_tmp, -1); + warc_write_block_from_file (warc_tmp); + warc_write_end_record (); + + if (! warc_write_ok) + { + logprintf (LOG_NOTQUIET, _("Error writing warcinfo record to WARC file.\n")); + } + + free (filename_copy); + fclose (warc_tmp); + return warc_write_ok; +} + +/* Opens a new WARC file. + If META is true, generates a filename ending with 'meta.warc.gz'. + + This method will: + 1. close the current WARC file (if there is one); + 2. increment warc_current_file_number; + 3. open a new WARC file; + 4. write the initial warcinfo record. + + Returns true on success, false otherwise. + */ +static bool +warc_start_new_file (bool meta) +{ + if (opt.warc_filename == NULL) + return false; + + if (warc_current_file != NULL) + fclose (warc_current_file); + if (warc_current_warcinfo_uuid_str) + free (warc_current_warcinfo_uuid_str); + if (warc_current_filename) + free (warc_current_filename); + + warc_current_file_number++; + + int base_filename_length = strlen (opt.warc_filename); + /* filename format: base + "-" + 5 digit serial number + ".warc.gz" */ + char *new_filename = malloc (base_filename_length + 1 + 5 + 8 + 1); + warc_current_filename = new_filename; + + char *extension = (opt.warc_compression_enabled ? "warc.gz" : "warc"); + + /* If max size is enabled, we add a serial number to the file names. */ + if (meta) + sprintf (new_filename, "%s-meta.%s", opt.warc_filename, extension); + else if (opt.warc_maxsize > 0) + sprintf (new_filename, "%s-%05d.%s", opt.warc_filename, warc_current_file_number, extension); + else + sprintf (new_filename, "%s.%s", opt.warc_filename, extension); + + logprintf (LOG_VERBOSE, _("Opening WARC file %s.\n\n"), quote (new_filename)); + + /* Open the WARC file. */ + warc_current_file = fopen (new_filename, "wb+"); + if (warc_current_file == NULL) + { + logprintf (LOG_NOTQUIET, _("Error opening WARC file %s.\n"), quote (new_filename)); + return false; + } + + if (! warc_write_warcinfo_record (new_filename)) + return false; + + /* Add warcinfo uuid to manifest. */ + if (warc_manifest_fp) + fprintf (warc_manifest_fp, "%s\n", warc_current_warcinfo_uuid_str); + + return true; +} + +/* Opens the CDX file for output. */ +static bool +warc_start_cdx_file () +{ + int filename_length = strlen (opt.warc_filename); + char *cdx_filename = alloca (filename_length + 4 + 1); + memcpy (cdx_filename, opt.warc_filename, filename_length); + memcpy (cdx_filename + filename_length, ".cdx", 5); + warc_current_cdx_file = fopen (cdx_filename, "a+"); + if (warc_current_cdx_file == NULL) + return false; + + /* Print the CDX header. + * + * a - original url + * b - date + * m - mime type + * s - response code + * k - new style checksum + * r - redirect + * M - meta tags + * V - compressed arc file offset + * g - file name + * u - record-id + */ + fprintf (warc_current_cdx_file, " CDX a b a m s k r M V g u\n"); + fflush (warc_current_cdx_file); + + return true; +} + +#define CDX_FIELDSEP " \t\r\n" + +/* Parse the CDX header and find the field numbers of the original url, + checksum and record ID fields. */ +static bool +warc_parse_cdx_header (char *lineptr, int *field_num_original_url, int *field_num_checksum, int *field_num_record_id) +{ + *field_num_original_url = -1; + *field_num_checksum = -1; + *field_num_record_id = -1; + + char *token; + char *save_ptr; + token = strtok_r (lineptr, CDX_FIELDSEP, &save_ptr); + + if (token != NULL && strcmp (token, "CDX") == 0) + { + int field_num = 0; + while (token != NULL) + { + token = strtok_r (NULL, CDX_FIELDSEP, &save_ptr); + if (token != NULL) + { + switch (token[0]) + { + case 'a': + *field_num_original_url = field_num; + break; + case 'k': + *field_num_checksum = field_num; + break; + case 'u': + *field_num_record_id = field_num; + break; + } + } + field_num++; + } + } + + return *field_num_original_url != -1 + && *field_num_checksum != -1 + && *field_num_record_id != -1; +} + +/* Parse the CDX record and add it to the warc_cdx_dedup_table hash table. */ +static void +warc_process_cdx_line (char *lineptr, int field_num_original_url, int field_num_checksum, int field_num_record_id) +{ + char *original_url = NULL; + char *checksum = NULL; + char *record_id = NULL; + + char *token; + char *save_ptr; + token = strtok_r (lineptr, CDX_FIELDSEP, &save_ptr); + + /* Read this line to get the fields we need. */ + int field_num = 0; + while (token != NULL) + { + char **val; + if (field_num == field_num_original_url) + val = &original_url; + else if (field_num == field_num_checksum) + val = &checksum; + else if (field_num == field_num_record_id) + val = &record_id; + else + val = NULL; + + if (val != NULL) + *val = strdup (token); + + token = strtok_r (NULL, CDX_FIELDSEP, &save_ptr); + field_num++; + } + + if (original_url != NULL && checksum != NULL && record_id != NULL) + { + /* For some extra efficiency, we decode the base32 encoded + checksum value. This should produce exactly SHA1_DIGEST_SIZE + bytes. */ + size_t checksum_l; + char * checksum_v; + base32_decode_alloc (checksum, strlen (checksum), &checksum_v, &checksum_l); + free (checksum); + + if (checksum_v != NULL && checksum_l == SHA1_DIGEST_SIZE) + { + /* This is a valid line with a valid checksum. */ + struct warc_cdx_record * rec = malloc (sizeof (struct warc_cdx_record)); + rec->url = original_url; + rec->uuid = record_id; + memcpy (rec->digest, checksum_v, SHA1_DIGEST_SIZE); + hash_table_put (warc_cdx_dedup_table, rec->digest, rec); + free (checksum_v); + } + else + { + free (original_url); + if (checksum_v != NULL) + free (checksum_v); + free (record_id); + } + } +} + +/* Loads the CDX file from opt.warc_cdx_dedup_filename and fills + the warc_cdx_dedup_table. */ +bool +warc_load_cdx_dedup_file () +{ + FILE *f = fopen (opt.warc_cdx_dedup_filename, "r"); + if (f == NULL) + return false; + + int field_num_original_url = -1; + int field_num_checksum = -1; + int field_num_record_id = -1; + + char *lineptr = NULL; + size_t n = 0; + size_t line_length; + + /* The first line should contain the CDX header. + Format: " CDX x x x x x" + where x are field type indicators. For our purposes, we only + need 'a' (the original url), 'k' (the SHA1 checksum) and + 'u' (the WARC record id). */ + line_length = getline (&lineptr, &n, f); + if (line_length != -1) + warc_parse_cdx_header (lineptr, &field_num_original_url, &field_num_checksum, &field_num_record_id); + + /* If the file contains all three fields, read the complete file. */ + if (field_num_original_url == -1 + || field_num_checksum == -1 + || field_num_record_id == -1) + { + if (field_num_original_url == -1) + logprintf (LOG_NOTQUIET, _("CDX file does not list original urls. (Missing column 'a'.)\n")); + if (field_num_checksum == -1) + logprintf (LOG_NOTQUIET, _("CDX file does not list checksums. (Missing column 'k'.)\n")); + if (field_num_record_id == -1) + logprintf (LOG_NOTQUIET, _("CDX file does not list record ids. (Missing column 'u'.)\n")); + } + else + { + /* Initialize the table. */ + warc_cdx_dedup_table = hash_table_new (1000, warc_hash_sha1_digest, warc_cmp_sha1_digest); + + do + { + line_length = getline (&lineptr, &n, f); + if (line_length != -1) + warc_process_cdx_line (lineptr, field_num_original_url, field_num_checksum, field_num_record_id); + + } + while (line_length != -1); + + /* Print results. */ + int nrecords = hash_table_count (warc_cdx_dedup_table); + logprintf (LOG_VERBOSE, ngettext ("Loaded %d record from CDX.\n\n", + "Loaded %d records from CDX.\n\n", nrecords), + nrecords); + } + + fclose (f); + + return true; +} +#undef CDX_FIELDSEP + +/* Returns the existing duplicate CDX record for the given url and payload + digest. Returns NULL if the url is not found or if the payload digest + does not match, or if CDX deduplication is disabled. */ +static struct warc_cdx_record * +warc_find_duplicate_cdx_record (char *url, char *sha1_digest_payload) +{ + if (warc_cdx_dedup_table == NULL) + return NULL; + + char *key; + struct warc_cdx_record *rec_existing; + hash_table_get_pair (warc_cdx_dedup_table, sha1_digest_payload, &key, &rec_existing); + + if (rec_existing != NULL && strcmp (rec_existing->url, url) == 0) + return rec_existing; + else + return NULL; +} + +/* Initializes the WARC writer (if opt.warc_filename is set). + This should be called before any WARC record is written. */ +void +warc_init () +{ + warc_write_ok = true; + + if (opt.warc_filename != NULL) + { + if (opt.warc_cdx_dedup_filename != NULL) + { + if (! warc_load_cdx_dedup_file ()) + { + logprintf (LOG_NOTQUIET, + _("Could not read CDX file %s for deduplication.\n"), + quote (opt.warc_cdx_dedup_filename)); + exit(1); + } + } + + warc_manifest_fp = warc_tempfile (); + if (warc_manifest_fp == NULL) + { + logprintf (LOG_NOTQUIET, _("Could not open temporary WARC manifest file.\n")); + exit(1); + } + + if (opt.warc_keep_log) + { + warc_log_fp = warc_tempfile (); + if (warc_log_fp == NULL) + { + logprintf (LOG_NOTQUIET, _("Could not open temporary WARC log file.\n")); + exit(1); + } + log_set_warc_log_fp (warc_log_fp); + } + + warc_current_file_number = -1; + if (! warc_start_new_file (false)) + { + logprintf (LOG_NOTQUIET, _("Could not open WARC file.\n")); + exit(1); + } + + if (opt.warc_cdx_enabled) + { + if (! warc_start_cdx_file ()) + { + logprintf (LOG_NOTQUIET, _("Could not open CDX file for output.\n")); + exit(1); + } + } + } +} + +/* Writes metadata (manifest, configuration, log file) to the WARC file. */ +void +warc_write_metadata () +{ + /* If there are multiple WARC files, the metadata should be written to a separate file. */ + if (opt.warc_maxsize > 0) + warc_start_new_file (true); + + char manifest_uuid [48]; + warc_uuid_str (manifest_uuid); + + fflush (warc_manifest_fp); + warc_write_resource_record (manifest_uuid, + "metadata://gnu.org/software/wget/warc/MANIFEST.txt", + NULL, NULL, NULL, "text/plain", + warc_manifest_fp, -1); + /* warc_write_resource_record has closed warc_manifest_fp. */ + + FILE * warc_tmp_fp = warc_tempfile (); + if (warc_tmp_fp == NULL) + { + logprintf (LOG_NOTQUIET, _("Could not open temporary WARC file.\n")); + exit(1); + } + fflush (warc_tmp_fp); + fprintf (warc_tmp_fp, "%s\n", program_argstring); + + warc_write_resource_record (manifest_uuid, + "metadata://gnu.org/software/wget/warc/wget_arguments.txt", + NULL, NULL, NULL, "text/plain", + warc_tmp_fp, -1); + /* warc_write_resource_record has closed warc_tmp_fp. */ + + if (warc_log_fp != NULL) + { + warc_write_resource_record (NULL, + "metadata://gnu.org/software/wget/warc/wget.log", + NULL, manifest_uuid, NULL, "text/plain", + warc_log_fp, -1); + /* warc_write_resource_record has closed warc_log_fp. */ + + warc_log_fp = NULL; + log_set_warc_log_fp (NULL); + } +} + +/* Finishes the WARC writing. + This should be called at the end of the program. */ +void +warc_close () +{ + if (warc_current_file != NULL) + { + warc_write_metadata (); + free (warc_current_warcinfo_uuid_str); + fclose (warc_current_file); + } + if (warc_current_cdx_file != NULL) + fclose (warc_current_cdx_file); + if (warc_log_fp != NULL) + { + fclose (warc_log_fp); + log_set_warc_log_fp (NULL); + } +} + +/* Creates a temporary file for writing WARC output. + The temporary file will be created in opt.warc_tempdir. + Returns the pointer to the temporary file, or NULL. */ +FILE * +warc_tempfile () +{ + char filename[100]; + if (path_search (filename, 100, opt.warc_tempdir, "wget", true) == -1) + return NULL; + + int fd = mkstemp (filename); + if (fd < 0) + return NULL; + + if (unlink (filename) < 0) + return NULL; + + return fdopen (fd, "wb+"); +} + + +/* Writes a request record to the WARC file. + url is the target uri of the request, + timestamp_str is the timestamp of the request (generated with warc_timestamp), + record_uuid is the uuid of the request (generated with warc_uuid_str), + body is a pointer to a file containing the request headers and body. + ip is the ip address of the server (or NULL), + Calling this function will close body. + Returns true on success, false on error. */ +bool +warc_write_request_record (char *url, char *timestamp_str, char *record_uuid, ip_address *ip, FILE *body, long int payload_offset) +{ + warc_write_start_record (); + warc_write_header ("WARC-Type", "request"); + warc_write_header ("WARC-Target-URI", url); + warc_write_header ("Content-Type", "application/http;msgtype=request"); + warc_write_date_header (timestamp_str); + warc_write_header ("WARC-Record-ID", record_uuid); + warc_write_ip_header (ip); + warc_write_header ("WARC-Warcinfo-ID", warc_current_warcinfo_uuid_str); + warc_write_digest_headers (body, payload_offset); + warc_write_block_from_file (body); + warc_write_end_record (); + + fclose (body); + + return warc_write_ok; +} + +/* Writes a response record to the CDX file. + url is the target uri of the request/response, + timestamp_str is the timestamp of the request that generated this response, + (generated with warc_timestamp), + mime_type is the mime type of the response body (will be printed to CDX), + response_code is the HTTP response code (will be printed to CDX), + payload_digest is the sha1 digest of the payload, + redirect_location is the contents of the Location: header, or NULL (will be printed to CDX), + offset is the position of the WARC record in the WARC file, + warc_filename is the filename of the WARC, + response_uuid is the uuid of the response. + Returns true on success, false on error. */ +static bool +warc_write_cdx_record (char *url, char *timestamp_str, char *mime_type, int response_code, char *payload_digest, char *redirect_location, size_t offset, char *warc_filename, char *response_uuid) +{ + /* Transform the timestamp. */ + char timestamp_str_cdx [15]; + memcpy (timestamp_str_cdx , timestamp_str , 4); /* "YYYY" "-" */ + memcpy (timestamp_str_cdx + 4, timestamp_str + 5, 2); /* "mm" "-" */ + memcpy (timestamp_str_cdx + 6, timestamp_str + 8, 2); /* "dd" "T" */ + memcpy (timestamp_str_cdx + 8, timestamp_str + 11, 2); /* "HH" ":" */ + memcpy (timestamp_str_cdx + 10, timestamp_str + 14, 2); /* "MM" ":" */ + memcpy (timestamp_str_cdx + 12, timestamp_str + 17, 2); /* "SS" "Z" */ + timestamp_str_cdx[14] = '\0'; + + /* Rewrite the checksum. */ + char *checksum; + if (payload_digest != NULL) + checksum = payload_digest + 5; /* Skip the "sha1:" */ + else + checksum = "-"; + + if (mime_type == NULL || strlen(mime_type) == 0) + mime_type = "-"; + if (redirect_location == NULL || strlen(redirect_location) == 0) + redirect_location = "-"; + + /* Print the CDX line. */ + fprintf (warc_current_cdx_file, "%s %s %s %s %d %s %s - %ld %s %s\n", url, timestamp_str_cdx, url, mime_type, response_code, checksum, redirect_location, offset, warc_current_filename, response_uuid); + fflush (warc_current_cdx_file); + + return true; +} + +/* Writes a revisit record to the WARC file. + url is the target uri of the request/response, + timestamp_str is the timestamp of the request that generated this response + (generated with warc_timestamp), + concurrent_to_uuid is the uuid of the request for that generated this response + (generated with warc_uuid_str), + refers_to_uuid is the uuid of the original response + (generated with warc_uuid_str), + payload_digest is the sha1 digest of the payload, + ip is the ip address of the server (or NULL), + body is a pointer to a file containing the response headers (without payload). + Calling this function will close body. + Returns true on success, false on error. */ +static bool +warc_write_revisit_record (char *url, char *timestamp_str, char *concurrent_to_uuid, char *payload_digest, char *refers_to, ip_address *ip, FILE *body) +{ + char revisit_uuid [48]; + warc_uuid_str (revisit_uuid); + + char *block_digest = NULL; + char sha1_res_block[SHA1_DIGEST_SIZE]; + sha1_stream (body, sha1_res_block); + block_digest = warc_base32_sha1_digest (sha1_res_block); + + warc_write_start_record (); + warc_write_header ("WARC-Type", "revisit"); + warc_write_header ("WARC-Record-ID", revisit_uuid); + warc_write_header ("WARC-Warcinfo-ID", warc_current_warcinfo_uuid_str); + warc_write_header ("WARC-Concurrent-To", concurrent_to_uuid); + warc_write_header ("WARC-Refers-To", refers_to); + warc_write_header ("WARC-Profile", "http://netpreserve.org/warc/1.0/revisit/identical-payload-digest"); + warc_write_header ("WARC-Truncated", "length"); + warc_write_header ("WARC-Target-URI", url); + warc_write_date_header (timestamp_str); + warc_write_ip_header (ip); + warc_write_header ("Content-Type", "application/http;msgtype=response"); + warc_write_header ("WARC-Block-Digest", block_digest); + warc_write_header ("WARC-Payload-Digest", payload_digest); + warc_write_block_from_file (body); + warc_write_end_record (); + + fclose (body); + free (block_digest); + + return warc_write_ok; +} + +/* Writes a response record to the WARC file. + url is the target uri of the request/response, + timestamp_str is the timestamp of the request that generated this response + (generated with warc_timestamp), + concurrent_to_uuid is the uuid of the request for that generated this response + (generated with warc_uuid_str), + ip is the ip address of the server (or NULL), + body is a pointer to a file containing the response headers and body. + mime_type is the mime type of the response body (will be printed to CDX), + response_code is the HTTP response code (will be printed to CDX), + redirect_location is the contents of the Location: header, or NULL (will be printed to CDX), + Calling this function will close body. + Returns true on success, false on error. */ +bool +warc_write_response_record (char *url, char *timestamp_str, char *concurrent_to_uuid, ip_address *ip, FILE *body, long int payload_offset, char *mime_type, int response_code, char *redirect_location) +{ + char *block_digest = NULL; + char *payload_digest = NULL; + char sha1_res_block[SHA1_DIGEST_SIZE]; + char sha1_res_payload[SHA1_DIGEST_SIZE]; + + if (opt.warc_digests_enabled) + { + /* Calculate the block and payload digests. */ + rewind (body); + if (warc_sha1_stream_with_payload (body, sha1_res_block, sha1_res_payload, payload_offset) == 0) + { + /* Decide (based on url + payload digest) if we have seen this + data before. */ + struct warc_cdx_record *rec_existing = warc_find_duplicate_cdx_record (url, sha1_res_payload); + if (rec_existing != NULL) + { + /* Found an existing record. */ + logprintf (LOG_VERBOSE, _("Found exact match in CDX file. Saving revisit record to WARC.\n")); + + /* Remove the payload from the file. */ + if (payload_offset > 0) + { + if (ftruncate (fileno (body), payload_offset) == -1) + return false; + } + + /* Send the original payload digest. */ + payload_digest = warc_base32_sha1_digest (sha1_res_payload); + bool result = warc_write_revisit_record (url, timestamp_str, concurrent_to_uuid, payload_digest, rec_existing->uuid, ip, body); + free (payload_digest); + + return result; + } + + block_digest = warc_base32_sha1_digest (sha1_res_block); + payload_digest = warc_base32_sha1_digest (sha1_res_payload); + } + } + + /* Not a revisit, just store the record. */ + + char response_uuid [48]; + warc_uuid_str (response_uuid); + + fseek (warc_current_file, 0L, SEEK_END); + size_t offset = ftell (warc_current_file); + + warc_write_start_record (); + warc_write_header ("WARC-Type", "response"); + warc_write_header ("WARC-Record-ID", response_uuid); + warc_write_header ("WARC-Warcinfo-ID", warc_current_warcinfo_uuid_str); + warc_write_header ("WARC-Concurrent-To", concurrent_to_uuid); + warc_write_header ("WARC-Target-URI", url); + warc_write_date_header (timestamp_str); + warc_write_ip_header (ip); + warc_write_header ("WARC-Block-Digest", block_digest); + warc_write_header ("WARC-Payload-Digest", payload_digest); + warc_write_header ("Content-Type", "application/http;msgtype=response"); + warc_write_block_from_file (body); + warc_write_end_record (); + + fclose (body); + + if (warc_write_ok && opt.warc_cdx_enabled) + { + /* Add this record to the CDX. */ + warc_write_cdx_record (url, timestamp_str, mime_type, response_code, payload_digest, redirect_location, offset, warc_current_filename, response_uuid); + } + + if (block_digest) + free (block_digest); + if (payload_digest) + free (payload_digest); + + return warc_write_ok; +} + +/* Writes a resource record to the WARC file. + resource_uuid is the uuid of the resource (or NULL), + url is the target uri of the resource, + timestamp_str is the timestamp (generated with warc_timestamp), + concurrent_to_uuid is the uuid of the request for that generated this resource + (generated with warc_uuid_str) or NULL, + ip is the ip address of the server (or NULL), + content_type is the mime type of the body (or NULL), + body is a pointer to a file containing the resource data. + Calling this function will close body. + Returns true on success, false on error. */ +bool +warc_write_resource_record (char *resource_uuid, char *url, char *timestamp_str, char *concurrent_to_uuid, ip_address *ip, char *content_type, FILE *body, long int payload_offset) +{ + if (resource_uuid == NULL) + { + resource_uuid = alloca (48); + warc_uuid_str (resource_uuid); + } + + if (content_type == NULL) + content_type = "application/octet-stream"; + + warc_write_start_record (); + warc_write_header ("WARC-Type", "resource"); + warc_write_header ("WARC-Record-ID", resource_uuid); + warc_write_header ("WARC-Warcinfo-ID", warc_current_warcinfo_uuid_str); + warc_write_header ("WARC-Concurrent-To", concurrent_to_uuid); + warc_write_header ("WARC-Target-URI", url); + warc_write_date_header (timestamp_str); + warc_write_ip_header (ip); + warc_write_digest_headers (body, payload_offset); + warc_write_header ("Content-Type", content_type); + warc_write_block_from_file (body); + warc_write_end_record (); + + fclose (body); + + return warc_write_ok; +} + diff --git a/src/warc.h b/src/warc.h new file mode 100644 index 00000000..2ade2a8b --- /dev/null +++ b/src/warc.h @@ -0,0 +1,19 @@ +/* Declarations of WARC helper methods. */ +#ifndef WARC_H +#define WARC_H + +#include "host.h" + +void warc_init (); +void warc_close (); +void warc_timestamp (char *timestamp); +void warc_uuid_str (char *id_str); + +FILE * warc_tempfile (); + +bool warc_write_request_record (char *url, char *timestamp_str, char *concurrent_to_uuid, ip_address *ip, FILE *body, long int payload_offset); +bool warc_write_response_record (char *url, char *timestamp_str, char *concurrent_to_uuid, ip_address *ip, FILE *body, long int payload_offset, char *mime_type, int response_code, char *redirect_location); +bool warc_write_resource_record (char *resource_uuid, char *url, char *timestamp_str, char *concurrent_to_uuid, ip_address *ip, char *content_type, FILE *body, long int payload_offset); + +#endif /* WARC_H */ + diff --git a/src/wget.h b/src/wget.h index c7c5e2cb..ee315b6f 100644 --- a/src/wget.h +++ b/src/wget.h @@ -353,7 +353,9 @@ typedef enum PROXERR, /* 50 */ AUTHFAILED, QUOTEXC, WRITEFAILED, SSLINITFAILED, VERIFCERTERR, - UNLINKERR, NEWLOCATION_KEEP_POST + UNLINKERR, NEWLOCATION_KEEP_POST, + + WARC_ERR, WARC_TMP_FOPENERR, WARC_TMP_FWRITEERR } uerr_t; /* 2005-02-19 SMS. From 127036d3ec956c1d5f7e9c206a1a74d65c0f9be8 Mon Sep 17 00:00:00 2001 From: Steven Schweda Date: Fri, 4 Nov 2011 22:31:48 +0100 Subject: [PATCH 05/75] gnutls: fix memory leak. --- src/ChangeLog | 3 ++- src/gnutls.c | 10 ++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/src/ChangeLog b/src/ChangeLog index 65c48072..d1358bfa 100644 --- a/src/ChangeLog +++ b/src/ChangeLog @@ -1,5 +1,6 @@ -2011-11-04 Giuseppe Scrivano +2011-11-01 Steven Schweda + * gnutls.c (ssl_init): Ensure GNU TLS is loaded only once. 2011-10-07 Steven Schweda diff --git a/src/gnutls.c b/src/gnutls.c index 40a04ef3..e4b0fc2e 100644 --- a/src/gnutls.c +++ b/src/gnutls.c @@ -63,6 +63,13 @@ static gnutls_certificate_credentials credentials; bool ssl_init () { + /* Becomes true if GnuTLS is initialized. */ + static bool ssl_initialized = false; + + /* GnuTLS should be initialized only once. */ + if (ssl_initialized) + return true; + const char *ca_directory; DIR *dir; @@ -104,6 +111,9 @@ ssl_init () if (opt.ca_cert) gnutls_certificate_set_x509_trust_file (credentials, opt.ca_cert, GNUTLS_X509_FMT_PEM); + + ssl_initialized = true; + return true; } From 5305f18c0a541d52e5e6bb8ed783ef6f7939dae1 Mon Sep 17 00:00:00 2001 From: Giuseppe Scrivano Date: Fri, 4 Nov 2011 22:34:51 +0100 Subject: [PATCH 06/75] NEWS: cite last changes. --- NEWS | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/NEWS b/NEWS index 79c25b3e..27f27743 100644 --- a/NEWS +++ b/NEWS @@ -11,6 +11,10 @@ Please send GNU Wget bug reports to . ** Add support for content-on-error. It allows to store the HTTP payload on 4xx or 5xx errors. +** Add support for WARC files. + +** Fix a memory leak problem in the GNU TLS backend. + * Changes in Wget 1.13.4 From eed850d938fde57f08ac39e2c7218c8542954e6c Mon Sep 17 00:00:00 2001 From: Steven Schweda Date: Sat, 5 Nov 2011 11:52:51 +0100 Subject: [PATCH 07/75] warc: Fix a problem under OS X. --- src/ChangeLog | 6 ++++++ src/warc.c | 8 +++++++- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/src/ChangeLog b/src/ChangeLog index d1358bfa..001d3909 100644 --- a/src/ChangeLog +++ b/src/ChangeLog @@ -1,3 +1,9 @@ +2011-11-04 Steven Schweda + + * warc.c [! WINDOWS]: Include . + (warc_write_warcinfo_record): Assign a new allocated buffer and + free it on errors. + 2011-11-01 Steven Schweda * gnutls.c (ssl_init): Ensure GNU TLS is loaded only once. diff --git a/src/warc.c b/src/warc.c index 77ef3692..33488c4e 100644 --- a/src/warc.c +++ b/src/warc.c @@ -19,6 +19,10 @@ #include #endif +#ifndef WINDOWS +#include +#endif + #include "warc.h" extern char *version_string; @@ -605,7 +609,7 @@ warc_write_warcinfo_record (char *filename) char *filename_copy, *filename_basename; filename_copy = strdup (filename); - filename_basename = basename (filename_copy); + filename_basename = strdup (basename (filename_copy)); warc_write_start_record (); warc_write_header ("WARC-Type", "warcinfo"); @@ -619,6 +623,7 @@ warc_write_warcinfo_record (char *filename) if (warc_tmp == NULL) { free (filename_copy); + free (filename_basename); return false; } @@ -646,6 +651,7 @@ warc_write_warcinfo_record (char *filename) } free (filename_copy); + free (filename_basename); fclose (warc_tmp); return warc_write_ok; } From 1316701791ad4ea914012c08bd2d24528cf43c85 Mon Sep 17 00:00:00 2001 From: Gijs van Tulder Date: Sun, 20 Nov 2011 18:28:19 +0100 Subject: [PATCH 08/75] Fix for gzip bug in WARC + zlib 1.2.4. --- src/ChangeLog | 5 +++++ src/warc.c | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/src/ChangeLog b/src/ChangeLog index 001d3909..8db453b0 100644 --- a/src/ChangeLog +++ b/src/ChangeLog @@ -1,3 +1,8 @@ +2011-11-09 Gijs van Tulder + + * warc.c: Call gzdopen() with wb9 instead of wb+9, which fails on + zlib version >= 1.2.4. + 2011-11-04 Steven Schweda * warc.c [! WINDOWS]: Include . diff --git a/src/warc.c b/src/warc.c index 33488c4e..680ac997 100644 --- a/src/warc.c +++ b/src/warc.c @@ -169,7 +169,7 @@ warc_write_start_record () fflush (warc_current_file); /* Start a new GZIP stream. */ - warc_current_gzfile = gzdopen (dup (fileno (warc_current_file)), "wb+9"); + warc_current_gzfile = gzdopen (dup (fileno (warc_current_file)), "wb9"); warc_current_gzfile_uncompressed_size = 0; if (warc_current_gzfile == NULL) From 0bfb1aa9be5ad7153378037d97f6f0ac77c2a3e3 Mon Sep 17 00:00:00 2001 From: Giuseppe Scrivano Date: Sun, 11 Dec 2011 15:18:11 +0100 Subject: [PATCH 09/75] trunc: check for `close'-ing the fd errors. --- ChangeLog | 5 +++++ util/trunc.c | 6 ++++++ 2 files changed, 11 insertions(+) diff --git a/ChangeLog b/ChangeLog index 7691445e..15006698 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,8 @@ +2011-12-11 Giuseppe Scrivano + + * util/trunc.c (main): Call `close' on the fd and check for errors. + Reported by: . + 2011-10-23 Giuseppe Scrivano * bootstrap.conf (gnulib_modules): Include module `vsnprintf'. diff --git a/util/trunc.c b/util/trunc.c index 55cb19d3..a5f1dcb0 100644 --- a/util/trunc.c +++ b/util/trunc.c @@ -128,5 +128,11 @@ main (int argc, char *argv[]) exit (EXIT_FAILURE); } + if (close (fd) < 0) + { + perror (PROGRAM_NAME ": close"); + exit (EXIT_FAILURE); + } + return 0; } From c2ee9283024b2cb2b4205f35b79c6f6093e0dce8 Mon Sep 17 00:00:00 2001 From: Giuseppe Scrivano Date: Mon, 12 Dec 2011 21:30:39 +0100 Subject: [PATCH 10/75] Fix regeneration of autotools files in a distributed tarball. --- ChangeLog | 5 +++++ Makefile.am | 2 +- NEWS | 1 + 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/ChangeLog b/ChangeLog index 15006698..39f591fe 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,8 @@ +2011-12-12 Giuseppe Scrivano + + * Makefile.am (EXTRA_DIST): Add build-aux/bzr-version-gen. + Reported by: Elan Ruusamäe . + 2011-12-11 Giuseppe Scrivano * util/trunc.c (main): Call `close' on the fd and check for errors. diff --git a/Makefile.am b/Makefile.am index 9fdc5713..24d95e07 100644 --- a/Makefile.am +++ b/Makefile.am @@ -46,7 +46,7 @@ SUBDIRS = lib src doc po tests util EXTRA_DIST = ChangeLog.README MAILING-LIST \ msdos/ChangeLog msdos/config.h msdos/Makefile.DJ \ msdos/Makefile.WC ABOUT-NLS \ - build-aux/build_info.pl .version + build-aux/build_info.pl build-aux/bzr-version-gen .version CLEANFILES = *~ *.bak $(DISTNAME).tar.gz diff --git a/NEWS b/NEWS index 27f27743..61a4983a 100644 --- a/NEWS +++ b/NEWS @@ -15,6 +15,7 @@ Please send GNU Wget bug reports to . ** Fix a memory leak problem in the GNU TLS backend. +** Autoreconf works again for distributed tarballs. * Changes in Wget 1.13.4 From 5e1badae1e595b378f039f4a6b09f6e44767a37a Mon Sep 17 00:00:00 2001 From: Sasikantha Babu Date: Mon, 9 Jan 2012 00:03:23 +0100 Subject: [PATCH 11/75] Properly format IPv6 addresses. --- src/ChangeLog | 5 +++++ src/connect.c | 37 ++++++++++++++++++++++++++++++++++++- src/connect.h | 1 + src/http.c | 12 +++++++++--- 4 files changed, 51 insertions(+), 4 deletions(-) diff --git a/src/ChangeLog b/src/ChangeLog index 8db453b0..e58decb5 100644 --- a/src/ChangeLog +++ b/src/ChangeLog @@ -1,3 +1,8 @@ +2012-01-09 Sasikantha Babu (tiny change) + * connect.c (connect_to_ip): properly formatted ipv6 address display. + (socket_family): New function - returns socket family type. + * http.c (gethttp): properly formatted ipv6 address display. + 2011-11-09 Gijs van Tulder * warc.c: Call gzdopen() with wb9 instead of wb+9, which fails on diff --git a/src/connect.c b/src/connect.c index 6008c3c2..34b40abc 100644 --- a/src/connect.c +++ b/src/connect.c @@ -298,7 +298,12 @@ connect_to_ip (const ip_address *ip, int port, const char *print) xfree (str); } else - logprintf (LOG_VERBOSE, _("Connecting to %s:%d... "), txt_addr, port); + { + if (ip->family == AF_INET) + logprintf (LOG_VERBOSE, _("Connecting to %s:%d... "), txt_addr, port); + else if (ip->family == AF_INET6) + logprintf (LOG_VERBOSE, _("Connecting to [%s]:%d... "), txt_addr, port); + } } /* Store the sockaddr info to SA. */ @@ -586,6 +591,36 @@ socket_ip_address (int sock, ip_address *ip, int endpoint) } } +/* Get the socket family of connection on FD and store + Return family type on success, -1 otherwise. + + If ENDPOINT is ENDPOINT_LOCAL, it returns the sock family of the local + (client) side of the socket. Else if ENDPOINT is ENDPOINT_PEER, it + returns the sock family of the remote (peer's) side of the socket. */ + +int +socket_family (int sock, int endpoint) +{ + struct sockaddr_storage storage; + struct sockaddr *sockaddr = (struct sockaddr *) &storage; + socklen_t addrlen = sizeof (storage); + int ret; + + memset (sockaddr, 0, addrlen); + + if (endpoint == ENDPOINT_LOCAL) + ret = getsockname (sock, sockaddr, &addrlen); + else if (endpoint == ENDPOINT_PEER) + ret = getpeername (sock, sockaddr, &addrlen); + else + abort (); + + if (ret < 0) + return -1; + + return sockaddr->sa_family; +} + /* Return true if the error from the connect code can be considered retryable. Wget normally retries after errors, but the exception are the "unsupported protocol" type errors (possible on IPv4/IPv6 diff --git a/src/connect.h b/src/connect.h index 20bb2439..bb3f26a7 100644 --- a/src/connect.h +++ b/src/connect.h @@ -51,6 +51,7 @@ enum { ENDPOINT_PEER }; bool socket_ip_address (int, ip_address *, int); +int socket_family (int sock, int endpoint); bool retryable_socket_connect_error (int); diff --git a/src/http.c b/src/http.c index 6a2ffe86..69789fcd 100644 --- a/src/http.c +++ b/src/http.c @@ -1951,11 +1951,17 @@ gethttp (struct url *u, struct http_stat *hs, int *dt, struct url *proxy, #endif &host_lookup_failed)) { + int family = socket_family (pconn.socket, ENDPOINT_PEER); sock = pconn.socket; using_ssl = pconn.ssl; - logprintf (LOG_VERBOSE, _("Reusing existing connection to %s:%d.\n"), - quotearg_style (escape_quoting_style, pconn.host), - pconn.port); + if (family == AF_INET6) + logprintf (LOG_VERBOSE, _("Reusing existing connection to [%s]:%d.\n"), + quotearg_style (escape_quoting_style, pconn.host), + pconn.port); + else + logprintf (LOG_VERBOSE, _("Reusing existing connection to %s:%d.\n"), + quotearg_style (escape_quoting_style, pconn.host), + pconn.port); DEBUGP (("Reusing fd %d.\n", sock)); if (pconn.authorized) /* If the connection is already authorized, the "Basic" From 0a8a898fbec5a66b3d1978d4db857a82d623f546 Mon Sep 17 00:00:00 2001 From: Gijs van Tulder Date: Wed, 11 Jan 2012 15:27:06 +0100 Subject: [PATCH 12/75] Fix a linker error if zlib is not found. --- ChangeLog | 4 ++++ configure.ac | 7 +++++++ src/ChangeLog | 7 +++++++ src/init.c | 6 ++++++ src/main.c | 4 ++++ src/warc.c | 14 ++++++++++++++ 6 files changed, 42 insertions(+) diff --git a/ChangeLog b/ChangeLog index 39f591fe..a391a786 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,7 @@ +2012-01-09 Gijs van Tulder + + * configure.ac: Always try to use libz, even without SSL. + 2011-12-12 Giuseppe Scrivano * Makefile.am (EXTRA_DIST): Add build-aux/bzr-version-gen. diff --git a/configure.ac b/configure.ac index 360f6c91..647e44e3 100644 --- a/configure.ac +++ b/configure.ac @@ -65,6 +65,9 @@ AC_ARG_WITH(ssl, [[ --without-ssl disable SSL autodetection --with-ssl={gnutls,openssl} specify the SSL backend. GNU TLS is the default.]]) +AC_ARG_WITH(zlib, +[[ --without-zlib disable zlib ]]) + AC_ARG_ENABLE(opie, [ --disable-opie disable support for opie or s/key FTP login], ENABLE_OPIE=$enableval, ENABLE_OPIE=yes) @@ -234,6 +237,10 @@ dnl dnl Checks for libraries. dnl +AS_IF([test x"$with_zlib" != xno], [ + AC_CHECK_LIB(z, compress) +]) + AS_IF([test x"$with_ssl" = xopenssl], [ dnl some versions of openssl use zlib compression AC_CHECK_LIB(z, compress) diff --git a/src/ChangeLog b/src/ChangeLog index e58decb5..141a47d4 100644 --- a/src/ChangeLog +++ b/src/ChangeLog @@ -1,3 +1,10 @@ +2012-01-09 Gijs van Tulder + + * init.c: Disable WARC compression if zlib is disabled. + * main.c: Do not show the 'no-warc-compression' option if zlib is + disabled. + * warc.c: Do not compress WARC files if zlib is disabled. + 2012-01-09 Sasikantha Babu (tiny change) * connect.c (connect_to_ip): properly formatted ipv6 address display. (socket_family): New function - returns socket family type. diff --git a/src/init.c b/src/init.c index 47fdea06..d2fba82c 100644 --- a/src/init.c +++ b/src/init.c @@ -267,7 +267,9 @@ static const struct { { "waitretry", &opt.waitretry, cmd_time }, { "warccdx", &opt.warc_cdx_enabled, cmd_boolean }, { "warccdxdedup", &opt.warc_cdx_dedup_filename, cmd_file }, +#ifdef HAVE_LIBZ { "warccompression", &opt.warc_compression_enabled, cmd_boolean }, +#endif { "warcdigests", &opt.warc_digests_enabled, cmd_boolean }, { "warcfile", &opt.warc_filename, cmd_file }, { "warcheader", NULL, cmd_spec_warc_header }, @@ -374,7 +376,11 @@ defaults (void) opt.show_all_dns_entries = false; opt.warc_maxsize = 0; /* 1024 * 1024 * 1024; */ +#ifdef HAVE_LIBZ opt.warc_compression_enabled = true; +#else + opt.warc_compression_enabled = false; +#endif opt.warc_digests_enabled = true; opt.warc_cdx_enabled = false; opt.warc_cdx_dedup_filename = NULL; diff --git a/src/main.c b/src/main.c index 28467359..5aa528de 100644 --- a/src/main.c +++ b/src/main.c @@ -289,7 +289,9 @@ static struct cmdline_option option_data[] = { "wait", 'w', OPT_VALUE, "wait", -1 }, { "waitretry", 0, OPT_VALUE, "waitretry", -1 }, { "warc-cdx", 0, OPT_BOOLEAN, "warccdx", -1 }, +#ifdef HAVE_LIBZ { "warc-compression", 0, OPT_BOOLEAN, "warccompression", -1 }, +#endif { "warc-dedup", 0, OPT_VALUE, "warccdxdedup", -1 }, { "warc-digests", 0, OPT_BOOLEAN, "warcdigests", -1 }, { "warc-file", 0, OPT_VALUE, "warcfile", -1 }, @@ -674,8 +676,10 @@ WARC options:\n"), --warc-cdx write CDX index files.\n"), N_("\ --warc-dedup=FILENAME do not store records listed in this CDX file.\n"), +#ifdef HAVE_LIBZ N_("\ --no-warc-compression do not compress WARC files with GZIP.\n"), +#endif N_("\ --no-warc-digests do not calculate SHA1 digests.\n"), N_("\ diff --git a/src/warc.c b/src/warc.c index 680ac997..a3cc8184 100644 --- a/src/warc.c +++ b/src/warc.c @@ -14,7 +14,9 @@ #include #include #include +#ifdef HAVE_LIBZ #include +#endif #ifdef HAVE_LIBUUID #include #endif @@ -42,6 +44,7 @@ static FILE *warc_manifest_fp; /* The current WARC file (or NULL, if WARC is disabled). */ static FILE *warc_current_file; +#ifdef HAVE_LIBZ /* The gzip stream for the current WARC file (or NULL, if WARC or gzip is disabled). */ static gzFile *warc_current_gzfile; @@ -51,6 +54,7 @@ static size_t warc_current_gzfile_offset; /* The uncompressed size (so far) of the current record. */ static size_t warc_current_gzfile_uncompressed_size; +# endif /* This is true until a warc_write_* method fails. */ static bool warc_write_ok; @@ -105,12 +109,14 @@ warc_cmp_sha1_digest (const void *digest1, const void *digest2) static size_t warc_write_buffer (const char *buffer, size_t size) { +#ifdef HAVE_LIBZ if (warc_current_gzfile) { warc_current_gzfile_uncompressed_size += size; return gzwrite (warc_current_gzfile, buffer, size); } else +#endif return fwrite (buffer, 1, size, warc_current_file); } @@ -155,6 +161,7 @@ warc_write_start_record () if (opt.warc_maxsize > 0 && ftell (warc_current_file) >= opt.warc_maxsize) warc_start_new_file (false); +#ifdef HAVE_LIBZ /* Start a GZIP stream, if required. */ if (opt.warc_compression_enabled) { @@ -179,6 +186,7 @@ warc_write_start_record () return false; } } +#endif warc_write_string ("WARC/1.0\r\n"); return warc_write_ok; @@ -247,6 +255,7 @@ warc_write_end_record () { warc_write_buffer ("\r\n\r\n", 4); +#ifdef HAVE_LIBZ /* We start a new gzip stream for each record. */ if (warc_write_ok && warc_current_gzfile) { @@ -325,6 +334,7 @@ warc_write_end_record () fflush (warc_current_file); fseek (warc_current_file, 0, SEEK_END); } +#endif /* HAVE_LIBZ */ return warc_write_ok; } @@ -687,7 +697,11 @@ warc_start_new_file (bool meta) char *new_filename = malloc (base_filename_length + 1 + 5 + 8 + 1); warc_current_filename = new_filename; +#ifdef HAVE_LIBZ char *extension = (opt.warc_compression_enabled ? "warc.gz" : "warc"); +#else + char *extension = "warc"; +#endif /* If max size is enabled, we add a serial number to the file names. */ if (meta) From 586ade4fb19021fb8893912cda13875ae4120236 Mon Sep 17 00:00:00 2001 From: Gijs van Tulder Date: Sat, 28 Jan 2012 14:08:52 +0100 Subject: [PATCH 13/75] Fix memory leak. --- src/ChangeLog | 5 +++++ src/http.c | 14 +++++++++++--- src/retr.c | 22 ++++++++++++++++------ 3 files changed, 32 insertions(+), 9 deletions(-) diff --git a/src/ChangeLog b/src/ChangeLog index 141a47d4..e10d4c02 100644 --- a/src/ChangeLog +++ b/src/ChangeLog @@ -1,3 +1,8 @@ +2012-01-27 Gijs van Tulder + + * retr.c (fd_read_body): Fix a memory leak with chunked responses. + * http.c (skip_short_body): Fix the same memory leak. + 2012-01-09 Gijs van Tulder * init.c: Disable WARC compression if zlib is disabled. diff --git a/src/http.c b/src/http.c index 69789fcd..78725796 100644 --- a/src/http.c +++ b/src/http.c @@ -951,9 +951,12 @@ skip_short_body (int fd, wgint contlen, bool chunked) break; remaining_chunk_size = strtol (line, &endl, 16); + xfree (line); + if (remaining_chunk_size == 0) { - fd_read_line (fd); + line = fd_read_line (fd); + xfree_null (line); break; } } @@ -978,8 +981,13 @@ skip_short_body (int fd, wgint contlen, bool chunked) { remaining_chunk_size -= ret; if (remaining_chunk_size == 0) - if (fd_read_line (fd) == NULL) - return false; + { + char *line = fd_read_line (fd); + if (line == NULL) + return false; + else + xfree (line); + } } /* Safe even if %.*s bogusly expects terminating \0 because diff --git a/src/retr.c b/src/retr.c index 3df582b8..f57b2c6d 100644 --- a/src/retr.c +++ b/src/retr.c @@ -307,11 +307,16 @@ fd_read_body (int fd, FILE *out, wgint toread, wgint startpos, } remaining_chunk_size = strtol (line, &endl, 16); + xfree (line); + if (remaining_chunk_size == 0) { ret = 0; - if (fd_read_line (fd) == NULL) + line = fd_read_line (fd); + if (line == NULL) ret = -1; + else + xfree (line); break; } } @@ -371,11 +376,16 @@ fd_read_body (int fd, FILE *out, wgint toread, wgint startpos, { remaining_chunk_size -= ret; if (remaining_chunk_size == 0) - if (fd_read_line (fd) == NULL) - { - ret = -1; - break; - } + { + char *line = fd_read_line (fd); + if (line == NULL) + { + ret = -1; + break; + } + else + xfree (line); + } } } From 6d67d793f51af4e0a5a840751c15308ab76ba8b6 Mon Sep 17 00:00:00 2001 From: Gijs van Tulder Date: Sat, 28 Jan 2012 14:09:29 +0100 Subject: [PATCH 14/75] Add support for chunks to the WARC outputter. --- src/ChangeLog | 6 ++++++ src/retr.c | 17 +++++++++++++++-- 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/src/ChangeLog b/src/ChangeLog index e10d4c02..141b7e18 100644 --- a/src/ChangeLog +++ b/src/ChangeLog @@ -1,3 +1,9 @@ +2012-01-27 Gijs van Tulder + + * retr.c (fd_read_body): If the response is chunked, the chunk + headers are now written to the WARC file, making the WARC file + an exact copy of the HTTP response. + 2012-01-27 Gijs van Tulder * retr.c (fd_read_body): Fix a memory leak with chunked responses. diff --git a/src/retr.c b/src/retr.c index f57b2c6d..8c8cdf5b 100644 --- a/src/retr.c +++ b/src/retr.c @@ -213,6 +213,9 @@ write_data (FILE *out, FILE *out2, const char *buf, int bufsize, the data is stored to ELAPSED. If OUT2 is non-NULL, the contents is also written to OUT2. + OUT2 will get an exact copy of the response: if this is a chunked + response, everything -- including the chunk headers -- is written + to OUT2. (OUT will only get the unchunked response.) The function exits and returns the amount of data read. In case of error while reading data, -1 is returned. In case of error while @@ -305,6 +308,8 @@ fd_read_body (int fd, FILE *out, wgint toread, wgint startpos, ret = -1; break; } + else if (out2 != NULL) + fwrite (line, 1, strlen (line), out2); remaining_chunk_size = strtol (line, &endl, 16); xfree (line); @@ -316,7 +321,11 @@ fd_read_body (int fd, FILE *out, wgint toread, wgint startpos, if (line == NULL) ret = -1; else - xfree (line); + { + if (out2 != NULL) + fwrite (line, 1, strlen (line), out2); + xfree (line); + } break; } } @@ -384,7 +393,11 @@ fd_read_body (int fd, FILE *out, wgint toread, wgint startpos, break; } else - xfree (line); + { + if (out2 != NULL) + fwrite (line, 1, strlen (line), out2); + xfree (line); + } } } } From c60530b3690a29f3c5b377c50ff8571923c7e863 Mon Sep 17 00:00:00 2001 From: Steven Schubiger Date: Fri, 17 Feb 2012 17:05:26 +0100 Subject: [PATCH 15/75] warc: add license header. --- src/ChangeLog | 4 ++++ src/warc.c | 30 +++++++++++++++++++++++++++++- 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/src/ChangeLog b/src/ChangeLog index 141b7e18..ffd6ccc9 100644 --- a/src/ChangeLog +++ b/src/ChangeLog @@ -1,3 +1,7 @@ +2012-02-17 Steven Schubiger + + * warc.c: Add license header. + 2012-01-27 Gijs van Tulder * retr.c (fd_read_body): If the response is chunked, the chunk diff --git a/src/warc.c b/src/warc.c index a3cc8184..f18e21b8 100644 --- a/src/warc.c +++ b/src/warc.c @@ -1,4 +1,32 @@ -/* Utility functions for writing WARC files. */ +/* Utility functions for writing WARC files. + Copyright (C) 2011, 2012 Free Software Foundation, Inc. + +This file is part of GNU Wget. + +GNU Wget is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; either version 3 of the License, or (at +your option) any later version. + +GNU Wget is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with Wget. If not, see . + +Additional permission under GNU GPL version 3 section 7 + +If you modify this program, or any covered work, by linking or +combining it with the OpenSSL project's OpenSSL library (or a +modified version of that library), containing parts covered by the +terms of the OpenSSL or SSLeay licenses, the Free Software Foundation +grants you additional permission to convey the resulting work. +Corresponding Source for a non-source form of such a combination +shall include the source code for the parts of OpenSSL used as well +as that of the covered work. */ + #define _GNU_SOURCE #include "wget.h" From 611a219fb0d606b00d9af335efbd3cee173fc51b Mon Sep 17 00:00:00 2001 From: Giuseppe Scrivano Date: Thu, 23 Feb 2012 11:11:49 +0100 Subject: [PATCH 16/75] gnutls: Remove two unused variables. --- src/ChangeLog | 4 ++++ src/gnutls.c | 4 ---- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/ChangeLog b/src/ChangeLog index ffd6ccc9..9364d54d 100644 --- a/src/ChangeLog +++ b/src/ChangeLog @@ -1,3 +1,7 @@ +2012-02-23 Giuseppe Scrivano + + * gnutls.c (wgnutls_read): Remove unused variables `timer' and `flags'. + 2012-02-17 Steven Schubiger * warc.c: Add license header. diff --git a/src/gnutls.c b/src/gnutls.c index e4b0fc2e..2a1d22b9 100644 --- a/src/gnutls.c +++ b/src/gnutls.c @@ -217,11 +217,7 @@ wgnutls_read_timeout (int fd, char *buf, int bufsize, void *arg, double timeout) static int wgnutls_read (int fd, char *buf, int bufsize, void *arg) { -#ifdef F_GETFL - int flags = 0; -#endif int ret = 0; - struct ptimer *timer; struct wgnutls_transport_context *ctx = arg; if (ctx->peeklen) From bcc2abf116f565d333ac3bdc64c10f5a7e1990fb Mon Sep 17 00:00:00 2001 From: Giuseppe Scrivano Date: Thu, 23 Feb 2012 11:45:05 +0100 Subject: [PATCH 17/75] Handle correctly some malloc failures. --- src/ChangeLog | 2 ++ src/main.c | 26 ++++++++++++++++++-------- 2 files changed, 20 insertions(+), 8 deletions(-) diff --git a/src/ChangeLog b/src/ChangeLog index 9364d54d..13ab1ae7 100644 --- a/src/ChangeLog +++ b/src/ChangeLog @@ -1,5 +1,7 @@ 2012-02-23 Giuseppe Scrivano + * main.c (main): Fail gracefully if `malloc' fails. + * gnutls.c (wgnutls_read): Remove unused variables `timer' and `flags'. 2012-02-17 Steven Schubiger diff --git a/src/main.c b/src/main.c index 5aa528de..9eefc98f 100644 --- a/src/main.c +++ b/src/main.c @@ -988,15 +988,20 @@ main (int argc, char **argv) for (i = 1; i < argc; i++) argstring_length += strlen (argv[i]) + 2 + 1; char *p = program_argstring = malloc (argstring_length * sizeof (char)); + if (p == NULL) + { + fprintf (stderr, _("Memory allocation problem\n")); + exit (2); + } for (i = 1; i < argc; i++) - { - *p++ = '"'; - int arglen = strlen (argv[i]); - memcpy (p, argv[i], arglen); - p += arglen; - *p++ = '"'; - *p++ = ' '; - } + { + *p++ = '"'; + int arglen = strlen (argv[i]); + memcpy (p, argv[i], arglen); + p += arglen; + *p++ = '"'; + *p++ = ' '; + } *p = '\0'; /* Load the hard-coded defaults. */ @@ -1355,6 +1360,11 @@ for details.\n\n")); /* Fill in the arguments. */ url = alloca_array (char *, nurl + 1); + if (url == NULL) + { + fprintf (stderr, _("Memory allocation problem\n")); + exit (2); + } for (i = 0; i < nurl; i++, optind++) { char *rewritten = rewrite_shorthand_url (argv[optind]); From 408126aae0277e5c9e995a32bd942f4fa5cd7a9d Mon Sep 17 00:00:00 2001 From: Giuseppe Scrivano Date: Thu, 23 Feb 2012 11:56:44 +0100 Subject: [PATCH 18/75] Print some diagnostic messages to stderr not to stdout. --- NEWS | 2 ++ src/ChangeLog | 2 ++ src/main.c | 11 ++++++----- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/NEWS b/NEWS index 61a4983a..3a6aca0d 100644 --- a/NEWS +++ b/NEWS @@ -16,6 +16,8 @@ Please send GNU Wget bug reports to . ** Fix a memory leak problem in the GNU TLS backend. ** Autoreconf works again for distributed tarballs. + +** Print some diagnostic messages to stderr not to stdout. * Changes in Wget 1.13.4 diff --git a/src/ChangeLog b/src/ChangeLog index 13ab1ae7..e5def2a8 100644 --- a/src/ChangeLog +++ b/src/ChangeLog @@ -1,5 +1,7 @@ 2012-02-23 Giuseppe Scrivano + * main.c (main): Write diagnostic messages to `stderr' not to `stdout'. + * main.c (main): Fail gracefully if `malloc' fails. * gnutls.c (wgnutls_read): Remove unused variables `timer' and `flags'. diff --git a/src/main.c b/src/main.c index 9eefc98f..352715a0 100644 --- a/src/main.c +++ b/src/main.c @@ -1034,7 +1034,7 @@ main (int argc, char **argv) } if (!userrc_ret) { - printf ("Exiting due to error in %s\n", optarg); + fprintf (stderr, "Exiting due to error in %s\n", optarg); exit (2); } else @@ -1062,9 +1062,10 @@ main (int argc, char **argv) { if (ret == '?') { - print_usage (0); - printf ("\n"); - printf (_("Try `%s --help' for more options.\n"), exec_name); + print_usage (1); + fprintf (stderr, "\n"); + fprintf (stderr, _("Try `%s --help' for more options.\n"), + exec_name); exit (2); } /* Find the short option character in the mapping. */ @@ -1307,7 +1308,7 @@ for details.\n\n")); /* No URL specified. */ fprintf (stderr, _("%s: missing URL\n"), exec_name); print_usage (1); - printf ("\n"); + fprintf (stderr, "\n"); /* #### Something nicer should be printed here -- similar to the pre-1.5 `--help' page. */ fprintf (stderr, _("Try `%s --help' for more options.\n"), exec_name); From 6a25955fe6db8e08805d5a0b07ff6b531a5515d3 Mon Sep 17 00:00:00 2001 From: Gijs van Tulder Date: Sat, 25 Feb 2012 11:58:21 +0100 Subject: [PATCH 19/75] warc: support large files. --- src/ChangeLog | 6 ++++++ src/http.c | 6 +++--- src/warc.c | 54 +++++++++++++++++++++++++-------------------------- src/warc.h | 6 +++--- 4 files changed, 39 insertions(+), 33 deletions(-) diff --git a/src/ChangeLog b/src/ChangeLog index e5def2a8..9f221012 100644 --- a/src/ChangeLog +++ b/src/ChangeLog @@ -1,3 +1,9 @@ +2012-02-01 Gijs van Tulder + + * warc.c: Fix large file support with ftello, fseeko. + * warc.h: Fix large file support. + * http.c: Fix large file support. + 2012-02-23 Giuseppe Scrivano * main.c (main): Write diagnostic messages to `stderr' not to `stdout'. diff --git a/src/http.c b/src/http.c index 78725796..61001f3b 100644 --- a/src/http.c +++ b/src/http.c @@ -1712,7 +1712,7 @@ gethttp (struct url *u, struct http_stat *hs, int *dt, struct url *proxy, char warc_timestamp_str [21]; char warc_request_uuid [48]; ip_address *warc_ip = NULL; - long int warc_payload_offset = -1; + off_t warc_payload_offset = -1; /* Whether this connection will be kept alive after the HTTP request is done. */ @@ -2127,7 +2127,7 @@ gethttp (struct url *u, struct http_stat *hs, int *dt, struct url *proxy, if (write_error >= 0 && warc_tmp != NULL) { /* Remember end of headers / start of payload. */ - warc_payload_offset = ftell (warc_tmp); + warc_payload_offset = ftello (warc_tmp); /* Write a copy of the data to the WARC record. */ int warc_tmp_written = fwrite (opt.post_data, 1, post_data_size, warc_tmp); @@ -2139,7 +2139,7 @@ gethttp (struct url *u, struct http_stat *hs, int *dt, struct url *proxy, { if (warc_tmp != NULL) /* Remember end of headers / start of payload. */ - warc_payload_offset = ftell (warc_tmp); + warc_payload_offset = ftello (warc_tmp); write_error = post_file (sock, opt.post_file_name, post_data_size, warc_tmp); } diff --git a/src/warc.c b/src/warc.c index f18e21b8..1c1e0797 100644 --- a/src/warc.c +++ b/src/warc.c @@ -78,10 +78,10 @@ static FILE *warc_current_file; static gzFile *warc_current_gzfile; /* The offset of the current gzip record in the WARC file. */ -static size_t warc_current_gzfile_offset; +static off_t warc_current_gzfile_offset; /* The uncompressed size (so far) of the current record. */ -static size_t warc_current_gzfile_uncompressed_size; +static off_t warc_current_gzfile_uncompressed_size; # endif /* This is true until a warc_write_* method fails. */ @@ -186,7 +186,7 @@ warc_write_start_record () return false; fflush (warc_current_file); - if (opt.warc_maxsize > 0 && ftell (warc_current_file) >= opt.warc_maxsize) + if (opt.warc_maxsize > 0 && ftello (warc_current_file) >= opt.warc_maxsize) warc_start_new_file (false); #ifdef HAVE_LIBZ @@ -194,7 +194,7 @@ warc_write_start_record () if (opt.warc_compression_enabled) { /* Record the starting offset of the new record. */ - warc_current_gzfile_offset = ftell (warc_current_file); + warc_current_gzfile_offset = ftello (warc_current_file); /* Reserve space for the extra GZIP header field. In warc_write_end_record we will fill this space @@ -245,8 +245,8 @@ warc_write_block_from_file (FILE *data_in) { /* Add the Content-Length header. */ char *content_length; - fseek (data_in, 0L, SEEK_END); - if (! asprintf (&content_length, "%ld", ftell (data_in))) + fseeko (data_in, 0L, SEEK_END); + if (! asprintf (&content_length, "%ld", ftello (data_in))) { warc_write_ok = false; return false; @@ -257,7 +257,7 @@ warc_write_block_from_file (FILE *data_in) /* End of the WARC header section. */ warc_write_string ("\r\n"); - if (fseek (data_in, 0L, SEEK_SET) != 0) + if (fseeko (data_in, 0L, SEEK_SET) != 0) warc_write_ok = false; /* Copy the data in the file to the WARC record. */ @@ -294,7 +294,7 @@ warc_write_end_record () } fflush (warc_current_file); - fseek (warc_current_file, 0, SEEK_END); + fseeko (warc_current_file, 0, SEEK_END); /* The WARC standard suggests that we add 'skip length' data in the extra header field of the GZIP stream. @@ -312,12 +312,12 @@ warc_write_end_record () */ /* Calculate the uncompressed and compressed sizes. */ - size_t current_offset = ftell (warc_current_file); - size_t uncompressed_size = current_offset - warc_current_gzfile_offset; - size_t compressed_size = warc_current_gzfile_uncompressed_size; + off_t current_offset = ftello (warc_current_file); + off_t uncompressed_size = current_offset - warc_current_gzfile_offset; + off_t compressed_size = warc_current_gzfile_uncompressed_size; /* Go back to the static GZIP header. */ - fseek (warc_current_file, warc_current_gzfile_offset + EXTRA_GZIP_HEADER_SIZE, SEEK_SET); + fseeko (warc_current_file, warc_current_gzfile_offset + EXTRA_GZIP_HEADER_SIZE, SEEK_SET); /* Read the header. */ char static_header[GZIP_STATIC_HEADER_SIZE]; @@ -332,7 +332,7 @@ warc_write_end_record () static_header[OFF_FLG] = static_header[OFF_FLG] | FLG_FEXTRA; /* Write the header back to the file, but starting at warc_current_gzfile_offset. */ - fseek (warc_current_file, warc_current_gzfile_offset, SEEK_SET); + fseeko (warc_current_file, warc_current_gzfile_offset, SEEK_SET); fwrite (static_header, 1, GZIP_STATIC_HEADER_SIZE, warc_current_file); /* Prepare the extra GZIP header. */ @@ -355,12 +355,12 @@ warc_write_end_record () extra_header[11] = (compressed_size >> 24) & 255; /* Write the extra header after the static header. */ - fseek (warc_current_file, warc_current_gzfile_offset + GZIP_STATIC_HEADER_SIZE, SEEK_SET); + fseeko (warc_current_file, warc_current_gzfile_offset + GZIP_STATIC_HEADER_SIZE, SEEK_SET); fwrite (extra_header, 1, EXTRA_GZIP_HEADER_SIZE, warc_current_file); /* Done, move back to the end of the file. */ fflush (warc_current_file); - fseek (warc_current_file, 0, SEEK_END); + fseeko (warc_current_file, 0, SEEK_END); } #endif /* HAVE_LIBZ */ @@ -408,14 +408,14 @@ warc_write_ip_header (ip_address *ip) the end of the file. The digest number will be written into the 16 bytes beginning ad RES_PAYLOAD. */ static int -warc_sha1_stream_with_payload (FILE *stream, void *res_block, void *res_payload, long int payload_offset) +warc_sha1_stream_with_payload (FILE *stream, void *res_block, void *res_payload, off_t payload_offset) { #define BLOCKSIZE 32768 struct sha1_ctx ctx_block; struct sha1_ctx ctx_payload; - long int pos; - size_t sum; + off_t pos; + off_t sum; char *buffer = malloc (BLOCKSIZE + 72); if (!buffer) @@ -434,7 +434,7 @@ warc_sha1_stream_with_payload (FILE *stream, void *res_block, void *res_payload, /* We read the file in blocks of BLOCKSIZE bytes. One call of the computation function processes the whole buffer so that with the next round of the loop another block can be read. */ - size_t n; + off_t n; sum = 0; /* Read block. Take care for partial reads. */ @@ -475,7 +475,7 @@ warc_sha1_stream_with_payload (FILE *stream, void *res_block, void *res_payload, if (payload_offset >= 0 && payload_offset < pos) { /* At least part of the buffer contains data from payload. */ - int start_of_payload = payload_offset - (pos - BLOCKSIZE); + off_t start_of_payload = payload_offset - (pos - BLOCKSIZE); if (start_of_payload <= 0) /* All bytes in the buffer belong to the payload. */ start_of_payload = 0; @@ -499,7 +499,7 @@ warc_sha1_stream_with_payload (FILE *stream, void *res_block, void *res_payload, if (payload_offset >= 0 && payload_offset < pos) { /* At least part of the buffer contains data from payload. */ - int start_of_payload = payload_offset - (pos - sum); + off_t start_of_payload = payload_offset - (pos - sum); if (start_of_payload <= 0) /* All bytes in the buffer belong to the payload. */ start_of_payload = 0; @@ -1134,7 +1134,7 @@ warc_tempfile () Calling this function will close body. Returns true on success, false on error. */ bool -warc_write_request_record (char *url, char *timestamp_str, char *record_uuid, ip_address *ip, FILE *body, long int payload_offset) +warc_write_request_record (char *url, char *timestamp_str, char *record_uuid, ip_address *ip, FILE *body, off_t payload_offset) { warc_write_start_record (); warc_write_header ("WARC-Type", "request"); @@ -1166,7 +1166,7 @@ warc_write_request_record (char *url, char *timestamp_str, char *record_uuid, ip response_uuid is the uuid of the response. Returns true on success, false on error. */ static bool -warc_write_cdx_record (char *url, char *timestamp_str, char *mime_type, int response_code, char *payload_digest, char *redirect_location, size_t offset, char *warc_filename, char *response_uuid) +warc_write_cdx_record (char *url, char *timestamp_str, char *mime_type, int response_code, char *payload_digest, char *redirect_location, off_t offset, char *warc_filename, char *response_uuid) { /* Transform the timestamp. */ char timestamp_str_cdx [15]; @@ -1258,7 +1258,7 @@ warc_write_revisit_record (char *url, char *timestamp_str, char *concurrent_to_u Calling this function will close body. Returns true on success, false on error. */ bool -warc_write_response_record (char *url, char *timestamp_str, char *concurrent_to_uuid, ip_address *ip, FILE *body, long int payload_offset, char *mime_type, int response_code, char *redirect_location) +warc_write_response_record (char *url, char *timestamp_str, char *concurrent_to_uuid, ip_address *ip, FILE *body, off_t payload_offset, char *mime_type, int response_code, char *redirect_location) { char *block_digest = NULL; char *payload_digest = NULL; @@ -1304,8 +1304,8 @@ warc_write_response_record (char *url, char *timestamp_str, char *concurrent_to_ char response_uuid [48]; warc_uuid_str (response_uuid); - fseek (warc_current_file, 0L, SEEK_END); - size_t offset = ftell (warc_current_file); + fseeko (warc_current_file, 0L, SEEK_END); + off_t offset = ftello (warc_current_file); warc_write_start_record (); warc_write_header ("WARC-Type", "response"); @@ -1349,7 +1349,7 @@ warc_write_response_record (char *url, char *timestamp_str, char *concurrent_to_ Calling this function will close body. Returns true on success, false on error. */ bool -warc_write_resource_record (char *resource_uuid, char *url, char *timestamp_str, char *concurrent_to_uuid, ip_address *ip, char *content_type, FILE *body, long int payload_offset) +warc_write_resource_record (char *resource_uuid, char *url, char *timestamp_str, char *concurrent_to_uuid, ip_address *ip, char *content_type, FILE *body, off_t payload_offset) { if (resource_uuid == NULL) { diff --git a/src/warc.h b/src/warc.h index 2ade2a8b..84daad4c 100644 --- a/src/warc.h +++ b/src/warc.h @@ -11,9 +11,9 @@ void warc_uuid_str (char *id_str); FILE * warc_tempfile (); -bool warc_write_request_record (char *url, char *timestamp_str, char *concurrent_to_uuid, ip_address *ip, FILE *body, long int payload_offset); -bool warc_write_response_record (char *url, char *timestamp_str, char *concurrent_to_uuid, ip_address *ip, FILE *body, long int payload_offset, char *mime_type, int response_code, char *redirect_location); -bool warc_write_resource_record (char *resource_uuid, char *url, char *timestamp_str, char *concurrent_to_uuid, ip_address *ip, char *content_type, FILE *body, long int payload_offset); +bool warc_write_request_record (char *url, char *timestamp_str, char *concurrent_to_uuid, ip_address *ip, FILE *body, off_t payload_offset); +bool warc_write_response_record (char *url, char *timestamp_str, char *concurrent_to_uuid, ip_address *ip, FILE *body, off_t payload_offset, char *mime_type, int response_code, char *redirect_location); +bool warc_write_resource_record (char *resource_uuid, char *url, char *timestamp_str, char *concurrent_to_uuid, ip_address *ip, char *content_type, FILE *body, off_t payload_offset); #endif /* WARC_H */ From 04f29f2f08da21cbcebbf86fe98de0522f024c64 Mon Sep 17 00:00:00 2001 From: Giuseppe Scrivano Date: Sun, 26 Feb 2012 02:41:07 +0100 Subject: [PATCH 20/75] Report stdout close errors. --- ChangeLog | 4 ++++ NEWS | 4 +++- bootstrap.conf | 3 ++- src/ChangeLog | 5 +++++ src/main.c | 6 ++++-- 5 files changed, 18 insertions(+), 4 deletions(-) diff --git a/ChangeLog b/ChangeLog index a391a786..39cfd525 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,7 @@ +2012-02-26 Giuseppe Scrivano + + * bootstrap.conf (gnulib_modules): Add module `closeout'. + 2012-01-09 Gijs van Tulder * configure.ac: Always try to use libz, even without SSL. diff --git a/NEWS b/NEWS index 3a6aca0d..5c8184b4 100644 --- a/NEWS +++ b/NEWS @@ -1,7 +1,7 @@ GNU Wget NEWS -- history of user-visible changes. Copyright (C) 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004, 2005, -2006, 2007, 2008, 2009, 2010, 2011 Free Software Foundation, Inc. +2006, 2007, 2008, 2009, 2010, 2011, 2012 Free Software Foundation, Inc. See the end for copying conditions. Please send GNU Wget bug reports to . @@ -18,6 +18,8 @@ Please send GNU Wget bug reports to . ** Autoreconf works again for distributed tarballs. ** Print some diagnostic messages to stderr not to stdout. + +** Report stdout close errors. * Changes in Wget 1.13.4 diff --git a/bootstrap.conf b/bootstrap.conf index 6473cbba..fff26a6d 100644 --- a/bootstrap.conf +++ b/bootstrap.conf @@ -1,5 +1,5 @@ # bootstrap.conf - Bootstrap configuration. -# Copyright (C) 2007, 2008, 2009, 2010, 2011 Free Software Foundation, +# Copyright (C) 2007, 2008, 2009, 2010, 2011, 2012 Free Software Foundation, # Inc. # # This file is part of GNU Wget. @@ -33,6 +33,7 @@ bind c-ctype clock-time close +closeout connect fcntl futimens diff --git a/src/ChangeLog b/src/ChangeLog index 9f221012..caebc5e4 100644 --- a/src/ChangeLog +++ b/src/ChangeLog @@ -1,3 +1,8 @@ +2012-02-26 Giuseppe Scrivano + + * main.c: Include "closeout.h" + (main): Register close_stdout at exit. + 2012-02-01 Gijs van Tulder * warc.c: Fix large file support with ftello, fseeko. diff --git a/src/main.c b/src/main.c index 352715a0..3e731e9d 100644 --- a/src/main.c +++ b/src/main.c @@ -1,6 +1,6 @@ /* Command line parsing. Copyright (C) 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004, - 2005, 2006, 2007, 2008, 2009, 2010, 2011 Free Software Foundation, + 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012 Free Software Foundation, Inc. This file is part of GNU Wget. @@ -56,7 +56,7 @@ as that of the covered work. */ #include "http.h" /* for save_cookies */ #include "ptimer.h" #include "warc.h" - +#include "closeout.h" #include #include #include @@ -966,6 +966,8 @@ main (int argc, char **argv) i18n_initialize (); + atexit (close_stdout); + /* Construct the name of the executable, without the directory part. */ #ifdef __VMS /* On VMS, lose the "dev:[dir]" prefix and the ".EXE;nnn" suffix. */ From b9b510ca5f9c13f8d3f129faae61f324f5c816d5 Mon Sep 17 00:00:00 2001 From: Sasikantha Babu Date: Mon, 5 Mar 2012 22:23:06 +0100 Subject: [PATCH 21/75] Accept --bit option --- NEWS | 2 ++ src/ChangeLog | 13 +++++++++++++ src/init.c | 1 + src/main.c | 6 ++++++ src/options.h | 1 + src/progress.c | 11 ++++++----- src/retr.c | 23 +++++++++++++++-------- src/retr.h | 2 ++ src/utils.c | 11 +++++++++++ 9 files changed, 57 insertions(+), 13 deletions(-) diff --git a/NEWS b/NEWS index 5c8184b4..311a2f1a 100644 --- a/NEWS +++ b/NEWS @@ -20,6 +20,8 @@ Please send GNU Wget bug reports to . ** Print some diagnostic messages to stderr not to stdout. ** Report stdout close errors. + +** Accept the --bit option. * Changes in Wget 1.13.4 diff --git a/src/ChangeLog b/src/ChangeLog index caebc5e4..a156c479 100644 --- a/src/ChangeLog +++ b/src/ChangeLog @@ -1,3 +1,16 @@ +2012-03-06 Sasikantha Babu + + * utils.c (convert_to_bits): Added new function convert_to_bits to + convert bytes to bits. + * retr.c (calc_rate): Modified the function to handle --bits + option and download rate calculated as bits per sec (SI-prefix) + for --bits otherwise bytes (IEC-prefix). + (retr_rate): Rates will display in bits per sec for --bits. + * options.h (struct opt): Added --bit option bool variable bits_fmt. + * main.c (print_help) : Added help for --bit. + * init.c: Defined command for --bit option. + * retr.h: Added function prototype. + 2012-02-26 Giuseppe Scrivano * main.c: Include "closeout.h" diff --git a/src/init.c b/src/init.c index d2fba82c..c890956c 100644 --- a/src/init.c +++ b/src/init.c @@ -127,6 +127,7 @@ static const struct { { "backups", &opt.backups, cmd_number }, { "base", &opt.base_href, cmd_string }, { "bindaddress", &opt.bind_address, cmd_string }, + { "bits", &opt.bits_fmt, cmd_boolean}, #ifdef HAVE_SSL { "cacertificate", &opt.ca_cert, cmd_file }, #endif diff --git a/src/main.c b/src/main.c index 3e731e9d..a12aaf1b 100644 --- a/src/main.c +++ b/src/main.c @@ -167,6 +167,7 @@ static struct cmdline_option option_data[] = { "backups", 0, OPT_BOOLEAN, "backups", -1 }, { "base", 'B', OPT_VALUE, "base", -1 }, { "bind-address", 0, OPT_VALUE, "bindaddress", -1 }, + { "bits", 0, OPT_BOOLEAN, "bits", -1 }, { IF_SSL ("ca-certificate"), 0, OPT_VALUE, "cacertificate", -1 }, { IF_SSL ("ca-directory"), 0, OPT_VALUE, "cadirectory", -1 }, { "cache", 0, OPT_BOOLEAN, "cache", -1 }, @@ -746,6 +747,11 @@ Recursive accept/reject:\n"), -np, --no-parent don't ascend to the parent directory.\n"), "\n", + N_("\ +Output format:\n"), + N_("\ + --bits Output bandwidth in bits.\n"), + "\n", N_("Mail bug reports and suggestions to .\n") }; diff --git a/src/options.h b/src/options.h index 0be66814..1f429906 100644 --- a/src/options.h +++ b/src/options.h @@ -266,6 +266,7 @@ struct options bool show_all_dns_entries; /* Show all the DNS entries when resolving a name. */ + bool bits_fmt; /*Output bandwidth in bits format*/ }; extern struct options opt; diff --git a/src/progress.c b/src/progress.c index 219b5bea..799d6e37 100644 --- a/src/progress.c +++ b/src/progress.c @@ -861,7 +861,7 @@ create_image (struct bar_progress *bp, double dl_total_time, bool done) struct bar_progress_hist *hist = &bp->hist; /* The progress bar should look like this: - xx% [=======> ] nn,nnn 12.34K/s eta 36m 51s + xx% [=======> ] nn,nnn 12.34KB/s eta 36m 51s Calculate the geometry. The idea is to assign as much room as possible to the progress bar. The other idea is to never let @@ -873,7 +873,7 @@ create_image (struct bar_progress *bp, double dl_total_time, bool done) "xx% " or "100%" - percentage - 4 chars "[]" - progress bar decorations - 2 chars " nnn,nnn,nnn" - downloaded bytes - 12 chars or very rarely more - " 12.5K/s" - download rate - 8 chars + " 12.5KB/s" - download rate - 9 chars " eta 36m 51s" - ETA - 14 chars "=====>..." - progress bar - the rest @@ -977,10 +977,11 @@ create_image (struct bar_progress *bp, double dl_total_time, bool done) *p++ = ' '; } - /* " 12.52K/s" */ + /* " 12.52Kb/s or 12.52KB/s" */ if (hist->total_time > 0 && hist->total_bytes) { - static const char *short_units[] = { "B/s", "K/s", "M/s", "G/s" }; + static const char *short_units[] = { "B/s", "KB/s", "MB/s", "GB/s" }; + static const char *short_units_bits[] = { "b/s", "Kb/s", "Mb/s", "Gb/s" }; int units = 0; /* Calculate the download speed using the history ring and recent data that hasn't made it to the ring yet. */ @@ -988,7 +989,7 @@ create_image (struct bar_progress *bp, double dl_total_time, bool done) double dltime = hist->total_time + (dl_total_time - bp->recent_start); double dlspeed = calc_rate (dlquant, dltime, &units); sprintf (p, " %4.*f%s", dlspeed >= 99.95 ? 0 : dlspeed >= 9.995 ? 1 : 2, - dlspeed, short_units[units]); + dlspeed, !opt.bits_fmt?short_units[units]:short_units_bits[units]); move_to_end (p); } else diff --git a/src/retr.c b/src/retr.c index 8c8cdf5b..5f33c7a7 100644 --- a/src/retr.c +++ b/src/retr.c @@ -620,6 +620,7 @@ retr_rate (wgint bytes, double secs) { static char res[20]; static const char *rate_names[] = {"B/s", "KB/s", "MB/s", "GB/s" }; + static const char *rate_names_bits[] = {"b/s", "Kb/s", "Mb/s", "Gb/s" }; int units; double dlrate = calc_rate (bytes, secs, &units); @@ -627,7 +628,7 @@ retr_rate (wgint bytes, double secs) e.g. "1022", "247", "12.5", "2.38". */ sprintf (res, "%.*f %s", dlrate >= 99.95 ? 0 : dlrate >= 9.995 ? 1 : 2, - dlrate, rate_names[units]); + dlrate, !opt.bits_fmt? rate_names[units]: rate_names_bits[units]); return res; } @@ -644,6 +645,11 @@ double calc_rate (wgint bytes, double secs, int *units) { double dlrate; + double bibyte = 1000.0; + + if (!opt.bits_fmt) + bibyte = 1024.0; + assert (secs >= 0); assert (bytes >= 0); @@ -655,16 +661,17 @@ calc_rate (wgint bytes, double secs, int *units) 0 and the timer's resolution, assume half the resolution. */ secs = ptimer_resolution () / 2.0; - dlrate = bytes / secs; - if (dlrate < 1024.0) + dlrate = convert_to_bits (bytes) / secs; + if (dlrate < bibyte) *units = 0; - else if (dlrate < 1024.0 * 1024.0) - *units = 1, dlrate /= 1024.0; - else if (dlrate < 1024.0 * 1024.0 * 1024.0) - *units = 2, dlrate /= (1024.0 * 1024.0); + else if (dlrate < (bibyte * bibyte)) + *units = 1, dlrate /= bibyte; + else if (dlrate < (bibyte * bibyte * bibyte)) + *units = 2, dlrate /= (bibyte * bibyte); + else /* Maybe someone will need this, one day. */ - *units = 3, dlrate /= (1024.0 * 1024.0 * 1024.0); + *units = 3, dlrate /= (bibyte * bibyte * bibyte); return dlrate; } diff --git a/src/retr.h b/src/retr.h index 22ab9ecd..776238b1 100644 --- a/src/retr.h +++ b/src/retr.h @@ -75,4 +75,6 @@ void set_local_file (const char **, const char *); bool input_file_url (const char *); +wgint convert_to_bits (wgint num); + #endif /* RETR_H */ diff --git a/src/utils.c b/src/utils.c index 509088b6..244b03cd 100644 --- a/src/utils.c +++ b/src/utils.c @@ -1825,6 +1825,17 @@ number_to_static_string (wgint number) ringpos = (ringpos + 1) % RING_SIZE; return buf; } + +/* Converts the byte to bits format if --bits option is enabled + */ +wgint +convert_to_bits (wgint num) +{ + if (opt.bits_fmt) + return num * 8; + return num; +} + /* Determine the width of the terminal we're running on. If that's not possible, return 0. */ From 0ccaa999a2eaa51b81e97cd77dff09662b56b914 Mon Sep 17 00:00:00 2001 From: Steven Schubiger Date: Thu, 8 Mar 2012 10:00:51 +0100 Subject: [PATCH 22/75] Fix typo. --- src/ChangeLog | 4 ++++ src/init.c | 4 ++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/src/ChangeLog b/src/ChangeLog index a156c479..265fdc11 100644 --- a/src/ChangeLog +++ b/src/ChangeLog @@ -1,3 +1,7 @@ +2012-03-07 Steven Schubiger + + * init.c (wgetrc_user_file_name): Correct typo. + 2012-03-06 Sasikantha Babu * utils.c (convert_to_bits): Added new function convert_to_bits to diff --git a/src/init.c b/src/init.c index c890956c..76cb2295 100644 --- a/src/init.c +++ b/src/init.c @@ -1,6 +1,6 @@ /* Reading/parsing the initialization file. Copyright (C) 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004, - 2005, 2006, 2007, 2008, 2009, 2010, 2011 Free Software Foundation, + 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012 Free Software Foundation, Inc. This file is part of GNU Wget. @@ -469,7 +469,7 @@ wgetrc_env_file_name (void) return NULL; } -/* Check for the existance of '$HOME/.wgetrc' and return it's path +/* Check for the existance of '$HOME/.wgetrc' and return its path if it exists and is set. */ char * wgetrc_user_file_name (void) From b3014041c505fec614b286379cf608d7b5474ff5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=81ngel=20Gonz=C3=A1lez?= Date: Tue, 20 Mar 2012 20:41:14 +0100 Subject: [PATCH 23/75] Add new gnulib modules. --- ChangeLog | 5 +++++ bootstrap.conf | 3 +++ 2 files changed, 8 insertions(+) diff --git a/ChangeLog b/ChangeLog index 39cfd525..fcc61070 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,8 @@ +2012-03-20 Ángel González + + * bootstrap.conf (gnulib_modules): Add modules `ftello', + `mkstemp' and `strtok_r'. + 2012-02-26 Giuseppe Scrivano * bootstrap.conf (gnulib_modules): Add module `closeout'. diff --git a/bootstrap.conf b/bootstrap.conf index fff26a6d..7ab3ad0a 100644 --- a/bootstrap.conf +++ b/bootstrap.conf @@ -37,6 +37,7 @@ closeout connect fcntl futimens +ftello getaddrinfo getopt-gnu getpass-gnu @@ -50,6 +51,7 @@ listen maintainer-makefile mbtowc mkdir +mkstemp crypto/md5 crypto/sha1 pipe @@ -66,6 +68,7 @@ socket stdbool strcasestr strerror_r-posix +strtok_r tmpdir unlocked-io update-copyright From 44ea82bc67a2f16cb485b51342f8265d3717c18d Mon Sep 17 00:00:00 2001 From: Ray Satiro Date: Sun, 25 Mar 2012 13:47:53 +0200 Subject: [PATCH 24/75] Fix build under mingw when OpenSSL is used. --- ChangeLog | 4 ++++ configure.ac | 10 ++++++---- src/ChangeLog | 5 +++++ src/build_info.c.in | 2 +- 4 files changed, 16 insertions(+), 5 deletions(-) diff --git a/ChangeLog b/ChangeLog index fcc61070..f3e4e566 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,7 @@ +2012-03-25 Ray Satiro + + * configure.ac: Fix build under mingw when OpenSSL is used. + 2012-03-20 Ángel González * bootstrap.conf (gnulib_modules): Add modules `ftello', diff --git a/configure.ac b/configure.ac index 647e44e3..eb1b8b7d 100644 --- a/configure.ac +++ b/configure.ac @@ -264,6 +264,9 @@ AS_IF([test x"$with_ssl" = xopenssl], [ AC_CHECK_LIB(ssl32, SSL_connect, [ ssl_found=yes AC_MSG_NOTICE([Enabling support for SSL via OpenSSL (shared)]) + AC_LIBOBJ([openssl]) + LIBS="${LIBS} -lssl32" + AC_DEFINE([HAVE_LIBSSL32], [1], [Define to 1 if you have the `ssl32' library (-lssl32).]) ], AC_MSG_ERROR([openssl not found: shared lib eay32 found but ssl32 not found])) @@ -289,6 +292,7 @@ AS_IF([test x$ssl_found != xyes], ], [SSL_library_init ()]) if test x"$LIBSSL" != x then + ssl_found=yes AC_MSG_NOTICE([compiling in support for SSL via OpenSSL]) AC_LIBOBJ([openssl]) LIBS="$LIBSSL $LIBS" @@ -296,9 +300,6 @@ AS_IF([test x$ssl_found != xyes], then AC_MSG_ERROR([--with-ssl=openssl was given, but SSL is not available.]) fi - - AC_LIBOBJ([openssl]) - ]) ], [ @@ -316,6 +317,7 @@ AS_IF([test x$ssl_found != xyes], ], [gnutls_global_init()]) if test x"$LIBGNUTLS" != x then + ssl_found=yes AC_MSG_NOTICE([compiling in support for SSL via GnuTLS]) AC_LIBOBJ([gnutls]) LIBS="$LIBGNUTLS $LIBS" @@ -328,7 +330,7 @@ AS_IF([test x$ssl_found != xyes], ]) # endif: --with-ssl == openssl? dnl Enable NTLM if requested and if SSL is available. -if test x"$LIBSSL" != x +if test x"$LIBSSL" != x || test "$ac_cv_lib_ssl32_SSL_connect" = yes then if test x"$ENABLE_NTLM" != xno then diff --git a/src/ChangeLog b/src/ChangeLog index 265fdc11..f52eae29 100644 --- a/src/ChangeLog +++ b/src/ChangeLog @@ -1,3 +1,8 @@ +2012-03-25 Ray Satiro + + * build_info.c.in: Check that HAVE_LIBSSL32 is defined when OpenSSL + is used. + 2012-03-07 Steven Schubiger * init.c (wgetrc_user_file_name): Correct typo. diff --git a/src/build_info.c.in b/src/build_info.c.in index 892962a4..c0b1677f 100644 --- a/src/build_info.c.in +++ b/src/build_info.c.in @@ -9,5 +9,5 @@ ntlm defined ENABLE_NTLM opie defined ENABLE_OPIE ssl choice: - openssl defined HAVE_LIBSSL + openssl defined HAVE_LIBSSL || defined HAVE_LIBSSL32 gnutls defined HAVE_LIBGNUTLS From 6533cf24528c1d3b2fd471d3ae5a312baac7ce15 Mon Sep 17 00:00:00 2001 From: Giuseppe Scrivano Date: Sun, 25 Mar 2012 17:49:55 +0200 Subject: [PATCH 25/75] Assume some headers files provided by gnulib are always present. --- src/ChangeLog | 9 +++++++++ src/connect.c | 13 +++---------- src/ptimer.c | 4 +--- src/utils.c | 4 +--- 4 files changed, 14 insertions(+), 16 deletions(-) diff --git a/src/ChangeLog b/src/ChangeLog index f52eae29..7c7fb93f 100644 --- a/src/ChangeLog +++ b/src/ChangeLog @@ -1,3 +1,12 @@ +2012-03-25 Giuseppe Scrivano + + * utils.c: Include . + + * ptimer.c: Include . + + * connect.c: Include , , . + Reported by: Ray Satiro . + 2012-03-25 Ray Satiro * build_info.c.in: Check that HAVE_LIBSSL32 is defined when OpenSSL diff --git a/src/connect.c b/src/connect.c index 34b40abc..119ccb71 100644 --- a/src/connect.c +++ b/src/connect.c @@ -36,13 +36,8 @@ as that of the covered work. */ #include #include -#ifdef HAVE_SYS_SOCKET_H -# include -#endif /* def HAVE_SYS_SOCKET_H */ - -#ifdef HAVE_SYS_SELECT_H -# include -#endif /* def HAVE_SYS_SELECT_H */ +#include +#include #ifndef WINDOWS # ifdef __VMS @@ -58,9 +53,7 @@ as that of the covered work. */ #include #include -#ifdef HAVE_SYS_TIME_H -# include -#endif +#include #include "utils.h" #include "host.h" #include "connect.h" diff --git a/src/ptimer.c b/src/ptimer.c index c06e8b90..c53b5e72 100644 --- a/src/ptimer.c +++ b/src/ptimer.c @@ -59,9 +59,7 @@ as that of the covered work. */ #include #include #include -#ifdef HAVE_SYS_TIME_H -# include -#endif +#include /* Cygwin currently (as of 2005-04-08, Cygwin 1.5.14) lacks clock_getres, but still defines _POSIX_TIMERS! Because of that we simply use the diff --git a/src/utils.c b/src/utils.c index 244b03cd..1486ed0b 100644 --- a/src/utils.c +++ b/src/utils.c @@ -62,9 +62,7 @@ as that of the covered work. */ #include /* For TIOCGWINSZ and friends: */ -#ifdef HAVE_SYS_IOCTL_H -# include -#endif +#include #ifdef HAVE_TERMIOS_H # include #endif From 2ffc3836540b437f8af8d8cfc15252b7057c1ff2 Mon Sep 17 00:00:00 2001 From: Tim Ruehsen Date: Thu, 29 Mar 2012 20:13:27 +0200 Subject: [PATCH 26/75] activate itimer support. --- src/ChangeLog | 4 ++++ src/utils.c | 2 ++ 2 files changed, 6 insertions(+) diff --git a/src/ChangeLog b/src/ChangeLog index 7c7fb93f..52790ca2 100644 --- a/src/ChangeLog +++ b/src/ChangeLog @@ -1,3 +1,7 @@ +2012-03-29 From: Tim Ruehsen (tiny change) + + * utils.c (library): Include . + 2012-03-25 Giuseppe Scrivano * utils.c: Include . diff --git a/src/utils.c b/src/utils.c index 1486ed0b..4188ced7 100644 --- a/src/utils.c +++ b/src/utils.c @@ -59,6 +59,8 @@ as that of the covered work. */ # endif #endif +#include + #include /* For TIOCGWINSZ and friends: */ From 2541e0b57933dc4bf39284b127bfaaf67868c040 Mon Sep 17 00:00:00 2001 From: Tim Ruehsen Date: Sun, 1 Apr 2012 13:59:46 +0200 Subject: [PATCH 27/75] warc: make warc_uuid_str implementation depend on HAVE_LIBUUID --- src/ChangeLog | 6 +++++- src/warc.c | 49 ++++++++++++++++++++++++------------------------- 2 files changed, 29 insertions(+), 26 deletions(-) diff --git a/src/ChangeLog b/src/ChangeLog index 52790ca2..4d2dd711 100644 --- a/src/ChangeLog +++ b/src/ChangeLog @@ -1,4 +1,8 @@ -2012-03-29 From: Tim Ruehsen (tiny change) +2012-03-30 Tim Ruehsen (tiny change) + + * warc.c: make warc_uuid_str() implementation depend on HAVE_LIBUUID. + +2012-03-29 Tim Ruehsen (tiny change) * utils.c (library): Include . diff --git a/src/warc.c b/src/warc.c index 1c1e0797..b34db425 100644 --- a/src/warc.c +++ b/src/warc.c @@ -580,15 +580,32 @@ warc_timestamp (char *timestamp) strftime (timestamp, 21, "%Y-%m-%dT%H:%M:%SZ", timeinfo); } -/* Fills uuid_str with a UUID based on random numbers. +#ifdef HAVE_LIBUUID +/* Fills urn_str with a UUID in the format required + for the WARC-Record-Id header. + The string will be 47 characters long. */ +void +warc_uuid_str (char *urn_str) +{ + char uuid_str[37]; + + uuid_t record_id; + uuid_generate (record_id); + uuid_unparse (record_id, uuid_str); + + sprintf (urn_str, "", uuid_str); +} +#else +/* Fills urn_str with a UUID based on random numbers in the format + required for the WARC-Record-Id header. (See RFC 4122, UUID version 4.) Note: this is a fallback method, it is much better to use the methods provided by libuuid. - The uuid_str will be 36 characters long. */ -static void -warc_uuid_random (char *uuid_str) + The string will be 47 characters long. */ +void +warc_uuid_str (char *urn_str) { // RFC 4122, a version 4 UUID with only random numbers @@ -605,32 +622,14 @@ warc_uuid_random (char *uuid_str) // clock_seq_hi_and_reserved to zero and one, respectively. uuid_data[8] = (uuid_data[8] & 0xBF) | 0x80; - sprintf (uuid_str, - "%02x%02x%02x%02x-%02x%02x-%02x%02x-%02x%02x-%02x%02x%02x%02x%02x%02x", + sprintf (urn_str, + "", uuid_data[0], uuid_data[1], uuid_data[2], uuid_data[3], uuid_data[4], uuid_data[5], uuid_data[6], uuid_data[7], uuid_data[8], uuid_data[9], uuid_data[10], uuid_data[11], uuid_data[12], uuid_data[13], uuid_data[14], uuid_data[15]); } - -/* Fills urn_str with a UUID in the format required - for the WARC-Record-Id header. - The string will be 47 characters long. */ -void -warc_uuid_str (char *urn_str) -{ - char uuid_str[37]; - -# ifdef HAVE_LIBUUID - uuid_t record_id; - uuid_generate (record_id); - uuid_unparse (record_id, uuid_str); -# else - warc_uuid_random (uuid_str); -# endif - - sprintf (urn_str, "", uuid_str); -} +#endif /* Write a warcinfo record to the current file. Updates warc_current_warcinfo_uuid_str. */ From 3bb17fca04e792fe9668365e2c2af43fdbedafea Mon Sep 17 00:00:00 2001 From: Giuseppe Scrivano Date: Sun, 1 Apr 2012 16:26:44 +0200 Subject: [PATCH 28/75] gnutls: do not access unitialized variable. --- src/ChangeLog | 6 ++++++ src/gnutls.c | 12 ++++++++---- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/src/ChangeLog b/src/ChangeLog index 4d2dd711..d58b1520 100644 --- a/src/ChangeLog +++ b/src/ChangeLog @@ -1,3 +1,9 @@ +2012-04-01 Giuseppe Scrivano + + * gnutls.c (wgnutls_read_timeout): Do not use timer if it is not + allocated. + Reported by: Xu Zhongxing + 2012-03-30 Tim Ruehsen (tiny change) * warc.c: make warc_uuid_str() implementation depend on HAVE_LIBUUID. diff --git a/src/gnutls.c b/src/gnutls.c index 2a1d22b9..2db5a90c 100644 --- a/src/gnutls.c +++ b/src/gnutls.c @@ -1,5 +1,5 @@ /* SSL support via GnuTLS library. - Copyright (C) 2005, 2006, 2007, 2008, 2009, 2010, 2011 Free Software + Copyright (C) 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012 Free Software Foundation, Inc. This file is part of GNU Wget. @@ -160,9 +160,13 @@ wgnutls_read_timeout (int fd, char *buf, int bufsize, void *arg, double timeout) do { - double next_timeout = timeout - ptimer_measure (timer); - if (timeout && next_timeout < 0) - break; + double next_timeout; + if (timeout > 0.0) + { + next_timeout = timeout - ptimer_measure (timer); + if (next_timeout < 0.0) + break; + } ret = GNUTLS_E_AGAIN; if (timeout == 0 || gnutls_record_check_pending (ctx->session) From b30ba732ade43b231f4fe6693f1f833b36f7ffe2 Mon Sep 17 00:00:00 2001 From: Giuseppe Scrivano Date: Sun, 1 Apr 2012 16:30:59 +0200 Subject: [PATCH 29/75] gnutls: Fix a memory leak. --- src/ChangeLog | 2 ++ src/gnutls.c | 21 ++++++++------------- 2 files changed, 10 insertions(+), 13 deletions(-) diff --git a/src/ChangeLog b/src/ChangeLog index d58b1520..2152cce3 100644 --- a/src/ChangeLog +++ b/src/ChangeLog @@ -1,5 +1,7 @@ 2012-04-01 Giuseppe Scrivano + * gnutls.c (wgnutls_read_timeout): Ensure timer is freed. + * gnutls.c (wgnutls_read_timeout): Do not use timer if it is not allocated. Reported by: Xu Zhongxing diff --git a/src/gnutls.c b/src/gnutls.c index 2db5a90c..442b1364 100644 --- a/src/gnutls.c +++ b/src/gnutls.c @@ -175,15 +175,13 @@ wgnutls_read_timeout (int fd, char *buf, int bufsize, void *arg, double timeout) if (timeout) { #ifdef F_GETFL - ret = fcntl (fd, F_SETFL, flags | O_NONBLOCK); - if (ret < 0) - return ret; + if (fcntl (fd, F_SETFL, flags | O_NONBLOCK)) + break; #else /* XXX: Assume it was blocking before. */ const int one = 1; - ret = ioctl (fd, FIONBIO, &one); - if (ret < 0) - return ret; + if (ioctl (fd, FIONBIO, &one) < 0) + break; #endif } @@ -191,16 +189,13 @@ wgnutls_read_timeout (int fd, char *buf, int bufsize, void *arg, double timeout) if (timeout) { - int status; #ifdef F_GETFL - status = fcntl (fd, F_SETFL, flags); - if (status < 0) - return status; + if (fcntl (fd, F_SETFL, flags) < 0) + break; #else const int zero = 0; - status = ioctl (fd, FIONBIO, &zero); - if (status < 0) - return status; + if (ioctl (fd, FIONBIO, &zero) < 0) + break; #endif } } From 08a147c672caca6bed6521ded5729ee4487e6a35 Mon Sep 17 00:00:00 2001 From: Gijs van Tulder Date: Sun, 1 Apr 2012 23:29:16 +0200 Subject: [PATCH 30/75] Fix a segfault on an incomplete STYLE tag. --- NEWS | 1 + src/ChangeLog | 4 ++++ src/html-url.c | 7 ++++--- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/NEWS b/NEWS index 311a2f1a..e0f81a99 100644 --- a/NEWS +++ b/NEWS @@ -34,6 +34,7 @@ Please send GNU Wget bug reports to . ** Return a network failure when FTP downloads fail and --timestamping is specified. +** Fix a segfault on an incomplete STYLE tag. * Changes in Wget 1.13.3 diff --git a/src/ChangeLog b/src/ChangeLog index 2152cce3..6e6d354f 100644 --- a/src/ChangeLog +++ b/src/ChangeLog @@ -1,3 +1,7 @@ +2012-04-01 Gijs van Tulder + + * html-url.c: Prevent crash on incomplete STYLE tag. + 2012-04-01 Giuseppe Scrivano * gnutls.c (wgnutls_read_timeout): Ensure timer is freed. diff --git a/src/html-url.c b/src/html-url.c index f5ab2932..855393a7 100644 --- a/src/html-url.c +++ b/src/html-url.c @@ -1,6 +1,6 @@ /* Collect URLs from HTML source. Copyright (C) 1998, 1999, 2000, 2001, 2002, 2003, 2004, 2005, 2006, - 2007, 2008, 2009, 2010, 2011 Free Software Foundation, Inc. + 2007, 2008, 2009, 2010, 2011, 2012 Free Software Foundation, Inc. This file is part of GNU Wget. @@ -675,8 +675,9 @@ collect_tags_mapper (struct taginfo *tag, void *arg) check_style_attr (tag, ctx); - if (tag->end_tag_p && (0 == strcasecmp (tag->name, "style")) && - tag->contents_begin && tag->contents_end) + if (tag->end_tag_p && (0 == strcasecmp (tag->name, "style")) + && tag->contents_begin && tag->contents_end + && tag->contents_begin <= tag->contents_end) { /* parse contents */ get_urls_css (ctx, tag->contents_begin - ctx->text, From 154d499be275e9af301e4e2676f72668bc7b21c0 Mon Sep 17 00:00:00 2001 From: Daniel Kahn Gillmor Date: Sat, 7 Apr 2012 14:43:12 +0200 Subject: [PATCH 31/75] Enable client certificates when GNU TLS is used. --- NEWS | 2 ++ src/ChangeLog | 5 +++++ src/gnutls.c | 44 ++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 51 insertions(+) diff --git a/NEWS b/NEWS index e0f81a99..5b8d8a63 100644 --- a/NEWS +++ b/NEWS @@ -22,6 +22,8 @@ Please send GNU Wget bug reports to . ** Report stdout close errors. ** Accept the --bit option. + +** Enable client certificates when GNU TLS is used. * Changes in Wget 1.13.4 diff --git a/src/ChangeLog b/src/ChangeLog index 6e6d354f..7c792d62 100644 --- a/src/ChangeLog +++ b/src/ChangeLog @@ -1,3 +1,8 @@ +2012-04-07 Daniel Kahn Gillmor (tiny change) + + * gnutls.c (key_type_to_gnutls_type): New function. + (ssl_init): Use correctly the specified gnutls certificate. + 2012-04-01 Gijs van Tulder * html-url.c: Prevent crash on incomplete STYLE tag. diff --git a/src/gnutls.c b/src/gnutls.c index 442b1364..291da895 100644 --- a/src/gnutls.c +++ b/src/gnutls.c @@ -54,6 +54,20 @@ as that of the covered work. */ # include "w32sock.h" #endif +static int +key_type_to_gnutls_type (enum keyfile_type type) +{ + switch (type) + { + case keyfile_pem: + return GNUTLS_X509_FMT_PEM; + case keyfile_asn1: + return GNUTLS_X509_FMT_DER; + default: + abort (); + } +} + /* Note: some of the functions private to this file have names that begin with "wgnutls_" (e.g. wgnutls_read) so that they wouldn't be confused with actual gnutls functions -- such as the gnutls_read @@ -108,6 +122,36 @@ ssl_init () closedir (dir); } + /* Use the private key from the cert file unless otherwise specified. */ + if (opt.cert_file && !opt.private_key) + { + opt.private_key = opt.cert_file; + opt.private_key_type = opt.cert_type; + } + /* Use the cert from the private key file unless otherwise specified. */ + if (!opt.cert_file && opt.private_key) + { + opt.cert_file = opt.private_key; + opt.cert_type = opt.private_key_type; + } + + if (opt.cert_file && opt.private_key) + { + int type; + if (opt.private_key_type != opt.cert_type) + { + /* GnuTLS can't handle this */ + logprintf (LOG_NOTQUIET, _("ERROR: GnuTLS requires the key and the \ +cert to be of the same type.\n")); + } + + type = key_type_to_gnutls_type (opt.private_key_type); + + gnutls_certificate_set_x509_key_file (credentials, opt.cert_file, + opt.private_key, + type); + } + if (opt.ca_cert) gnutls_certificate_set_x509_trust_file (credentials, opt.ca_cert, GNUTLS_X509_FMT_PEM); From bd4f1e60423c07475db39c979bb4c0c7b7acd22d Mon Sep 17 00:00:00 2001 From: Tim Ruehsen Date: Fri, 13 Apr 2012 21:35:29 +0200 Subject: [PATCH 32/75] Fix a memory leak. --- src/ChangeLog | 5 +++++ src/warc.c | 1 + 2 files changed, 6 insertions(+) diff --git a/src/ChangeLog b/src/ChangeLog index 7c792d62..65e881e7 100644 --- a/src/ChangeLog +++ b/src/ChangeLog @@ -1,3 +1,8 @@ +2012-04-13 Tim Ruehsen (tiny change) + + * warc.c (warc_load_cdx_dedup_file): Fix a memory leak by freeing + `lineptr'. + 2012-04-07 Daniel Kahn Gillmor (tiny change) * gnutls.c (key_type_to_gnutls_type): New function. diff --git a/src/warc.c b/src/warc.c index b34db425..911cebd7 100644 --- a/src/warc.c +++ b/src/warc.c @@ -956,6 +956,7 @@ warc_load_cdx_dedup_file () nrecords); } + free (lineptr); fclose (f); return true; From fd582e454378db9a1e218acf79f24fbe042bed98 Mon Sep 17 00:00:00 2001 From: Phil Pennock Date: Fri, 13 Apr 2012 23:58:46 +0200 Subject: [PATCH 33/75] Add support for TLS SNI --- NEWS | 2 ++ src/ChangeLog | 8 ++++++++ src/gnutls.c | 12 +++++++++++- src/host.c | 17 ++++++++++++++++- src/host.h | 4 +++- src/http.c | 4 ++-- src/openssl.c | 17 +++++++++++++++-- src/ssl.h | 4 ++-- 8 files changed, 59 insertions(+), 9 deletions(-) diff --git a/NEWS b/NEWS index 5b8d8a63..84040909 100644 --- a/NEWS +++ b/NEWS @@ -24,6 +24,8 @@ Please send GNU Wget bug reports to . ** Accept the --bit option. ** Enable client certificates when GNU TLS is used. + +** Add support for TLS Server Name Indication. * Changes in Wget 1.13.4 diff --git a/src/ChangeLog b/src/ChangeLog index 65e881e7..86d0dd13 100644 --- a/src/ChangeLog +++ b/src/ChangeLog @@ -1,3 +1,11 @@ +2009-06-14 Phil Pennock (tiny change) + * host.h: Declare `is_valid_ip_address'. + * host.c (is_valid_ip_address): New function. + * http.c (gethttp): Specify the hostname to ssl_connect_wget. + * gnutls.c (ssl_connect_wget): Specify the server name. + * openssl.c (ssl_connect_wget): Likewise. + * ssl.h: Change method signature for ssl_connect_wget. + 2012-04-13 Tim Ruehsen (tiny change) * warc.c (warc_load_cdx_dedup_file): Fix a memory leak by freeing diff --git a/src/gnutls.c b/src/gnutls.c index 291da895..cbd5e1da 100644 --- a/src/gnutls.c +++ b/src/gnutls.c @@ -54,6 +54,8 @@ as that of the covered work. */ # include "w32sock.h" #endif +#include "host.h" + static int key_type_to_gnutls_type (enum keyfile_type type) { @@ -369,12 +371,20 @@ static struct transport_implementation wgnutls_transport = }; bool -ssl_connect_wget (int fd) +ssl_connect_wget (int fd, const char *hostname) { struct wgnutls_transport_context *ctx; gnutls_session session; int err; gnutls_init (&session, GNUTLS_CLIENT); + + /* We set the server name but only if it's not an IP address. */ + if (! is_valid_ip_address (hostname)) + { + gnutls_server_name_set (session, GNUTLS_NAME_DNS, hostname, + strlen (hostname)); + } + gnutls_set_default_priority (session); gnutls_credentials_set (session, GNUTLS_CRD_CERTIFICATE, credentials); #ifndef FD_TO_SOCKET diff --git a/src/host.c b/src/host.c index 86f107a3..86bf83b3 100644 --- a/src/host.c +++ b/src/host.c @@ -1,6 +1,6 @@ /* Host name resolution and matching. Copyright (C) 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004, - 2005, 2006, 2007, 2008, 2009, 2010, 2011 Free Software Foundation, + 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012 Free Software Foundation, Inc. This file is part of GNU Wget. @@ -914,3 +914,18 @@ host_cleanup (void) host_name_addresses_map = NULL; } } + +bool +is_valid_ip_address (const char *name) +{ + const char *endp; + + endp = name + strlen(name); + if (is_valid_ipv4_address (name, endp)) + return true; +#ifdef ENABLE_IPV6 + if (is_valid_ipv6_address (name, endp)) + return true; +#endif + return false; +} diff --git a/src/host.h b/src/host.h index 3f4a02a2..3f27ea0f 100644 --- a/src/host.h +++ b/src/host.h @@ -1,6 +1,6 @@ /* Declarations for host.c Copyright (C) 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004, - 2005, 2006, 2007, 2008, 2009, 2010, 2011 Free Software Foundation, + 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012 Free Software Foundation, Inc. This file is part of GNU Wget. @@ -98,6 +98,8 @@ const char *print_address (const ip_address *); bool is_valid_ipv6_address (const char *, const char *); #endif +bool is_valid_ip_address (const char *name); + bool accept_domain (struct url *); bool sufmatch (const char **, const char *); diff --git a/src/http.c b/src/http.c index 61001f3b..87d3748c 100644 --- a/src/http.c +++ b/src/http.c @@ -1,6 +1,6 @@ /* HTTP support. Copyright (C) 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004, - 2005, 2006, 2007, 2008, 2009, 2010, 2011 Free Software Foundation, + 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012 Free Software Foundation, Inc. This file is part of GNU Wget. @@ -2082,7 +2082,7 @@ gethttp (struct url *u, struct http_stat *hs, int *dt, struct url *proxy, if (conn->scheme == SCHEME_HTTPS) { - if (!ssl_connect_wget (sock)) + if (!ssl_connect_wget (sock, u->host)) { fd_close (sock); return CONSSLERR; diff --git a/src/openssl.c b/src/openssl.c index bc374915..f976455f 100644 --- a/src/openssl.c +++ b/src/openssl.c @@ -1,6 +1,6 @@ /* SSL support via OpenSSL library. Copyright (C) 2000, 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, - 2009, 2010, 2011 Free Software Foundation, Inc. + 2009, 2010, 2011, 2012 Free Software Foundation, Inc. Originally contributed by Christian Fraenkel. This file is part of GNU Wget. @@ -395,7 +395,7 @@ static struct transport_implementation openssl_transport = { Returns true on success, false on failure. */ bool -ssl_connect_wget (int fd) +ssl_connect_wget (int fd, const char *hostname) { SSL *conn; struct openssl_transport_context *ctx; @@ -406,6 +406,19 @@ ssl_connect_wget (int fd) conn = SSL_new (ssl_ctx); if (!conn) goto error; +#if OPENSSL_VERSION_NUMBER >= 0x0090806fL && !defined(OPENSSL_NO_TLSEXT) + /* If the SSL library was build with support for ServerNameIndication + then use it whenever we have a hostname. If not, don't, ever. */ + if (! is_valid_ip_address (hostname)) + { + if (! SSL_set_tlsext_host_name (conn, hostname)) + { + DEBUGP (("Failed to set TLS server-name indication.")); + goto error; + } + } +#endif + #ifndef FD_TO_SOCKET # define FD_TO_SOCKET(X) (X) #endif diff --git a/src/ssl.h b/src/ssl.h index 0532c402..e365c4f4 100644 --- a/src/ssl.h +++ b/src/ssl.h @@ -1,6 +1,6 @@ /* SSL support. Copyright (C) 2000, 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, - 2009, 2010, 2011 Free Software Foundation, Inc. + 2009, 2010, 2011, 2012 Free Software Foundation, Inc. Originally contributed by Christian Fraenkel. This file is part of GNU Wget. @@ -33,7 +33,7 @@ as that of the covered work. */ #define GEN_SSLFUNC_H bool ssl_init (void); -bool ssl_connect_wget (int); +bool ssl_connect_wget (int, const char *); bool ssl_check_certificate (int, const char *); #endif /* GEN_SSLFUNC_H */ From f1d4aeaffb7f682d825ecd6d85a71b1e4a30f189 Mon Sep 17 00:00:00 2001 From: Tim Ruehsen Date: Sat, 21 Apr 2012 12:08:45 +0200 Subject: [PATCH 34/75] Fix memory leak. --- src/ChangeLog | 4 ++++ src/ftp-basic.c | 10 ++++++++-- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/src/ChangeLog b/src/ChangeLog index 86d0dd13..a22d2527 100644 --- a/src/ChangeLog +++ b/src/ChangeLog @@ -1,3 +1,7 @@ +2012-04-21 Tim Ruehsen + + * ftp-basic.c (ftp_pasv): Fix memory leak. + 2009-06-14 Phil Pennock (tiny change) * host.h: Declare `is_valid_ip_address'. * host.c (is_valid_ip_address): New function. diff --git a/src/ftp-basic.c b/src/ftp-basic.c index 178fdfea..045d125e 100644 --- a/src/ftp-basic.c +++ b/src/ftp-basic.c @@ -524,7 +524,10 @@ ftp_pasv (int csock, ip_address *addr, int *port) for (s += 4; *s && !c_isdigit (*s); s++) ; if (!*s) - return FTPINVPASV; + { + xfree (respline); + return FTPINVPASV; + } for (i = 0; i < 6; i++) { tmp[i] = 0; @@ -593,7 +596,10 @@ ftp_lpsv (int csock, ip_address *addr, int *port) for (s += 4; *s && !c_isdigit (*s); s++) ; if (!*s) - return FTPINVPASV; + { + xfree (respline); + return FTPINVPASV; + } /* First, get the address family */ af = 0; From 0fcd1bb235fd3a29df49a70683ac4156658b4e17 Mon Sep 17 00:00:00 2001 From: Tim Ruehsen Date: Sat, 21 Apr 2012 12:19:25 +0200 Subject: [PATCH 35/75] Fix memory leak. --- src/ChangeLog | 2 ++ src/http.c | 10 ++++++++++ 2 files changed, 12 insertions(+) diff --git a/src/ChangeLog b/src/ChangeLog index a22d2527..f12875e1 100644 --- a/src/ChangeLog +++ b/src/ChangeLog @@ -2,6 +2,8 @@ * ftp-basic.c (ftp_pasv): Fix memory leak. + * http.c (gethttp): Fix memory leak. + 2009-06-14 Phil Pennock (tiny change) * host.h: Declare `is_valid_ip_address'. * host.c (is_valid_ip_address): New function. diff --git a/src/http.c b/src/http.c index 87d3748c..cf901929 100644 --- a/src/http.c +++ b/src/http.c @@ -2030,6 +2030,7 @@ gethttp (struct url *u, struct http_stat *hs, int *dt, struct url *proxy, if (write_error < 0) { CLOSE_INVALIDATE (sock); + request_free (req); return WRITEFAILED; } @@ -2039,6 +2040,7 @@ gethttp (struct url *u, struct http_stat *hs, int *dt, struct url *proxy, logprintf (LOG_VERBOSE, _("Failed reading proxy response: %s\n"), fd_errstr (sock)); CLOSE_INVALIDATE (sock); + request_free (req); return HERR; } message = NULL; @@ -2059,6 +2061,7 @@ gethttp (struct url *u, struct http_stat *hs, int *dt, struct url *proxy, quotearg_style (escape_quoting_style, _("Malformed status line"))); xfree (head); + request_free (req); return HERR; } hs->message = xstrdup (message); @@ -2070,6 +2073,7 @@ gethttp (struct url *u, struct http_stat *hs, int *dt, struct url *proxy, logprintf (LOG_NOTQUIET, _("Proxy tunneling failed: %s"), message ? quotearg_style (escape_quoting_style, message) : "?"); xfree_null (message); + request_free (req); return CONSSLERR; } xfree_null (message); @@ -2085,11 +2089,13 @@ gethttp (struct url *u, struct http_stat *hs, int *dt, struct url *proxy, if (!ssl_connect_wget (sock, u->host)) { fd_close (sock); + request_free (req); return CONSSLERR; } else if (!ssl_check_certificate (sock, u->host)) { fd_close (sock); + request_free (req); return VERIFCERTERR; } using_ssl = true; @@ -2222,6 +2228,7 @@ read_header: quotearg_style (escape_quoting_style, _("Malformed status line"))); CLOSE_INVALIDATE (sock); + resp_free (resp); request_free (req); xfree (head); return HERR; @@ -2230,6 +2237,7 @@ read_header: if (H_10X (statcode)) { DEBUGP (("Ignoring response\n")); + resp_free (resp); xfree (head); goto read_header; } @@ -2450,6 +2458,8 @@ read_header: retrieve the file. But if the output_document was given, then this test was already done and the file didn't exist. Hence the !opt.output_document */ get_file_flags (hs->local_file, dt); + request_free (req); + resp_free (resp); xfree (head); xfree_null (message); return RETRUNNEEDED; From 196f70a7df2f2249fcb84446a899f265cdb4129c Mon Sep 17 00:00:00 2001 From: Tim Ruehsen Date: Sat, 21 Apr 2012 13:48:18 +0200 Subject: [PATCH 36/75] Silent compiler warning. --- src/ChangeLog | 2 ++ src/ftp.c | 35 +++++++++++------------------------ 2 files changed, 13 insertions(+), 24 deletions(-) diff --git a/src/ChangeLog b/src/ChangeLog index f12875e1..08266a73 100644 --- a/src/ChangeLog +++ b/src/ChangeLog @@ -4,6 +4,8 @@ * http.c (gethttp): Fix memory leak. + * ftp.c (getftp): Silent compiler warning. + 2009-06-14 Phil Pennock (tiny change) * host.h: Declare `is_valid_ip_address'. * host.c (is_valid_ip_address): New function. diff --git a/src/ftp.c b/src/ftp.c index 989a1dda..669e6637 100644 --- a/src/ftp.c +++ b/src/ftp.c @@ -247,9 +247,8 @@ getftp (struct url *u, wgint passed_expected_bytes, wgint *qtyread, int csock, dtsock, local_sock, res; uerr_t err = RETROK; /* appease the compiler */ FILE *fp; - char *user, *passwd, *respline; - char *tms; - const char *tmrate; + char *respline, *tms; + const char *user, *passwd, *tmrate; int cmd = con->cmd; bool pasv_mode_open = false; wgint expected_bytes = 0; @@ -289,13 +288,6 @@ getftp (struct url *u, wgint passed_expected_bytes, wgint *qtyread, { char *host = con->proxy ? con->proxy->host : u->host; int port = con->proxy ? con->proxy->port : u->port; - char *logname = user; - - if (con->proxy) - { - /* If proxy is in use, log in as username@target-site. */ - logname = concat_strings (user, "@", u->host, (char *) 0); - } /* Login to the server: */ @@ -303,20 +295,10 @@ getftp (struct url *u, wgint passed_expected_bytes, wgint *qtyread, csock = connect_to_host (host, port); if (csock == E_HOST) - { - if (con->proxy) - xfree (logname); - return HOSTERR; - } else if (csock < 0) - { - if (con->proxy) - xfree (logname); - return (retryable_socket_connect_error (errno) ? CONERROR : CONIMPOSSIBLE); - } if (cmd & LEAVE_PENDING) con->csock = csock; @@ -328,10 +310,15 @@ getftp (struct url *u, wgint passed_expected_bytes, wgint *qtyread, quotearg_style (escape_quoting_style, user)); if (opt.server_response) logputs (LOG_ALWAYS, "\n"); - err = ftp_login (csock, logname, passwd); - if (con->proxy) - xfree (logname); + { + /* If proxy is in use, log in as username@target-site. */ + char *logname = concat_strings (user, "@", u->host, (char *) 0); + err = ftp_login (csock, logname, passwd); + xfree (logname); + } + else + err = ftp_login (csock, user, passwd); /* FTPRERR, FTPSRVERR, WRITEFAILED, FTPLOGREFUSED, FTPLOGINC */ switch (err) @@ -514,7 +501,7 @@ Error in server response, closing control connection.\n")); logputs (LOG_VERBOSE, _("==> CWD not needed.\n")); else { - char *targ = NULL; + const char *targ = NULL; int cwd_count; int cwd_end; int cwd_start; From c6889dab18d3fcbcb9e9ac28ccdf684131a0089f Mon Sep 17 00:00:00 2001 From: Tim Ruehsen Date: Sun, 22 Apr 2012 18:36:09 +0200 Subject: [PATCH 37/75] Fix a possible invalid `free'. --- src/ChangeLog | 4 ++++ src/main.c | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/src/ChangeLog b/src/ChangeLog index 08266a73..44d4b54d 100644 --- a/src/ChangeLog +++ b/src/ChangeLog @@ -1,3 +1,7 @@ +2012-04-22 Tim Ruehsen + + * main.c (main): Dynamically allocate `opt.progress_type'. + 2012-04-21 Tim Ruehsen * ftp-basic.c (ftp_pasv): Fix memory leak. diff --git a/src/main.c b/src/main.c index a12aaf1b..438085e4 100644 --- a/src/main.c +++ b/src/main.c @@ -1299,7 +1299,7 @@ for details.\n\n")); } if (opt.warc_keep_log) { - opt.progress_type = "dot"; + opt.progress_type = xstrdup ("dot"); } } From 378c2030799bdc51d7240321ad06607c5b2d1a31 Mon Sep 17 00:00:00 2001 From: Tim Ruehsen Date: Tue, 24 Apr 2012 21:46:06 +0200 Subject: [PATCH 38/75] Use empty query in local filenames. --- src/ChangeLog | 4 ++++ src/url.c | 9 ++++----- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/src/ChangeLog b/src/ChangeLog index 44d4b54d..c7675e88 100644 --- a/src/ChangeLog +++ b/src/ChangeLog @@ -1,3 +1,7 @@ +2012-03-30 Tim Ruehsen + + * url.c: Use empty query in local filenames. + 2012-04-22 Tim Ruehsen * main.c (main): Dynamically allocate `opt.progress_type'. diff --git a/src/url.c b/src/url.c index 2593d09e..ddde798c 100644 --- a/src/url.c +++ b/src/url.c @@ -1502,7 +1502,7 @@ url_file_name (const struct url *u, char *replaced_filename) { struct growable fnres; /* stands for "file name result" */ - const char *u_file, *u_query; + const char *u_file; char *fname, *unique; char *index_filename = "index.html"; /* The default index file is index.html */ @@ -1561,12 +1561,11 @@ url_file_name (const struct url *u, char *replaced_filename) u_file = *u->file ? u->file : index_filename; append_uri_pathel (u_file, u_file + strlen (u_file), false, &fnres); - /* Append "?query" to the file name. */ - u_query = u->query && *u->query ? u->query : NULL; - if (u_query) + /* Append "?query" to the file name, even if empty */ + if (u->query) { append_char (FN_QUERY_SEP, &fnres); - append_uri_pathel (u_query, u_query + strlen (u_query), + append_uri_pathel (u->query, u->query + strlen (u->query), true, &fnres); } } From 0aa3c5d33c5faa8902fa638c36314deae45460f3 Mon Sep 17 00:00:00 2001 From: Tim Ruehsen Date: Sat, 5 May 2012 15:24:35 +0200 Subject: [PATCH 39/75] Fix some compiler warnings. --- src/ChangeLog | 32 ++++++++++++++++++++++++++++++++ src/convert.c | 2 +- src/cookies.c | 11 ++++------- src/css-url.c | 3 ++- src/css-url.h | 1 + src/gnutls.c | 2 +- src/html-parse.c | 6 +++--- src/html-url.c | 2 +- src/progress.c | 4 ++-- src/retr.h | 2 -- src/spider.c | 2 +- src/utils.h | 1 + src/warc.c | 22 +++++++++++----------- src/warc.h | 6 +++--- 14 files changed, 63 insertions(+), 33 deletions(-) diff --git a/src/ChangeLog b/src/ChangeLog index c7675e88..a1d2c2a5 100644 --- a/src/ChangeLog +++ b/src/ChangeLog @@ -1,3 +1,35 @@ +2012-03-30 Tim Ruehsen + + * convert.c (convert_links_in_hashtable): Mmake it static. + * cookies.c (parse_set_cookie): Remove empty else branches. + * css-url.c: Include "css-url.h". + (get_uri_string): Make it static. + * css-url.h (get_urls_css): Add protoype. + * gnutls.c (ssl_init): Add prototype. + * html-parse.c (tagstack_push): Make it static. + * html-parse.c (tagstack_pop): Make it static. + * html-parse.c (tagstack_find): Make it static. + * html-url.c (cleanup_html_url): Make it static. + * progress.c (count_cols): Make it static. + * progress.c (get_eta): Make it static. + * retr.h (convert_to_bits): Remove prototype. + * util.h (convert_to_bits): Add prototype. + * spider.c (spider_cleanup): Make it static. + * warc.c (warc_write_start_record): Add prototype. + * warc.c (warc_write_end_record): Add prototype. + * warc.c (warc_start_cdx_file): Add prototype. + * warc.c (warc_init): Add prototype. + * warc.c (warc_load_cdx_dedup_file): Add prototype. + * warc.c (warc_write_metadata): Add prototype. + * warc.c (warc_close): Add prototype. + * warc.c (warc_tempfile): Add prototype. + * warc.c (warc_write_warcinfo_record): Make it static. + * warc.c (warc_load_cdx_dedup_file): Make it static. + * warc.c (warc_write_metadata): Make it static. + * warc.h (warc_init): Fix prototype. + * warc.h (warc_close): Fix prototype. + * warc.h (warc_tempfile): Fix prototype. + 2012-03-30 Tim Ruehsen * url.c: Use empty query in local filenames. diff --git a/src/convert.c b/src/convert.c index c6ccf534..6cf6f272 100644 --- a/src/convert.c +++ b/src/convert.c @@ -58,7 +58,7 @@ struct hash_table *downloaded_css_set; static void convert_links (const char *, struct urlpos *); -void +static void convert_links_in_hashtable (struct hash_table *downloaded_set, int is_css, int *file_count) diff --git a/src/cookies.c b/src/cookies.c index 7c3fb1cb..a10971ca 100644 --- a/src/cookies.c +++ b/src/cookies.c @@ -391,6 +391,9 @@ parse_set_cookie (const char *set_cookie, bool silent) goto error; BOUNDED_TO_ALLOCA (value.b, value.e, value_copy); + /* Check if expiration spec is valid. + If not, assume default (cookie doesn't expire, but valid only for + this session.) */ expires = http_atotm (value_copy); if (expires != (time_t) -1) { @@ -402,10 +405,6 @@ parse_set_cookie (const char *set_cookie, bool silent) if (cookie->expiry_time < cookies_now) cookie->discard_requested = 1; } - else - /* Error in expiration spec. Assume default (cookie doesn't - expire, but valid only for this session.) */ - ; } else if (TOKEN_IS (name, "max-age")) { @@ -433,9 +432,7 @@ parse_set_cookie (const char *set_cookie, bool silent) /* ignore value completely */ cookie->secure = 1; } - else - /* Ignore unrecognized attribute. */ - ; + /* else: Ignore unrecognized attribute. */ } if (*ptr) /* extract_param has encountered a syntax error */ diff --git a/src/css-url.c b/src/css-url.c index de1caad9..f97690d6 100644 --- a/src/css-url.c +++ b/src/css-url.c @@ -55,6 +55,7 @@ as that of the covered work. */ #include "convert.h" #include "html-url.h" #include "css-tokens.h" +#include "css-url.h" /* from lex.yy.c */ extern char *yytext; @@ -107,7 +108,7 @@ const char *token_names[] = { whitespace after the opening parenthesis and before the closing parenthesis. */ -char * +static char * get_uri_string (const char *at, int *pos, int *length) { char *uri; diff --git a/src/css-url.h b/src/css-url.h index 8d32c34f..7f940e69 100644 --- a/src/css-url.h +++ b/src/css-url.h @@ -30,6 +30,7 @@ as that of the covered work. */ #ifndef CSS_URL_H #define CSS_URL_H +void get_urls_css (struct map_context *, int, int); void get_urls_css (struct map_context *, int, int); struct urlpos *get_urls_css_file (const char *, const char *); diff --git a/src/gnutls.c b/src/gnutls.c index cbd5e1da..7cc2e718 100644 --- a/src/gnutls.c +++ b/src/gnutls.c @@ -77,7 +77,7 @@ key_type_to_gnutls_type (enum keyfile_type type) static gnutls_certificate_credentials credentials; bool -ssl_init () +ssl_init (void) { /* Becomes true if GnuTLS is initialized. */ static bool ssl_initialized = false; diff --git a/src/html-parse.c b/src/html-parse.c index 9fafd8f5..20791cd8 100644 --- a/src/html-parse.c +++ b/src/html-parse.c @@ -280,7 +280,7 @@ struct tagstack_item { struct tagstack_item *next; }; -struct tagstack_item * +static struct tagstack_item * tagstack_push (struct tagstack_item **head, struct tagstack_item **tail) { struct tagstack_item *ts = xmalloc(sizeof(struct tagstack_item)); @@ -301,7 +301,7 @@ tagstack_push (struct tagstack_item **head, struct tagstack_item **tail) } /* remove ts and everything after it from the stack */ -void +static void tagstack_pop (struct tagstack_item **head, struct tagstack_item **tail, struct tagstack_item *ts) { @@ -343,7 +343,7 @@ tagstack_pop (struct tagstack_item **head, struct tagstack_item **tail, } } -struct tagstack_item * +static struct tagstack_item * tagstack_find (struct tagstack_item *tail, const char *tagname_begin, const char *tagname_end) { diff --git a/src/html-url.c b/src/html-url.c index 855393a7..55563e2d 100644 --- a/src/html-url.c +++ b/src/html-url.c @@ -830,7 +830,7 @@ get_urls_file (const char *file) return head; } -void +static void cleanup_html_url (void) { /* Destroy the hash tables. The hash table keys and values are not diff --git a/src/progress.c b/src/progress.c index 799d6e37..f61c95e5 100644 --- a/src/progress.c +++ b/src/progress.c @@ -766,7 +766,7 @@ update_speed_ring (struct bar_progress *bp, wgint howmuch, double dltime) } #if USE_NLS_PROGRESS_BAR -int +static int count_cols (const char *mbs) { wchar_t wc; @@ -795,7 +795,7 @@ count_cols (const char *mbs) # define count_cols(mbs) ((int)(strlen(mbs))) #endif -const char * +static const char * get_eta (int *bcd) { /* TRANSLATORS: "ETA" is English-centric, but this must diff --git a/src/retr.h b/src/retr.h index 776238b1..22ab9ecd 100644 --- a/src/retr.h +++ b/src/retr.h @@ -75,6 +75,4 @@ void set_local_file (const char **, const char *); bool input_file_url (const char *); -wgint convert_to_bits (wgint num); - #endif /* RETR_H */ diff --git a/src/spider.c b/src/spider.c index ae2f392c..dad9a23d 100644 --- a/src/spider.c +++ b/src/spider.c @@ -45,7 +45,7 @@ static struct hash_table *nonexisting_urls_set; /* Cleanup the data structures associated with this file. */ -void +static void spider_cleanup (void) { if (nonexisting_urls_set) diff --git a/src/utils.h b/src/utils.h index 8b1a8a11..514c5f26 100644 --- a/src/utils.h +++ b/src/utils.h @@ -127,6 +127,7 @@ char *human_readable (HR_NUMTYPE); int numdigit (wgint); char *number_to_string (char *, wgint); char *number_to_static_string (wgint); +wgint convert_to_bits (wgint); int determine_screen_width (void); int random_number (int); diff --git a/src/warc.c b/src/warc.c index 911cebd7..fa0830dc 100644 --- a/src/warc.c +++ b/src/warc.c @@ -180,7 +180,7 @@ warc_write_string (const char *str) Returns false and set warc_write_ok to false if there is an error. */ static bool -warc_write_start_record () +warc_write_start_record (void) { if (!warc_write_ok) return false; @@ -279,7 +279,7 @@ warc_write_block_from_file (FILE *data_in) with the uncompressed and compressed length of the record. */ static bool -warc_write_end_record () +warc_write_end_record (void) { warc_write_buffer ("\r\n\r\n", 4); @@ -633,7 +633,7 @@ warc_uuid_str (char *urn_str) /* Write a warcinfo record to the current file. Updates warc_current_warcinfo_uuid_str. */ -bool +static bool warc_write_warcinfo_record (char *filename) { /* Write warc-info record as the first record of the file. */ @@ -760,7 +760,7 @@ warc_start_new_file (bool meta) /* Opens the CDX file for output. */ static bool -warc_start_cdx_file () +warc_start_cdx_file (void) { int filename_length = strlen (opt.warc_filename); char *cdx_filename = alloca (filename_length + 4 + 1); @@ -899,8 +899,8 @@ warc_process_cdx_line (char *lineptr, int field_num_original_url, int field_num_ /* Loads the CDX file from opt.warc_cdx_dedup_filename and fills the warc_cdx_dedup_table. */ -bool -warc_load_cdx_dedup_file () +static bool +warc_load_cdx_dedup_file (void) { FILE *f = fopen (opt.warc_cdx_dedup_filename, "r"); if (f == NULL) @@ -985,7 +985,7 @@ warc_find_duplicate_cdx_record (char *url, char *sha1_digest_payload) /* Initializes the WARC writer (if opt.warc_filename is set). This should be called before any WARC record is written. */ void -warc_init () +warc_init (void) { warc_write_ok = true; @@ -1039,8 +1039,8 @@ warc_init () } /* Writes metadata (manifest, configuration, log file) to the WARC file. */ -void -warc_write_metadata () +static void +warc_write_metadata (void) { /* If there are multiple WARC files, the metadata should be written to a separate file. */ if (opt.warc_maxsize > 0) @@ -1087,7 +1087,7 @@ warc_write_metadata () /* Finishes the WARC writing. This should be called at the end of the program. */ void -warc_close () +warc_close (void) { if (warc_current_file != NULL) { @@ -1108,7 +1108,7 @@ warc_close () The temporary file will be created in opt.warc_tempdir. Returns the pointer to the temporary file, or NULL. */ FILE * -warc_tempfile () +warc_tempfile (void) { char filename[100]; if (path_search (filename, 100, opt.warc_tempdir, "wget", true) == -1) diff --git a/src/warc.h b/src/warc.h index 84daad4c..41829d12 100644 --- a/src/warc.h +++ b/src/warc.h @@ -4,12 +4,12 @@ #include "host.h" -void warc_init (); -void warc_close (); +void warc_init (void); +void warc_close (void); void warc_timestamp (char *timestamp); void warc_uuid_str (char *id_str); -FILE * warc_tempfile (); +FILE * warc_tempfile (void); bool warc_write_request_record (char *url, char *timestamp_str, char *concurrent_to_uuid, ip_address *ip, FILE *body, off_t payload_offset); bool warc_write_response_record (char *url, char *timestamp_str, char *concurrent_to_uuid, ip_address *ip, FILE *body, off_t payload_offset, char *mime_type, int response_code, char *redirect_location); From f5a10978710a3e9907fbef31b7df9f414acccb16 Mon Sep 17 00:00:00 2001 From: Gijs van Tulder Date: Wed, 9 May 2012 21:18:23 +0200 Subject: [PATCH 40/75] Add support for -accept-regex and --reject-regex. --- ChangeLog | 5 +++ bootstrap.conf | 1 + configure.ac | 12 ++++++ src/ChangeLog | 9 +++++ src/init.c | 29 ++++++++++++++ src/main.c | 43 +++++++++++++++++++++ src/options.h | 13 +++++++ src/recur.c | 5 +++ src/utils.c | 101 +++++++++++++++++++++++++++++++++++++++++++++++++ src/utils.h | 9 +++++ 10 files changed, 227 insertions(+) diff --git a/ChangeLog b/ChangeLog index f3e4e566..51632aa2 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,8 @@ +2012-04-11 Gijs van Tulder + + * bootstrap.conf (gnulib_modules): Include module `regex'. + * configure.ac: Check for PCRE library. + 2012-03-25 Ray Satiro * configure.ac: Fix build under mingw when OpenSSL is used. diff --git a/bootstrap.conf b/bootstrap.conf index 7ab3ad0a..56b7b278 100644 --- a/bootstrap.conf +++ b/bootstrap.conf @@ -58,6 +58,7 @@ pipe quote quotearg recv +regex select send setsockopt diff --git a/configure.ac b/configure.ac index eb1b8b7d..45cebcab 100644 --- a/configure.ac +++ b/configure.ac @@ -532,6 +532,18 @@ AC_CHECK_HEADER(uuid/uuid.h, ]) ) +dnl +dnl Check for PCRE +dnl + +AC_CHECK_HEADER(pcre.h, + AC_CHECK_LIB(pcre, pcre_compile, + [LIBS="${LIBS} -lpcre" + AC_DEFINE([HAVE_LIBPCRE], 1, + [Define if libpcre is available.]) + ]) +) + dnl Needed by src/Makefile.am AM_CONDITIONAL([IRI_IS_ENABLED], [test "X$iri" != "Xno"]) diff --git a/src/ChangeLog b/src/ChangeLog index a1d2c2a5..32d58192 100644 --- a/src/ChangeLog +++ b/src/ChangeLog @@ -1,3 +1,12 @@ +2012-04-11 Gijs van Tulder + + * init.c: Add --accept-regex, --reject-regex and --regex-type. + * main.c: Likewise. + * options.c: Likewise. + * recur.c: Likewise. + * utils.c: Add regex-related functions. + * utils.h: Add regex-related functions. + 2012-03-30 Tim Ruehsen * convert.c (convert_links_in_hashtable): Mmake it static. diff --git a/src/init.c b/src/init.c index 76cb2295..57a4f00d 100644 --- a/src/init.c +++ b/src/init.c @@ -46,6 +46,10 @@ as that of the covered work. */ # endif #endif +#include +#ifdef HAVE_LIBPCRE +# include +#endif #ifdef HAVE_PWD_H # include @@ -94,6 +98,7 @@ CMD_DECLARE (cmd_spec_mirror); CMD_DECLARE (cmd_spec_prefer_family); CMD_DECLARE (cmd_spec_progress); CMD_DECLARE (cmd_spec_recursive); +CMD_DECLARE (cmd_spec_regex_type); CMD_DECLARE (cmd_spec_restrict_file_names); #ifdef HAVE_SSL CMD_DECLARE (cmd_spec_secure_protocol); @@ -116,6 +121,7 @@ static const struct { } commands[] = { /* KEEP THIS LIST ALPHABETICALLY SORTED */ { "accept", &opt.accepts, cmd_vector }, + { "acceptregex", &opt.acceptregex_s, cmd_string }, { "addhostdir", &opt.add_hostdir, cmd_boolean }, { "adjustextension", &opt.adjust_extension, cmd_boolean }, { "alwaysrest", &opt.always_rest, cmd_boolean }, /* deprecated */ @@ -236,7 +242,9 @@ static const struct { { "reclevel", &opt.reclevel, cmd_number_inf }, { "recursive", NULL, cmd_spec_recursive }, { "referer", &opt.referer, cmd_string }, + { "regextype", &opt.regex_type, cmd_spec_regex_type }, { "reject", &opt.rejects, cmd_vector }, + { "rejectregex", &opt.rejectregex_s, cmd_string }, { "relativeonly", &opt.relative_only, cmd_boolean }, { "remoteencoding", &opt.encoding_remote, cmd_string }, { "removelisting", &opt.remove_listing, cmd_boolean }, @@ -361,6 +369,8 @@ defaults (void) opt.restrict_files_nonascii = false; opt.restrict_files_case = restrict_no_case_restriction; + opt.regex_type = regex_type_posix; + opt.max_redirect = 20; opt.waitretry = 10; @@ -1368,6 +1378,25 @@ cmd_spec_recursive (const char *com, const char *val, void *place_ignored) return true; } +/* Validate --regex-type and set the choice. */ + +static bool +cmd_spec_regex_type (const char *com, const char *val, void *place_ignored) +{ + static const struct decode_item choices[] = { + { "posix", regex_type_posix }, +#ifdef HAVE_LIBPCRE + { "pcre", regex_type_pcre }, +#endif + }; + int regex_type = regex_type_posix; + int ok = decode_string (val, choices, countof (choices), ®ex_type); + if (!ok) + fprintf (stderr, _("%s: %s: Invalid value %s.\n"), exec_name, com, quote (val)); + opt.regex_type = regex_type; + return ok; +} + static bool cmd_spec_restrict_file_names (const char *com, const char *val, void *place_ignored) { diff --git a/src/main.c b/src/main.c index 438085e4..aac01ac8 100644 --- a/src/main.c +++ b/src/main.c @@ -158,6 +158,7 @@ struct cmdline_option { static struct cmdline_option option_data[] = { { "accept", 'A', OPT_VALUE, "accept", -1 }, + { "accept-regex", 0, OPT_VALUE, "acceptregex", -1 }, { "adjust-extension", 'E', OPT_BOOLEAN, "adjustextension", -1 }, { "append-output", 'a', OPT__APPEND_OUTPUT, NULL, required_argument }, { "ask-password", 0, OPT_BOOLEAN, "askpassword", -1 }, @@ -262,7 +263,9 @@ static struct cmdline_option option_data[] = { "read-timeout", 0, OPT_VALUE, "readtimeout", -1 }, { "recursive", 'r', OPT_BOOLEAN, "recursive", -1 }, { "referer", 0, OPT_VALUE, "referer", -1 }, + { "regex-type", 0, OPT_VALUE, "regextype", -1 }, { "reject", 'R', OPT_VALUE, "reject", -1 }, + { "reject-regex", 0, OPT_VALUE, "rejectregex", -1 }, { "relative", 'L', OPT_BOOLEAN, "relativeonly", -1 }, { "remote-encoding", 0, OPT_VALUE, "remoteencoding", -1 }, { "remove-listing", 0, OPT_BOOLEAN, "removelisting", -1 }, @@ -722,6 +725,17 @@ Recursive accept/reject:\n"), -A, --accept=LIST comma-separated list of accepted extensions.\n"), N_("\ -R, --reject=LIST comma-separated list of rejected extensions.\n"), + N_("\ + --accept-regex=REGEX regex matching accepted URLs.\n"), + N_("\ + --reject-regex=REGEX regex matching rejected URLs.\n"), +#ifdef HAVE_LIBPCRE + N_("\ + --regex-type=TYPE regex type (posix|pcre).\n"), +#else + N_("\ + --regex-type=TYPE regex type (posix).\n"), +#endif N_("\ -D, --domains=LIST comma-separated list of accepted domains.\n"), N_("\ @@ -1323,6 +1337,35 @@ for details.\n\n")); exit (1); } + /* Compile the regular expressions. */ + switch (opt.regex_type) + { +#ifdef HAVE_LIBPCRE + case regex_type_pcre: + opt.regex_compile_fun = compile_pcre_regex; + opt.regex_match_fun = match_pcre_regex; + break; +#endif + + case regex_type_posix: + default: + opt.regex_compile_fun = compile_posix_regex; + opt.regex_match_fun = match_posix_regex; + break; + } + if (opt.acceptregex_s) + { + opt.acceptregex = opt.regex_compile_fun (opt.acceptregex_s); + if (!opt.acceptregex) + exit (1); + } + if (opt.rejectregex_s) + { + opt.rejectregex = opt.regex_compile_fun (opt.rejectregex_s); + if (!opt.rejectregex) + exit (1); + } + #ifdef ENABLE_IRI if (opt.enable_iri) { diff --git a/src/options.h b/src/options.h index 1f429906..0da79379 100644 --- a/src/options.h +++ b/src/options.h @@ -74,6 +74,19 @@ struct options bool ignore_case; /* Whether to ignore case when matching dirs and files */ + char *acceptregex_s; /* Patterns to accept (a regex string). */ + char *rejectregex_s; /* Patterns to reject (a regex string). */ + void *acceptregex; /* Patterns to accept (a regex struct). */ + void *rejectregex; /* Patterns to reject (a regex struct). */ + enum { +#ifdef HAVE_LIBPCRE + regex_type_pcre, +#endif + regex_type_posix + } regex_type; /* The regex library. */ + void *(*regex_compile_fun)(const char *); /* Function to compile a regex. */ + bool (*regex_match_fun)(const void *, const char *); /* Function to match a string to a regex. */ + char **domains; /* See host.c */ char **exclude_domains; bool dns_cache; /* whether we cache DNS lookups. */ diff --git a/src/recur.c b/src/recur.c index 139fe2e3..72274fb5 100644 --- a/src/recur.c +++ b/src/recur.c @@ -586,6 +586,11 @@ download_child_p (const struct urlpos *upos, struct url *parent, int depth, goto out; } } + if (!accept_url (url)) + { + DEBUGP (("%s is excluded/not-included through regex.\n", url)); + goto out; + } /* 6. Check for acceptance/rejection rules. We ignore these rules for directories (no file name to match) and for non-leaf HTMLs, diff --git a/src/utils.c b/src/utils.c index 4188ced7..55a8a8d2 100644 --- a/src/utils.c +++ b/src/utils.c @@ -73,6 +73,11 @@ as that of the covered work. */ #include #include +#include +#ifdef HAVE_LIBPCRE +# include +#endif + #ifndef HAVE_SIGSETJMP /* If sigsetjmp is a macro, configure won't pick it up. */ # ifdef sigsetjmp @@ -917,6 +922,19 @@ acceptable (const char *s) return true; } +/* Determine whether an URL is acceptable to be followed, according to + regex patterns to accept/reject. */ +bool +accept_url (const char *s) +{ + if (opt.acceptregex && !opt.regex_match_fun (opt.acceptregex, s)) + return false; + if (opt.rejectregex && opt.regex_match_fun (opt.rejectregex, s)) + return false; + + return true; +} + /* Check if D2 is a subdirectory of D1. E.g. if D1 is `/something', subdir_p() will return true if and only if D2 begins with `/something/' or is exactly '/something'. */ @@ -2309,6 +2327,89 @@ base64_decode (const char *base64, void *dest) return q - (char *) dest; } +#ifdef HAVE_LIBPCRE +/* Compiles the PCRE regex. */ +void * +compile_pcre_regex (const char *str) +{ + const char *errbuf; + int erroffset; + pcre *regex = pcre_compile (str, 0, &errbuf, &erroffset, 0); + if (! regex) + { + fprintf (stderr, _("Invalid regular expression %s, %s\n"), + quote (str), errbuf); + return false; + } + return regex; +} +#endif + +/* Compiles the POSIX regex. */ +void * +compile_posix_regex (const char *str) +{ + regex_t *regex = xmalloc (sizeof (regex_t)); + int errcode = regcomp ((regex_t *) regex, str, REG_EXTENDED | REG_NOSUB); + if (errcode != 0) + { + int errbuf_size = regerror (errcode, (regex_t *) regex, NULL, 0); + char *errbuf = xmalloc (errbuf_size); + errbuf_size = regerror (errcode, (regex_t *) regex, errbuf, errbuf_size); + fprintf (stderr, _("Invalid regular expression %s, %s\n"), + quote (str), errbuf); + xfree (errbuf); + return NULL; + } + + return regex; +} + +#ifdef HAVE_LIBPCRE +#define OVECCOUNT 30 +/* Matches a PCRE regex. */ +bool +match_pcre_regex (const void *regex, const char *str) +{ + int l = strlen (str); + int ovector[OVECCOUNT]; + + int rc = pcre_exec ((pcre *) regex, 0, str, l, 0, 0, ovector, OVECCOUNT); + if (rc == PCRE_ERROR_NOMATCH) + return false; + else if (rc < 0) + { + logprintf (LOG_VERBOSE, _("Error while matching %s: %d\n"), + quote (str), rc); + return false; + } + else + return true; +} +#undef OVECCOUNT +#endif + +/* Matches a POSIX regex. */ +bool +match_posix_regex (const void *regex, const char *str) +{ + int rc = regexec ((regex_t *) regex, str, 0, NULL, 0); + if (rc == REG_NOMATCH) + return false; + else if (rc == 0) + return true; + else + { + int errbuf_size = regerror (rc, opt.acceptregex, NULL, 0); + char *errbuf = xmalloc (errbuf_size); + errbuf_size = regerror (rc, opt.acceptregex, errbuf, errbuf_size); + logprintf (LOG_VERBOSE, _("Error while matching %s: %d\n"), + quote (str), rc); + xfree (errbuf); + return false; + } +} + #undef IS_ASCII #undef NEXT_CHAR diff --git a/src/utils.h b/src/utils.h index 514c5f26..409cdc5a 100644 --- a/src/utils.h +++ b/src/utils.h @@ -90,6 +90,7 @@ char *file_merge (const char *, const char *); int fnmatch_nocase (const char *, const char *, int); bool acceptable (const char *); +bool accept_url (const char *); bool accdir (const char *s); char *suffix (const char *s); bool match_tail (const char *, const char *, bool); @@ -142,6 +143,14 @@ void xsleep (double); int base64_encode (const void *, int, char *); int base64_decode (const char *, void *); +#ifdef HAVE_LIBPCRE +void *compile_pcre_regex (const char *); +bool match_pcre_regex (const void *, const char *); +#endif + +void *compile_posix_regex (const char *); +bool match_posix_regex (const void *, const char *); + void stable_sort (void *, size_t, size_t, int (*) (const void *, const void *)); const char *print_decimal (double); From e41d044e1655e782a8d52dc6218c2f031321bfd0 Mon Sep 17 00:00:00 2001 From: Giuseppe Scrivano Date: Wed, 9 May 2012 21:19:58 +0200 Subject: [PATCH 41/75] NEWS: cite the new feature. --- NEWS | 2 ++ 1 file changed, 2 insertions(+) diff --git a/NEWS b/NEWS index 84040909..39c1c325 100644 --- a/NEWS +++ b/NEWS @@ -26,6 +26,8 @@ Please send GNU Wget bug reports to . ** Enable client certificates when GNU TLS is used. ** Add support for TLS Server Name Indication. + +** Accept the arguments --accept-reject and --reject-regex. * Changes in Wget 1.13.4 From 0b4c04b5836badc480b4bfb90a5bbf0ec897afc5 Mon Sep 17 00:00:00 2001 From: Tim Ruehsen Date: Fri, 11 May 2012 15:45:44 +0200 Subject: [PATCH 42/75] gnutls: remove deprecated gnutls types. --- src/ChangeLog | 7 +++++++ src/gnutls.c | 12 ++++++------ 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/src/ChangeLog b/src/ChangeLog index 32d58192..87e4b75e 100644 --- a/src/ChangeLog +++ b/src/ChangeLog @@ -1,3 +1,10 @@ +2012-05-13 Tim Ruehsen + + * gnutls.c (credentials): Change type to + gnutls_certificate_credentials_t. + (ssl_init): Do not use deprecated types. + (ssl_connect_wget): Likewise. + 2012-04-11 Gijs van Tulder * init.c: Add --accept-regex, --reject-regex and --regex-type. diff --git a/src/gnutls.c b/src/gnutls.c index 7cc2e718..2b13875f 100644 --- a/src/gnutls.c +++ b/src/gnutls.c @@ -75,7 +75,7 @@ key_type_to_gnutls_type (enum keyfile_type type) confused with actual gnutls functions -- such as the gnutls_read preprocessor macro. */ -static gnutls_certificate_credentials credentials; +static gnutls_certificate_credentials_t credentials; bool ssl_init (void) { @@ -165,7 +165,7 @@ cert to be of the same type.\n")); struct wgnutls_transport_context { - gnutls_session session; /* GnuTLS session handle */ + gnutls_session_t session; /* GnuTLS session handle */ int last_error; /* last error returned by read/write/... */ /* Since GnuTLS doesn't support the equivalent to recv(..., @@ -374,7 +374,7 @@ bool ssl_connect_wget (int fd, const char *hostname) { struct wgnutls_transport_context *ctx; - gnutls_session session; + gnutls_session_t session; int err; gnutls_init (&session, GNUTLS_CLIENT); @@ -390,7 +390,7 @@ ssl_connect_wget (int fd, const char *hostname) #ifndef FD_TO_SOCKET # define FD_TO_SOCKET(X) (X) #endif - gnutls_transport_set_ptr (session, (gnutls_transport_ptr) FD_TO_SOCKET (fd)); + gnutls_transport_set_ptr (session, (gnutls_transport_ptr_t) FD_TO_SOCKET (fd)); err = 0; #if HAVE_GNUTLS_PRIORITY_SET_DIRECT @@ -497,8 +497,8 @@ ssl_check_certificate (int fd, const char *host) if (gnutls_certificate_type_get (ctx->session) == GNUTLS_CRT_X509) { time_t now = time (NULL); - gnutls_x509_crt cert; - const gnutls_datum *cert_list; + gnutls_x509_crt_t cert; + const gnutls_datum_t *cert_list; unsigned int cert_list_size; if ((err = gnutls_x509_crt_init (&cert)) < 0) From f4122c50949f5bd5a9867a6074d39b56d29c660f Mon Sep 17 00:00:00 2001 From: Giuseppe Scrivano Date: Sun, 13 May 2012 16:46:15 +0200 Subject: [PATCH 43/75] Use git-version-gen instead of bzr-version-gen. --- ChangeLog | 9 +++++++ Makefile.am | 2 +- bootstrap.conf | 1 + build-aux/bzr-version-gen | 57 --------------------------------------- configure.ac | 2 +- 5 files changed, 12 insertions(+), 59 deletions(-) delete mode 100755 build-aux/bzr-version-gen diff --git a/ChangeLog b/ChangeLog index 51632aa2..aa249b06 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,12 @@ +2012-05-13 Giuseppe Scrivano + + * bootstrap.conf (gnulib_modules): Add `git-version-gen'. + * build-aux/bzr-version-gen: Remove file. + * configure.ac: Invoke `build-aux/git-version-gen' to get the dist + version. + * Makefile.am (EXTRA_DIST): Distribute build-aux/git-version-gen instead + of build-aux/bzr-version-gen. + 2012-04-11 Gijs van Tulder * bootstrap.conf (gnulib_modules): Include module `regex'. diff --git a/Makefile.am b/Makefile.am index 24d95e07..7a500ba0 100644 --- a/Makefile.am +++ b/Makefile.am @@ -46,7 +46,7 @@ SUBDIRS = lib src doc po tests util EXTRA_DIST = ChangeLog.README MAILING-LIST \ msdos/ChangeLog msdos/config.h msdos/Makefile.DJ \ msdos/Makefile.WC ABOUT-NLS \ - build-aux/build_info.pl build-aux/bzr-version-gen .version + build-aux/build_info.pl build-aux/git-version-gen .version CLEANFILES = *~ *.bak $(DISTNAME).tar.gz diff --git a/bootstrap.conf b/bootstrap.conf index 56b7b278..a784a906 100644 --- a/bootstrap.conf +++ b/bootstrap.conf @@ -43,6 +43,7 @@ getopt-gnu getpass-gnu getpeername getsockname +git-version-gen gnupload ioctl iconv diff --git a/build-aux/bzr-version-gen b/build-aux/bzr-version-gen deleted file mode 100755 index f22d14fd..00000000 --- a/build-aux/bzr-version-gen +++ /dev/null @@ -1,57 +0,0 @@ -#!/bin/sh - -scriptversion=2011-08-11.08; # UTC - -# Copyright (C) 2010, 2011 Free Software Foundation, Inc. - -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 3 of the License, or -# (at your option) any later version. - -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. - -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -# Additional permission under GNU GPL version 3 section 7 - - -# Written by Giuseppe Scrivano. - -if test -f .tarball-version -then - cat .tarball-version | tr -d '\n' - exit 0 -fi - -DIRTY="" - -test -n "`bzr diff | tr -d '\n'`" && DIRTY="-dirty" - -REVNO=`bzr revno` - -TAG=`bzr tags -r $REVNO | cut -d' ' -f1` -if test -z "$TAG" -then - TAG=`bzr tags --sort=time -r ..$REVNO | tail -n1 | cut -d' ' -f1` - - # No tags yet - test -z "$TAG" && TAG="unknown" - - TAG=$TAG-$REVNO -fi - - -printf "%s%s" "$TAG" "$DIRTY" - -# Local variables: -# eval: (add-hook 'write-file-hooks 'time-stamp) -# time-stamp-start: "scriptversion=" -# time-stamp-format: "%:y-%02m-%02d.%02H" -# time-stamp-time-zone: "UTC" -# time-stamp-end: "; # UTC" -# End: diff --git a/configure.ac b/configure.ac index 45cebcab..873c3c92 100644 --- a/configure.ac +++ b/configure.ac @@ -31,7 +31,7 @@ dnl Process this file with autoconf to produce a configure script. dnl AC_INIT([wget], - [m4_esyscmd([build-aux/bzr-version-gen])], + m4_esyscmd([build-aux/git-version-gen .tarball-version]), [bug-wget@gnu.org]) AC_PREREQ(2.61) From e24e81725943d7bc54e1b0b902806c816b983785 Mon Sep 17 00:00:00 2001 From: Giuseppe Scrivano Date: Sun, 13 May 2012 17:38:00 +0200 Subject: [PATCH 44/75] doc: Document --accept-regex and --reject-regex. --- doc/ChangeLog | 5 +++++ doc/wget.texi | 10 ++++++++++ 2 files changed, 15 insertions(+) diff --git a/doc/ChangeLog b/doc/ChangeLog index 36c07bda..a163bf34 100644 --- a/doc/ChangeLog +++ b/doc/ChangeLog @@ -1,3 +1,8 @@ +2012-05-13 Giuseppe Scrivano + + * wget.texi (Types of Files): Document --accept-regex and + --reject-regex. + 2011-10-02 Henrik Holst (tiny change) * wget.texi (HTTP Options): Document option --content-on-error. diff --git a/doc/wget.texi b/doc/wget.texi index 7a77a7b6..cd379e97 100644 --- a/doc/wget.texi +++ b/doc/wget.texi @@ -2284,6 +2284,8 @@ in @file{.wgetrc}. @item -A @var{acclist} @itemx --accept @var{acclist} @itemx accept = @var{acclist} +@itemx --accept-regex @var{urlregex} +@itemx accept-regex = @var{urlregex} The argument to @samp{--accept} option is a list of file suffixes or patterns that Wget will download during recursive retrieval. A suffix is the ending part of a file, and consists of ``normal'' letters, @@ -2300,6 +2302,9 @@ a description of how pattern matching works. Of course, any number of suffixes and patterns can be combined into a comma-separated list, and given as an argument to @samp{-A}. +The argument to @samp{--accept-regex} option is a regular expression which +is matched against the complete URL. + @cindex reject wildcards @cindex reject suffixes @cindex wildcards, reject @@ -2307,6 +2312,8 @@ comma-separated list, and given as an argument to @samp{-A}. @item -R @var{rejlist} @itemx --reject @var{rejlist} @itemx reject = @var{rejlist} +@itemx --reject-regex @var{urlregex} +@itemx reject-regex = @var{urlregex} The @samp{--reject} option works the same way as @samp{--accept}, only its logic is the reverse; Wget will download all files @emph{except} the ones matching the suffixes (or patterns) in the list. @@ -2318,6 +2325,9 @@ Analogously, to download all files except the ones beginning with expansion by the shell. @end table +The argument to @samp{--accept-regex} option is a regular expression which +is matched against the complete URL. + @noindent The @samp{-A} and @samp{-R} options may be combined to achieve even better fine-tuning of which files to retrieve. E.g. @samp{wget -A From d19cc259cb9dadfa55f1065a4cdfe338eec28b9d Mon Sep 17 00:00:00 2001 From: Tim Ruehsen Date: Mon, 14 May 2012 14:52:44 +0200 Subject: [PATCH 45/75] gnutls: do not call fcntl in a loop. * gnutls.c (wgnutls_read_timeout): removed warnings, moved fcntl stuff outside loop. --- src/ChangeLog | 5 ++++ src/gnutls.c | 63 +++++++++++++++++++++++---------------------------- 2 files changed, 33 insertions(+), 35 deletions(-) diff --git a/src/ChangeLog b/src/ChangeLog index 87e4b75e..8c6f4c80 100644 --- a/src/ChangeLog +++ b/src/ChangeLog @@ -1,3 +1,8 @@ +2012-05-14 Tim Ruehsen + + * gnutls.c: wgnutls_read_timeout (wgnutls_read_timeout): removed + warnings, moved fcntl stuff outside loop. + 2012-05-13 Tim Ruehsen * gnutls.c (credentials): Change type to diff --git a/src/gnutls.c b/src/gnutls.c index 2b13875f..9847ab47 100644 --- a/src/gnutls.c +++ b/src/gnutls.c @@ -188,7 +188,7 @@ wgnutls_read_timeout (int fd, char *buf, int bufsize, void *arg, double timeout) int flags = 0; #endif int ret = 0; - struct ptimer *timer; + struct ptimer *timer = NULL; struct wgnutls_transport_context *ctx = arg; int timed_out = 0; @@ -198,19 +198,27 @@ wgnutls_read_timeout (int fd, char *buf, int bufsize, void *arg, double timeout) flags = fcntl (fd, F_GETFL, 0); if (flags < 0) return flags; + if (fcntl (fd, F_SETFL, flags | O_NONBLOCK)) + return -1; +#else + /* XXX: Assume it was blocking before. */ + const int one = 1; + if (ioctl (fd, FIONBIO, &one) < 0) + return -1; #endif + timer = ptimer_new (); - if (timer == 0) + if (timer == NULL) return -1; } do { - double next_timeout; - if (timeout > 0.0) + double next_timeout = 0; + if (timeout) { next_timeout = timeout - ptimer_measure (timer); - if (next_timeout < 0.0) + if (next_timeout < 0) break; } @@ -218,43 +226,28 @@ wgnutls_read_timeout (int fd, char *buf, int bufsize, void *arg, double timeout) if (timeout == 0 || gnutls_record_check_pending (ctx->session) || select_fd (fd, next_timeout, WAIT_FOR_READ)) { - if (timeout) - { -#ifdef F_GETFL - if (fcntl (fd, F_SETFL, flags | O_NONBLOCK)) - break; -#else - /* XXX: Assume it was blocking before. */ - const int one = 1; - if (ioctl (fd, FIONBIO, &one) < 0) - break; -#endif - } - ret = gnutls_record_recv (ctx->session, buf, bufsize); - - if (timeout) - { -#ifdef F_GETFL - if (fcntl (fd, F_SETFL, flags) < 0) - break; -#else - const int zero = 0; - if (ioctl (fd, FIONBIO, &zero) < 0) - break; -#endif - } + timed_out = timeout && ptimer_measure (timer) >= timeout; } - - timed_out = timeout && ptimer_measure (timer) >= timeout; } while (ret == GNUTLS_E_INTERRUPTED || (ret == GNUTLS_E_AGAIN && !timed_out)); if (timeout) - ptimer_destroy (timer); + { + ptimer_destroy (timer); - if (timeout && timed_out && ret == GNUTLS_E_AGAIN) - errno = ETIMEDOUT; +#ifdef F_GETFL + if (fcntl (fd, F_SETFL, flags) < 0) + return -1; +#else + const int zero = 0; + if (ioctl (fd, FIONBIO, &zero) < 0) + return -1; +#endif + + if (timed_out && ret == GNUTLS_E_AGAIN) + errno = ETIMEDOUT; + } return ret; } From 2e86829809c4a89eec3f13f8ad86a26c6c9c55de Mon Sep 17 00:00:00 2001 From: Tim Ruehsen Date: Mon, 14 May 2012 17:32:55 +0200 Subject: [PATCH 46/75] removed 'const' warnings. * hash.h (hash_table_put): Make argument "value" const. * hash.c (hash_table_put): Make argument value const. Cast `value' to void. * http.c (request_set_header): Make argument `name' const. Cast `value' and `name' to void*. (request_remove_header): Make argument `name' const. * url.c (url_file_name): Make `index_filename' static. * warc.h (warc_write_cdx_record): Make `url', `timestamp', `mime_type', `payload_digest', `redirect_location', `warc_filename', response_uuid' arguments const. Make `checksum' const. * warc.c (warc_write_date_header): Make the `timestamp' argument const. Make `extension' const. (warc_write_cdx_record): Make `url', `timestamp', `mime_type', `payload_digest', `redirect_location', `warc_filename', response_uuid' arguments const. Make `checksum' const. --- src/ChangeLog | 20 ++++++++++++++++++-- src/hash.c | 6 +++--- src/hash.h | 2 +- src/http.c | 14 +++++++------- src/url.c | 2 +- src/warc.c | 12 ++++++------ src/warc.h | 2 +- 7 files changed, 37 insertions(+), 21 deletions(-) diff --git a/src/ChangeLog b/src/ChangeLog index 8c6f4c80..68df65bb 100644 --- a/src/ChangeLog +++ b/src/ChangeLog @@ -1,7 +1,23 @@ 2012-05-14 Tim Ruehsen - * gnutls.c: wgnutls_read_timeout (wgnutls_read_timeout): removed - warnings, moved fcntl stuff outside loop. + * gnutls.c (wgnutls_read_timeout): removed warnings, moved fcntl stuff + outside loop. + + * hash.h (hash_table_put): Make argument "value" const. + * hash.c (hash_table_put): Make argument value const. Cast `value' to + void. + * http.c (request_set_header): Make argument `name' const. Cast `value' + and `name' to void*. + (request_remove_header): Make argument `name' const. + * url.c (url_file_name): Make `index_filename' static. + * warc.h (warc_write_cdx_record): Make `url', `timestamp', `mime_type', + `payload_digest', `redirect_location', `warc_filename', response_uuid' + arguments const. Make `checksum' const. + * warc.c (warc_write_date_header): Make the `timestamp' argument const. + Make `extension' const. + (warc_write_cdx_record): Make `url', `timestamp', `mime_type', + `payload_digest', `redirect_location', `warc_filename', response_uuid' + arguments const. Make `checksum' const. 2012-05-13 Tim Ruehsen diff --git a/src/hash.c b/src/hash.c index 6c40801f..129ead1a 100644 --- a/src/hash.c +++ b/src/hash.c @@ -423,14 +423,14 @@ grow_hash_table (struct hash_table *ht) table if necessary. */ void -hash_table_put (struct hash_table *ht, const void *key, void *value) +hash_table_put (struct hash_table *ht, const void *key, const void *value) { struct cell *c = find_cell (ht, key); if (CELL_OCCUPIED (c)) { /* update existing item */ c->key = (void *)key; /* const? */ - c->value = value; + c->value = (void *)value; return; } @@ -445,7 +445,7 @@ hash_table_put (struct hash_table *ht, const void *key, void *value) /* add new item */ ++ht->count; c->key = (void *)key; /* const? */ - c->value = value; + c->value = (void *)value; } /* Remove KEY->value mapping from HT. Return 0 if there was no such diff --git a/src/hash.h b/src/hash.h index 1dadf093..85767609 100644 --- a/src/hash.h +++ b/src/hash.h @@ -42,7 +42,7 @@ int hash_table_get_pair (const struct hash_table *, const void *, void *, void *); int hash_table_contains (const struct hash_table *, const void *); -void hash_table_put (struct hash_table *, const void *, void *); +void hash_table_put (struct hash_table *, const void *, const void *); int hash_table_remove (struct hash_table *, const void *); void hash_table_clear (struct hash_table *); diff --git a/src/http.c b/src/http.c index cf901929..8d4edba5 100644 --- a/src/http.c +++ b/src/http.c @@ -231,7 +231,7 @@ release_header (struct request_header *hdr) */ static void -request_set_header (struct request *req, char *name, char *value, +request_set_header (struct request *req, const char *name, const char *value, enum rp release_policy) { struct request_header *hdr; @@ -242,7 +242,7 @@ request_set_header (struct request *req, char *name, char *value, /* A NULL value is a no-op; if freeing the name is requested, free it now to avoid leaks. */ if (release_policy == rel_name || release_policy == rel_both) - xfree (name); + xfree ((void *)name); return; } @@ -253,8 +253,8 @@ request_set_header (struct request *req, char *name, char *value, { /* Replace existing header. */ release_header (hdr); - hdr->name = name; - hdr->value = value; + hdr->name = (void *)name; + hdr->value = (void *)value; hdr->release_policy = release_policy; return; } @@ -268,8 +268,8 @@ request_set_header (struct request *req, char *name, char *value, req->headers = xrealloc (req->headers, req->hcapacity * sizeof (*hdr)); } hdr = &req->headers[req->hcount++]; - hdr->name = name; - hdr->value = value; + hdr->name = (void *)name; + hdr->value = (void *)value; hdr->release_policy = release_policy; } @@ -296,7 +296,7 @@ request_set_user_header (struct request *req, const char *header) the header was actually removed, false otherwise. */ static bool -request_remove_header (struct request *req, char *name) +request_remove_header (struct request *req, const char *name) { int i; for (i = 0; i < req->hcount; i++) diff --git a/src/url.c b/src/url.c index ddde798c..e44dfcd2 100644 --- a/src/url.c +++ b/src/url.c @@ -1504,7 +1504,7 @@ url_file_name (const struct url *u, char *replaced_filename) const char *u_file; char *fname, *unique; - char *index_filename = "index.html"; /* The default index file is index.html */ + const char *index_filename = "index.html"; /* The default index file is index.html */ fnres.base = NULL; fnres.size = 0; diff --git a/src/warc.c b/src/warc.c index fa0830dc..a2cf102a 100644 --- a/src/warc.c +++ b/src/warc.c @@ -372,7 +372,7 @@ warc_write_end_record (void) the current WARC record. If timestamp is NULL, the current time will be used. */ static bool -warc_write_date_header (char *timestamp) +warc_write_date_header (const char *timestamp) { if (timestamp == NULL) { @@ -725,9 +725,9 @@ warc_start_new_file (bool meta) warc_current_filename = new_filename; #ifdef HAVE_LIBZ - char *extension = (opt.warc_compression_enabled ? "warc.gz" : "warc"); + const char *extension = (opt.warc_compression_enabled ? "warc.gz" : "warc"); #else - char *extension = "warc"; + const char *extension = "warc"; #endif /* If max size is enabled, we add a serial number to the file names. */ @@ -1166,7 +1166,7 @@ warc_write_request_record (char *url, char *timestamp_str, char *record_uuid, ip response_uuid is the uuid of the response. Returns true on success, false on error. */ static bool -warc_write_cdx_record (char *url, char *timestamp_str, char *mime_type, int response_code, char *payload_digest, char *redirect_location, off_t offset, char *warc_filename, char *response_uuid) +warc_write_cdx_record (const char *url, const char *timestamp_str, const char *mime_type, int response_code, const char *payload_digest, const char *redirect_location, off_t offset, const char *warc_filename, const char *response_uuid) { /* Transform the timestamp. */ char timestamp_str_cdx [15]; @@ -1179,7 +1179,7 @@ warc_write_cdx_record (char *url, char *timestamp_str, char *mime_type, int resp timestamp_str_cdx[14] = '\0'; /* Rewrite the checksum. */ - char *checksum; + const char *checksum; if (payload_digest != NULL) checksum = payload_digest + 5; /* Skip the "sha1:" */ else @@ -1349,7 +1349,7 @@ warc_write_response_record (char *url, char *timestamp_str, char *concurrent_to_ Calling this function will close body. Returns true on success, false on error. */ bool -warc_write_resource_record (char *resource_uuid, char *url, char *timestamp_str, char *concurrent_to_uuid, ip_address *ip, char *content_type, FILE *body, off_t payload_offset) +warc_write_resource_record (char *resource_uuid, const char *url, const char *timestamp_str, const char *concurrent_to_uuid, ip_address *ip, const char *content_type, FILE *body, off_t payload_offset) { if (resource_uuid == NULL) { diff --git a/src/warc.h b/src/warc.h index 41829d12..ecfff600 100644 --- a/src/warc.h +++ b/src/warc.h @@ -13,7 +13,7 @@ FILE * warc_tempfile (void); bool warc_write_request_record (char *url, char *timestamp_str, char *concurrent_to_uuid, ip_address *ip, FILE *body, off_t payload_offset); bool warc_write_response_record (char *url, char *timestamp_str, char *concurrent_to_uuid, ip_address *ip, FILE *body, off_t payload_offset, char *mime_type, int response_code, char *redirect_location); -bool warc_write_resource_record (char *resource_uuid, char *url, char *timestamp_str, char *concurrent_to_uuid, ip_address *ip, char *content_type, FILE *body, off_t payload_offset); +bool warc_write_resource_record (char *resource_uuid, const char *url, const char *timestamp_str, const char *concurrent_to_uuid, ip_address *ip, const char *content_type, FILE *body, off_t payload_offset); #endif /* WARC_H */ From 8ac9c05fc03c9f12ae201fda56622f42d78ff697 Mon Sep 17 00:00:00 2001 From: Giuseppe Scrivano Date: Wed, 16 May 2012 21:39:24 +0200 Subject: [PATCH 47/75] warc: Cut long lines to 80 columns. --- src/ChangeLog | 5 ++ src/warc.c | 168 +++++++++++++++++++++++++++++++++----------------- src/warc.h | 12 ++-- 3 files changed, 124 insertions(+), 61 deletions(-) diff --git a/src/ChangeLog b/src/ChangeLog index 68df65bb..fe0a7afc 100644 --- a/src/ChangeLog +++ b/src/ChangeLog @@ -1,3 +1,8 @@ +2012-05-16 Giuseppe Scrivano + + * warc.h: Cut length lines to 80 columns. + * warc.c: Likewise. + 2012-05-14 Tim Ruehsen * gnutls.c (wgnutls_read_timeout): removed warnings, moved fcntl stuff diff --git a/src/warc.c b/src/warc.c index a2cf102a..57fdcad0 100644 --- a/src/warc.c +++ b/src/warc.c @@ -209,7 +209,8 @@ warc_write_start_record (void) if (warc_current_gzfile == NULL) { - logprintf (LOG_NOTQUIET, _("Error opening GZIP stream to WARC file.\n")); + logprintf (LOG_NOTQUIET, +_("Error opening GZIP stream to WARC file.\n")); warc_write_ok = false; return false; } @@ -298,12 +299,12 @@ warc_write_end_record (void) /* The WARC standard suggests that we add 'skip length' data in the extra header field of the GZIP stream. - + In warc_write_start_record we reserved space for this extra header. This extra space starts at warc_current_gzfile_offset and fills EXTRA_GZIP_HEADER_SIZE bytes. The static GZIP header starts at warc_current_gzfile_offset + EXTRA_GZIP_HEADER_SIZE. - + We need to do three things: 1. Move the static GZIP header to warc_current_gzfile_offset; 2. Set the FEXTRA flag in the GZIP header; @@ -317,11 +318,13 @@ warc_write_end_record (void) off_t compressed_size = warc_current_gzfile_uncompressed_size; /* Go back to the static GZIP header. */ - fseeko (warc_current_file, warc_current_gzfile_offset + EXTRA_GZIP_HEADER_SIZE, SEEK_SET); + fseeko (warc_current_file, warc_current_gzfile_offset + + EXTRA_GZIP_HEADER_SIZE, SEEK_SET); /* Read the header. */ char static_header[GZIP_STATIC_HEADER_SIZE]; - size_t result = fread (static_header, 1, GZIP_STATIC_HEADER_SIZE, warc_current_file); + size_t result = fread (static_header, 1, GZIP_STATIC_HEADER_SIZE, + warc_current_file); if (result != GZIP_STATIC_HEADER_SIZE) { warc_write_ok = false; @@ -331,7 +334,8 @@ warc_write_end_record (void) /* Set the FEXTRA flag in the flags byte of the header. */ static_header[OFF_FLG] = static_header[OFF_FLG] | FLG_FEXTRA; - /* Write the header back to the file, but starting at warc_current_gzfile_offset. */ + /* Write the header back to the file, but starting at + warc_current_gzfile_offset. */ fseeko (warc_current_file, warc_current_gzfile_offset, SEEK_SET); fwrite (static_header, 1, GZIP_STATIC_HEADER_SIZE, warc_current_file); @@ -355,7 +359,8 @@ warc_write_end_record (void) extra_header[11] = (compressed_size >> 24) & 255; /* Write the extra header after the static header. */ - fseeko (warc_current_file, warc_current_gzfile_offset + GZIP_STATIC_HEADER_SIZE, SEEK_SET); + fseeko (warc_current_file, warc_current_gzfile_offset + + GZIP_STATIC_HEADER_SIZE, SEEK_SET); fwrite (extra_header, 1, EXTRA_GZIP_HEADER_SIZE, warc_current_file); /* Done, move back to the end of the file. */ @@ -402,13 +407,14 @@ warc_write_ip_header (ip_address *ip) Compute SHA1 message digests for bytes read from STREAM. The digest of the complete file will be written into the 16 bytes beginning at RES_BLOCK. - + If payload_offset >= 0, a second digest will be calculated of the portion of the file starting at payload_offset and continuing to the end of the file. The digest number will be written into the 16 bytes beginning ad RES_PAYLOAD. */ static int -warc_sha1_stream_with_payload (FILE *stream, void *res_block, void *res_payload, off_t payload_offset) +warc_sha1_stream_with_payload (FILE *stream, void *res_block, void *res_payload, + off_t payload_offset) { #define BLOCKSIZE 32768 @@ -486,7 +492,8 @@ warc_sha1_stream_with_payload (FILE *stream, void *res_block, void *res_payload, have to start with a full block, there may still be some bytes left from the previous buffer. Therefore, we need to continue with sha1_process_bytes. */ - sha1_process_bytes (buffer + start_of_payload, BLOCKSIZE - start_of_payload, &ctx_payload); + sha1_process_bytes (buffer + start_of_payload, + BLOCKSIZE - start_of_payload, &ctx_payload); } } @@ -505,7 +512,8 @@ warc_sha1_stream_with_payload (FILE *stream, void *res_block, void *res_payload, start_of_payload = 0; /* Process the payload part of the buffer. */ - sha1_process_bytes (buffer + start_of_payload, sum - start_of_payload, &ctx_payload); + sha1_process_bytes (buffer + start_of_payload, + sum - start_of_payload, &ctx_payload); } } @@ -526,7 +534,8 @@ warc_base32_sha1_digest (char *sha1_digest) { // length: "sha1:" + digest + "\0" char *sha1_base32 = malloc (BASE32_LENGTH(SHA1_DIGEST_SIZE) + 1 + 5 ); - base32_encode (sha1_digest, SHA1_DIGEST_SIZE, sha1_base32 + 5, BASE32_LENGTH(SHA1_DIGEST_SIZE) + 1); + base32_encode (sha1_digest, SHA1_DIGEST_SIZE, sha1_base32 + 5, + BASE32_LENGTH(SHA1_DIGEST_SIZE) + 1); memcpy (sha1_base32, "sha1:", 5); sha1_base32[BASE32_LENGTH(SHA1_DIGEST_SIZE) + 5] = '\0'; return sha1_base32; @@ -547,7 +556,8 @@ warc_write_digest_headers (FILE *file, long payload_offset) char sha1_res_payload[SHA1_DIGEST_SIZE]; rewind (file); - if (warc_sha1_stream_with_payload (file, sha1_res_block, sha1_res_payload, payload_offset) == 0) + if (warc_sha1_stream_with_payload (file, sha1_res_block, + sha1_res_payload, payload_offset) == 0) { char *digest; @@ -637,7 +647,8 @@ static bool warc_write_warcinfo_record (char *filename) { /* Write warc-info record as the first record of the file. */ - /* We add the record id of this info record to the other records in the file. */ + /* We add the record id of this info record to the other records in the + file. */ warc_current_warcinfo_uuid_str = (char *) malloc (48); warc_uuid_str (warc_current_warcinfo_uuid_str); @@ -666,7 +677,8 @@ warc_write_warcinfo_record (char *filename) fprintf (warc_tmp, "software: Wget/%s (%s)\r\n", version_string, OS_TYPE); fprintf (warc_tmp, "format: WARC File Format 1.0\r\n"); - fprintf (warc_tmp, "conformsTo: http://bibnum.bnf.fr/WARC/WARC_ISO_28500_version1_latestdraft.pdf\r\n"); + fprintf (warc_tmp, +"conformsTo: http://bibnum.bnf.fr/WARC/WARC_ISO_28500_version1_latestdraft.pdf\r\n"); fprintf (warc_tmp, "robots: %s\r\n", (opt.use_robots ? "classic" : "off")); fprintf (warc_tmp, "wget-arguments: %s\r\n", program_argstring); /* Add the user headers, if any. */ @@ -683,9 +695,7 @@ warc_write_warcinfo_record (char *filename) warc_write_end_record (); if (! warc_write_ok) - { - logprintf (LOG_NOTQUIET, _("Error writing warcinfo record to WARC file.\n")); - } + logprintf (LOG_NOTQUIET, _("Error writing warcinfo record to WARC file.\n")); free (filename_copy); free (filename_basename); @@ -695,7 +705,7 @@ warc_write_warcinfo_record (char *filename) /* Opens a new WARC file. If META is true, generates a filename ending with 'meta.warc.gz'. - + This method will: 1. close the current WARC file (if there is one); 2. increment warc_current_file_number; @@ -734,7 +744,10 @@ warc_start_new_file (bool meta) if (meta) sprintf (new_filename, "%s-meta.%s", opt.warc_filename, extension); else if (opt.warc_maxsize > 0) - sprintf (new_filename, "%s-%05d.%s", opt.warc_filename, warc_current_file_number, extension); + { + sprintf (new_filename, "%s-%05d.%s", opt.warc_filename, + warc_current_file_number, extension); + } else sprintf (new_filename, "%s.%s", opt.warc_filename, extension); @@ -744,7 +757,8 @@ warc_start_new_file (bool meta) warc_current_file = fopen (new_filename, "wb+"); if (warc_current_file == NULL) { - logprintf (LOG_NOTQUIET, _("Error opening WARC file %s.\n"), quote (new_filename)); + logprintf (LOG_NOTQUIET, _("Error opening WARC file %s.\n"), + quote (new_filename)); return false; } @@ -794,7 +808,8 @@ warc_start_cdx_file (void) /* Parse the CDX header and find the field numbers of the original url, checksum and record ID fields. */ static bool -warc_parse_cdx_header (char *lineptr, int *field_num_original_url, int *field_num_checksum, int *field_num_record_id) +warc_parse_cdx_header (char *lineptr, int *field_num_original_url, + int *field_num_checksum, int *field_num_record_id) { *field_num_original_url = -1; *field_num_checksum = -1; @@ -803,7 +818,7 @@ warc_parse_cdx_header (char *lineptr, int *field_num_original_url, int *field_nu char *token; char *save_ptr; token = strtok_r (lineptr, CDX_FIELDSEP, &save_ptr); - + if (token != NULL && strcmp (token, "CDX") == 0) { int field_num = 0; @@ -836,7 +851,8 @@ warc_parse_cdx_header (char *lineptr, int *field_num_original_url, int *field_nu /* Parse the CDX record and add it to the warc_cdx_dedup_table hash table. */ static void -warc_process_cdx_line (char *lineptr, int field_num_original_url, int field_num_checksum, int field_num_record_id) +warc_process_cdx_line (char *lineptr, int field_num_original_url, + int field_num_checksum, int field_num_record_id) { char *original_url = NULL; char *checksum = NULL; @@ -874,13 +890,15 @@ warc_process_cdx_line (char *lineptr, int field_num_original_url, int field_num_ bytes. */ size_t checksum_l; char * checksum_v; - base32_decode_alloc (checksum, strlen (checksum), &checksum_v, &checksum_l); + base32_decode_alloc (checksum, strlen (checksum), &checksum_v, + &checksum_l); free (checksum); if (checksum_v != NULL && checksum_l == SHA1_DIGEST_SIZE) { /* This is a valid line with a valid checksum. */ - struct warc_cdx_record * rec = malloc (sizeof (struct warc_cdx_record)); + struct warc_cdx_record *rec; + rec = malloc (sizeof (struct warc_cdx_record)); rec->url = original_url; rec->uuid = record_id; memcpy (rec->digest, checksum_v, SHA1_DIGEST_SIZE); @@ -921,7 +939,8 @@ warc_load_cdx_dedup_file (void) 'u' (the WARC record id). */ line_length = getline (&lineptr, &n, f); if (line_length != -1) - warc_parse_cdx_header (lineptr, &field_num_original_url, &field_num_checksum, &field_num_record_id); + warc_parse_cdx_header (lineptr, &field_num_original_url, + &field_num_checksum, &field_num_record_id); /* If the file contains all three fields, read the complete file. */ if (field_num_original_url == -1 @@ -929,22 +948,29 @@ warc_load_cdx_dedup_file (void) || field_num_record_id == -1) { if (field_num_original_url == -1) - logprintf (LOG_NOTQUIET, _("CDX file does not list original urls. (Missing column 'a'.)\n")); + logprintf (LOG_NOTQUIET, +_("CDX file does not list original urls. (Missing column 'a'.)\n")); if (field_num_checksum == -1) - logprintf (LOG_NOTQUIET, _("CDX file does not list checksums. (Missing column 'k'.)\n")); + logprintf (LOG_NOTQUIET, +_("CDX file does not list checksums. (Missing column 'k'.)\n")); if (field_num_record_id == -1) - logprintf (LOG_NOTQUIET, _("CDX file does not list record ids. (Missing column 'u'.)\n")); + logprintf (LOG_NOTQUIET, +_("CDX file does not list record ids. (Missing column 'u'.)\n")); } else { /* Initialize the table. */ - warc_cdx_dedup_table = hash_table_new (1000, warc_hash_sha1_digest, warc_cmp_sha1_digest); + warc_cdx_dedup_table = hash_table_new (1000, warc_hash_sha1_digest, + warc_cmp_sha1_digest); do { line_length = getline (&lineptr, &n, f); if (line_length != -1) - warc_process_cdx_line (lineptr, field_num_original_url, field_num_checksum, field_num_record_id); + { + warc_process_cdx_line (lineptr, field_num_original_url, + field_num_checksum, field_num_record_id); + } } while (line_length != -1); @@ -952,7 +978,8 @@ warc_load_cdx_dedup_file (void) /* Print results. */ int nrecords = hash_table_count (warc_cdx_dedup_table); logprintf (LOG_VERBOSE, ngettext ("Loaded %d record from CDX.\n\n", - "Loaded %d records from CDX.\n\n", nrecords), + "Loaded %d records from CDX.\n\n", + nrecords), nrecords); } @@ -974,7 +1001,8 @@ warc_find_duplicate_cdx_record (char *url, char *sha1_digest_payload) char *key; struct warc_cdx_record *rec_existing; - hash_table_get_pair (warc_cdx_dedup_table, sha1_digest_payload, &key, &rec_existing); + hash_table_get_pair (warc_cdx_dedup_table, sha1_digest_payload, &key, + &rec_existing); if (rec_existing != NULL && strcmp (rec_existing->url, url) == 0) return rec_existing; @@ -1005,7 +1033,8 @@ warc_init (void) warc_manifest_fp = warc_tempfile (); if (warc_manifest_fp == NULL) { - logprintf (LOG_NOTQUIET, _("Could not open temporary WARC manifest file.\n")); + logprintf (LOG_NOTQUIET, + _("Could not open temporary WARC manifest file.\n")); exit(1); } @@ -1014,7 +1043,8 @@ warc_init (void) warc_log_fp = warc_tempfile (); if (warc_log_fp == NULL) { - logprintf (LOG_NOTQUIET, _("Could not open temporary WARC log file.\n")); + logprintf (LOG_NOTQUIET, + _("Could not open temporary WARC log file.\n")); exit(1); } log_set_warc_log_fp (warc_log_fp); @@ -1031,7 +1061,8 @@ warc_init (void) { if (! warc_start_cdx_file ()) { - logprintf (LOG_NOTQUIET, _("Could not open CDX file for output.\n")); + logprintf (LOG_NOTQUIET, + _("Could not open CDX file for output.\n")); exit(1); } } @@ -1066,7 +1097,7 @@ warc_write_metadata (void) fprintf (warc_tmp_fp, "%s\n", program_argstring); warc_write_resource_record (manifest_uuid, - "metadata://gnu.org/software/wget/warc/wget_arguments.txt", + "metadata://gnu.org/software/wget/warc/wget_arguments.txt", NULL, NULL, NULL, "text/plain", warc_tmp_fp, -1); /* warc_write_resource_record has closed warc_tmp_fp. */ @@ -1074,7 +1105,7 @@ warc_write_metadata (void) if (warc_log_fp != NULL) { warc_write_resource_record (NULL, - "metadata://gnu.org/software/wget/warc/wget.log", + "metadata://gnu.org/software/wget/warc/wget.log", NULL, manifest_uuid, NULL, "text/plain", warc_log_fp, -1); /* warc_write_resource_record has closed warc_log_fp. */ @@ -1134,7 +1165,8 @@ warc_tempfile (void) Calling this function will close body. Returns true on success, false on error. */ bool -warc_write_request_record (char *url, char *timestamp_str, char *record_uuid, ip_address *ip, FILE *body, off_t payload_offset) +warc_write_request_record (char *url, char *timestamp_str, char *record_uuid, + ip_address *ip, FILE *body, off_t payload_offset) { warc_write_start_record (); warc_write_header ("WARC-Type", "request"); @@ -1147,7 +1179,7 @@ warc_write_request_record (char *url, char *timestamp_str, char *record_uuid, ip warc_write_digest_headers (body, payload_offset); warc_write_block_from_file (body); warc_write_end_record (); - + fclose (body); return warc_write_ok; @@ -1166,7 +1198,11 @@ warc_write_request_record (char *url, char *timestamp_str, char *record_uuid, ip response_uuid is the uuid of the response. Returns true on success, false on error. */ static bool -warc_write_cdx_record (const char *url, const char *timestamp_str, const char *mime_type, int response_code, const char *payload_digest, const char *redirect_location, off_t offset, const char *warc_filename, const char *response_uuid) +warc_write_cdx_record (const char *url, const char *timestamp_str, + const char *mime_type, int response_code, + const char *payload_digest, const char *redirect_location, + off_t offset, const char *warc_filename, + const char *response_uuid) { /* Transform the timestamp. */ char timestamp_str_cdx [15]; @@ -1177,7 +1213,7 @@ warc_write_cdx_record (const char *url, const char *timestamp_str, const char *m memcpy (timestamp_str_cdx + 10, timestamp_str + 14, 2); /* "MM" ":" */ memcpy (timestamp_str_cdx + 12, timestamp_str + 17, 2); /* "SS" "Z" */ timestamp_str_cdx[14] = '\0'; - + /* Rewrite the checksum. */ const char *checksum; if (payload_digest != NULL) @@ -1191,7 +1227,9 @@ warc_write_cdx_record (const char *url, const char *timestamp_str, const char *m redirect_location = "-"; /* Print the CDX line. */ - fprintf (warc_current_cdx_file, "%s %s %s %s %d %s %s - %ld %s %s\n", url, timestamp_str_cdx, url, mime_type, response_code, checksum, redirect_location, offset, warc_current_filename, response_uuid); + fprintf (warc_current_cdx_file, "%s %s %s %s %d %s %s - %ld %s %s\n", url, + timestamp_str_cdx, url, mime_type, response_code, checksum, + redirect_location, offset, warc_current_filename, response_uuid); fflush (warc_current_cdx_file); return true; @@ -1211,7 +1249,9 @@ warc_write_cdx_record (const char *url, const char *timestamp_str, const char *m Calling this function will close body. Returns true on success, false on error. */ static bool -warc_write_revisit_record (char *url, char *timestamp_str, char *concurrent_to_uuid, char *payload_digest, char *refers_to, ip_address *ip, FILE *body) +warc_write_revisit_record (char *url, char *timestamp_str, + char *concurrent_to_uuid, char *payload_digest, + char *refers_to, ip_address *ip, FILE *body) { char revisit_uuid [48]; warc_uuid_str (revisit_uuid); @@ -1237,7 +1277,7 @@ warc_write_revisit_record (char *url, char *timestamp_str, char *concurrent_to_u warc_write_header ("WARC-Payload-Digest", payload_digest); warc_write_block_from_file (body); warc_write_end_record (); - + fclose (body); free (block_digest); @@ -1258,7 +1298,10 @@ warc_write_revisit_record (char *url, char *timestamp_str, char *concurrent_to_u Calling this function will close body. Returns true on success, false on error. */ bool -warc_write_response_record (char *url, char *timestamp_str, char *concurrent_to_uuid, ip_address *ip, FILE *body, off_t payload_offset, char *mime_type, int response_code, char *redirect_location) +warc_write_response_record (char *url, char *timestamp_str, + char *concurrent_to_uuid, ip_address *ip, + FILE *body, off_t payload_offset, char *mime_type, + int response_code, char *redirect_location) { char *block_digest = NULL; char *payload_digest = NULL; @@ -1269,15 +1312,20 @@ warc_write_response_record (char *url, char *timestamp_str, char *concurrent_to_ { /* Calculate the block and payload digests. */ rewind (body); - if (warc_sha1_stream_with_payload (body, sha1_res_block, sha1_res_payload, payload_offset) == 0) + if (warc_sha1_stream_with_payload (body, sha1_res_block, sha1_res_payload, + payload_offset) == 0) { /* Decide (based on url + payload digest) if we have seen this data before. */ - struct warc_cdx_record *rec_existing = warc_find_duplicate_cdx_record (url, sha1_res_payload); + struct warc_cdx_record *rec_existing; + rec_existing = warc_find_duplicate_cdx_record (url, sha1_res_payload); if (rec_existing != NULL) { + bool result; + /* Found an existing record. */ - logprintf (LOG_VERBOSE, _("Found exact match in CDX file. Saving revisit record to WARC.\n")); + logprintf (LOG_VERBOSE, + _("Found exact match in CDX file. Saving revisit record to WARC.\n")); /* Remove the payload from the file. */ if (payload_offset > 0) @@ -1288,7 +1336,9 @@ warc_write_response_record (char *url, char *timestamp_str, char *concurrent_to_ /* Send the original payload digest. */ payload_digest = warc_base32_sha1_digest (sha1_res_payload); - bool result = warc_write_revisit_record (url, timestamp_str, concurrent_to_uuid, payload_digest, rec_existing->uuid, ip, body); + result = warc_write_revisit_record (url, timestamp_str, + concurrent_to_uuid, payload_digest, rec_existing->uuid, + ip, body); free (payload_digest); return result; @@ -1326,7 +1376,9 @@ warc_write_response_record (char *url, char *timestamp_str, char *concurrent_to_ if (warc_write_ok && opt.warc_cdx_enabled) { /* Add this record to the CDX. */ - warc_write_cdx_record (url, timestamp_str, mime_type, response_code, payload_digest, redirect_location, offset, warc_current_filename, response_uuid); + warc_write_cdx_record (url, timestamp_str, mime_type, response_code, + payload_digest, redirect_location, offset, warc_current_filename, + response_uuid); } if (block_digest) @@ -1341,15 +1393,18 @@ warc_write_response_record (char *url, char *timestamp_str, char *concurrent_to_ resource_uuid is the uuid of the resource (or NULL), url is the target uri of the resource, timestamp_str is the timestamp (generated with warc_timestamp), - concurrent_to_uuid is the uuid of the request for that generated this resource - (generated with warc_uuid_str) or NULL, + concurrent_to_uuid is the uuid of the request for that generated this + resource (generated with warc_uuid_str) or NULL, ip is the ip address of the server (or NULL), content_type is the mime type of the body (or NULL), body is a pointer to a file containing the resource data. Calling this function will close body. Returns true on success, false on error. */ bool -warc_write_resource_record (char *resource_uuid, const char *url, const char *timestamp_str, const char *concurrent_to_uuid, ip_address *ip, const char *content_type, FILE *body, off_t payload_offset) +warc_write_resource_record (char *resource_uuid, const char *url, + const char *timestamp_str, const char *concurrent_to_uuid, + ip_address *ip, const char *content_type, FILE *body, + off_t payload_offset) { if (resource_uuid == NULL) { @@ -1372,9 +1427,8 @@ warc_write_resource_record (char *resource_uuid, const char *url, const char *ti warc_write_header ("Content-Type", content_type); warc_write_block_from_file (body); warc_write_end_record (); - + fclose (body); return warc_write_ok; } - diff --git a/src/warc.h b/src/warc.h index ecfff600..eba640d5 100644 --- a/src/warc.h +++ b/src/warc.h @@ -11,9 +11,13 @@ void warc_uuid_str (char *id_str); FILE * warc_tempfile (void); -bool warc_write_request_record (char *url, char *timestamp_str, char *concurrent_to_uuid, ip_address *ip, FILE *body, off_t payload_offset); -bool warc_write_response_record (char *url, char *timestamp_str, char *concurrent_to_uuid, ip_address *ip, FILE *body, off_t payload_offset, char *mime_type, int response_code, char *redirect_location); -bool warc_write_resource_record (char *resource_uuid, const char *url, const char *timestamp_str, const char *concurrent_to_uuid, ip_address *ip, const char *content_type, FILE *body, off_t payload_offset); +bool warc_write_request_record (char *url, char *timestamp_str, + char *concurrent_to_uuid, ip_address *ip, FILE *body, off_t payload_offset); +bool warc_write_response_record (char *url, char *timestamp_str, + char *concurrent_to_uuid, ip_address *ip, FILE *body, off_t payload_offset, + char *mime_type, int response_code, char *redirect_location); +bool warc_write_resource_record (char *resource_uuid, const char *url, + const char *timestamp_str, const char *concurrent_to_uuid, ip_address *ip, + const char *content_type, FILE *body, off_t payload_offset); #endif /* WARC_H */ - From e93bb4fa28bfb58622db8ca0b084f52fd52eb956 Mon Sep 17 00:00:00 2001 From: illusionoflife Date: Sun, 20 May 2012 21:02:25 +0200 Subject: [PATCH 48/75] Remove unused arguments. --- src/ChangeLog | 7 +++++++ src/convert.c | 4 ++-- src/convert.h | 4 ++-- src/retr.c | 4 ++-- 4 files changed, 13 insertions(+), 6 deletions(-) diff --git a/src/ChangeLog b/src/ChangeLog index fe0a7afc..f38047cb 100644 --- a/src/ChangeLog +++ b/src/ChangeLog @@ -1,3 +1,10 @@ +2012-05-19 illusionoflife (tiny change) + + * convert.c (register_html,register_css): Fixed functions signature to + not accept unused argument + * retr.c (retrieve_url): Changed register_{css,html} usage according + new signature. + 2012-05-16 Giuseppe Scrivano * warc.h: Cut length lines to 80 columns. diff --git a/src/convert.c b/src/convert.c index 6cf6f272..e1c58e9a 100644 --- a/src/convert.c +++ b/src/convert.c @@ -870,7 +870,7 @@ register_delete_file (const char *file) /* Register that FILE is an HTML file that has been downloaded. */ void -register_html (const char *url, const char *file) +register_html (const char *file) { if (!downloaded_html_set) downloaded_html_set = make_string_hash_table (0); @@ -880,7 +880,7 @@ register_html (const char *url, const char *file) /* Register that FILE is a CSS file that has been downloaded. */ void -register_css (const char *url, const char *file) +register_css (const char *file) { if (!downloaded_css_set) downloaded_css_set = make_string_hash_table (0); diff --git a/src/convert.h b/src/convert.h index 1f034e57..cdd0a482 100644 --- a/src/convert.h +++ b/src/convert.h @@ -101,8 +101,8 @@ downloaded_file_t downloaded_file (downloaded_file_t, const char *); void register_download (const char *, const char *); void register_redirection (const char *, const char *); -void register_html (const char *, const char *); -void register_css (const char *, const char *); +void register_html (const char *); +void register_css (const char *); void register_delete_file (const char *); void convert_all_links (void); void convert_cleanup (void); diff --git a/src/retr.c b/src/retr.c index 5f33c7a7..8bc54425 100644 --- a/src/retr.c +++ b/src/retr.c @@ -932,10 +932,10 @@ retrieve_url (struct url * orig_parsed, const char *origurl, char **file, register_redirection (origurl, u->url); if (*dt & TEXTHTML) - register_html (u->url, local_file); + register_html (local_file); if (*dt & TEXTCSS) - register_css (u->url, local_file); + register_css (local_file); } if (file) From 370f96d36cac952a52f71cb2a2b03f4a64a6d93e Mon Sep 17 00:00:00 2001 From: Tim Ruehsen Date: Fri, 18 May 2012 13:23:56 +0200 Subject: [PATCH 49/75] gnutls: honor the specified timeout value * gnutls.c (wgnutls_poll): Honor the specified `timeout' value. (wgnutls_peek): Likewise. --- src/ChangeLog | 5 +++++ src/gnutls.c | 30 +++++++++++++++++++----------- 2 files changed, 24 insertions(+), 11 deletions(-) diff --git a/src/ChangeLog b/src/ChangeLog index f38047cb..533a39e6 100644 --- a/src/ChangeLog +++ b/src/ChangeLog @@ -1,3 +1,8 @@ +2012-05-18 Tim Ruehsen + + * gnutls.c (wgnutls_poll): Honor the specified `timeout' value. + (wgnutls_peek): Likewise. + 2012-05-19 illusionoflife (tiny change) * convert.c (register_html,register_css): Fixed functions signature to diff --git a/src/gnutls.c b/src/gnutls.c index 9847ab47..32c6d174 100644 --- a/src/gnutls.c +++ b/src/gnutls.c @@ -216,11 +216,11 @@ wgnutls_read_timeout (int fd, char *buf, int bufsize, void *arg, double timeout) { double next_timeout = 0; if (timeout) - { - next_timeout = timeout - ptimer_measure (timer); - if (next_timeout < 0) - break; - } + { + next_timeout = timeout - ptimer_measure (timer); + if (next_timeout < 0) + break; + } ret = GNUTLS_E_AGAIN; if (timeout == 0 || gnutls_record_check_pending (ctx->session) @@ -294,8 +294,12 @@ static int wgnutls_poll (int fd, double timeout, int wait_for, void *arg) { struct wgnutls_transport_context *ctx = arg; - return ctx->peeklen || gnutls_record_check_pending (ctx->session) - || select_fd (fd, timeout, wait_for); + + if (timeout) + return ctx->peeklen || gnutls_record_check_pending (ctx->session) + || select_fd (fd, timeout, wait_for); + else + return ctx->peeklen || gnutls_record_check_pending (ctx->session); } static int @@ -304,15 +308,19 @@ wgnutls_peek (int fd, char *buf, int bufsize, void *arg) int read = 0; struct wgnutls_transport_context *ctx = arg; int offset = MIN (bufsize, ctx->peeklen); + + if (ctx->peeklen) + { + memcpy (buf, ctx->peekbuf, offset); + return offset; + } + if (bufsize > sizeof ctx->peekbuf) bufsize = sizeof ctx->peekbuf; - if (ctx->peeklen) - memcpy (buf, ctx->peekbuf, offset); - if (bufsize > offset) { - if (gnutls_record_check_pending (ctx->session) <= 0 + if (opt.read_timeout && gnutls_record_check_pending (ctx->session) == 0 && select_fd (fd, 0.0, WAIT_FOR_READ) <= 0) read = 0; else From 620ca36038946f88004c7ea18c4d3ebb4bd0893b Mon Sep 17 00:00:00 2001 From: Giuseppe Scrivano Date: Mon, 21 May 2012 22:54:57 +0200 Subject: [PATCH 50/75] NEWS: cite the last change. --- NEWS | 2 ++ 1 file changed, 2 insertions(+) diff --git a/NEWS b/NEWS index 39c1c325..ba67de97 100644 --- a/NEWS +++ b/NEWS @@ -28,6 +28,8 @@ Please send GNU Wget bug reports to . ** Add support for TLS Server Name Indication. ** Accept the arguments --accept-reject and --reject-regex. + +** The GNU TLS backend honors correctly the timeout value. * Changes in Wget 1.13.4 From 9cc514d21ced5b18c9db1bec418c04a99d3d6c5c Mon Sep 17 00:00:00 2001 From: Giuseppe Scrivano Date: Sat, 26 May 2012 02:55:53 +0200 Subject: [PATCH 51/75] Use the right type as result from readline. --- src/ChangeLog | 6 ++++++ src/warc.c | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/src/ChangeLog b/src/ChangeLog index 533a39e6..8b96c628 100644 --- a/src/ChangeLog +++ b/src/ChangeLog @@ -1,3 +1,9 @@ +2012-05-26 Giuseppe Scrivano + + * warc.c (warc_load_cdx_dedup_file): Change type of `line_length' to + ssize_t. + Suggested by: Ángel González + 2012-05-18 Tim Ruehsen * gnutls.c (wgnutls_poll): Honor the specified `timeout' value. diff --git a/src/warc.c b/src/warc.c index 57fdcad0..6935aaf1 100644 --- a/src/warc.c +++ b/src/warc.c @@ -930,7 +930,7 @@ warc_load_cdx_dedup_file (void) char *lineptr = NULL; size_t n = 0; - size_t line_length; + ssize_t line_length; /* The first line should contain the CDX header. Format: " CDX x x x x x" From 639a4545287b79baa8783c90c38a572b406714c7 Mon Sep 17 00:00:00 2001 From: Mike Frysinger Date: Sat, 26 May 2012 14:05:56 +0200 Subject: [PATCH 52/75] warc: use the right type for the gzip stream --- src/ChangeLog | 4 ++++ src/warc.c | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/src/ChangeLog b/src/ChangeLog index 8b96c628..d7a13ac0 100644 --- a/src/ChangeLog +++ b/src/ChangeLog @@ -1,3 +1,7 @@ +2012-05-26 Mike Frysinger + + * warc.c: Change type of `warc_current_gzfile' to gzFile. + 2012-05-26 Giuseppe Scrivano * warc.c (warc_load_cdx_dedup_file): Change type of `line_length' to diff --git a/src/warc.c b/src/warc.c index 6935aaf1..24751dbf 100644 --- a/src/warc.c +++ b/src/warc.c @@ -75,7 +75,7 @@ static FILE *warc_current_file; #ifdef HAVE_LIBZ /* The gzip stream for the current WARC file (or NULL, if WARC or gzip is disabled). */ -static gzFile *warc_current_gzfile; +static gzFile warc_current_gzfile; /* The offset of the current gzip record in the WARC file. */ static off_t warc_current_gzfile_offset; From 2b1dd8d23b987a82d03cfa803aabcb02f80e1066 Mon Sep 17 00:00:00 2001 From: Steven Schweda Date: Sat, 26 May 2012 14:39:13 +0200 Subject: [PATCH 53/75] Guard inclusion of some headers. --- src/ChangeLog | 4 ++++ src/connect.c | 9 +++++++-- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/src/ChangeLog b/src/ChangeLog index d7a13ac0..7e16b17c 100644 --- a/src/ChangeLog +++ b/src/ChangeLog @@ -1,3 +1,7 @@ +2011-05-26 Steven Schweda + * connect.c [HAVE_SYS_SOCKET_H]: Include . + [HAVE_SYS_SELECT_H]: Include . + 2012-05-26 Mike Frysinger * warc.c: Change type of `warc_current_gzfile' to gzFile. diff --git a/src/connect.c b/src/connect.c index 119ccb71..6eca1ded 100644 --- a/src/connect.c +++ b/src/connect.c @@ -36,8 +36,13 @@ as that of the covered work. */ #include #include -#include -#include +#ifdef HAVE_SYS_SOCKET_H +# include +#endif /* def HAVE_SYS_SOCKET_H */ + +#ifdef HAVE_SYS_SELECT_H +# include +#endif /* def HAVE_SYS_SELECT_H */ #ifndef WINDOWS # ifdef __VMS From ee9d4a905769c7bdfad94947c48aaada329fc9de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=81ngel=20Gonz=C3=A1lez?= Date: Thu, 31 May 2012 22:57:41 +0200 Subject: [PATCH 54/75] fix segfault on wrong urls (bug 36570) --- ChangeLog | 4 ++++ src/convert.c | 3 +++ 2 files changed, 7 insertions(+) diff --git a/ChangeLog b/ChangeLog index aa249b06..2f0f9655 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,7 @@ +2012-05-31 Ángel González + + * convert.c: fix segfault on wrong urls (bug 36570) + 2012-05-13 Giuseppe Scrivano * bootstrap.conf (gnulib_modules): Add `git-version-gen'. diff --git a/src/convert.c b/src/convert.c index e1c58e9a..f5a9cba3 100644 --- a/src/convert.c +++ b/src/convert.c @@ -124,6 +124,9 @@ convert_links_in_hashtable (struct hash_table *downloaded_set, set_uri_encoding (pi, opt.locale, true); u = url_parse (cur_url->url->url, NULL, pi, true); + if (!u) + continue; + local_name = hash_table_get (dl_url_file_map, u->url); /* Decide on the conversion type. */ From 1d14c18d7f81c03fdf79358055ed909c6d65caa1 Mon Sep 17 00:00:00 2001 From: Gijs van Tulder Date: Wed, 30 May 2012 23:00:04 +0200 Subject: [PATCH 55/75] warc: Fix segfault if CDX record is not found. --- src/ChangeLog | 4 ++++ src/warc.c | 6 +++--- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/src/ChangeLog b/src/ChangeLog index 7e16b17c..9e74e474 100644 --- a/src/ChangeLog +++ b/src/ChangeLog @@ -1,3 +1,7 @@ +2012-05-30 Gijs van Tulder + + * warc.c: Fix segfault if CDX record is not found. + 2011-05-26 Steven Schweda * connect.c [HAVE_SYS_SOCKET_H]: Include . [HAVE_SYS_SELECT_H]: Include . diff --git a/src/warc.c b/src/warc.c index 24751dbf..92a49ef8 100644 --- a/src/warc.c +++ b/src/warc.c @@ -1001,10 +1001,10 @@ warc_find_duplicate_cdx_record (char *url, char *sha1_digest_payload) char *key; struct warc_cdx_record *rec_existing; - hash_table_get_pair (warc_cdx_dedup_table, sha1_digest_payload, &key, - &rec_existing); + int found = hash_table_get_pair (warc_cdx_dedup_table, sha1_digest_payload, + &key, &rec_existing); - if (rec_existing != NULL && strcmp (rec_existing->url, url) == 0) + if (found && strcmp (rec_existing->url, url) == 0) return rec_existing; else return NULL; From 6741bc4233d6060dd046e35e6e69a2fe2772e601 Mon Sep 17 00:00:00 2001 From: Giuseppe Scrivano Date: Sat, 2 Jun 2012 19:36:45 +0200 Subject: [PATCH 56/75] Revert 2b1dd8d23b987a82d03cfa803aabcb02f80e1066 --- src/ChangeLog | 4 ++++ src/connect.c | 9 ++------- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/src/ChangeLog b/src/ChangeLog index 9e74e474..5639fd12 100644 --- a/src/ChangeLog +++ b/src/ChangeLog @@ -1,3 +1,7 @@ +2012-06-02 Giuseppe Scrivano + + * connect.c: Include and . + 2012-05-30 Gijs van Tulder * warc.c: Fix segfault if CDX record is not found. diff --git a/src/connect.c b/src/connect.c index 6eca1ded..119ccb71 100644 --- a/src/connect.c +++ b/src/connect.c @@ -36,13 +36,8 @@ as that of the covered work. */ #include #include -#ifdef HAVE_SYS_SOCKET_H -# include -#endif /* def HAVE_SYS_SOCKET_H */ - -#ifdef HAVE_SYS_SELECT_H -# include -#endif /* def HAVE_SYS_SELECT_H */ +#include +#include #ifndef WINDOWS # ifdef __VMS From 321b5dce853e856df342cc8bddea0dab1f7193b2 Mon Sep 17 00:00:00 2001 From: Tim Ruehsen Date: Mon, 4 Jun 2012 12:25:30 +0200 Subject: [PATCH 57/75] * fix a few little dissonances --- src/ChangeLog | 8 ++++++++ src/main.c | 2 +- src/openssl.c | 2 +- src/utils.c | 4 ++-- src/warc.c | 6 ++++++ 5 files changed, 18 insertions(+), 4 deletions(-) diff --git a/src/ChangeLog b/src/ChangeLog index 5639fd12..c91af04d 100644 --- a/src/ChangeLog +++ b/src/ChangeLog @@ -1,3 +1,11 @@ +2012-06-04 Tim Ruehsen + + * main.c (main): Check for filename != NULL. + * warc.c (warc_process_cdx_line): Fix memory leak. + * utils.c (match_posix_regex, compile_posix_regex): Remove dead + assignment. + * openssl.c (ssl_init): Fix old-style function definition. + 2012-06-02 Giuseppe Scrivano * connect.c: Include and . diff --git a/src/main.c b/src/main.c index aac01ac8..dda91c72 100644 --- a/src/main.c +++ b/src/main.c @@ -1566,7 +1566,7 @@ outputting to a regular file.\n")); &dt, opt.recursive, iri, true); } - if (opt.delete_after && file_exists_p(filename)) + if (opt.delete_after && filename != NULL && file_exists_p (filename)) { DEBUGP (("Removing file due to --delete-after in main():\n")); logprintf (LOG_VERBOSE, _("Removing %s.\n"), filename); diff --git a/src/openssl.c b/src/openssl.c index f976455f..3924e41e 100644 --- a/src/openssl.c +++ b/src/openssl.c @@ -159,7 +159,7 @@ key_type_to_ssl_type (enum keyfile_type type) Returns true on success, false otherwise. */ bool -ssl_init () +ssl_init (void) { SSL_METHOD const *meth; diff --git a/src/utils.c b/src/utils.c index 55a8a8d2..fb3ccd45 100644 --- a/src/utils.c +++ b/src/utils.c @@ -2355,7 +2355,7 @@ compile_posix_regex (const char *str) { int errbuf_size = regerror (errcode, (regex_t *) regex, NULL, 0); char *errbuf = xmalloc (errbuf_size); - errbuf_size = regerror (errcode, (regex_t *) regex, errbuf, errbuf_size); + regerror (errcode, (regex_t *) regex, errbuf, errbuf_size); fprintf (stderr, _("Invalid regular expression %s, %s\n"), quote (str), errbuf); xfree (errbuf); @@ -2402,7 +2402,7 @@ match_posix_regex (const void *regex, const char *str) { int errbuf_size = regerror (rc, opt.acceptregex, NULL, 0); char *errbuf = xmalloc (errbuf_size); - errbuf_size = regerror (rc, opt.acceptregex, errbuf, errbuf_size); + regerror (rc, opt.acceptregex, errbuf, errbuf_size); logprintf (LOG_VERBOSE, _("Error while matching %s: %d\n"), quote (str), rc); xfree (errbuf); diff --git a/src/warc.c b/src/warc.c index 92a49ef8..69f80beb 100644 --- a/src/warc.c +++ b/src/warc.c @@ -913,6 +913,12 @@ warc_process_cdx_line (char *lineptr, int field_num_original_url, free (record_id); } } + else + { + xfree_null(checksum); + xfree_null(original_url); + xfree_null(record_id); + } } /* Loads the CDX file from opt.warc_cdx_dedup_filename and fills From 96418c68851d00fd66014d2dc7bd46b3d715e525 Mon Sep 17 00:00:00 2001 From: Giuseppe Scrivano Date: Wed, 6 Jun 2012 14:10:07 +0200 Subject: [PATCH 58/75] Rename --bits to --report-bps. --- NEWS | 2 +- src/ChangeLog | 10 ++++++++++ src/init.c | 2 +- src/main.c | 4 ++-- src/options.h | 2 +- src/progress.c | 2 +- src/retr.c | 4 ++-- src/utils.c | 4 ++-- 8 files changed, 20 insertions(+), 10 deletions(-) diff --git a/NEWS b/NEWS index ba67de97..20123a2b 100644 --- a/NEWS +++ b/NEWS @@ -21,7 +21,7 @@ Please send GNU Wget bug reports to . ** Report stdout close errors. -** Accept the --bit option. +** Accept the --report-bps option. ** Enable client certificates when GNU TLS is used. diff --git a/src/ChangeLog b/src/ChangeLog index c91af04d..e8c6ba49 100644 --- a/src/ChangeLog +++ b/src/ChangeLog @@ -1,3 +1,13 @@ +2012-06-06 Giuseppe Scrivano + + * options.h (struct options): Rename `bits_fmt' to `report_bps'. + * main.c (print_help): Rename --bits to --report-bps. + (cmdline_options): Likewise. + * init.c (commands): Likewise + * progress.c (create_image): Adjust caller. + * retr.c (retr_rate): Likewise. + * utils.c (convert_to_bits): Likewise. + 2012-06-04 Tim Ruehsen * main.c (main): Check for filename != NULL. diff --git a/src/init.c b/src/init.c index 57a4f00d..b55aa968 100644 --- a/src/init.c +++ b/src/init.c @@ -133,7 +133,6 @@ static const struct { { "backups", &opt.backups, cmd_number }, { "base", &opt.base_href, cmd_string }, { "bindaddress", &opt.bind_address, cmd_string }, - { "bits", &opt.bits_fmt, cmd_boolean}, #ifdef HAVE_SSL { "cacertificate", &opt.ca_cert, cmd_file }, #endif @@ -248,6 +247,7 @@ static const struct { { "relativeonly", &opt.relative_only, cmd_boolean }, { "remoteencoding", &opt.encoding_remote, cmd_string }, { "removelisting", &opt.remove_listing, cmd_boolean }, + { "reportbps", &opt.report_bps, cmd_boolean}, { "restrictfilenames", NULL, cmd_spec_restrict_file_names }, { "retrsymlinks", &opt.retr_symlinks, cmd_boolean }, { "retryconnrefused", &opt.retry_connrefused, cmd_boolean }, diff --git a/src/main.c b/src/main.c index dda91c72..5e87f733 100644 --- a/src/main.c +++ b/src/main.c @@ -168,7 +168,6 @@ static struct cmdline_option option_data[] = { "backups", 0, OPT_BOOLEAN, "backups", -1 }, { "base", 'B', OPT_VALUE, "base", -1 }, { "bind-address", 0, OPT_VALUE, "bindaddress", -1 }, - { "bits", 0, OPT_BOOLEAN, "bits", -1 }, { IF_SSL ("ca-certificate"), 0, OPT_VALUE, "cacertificate", -1 }, { IF_SSL ("ca-directory"), 0, OPT_VALUE, "cadirectory", -1 }, { "cache", 0, OPT_BOOLEAN, "cache", -1 }, @@ -269,6 +268,7 @@ static struct cmdline_option option_data[] = { "relative", 'L', OPT_BOOLEAN, "relativeonly", -1 }, { "remote-encoding", 0, OPT_VALUE, "remoteencoding", -1 }, { "remove-listing", 0, OPT_BOOLEAN, "removelisting", -1 }, + { "report-bps", 0, OPT_BOOLEAN, "reportbps", -1 }, { "restrict-file-names", 0, OPT_BOOLEAN, "restrictfilenames", -1 }, { "retr-symlinks", 0, OPT_BOOLEAN, "retrsymlinks", -1 }, { "retry-connrefused", 0, OPT_BOOLEAN, "retryconnrefused", -1 }, @@ -764,7 +764,7 @@ Recursive accept/reject:\n"), N_("\ Output format:\n"), N_("\ - --bits Output bandwidth in bits.\n"), + --report-bps Output bandwidth in bits.\n"), "\n", N_("Mail bug reports and suggestions to .\n") }; diff --git a/src/options.h b/src/options.h index 0da79379..44e0a703 100644 --- a/src/options.h +++ b/src/options.h @@ -279,7 +279,7 @@ struct options bool show_all_dns_entries; /* Show all the DNS entries when resolving a name. */ - bool bits_fmt; /*Output bandwidth in bits format*/ + bool report_bps; /*Output bandwidth in bits format*/ }; extern struct options opt; diff --git a/src/progress.c b/src/progress.c index f61c95e5..2e888a90 100644 --- a/src/progress.c +++ b/src/progress.c @@ -989,7 +989,7 @@ create_image (struct bar_progress *bp, double dl_total_time, bool done) double dltime = hist->total_time + (dl_total_time - bp->recent_start); double dlspeed = calc_rate (dlquant, dltime, &units); sprintf (p, " %4.*f%s", dlspeed >= 99.95 ? 0 : dlspeed >= 9.995 ? 1 : 2, - dlspeed, !opt.bits_fmt?short_units[units]:short_units_bits[units]); + dlspeed, !opt.report_bps ? short_units[units] : short_units_bits[units]); move_to_end (p); } else diff --git a/src/retr.c b/src/retr.c index 8bc54425..6204839c 100644 --- a/src/retr.c +++ b/src/retr.c @@ -628,7 +628,7 @@ retr_rate (wgint bytes, double secs) e.g. "1022", "247", "12.5", "2.38". */ sprintf (res, "%.*f %s", dlrate >= 99.95 ? 0 : dlrate >= 9.995 ? 1 : 2, - dlrate, !opt.bits_fmt? rate_names[units]: rate_names_bits[units]); + dlrate, !opt.report_bps ? rate_names[units]: rate_names_bits[units]); return res; } @@ -647,7 +647,7 @@ calc_rate (wgint bytes, double secs, int *units) double dlrate; double bibyte = 1000.0; - if (!opt.bits_fmt) + if (!opt.report_bps) bibyte = 1024.0; diff --git a/src/utils.c b/src/utils.c index fb3ccd45..567dc359 100644 --- a/src/utils.c +++ b/src/utils.c @@ -1844,12 +1844,12 @@ number_to_static_string (wgint number) return buf; } -/* Converts the byte to bits format if --bits option is enabled +/* Converts the byte to bits format if --report-bps option is enabled */ wgint convert_to_bits (wgint num) { - if (opt.bits_fmt) + if (opt.report_bps) return num * 8; return num; } From 6b5c0c742d251dd299ad46b43ea6bb576b7b8997 Mon Sep 17 00:00:00 2001 From: Giuseppe Scrivano Date: Wed, 6 Jun 2012 20:41:25 +0200 Subject: [PATCH 59/75] Rename, again, --reports-bits to report-speed. --- NEWS | 2 +- src/ChangeLog | 6 ++++++ src/init.c | 12 +++++++++++- src/main.c | 4 ++-- 4 files changed, 20 insertions(+), 4 deletions(-) diff --git a/NEWS b/NEWS index 20123a2b..3742159c 100644 --- a/NEWS +++ b/NEWS @@ -21,7 +21,7 @@ Please send GNU Wget bug reports to . ** Report stdout close errors. -** Accept the --report-bps option. +** Accept the --report-speed option. ** Enable client certificates when GNU TLS is used. diff --git a/src/ChangeLog b/src/ChangeLog index e8c6ba49..fb352ad3 100644 --- a/src/ChangeLog +++ b/src/ChangeLog @@ -1,9 +1,15 @@ 2012-06-06 Giuseppe Scrivano + * main.c (print_help): Rename --bits to --report-bps. + (cmdline_options): Likewise. + * init.c (commands): Rename --report-bps to --report-speed. + (cmd_spec_report_speed): New function. + * options.h (struct options): Rename `bits_fmt' to `report_bps'. * main.c (print_help): Rename --bits to --report-bps. (cmdline_options): Likewise. * init.c (commands): Likewise + * progress.c (create_image): Adjust caller. * retr.c (retr_rate): Likewise. * utils.c (convert_to_bits): Likewise. diff --git a/src/init.c b/src/init.c index b55aa968..d5f9a4f0 100644 --- a/src/init.c +++ b/src/init.c @@ -100,6 +100,7 @@ CMD_DECLARE (cmd_spec_progress); CMD_DECLARE (cmd_spec_recursive); CMD_DECLARE (cmd_spec_regex_type); CMD_DECLARE (cmd_spec_restrict_file_names); +CMD_DECLARE (cmd_spec_report_speed); #ifdef HAVE_SSL CMD_DECLARE (cmd_spec_secure_protocol); #endif @@ -247,7 +248,7 @@ static const struct { { "relativeonly", &opt.relative_only, cmd_boolean }, { "remoteencoding", &opt.encoding_remote, cmd_string }, { "removelisting", &opt.remove_listing, cmd_boolean }, - { "reportbps", &opt.report_bps, cmd_boolean}, + { "reportspeed", &opt.report_bps, cmd_spec_report_speed}, { "restrictfilenames", NULL, cmd_spec_restrict_file_names }, { "retrsymlinks", &opt.retr_symlinks, cmd_boolean }, { "retryconnrefused", &opt.retry_connrefused, cmd_boolean }, @@ -1451,6 +1452,15 @@ cmd_spec_restrict_file_names (const char *com, const char *val, void *place_igno return true; } +static bool +cmd_spec_report_speed (const char *com, const char *val, void *place_ignored) +{ + opt.report_bps = strcasecmp (val, "bits") == 0; + if (!opt.report_bps) + fprintf (stderr, _("%s: %s: Invalid value %s.\n"), exec_name, com, quote (val)); + return opt.report_bps; +} + #ifdef HAVE_SSL static bool cmd_spec_secure_protocol (const char *com, const char *val, void *place) diff --git a/src/main.c b/src/main.c index 5e87f733..e5a60e66 100644 --- a/src/main.c +++ b/src/main.c @@ -268,7 +268,7 @@ static struct cmdline_option option_data[] = { "relative", 'L', OPT_BOOLEAN, "relativeonly", -1 }, { "remote-encoding", 0, OPT_VALUE, "remoteencoding", -1 }, { "remove-listing", 0, OPT_BOOLEAN, "removelisting", -1 }, - { "report-bps", 0, OPT_BOOLEAN, "reportbps", -1 }, + { "report-speed", 0, OPT_BOOLEAN, "reportspeed", -1 }, { "restrict-file-names", 0, OPT_BOOLEAN, "restrictfilenames", -1 }, { "retr-symlinks", 0, OPT_BOOLEAN, "retrsymlinks", -1 }, { "retry-connrefused", 0, OPT_BOOLEAN, "retryconnrefused", -1 }, @@ -764,7 +764,7 @@ Recursive accept/reject:\n"), N_("\ Output format:\n"), N_("\ - --report-bps Output bandwidth in bits.\n"), + --report-speed=TYPE Output bandwidth as TYPE. TYPE can be bits.\n"), "\n", N_("Mail bug reports and suggestions to .\n") }; From 3806fd1e0277bf56f66e5f14fd8380229740db0b Mon Sep 17 00:00:00 2001 From: Giuseppe Scrivano Date: Sat, 9 Jun 2012 12:41:57 +0200 Subject: [PATCH 60/75] texi2pod.pl: Revert change from 2011-08-06. --- doc/ChangeLog | 4 +++ doc/texi2pod.pl | 93 ++++++++++++------------------------------------- 2 files changed, 26 insertions(+), 71 deletions(-) diff --git a/doc/ChangeLog b/doc/ChangeLog index a163bf34..2925a5bf 100644 --- a/doc/ChangeLog +++ b/doc/ChangeLog @@ -1,3 +1,7 @@ +2012-06-09 Giuseppe Scrivano + + * texi2pod.pl: Revert change from 2011-08-06. + 2012-05-13 Giuseppe Scrivano * wget.texi (Types of Files): Document --accept-regex and diff --git a/doc/texi2pod.pl b/doc/texi2pod.pl index 9c0e94c6..86c4b189 100755 --- a/doc/texi2pod.pl +++ b/doc/texi2pod.pl @@ -1,6 +1,7 @@ #! /usr/bin/env perl -# Copyright (C) 1999, 2000, 2001, 2003, 2010 Free Software Foundation, Inc. +# Copyright (C) 1999, 2000, 2001, 2003, 2007, 2009, 2010, 2011 Free +# Software Foundation, Inc. # This file is part of GCC. @@ -15,14 +16,15 @@ # GNU General Public License for more details. # You should have received a copy of the GNU General Public License -# along with GCC; see the file COPYING. If not, write to -# the Free Software Foundation, 51 Franklin Street, Fifth Floor, -# Boston MA 02110-1301, USA. +# along with GCC. If not, see . # This does trivial (and I mean _trivial_) conversion of Texinfo # markup to Perl POD format. It's intended to be used to extract # something suitable for a manpage from a Texinfo document. +use warnings; +BEGIN { eval { require warnings; } and warnings->import; } + $output = 0; $skipping = 0; %sects = (); @@ -36,7 +38,6 @@ $shift = ""; $fnno = 1; $inf = ""; $ibase = ""; -@ipath = (); while ($_ = shift) { if (/^-D(.*)$/) { @@ -52,13 +53,6 @@ while ($_ = shift) { die "flags may only contain letters, digits, hyphens, dashes and underscores\n" unless $flag =~ /^[a-zA-Z0-9_-]+$/; $defs{$flag} = $value; - } elsif (/^-I(.*)$/) { - if ($1 ne "") { - $flag = $1; - } else { - $flag = shift; - } - push (@ipath, $flag); } elsif (/^-/) { usage(); } else { @@ -162,8 +156,6 @@ while(<$inf>) { } elsif ($ended =~ /^(?:itemize|enumerate|[fv]?table)$/) { $_ = "\n=back\n"; $ic = pop @icstack; - } elsif ($ended eq "multitable") { - $_ = "\n=back\n"; } else { die "unknown command \@end $ended at line $.\n"; } @@ -213,18 +205,14 @@ while(<$inf>) { # Now the ones that have to be replaced by special escapes # (which will be turned back into text by unmunge()) - # Replace @@ before @{ and @} in order to parse @samp{@@} correctly. s/&/&/g; s/\@\@/&at;/g; s/\@\{/{/g; s/\@\}/}/g; - s/\@`\{(.)\}/&$1grave;/g; - # Inside a verbatim block, handle @var, @samp and @url specially. + # Inside a verbatim block, handle @var specially. if ($shift ne "") { s/\@var\{([^\}]*)\}/<$1>/g; - s/\@samp\{([^\}]*)\}/"$1"/g; - s/\@url\{([^\}]*)\}/<$1>/g; } # POD doesn't interpret E<> inside a verbatim block. @@ -243,23 +231,17 @@ while(<$inf>) { $inf = gensym(); $file = postprocess($1); - # Try cwd and $ibase, then explicit -I paths. - $done = 0; - foreach $path ("", $ibase, @ipath) { - $mypath = $file; - $mypath = $path . "/" . $mypath if ($path ne ""); - open($inf, "<" . $mypath) and ($done = 1, last); - } - die "cannot find $file" if !$done; + # Try cwd and $ibase. + open($inf, "<" . $file) + or open($inf, "<" . $ibase . "/" . $file) + or die "cannot open $file or $ibase/$file: $!\n"; next; }; - /^\@(?:section|unnumbered|unnumberedsec|center|heading)\s+(.+)$/ + /^\@(?:section|unnumbered|unnumberedsec|center)\s+(.+)$/ and $_ = "\n=head2 $1\n"; /^\@subsection\s+(.+)$/ and $_ = "\n=head3 $1\n"; - /^\@subsubsection\s+(.+)$/ - and $_ = "\n=head4 $1\n"; # Block command handlers: /^\@itemize(?:\s+(\@[a-z]+|\*|-))?/ and do { @@ -268,7 +250,7 @@ while(<$inf>) { if (defined $1) { $ic = $1; } else { - $ic = '*'; + $ic = '@bullet'; } $_ = "\n=over 4\n"; $endw = "itemize"; @@ -286,12 +268,6 @@ while(<$inf>) { $endw = "enumerate"; }; - /^\@multitable\s.*/ and do { - push @endwstack, $endw; - $endw = "multitable"; - $_ = "\n=over 4\n"; - }; - /^\@([fv]?table)\s+(\@[a-z]+)/ and do { push @endwstack, $endw; push @icstack, $ic; @@ -301,7 +277,6 @@ while(<$inf>) { $ic =~ s/\@(?:code|kbd)/C/; $ic =~ s/\@(?:dfn|var|emph|cite|i)/I/; $ic =~ s/\@(?:file)/F/; - $ic =~ s/\@(?:asis)//; $_ = "\n=over 4\n"; }; @@ -312,29 +287,14 @@ while(<$inf>) { $_ = ""; # need a paragraph break }; - /^\@item\s+(.*\S)\s*$/ and $endw eq "multitable" and do { - @columns = (); - for $column (split (/\s*\@tab\s*/, $1)) { - # @strong{...} is used a @headitem work-alike - $column =~ s/^\@strong{(.*)}$/$1/; - push @columns, $column; - } - $_ = "\n=item ".join (" : ", @columns)."\n"; - }; - /^\@itemx?\s*(.+)?$/ and do { if (defined $1) { - if ($ic) { - if ($endw eq "enumerate") { - $_ = "\n=item $ic $1\n"; - $ic =~ s/(\d+)/$1 + 1/eg; - } else { - # Entity escapes prevent munging by the <> - # processing below. - $_ = "\n=item $ic\<$1\>\n"; - } + my $thing = $1; + if ($ic =~ /\@asis/) { + $_ = "\n=item $thing\n"; } else { - $_ = "\n=item $1\n"; + # Entity escapes prevent munging by the <> processing below. + $_ = "\n=item $ic\<$thing\>\n"; } } else { $_ = "\n=item $ic\n"; @@ -355,11 +315,12 @@ die "No filename or title\n" unless defined $fn && defined $tl; $sects{NAME} = "$fn \- $tl\n"; $sects{FOOTNOTES} .= "=back\n" if exists $sects{FOOTNOTES}; -for $sect (qw(NAME SYNOPSIS DESCRIPTION OPTIONS ENVIRONMENT FILES - BUGS NOTES FOOTNOTES SEEALSO AUTHOR COPYRIGHT)) { +for $sect (qw(NAME SYNOPSIS DESCRIPTION OPTIONS ENVIRONMENT EXITSTATUS + FILES BUGS NOTES FOOTNOTES SEEALSO AUTHOR COPYRIGHT)) { if(exists $sects{$sect}) { $head = $sect; $head =~ s/SEEALSO/SEE ALSO/; + $head =~ s/EXITSTATUS/EXIT STATUS/; print "=head1 $head\n\n"; print scalar unmunge ($sects{$sect}); print "\n"; @@ -391,13 +352,11 @@ sub postprocess s/\@r\{([^\}]*)\}/R<$1>/g; s/\@(?:dfn|var|emph|cite|i)\{([^\}]*)\}/I<$1>/g; s/\@(?:code|kbd)\{([^\}]*)\}/C<$1>/g; - s/\@(?:samp|strong|key|option|env|command|b)\{([^\}]*)\}/B<$1>/g; + s/\@(?:gccoptlist|samp|strong|key|option|env|command|b)\{([^\}]*)\}/B<$1>/g; s/\@sc\{([^\}]*)\}/\U$1/g; - s/\@acronym\{([^\}]*)\}/\U$1/g; s/\@file\{([^\}]*)\}/F<$1>/g; s/\@w\{([^\}]*)\}/S<$1>/g; s/\@(?:dmn|math)\{([^\}]*)\}/$1/g; - s/\@\///g; # keep references of the form @ref{...}, print them bold s/\@(?:ref)\{([^\}]*)\}/B<$1>/g; @@ -419,9 +378,6 @@ sub postprocess s/\@gol//g; s/\@\*\s*\n?//g; - # Anchors are thrown away - s/\@anchor\{(?:[^\}]*)\}//g; - # @uref can take one, two, or three arguments, with different # semantics each time. @url and @email are just like @uref with # one argument, for our purposes. @@ -429,10 +385,6 @@ sub postprocess s/\@uref\{([^\},]*),([^\},]*)\}/$2 (C<$1>)/g; s/\@uref\{([^\},]*),([^\},]*),([^\},]*)\}/$3/g; - # Handle gccoptlist here, so it can contain the above formatting - # commands. - s/\@gccoptlist\{([^\}]*)\}/B<$1>/g; - # Un-escape <> at this point. s/<//g; @@ -466,7 +418,6 @@ sub unmunge # Replace escaped symbols with their equivalents. local $_ = $_[0]; - s/&(.)grave;/E<$1grave>/g; s/</E/g; s/>/E/g; s/{/\{/g; From aa2f287c68a9c45329dad97700fa562c4207b9ce Mon Sep 17 00:00:00 2001 From: Giuseppe Scrivano Date: Sat, 9 Jun 2012 13:13:28 +0200 Subject: [PATCH 61/75] help: Move --report-speed under 'Logging and input file'. --- src/ChangeLog | 5 +++++ src/main.c | 8 ++------ 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/src/ChangeLog b/src/ChangeLog index fb352ad3..3cb29635 100644 --- a/src/ChangeLog +++ b/src/ChangeLog @@ -1,3 +1,8 @@ +2012-06-09 Giuseppe Scrivano + + * main.c (print_help): Move --report-speed under the section + "Logging and input file". + 2012-06-06 Giuseppe Scrivano * main.c (print_help): Rename --bits to --report-bps. diff --git a/src/main.c b/src/main.c index e5a60e66..94a33e75 100644 --- a/src/main.c +++ b/src/main.c @@ -460,6 +460,8 @@ Logging and input file:\n"), -v, --verbose be verbose (this is the default).\n"), N_("\ -nv, --no-verbose turn off verboseness, without being quiet.\n"), + N_("\ + --report-speed=TYPE Output bandwidth as TYPE. TYPE can be bits.\n"), N_("\ -i, --input-file=FILE download URLs found in local or external FILE.\n"), N_("\ @@ -760,12 +762,6 @@ Recursive accept/reject:\n"), N_("\ -np, --no-parent don't ascend to the parent directory.\n"), "\n", - - N_("\ -Output format:\n"), - N_("\ - --report-speed=TYPE Output bandwidth as TYPE. TYPE can be bits.\n"), - "\n", N_("Mail bug reports and suggestions to .\n") }; From 29861463de41aeaa9b72c66321b125559ea958aa Mon Sep 17 00:00:00 2001 From: Giuseppe Scrivano Date: Sat, 9 Jun 2012 13:14:51 +0200 Subject: [PATCH 62/75] doc: document new options. --- doc/ChangeLog | 3 +++ doc/wget.texi | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 36 insertions(+) diff --git a/doc/ChangeLog b/doc/ChangeLog index 2925a5bf..f3af7ea3 100644 --- a/doc/ChangeLog +++ b/doc/ChangeLog @@ -1,5 +1,8 @@ 2012-06-09 Giuseppe Scrivano + * wget.texi (Logging and Input File Options): Document "--report-speed". + (HTTPS (SSL/TLS) Options): Document WARC. + * texi2pod.pl: Revert change from 2011-08-06. 2012-05-13 Giuseppe Scrivano diff --git a/doc/wget.texi b/doc/wget.texi index cd379e97..5d6a28fc 100644 --- a/doc/wget.texi +++ b/doc/wget.texi @@ -479,6 +479,10 @@ Turn off verbose without being completely quiet (use @samp{-q} for that), which means that error messages and basic information still get printed. +@item -nv +@itemx --report-speed=@var{type} +Output bandwidth as @var{type}. The only accepted value is @samp{bits}. + @cindex input-file @item -i @var{file} @itemx --input-file=@var{file} @@ -1658,6 +1662,35 @@ not used), EGD is never contacted. EGD is not needed on modern Unix systems that support @file{/dev/random}. @end table +@cindex WARC +@item --warc-file=@var{file} +Use @var{file} as the destination WARC file. + +@item --warc-header=@var{string} +Use @var{string} into as the warcinfo record. + +@item --warc-max-size=@var{size} +Set the maximum size of the WARC files to @var{size}. + +@item --warc-cdx +Write CDX index files. + +@item --warc-dedup=@var{file} +Do not store records listed in this CDX file. + +@item --no-warc-compression +Do not compress WARC files with GZIP. + +@item --no-warc-digests +Do not calculate SHA1 digests. + +@item --no-warc-keep-log +Do not store the log file in a WARC record. + +@item--warc-tempdir=@var{dir} +Specify the location for temporary files created by the WARC writer. + + @node FTP Options, Recursive Retrieval Options, HTTPS (SSL/TLS) Options, Invoking @section FTP Options From 4661f141bb6e694592b1f26c49f11f7036093162 Mon Sep 17 00:00:00 2001 From: Giuseppe Scrivano Date: Sat, 9 Jun 2012 13:17:27 +0200 Subject: [PATCH 63/75] Fix the last commit. --- doc/wget.texi | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/doc/wget.texi b/doc/wget.texi index 5d6a28fc..73341ec6 100644 --- a/doc/wget.texi +++ b/doc/wget.texi @@ -1663,6 +1663,7 @@ systems that support @file{/dev/random}. @end table @cindex WARC +@table @samp @item --warc-file=@var{file} Use @var{file} as the destination WARC file. @@ -1687,9 +1688,9 @@ Do not calculate SHA1 digests. @item --no-warc-keep-log Do not store the log file in a WARC record. -@item--warc-tempdir=@var{dir} +@item --warc-tempdir=@var{dir} Specify the location for temporary files created by the WARC writer. - +@end table @node FTP Options, Recursive Retrieval Options, HTTPS (SSL/TLS) Options, Invoking @section FTP Options From 93720df4c8d76af7848267f8197ed76eea21f181 Mon Sep 17 00:00:00 2001 From: Giuseppe Scrivano Date: Sat, 16 Jun 2012 12:15:03 +0200 Subject: [PATCH 64/75] Do not close stdout twice. --- ChangeLog | 5 +++++ bootstrap.conf | 1 - src/ChangeLog | 6 ++++++ src/main.c | 3 --- 4 files changed, 11 insertions(+), 4 deletions(-) diff --git a/ChangeLog b/ChangeLog index 2f0f9655..ef8bbefe 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,8 @@ +2012-06-16 Giuseppe Scrivano + + * bootstrap.conf (gnulib_modules): Remove `closeout'. + Reported by: Micah Cowan . + 2012-05-31 Ángel González * convert.c: fix segfault on wrong urls (bug 36570) diff --git a/bootstrap.conf b/bootstrap.conf index a784a906..febd11da 100644 --- a/bootstrap.conf +++ b/bootstrap.conf @@ -33,7 +33,6 @@ bind c-ctype clock-time close -closeout connect fcntl futimens diff --git a/src/ChangeLog b/src/ChangeLog index 3cb29635..bd594522 100644 --- a/src/ChangeLog +++ b/src/ChangeLog @@ -1,3 +1,9 @@ +2012-06-16 Giuseppe Scrivano + + * main.c: Do not include "stdout.h". + (main): Do not register `close_stdout' at exit. + Reported by: Micah Cowan . + 2012-06-09 Giuseppe Scrivano * main.c (print_help): Move --report-speed under the section diff --git a/src/main.c b/src/main.c index 94a33e75..291fe077 100644 --- a/src/main.c +++ b/src/main.c @@ -56,7 +56,6 @@ as that of the covered work. */ #include "http.h" /* for save_cookies */ #include "ptimer.h" #include "warc.h" -#include "closeout.h" #include #include #include @@ -982,8 +981,6 @@ main (int argc, char **argv) i18n_initialize (); - atexit (close_stdout); - /* Construct the name of the executable, without the directory part. */ #ifdef __VMS /* On VMS, lose the "dev:[dir]" prefix and the ".EXE;nnn" suffix. */ From 90e9d9e1bd98dcb1ab286d696acf790cf3758a6a Mon Sep 17 00:00:00 2001 From: Giuseppe Scrivano Date: Sat, 16 Jun 2012 12:20:33 +0200 Subject: [PATCH 65/75] Move cleanup related code to `cleanup' --- src/ChangeLog | 3 +++ src/init.c | 9 +++++++++ src/main.c | 8 -------- 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/src/ChangeLog b/src/ChangeLog index bd594522..b3937051 100644 --- a/src/ChangeLog +++ b/src/ChangeLog @@ -1,5 +1,8 @@ 2012-06-16 Giuseppe Scrivano + * main.c (main): Move some cleanup related function to... + * init.c (cleanup): ...here. + * main.c: Do not include "stdout.h". (main): Do not register `close_stdout' at exit. Reported by: Micah Cowan . diff --git a/src/init.c b/src/init.c index d5f9a4f0..40b62b27 100644 --- a/src/init.c +++ b/src/init.c @@ -1675,6 +1675,12 @@ cleanup (void) { /* Free external resources, close files, etc. */ + /* Close WARC file. */ + if (opt.warc_filename != 0) + warc_close (); + + log_close (); + if (output_stream) fclose (output_stream); /* No need to check for error because Wget flushes its output (and @@ -1696,6 +1702,9 @@ cleanup (void) host_cleanup (); log_cleanup (); + for (i = 0; i < nurl; i++) + xfree (url[i]); + { extern acc_t *netrc_list; free_netrc (netrc_list); diff --git a/src/main.c b/src/main.c index 291fe077..96d7d57f 100644 --- a/src/main.c +++ b/src/main.c @@ -1626,14 +1626,6 @@ outputting to a regular file.\n")); if (opt.convert_links && !opt.delete_after) convert_all_links (); - /* Close WARC file. */ - if (opt.warc_filename != 0) - warc_close (); - - log_close (); - - for (i = 0; i < nurl; i++) - xfree (url[i]); cleanup (); exit (get_exit_status ()); From 6aa2a7cc9879228cd6c910734beeb8f8688811e9 Mon Sep 17 00:00:00 2001 From: Giuseppe Scrivano Date: Sat, 16 Jun 2012 13:05:03 +0200 Subject: [PATCH 66/75] Add new test --- tests/ChangeLog | 6 ++++++ tests/Makefile.am | 1 + tests/Test-stdouterr.px | 48 +++++++++++++++++++++++++++++++++++++++++ tests/run-px | 1 + 4 files changed, 56 insertions(+) create mode 100755 tests/Test-stdouterr.px diff --git a/tests/ChangeLog b/tests/ChangeLog index f686c03d..e12d8b54 100644 --- a/tests/ChangeLog +++ b/tests/ChangeLog @@ -1,3 +1,9 @@ +2012-06-16 Giuseppe Scrivano + + * Makefile.am (EXTRA_DIST): Add Test-stdouterr.px. + * run-px (tests): Likewise. + * Test-stdouterr.px: New file. + 2011-06-03 Merinov Nikolay * Test-idn-cmd-utf8.px: Added test for idn with utf-8 local encoding. diff --git a/tests/Makefile.am b/tests/Makefile.am index 6cdbb991..9ff302cd 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -124,6 +124,7 @@ EXTRA_DIST = FTPServer.pm FTPTest.pm HTTPServer.pm HTTPTest.pm \ Test-restrict-ascii.px \ Test-Restrict-Lowercase.px \ Test-Restrict-Uppercase.px \ + Test-stdouterr.px \ Test--spider-fail.px \ Test--spider.px \ Test--spider-r-HTTP-Content-Disposition.px \ diff --git a/tests/Test-stdouterr.px b/tests/Test-stdouterr.px new file mode 100755 index 00000000..d594ead0 --- /dev/null +++ b/tests/Test-stdouterr.px @@ -0,0 +1,48 @@ +#!/usr/bin/env perl + +use strict; +use warnings; + +use HTTPTest; + + +############################################################################### + +# code, msg, headers, content +my %urls = ( + '/somefile.txt' => { + code => "200", + msg => "Dontcare", + headers => { + "Content-type" => "text/plain", + }, + content => "blabla", + }, +); + +unless(-e "/dev/full") { + exit(2); # skip +} + +my $cmdline = $WgetTest::WGETPATH . " -c http://localhost:{{port}}/somefile.txt -O /dev/full"; + +my $expected_error_code = 3; + +my %existing_files = ( +); + +my %expected_downloaded_files = ( +); + +############################################################################### + +my $the_test = HTTPTest->new (name => "Test-stdouterr", + input => \%urls, + cmdline => $cmdline, + errcode => $expected_error_code, + existing => \%existing_files, + output => \%expected_downloaded_files); +exit $the_test->run(); + +# vim: et ts=4 sw=4 + diff --git a/tests/run-px b/tests/run-px index 21074cc9..657194fc 100755 --- a/tests/run-px +++ b/tests/run-px @@ -74,6 +74,7 @@ my @tests = ( 'Test-restrict-ascii.px', 'Test-Restrict-Lowercase.px', 'Test-Restrict-Uppercase.px', + 'Test-stdouterr.px', 'Test--spider-fail.px', 'Test--spider-r-HTTP-Content-Disposition.px', 'Test--spider-r--no-content-disposition.px', From ae0598df9bc459652c167fa9826a72b10b775a7a Mon Sep 17 00:00:00 2001 From: Giuseppe Scrivano Date: Sun, 17 Jun 2012 22:24:32 +0200 Subject: [PATCH 67/75] Check for fclose errors. --- src/ChangeLog | 7 +++++++ src/exits.c | 2 +- src/init.c | 5 ++++- src/wget.h | 2 +- 4 files changed, 13 insertions(+), 3 deletions(-) diff --git a/src/ChangeLog b/src/ChangeLog index b3937051..7b2bde0e 100644 --- a/src/ChangeLog +++ b/src/ChangeLog @@ -1,3 +1,10 @@ +2012-06-17 Giuseppe Scrivano + + * wget.h: Define `CLOSEFAILED'. + * init.c: Include "exits.h". + (cleanup): Check `fclose' failure. + * exits.c (get_status_for_err): Handle `CLOSEFAILED'. + 2012-06-16 Giuseppe Scrivano * main.c (main): Move some cleanup related function to... diff --git a/src/exits.c b/src/exits.c index 3d846b56..2233cdc1 100644 --- a/src/exits.c +++ b/src/exits.c @@ -60,7 +60,7 @@ get_status_for_err (uerr_t err) case RETROK: return WGET_EXIT_SUCCESS; case FOPENERR: case FOPEN_EXCL_ERR: case FWRITEERR: case WRITEFAILED: - case UNLINKERR: + case UNLINKERR: case CLOSEFAILED: return WGET_EXIT_IO_FAIL; case NOCONERROR: case HOSTERR: case CONSOCKERR: case CONERROR: case CONSSLERR: case CONIMPOSSIBLE: case FTPRERR: case FTPINVPASV: diff --git a/src/init.c b/src/init.c index 40b62b27..4188ca16 100644 --- a/src/init.c +++ b/src/init.c @@ -30,6 +30,7 @@ shall include the source code for the parts of OpenSSL used as well as that of the covered work. */ #include "wget.h" +#include "exits.h" #include #include @@ -1682,7 +1683,9 @@ cleanup (void) log_close (); if (output_stream) - fclose (output_stream); + if (fclose (output_stream) == EOF) + inform_exit_status (CLOSEFAILED); + /* No need to check for error because Wget flushes its output (and checks for errors) after any data arrives. */ diff --git a/src/wget.h b/src/wget.h index ee315b6f..ca4a702d 100644 --- a/src/wget.h +++ b/src/wget.h @@ -353,7 +353,7 @@ typedef enum PROXERR, /* 50 */ AUTHFAILED, QUOTEXC, WRITEFAILED, SSLINITFAILED, VERIFCERTERR, - UNLINKERR, NEWLOCATION_KEEP_POST, + UNLINKERR, NEWLOCATION_KEEP_POST, CLOSEFAILED, WARC_ERR, WARC_TMP_FOPENERR, WARC_TMP_FWRITEERR } uerr_t; From 172a11764713faeb7eb6ce569c032b1b1e37e370 Mon Sep 17 00:00:00 2001 From: Giuseppe Scrivano Date: Sat, 7 Jul 2012 10:27:09 +0200 Subject: [PATCH 68/75] Fix some log messages. --- src/ChangeLog | 6 ++++++ src/http.c | 2 +- src/main.c | 2 +- 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/src/ChangeLog b/src/ChangeLog index 7b2bde0e..f8f7a49e 100644 --- a/src/ChangeLog +++ b/src/ChangeLog @@ -1,3 +1,9 @@ +2012-07-07 Giuseppe Scrivano + + * http.c (http_loop): Fix log message. + * main.c (main): Likewise. + Reported by: Petr Pisar + 2012-06-17 Giuseppe Scrivano * wget.h: Define `CLOSEFAILED'. diff --git a/src/http.c b/src/http.c index 8d4edba5..1a4b2e10 100644 --- a/src/http.c +++ b/src/http.c @@ -3118,7 +3118,7 @@ Spider mode enabled. Check if remote file exists.\n")); case WARC_ERR: /* A fatal WARC error. */ logputs (LOG_VERBOSE, "\n"); - logprintf (LOG_NOTQUIET, _("Cannot write to WARC file..\n")); + logprintf (LOG_NOTQUIET, _("Cannot write to WARC file.\n")); ret = err; goto exit; case WARC_TMP_FOPENERR: case WARC_TMP_FWRITEERR: diff --git a/src/main.c b/src/main.c index 96d7d57f..b8b28699 100644 --- a/src/main.c +++ b/src/main.c @@ -1188,7 +1188,7 @@ main (int argc, char **argv) { fprintf (stderr, _("Both --no-clobber and --convert-links were specified," - "only --convert-links will be used.\n")); + " only --convert-links will be used.\n")); opt.noclobber = false; } From 4fe805a7ecad8a1717cdba78710fba2c543d39ce Mon Sep 17 00:00:00 2001 From: Tim Ruehsen Date: Thu, 28 Jun 2012 17:45:18 +0200 Subject: [PATCH 69/75] Add support for RFC 2617 Digest Access Authentication --- src/ChangeLog | 5 +++ src/http.c | 98 +++++++++++++++++++++++++++++++++++++++------------ 2 files changed, 81 insertions(+), 22 deletions(-) diff --git a/src/ChangeLog b/src/ChangeLog index f8f7a49e..11566b4f 100644 --- a/src/ChangeLog +++ b/src/ChangeLog @@ -1,3 +1,8 @@ +2012-07-07 Tim Ruehsen + + (digest_authentication_encode): Add support for RFC 2617 Digest + Access Authentication. + 2012-07-07 Giuseppe Scrivano * http.c (http_loop): Fix log message. diff --git a/src/http.c b/src/http.c index 1a4b2e10..fa2d5ed2 100644 --- a/src/http.c +++ b/src/http.c @@ -3655,19 +3655,23 @@ digest_authentication_encode (const char *au, const char *user, const char *passwd, const char *method, const char *path) { - static char *realm, *opaque, *nonce; + static char *realm, *opaque, *nonce, *qop; static struct { const char *name; char **variable; } options[] = { { "realm", &realm }, { "opaque", &opaque }, - { "nonce", &nonce } + { "nonce", &nonce }, + { "qop", &qop } }; + char cnonce[16] = ""; char *res; + size_t res_size; param_token name, value; - realm = opaque = nonce = NULL; + + realm = opaque = nonce = qop = NULL; au += 6; /* skip over `Digest' */ while (extract_param (&au, &name, &value, ',')) @@ -3683,11 +3687,19 @@ digest_authentication_encode (const char *au, const char *user, break; } } + + if (qop != NULL && strcmp(qop,"auth")) + { + logprintf (LOG_NOTQUIET, _("Unsupported quality of protection '%s'.\n"), qop); + user = NULL; /* force freeing mem and return */ + } + if (!realm || !nonce || !user || !passwd || !path || !method) { xfree_null (realm); xfree_null (opaque); xfree_null (nonce); + xfree_null (qop); return NULL; } @@ -3716,27 +3728,69 @@ digest_authentication_encode (const char *au, const char *user, md5_finish_ctx (&ctx, hash); dump_hash (a2buf, hash); - /* RESPONSE_DIGEST = H(A1BUF ":" nonce ":" A2BUF) */ - md5_init_ctx (&ctx); - md5_process_bytes ((unsigned char *)a1buf, MD5_DIGEST_SIZE * 2, &ctx); - md5_process_bytes ((unsigned char *)":", 1, &ctx); - md5_process_bytes ((unsigned char *)nonce, strlen (nonce), &ctx); - md5_process_bytes ((unsigned char *)":", 1, &ctx); - md5_process_bytes ((unsigned char *)a2buf, MD5_DIGEST_SIZE * 2, &ctx); - md5_finish_ctx (&ctx, hash); + if (!strcmp(qop,"auth")) + { + /* RFC 2617 Digest Access Authentication */ + /* generate random hex string */ + snprintf(cnonce, sizeof(cnonce), "%08x", random_number(INT_MAX)); + + /* RESPONSE_DIGEST = H(A1BUF ":" nonce ":" noncecount ":" clientnonce ":" qop ": " A2BUF) */ + md5_init_ctx (&ctx); + md5_process_bytes ((unsigned char *)a1buf, MD5_DIGEST_SIZE * 2, &ctx); + md5_process_bytes ((unsigned char *)":", 1, &ctx); + md5_process_bytes ((unsigned char *)nonce, strlen (nonce), &ctx); + md5_process_bytes ((unsigned char *)":", 1, &ctx); + md5_process_bytes ((unsigned char *)"00000001", 8, &ctx); /* TODO: keep track of server nonce values */ + md5_process_bytes ((unsigned char *)":", 1, &ctx); + md5_process_bytes ((unsigned char *)cnonce, strlen(cnonce), &ctx); + md5_process_bytes ((unsigned char *)":", 1, &ctx); + md5_process_bytes ((unsigned char *)qop, strlen(qop), &ctx); + md5_process_bytes ((unsigned char *)":", 1, &ctx); + md5_process_bytes ((unsigned char *)a2buf, MD5_DIGEST_SIZE * 2, &ctx); + md5_finish_ctx (&ctx, hash); + } + else + { + /* RFC 2069 Digest Access Authentication */ + /* RESPONSE_DIGEST = H(A1BUF ":" nonce ":" A2BUF) */ + md5_init_ctx (&ctx); + md5_process_bytes ((unsigned char *)a1buf, MD5_DIGEST_SIZE * 2, &ctx); + md5_process_bytes ((unsigned char *)":", 1, &ctx); + md5_process_bytes ((unsigned char *)nonce, strlen (nonce), &ctx); + md5_process_bytes ((unsigned char *)":", 1, &ctx); + md5_process_bytes ((unsigned char *)a2buf, MD5_DIGEST_SIZE * 2, &ctx); + md5_finish_ctx (&ctx, hash); + } + dump_hash (response_digest, hash); - res = xmalloc (strlen (user) - + strlen (user) - + strlen (realm) - + strlen (nonce) - + strlen (path) - + 2 * MD5_DIGEST_SIZE /*strlen (response_digest)*/ - + (opaque ? strlen (opaque) : 0) - + 128); - sprintf (res, "Digest \ -username=\"%s\", realm=\"%s\", nonce=\"%s\", uri=\"%s\", response=\"%s\"", - user, realm, nonce, path, response_digest); + res_size = strlen (user) + + strlen (user) + + strlen (realm) + + strlen (nonce) + + strlen (path) + + 2 * MD5_DIGEST_SIZE /*strlen (response_digest)*/ + + (opaque ? strlen (opaque) : 0) + + (qop ? 128: 0) + + 128; + + res = xmalloc (res_size); + + if (!strcmp(qop,"auth")) + { + snprintf (res, res_size, "Digest "\ + "username=\"%s\", realm=\"%s\", nonce=\"%s\", uri=\"%s\", response=\"%s\""\ + ", qop=auth, nc=00000001, cnonce=\"%s\"", + user, realm, nonce, path, response_digest, cnonce); + + } + else + { + snprintf (res, res_size, "Digest "\ + "username=\"%s\", realm=\"%s\", nonce=\"%s\", uri=\"%s\", response=\"%s\"", + user, realm, nonce, path, response_digest); + } + if (opaque) { char *p = res + strlen (res); From f9768d368dfda89b576d3a5d6e0f940faadd3d80 Mon Sep 17 00:00:00 2001 From: Giuseppe Scrivano Date: Sun, 8 Jul 2012 11:29:09 +0200 Subject: [PATCH 70/75] Cite new change in the NEWS file. --- NEWS | 2 ++ 1 file changed, 2 insertions(+) diff --git a/NEWS b/NEWS index 3742159c..25c401e5 100644 --- a/NEWS +++ b/NEWS @@ -30,6 +30,8 @@ Please send GNU Wget bug reports to . ** Accept the arguments --accept-reject and --reject-regex. ** The GNU TLS backend honors correctly the timeout value. + +** Add support for RFC 2617 Digest Access Authentication. * Changes in Wget 1.13.4 From c32ef46f99ac78f0172a7784ca8055382857b4f6 Mon Sep 17 00:00:00 2001 From: Steven Schubiger Date: Sun, 8 Jul 2012 11:30:53 +0200 Subject: [PATCH 71/75] Fix header comments for exits.h and exits.c. --- src/ChangeLog | 5 +++++ src/exits.c | 6 ++---- src/exits.h | 4 ++-- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/src/ChangeLog b/src/ChangeLog index 11566b4f..e91cc218 100644 --- a/src/ChangeLog +++ b/src/ChangeLog @@ -1,3 +1,8 @@ +2012-07-08 Steven Schubiger + + * exits.h: Fix comment. + * exits.c: Likewise. + 2012-07-07 Tim Ruehsen (digest_authentication_encode): Add support for RFC 2617 Digest diff --git a/src/exits.c b/src/exits.c index 2233cdc1..e23fc1c9 100644 --- a/src/exits.c +++ b/src/exits.c @@ -1,7 +1,5 @@ -/* Command line parsing. - Copyright (C) 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004, - 2005, 2006, 2007, 2008, 2009, 2010, 2011 Free Software Foundation, - Inc. +/* Exit status handling. + Copyright (C) 2009, 2010, 2011, 2012 Free Software Foundation, Inc. This file is part of GNU Wget. diff --git a/src/exits.h b/src/exits.h index dfe95164..98dde9a7 100644 --- a/src/exits.h +++ b/src/exits.h @@ -1,5 +1,5 @@ -/* Internationalization related declarations. - Copyright (C) 2008, 2009, 2010, 2011 Free Software Foundation, Inc. +/* Exit status related declarations. + Copyright (C) 2009, 2010, 2011, 2012 Free Software Foundation, Inc. This file is part of GNU Wget. From 31674653eb2ee894e8f8d67882514925acf639f2 Mon Sep 17 00:00:00 2001 From: Steven Schubiger Date: Sun, 8 Jul 2012 11:36:54 +0200 Subject: [PATCH 72/75] Include missing header. --- src/ChangeLog | 4 ++++ src/init.c | 1 + 2 files changed, 5 insertions(+) diff --git a/src/ChangeLog b/src/ChangeLog index e91cc218..8fcd0bf0 100644 --- a/src/ChangeLog +++ b/src/ChangeLog @@ -1,3 +1,7 @@ +2012-07-03 Steven Schubiger + + * init.c: Include warc.h for warc_close in cleanup function. + 2012-07-08 Steven Schubiger * exits.h: Fix comment. diff --git a/src/init.c b/src/init.c index 4188ca16..365fb5ba 100644 --- a/src/init.c +++ b/src/init.c @@ -67,6 +67,7 @@ as that of the covered work. */ #include "res.h" /* for res_cleanup */ #include "http.h" /* for http_cleanup */ #include "retr.h" /* for output_stream */ +#include "warc.h" /* for warc_close */ #ifdef TESTING #include "test.h" From 22f016ca3ac1eea102bad274e14caa9ca6f91d56 Mon Sep 17 00:00:00 2001 From: Giuseppe Scrivano Date: Sun, 8 Jul 2012 14:34:16 +0200 Subject: [PATCH 73/75] bootstrap: update from gnulib. --- ChangeLog | 6 + bootstrap | 497 ++++++++++++++++++++++++------------------------ bootstrap.conf | 1 - lib/Makefile.am | 18 -- 4 files changed, 252 insertions(+), 270 deletions(-) delete mode 100644 lib/Makefile.am diff --git a/ChangeLog b/ChangeLog index ef8bbefe..59705fc8 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,9 @@ +2012-07-08 Giuseppe Scrivano + + * bootstrap: Update from gnulib. + * bootstrap.conf (gnulib_extra_files): Remove $build_aux/missing. + * lib/Makefile.am: Delete file. + 2012-06-16 Giuseppe Scrivano * bootstrap.conf (gnulib_modules): Remove `closeout'. diff --git a/bootstrap b/bootstrap index 7cbb5dc4..1d61e5c1 100755 --- a/bootstrap +++ b/bootstrap @@ -1,10 +1,10 @@ #! /bin/sh # Print a version string. -scriptversion=2011-04-05.18; # UTC +scriptversion=2012-07-06.11; # UTC # Bootstrap this package from checked-out sources. -# Copyright (C) 2003-2011 Free Software Foundation, Inc. +# Copyright (C) 2003-2012 Free Software Foundation, Inc. # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -36,12 +36,12 @@ nl=' LC_ALL=C export LC_ALL +# Ensure that CDPATH is not set. Otherwise, the output from cd +# would cause trouble in at least one use below. +(unset CDPATH) >/dev/null 2>&1 && unset CDPATH + local_gl_dir=gl -# Temporary directory names. -bt='._bootmp' -bt_regex=`echo "$bt"| sed 's/\./[.]/g'` -bt2=${bt}2 me=$0 usage() { @@ -77,6 +77,16 @@ Running without arguments will suffice in most cases. EOF } +warn() +{ + for i + do + echo "$i" + done | sed -e "s/^/$me: /" >&2 +} + +die() { warn "$@"; exit 1; } + # Configuration. # Name of the Makefile.am @@ -88,9 +98,12 @@ gnulib_modules= # Any gnulib files needed that are not in modules. gnulib_files= -# A function to be called to edit gnulib.mk right after it's created. +: ${AUTOPOINT=autopoint} +: ${AUTORECONF=autoreconf} + +# A function to be called right after gnulib-tool is run. # Override it via your own definition in bootstrap.conf. -gnulib_mk_hook() { :; } +bootstrap_post_import_hook() { :; } # A function to be called after everything else in this script. # Override it via your own definition in bootstrap.conf. @@ -105,6 +118,11 @@ po_download_command_format=\ "rsync --delete --exclude '*.s1' -Lrtvz \ 'translationproject.org::tp/latest/%s/' '%s'" +# Fallback for downloading .po files (if rsync fails). +po_download_command_format2=\ +"wget --mirror -nd -q -np -A.po -P '%s' \ + http://translationproject.org/latest/%s/" + extract_package_name=' /^AC_INIT(/{ /.*,.*,.*, */{ @@ -122,7 +140,8 @@ extract_package_name=' p } ' -package=`sed -n "$extract_package_name" configure.ac` || exit +package=$(sed -n "$extract_package_name" configure.ac) \ + || die 'cannot find package name in configure.ac' gnulib_name=lib$package build_aux=build-aux @@ -195,19 +214,15 @@ find_tool () else find_tool_error_prefix="\$$find_tool_envvar: " fi - if test x"$find_tool_res" = x; then - echo >&2 "$me: one of these is required: $find_tool_names" - exit 1 - fi - ($find_tool_res --version /dev/null 2>&1 || { - echo >&2 "$me: ${find_tool_error_prefix}cannot run $find_tool_res --version" - exit 1 - } + test x"$find_tool_res" != x \ + || die "one of these is required: $find_tool_names" + ($find_tool_res --version /dev/null 2>&1 \ + || die "${find_tool_error_prefix}cannot run $find_tool_res --version" eval "$find_tool_envvar=\$find_tool_res" eval "export $find_tool_envvar" } -# Find sha1sum, named gsha1sum on MacPorts, and shasum on MacOS 10.6. +# Find sha1sum, named gsha1sum on MacPorts, and shasum on Mac OS X 10.6. find_tool SHA1SUM sha1sum gsha1sum shasum # Override the default configuration, if necessary. @@ -222,7 +237,6 @@ esac test -z "${gnulib_extra_files}" && \ gnulib_extra_files=" $build_aux/install-sh - $build_aux/missing $build_aux/mdate-sh $build_aux/texinfo.tex $build_aux/depcomp @@ -248,7 +262,7 @@ do usage exit;; --gnulib-srcdir=*) - GNULIB_SRCDIR=`expr "X$option" : 'X--gnulib-srcdir=\(.*\)'`;; + GNULIB_SRCDIR=${option#--gnulib-srcdir=};; --skip-po) SKIP_PO=t;; --force) @@ -262,21 +276,15 @@ do --no-git) use_git=false;; *) - echo >&2 "$0: $option: unknown option" - exit 1;; + die "$option: unknown option";; esac done -if $use_git || test -d "$GNULIB_SRCDIR"; then - : -else - echo "$0: Error: --no-git requires --gnulib-srcdir" >&2 - exit 1 -fi +$use_git || test -d "$GNULIB_SRCDIR" \ + || die "Error: --no-git requires --gnulib-srcdir" if test -n "$checkout_only_file" && test ! -r "$checkout_only_file"; then - echo "$0: Bootstrapping from a non-checked-out distribution is risky." >&2 - exit 1 + die "Bootstrapping from a non-checked-out distribution is risky." fi # Ensure that lines starting with ! sort last, per gitignore conventions @@ -290,7 +298,7 @@ sort_patterns() { P x s/^\n// - }' + }' | sed '/^$/d' } # If $STR is not already on a line by itself in $FILE, insert it, @@ -299,10 +307,10 @@ insert_sorted_if_absent() { file=$1 str=$2 test -f $file || touch $file - echo "$str" | sort_patterns - $file | cmp - $file > /dev/null \ + echo "$str" | sort_patterns - $file | cmp -s - $file > /dev/null \ || { echo "$str" | sort_patterns - $file > $file.bak \ && mv $file.bak $file; } \ - || exit 1 + || die "insert_sorted_if_absent $file $str: failed" } # Adjust $PATTERN for $VC_IGNORE_FILE and insert it with @@ -312,10 +320,10 @@ insert_vc_ignore() { pattern="$2" case $vc_ignore_file in *.gitignore) - # A .gitignore entry that does not start with `/' applies - # recursively to subdirectories, so prepend `/' to every + # A .gitignore entry that does not start with '/' applies + # recursively to subdirectories, so prepend '/' to every # .gitignore entry. - pattern=`echo "$pattern" | sed s,^,/,`;; + pattern=$(echo "$pattern" | sed s,^,/,);; esac insert_sorted_if_absent "$vc_ignore_file" "$pattern" } @@ -326,11 +334,9 @@ grep '^[ ]*AC_CONFIG_AUX_DIR(\['"$build_aux"'\])' configure.ac \ >/dev/null && found_aux_dir=yes grep '^[ ]*AC_CONFIG_AUX_DIR('"$build_aux"')' configure.ac \ >/dev/null && found_aux_dir=yes -if test $found_aux_dir = no; then - echo "$0: expected line not found in configure.ac. Add the following:" >&2 - echo " AC_CONFIG_AUX_DIR([$build_aux])" >&2 - exit 1 -fi +test $found_aux_dir = yes \ + || die "expected line not found in configure.ac. Add the following:" \ + " AC_CONFIG_AUX_DIR([$build_aux])" # If $build_aux doesn't exist, create it now, otherwise some bits # below will malfunction. If creating it, also mark it as ignored. @@ -419,20 +425,50 @@ check_versions() { $use_git || continue fi # Honor $APP variables ($TAR, $AUTOCONF, etc.) - appvar=`echo $app | tr '[a-z]-' '[A-Z]_'` + appvar=$(echo $app | LC_ALL=C tr '[a-z]-' '[A-Z]_') test "$appvar" = TAR && appvar=AMTAR - eval "app=\${$appvar-$app}" - inst_ver=$(get_version $app) - if [ ! "$inst_ver" ]; then - echo "$me: Error: '$app' not found" >&2 - ret=1 - elif [ ! "$req_ver" = "-" ]; then - latest_ver=$(sort_ver $req_ver $inst_ver | cut -d' ' -f2) - if [ ! "$latest_ver" = "$inst_ver" ]; then - echo "$me: Error: '$app' version == $inst_ver is too old" >&2 - echo " '$app' version >= $req_ver is required" >&2 + case $appvar in + GZIP) ;; # Do not use $GZIP: it contains gzip options. + *) eval "app=\${$appvar-$app}" ;; + esac + + # Handle the still-experimental Automake-NG programs specially. + # They remain named as the mainstream Automake programs ("automake", + # and "aclocal") to avoid gratuitous incompatibilities with + # pre-existing usages (by, say, autoreconf, or custom autogen.sh + # scripts), but correctly identify themselves (as being part of + # "GNU automake-ng") when asked their version. + case $app in + automake-ng|aclocal-ng) + app=${app%-ng} + ($app --version | grep '(GNU automake-ng)') >/dev/null 2>&1 || { + warn "Error: '$app' not found or not from Automake-NG" + ret=1 + continue + } ;; + esac + if [ "$req_ver" = "-" ]; then + # Merely require app to exist; not all prereq apps are well-behaved + # so we have to rely on $? rather than get_version. + $app --version >/dev/null 2>&1 + if [ 126 -le $? ]; then + warn "Error: '$app' not found" ret=1 fi + else + # Require app to produce a new enough version string. + inst_ver=$(get_version $app) + if [ ! "$inst_ver" ]; then + warn "Error: '$app' not found" + ret=1 + else + latest_ver=$(sort_ver $req_ver $inst_ver | cut -d' ' -f2) + if [ ! "$latest_ver" = "$inst_ver" ]; then + warn "Error: '$app' version == $inst_ver is too old" \ + " '$app' version >= $req_ver is required" + ret=1 + fi + fi fi done @@ -459,14 +495,37 @@ if test $use_libtool = 1; then find_tool LIBTOOLIZE glibtoolize libtoolize fi +# gnulib-tool requires at least automake and autoconf. +# If either is not listed, add it (with minimum version) as a prerequisite. +case $buildreq in + *automake*) ;; + *) buildreq="automake 1.9 +$buildreq" ;; +esac +case $buildreq in + *autoconf*) ;; + *) buildreq="autoconf 2.59 +$buildreq" ;; +esac + +# When we can deduce that gnulib-tool will require patch, +# and when patch is not already listed as a prerequisite, add it, too. +if test -d "$local_gl_dir" \ + && ! find "$local_gl_dir" -name '*.diff' -exec false {} +; then + case $buildreq in + *patch*) ;; + *) buildreq="patch - +$buildreq" ;; + esac +fi + if ! printf "$buildreq" | check_versions; then echo >&2 if test -f README-prereq; then - echo "$0: See README-prereq for how to get the prerequisite programs" >&2 + die "See README-prereq for how to get the prerequisite programs" else - echo "$0: Please install the prerequisite programs" >&2 + die "Please install the prerequisite programs" fi - exit 1 fi echo "$0: Bootstrapping from checked-out $package sources..." @@ -495,7 +554,7 @@ git_modules_config () { test -f .gitmodules && git config --file .gitmodules "$@" } -gnulib_path=`git_modules_config submodule.gnulib.path` +gnulib_path=$(git_modules_config submodule.gnulib.path) test -z "$gnulib_path" && gnulib_path=gnulib # Get gnulib files. @@ -560,7 +619,7 @@ if $bootstrap_sync; then fi gnulib_tool=$GNULIB_SRCDIR/gnulib-tool -<$gnulib_tool || exit +<$gnulib_tool || exit $? # Get translations. @@ -568,7 +627,10 @@ download_po_files() { subdir=$1 domain=$2 echo "$me: getting translations into $subdir for $domain..." - cmd=`printf "$po_download_command_format" "$domain" "$subdir"` + cmd=$(printf "$po_download_command_format" "$domain" "$subdir") + eval "$cmd" && return + # Fallback to HTTP. + cmd=$(printf "$po_download_command_format2" "$subdir" "$domain") eval "$cmd" } @@ -591,7 +653,7 @@ update_po_files() { && ls "$ref_po_dir"/*.po 2>/dev/null | sed 's|.*/||; s|\.po$||' > "$po_dir/LINGUAS" || return - langs=`cd $ref_po_dir && echo *.po|sed 's/\.po//g'` + langs=$(cd $ref_po_dir && echo *.po | sed 's/\.po//g') test "$langs" = '*' && langs=x for po in $langs; do case $po in x) continue;; esac @@ -628,18 +690,18 @@ symlink_to_dir() # If the destination directory doesn't exist, create it. # This is required at least for "lib/uniwidth/cjk.h". - dst_dir=`dirname "$dst"` + dst_dir=$(dirname "$dst") if ! test -d "$dst_dir"; then mkdir -p "$dst_dir" # If we've just created a directory like lib/uniwidth, # tell version control system(s) it's ignorable. # FIXME: for now, this does only one level - parent=`dirname "$dst_dir"` + parent=$(dirname "$dst_dir") for dot_ig in x $vc_ignore; do test $dot_ig = x && continue ig=$parent/$dot_ig - insert_vc_ignore $ig `echo "$dst_dir"|sed 's,.*/,,'` + insert_vc_ignore $ig "${dst_dir##*/}" done fi @@ -656,21 +718,28 @@ symlink_to_dir() cp -fp "$src" "$dst" } else + # Leave any existing symlink alone, if it already points to the source, + # so that broken build tools that care about symlink times + # aren't confused into doing unnecessary builds. Conversely, if the + # existing symlink's time stamp is older than the source, make it afresh, + # so that broken tools aren't confused into skipping needed builds. See + # . test -h "$dst" && - src_ls=`ls -diL "$src" 2>/dev/null` && set $src_ls && src_i=$1 && - dst_ls=`ls -diL "$dst" 2>/dev/null` && set $dst_ls && dst_i=$1 && - test "$src_i" = "$dst_i" || { + src_ls=$(ls -diL "$src" 2>/dev/null) && set $src_ls && src_i=$1 && + dst_ls=$(ls -diL "$dst" 2>/dev/null) && set $dst_ls && dst_i=$1 && + test "$src_i" = "$dst_i" && + both_ls=$(ls -dt "$src" "$dst") && + test "X$both_ls" = "X$dst$nl$src" || { dot_dots= case $src in /*) ;; *) case /$dst/ in *//* | */../* | */./* | /*/*/*/*/*/) - echo >&2 "$me: invalid symlink calculation: $src -> $dst" - exit 1;; - /*/*/*/*/) dot_dots=../../../;; - /*/*/*/) dot_dots=../../;; - /*/*/) dot_dots=../;; + die "invalid symlink calculation: $src -> $dst";; + /*/*/*/*/) dot_dots=../../../;; + /*/*/*/) dot_dots=../../;; + /*/*/) dot_dots=../;; esac;; esac @@ -681,164 +750,94 @@ symlink_to_dir() } } -cp_mark_as_generated() -{ - cp_src=$1 - cp_dst=$2 - - if cmp -s "$cp_src" "$GNULIB_SRCDIR/$cp_dst"; then - symlink_to_dir "$GNULIB_SRCDIR" "$cp_dst" - elif cmp -s "$cp_src" "$local_gl_dir/$cp_dst"; then - symlink_to_dir $local_gl_dir "$cp_dst" - else - case $cp_dst in - *.[ch]) c1='/* '; c2=' */';; - *.texi) c1='@c '; c2= ;; - *.m4|*/Make*|Make*) c1='# ' ; c2= ;; - *) c1= ; c2= ;; - esac - - # If the destination directory doesn't exist, create it. - # This is required at least for "lib/uniwidth/cjk.h". - dst_dir=`dirname "$cp_dst"` - test -d "$dst_dir" || mkdir -p "$dst_dir" - - if test -z "$c1"; then - cmp -s "$cp_src" "$cp_dst" || { - # Copy the file first to get proper permissions if it - # doesn't already exist. Then overwrite the copy. - echo "$me: cp -f $cp_src $cp_dst" && - rm -f "$cp_dst" && - cp "$cp_src" "$cp_dst-t" && - sed "s!$bt_regex/!!g" "$cp_src" > "$cp_dst-t" && - mv -f "$cp_dst-t" "$cp_dst" - } - else - # Copy the file first to get proper permissions if it - # doesn't already exist. Then overwrite the copy. - cp "$cp_src" "$cp_dst-t" && - ( - echo "$c1-*- buffer-read-only: t -*- vi: set ro:$c2" && - echo "${c1}DO NOT EDIT! GENERATED AUTOMATICALLY!$c2" && - sed "s!$bt_regex/!!g" "$cp_src" - ) > $cp_dst-t && - if cmp -s "$cp_dst-t" "$cp_dst"; then - rm -f "$cp_dst-t" - else - echo "$me: cp $cp_src $cp_dst # with edits" && - mv -f "$cp_dst-t" "$cp_dst" - fi - fi - fi -} - version_controlled_file() { - dir=$1 + parent=$1 file=$2 - found=no - if test -d CVS; then - grep -F "/$file/" $dir/CVS/Entries 2>/dev/null | - grep '^/[^/]*/[0-9]' > /dev/null && found=yes - elif test -d .git; then - git rm -n "$dir/$file" > /dev/null 2>&1 && found=yes + if test -d .git; then + git rm -n "$file" > /dev/null 2>&1 elif test -d .svn; then - svn log -r HEAD "$dir/$file" > /dev/null 2>&1 && found=yes + svn log -r HEAD "$file" > /dev/null 2>&1 + elif test -d CVS; then + grep -F "/${file##*/}/" "$parent/CVS/Entries" 2>/dev/null | + grep '^/[^/]*/[0-9]' > /dev/null else - echo "$me: no version control for $dir/$file?" >&2 + warn "no version control for $file?" + false fi - test $found = yes } -slurp() { - for dir in . `(cd $1 && find * -type d -print)`; do - copied= - sep= - for file in `ls -a $1/$dir`; do - case $file in - .|..) continue;; - # FIXME: should all file names starting with "." be ignored? - .*) continue;; - esac - test -d $1/$dir/$file && continue - for excluded_file in $excluded_files; do - test "$dir/$file" = "$excluded_file" && continue 2 +# NOTE: we have to be careful to run both autopoint and libtoolize +# before gnulib-tool, since gnulib-tool is likely to provide newer +# versions of files "installed" by these two programs. +# Then, *after* gnulib-tool (see below), we have to be careful to +# run autoreconf in such a way that it does not run either of these +# two just-pre-run programs. + +# Import from gettext. +with_gettext=yes +grep '^[ ]*AM_GNU_GETTEXT_VERSION(' configure.ac >/dev/null || \ + with_gettext=no + +if test $with_gettext = yes || test $use_libtool = 1; then + + tempbase=.bootstrap$$ + trap "rm -f $tempbase.0 $tempbase.1" 1 2 13 15 + + > $tempbase.0 > $tempbase.1 && + find . ! -type d -print | sort > $tempbase.0 || exit + + if test $with_gettext = yes; then + # Released autopoint has the tendency to install macros that have been + # obsoleted in current gnulib, so run this before gnulib-tool. + echo "$0: $AUTOPOINT --force" + $AUTOPOINT --force || exit + fi + + # Autoreconf runs aclocal before libtoolize, which causes spurious + # warnings if the initial aclocal is confused by the libtoolized + # (or worse out-of-date) macro directory. + # libtoolize 1.9b added the --install option; but we support back + # to libtoolize 1.5.22, where the install action was default. + if test $use_libtool = 1; then + install= + case $($LIBTOOLIZE --help) in + *--install*) install=--install ;; + esac + echo "running: $LIBTOOLIZE $install --copy" + $LIBTOOLIZE $install --copy + fi + + find . ! -type d -print | sort >$tempbase.1 + old_IFS=$IFS + IFS=$nl + for file in $(comm -13 $tempbase.0 $tempbase.1); do + IFS=$old_IFS + parent=${file%/*} + version_controlled_file "$parent" "$file" || { + for dot_ig in x $vc_ignore; do + test $dot_ig = x && continue + ig=$parent/$dot_ig + insert_vc_ignore "$ig" "${file##*/}" done - if test $file = Makefile.am && test "X$gnulib_mk" != XMakefile.am; then - copied=$copied${sep}$gnulib_mk; sep=$nl - remove_intl='/^[^#].*\/intl/s/^/#/;'"s!$bt_regex/!!g" - sed "$remove_intl" $1/$dir/$file | - cmp - $dir/$gnulib_mk > /dev/null || { - echo "$me: Copying $1/$dir/$file to $dir/$gnulib_mk ..." && - rm -f $dir/$gnulib_mk && - sed "$remove_intl" $1/$dir/$file >$dir/$gnulib_mk && - gnulib_mk_hook $dir/$gnulib_mk - } - elif { test "${2+set}" = set && test -r $2/$dir/$file; } || - version_controlled_file $dir $file; then - echo "$me: $dir/$file overrides $1/$dir/$file" - else - copied=$copied$sep$file; sep=$nl - if test $file = gettext.m4; then - echo "$me: patching m4/gettext.m4 to remove need for intl/* ..." - rm -f $dir/$file - sed ' - /^AC_DEFUN(\[AM_INTL_SUBDIR],/,/^]/c\ - AC_DEFUN([AM_INTL_SUBDIR], []) - /^AC_DEFUN(\[gt_INTL_SUBDIR_CORE],/,/^]/c\ - AC_DEFUN([gt_INTL_SUBDIR_CORE], []) - $a\ - AC_DEFUN([gl_LOCK_EARLY], []) - ' $1/$dir/$file >$dir/$file - else - cp_mark_as_generated $1/$dir/$file $dir/$file - fi - fi || exit - done - - for dot_ig in x $vc_ignore; do - test $dot_ig = x && continue - ig=$dir/$dot_ig - if test -n "$copied"; then - insert_vc_ignore $ig "$copied" - # If an ignored file name ends with .in.h, then also add - # the name with just ".h". Many gnulib headers are generated, - # e.g., stdint.in.h -> stdint.h, dirent.in.h ->..., etc. - # Likewise for .gperf -> .h, .y -> .c, and .sin -> .sed - f=`echo "$copied" | - sed ' - s/\.in\.h$/.h/ - s/\.sin$/.sed/ - s/\.y$/.c/ - s/\.gperf$/.h/ - ' - ` - insert_vc_ignore $ig "$f" - - # For files like sys_stat.in.h and sys_time.in.h, record as - # ignorable the directory we might eventually create: sys/. - f=`echo "$copied"|sed 's/sys_.*\.in\.h$/sys/'` - insert_vc_ignore $ig "$f" - fi - done + } done -} + IFS=$old_IFS - -# Create boot temporary directories to import from gnulib and gettext. -rm -fr $bt $bt2 && -mkdir $bt $bt2 || exit + rm -f $tempbase.0 $tempbase.1 + trap - 1 2 13 15 +fi # Import from gnulib. gnulib_tool_options="\ --import\ --no-changelog\ - --aux-dir $bt/$build_aux\ - --doc-base $bt/$doc_base\ + --aux-dir $build_aux\ + --doc-base $doc_base\ --lib $gnulib_name\ - --m4-base $bt/$m4_base/\ - --source-base $bt/$source_base/\ - --tests-base $bt/$tests_base\ + --m4-base $m4_base/\ + --source-base $source_base/\ + --tests-base $tests_base\ --local-dir $local_gl_dir\ $gnulib_tool_option_extras\ " @@ -850,25 +849,14 @@ if test $use_libtool = 1; then fi echo "$0: $gnulib_tool $gnulib_tool_options --import ..." $gnulib_tool $gnulib_tool_options --import $gnulib_modules && -slurp $bt || exit for file in $gnulib_files; do - symlink_to_dir "$GNULIB_SRCDIR" $file || exit + symlink_to_dir "$GNULIB_SRCDIR" $file \ + || die "failed to symlink $file" done - -# Import from gettext. -with_gettext=yes -grep '^[ ]*AM_GNU_GETTEXT_VERSION(' configure.ac >/dev/null || \ - with_gettext=no - -if test $with_gettext = yes; then - echo "$0: (cd $bt2; ${AUTOPOINT-autopoint}) ..." - cp configure.ac $bt2 && - (cd $bt2 && ${AUTOPOINT-autopoint} && rm configure.ac) && - slurp $bt2 $bt || exit -fi -rm -fr $bt $bt2 || exit +bootstrap_post_import_hook \ + || die "bootstrap_post_import_hook failed" # Remove any dangling symlink matching "*.m4" or "*.[ch]" in some # gnulib-populated directories. Such .m4 files would cause aclocal to fail. @@ -882,37 +870,31 @@ find "$m4_base" "$source_base" \ -depth \( -name '*.m4' -o -name '*.[ch]' \) \ -type l -xtype l -delete > /dev/null 2>&1 -# Reconfigure, getting other files. - -# Skip autoheader if it's not needed. -grep -E '^[ ]*AC_CONFIG_HEADERS?\>' configure.ac >/dev/null || - AUTOHEADER=true - -for command in \ - libtool \ - "${ACLOCAL-aclocal} --force -I '$m4_base' $ACLOCAL_FLAGS" \ - "${AUTOCONF-autoconf} --force" \ - "${AUTOHEADER-autoheader} --force" \ - "${AUTOMAKE-automake} --add-missing --copy --force-missing" -do - if test "$command" = libtool; then - test $use_libtool = 0 \ - && continue - command="${LIBTOOLIZE-libtoolize} -c -f" - fi - echo "$0: $command ..." - eval "$command" || exit -done +# Some systems (RHEL 5) are using ancient autotools, for which the +# --no-recursive option had not been invented. Detect that lack and +# omit the option when it's not supported. FIXME in 2017: remove this +# hack when RHEL 5 autotools are updated, or when they become irrelevant. +no_recursive= +case $($AUTORECONF --help) in + *--no-recursive*) no_recursive=--no-recursive;; +esac +# Tell autoreconf not to invoke autopoint or libtoolize; they were run above. +echo "running: AUTOPOINT=true LIBTOOLIZE=true " \ + "$AUTORECONF --verbose --install $no_recursive -I $m4_base $ACLOCAL_FLAGS" +AUTOPOINT=true LIBTOOLIZE=true \ + $AUTORECONF --verbose --install $no_recursive -I $m4_base $ACLOCAL_FLAGS \ + || die "autoreconf failed" # Get some extra files from gnulib, overriding existing files. for file in $gnulib_extra_files; do case $file in */INSTALL) dst=INSTALL;; - build-aux/*) dst=$build_aux/`expr "$file" : 'build-aux/\(.*\)'`;; + build-aux/*) dst=$build_aux/${file#build-aux/};; *) dst=$file;; esac - symlink_to_dir "$GNULIB_SRCDIR" $file $dst || exit + symlink_to_dir "$GNULIB_SRCDIR" $file $dst \ + || die "failed to symlink $file" done if test $with_gettext = yes; then @@ -928,7 +910,19 @@ if test $with_gettext = yes; then a\ '"$XGETTEXT_OPTIONS"' $${end_of_xgettext_options+} } - ' po/Makevars.template >po/Makevars || exit 1 + ' po/Makevars.template >po/Makevars \ + || die 'cannot generate po/Makevars' + + # If the 'gettext' module is in use, grab the latest Makefile.in.in. + # If only the 'gettext-h' module is in use, assume autopoint already + # put the correct version of this file into place. + case $gnulib_modules in + *gettext-h*) ;; + *gettext*) + cp $GNULIB_SRCDIR/build-aux/po/Makefile.in.in po/Makefile.in.in \ + || die "cannot create po/Makefile.in.in" + ;; + esac if test -d runtime-po; then # Similarly for runtime-po/Makevars, but not quite the same. @@ -942,7 +936,8 @@ if test $with_gettext = yes; then a\ '"$XGETTEXT_OPTIONS_RUNTIME"' $${end_of_xgettext_options+} } - ' po/Makevars.template >runtime-po/Makevars || exit 1 + ' po/Makevars.template >runtime-po/Makevars \ + || die 'cannot generate runtime-po/Makevars' # Copy identical files from po to runtime-po. (cd po && cp -p Makefile.in.in *-quot *.header *.sed *.sin ../runtime-po) diff --git a/bootstrap.conf b/bootstrap.conf index febd11da..efb1bc21 100644 --- a/bootstrap.conf +++ b/bootstrap.conf @@ -80,7 +80,6 @@ write gnulib_extra_files=" $build_aux/install-sh - $build_aux/missing $build_aux/mdate-sh $build_aux/texinfo.tex $build_aux/depcomp diff --git a/lib/Makefile.am b/lib/Makefile.am deleted file mode 100644 index 1aaa839a..00000000 --- a/lib/Makefile.am +++ /dev/null @@ -1,18 +0,0 @@ -# GNU Wget - -# Copyright (C) 2010, 2011 Free Software Foundation, Inc. - -## This program is free software: you can redistribute it and/or modify -## it under the terms of the GNU General Public License as published by -## the Free Software Foundation, either version 3 of the License, or -## (at your option) any later version. -## -## This program is distributed in the hope that it will be useful, -## but WITHOUT ANY WARRANTY -## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -## GNU General Public License for more details. -## -## You should have received a copy of the GNU General Public License -## along with this program. If not, see . - -include gnulib.mk From 3db55372c70baca71c99d292d99bfecd21e89b33 Mon Sep 17 00:00:00 2001 From: mancha Date: Thu, 2 Aug 2012 15:50:40 +0200 Subject: [PATCH 74/75] doc: add ENVIRONMENT section to manpage and minor adjustments. Signed-off-by: mancha --- doc/ChangeLog | 4 ++++ doc/wget.texi | 12 +++++++----- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/doc/ChangeLog b/doc/ChangeLog index f3af7ea3..df76c53e 100644 --- a/doc/ChangeLog +++ b/doc/ChangeLog @@ -1,3 +1,7 @@ +2012-08-04 mancha (tiny change) + + * wget.texi: Export ENVIRONMENT to the man page. + 2012-06-09 Giuseppe Scrivano * wget.texi (Logging and Input File Options): Document "--report-speed". diff --git a/doc/wget.texi b/doc/wget.texi index 73341ec6..7efdc725 100644 --- a/doc/wget.texi +++ b/doc/wget.texi @@ -3576,28 +3576,30 @@ internal networks from the rest of Internet. In order to obtain information from the Web, their users connect and retrieve remote data using an authorized proxy. +@c man begin ENVIRONMENT Wget supports proxies for both @sc{http} and @sc{ftp} retrievals. The standard way to specify proxy location, which Wget recognizes, is using the following environment variables: -@table @code +@table @env @item http_proxy @itemx https_proxy -If set, the @code{http_proxy} and @code{https_proxy} variables should +If set, the @env{http_proxy} and @env{https_proxy} variables should contain the @sc{url}s of the proxies for @sc{http} and @sc{https} connections respectively. @item ftp_proxy This variable should contain the @sc{url} of the proxy for @sc{ftp} -connections. It is quite common that @code{http_proxy} and -@code{ftp_proxy} are set to the same @sc{url}. +connections. It is quite common that @env{http_proxy} and +@env{ftp_proxy} are set to the same @sc{url}. @item no_proxy This variable should contain a comma-separated list of domain extensions proxy should @emph{not} be used for. For instance, if the value of -@code{no_proxy} is @samp{.mit.edu}, proxy will not be used to retrieve +@env{no_proxy} is @samp{.mit.edu}, proxy will not be used to retrieve documents from MIT. @end table +@c man end In addition to the environment variables, proxy location and settings may be specified from within Wget itself. From e1df67a4f09d4fa86729e71b6e6ceb53c4f0b52b Mon Sep 17 00:00:00 2001 From: Giuseppe Scrivano Date: Sun, 5 Aug 2012 22:14:30 +0200 Subject: [PATCH 75/75] Prepare the new release --- NEWS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NEWS b/NEWS index 25c401e5..6ee857c1 100644 --- a/NEWS +++ b/NEWS @@ -6,7 +6,7 @@ See the end for copying conditions. Please send GNU Wget bug reports to . -* Changes in Wget X.Y.Z +* Changes in Wget 1.14 ** Add support for content-on-error. It allows to store the HTTP payload on 4xx or 5xx errors.