NEWS: add other items and tag 1.16.2

* warc.c (windows_uuid_str) [WINDOWS]: New function specific to
MS-Windows.
(warc_uuid_str) [WINDOWS]: If windows_uuid_str succeeds, use its
result; otherwise use the fallback method.

.gitignore

@@ -1,19 +1,7 @@
 # Project Root
-# Wildcard Ignores
-# build-aux/
-# doc/
-# m4/
-# others
-# po/
-# src/
-# testenv/
-# tests/
-# util/
-*~
 /.sc-start-sc_bindtextdomain
 /.sc-start-sc_prohibit_HAVE_MBRTOWC
 /.version
-/ABOUT-NLS~
 /GNUmakefile
 /INSTALL
 /Makefile
@@ -24,10 +12,18 @@
 /config.log
 /config.status
 /configure
-/cscope.out
 /lib/
 /maint.mk
-/tags
+# Wildcard Ignores
+*~
+po/*.gmo*
+po/*.po*
+src/*.o
+tests/*.log
+tests/*.trs
+testenv/*.log
+testenv/*.trs
+# build-aux/
 build-aux/.gitignore
 build-aux/compile
 build-aux/config.guess
@@ -39,6 +35,7 @@ build-aux/missing
 build-aux/snippet/
 build-aux/test-driver
 build-aux/texinfo.tex
+# doc/
 doc/Makefile
 doc/Makefile.in
 doc/sample.wgetrc.munged_for_texi_inclusion
@@ -47,10 +44,10 @@ doc/version.texi
 doc/wget.1
 doc/wget.info
 doc/wget.pod
+# m4/
 m4/.gitignore
 m4/gnulib-cache.m4
-po/*.gmo*
-po/*.po*
+# po/
 po/.gitignore
 po/.reference/
 po/LINGUAS
@@ -59,7 +56,8 @@ po/Makefile.in
 po/Makevars
 po/POTFILES
 po/wget.pot
-src/*.o
+po/stamp-po
+# src/
 src/.deps/
 src/Makefile
 src/Makefile.in
@@ -74,20 +72,22 @@ src/stamp-h1
 src/tags
 src/version.c
 src/wget
-testenv/*.log
-testenv/*.trs
-testenv/Makefile
-testenv/Makefile.in
+# testenv/
 testenv/conf/__pycache__/
 testenv/exc/__pycache__/
 testenv/misc/__pycache__/
 testenv/server/__pycache__/
 testenv/server/http/__pycache__/
 testenv/test/__pycache__/
-tests/*.log
-tests/*.trs
+testenv/Makefile
+testenv/Makefile.in
+# tests/
 tests/Makefile
 tests/Makefile.in
 tests/unit-tests
+# util/
 util/Makefile
 util/Makefile.in
+# others
+/tags
+/cscope.out

ChangeLog.README

@@ -1,16 +0,0 @@
-Please note that Wget has more than one ChangeLog file:
-
-    ./ChangeLog: documents changes to files in the top-level directory
-                 and to files in subdirectories like po/ that don't have
-                 their own ChangeLogs
-
-    src/ChangeLog: documents only changes to files in the src directory
-
-    doc/ChangeLog: documents only changes to files in the doc directory
-
-    windows/ChangeLog: documents only changes to files in the windows directory
-
-    msdos/ChangeLog: documents only changes to files in the msdos directory
-
-When checking to see if a patch you sent in has been applied, please
-look in the appropriate ChangeLog(s).

Makefile.am

@@ -27,7 +27,7 @@
 # as that of the covered work.
 
 #
-# Version: $(VERSION)
+# Version: @VERSION@
 #
 
 
@@ -43,8 +43,8 @@ ACLOCAL_AMFLAGS = -I m4
 # subdirectories in the distribution
 SUBDIRS = lib src doc po tests util testenv
 
-EXTRA_DIST = ChangeLog.README MAILING-LIST \
-             msdos/ChangeLog msdos/config.h msdos/Makefile.DJ \
+EXTRA_DIST = MAILING-LIST \
+             msdos/config.h msdos/Makefile.DJ \
              msdos/Makefile.WC ABOUT-NLS \
              build-aux/build_info.pl build-aux/git-version-gen .version
 
@@ -60,5 +60,20 @@ clean-generic:
 
 # Arrange so that .tarball-version appears only in the distribution
 # tarball, and never in a checked-out repository.
-dist-hook:
+dist-hook: gen-ChangeLog
 	$(AM_V_GEN)echo $(VERSION) > $(distdir)/.tarball-version
+
+gen_start_date = 2014-12-10
+.PHONY: gen-ChangeLog
+gen-ChangeLog:
+	$(AM_V_GEN)if test -d .git; then				\
+	  log_fix="$(srcdir)/build-aux/git-log-fix";			\
+	  test -e "$$log_fix"						\
+	    || amend_git_log=;						\
+	  $(top_srcdir)/build-aux/gitlog-to-changelog			\
+	    $$amend_git_log --since=$(gen_start_date) > $(distdir)/cl-t && \
+            echo >>  $(distdir)/cl-t &&                                 \
+            cat ChangeLog-2014-12-10 >>  $(distdir)/cl-t &&             \
+	    { rm -f $(distdir)/ChangeLog &&				\
+	      mv $(distdir)/cl-t $(distdir)/ChangeLog; }		\
+	fi

NEWS

@@ -1,11 +1,50 @@
 GNU Wget NEWS -- history of user-visible changes.
 
 Copyright (C) 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004, 2005,
-2006, 2007, 2008, 2009, 2010, 2011, 2012 Free Software Foundation, Inc.
+2006, 2007, 2008, 2009, 2010, 2011, 2012, 2013, 2014 Free Software
+Foundation, Inc.
 See the end for copying conditions.
 
 Please send GNU Wget bug reports to <bug-wget@gnu.org>.
-
+
+* Changes in Wget 1.16.2
+
+** Native uuid generation on Windows
+
+** Fix build on Solaris
+
+** Allow progress bar on stderr when -o is used
+
+** Accept 5-digit port numbers in FTP EPSV responses.
+
+** Support older versions of flex.
+
+** Updated translations.
+
+* Changes in Wget 1.16.1
+
+** Add --enable-assert configure option.
+
+** Use pkg-config to check for libraries presence.
+
+** Do not limit --secure-protocol=auto|pfs to TLSv1.0.
+
+** Add --secure-protocol=TLSv1_1|TLSv1_2 .
+
+** Full C89 source code compliance.
+
+** Select and use the most secure authentication scheme with HTTP connections.
+
+** Fix issues with turkish locales.
+
+** Handle 504 Gateway Timeout.
+
+** New option --crl-file to load Certificate Revocation Lists.
+
+** Add valgrind support to tests suite.
+
+** Fix an off-by-one problem in the progress bar (introduced in 1.16).
+
 * Changes in Wget 1.16
 
 ** No longer create local symbolic links by default.  Closes CVE-2014-4877.
@@ -21,7 +60,7 @@ Please send GNU Wget bug reports to <bug-wget@gnu.org>.
 ** Introduce --start-pos to allow starting downloads from a specified position.
 
 ** Fix a problem with ISA Server Proxy and keep-alive connections.
-
+
 * Changes in Wget 1.15
 
 ** Add support for --method.
@@ -53,7 +92,7 @@ Please send GNU Wget bug reports to <bug-wget@gnu.org>.
 ** Specify Host with the HTTP CONNECT method.
 
 ** Use the correct HTTP method on a redirection.
-
+
 * Changes in Wget 1.14
 
 ** Add support for content-on-error.  It allows to store the HTTP
@@ -80,7 +119,7 @@ Please send GNU Wget bug reports to <bug-wget@gnu.org>.
 ** The GNU TLS backend honors correctly the timeout value.
 
 ** Add support for RFC 2617 Digest Access Authentication.
-
+
 * Changes in Wget 1.13.4
 
 ** Now --version and --help work again.
@@ -93,7 +132,7 @@ Please send GNU Wget bug reports to <bug-wget@gnu.org>.
    is specified.
 
 ** Fix a segfault on an incomplete STYLE tag.
-
+
 * Changes in Wget 1.13.3
 
 ** Support HTTP/1.1
@@ -159,7 +198,7 @@ Please send GNU Wget bug reports to <bug-wget@gnu.org>.
 
 ** Introduce `show_all_dns_entries' to print all IP addresses corresponding to
    a DNS name when it is resolved.
-
+
 * Changes in Wget 1.12
 
 ** Mailing list MOVED to bug-wget@gnu.org
@@ -213,7 +252,7 @@ forces the percent-encoding of all non-ASCII bytes
 now documented: save_headers, spider, and user_agent,
 auth_no_challenge, and keep_session_cookies. Also added documentation
 for the "lowercase" and "uppercase" values for --restrict-file-names, which had been present since Wget 1.11.
-
+
 * Changes in Wget 1.11.4
 
 ** Fixed an issue (apparently a regression) where -O would refuse to
@@ -227,13 +266,13 @@ locally with content.
 to a pointer-to-time_t.
 
 ** Translation updates for Catalan.
-
+
 * Changes in Wget 1.11.3
 
 ** Downgraded -N with -O to a warning, rather than an error.
 
 ** Translation updates
-
+
 * Changes in Wget 1.11.2
 
 ** Fixed a problem in authenticating over HTTPS through a proxy.
@@ -255,7 +294,7 @@ multiple times. This has been fixed in 1.11.2.
 for improved conformance with RFC 3986. However, this behavior presents
 problems for some FTP setups, and so they are now preserved again, for
 FTP URLs only.
-
+
 * Changes in Wget 1.11.1.
 
 ** Interrupted downloads no longer result in renaming the file
@@ -277,7 +316,7 @@ be surprising, and notes that they may change in the future.
 
 ** Documentation of --no-parents now explains how a trailing slash, or
 lack thereof, in the specified URL, will affect behavior.
-
+
 * Changes in Wget 1.11.
 
 ** Timestamping now uses the value from the most recent HTTP response,
@@ -323,7 +362,7 @@ available at http://hg.addictivecode.org/. Prior to this, the source
 code was hosted on Subversion (migrated from the original CVS); you can
 still get access to older tags and branches for Wget in the Subversion
 repository at http://addictivecode.org/svn/wget/.
-
+
 * Changes in Wget 1.10.
 
 ** Downloading files larger than 2GB, sometimes referred to as "large
@@ -434,9 +473,9 @@ http_password and proxy_password respectively.  The login and passwd
 .wgetrc commands have been deprecated.
 
 * `wget -b' now works correctly under Windows.
-
+
 * Wget 1.9.1 is a bugfix release with no user-visible changes.
-
+
 * Changes in Wget 1.9.
 
 ** It is now possible to specify that POST method be used for HTTP
@@ -488,11 +527,11 @@ values "yes" and "no" along with the traditional "on" and "off".
 ** It is now possible to specify decimal values for timeouts, waiting
 periods, and download rate.  For instance, `--wait=0.5' now works as
 expected, as does `--dns-timeout=0.5' and even `--limit-rate=2.5k'.
-
+
 * Wget 1.8.2 is a bugfix release with no user-visible changes.
-
+
 * Wget 1.8.1 is a bugfix release with no user-visible changes.
-
+
 * Changes in Wget 1.8.
 
 ** A new progress indicator is now available and used by default.
@@ -552,9 +591,9 @@ no longer translated to `?', nor `%2B' to `+'.  Unsafe characters
 which are not reserved are still escaped, of course.
 
 ** No more than 20 successive redirections are allowed.
-
+
 * Wget 1.7.1 is a bugfix release with no user-visible changes.
-
+
 * Changes in Wget 1.7.
 
 ** SSL (`https') pages now work if you compile Wget with SSL support;
@@ -596,7 +635,7 @@ large sites (thousands of documents).
 documentation.  (The last version that shipped with a man page was
 1.4.5).  To get this, you need to have pod2man from the Perl
 distribution installed on your system.
-
+
 * Changes in Wget 1.6
 
 ** Administrative changes.
@@ -647,13 +686,13 @@ passive_ftp is the only .wgetrc command which takes a lockable Boolean.
 bound to multiple IP addresses.
 
 ** wget now accepts (illegal per HTTP spec) relative URLs in HTTP redirects.
-
+
 * Wget 1.5.3 is a bugfix release with no user-visible changes.
-
+
 * Wget 1.5.2 is a bugfix release with no user-visible changes.
-
+
 * Wget 1.5.1 is a bugfix release with no user-visible changes.
-
+
 * Changes in Wget 1.5.0
 
 ** Wget speaks many languages!
@@ -699,11 +738,11 @@ download.
 
 ** The manual page, now hopelessly out of date, is no longer
 distributed with Wget.
-
+
 * Wget 1.4.5 is a bugfix release with no user-visible changes.
-
+
 * Wget 1.4.4 is a bugfix release with no user-visible changes.
-
+
 * Changes in Wget 1.4.3
 
 ** Wget is now a GNU utility.
@@ -719,7 +758,7 @@ distributed with Wget.
 ** Global wgetrc now goes to /usr/local/etc (i.e. $sysconfdir).
 
 ** Lots of bugfixes.
-
+
 * Changes in Wget 1.4.2
 
 ** New mirror site at ftp://sunsite.auc.dk/pub/infosystems/wget/,
@@ -737,7 +776,7 @@ files.
 ** --convert-links should work now.
 
 ** Minor bugfixes.
-
+
 * Changes in Wget 1.4.1
 
 ** Minor bugfixes.
@@ -745,7 +784,7 @@ files.
 ** Added -I (the opposite of -X).
 
 ** Dot tracing is now customizable; try wget --dot-style=binary
-
+
 * Changes in Wget 1.4.0
 
 ** Wget 1.4.0 [formerly known as Geturl] is an extensive rewrite of
@@ -826,7 +865,7 @@ turn on mirroring options).
 ** Lots of small features I can't remember. :-)
 
 ** A host of bugfixes.
-
+
 * Changes in Geturl 1.3
 
 ** Added FTP globbing support (ftp://fly.cc.fer.hr/*)
@@ -844,9 +883,9 @@ turn on mirroring options).
 ** Added support for robots.txt
 
 ** Fixed some minor bugs
-
+
 * Geturl 1.2 is a bugfix release with no user-visible changes.
-
+
 * Changes in Geturl 1.1
 
 ** REST supported in FTP
@@ -864,7 +903,7 @@ geturl -vo log http://fly.cc.fer.hr/
 ** <base href="xxx"> supported
 
 ** autoconf supported
-
+
 ----------------------------------------------------------------------
 Copyright information:
 

README.checkout

@@ -4,8 +4,8 @@ Compiling From Repository Sources
    does not contain automatically-generated files, even when these are
    normally present in the distribution tarballs. Therefore, to build GNU
    Wget from the sources in the repository, you'll need to have one or
-   more of the following (note that gettext, OpenSSL, libidn, libiconv
-   and libpsl are not absolutely required):
+   more of the following (note that gettext, OpenSSL, GnuTLS, libidn,
+   libiconv, libpsl, libpcre and pkg-config are not absolutely required):
 
      * [20]autoconf (currently, GNU Wget requires version 2.61). This is
        needed to generate the configure script from configure.in. This is
@@ -24,11 +24,11 @@ Compiling From Repository Sources
        tests in the tests/ sub directory. Tarball distributions include an
        already-generated wget.1 manual. The command "make check" runs the
        test suite written in perl. To execute all the tests you need
-       [24]libwww-perl perl library. If "perl -MCPAN -e 'install
-       Bundle::LWP'" fails then you most likely don't have cpan module
-       installed. First download [25]CPAN and [26]install it. Then execute
-       "perl -MCPAN -e 'install Bundle::LWP'". Now "make check" should
-       pass most of the tests in the test suite.
+       [24]libwww-perl and libio-socket-ssl-perl perl library. If
+       "perl -MCPAN -e 'install Bundle::LWP'" fails then you most likely don't
+       have cpan module installed. First download [25]CPAN and [26]install it.
+       Then execute "perl -MCPAN -e 'install Bundle::LWP'".
+       Now "make check" should pass most of the tests in the test suite.
 
      * [27]texinfo in order to generate Info, PostScript and/or HTML
        documentation. You don't need texinfo in order to generate the
@@ -46,35 +46,54 @@ Compiling From Repository Sources
        to export M4=<new m4 path>, to be sure that autoconf/automake use
        it instead of the old one.
 
-     * [29]OpenSSL to handle encrypted web sessions. You almost certainly
-       want this. It is not enough to have the library installed; you need
-       to have the header files available to compile against as well. For
-       GNU/Linux systems, this usually means installing a "developer"
-       package for openssl (for example, [30]Debian systems require
-       installation of the libssl-dev package). Alternatively, you can
-       compile GNU Wget against the [31]GNUTLS library instead, by
-       specifying --with-ssl=gnutls to the ./configure script; however,
-       building against GNUTLS is currently experimental, and not
-       officially supported. If you have OpenSSL and its development
-       headers installed, but don't want to build Wget with it, specify
-       --without-ssl to the ./configure script.
+     * [29]GnuTLS to allow encrypted data transfer (HTTPS).
+       You need the header files and the library installed.
+       As an alternative, you can use [30]OpenSSL by specifying
+       --with-ssl=openssl to the ./configure script.
+       If you do not want HTTPS support, specify --without-ssl to the
+       ./configure script.
+       If you want to compile+link a non-system library version use
+       --with-libgnutls-prefix (or if having pkg-config: see description
+       below).
 
-     * [32]libidn is required for IDN/IRI support (non-ASCII characters
+     * [30]OpenSSL to allow encrypted data transfer (HTTPS) an
+       alternative to [31]GnuTLS.
+       You need the header files and the library installed.
+       If you want to compile+link a non-system library version use
+       --with-libssl-prefix (or if having pkg-config: see description below).
+
+     * [31]libidn is required for IDN/IRI support (non-ASCII characters
        within what would otherwise be URLs).
 
-     * [33]libiconv is required on non-GNU systems, for IDN/IRI support.
+     * [32]libiconv is required on non-GNU systems, for IDN/IRI support.
        On GNU systems, the functionality provided by libiconv is already
        present in the system libraries.
 
-     * [34]git is used to fetch gnulib files trough the bootstrap.sh script.
+     * [33]git is used to fetch gnulib files trough the bootstrap.sh script.
+
+     * [34]libpsl is required for using a public suffix list to check for valid
+       cookie domains. You need the header files and the library installed.
+
+     * [35]libpcre is required for using Perl-compatible regular expressions
+       with --accept-regex and --reject-regex. You need the header files and
+       the library installed to compile and link Wget with PCRE support.
+
+     * [36]pkg-config helps the ./configure script to find installed libraries.
+       Most libraries provide a pkg-config file (.pc extension) with
+       information about dependencies, header file and library locations.
+       Distributions deliver their specific .pc file to each library.
+       If you want to compile+link against your own library version, make a
+       copy of the appropriate .pc file and amend it to your needs (e.g. edit
+       the line starting with prefix=). Before you execute the ./configure
+       script, set (and export) PKG_CONFIG_PATH to the directory where you
+       saved the .pc file. Example:
+       $ PKG_CONFIG_PATH="." ./configure
 
-     * [35]libpsl is required for using a public suffix list to check for valid
-       cookie domains.
 
    For those who might be confused as to what to do once they check out
    the source code, considering configure and Makefile do not yet exist at
-   that point, a shell script called bootstrap.sh has been provided. After
-   calling ./bootstrap.sh you're ready to build GNU Wget in the normal
+   that point, a shell script called bootstrap has been provided. After
+   calling ./bootstrap you're ready to build GNU Wget in the normal
    fashion, with ./configure and make.
 
    So, to sum up, after checking out the source code as described above,
@@ -83,15 +102,23 @@ Compiling From Repository Sources
  $  cd wget        # assumes you've cloned a repository to "./wget"
     2. Generate all the automatically-generated files required prior to
        configuring the package:
- $  ./bootstrap.sh
+ $  ./bootstrap
     3. Configure the package and compile it:
- $  ./configure [some_parameters]
+ $  ./configure --enable-assert [some_parameters]
  $  make
     4. Hack, compile, test, hack, compile, test...
  $  src/wget --version
  GNU Wget 1.12-devel (9cb2563197bc)
 
- Copyright © 2008,2010 Free Software Foundation, Inc.
+ All developers are requested to enable the assertions on their development
+ builds to ensure a stable codebase. Assertions are added to state certain
+ assumptions about the code and its data which all developers should be mindful
+ of. To enable assertions, run the configure command with the --enable-assert
+ option, like this:
+
+ $ ./configure --enable-assert [other configure options]
+
+ Copyright (C) 2008,2010,2014,2015 Free Software Foundation, Inc.
  License GPLv3+: GNU GPL version 3 or later
  <http://www.gnu.org/licenses/gpl.html>.
  This is free software: you are free to change and redistribute it.
@@ -112,8 +139,8 @@ Compiling From Repository Sources
      * [43]Valid HTML 4.01
      __________________________________________________________________
 
-   All content © 2007 Free Software Foundation. For terms of use,
-   redistribution, and modification, please see the [44]WikiLicense page.
+   All content (C) 2007 Free Software Foundation. For terms of use,
+   redistribution, and modification, please see the [45]WikiLicense page.
 
 References
 
@@ -126,13 +153,13 @@ References
   26. http://apthorpe.cynistar.net/code/configuring_cpan.html
   27. http://www.gnu.org/software/texinfo/
   28. http://www.gnu.org/software/gettext/
-  29. http://www.openssl.org/
-  30. http://www.debian.org/
-  31. http://www.gnu.org/software/gnutls/
-  32. http://www.gnu.org/software/libidn/
-  33. http://www.gnu.org/software/libiconv/
-  34. http://git-scm.com/
-  35. https://github.com/rockdaboot/libpsl
+  29. http://www.gnu.org/software/gnutls/
+  30. http://www.openssl.org/
+  31. http://www.gnu.org/software/libidn/
+  32. http://www.gnu.org/software/libiconv/
+  33. http://git-scm.com/
+  34. https://github.com/rockdaboot/libpsl
+  35. http://www.pcre.org
   36. http://wget.addictivecode.org/CompilingRepoSources?action=edit&editor=text
   37. http://wget.addictivecode.org/CompilingRepoSources
   38. http://wget.addictivecode.org/CompilingRepoSources?action=info

bootstrap

@@ -1,10 +1,10 @@
 #! /bin/sh
 # Print a version string.
-scriptversion=2012-07-19.14; # UTC
+scriptversion=2013-12-05.23; # UTC
 
 # Bootstrap this package from checked-out sources.
 
-# Copyright (C) 2003-2012 Free Software Foundation, Inc.
+# Copyright (C) 2003-2014 Free Software Foundation, Inc.
 
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -140,20 +140,21 @@ po_download_command_format2=\
 "wget --mirror -nd -q -np -A.po -P '%s' \
  http://translationproject.org/latest/%s/"
 
+# Prefer a non-empty tarname (4th argument of AC_INIT if given), else
+# fall back to the package name (1st argument with munging)
 extract_package_name='
-  /^AC_INIT(/{
-     /.*,.*,.*, */{
-       s///
-       s/[][]//g
-       s/)$//
+  /^AC_INIT(\[*/{
+     s///
+     /^[^,]*,[^,]*,[^,]*,[ []*\([^][ ,)]\)/{
+       s//\1/
+       s/[],)].*//
        p
        q
      }
-     s/AC_INIT(\[*//
-     s/]*,.*//
+     s/[],)].*//
      s/^GNU //
      y/ABCDEFGHIJKLMNOPQRSTUVWXYZ/abcdefghijklmnopqrstuvwxyz/
-     s/[^A-Za-z0-9_]/-/g
+     s/[^abcdefghijklmnopqrstuvwxyz0123456789_]/-/g
      p
   }
 '
@@ -208,12 +209,26 @@ bootstrap_sync=false
 # Use git to update gnulib sources
 use_git=true
 
+check_exists() {
+  if test "$1" = "--verbose"; then
+    ($2 --version </dev/null) >/dev/null 2>&1
+    if test $? -ge 126; then
+      # If not found, run with diagnostics as one may be
+      # presented with env variables to set to find the right version
+      ($2 --version </dev/null)
+    fi
+  else
+    ($1 --version </dev/null) >/dev/null 2>&1
+  fi
+
+  test $? -lt 126
+}
+
 # find_tool ENVVAR NAMES...
 # -------------------------
 # Search for a required program.  Use the value of ENVVAR, if set,
-# otherwise find the first of the NAMES that can be run (i.e.,
-# supports --version).  If found, set ENVVAR to the program name,
-# die otherwise.
+# otherwise find the first of the NAMES that can be run.
+# If found, set ENVVAR to the program name, die otherwise.
 #
 # FIXME: code duplication, see also gnu-web-doc-update.
 find_tool ()
@@ -223,27 +238,21 @@ find_tool ()
   find_tool_names=$@
   eval "find_tool_res=\$$find_tool_envvar"
   if test x"$find_tool_res" = x; then
-    for i
-    do
-      if ($i --version </dev/null) >/dev/null 2>&1; then
-       find_tool_res=$i
-       break
+    for i; do
+      if check_exists $i; then
+        find_tool_res=$i
+        break
       fi
     done
-  else
-    find_tool_error_prefix="\$$find_tool_envvar: "
   fi
-  test x"$find_tool_res" != x \
-    || die "one of these is required: $find_tool_names"
-  ($find_tool_res --version </dev/null) >/dev/null 2>&1 \
-    || die "${find_tool_error_prefix}cannot run $find_tool_res --version"
+  if test x"$find_tool_res" = x; then
+    warn_ "one of these is required: $find_tool_names;"
+    die   "alternatively set $find_tool_envvar to a compatible tool"
+  fi
   eval "$find_tool_envvar=\$find_tool_res"
   eval "export $find_tool_envvar"
 }
 
-# Find sha1sum, named gsha1sum on MacPorts, and shasum on Mac OS X 10.6.
-find_tool SHA1SUM sha1sum gsha1sum shasum
-
 # Override the default configuration, if necessary.
 # Make sure that bootstrap.conf is sourced from the current directory
 # if we were invoked as "sh bootstrap".
@@ -255,12 +264,12 @@ esac
 # Extra files from gnulib, which override files from other sources.
 test -z "${gnulib_extra_files}" && \
   gnulib_extra_files="
-        $build_aux/install-sh
-        $build_aux/mdate-sh
-        $build_aux/texinfo.tex
-        $build_aux/depcomp
-        $build_aux/config.guess
-        $build_aux/config.sub
+        build-aux/install-sh
+        build-aux/mdate-sh
+        build-aux/texinfo.tex
+        build-aux/depcomp
+        build-aux/config.guess
+        build-aux/config.sub
         doc/INSTALL
 "
 
@@ -306,34 +315,34 @@ if test -n "$checkout_only_file" && test ! -r "$checkout_only_file"; then
   die "Bootstrapping from a non-checked-out distribution is risky."
 fi
 
-# Ensure that lines starting with ! sort last, per gitignore conventions
-# for whitelisting exceptions after a more generic blacklist pattern.
-sort_patterns() {
-  sort -u "$@" | sed '/^!/ {
-    H
-    d
-  }
-  $ {
-    P
-    x
-    s/^\n//
-  }' | sed '/^$/d'
+# Strip blank and comment lines to leave significant entries.
+gitignore_entries() {
+  sed '/^#/d; /^$/d' "$@"
 }
 
-# If $STR is not already on a line by itself in $FILE, insert it,
-# sorting the new contents of the file and replacing $FILE with the result.
-insert_sorted_if_absent() {
+# If $STR is not already on a line by itself in $FILE, insert it at the start.
+# Entries are inserted at the start of the ignore list to ensure existing
+# entries starting with ! are not overridden.  Such entries support
+# whitelisting exceptions after a more generic blacklist pattern.
+insert_if_absent() {
   file=$1
   str=$2
   test -f $file || touch $file
-  echo "$str" | sort_patterns - $file | cmp -s - $file > /dev/null \
-    || { echo "$str" | sort_patterns - $file > $file.bak \
-      && mv $file.bak $file; } \
-    || die "insert_sorted_if_absent $file $str: failed"
+  test -r $file || die "Error: failed to read ignore file: $file"
+  duplicate_entries=$(gitignore_entries $file | sort | uniq -d)
+  if [ "$duplicate_entries" ] ; then
+    die "Error: Duplicate entries in $file: " $duplicate_entries
+  fi
+  linesold=$(gitignore_entries $file | wc -l)
+  linesnew=$( { echo "$str"; cat $file; } | gitignore_entries | sort -u | wc -l)
+  if [ $linesold != $linesnew ] ; then
+    { echo "$str" | cat - $file > $file.bak && mv $file.bak $file; } \
+      || die "insert_if_absent $file $str: failed"
+  fi
 }
 
 # Adjust $PATTERN for $VC_IGNORE_FILE and insert it with
-# insert_sorted_if_absent.
+# insert_if_absent.
 insert_vc_ignore() {
   vc_ignore_file="$1"
   pattern="$2"
@@ -344,7 +353,7 @@ insert_vc_ignore() {
     # .gitignore entry.
     pattern=$(echo "$pattern" | sed s,^,/,);;
   esac
-  insert_sorted_if_absent "$vc_ignore_file" "$pattern"
+  insert_if_absent "$vc_ignore_file" "$pattern"
 }
 
 # Die if there is no AC_CONFIG_AUX_DIR($build_aux) line in configure.ac.
@@ -409,7 +418,7 @@ sort_ver() { # sort -V is not generally available
 get_version() {
   app=$1
 
-  $app --version >/dev/null 2>&1 || return 1
+  $app --version >/dev/null 2>&1 || { $app --version; return 1; }
 
   $app --version 2>&1 |
   sed -n '# Move version to start of line.
@@ -468,8 +477,7 @@ check_versions() {
     if [ "$req_ver" = "-" ]; then
       # Merely require app to exist; not all prereq apps are well-behaved
       # so we have to rely on $? rather than get_version.
-      $app --version >/dev/null 2>&1
-      if [ 126 -le $? ]; then
+      if ! check_exists --verbose $app; then
         warn_ "Error: '$app' not found"
         ret=1
       fi
@@ -502,6 +510,12 @@ print_versions() {
   # can't depend on column -t
 }
 
+# Find sha1sum, named gsha1sum on MacPorts, shasum on Mac OS X 10.6.
+# Also find the compatible sha1 utility on the BSDs
+if test x"$SKIP_PO" = x; then
+  find_tool SHA1SUM sha1sum gsha1sum shasum sha1
+fi
+
 use_libtool=0
 # We'd like to use grep -E, to see if any of LT_INIT,
 # AC_PROG_LIBTOOL, AM_PROG_LIBTOOL is used in configure.ac,
@@ -547,13 +561,21 @@ if ! printf "$buildreq" | check_versions; then
   fi
 fi
 
+# Warn the user if autom4te appears to be broken; this causes known
+# issues with at least gettext 0.18.3.
+probe=$(echo 'm4_quote([hi])' | autom4te -l M4sugar -t 'm4_quote:$%' -)
+if test "x$probe" != xhi; then
+  warn_ "WARNING: your autom4te wrapper eats stdin;"
+  warn_ "if bootstrap fails, consider upgrading your autotools"
+fi
+
 echo "$0: Bootstrapping from checked-out $package sources..."
 
 # See if we can use gnulib's git-merge-changelog merge driver.
-if test -d .git && (git --version) >/dev/null 2>/dev/null ; then
+if $use_git && test -d .git && check_exists git; then
   if git config merge.merge-changelog.driver >/dev/null ; then
     :
-  elif (git-merge-changelog --version) >/dev/null 2>/dev/null ; then
+  elif check_exists git-merge-changelog; then
     echo "$0: initializing git-merge-changelog driver"
     git config merge.merge-changelog.name 'GNU-style ChangeLog merge driver'
     git config merge.merge-changelog.driver 'git-merge-changelog %O %A %B'
@@ -573,17 +595,21 @@ git_modules_config () {
   test -f .gitmodules && git config --file .gitmodules "$@"
 }
 
-gnulib_path=$(git_modules_config submodule.gnulib.path)
-test -z "$gnulib_path" && gnulib_path=gnulib
+if $use_git; then
+  gnulib_path=$(git_modules_config submodule.gnulib.path)
+  test -z "$gnulib_path" && gnulib_path=gnulib
+fi
 
-# Get gnulib files.
+# Get gnulib files.  Populate $GNULIB_SRCDIR, possibly updating a
+# submodule, for use in the rest of the script.
 
 case ${GNULIB_SRCDIR--} in
 -)
+  # Note that $use_git is necessarily true in this case.
   if git_modules_config submodule.gnulib.url >/dev/null; then
     echo "$0: getting gnulib files..."
-    git submodule init || exit $?
-    git submodule update || exit $?
+    git submodule init -- "$gnulib_path" || exit $?
+    git submodule update -- "$gnulib_path" || exit $?
 
   elif [ ! -d "$gnulib_path" ]; then
     echo "$0: getting gnulib files..."
@@ -600,8 +626,8 @@ case ${GNULIB_SRCDIR--} in
   GNULIB_SRCDIR=$gnulib_path
   ;;
 *)
-  # Use GNULIB_SRCDIR as a reference.
-  if test -d "$GNULIB_SRCDIR"/.git && \
+  # Use GNULIB_SRCDIR directly or as a reference.
+  if $use_git && test -d "$GNULIB_SRCDIR"/.git && \
         git_modules_config submodule.gnulib.url >/dev/null; then
     echo "$0: getting gnulib files..."
     if git submodule -h|grep -- --reference > /dev/null; then
@@ -612,13 +638,14 @@ case ${GNULIB_SRCDIR--} in
       # This fallback allows at least git 1.5.5.
       if test -f "$gnulib_path"/gnulib-tool; then
         # Since file already exists, assume submodule init already complete.
-        git submodule update || exit $?
+        git submodule update -- "$gnulib_path" || exit $?
       else
         # Older git can't clone into an empty directory.
         rmdir "$gnulib_path" 2>/dev/null
         git clone --reference "$GNULIB_SRCDIR" \
           "$(git_modules_config submodule.gnulib.url)" "$gnulib_path" \
-          && git submodule init && git submodule update \
+          && git submodule init -- "$gnulib_path" \
+          && git submodule update -- "$gnulib_path" \
           || exit $?
       fi
     fi
@@ -627,12 +654,19 @@ case ${GNULIB_SRCDIR--} in
   ;;
 esac
 
+# $GNULIB_SRCDIR now points to the version of gnulib to use, and
+# we no longer need to use git or $gnulib_path below here.
+
 if $bootstrap_sync; then
   cmp -s "$0" "$GNULIB_SRCDIR/build-aux/bootstrap" || {
     echo "$0: updating bootstrap and restarting..."
+    case $(sh -c 'echo "$1"' -- a) in
+      a) ignored=--;;
+      *) ignored=ignored;;
+    esac
     exec sh -c \
       'cp "$1" "$2" && shift && exec "${CONFIG_SHELL-/bin/sh}" "$@"' \
-      -- "$GNULIB_SRCDIR/build-aux/bootstrap" \
+      $ignored "$GNULIB_SRCDIR/build-aux/bootstrap" \
       "$0" "$@" --no-bootstrap-sync
   }
 fi
@@ -680,11 +714,10 @@ update_po_files() {
     cksum_file="$ref_po_dir/$po.s1"
     if ! test -f "$cksum_file" ||
         ! test -f "$po_dir/$po.po" ||
-        ! $SHA1SUM -c --status "$cksum_file" \
-            < "$new_po" > /dev/null; then
+        ! $SHA1SUM -c "$cksum_file" < "$new_po" > /dev/null 2>&1; then
       echo "$me: updated $po_dir/$po.po..."
       cp "$new_po" "$po_dir/$po.po" \
-          && $SHA1SUM < "$new_po" > "$cksum_file"
+          && $SHA1SUM < "$new_po" > "$cksum_file" || return
     fi
   done
 }
@@ -889,20 +922,21 @@ find "$m4_base" "$source_base" \
   -depth \( -name '*.m4' -o -name '*.[ch]' \) \
   -type l -xtype l -delete > /dev/null 2>&1
 
+# Invoke autoreconf with --force --install to ensure upgrades of tools
+# such as ylwrap.
+AUTORECONFFLAGS="--verbose --install --force -I $m4_base $ACLOCAL_FLAGS"
+
 # Some systems (RHEL 5) are using ancient autotools, for which the
 # --no-recursive option had not been invented.  Detect that lack and
 # omit the option when it's not supported.  FIXME in 2017: remove this
 # hack when RHEL 5 autotools are updated, or when they become irrelevant.
-no_recursive=
 case $($AUTORECONF --help) in
-  *--no-recursive*) no_recursive=--no-recursive;;
+  *--no-recursive*) AUTORECONFFLAGS="$AUTORECONFFLAGS --no-recursive";;
 esac
 
 # Tell autoreconf not to invoke autopoint or libtoolize; they were run above.
-echo "running: AUTOPOINT=true LIBTOOLIZE=true " \
-    "$AUTORECONF --verbose --install $no_recursive -I $m4_base $ACLOCAL_FLAGS"
-AUTOPOINT=true LIBTOOLIZE=true \
-    $AUTORECONF --verbose --install $no_recursive -I $m4_base $ACLOCAL_FLAGS \
+echo "running: AUTOPOINT=true LIBTOOLIZE=true $AUTORECONF $AUTORECONFFLAGS"
+AUTOPOINT=true LIBTOOLIZE=true $AUTORECONF $AUTORECONFFLAGS \
   || die "autoreconf failed"
 
 # Get some extra files from gnulib, overriding existing files.

bootstrap.conf

@@ -31,17 +31,24 @@ announce-gen
 base32
 bind
 c-ctype
+c-strcase
+c-strcasestr
 clock-time
 close
 connect
+dirname
 fcntl
+fnmatch
 futimens
 ftello
 getaddrinfo
+getline
 getopt-gnu
 getpass-gnu
 getpeername
 getsockname
+gettext
+gitlog-to-changelog
 git-version-gen
 gnupload
 ioctl
@@ -51,12 +58,12 @@ listen
 maintainer-makefile
 mbiter
 mbtowc
+memrchr
 mkdir
 mkstemp
 mkostemp
 crypto/md5
 crypto/sha1
-pipe
 quote
 quotearg
 recv
@@ -68,10 +75,14 @@ sigprocmask
 sigpipe
 snprintf
 socket
+spawn-pipe
 stdbool
-strcasestr
+strcase
 strerror_r-posix
+strptime
 strtok_r
+strtoll
+timegm
 tmpdir
 unlocked-io
 update-copyright
@@ -104,4 +115,5 @@ perl       5.5
 rsync      -
 tar        -
 xz         -
+flex       -
 "

build-aux/build_info.pl

@@ -137,3 +137,4 @@ __DATA__
 
 #include "wget.h"
 #include <stdio.h>
+#include "version.h"

cfg.mk

@@ -19,10 +19,11 @@ local-checks-to-skip = \
     sc_prohibit_S_IS_definition \
     \
     sc_program_name \
-
+    sc_makefile_at_at_check \
+    sc_cast_of_argument_to_free \
 
 #SHELL=bash -x
 show-vc-list-except:
 	@$(VC_LIST_EXCEPT)
 
-VC_LIST_ALWAYS_EXCLUDE_REGEX = ^ABOUT-NLS|((msdos|vms)/.*)$$
+VC_LIST_ALWAYS_EXCLUDE_REGEX = ^ABOUT-NLS|((msdos|vms)/.*)|ChangeLog-2014-12-10$$

configure.ac

@@ -57,48 +57,101 @@ AC_CANONICAL_HOST
 AC_DEFINE_UNQUOTED([OS_TYPE], "$host_os",
                    [Define to be the name of the operating system.])
 
+
 dnl
-dnl Process features.
+dnl Process External Libraries
 dnl
 
-ENABLE_PSL=no
-AC_ARG_WITH(libpsl,
-    AS_HELP_STRING([--without-libpsl],
-                   [disable support for libpsl cookie checking.]),
-    [],
-    [AC_SEARCH_LIBS(psl_builtin, psl,
-                    [ENABLE_PSL=yes; AC_DEFINE([HAVE_LIBPSL], [1], [PSL Support Enabled])],
-                    [AC_MSG_WARN(*** libpsl not found. Falling back to Wget builtin cookie checking.)])
-    ])
+dnl Libpsl: Public Suffix List checking
+AC_ARG_WITH([libpsl],
+  [AS_HELP_STRING([--without-libpsl], [disable support for libpsl cookie checking.])])
 
-AC_ARG_WITH(ssl,
-[[  --without-ssl           disable SSL autodetection
-  --with-ssl={gnutls,openssl} specify the SSL backend.  GNU TLS is the default.]])
+dnl SSL: Configure SSL backend to use
+AC_ARG_WITH([ssl],
+  [AS_HELP_STRING([--with-ssl={gnutls,openssl}], [specify SSL backend. GNU TLS is the default.])])
 
-AC_ARG_WITH(zlib,
-[[  --without-zlib          disable zlib ]])
 
-AC_ARG_ENABLE(opie,
-[  --disable-opie          disable support for opie or s/key FTP login],
-ENABLE_OPIE=$enableval, ENABLE_OPIE=yes)
-test x"${ENABLE_OPIE}" = xyes && AC_DEFINE([ENABLE_OPIE], 1,
-   [Define if you want the Opie support for FTP compiled in.])
+dnl Zlib: Configure use iof zlib for compression
+AC_ARG_WITH([zlib],
+  [AS_HELP_STRING([--without-zlib], [disable zlib.])])
 
-AC_ARG_ENABLE(digest,
-[  --disable-digest        disable support for HTTP digest authorization],
-ENABLE_DIGEST=$enableval, ENABLE_DIGEST=yes)
-test x"${ENABLE_DIGEST}" = xyes && AC_DEFINE([ENABLE_DIGEST], 1,
-   [Define if you want the HTTP Digest Authorization compiled in.])
 
-AC_ARG_ENABLE(ntlm,
-[  --disable-ntlm          disable support for NTLM authorization],
-[ENABLE_NTLM=$enableval], [ENABLE_NTLM=auto])
+dnl
+dnl Process features
+dnl
 
-AC_ARG_ENABLE(debug,
-[  --disable-debug         disable support for debugging output],
-ENABLE_DEBUG=$enableval, ENABLE_DEBUG=yes)
-test x"${ENABLE_DEBUG}" = xyes && AC_DEFINE([ENABLE_DEBUG], 1,
-   [Define if you want the debug output support compiled in.])
+dnl Opie: Support for opie s/key FTP logins
+AC_ARG_ENABLE([opie],
+  [AS_HELP_STRING([--disable-opie], [disable support for opie or s/key FTP login])],
+  [ENABLE_OPIE=$enableval],
+  [ENABLE_OPIE=yes])
+
+AS_IF([test "x$ENABLE_OPIE" = xyes],
+  [AC_DEFINE([ENABLE_OPIE], [1], [Define if you want Opie support for FTP compiled in.])],
+  []
+)
+
+
+dnl Digest: Support for HTTP Digest Authentication
+AC_ARG_ENABLE([digest],
+  [AS_HELP_STRING([--disable-digest], [disable support for HTTP digest authorization])],
+  [ENABLE_DIGEST=$enableval],
+  [ENABLE_DIGEST=yes])
+
+AS_IF([test "x$ENABLE_DIGEST" = xyes],
+  [AC_DEFINE([ENABLE_DIGEST], [1], [Define if you want the HTTP Digest Authorization compiled in.])],
+  []
+)
+
+
+dnl NTLM: Support for HTTP NTLM Authentication
+AC_ARG_ENABLE([ntlm],
+  [AS_HELP_STRING([--disable-ntlm], [disable support for NTLM authorization])],
+  [ENABLE_NTLM=$enableval],
+  [ENABLE_NTLM=auto]
+)
+
+
+dnl Debug: Support for printing debugging output
+AC_ARG_ENABLE([debug],
+  [AS_HELP_STRING([--disable-debug], [disable support for debugging output])],
+  [ENABLE_DEBUG=$enableval],
+  [ENABLE_DEBUG=yes])
+
+AS_IF([test "x$ENABLE_DEBUG" = xyes],
+  [AC_DEFINE([ENABLE_DEBUG], [1], [Define if you want the debug output support compiled in.])],
+  []
+)
+
+dnl Valgrind-tests: Should test suite be run under valgrind?
+AC_ARG_ENABLE(valgrind-tests,
+  [AS_HELP_STRING([--enable-valgrind-tests], [enable using Valgrind for tests])],
+  [ENABLE_VALGRIND=$enableval],
+  [ENABLE_VALGRIND=no])
+
+AS_IF([test "x$ENABLE_VALGRIND" != xno], [
+  AC_CHECK_PROG(HAVE_VALGRIND, valgrind, yes, no)
+  AS_IF([test "x$HAVE_VALGRIND" = xyes], [
+    VALGRIND_TESTS="1"
+    AC_SUBST(VALGRIND_TESTS)
+    VALGRIND_INFO="Test suite will be run under Valgrind"
+  ], [
+    VALGRIND_INFO="Valgrind not found"
+  ])
+], [
+  VALGRIND_INFO="Valgrind testing not enabled"
+])
+
+AC_ARG_ENABLE(assert,
+  [AS_HELP_STRING([--enable-assert], [enable assertions in code base])],
+  [ENABLE_ASSERTION=$enableval],
+  [ENABLE_ASSERTION=no]
+)
+
+AS_IF([test "x$ENABLE_ASSERTION" != xyes], [
+  CFLAGS="-DNDEBUG $CFLAGS"
+  ],
+  [])
 
 dnl
 dnl Find the compiler
@@ -114,6 +167,12 @@ AC_AIX
 
 gl_EARLY
 
+dnl
+dnl Find python3
+dnl
+AM_PATH_PYTHON([3.0],,[:])
+AM_CONDITIONAL([HAVE_PYTHON3], [test "$PYTHON" != :])
+
 dnl
 dnl Gettext
 dnl
@@ -131,9 +190,11 @@ dnl   if compiler is gcc, then
 dnl     use -O2 and some warning flags
 dnl   else
 dnl     use os-specific flags or -O
+dnl
+dnl For these tests, the system considers clang to be gcc
 if test -n "$auto_cflags"; then
   if test -n "$GCC"; then
-    CFLAGS="$CFLAGS -O2 -Wall"
+    CFLAGS="$CFLAGS -O2 -Wall -Wextra"
   else
     case "$host_os" in
       *hpux*)  CFLAGS="$CFLAGS +O3"                      ;;
@@ -207,7 +268,7 @@ AC_FUNC_MMAP
 AC_FUNC_FSEEKO
 AC_CHECK_FUNCS(strptime timegm vsnprintf vasprintf drand48 pathconf)
 AC_CHECK_FUNCS(strtoll usleep ftello sigblock sigsetjmp memrchr wcwidth mbtowc)
-AC_CHECK_FUNCS(sleep symlink utime)
+AC_CHECK_FUNCS(sleep symlink utime strlcpy random)
 
 if test x"$ENABLE_OPIE" = xyes; then
   AC_LIBOBJ([ftp-opie])
@@ -244,48 +305,84 @@ dnl
 dnl Checks for libraries.
 dnl
 
+PKG_PROG_PKG_CONFIG
+
+AS_IF([test "x$with_libpsl" != xno], [
+  PKG_CHECK_MODULES([LIBPSL], libpsl, [
+    with_libpsl=yes
+    # correct $LIBPSL_LIBS (in libpsl <= 0.6.0)
+    AS_IF([test "x$LIBPSL_LIBS" = "x-llibpsl "], [LIBPSL_LIBS="-lpsl"])
+    LIBS="$LIBPSL_LIBS $LIBS"
+    CFLAGS="$LIBPSL_CFLAGS $CFLAGS"
+    AC_DEFINE([HAVE_LIBPSL], [1], [PSL support enabled])
+  ], [
+    AC_SEARCH_LIBS(psl_builtin, psl,
+      [with_libpsl=yes; AC_DEFINE([HAVE_LIBPSL], [1], [PSL support enabled])],
+      [with_libpsl=no;  AC_MSG_WARN(*** libpsl was not found. Fallback to builtin cookie checking.)])
+  ])
+])
+
 AS_IF([test x"$with_zlib" != xno], [
   with_zlib=yes
-  AC_CHECK_LIB(z, compress)
+  PKG_CHECK_MODULES([ZLIB], zlib, [
+    LIBS="$ZLIB_LIBS $LIBS"
+    CFLAGS="$ZLIB_CFLAGS $CFLAGS"
+    AC_DEFINE([HAVE_LIBZ], [1], [Define if using zlib.])
+  ], [
+    AC_CHECK_LIB(z, compress)
+  ])
 ])
 
 AS_IF([test x"$with_ssl" = xopenssl], [
-  dnl As of this writing (OpenSSL 0.9.6), the libcrypto shared library
-  dnl doesn't record its dependency on libdl, so we need to make sure
-  dnl -ldl ends up in LIBS on systems that have it.  Most OSes use
-  dnl dlopen(), but HP-UX uses shl_load().
-  AC_CHECK_LIB(dl, dlopen, [], [
-    AC_CHECK_LIB(dl, shl_load)
-  ])
+  if [test x"$with_libssl_prefix" = x]; then
+    PKG_CHECK_MODULES([OPENSSL], [openssl], [
+      AC_MSG_NOTICE([compiling in support for SSL via OpenSSL])
+      AC_LIBOBJ([openssl])
+      LIBS="$OPENSSL_LIBS $LIBS"
+      CFLAGS="$OPENSSL_CFLAGS -DHAVE_LIBSSL $CFLAGS"
+      LIBSSL=" " # ntlm check below wants this
+      AC_CHECK_FUNCS([RAND_egd])
+      AC_DEFINE([HAVE_LIBSSL], [1], [Define if using openssl.])
+      ssl_found=yes
+    ])
+  fi
+  if [test x"$ssl_found" != xyes]; then
+    dnl As of this writing (OpenSSL 0.9.6), the libcrypto shared library
+    dnl doesn't record its dependency on libdl, so we need to make sure
+    dnl -ldl ends up in LIBS on systems that have it.  Most OSes use
+    dnl dlopen(), but HP-UX uses shl_load().
+    AC_CHECK_LIB(dl, dlopen, [], [
+      AC_CHECK_LIB(dl, shl_load)
+    ])
 
-  ssl_found=no
-  case $host_os in
-    *mingw32* )
-      dnl prefer link to openssl dlls if possible. if not then fallback on static libs. if not then error
+    ssl_found=no
+    case $host_os in
+      *mingw32* )
+        dnl prefer link to openssl dlls if possible. if not then fallback on static libs. if not then error
 
-      AC_CHECK_LIB(eay32, EVP_MD_CTX_init)
-      if test x"$ac_cv_lib_eay32_EVP_MD_CTX_init" != xno
-      then
-        AC_CHECK_LIB(ssl32, SSL_connect, [
-          ssl_found=yes
-          AC_MSG_NOTICE([Enabling support for SSL via OpenSSL (shared)])
-          AC_LIBOBJ([openssl])
-          LIBS="${LIBS} -lssl32"
-          AC_DEFINE([HAVE_LIBSSL32], [1], [Define to 1 if you have the `ssl32' library (-lssl32).])
-        ],
-        AC_MSG_ERROR([openssl not found: shared lib eay32 found but ssl32 not found]))
+        AC_CHECK_LIB(eay32, EVP_MD_CTX_init)
+        if test x"$ac_cv_lib_eay32_EVP_MD_CTX_init" != xno
+        then
+          AC_CHECK_LIB(ssl32, SSL_connect, [
+            ssl_found=yes
+            AC_MSG_NOTICE([Enabling support for SSL via OpenSSL (shared)])
+            AC_LIBOBJ([openssl])
+            LIBS="${LIBS} -lssl32"
+            AC_DEFINE([HAVE_LIBSSL32], [1], [Define to 1 if you have the `ssl32' library (-lssl32).])
+          ],
+          AC_MSG_ERROR([openssl not found: shared lib eay32 found but ssl32 not found]))
 
-      else
-        LIBS+=' -lgdi32'
-        dnl fallback and test static libs
-      fi
-      dnl add zdll lib as dep for above tests?
-    ;;
-  esac
+        else
+          LIBS+=' -lgdi32'
+          dnl fallback and test static libs
+        fi
+        dnl add zdll lib as dep for above tests?
+      ;;
+    esac
 
-  AS_IF([test x$ssl_found != xyes], [
-    dnl Now actually check for -lssl if it wasn't already found
-    AC_LIB_HAVE_LINKFLAGS([ssl], [crypto], [
+    AS_IF([test x$ssl_found != xyes], [
+      dnl Now actually check for -lssl if it wasn't already found
+      AC_LIB_HAVE_LINKFLAGS([ssl], [crypto], [
 #include <openssl/ssl.h>
 #include <openssl/x509.h>
 #include <openssl/err.h>
@@ -293,19 +390,20 @@ AS_IF([test x"$with_ssl" = xopenssl], [
 #include <openssl/des.h>
 #include <openssl/md4.h>
 #include <openssl/md5.h>
-    ], [SSL_library_init ()])
-    if test x"$LIBSSL" != x
-    then
-      ssl_found=yes
-      AC_MSG_NOTICE([compiling in support for SSL via OpenSSL])
-      AC_LIBOBJ([openssl])
-      LIBS="$LIBSSL $LIBS"
-    elif test x"$with_ssl" != x
-    then
-      AC_MSG_ERROR([--with-ssl=openssl was given, but SSL is not available.])
-    fi
-  ])
-
+      ], [SSL_library_init ()])
+      if test x"$LIBSSL" != x
+      then
+        ssl_found=yes
+        AC_MSG_NOTICE([compiling in support for SSL via OpenSSL])
+        AC_LIBOBJ([openssl])
+        LIBS="$LIBSSL $LIBS"
+        AC_CHECK_FUNCS([RAND_egd])
+      elif test x"$with_ssl" != x
+      then
+        AC_MSG_ERROR([--with-ssl=openssl was given, but SSL is not available.])
+      fi
+    ])
+  fi
 ], [
   # --with-ssl is not openssl: check if it's no
   AS_IF([test x"$with_ssl" != xno], [
@@ -313,17 +411,29 @@ AS_IF([test x"$with_ssl" = xopenssl], [
     with_ssl=gnutls
 
     dnl Now actually check for -lgnutls
-    AC_LIB_HAVE_LINKFLAGS([gnutls], [], [
+    if [test x"$with_libgnutls_prefix" = x]; then
+      PKG_CHECK_MODULES([GNUTLS], [gnutls], [
+        AC_MSG_NOTICE([compiling in support for SSL via GnuTLS])
+        AC_LIBOBJ([gnutls])
+        LIBS="$GNUTLS_LIBS $LIBS"
+        CFLAGS="$GNUTLS_CFLAGS -DHAVE_LIBGNUTLS $CFLAGS"
+        AC_DEFINE([HAVE_LIBGNUTLS], [1], [Define if using gnutls.])
+        ssl_found=yes
+      ])
+    fi
+    if [test x"$ssl_found" != xyes]; then
+      AC_LIB_HAVE_LINKFLAGS([gnutls], [], [
 #include <gnutls/gnutls.h>
-    ], [gnutls_global_init()])
-    if test x"$LIBGNUTLS" != x
-    then
-      ssl_found=yes
-      AC_MSG_NOTICE([compiling in support for SSL via GnuTLS])
-      AC_LIBOBJ([gnutls])
-      LIBS="$LIBGNUTLS $LIBS"
-    else
-      AC_MSG_ERROR([--with-ssl=gnutls was given, but GNUTLS is not available.])
+      ], [gnutls_global_init()])
+      if test x"$LIBGNUTLS" != x
+      then
+        ssl_found=yes
+        AC_MSG_NOTICE([compiling in support for SSL via GnuTLS])
+        AC_LIBOBJ([gnutls])
+        LIBS="$LIBGNUTLS $LIBS"
+      else
+        AC_MSG_ERROR([GnuTLS has not been found. Use --with-ssl=openssl if you explicitly want OpenSSL.])
+      fi
     fi
 
     AC_CHECK_FUNCS(gnutls_priority_set_direct)
@@ -542,32 +652,64 @@ fi
 dnl
 dnl Check for UUID
 dnl
-AC_CHECK_HEADER(uuid.h,
-                AC_CHECK_FUNC(uuid, uuid_create,
-                  [AC_DEFINE([HAVE_UUID_CREATE], 1,
-                             [Define if uuid_create is available.])
-                  ])
-)
 
-AC_CHECK_HEADER(uuid/uuid.h,
-                AC_CHECK_LIB(uuid, uuid_generate,
-                  [LIBS="${LIBS} -luuid"
-                   AC_DEFINE([HAVE_LIBUUID], 1,
-                             [Define if libuuid is available.])
-                  ])
-)
+AC_ARG_WITH(libuuid, AC_HELP_STRING([--without-libuuid],
+                                    [Generate UUIDs for WARC files via libuuid]))
+
+AS_IF([test "x$with_libuuid" = xyes], [
+  # libuuid was explicitly requested
+  PKG_CHECK_MODULES([UUID], uuid, [
+    LIBS="$UUID_LIBS $LIBS"
+    CFLAGS="$UUID_CFLAGS $CFLAGS"
+    uuid_mode=1
+  ], [
+    AC_SEARCH_LIBS(uuid_generate, uuid,
+      [uuid_mode=1],
+      [AC_MSG_ERROR(*** libuuid was explicitly requested but wasn't found.)])
+  ])
+], [test "x$with_libuuid" = xno], [
+  # libuuid was explicitly *not* requested
+  AC_CHECK_HEADER(uuid.h,
+    AC_CHECK_FUNC(uuid_create, [uuid_mode=2]))
+], [
+  # default:
+  AC_CHECK_HEADER(uuid.h, [
+    AC_CHECK_FUNC(uuid_create, [uuid_mode=2])
+  ], [
+    AC_CHECK_HEADER(uuid/uuid.h,
+      AC_SEARCH_LIBS(uuid_generate, uuid, [uuid_mode=1]))
+  ])
+])
+
+AS_IF([test "x$uuid_mode" = x1], [
+  AC_DEFINE([HAVE_LIBUUID], [1], [Define if using libuuid.])
+], [test "x$uuid_mode" = x2], [
+  AC_DEFINE([HAVE_UUID_CREATE], 1, [Define if uuid_create is available.])
+])
 
 dnl
 dnl Check for PCRE
 dnl
 
-AC_CHECK_HEADER(pcre.h,
-                AC_CHECK_LIB(pcre, pcre_compile,
-                  [LIBS="${LIBS} -lpcre"
-                   AC_DEFINE([HAVE_LIBPCRE], 1,
-                             [Define if libpcre is available.])
-                  ])
-)
+AC_ARG_ENABLE(pcre, AC_HELP_STRING([--disable-pcre],
+                                   [Disable PCRE style regular expressions]))
+
+AS_IF([test "X$enable_pcre" != "Xno"],[
+  PKG_CHECK_MODULES([PCRE], libpcre, [
+    CFLAGS="$PCRE_CFLAGS $CFLAGS"
+    AC_CHECK_HEADER(pcre.h, [
+      LIBS="$PCRE_LIBS $LIBS"
+      AC_DEFINE([HAVE_LIBPCRE], [1], [Define if libpcre is available.])
+    ])
+  ], [
+    AC_CHECK_HEADER(pcre.h, [
+      AC_CHECK_LIB(pcre, pcre_compile, [
+        LIBS="${LIBS} -lpcre"
+        AC_DEFINE([HAVE_LIBPCRE], 1, [Define if libpcre is available.])
+      ])
+    ])
+  ])
+])
 
 
 dnl Needed by src/Makefile.am
@@ -594,9 +736,11 @@ AC_MSG_NOTICE([Summary of build options:
   Libs:              $LIBS
   SSL:               $with_ssl
   Zlib:              $with_zlib
-  PSL:               $ENABLE_PSL
+  PSL:               $with_libpsl
   Digest:            $ENABLE_DIGEST
   NTLM:              $ENABLE_NTLM
   OPIE:              $ENABLE_OPIE
   Debugging:         $ENABLE_DEBUG
+  Assertions:        $ENABLE_ASSERTION
+  Valgrind:          $VALGRIND_INFO
 ])

contrib/check-hard

@@ -0,0 +1,52 @@
+#!/bin/sh
+#
+# Do some checking before 'git push'.
+
+CORES=`grep -c ^processor /proc/cpuinfo 2>/dev/null || echo 0`
+if [ $CORES -lt 1 ]; then CORES=1; fi
+
+make -f Makefile distclean 2>/dev/null
+
+# automatically exit on error
+set -e
+
+export CFLAGS="-std=c89 -pedantic -O2 -g -Wall -Wextra -Wstrict-prototypes -Wold-style-definition -Wwrite-strings -Wshadow -Wformat -Wformat-security -Wunreachable-code -Wstrict-prototypes -Wmissing-prototypes -Wold-style-definition"
+
+CACHEFILE=$PWD/config_check.cache
+
+# measure time consumed and print it at the end of the script
+START=$(date +%s.%N)
+
+echo "./bootstrap"
+./bootstrap >/dev/null
+echo
+
+for CC in gcc clang; do
+  export CC
+  echo "*** checking CC=$CC"
+
+  # the compiler changed, so we have to remove the cache file here
+  rm -f $CACHEFILE
+
+  for options in "" "--with-ssl=openssl"; do
+    export DISTCHECK_CONFIGURE_FLAGS="-C --cache-file=$CACHEFILE $options"
+    echo "  ./configure $DISTCHECK_CONFIGURE_FLAGS"
+    ./configure $DISTCHECK_CONFIGURE_FLAGS >/dev/null
+
+    for xVALGRIND in 0 1; do
+      for xLCALL in C tr_TR.utf8; do
+        export TESTS_ENVIRONMENT="LC_ALL=$xLCALL VALGRIND_TESTS=$xVALGRIND"
+        echo "    TESTS_ENVIRONMENT=\"$TESTS_ENVIRONMENT\"" make check -j$CORES
+        make check -j$CORES >/dev/null
+      done
+    done
+
+    unset TESTS_ENVIRONMENT
+    export TESTS_ENVIRONMENT
+    echo "  make distcheck -j$CORES"
+    make distcheck -j$CORES
+  done
+done
+
+END=$(date +%s.%N)
+echo "Duration: "$(echo "$END - $START" | bc)

contrib/tsocked-wget

@@ -0,0 +1,90 @@
+#!/bin/bash
+
+# Script that executes wget using a socks proxy if the environment variable
+# socks_proxy is set.
+#
+# The socks_proxy variable shall have one of the forms:
+#   socks://username:password@host:port
+#   socks4://username:password@host:port
+#   socks5://username:password@host:port
+# with username, password and port fields being optional
+#
+# As socksification applies to the whole process, domains defined in the
+# no_proxy setting are *not* excluded.
+#
+
+# Requisites: bash (version 3.2.3 or later), tsocks and wget
+
+####################################
+#  Copyright (C) 2014 Ángel González
+#  Permission is hereby granted, free of charge, to any person obtaining a copy
+#  of this software and associated documentation files (the "Software"), to deal
+#  in the Software without restriction, including without limitation the rights
+#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+#  copies of the Software, and to permit persons to whom the Software is
+#  furnished to do so, subject to the following conditions:
+#
+#  The above copyright notice and this permission notice shall be included in
+#  all copies or substantial portions of the Software.
+#
+#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+#  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+#  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+#  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+#  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+#  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+#  THE SOFTWARE.
+#
+
+set -eu
+
+# Binary to be executed. Use an absolute path if installing this script as 'wget'
+WGET=wget
+
+if [ -z "${socks_proxy:-}" ]; then
+exec "$WGET" "$@"
+fi
+
+CONFIG=""
+
+if [[ "${socks_proxy}" =~ ^socks[45]?:// ]]; then
+ if [[ "${socks_proxy:5:1}" != ":" ]]; then
+  CONFIG+="server_type = ${socks_proxy:5:1}"
+  socks_proxy="${socks_proxy:9}"
+ else
+  socks_proxy="${socks_proxy:8}"
+ fi
+elif [[ "${socks_proxy}" =~ ^[[:alnum:]]*:// ]]; then
+ echo "Bad value specified for socks_proxy: $socks_proxy" >&2
+ exit 2
+fi
+
+if [[ "${socks_proxy}" =~ ^([^@:]*)(:([^@]*))?@ ]]; then
+ unset TSOCKS_USERNAME
+ CONFIG+="
+   default_user = ${BASH_REMATCH[1]}"
+
+ if [ ! -z "${BASH_REMATCH[3]}" ]; then
+  unset TSOCKS_PASSWORD
+  CONFIG+="
+   default_pass = ${BASH_REMATCH[3]}"
+ fi
+ socks_proxy="${socks_proxy:${#BASH_REMATCH[0]}}"
+fi
+
+
+# Get rid of trailing slashes
+if [[ "${socks_proxy}" =~ ^([^/]*)/ ]]; then
+ socks_proxy="${socks_proxy:0:${#BASH_REMATCH[1]}}"
+fi
+
+if [[ "${socks_proxy}" =~ :([0-9]+)$ ]]; then
+  CONFIG+="
+   server_port = ${BASH_REMATCH[1]}"
+  socks_proxy=${socks_proxy:0:${#socks_proxy} - ${#BASH_REMATCH[0]}}
+fi
+
+CONFIG+="
+   server = ${socks_proxy}"
+
+TSOCKS_CONF_FILE=<(echo "$CONFIG") exec tsocks "$WGET" --no-proxy "$@"

doc/Makefile.am

@@ -16,7 +16,7 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 #
-# Version: $(VERSION)
+# Version: @VERSION@
 #
 
 # Program to convert DVI files to PostScript
@@ -28,6 +28,7 @@ manext     = 1
 RM = rm -f
 
 TEXI2POD     = $(srcdir)/texi2pod.pl
+POD2MAN      = @POD2MAN@
 MAN          = wget.$(manext)
 WGETRC       = $(sysconfdir)/wgetrc
 SAMPLERCTEXI = sample.wgetrc.munged_for_texi_inclusion
@@ -38,7 +39,7 @@ SAMPLERCTEXI = sample.wgetrc.munged_for_texi_inclusion
 
 man_MANS = $(MAN)
 
-all: wget.info $(COMMENT_IF_NO_POD2MAN)$(MAN)
+all: wget.info @COMMENT_IF_NO_POD2MAN@$(MAN)
 
 everything: all wget_us.ps wget_a4.ps wget_toc.html
 
@@ -56,7 +57,7 @@ wget.pod: $(srcdir)/wget.texi version.texi
 	$(TEXI2POD) -D VERSION="$(VERSION)" $(srcdir)/wget.texi $@
 
 $(MAN): wget.pod
-	$(POD2MAN) --center="GNU Wget" --release="GNU Wget $(VERSION)" $? > $@
+	$(POD2MAN) --center="GNU Wget" --release="GNU Wget @VERSION@" $? > $@
 
 #wget.cat: $(MAN)
 #	nroff -man $? > $@
@@ -75,10 +76,10 @@ wget_toc.html: $(srcdir)/wget.texi
 #
 
 # install all the documentation
-install-data-local: install.wgetrc $(COMMENT_IF_NO_POD2MAN)install.man
+install-data-local: install.wgetrc @COMMENT_IF_NO_POD2MAN@install.man
 
 # uninstall all the documentation
-uninstall-local: $(COMMENT_IF_NO_POD2MAN)uninstall.man
+uninstall-local: @COMMENT_IF_NO_POD2MAN@uninstall.man
 
 
 # install man page, creating install directory if necessary

doc/sample.wgetrc

@@ -16,6 +16,9 @@
 ## To use the settings in this file, you will have to uncomment them,
 ## as well as change them, in most cases, as the values on the
 ## commented-out lines are the default values (e.g. "off").
+##
+## Command are case-, underscore- and minus-insensitive.
+## For example ftp_proxy, ftp-proxy and ftpproxy are the same.
 
 
 ##
@@ -122,7 +125,7 @@
 #iri = off
 
 # Force the default system encoding
-#locale = UTF-8
+#localencoding = UTF-8
 
 # Force the default remote server encoding
 #remoteencoding = UTF-8

doc/wget.texi

@@ -774,12 +774,15 @@ use @samp{--progress=bar:force:noscroll}.
 Force wget to display the progress bar in any verbosity.
 
 By default, wget only displays the progress bar in verbose mode.  One may
-however want wget to display the progress bar on screen in conjunction with
+however, want wget to display the progress bar on screen in conjunction with
 any other verbosity modes like @samp{--no-verbose} or @samp{--quiet}.  This
 is often a desired a property when invoking wget to download several small/large
 files.  In such a case, wget could simply be invoked with this parameter to get
 a much cleaner output on the screen.
 
+This option will also force the progress bar to be printed to @file{stderr} when
+used alongside the @samp{--logfile} option.
+
 @item -N
 @itemx --timestamping
 Turn on time-stamping.  @xref{Time-Stamping}, for details.
@@ -1725,19 +1728,23 @@ it allows Wget to fetch certificates on demand.
 Without this option Wget looks for CA certificates at the
 system-specified locations, chosen at OpenSSL installation time.
 
+@cindex SSL CRL, certificate revocation list
+@item --crl-file=@var{file}
+Specifies a CRL file in @var{file}.  This is needed for certificates
+that have been revocated by the CAs.
+
 @cindex entropy, specifying source of
 @cindex randomness, specifying source of
 @item --random-file=@var{file}
+[OpenSSL and LibreSSL only]
 Use @var{file} as the source of random data for seeding the
-pseudo-random number generator on systems without @file{/dev/random}.
+pseudo-random number generator on systems without @file{/dev/urandom}.
 
 On such systems the SSL library needs an external source of randomness
 to initialize.  Randomness may be provided by EGD (see
 @samp{--egd-file} below) or read from an external source specified by
 the user.  If this option is not specified, Wget looks for random data
-in @code{$RANDFILE} or, if that is unset, in @file{$HOME/.rnd}.  If
-none of those are available, it is likely that SSL encryption will not
-be usable.
+in @code{$RANDFILE} or, if that is unset, in @file{$HOME/.rnd}.
 
 If you're getting the ``Could not seed OpenSSL PRNG; disabling SSL.'' 
 error, you should provide random data using some of the methods
@@ -1745,6 +1752,7 @@ described above.
 
 @cindex EGD
 @item --egd-file=@var{file}
+[OpenSSL only]
 Use @var{file} as the EGD socket.  EGD stands for @dfn{Entropy
 Gathering Daemon}, a user-space program that collects data from
 various unpredictable system sources and makes it available to other
@@ -1759,7 +1767,7 @@ read random data from EGD socket specified using this option.
 
 If this option is not specified (and the equivalent startup command is
 not used), EGD is never contacted.  EGD is not needed on modern Unix
-systems that support @file{/dev/random}.
+systems that support @file{/dev/urandom}.
 @end table
 
 @cindex WARC
@@ -2893,10 +2901,10 @@ variable = value
 The @dfn{variable} will also be called @dfn{command}.  Valid
 @dfn{values} are different for different commands.
 
-The commands are case-insensitive and underscore-insensitive.  Thus
-@samp{DIr__PrefiX} is the same as @samp{dirprefix}.  Empty lines, lines
-beginning with @samp{#} and lines containing white-space only are
-discarded.
+The commands are case-, underscore- and minus-insensitive.  Thus
+@samp{DIr__PrefiX}, @samp{DIr-PrefiX} and @samp{dirprefix} are the same.
+Empty lines, lines beginning with @samp{#} and lines containing white-space
+only are discarded.
 
 Commands that expect a comma-separated list will clear the list on an
 empty command.  So, if you wish to reset the rejection list specified in
@@ -3126,6 +3134,9 @@ environment.
 Set @sc{http} user to @var{string}, equivalent to
 @samp{--http-user=@var{string}}.
 
+@item https_only = on/off
+When in recursive mode, only HTTPS links are followed (defaults to off).
+
 @item https_proxy = @var{string}
 Use @var{string} as @sc{https} proxy, instead of the one specified in
 environment.

msdos/ChangeLog

@@ -1,61 +0,0 @@
-2014-06-11  Giuseppe Scrivano  <gscrivan@redhat.com>
-
-	* Makefile.DJ: Remove terminating empty lines.
-	* Makefile.WC: Likewise.
-	* msdos.c: Likewise.
-
-2009-09-06  Gisle Vanem  <gvanem@broadpark.no>
-
-	* Makefile.WC: Added compilation of new file msdos.c.
-	Simplified; no need to have explicit rules for files in ./lib and
-	./md5. Just use the suffix search path mechanism in wmake.
-
-2009-09-06  Gisle Vanem  <gvanem@broadpark.no>
-
-	* Makefile.DJ: Added compilation of new file msdos.c.
-
-2009-09-05  Gisle Vanem  <gvanem@broadpark.no>
-
-	* config.h: Added 'HAVE_ALLOCA_H' for Watcom 1.5+.  Added
-	'USE_WATT32' since all DOS-targets use the Watt-32 tcp/ip stack.
-	Added meaningless dummy LOCALEDIR.
-
-	* Makefile.WC: A much needed update. Added rules for many files
-	in ./lib, css.c and version.c.
-
-	* Makefile.DJ: A much needed update. Added rules for css.c and
-	version.c.
-
-2008-01-25  Micah Cowan  <micah@cowan.name>
-
-	* config.h: Updated copyright year.
-
-2007-11-28  Micah Cowan <micah@cowan.name>
-
-	* config.h: Updated license exception for OpenSSL, per the SFLC.
-
-2007-10-15  Gisle Vanem  <gvanem@broadpark.no>
-
-	* config.h: Added some HAVE_* for djgpp 2.04 and Watcom 1.7+.
-
-	* Makefile.DJ: rewritten for including some files from ../lib.
-
-	* Makefile.WC: Ditto. Handling source-files out-of current
-	directory makes compliation a bit more painfull. AFAICS, one
-	must use explicit rules.
-
-2007-10-02  Gisle Vanem  <gvanem@broadpark.no>
-
-	* config.h: Removed unused defines, added needed 'HAVE_*' defines.
-
-	* Makefile.DJ: rewritten to be used from './src' directory.
-	Added '-DOPENSSL_NO_KRB5' for OpenSSL build. Target is
-	now wget.exe.
-
-	* Makefile.WC: Added for building with OpenWatcom targeting
-	32-bit DOS (DOS32A extender). 
-
-2007-09-24  Gisle Vanem  <giva@bgnett.no>
-
-	* Makefile.DJ, config.h: Added to support building on MS-DOS via
-	DJGPP.

po/stamp-po

@@ -1 +0,0 @@
-timestamp

src/Makefile.am

@@ -28,7 +28,7 @@
 # as that of the covered work.
 
 #
-# Version: $(VERSION)
+# Version: @VERSION@
 #
 
 if IRI_IS_ENABLED
@@ -36,13 +36,13 @@ IRI_OBJ = iri.c
 endif
 
 # The following line is losing on some versions of make!
-DEFS     += -DSYSTEM_WGETRC=\"$(sysconfdir)/wgetrc\" -DLOCALEDIR=\"$(localedir)\"
-LIBS     += $(LIBICONV) $(LIBINTL) $(LIB_CLOCK_GETTIME)
+DEFS     = @DEFS@ -DSYSTEM_WGETRC=\"$(sysconfdir)/wgetrc\" -DLOCALEDIR=\"$(localedir)\"
+LIBS     = @LIBICONV@ @LIBINTL@ @LIBS@ $(LIB_CLOCK_GETTIME)
 
 EXTRA_DIST = css.l css.c css_.c build_info.c.in
 
 bin_PROGRAMS = wget
-wget_SOURCES = cmpt.c connect.c convert.c cookies.c ftp.c	\
+wget_SOURCES = connect.c convert.c cookies.c ftp.c	\
 		css_.c css-url.c	\
 		ftp-basic.c ftp-ls.c hash.c host.c html-parse.c html-url.c	\
 		http.c init.c log.c main.c netrc.c progress.c ptimer.c	\
@@ -53,7 +53,7 @@ wget_SOURCES = cmpt.c connect.c convert.c cookies.c ftp.c	\
 		http.h http-ntlm.h init.h log.h mswindows.h netrc.h	\
 		options.h progress.h ptimer.h recur.h res.h retr.h	\
 		spider.h ssl.h sysdep.h url.h warc.h utils.h wget.h iri.h	\
-		exits.h gettext.h
+		exits.h version.h
 nodist_wget_SOURCES = version.c
 EXTRA_wget_SOURCES = iri.c
 LDADD = $(LIBOBJS) ../lib/libgnu.a
@@ -74,7 +74,8 @@ version.c:  $(wget_SOURCES) ../lib/libgnu.a
 	echo '/* version.c */' > $@
 	echo '/* Autogenerated by Makefile - DO NOT EDIT */' >> $@
 	echo '' >> $@
-	echo 'const char *version_string = "$(VERSION)";' >> $@
+	echo '#include "version.h"' >> $@
+	echo 'const char *version_string = "@VERSION@";' >> $@
 	echo 'const char *compilation_string = "'$(COMPILE)'";' \
 	    | $(ESCAPEQUOTE) >> $@
 	echo 'const char *link_string = "'$(CCLD) $(AM_CFLAGS) $(CFLAGS) \
@@ -82,7 +83,7 @@ version.c:  $(wget_SOURCES) ../lib/libgnu.a
 	    | $(ESCAPEQUOTE) >> $@
 
 css.c: $(srcdir)/css.l
-	$(LEX) $(LFLAGS) -o $@ $^
+	$(LEX) $(LFLAGS) -o$@ $^
 
 css_.c: css.c
 	echo '#include "wget.h"' > $@

src/build_info.c.in

@@ -2,7 +2,7 @@ digest          defined ENABLE_DIGEST
 https           defined HAVE_SSL
 ipv6            defined ENABLE_IPV6
 iri             defined ENABLE_IRI
-large-file      SIZEOF_OFF_T >= 8
+large-file      SIZEOF_OFF_T >= 8 || defined WINDOWS
 
 nls             defined ENABLE_NLS
 ntlm            defined ENABLE_NTLM

src/connect.c

@@ -54,6 +54,11 @@ as that of the covered work.  */
 #include <errno.h>
 #include <string.h>
 #include <sys/time.h>
+
+#ifdef ENABLE_IRI
+#include <idn-free.h>
+#endif
+
 #include "utils.h"
 #include "host.h"
 #include "connect.h"
@@ -168,7 +173,7 @@ sockaddr_size (const struct sockaddr *sa)
       abort ();
     }
 }
-
+
 /* Resolve the bind address specified via --bind-address and store it
    to SA.  The resolved value is stored in a static variable and
    reused after the first invocation of this function.
@@ -213,7 +218,7 @@ resolve_bind_address (struct sockaddr *sa)
   should_bind = true;
   return true;
 }
-
+
 struct cwt_context {
   int fd;
   const struct sockaddr *addr;
@@ -250,7 +255,7 @@ connect_with_timeout (int fd, const struct sockaddr *addr, socklen_t addrlen,
     errno = ETIMEDOUT;
   return ctx.result;
 }
-
+
 /* Connect via TCP to the specified address and port.
 
    If PRINT is non-NULL, it is the host name to print that we're
@@ -278,14 +283,13 @@ connect_to_ip (const ip_address *ip, int port, const char *print)
               str = xmalloc (len);
               snprintf (str, len, "%s (%s)", name, print);
               str[len-1] = '\0';
-              xfree (name);
+              idn_free (name);
             }
 
           logprintf (LOG_VERBOSE, _("Connecting to %s|%s|:%d... "),
                      str ? str : escnonprint_uri (print), txt_addr, port);
 
-          if (str)
-              xfree (str);
+          xfree (str);
         }
       else
         {
@@ -367,7 +371,7 @@ connect_to_ip (const ip_address *ip, int port, const char *print)
     if (sock >= 0)
       fd_close (sock);
     if (print)
-      logprintf (LOG_VERBOSE, _("failed: %s.\n"), strerror (errno));
+      logprintf (LOG_NOTQUIET, _("failed: %s.\n"), strerror (errno));
     errno = save_errno;
     return -1;
   }
@@ -429,7 +433,7 @@ connect_to_host (const char *host, int port)
 
   return -1;
 }
-
+
 /* Create a socket, bind it to local interface BIND_ADDRESS on port
    *PORT, set up a listen backlog, and return the resulting socket, or
    -1 in case of error.
@@ -746,7 +750,7 @@ wget uses blocking sockets so we must convert them back to blocking
        or EOF/error. */
     return false;
 }
-
+
 /* Basic socket operations, mostly EINTR wrappers.  */
 
 static int
@@ -794,7 +798,7 @@ sock_close (int fd)
 #undef read
 #undef write
 #undef close
-
+
 /* Reading and writing from the network.  We build around the socket
    (file descriptor) API, but support "extended" operations for things
    that are not mere file descriptors under the hood, such as SSL
@@ -845,7 +849,7 @@ void *
 fd_transport_context (int fd)
 {
   struct transport_info *info = hash_table_get (transport_map, (void *)(intptr_t) fd);
-  return info->ctx;
+  return info ? info->ctx : NULL;
 }
 
 /* When fd_read/fd_write are called multiple times in a loop, they should

src/convert.c

@@ -661,7 +661,7 @@ local_quote_string (const char *file, bool no_html_quote)
 
   return no_html_quote ? strdup (newname) : html_quote_string (newname);
 }
-
+
 /* Book-keeping code for dl_file_url_map, dl_url_file_map,
    downloaded_html_list, and downloaded_html_set.  Other code calls
    these functions to let us know that a file has been downloaded.  */
@@ -916,7 +916,7 @@ convert_cleanup (void)
   if (converted_files)
     string_set_free (converted_files);
 }
-
+
 /* Book-keeping code for downloaded files that enables extension
    hacks.  */
 
@@ -1012,7 +1012,7 @@ downloaded_files_free (void)
       downloaded_files_hash = NULL;
     }
 }
-
+
 /* The function returns the pointer to the malloc-ed quoted version of
    string s.  It will recognize and quote numeric and special graphic
    entities, as per RFC1866:

src/cookies.c

@@ -58,7 +58,9 @@ as that of the covered work.  */
 #include "hash.h"
 #include "cookies.h"
 #include "http.h"               /* for http_atotm */
-
+#include "c-strcase.h"
+
+
 /* Declarations of `struct cookie' and the most basic functions. */
 
 /* Cookie jar serves as cookie storage and a means of retrieving
@@ -151,13 +153,13 @@ cookie_expired_p (const struct cookie *c)
 static void
 delete_cookie (struct cookie *cookie)
 {
-  xfree_null (cookie->domain);
-  xfree_null (cookie->path);
-  xfree_null (cookie->attr);
-  xfree_null (cookie->value);
+  xfree (cookie->domain);
+  xfree (cookie->path);
+  xfree (cookie->attr);
+  xfree (cookie->value);
   xfree (cookie);
 }
-
+
 /* Functions for storing cookies.
 
    All cookies can be reached beginning with jar->chains.  The key in
@@ -302,7 +304,12 @@ discard_matching_cookie (struct cookie_jar *jar, struct cookie *cookie)
 
           res = hash_table_get_pair (jar->chains, victim->domain,
                                      &chain_key, NULL);
-          assert (res != 0);
+
+          if (res == 0)
+            {
+              logprintf (LOG_VERBOSE, _("Unable to get cookie for %s\n"),
+                         victim->domain);
+            }
           if (!victim->next)
             {
               /* VICTIM was the only cookie in the chain.  Destroy the
@@ -317,7 +324,7 @@ discard_matching_cookie (struct cookie_jar *jar, struct cookie *cookie)
       DEBUGP (("Discarded old cookie.\n"));
     }
 }
-
+
 /* Functions for parsing the `Set-Cookie' header, and creating new
    cookies from the wire.  */
 
@@ -369,7 +376,7 @@ parse_set_cookie (const char *set_cookie, bool silent)
         {
           if (!TOKEN_NON_EMPTY (value))
             goto error;
-          xfree_null (cookie->domain);
+          xfree (cookie->domain);
           /* Strictly speaking, we should set cookie->domain_exact if the
              domain doesn't begin with a dot.  But many sites set the
              domain to "foo.com" and expect "subhost.foo.com" to get the
@@ -382,7 +389,7 @@ parse_set_cookie (const char *set_cookie, bool silent)
         {
           if (!TOKEN_NON_EMPTY (value))
             goto error;
-          xfree_null (cookie->path);
+          xfree (cookie->path);
           cookie->path = strdupdelim (value.b, value.e);
         }
       else if (TOKEN_IS (name, "expires"))
@@ -456,7 +463,7 @@ parse_set_cookie (const char *set_cookie, bool silent)
 
 #undef TOKEN_IS
 #undef TOKEN_NON_EMPTY
-
+
 /* Sanity checks.  These are important, otherwise it is possible for
    mailcious attackers to destroy important cookie information and/or
    violate your privacy.  */
@@ -518,12 +525,12 @@ check_domain_match (const char *cookie_domain, const char *host)
 {
 
 #ifdef HAVE_LIBPSL
-  DEBUGP (("cdm: 1"));
   char *cookie_domain_lower = NULL;
   char *host_lower = NULL;
   const psl_ctx_t *psl;
   int is_acceptable;
 
+  DEBUGP (("cdm: 1"));
   if (!(psl = psl_builtin()))
     {
       DEBUGP (("\nlibpsl not built with a public suffix list. "
@@ -692,7 +699,7 @@ check_path_match (const char *cookie_path, const char *path)
   s = PS_newstr;                                                \
 } while (0)
 
-
+
 /* Process the HTTP `Set-Cookie' header.  This results in storing the
    cookie or discarding a matching one, or ignoring it completely, all
    depending on the contents.  */
@@ -776,7 +783,7 @@ cookie_handle_set_cookie (struct cookie_jar *jar,
   if (cookie)
     delete_cookie (cookie);
 }
-
+
 /* Support for sending out cookies in HTTP requests, based on
    previously stored cookies.  Entry point is
    `build_cookies_request'.  */
@@ -1107,7 +1114,7 @@ cookie_header (struct cookie_jar *jar, const char *host,
   assert (pos == result_size);
   return result;
 }
-
+
 /* Support for loading and saving cookies.  The format used for
    loading and saving should be the format of the `cookies.txt' file
    used by Netscape and Mozilla, at least the Unix versions.
@@ -1346,7 +1353,7 @@ cookie_jar_save (struct cookie_jar *jar, const char *file)
 
   DEBUGP (("Done saving cookies.\n"));
 }
-
+
 /* Clean up cookie-related data. */
 
 void
@@ -1369,7 +1376,7 @@ cookie_jar_delete (struct cookie_jar *jar)
   hash_table_destroy (jar->chains);
   xfree (jar);
 }
-
+
 /* Test cases.  Currently this is only tests parse_set_cookies.  To
    use, recompile Wget with -DTEST_COOKIES and call test_cookies()
    from main.  */
@@ -1438,8 +1445,8 @@ test_cookies (void)
               printf ("Invalid value %d for '%s' (expected '%s', got '%s')\n",
                       j / 2 + 1, data, expected[j + 1], v);
             j += 2;
-            free (n);
-            free (v);
+            xfree (n);
+            xfree (v);
           }
         if (expected[j])
           printf ("Too few parameters for '%s'\n", data);

src/exits.c

@@ -56,7 +56,7 @@ get_status_for_err (uerr_t err)
     case FTPNSFOD: case FTPUNKNOWNTYPE: case FTPSRVERR:
     case FTPRETRINT: case FTPRESTFAIL: case FTPNOPASV:
     case CONTNOTSUPPORTED: case RANGEERR: case RETRBADPATTERN:
-    case PROXERR:
+    case PROXERR: case GATEWAYTIMEOUT:
       return WGET_EXIT_SERVER_ERROR;
     case URLERROR: case QUOTEXC: case SSLINITFAILED: case UNKNOWNATTR:
     default:

src/ftp-basic.c

@@ -43,8 +43,9 @@ as that of the covered work.  */
 #include "host.h"
 #include "ftp.h"
 #include "retr.h"
+#include "c-strcase.h"
+
 
-
 /* Get the response of FTP server and allocate enough room to handle
    it.  <CR> and <LF> characters are stripped from the line, and the
    line is 0-terminated.  All the response lines but the last one are
@@ -190,7 +191,7 @@ ftp_login (int csock, const char *acc, const char *pass)
     for (i = 0; i < countof (skey_head); i++)
       {
         int l = strlen (skey_head[i]);
-        if (0 == strncasecmp (skey_head[i], respline, l))
+        if (0 == c_strncasecmp (skey_head[i], respline, l))
           {
             seed = respline + l;
             break;
@@ -784,16 +785,8 @@ ftp_epsv (int csock, ip_address *ip, int *port)
     }
 
   /* Finally, get the port number */
-  tport = 0;
-  for (i = 1; c_isdigit (*s); s++)
-    {
-      if (i > 5)
-        {
-          xfree (respline);
-          return FTPINVPASV;
-        }
+  for (tport = 0, i = 0; i < 5 && c_isdigit (*s); i++, s++)
       tport = (*s - '0') + 10 * tport;
-    }
 
   /* Make sure that the response terminates correcty */
   if (*s++ != delim)
@@ -965,16 +958,18 @@ ftp_list (int csock, const char *file, bool avoid_list_a, bool avoid_list,
   bool ok = false;
   size_t i = 0;
 
-  *list_a_used = false;
-
   /* 2013-10-12 Andrea Urbani (matfanjol)
      For more information about LIST and "LIST -a" please look at ftp.c,
      function getftp, text "__LIST_A_EXPLANATION__".
 
      If somebody changes the following commands, please, checks also the
      later "i" variable.  */
-  const char *list_commands[] = { "LIST -a",
-                                  "LIST" };
+  static const char *list_commands[] = {
+    "LIST -a",
+    "LIST"
+  };
+
+  *list_a_used = false;
 
   if (avoid_list_a)
     {
@@ -1068,25 +1063,25 @@ ftp_syst (int csock, enum stype *server_type, enum ustype *unix_type)
 
   if (request == NULL)
     *server_type = ST_OTHER;
-  else if (!strcasecmp (request, "VMS"))
+  else if (!c_strcasecmp (request, "VMS"))
     *server_type = ST_VMS;
-  else if (!strcasecmp (request, "UNIX"))
+  else if (!c_strcasecmp (request, "UNIX"))
     {
       *server_type = ST_UNIX;
       /* 2013-10-17 Andrea Urbani (matfanjol)
          I check more in depth the system type */
-      if (!strncasecmp (ftp_last_respline, "215 UNIX Type: L8", 17))
+      if (!c_strncasecmp (ftp_last_respline, "215 UNIX Type: L8", 17))
         *unix_type = UST_TYPE_L8;
-      else if (!strncasecmp (ftp_last_respline,
+      else if (!c_strncasecmp (ftp_last_respline,
                              "215 UNIX MultiNet Unix Emulation V5.3(93)", 41))
         *unix_type = UST_MULTINET;
     }
-  else if (!strcasecmp (request, "WINDOWS_NT")
-           || !strcasecmp (request, "WINDOWS2000"))
+  else if (!c_strcasecmp (request, "WINDOWS_NT")
+           || !c_strcasecmp (request, "WINDOWS2000"))
     *server_type = ST_WINNT;
-  else if (!strcasecmp (request, "MACOS"))
+  else if (!c_strcasecmp (request, "MACOS"))
     *server_type = ST_MACOS;
-  else if (!strcasecmp (request, "OS/400"))
+  else if (!c_strcasecmp (request, "OS/400"))
     *server_type = ST_OS400;
   else
     *server_type = ST_OTHER;
@@ -1135,7 +1130,7 @@ ftp_pwd (int csock, char **pwd)
     goto err;
 
   /* Has the `pwd' been already allocated?  Free! */
-  xfree_null (*pwd);
+  xfree (*pwd);
 
   *pwd = xstrdup (request);
 

src/ftp-ls.c

@@ -42,6 +42,7 @@ as that of the covered work.  */
 #include "url.h"
 #include "convert.h"            /* for html_quote_string prototype */
 #include "retr.h"               /* for output_stream */
+#include "c-strcase.h"
 
 /* Converts symbolic permissions to number-style ones, e.g. string
    rwxr-xr-x to 755.  For now, it knows nothing of
@@ -121,7 +122,7 @@ ftp_parse_unix_ls (const char *file, int ignore_perms)
     {
       len = clean_line (line, len);
       /* Skip if total...  */
-      if (!strncasecmp (line, "total", 5))
+      if (!c_strncasecmp (line, "total", 5))
         continue;
       /* Get the first token (permissions).  */
       tok = strtok (line, " ");
@@ -199,7 +200,7 @@ ftp_parse_unix_ls (const char *file, int ignore_perms)
           if (next < 0)         /* a month name was not encountered */
             {
               for (i = 0; i < 12; i++)
-                if (!strcasecmp (tok, months[i]))
+                if (!c_strcasecmp (tok, months[i]))
                   break;
               /* If we got a month, it means the token before it is the
                  size, and the filename is three tokens away.  */
@@ -362,8 +363,8 @@ ftp_parse_unix_ls (const char *file, int ignore_perms)
       if (error || ignore)
         {
           DEBUGP (("Skipping.\n"));
-          xfree_null (cur.name);
-          xfree_null (cur.linkto);
+          xfree (cur.name);
+          xfree (cur.linkto);
           continue;
         }
 
@@ -775,14 +776,14 @@ ftp_parse_vms_ls (const char *file)
          what will work in a CWD command.
       */
       len = strlen (tok);
-      if (!strncasecmp((tok + (len - 4)), ".DIR", 4))
+      if (!c_strncasecmp((tok + (len - 4)), ".DIR", 4))
         {
           *(tok+ (len - 4)) = '\0'; /* Discard ".DIR". */
           cur.type  = FT_DIRECTORY;
           cur.perms = VMS_DEFAULT_PROT_DIR;
           DEBUGP (("Directory (nv)\n"));
         }
-      else if (!strncasecmp ((tok + (len - 6)), ".DIR;1", 6))
+      else if (!c_strncasecmp ((tok + (len - 6)), ".DIR;1", 6))
         {
           *(tok+ (len - 6)) = '\0'; /* Discard ".DIR;1". */
           cur.type  = FT_DIRECTORY;
@@ -1051,7 +1052,7 @@ Unsupported listing type, trying Unix listing parser.\n"));
       return ftp_parse_unix_ls (file, 0);
     }
 }
-
+
 /* Stuff for creating FTP index. */
 
 /* The function creates an HTML index containing references to given
@@ -1088,7 +1089,7 @@ ftp_index (const char *file, struct url *u, struct fileinfo *f)
       else
         upwd = concat_strings (tmpu, "@", (char *) 0);
       xfree (tmpu);
-      xfree_null (tmpp);
+      xfree (tmpp);
     }
   else
     upwd = xstrdup ("");

src/ftp.c

@@ -50,6 +50,7 @@ as that of the covered work.  */
 #include "convert.h"            /* for downloaded_file */
 #include "recur.h"              /* for INFINITE_RECURSION */
 #include "warc.h"
+#include "c-strcase.h"
 
 #ifdef __VMS
 # include "vms.h"
@@ -76,7 +77,6 @@ typedef struct
   struct url *proxy;            /* FTWK-style proxy */
 } ccon;
 
-extern int numurls;
 
 /* Look for regexp "( *[0-9]+ *byte" (literal parenthesis) anywhere in
    the string S, and return the number converted to wgint, if found, 0
@@ -102,7 +102,7 @@ ftp_expected_bytes (const char *s)
         return 0;
       if (c_tolower (*s) != 'b')
         continue;
-      if (strncasecmp (s, "byte", 4))
+      if (c_strncasecmp (s, "byte", 4))
         continue;
       else
         break;
@@ -449,7 +449,7 @@ Error in server response, closing control connection.\n"));
           return err;
         case FTPSRVERR :
           /* PWD unsupported -- assume "/". */
-          xfree_null (con->id);
+          xfree (con->id);
           con->id = xstrdup ("/");
           break;
         case FTPOK:
@@ -700,74 +700,78 @@ Error in server response, closing control connection.\n"));
 
           for (cwd_count = cwd_start; cwd_count < cwd_end; cwd_count++)
             {
-          switch (cwd_count)
-            {
-              case 0:
-                /* Step one (optional): Go to the initial directory,
-                   exactly as reported by the server.
-                */
-                targ = con->id;
-                break;
+              switch (cwd_count)
+                {
+                  case 0:
+                    /* Step one (optional): Go to the initial directory,
+                       exactly as reported by the server.
+                    */
+                    targ = con->id;
+                    break;
 
-              case 1:
-                /* Step two: Go to the target directory.  (Absolute or
-                   relative will work now.)
-                */
-                targ = target;
-                break;
+                  case 1:
+                    /* Step two: Go to the target directory.  (Absolute or
+                       relative will work now.)
+                    */
+                    targ = target;
+                    break;
 
-              case 2:
-                /* Step three (optional): "CWD []" to restore server
-                   VMS-ness.
-                */
-                targ = "[]";
-                break;
+                  case 2:
+                    /* Step three (optional): "CWD []" to restore server
+                       VMS-ness.
+                    */
+                    targ = "[]";
+                    break;
 
-              default:
-                /* Can't happen. */
-                assert (1);
-            }
+                  default:
+                    logprintf (LOG_ALWAYS, _("Logically impossible section reached in getftp()"));
+                    logprintf (LOG_ALWAYS, _("cwd_count: %d\ncwd_start: %d\ncwd_end: %d\n"),
+                                             cwd_count, cwd_start, cwd_end);
+                    abort ();
+                }
 
-          if (!opt.server_response)
-            logprintf (LOG_VERBOSE, "==> CWD (%d) %s ... ", cwd_count,
-                       quotearg_style (escape_quoting_style, target));
-          err = ftp_cwd (csock, targ);
-          /* FTPRERR, WRITEFAILED, FTPNSFOD */
-          switch (err)
-            {
-            case FTPRERR:
-              logputs (LOG_VERBOSE, "\n");
-              logputs (LOG_NOTQUIET, _("\
+              if (!opt.server_response)
+                logprintf (LOG_VERBOSE, "==> CWD (%d) %s ... ", cwd_count,
+                           quotearg_style (escape_quoting_style, target));
+
+              err = ftp_cwd (csock, targ);
+
+              /* FTPRERR, WRITEFAILED, FTPNSFOD */
+              switch (err)
+                {
+                  case FTPRERR:
+                    logputs (LOG_VERBOSE, "\n");
+                    logputs (LOG_NOTQUIET, _("\
 Error in server response, closing control connection.\n"));
-              fd_close (csock);
-              con->csock = -1;
-              return err;
-            case WRITEFAILED:
-              logputs (LOG_VERBOSE, "\n");
-              logputs (LOG_NOTQUIET,
-                       _("Write failed, closing control connection.\n"));
-              fd_close (csock);
-              con->csock = -1;
-              return err;
-            case FTPNSFOD:
-              logputs (LOG_VERBOSE, "\n");
-              logprintf (LOG_NOTQUIET, _("No such directory %s.\n\n"),
-                         quote (u->dir));
-              fd_close (csock);
-              con->csock = -1;
-              return err;
-            case FTPOK:
-              break;
-            default:
-              abort ();
-            }
-          if (!opt.server_response)
-            logputs (LOG_VERBOSE, _("done.\n"));
+                    fd_close (csock);
+                    con->csock = -1;
+                    return err;
+                  case WRITEFAILED:
+                    logputs (LOG_VERBOSE, "\n");
+                    logputs (LOG_NOTQUIET,
+                             _("Write failed, closing control connection.\n"));
+                    fd_close (csock);
+                    con->csock = -1;
+                    return err;
+                  case FTPNSFOD:
+                    logputs (LOG_VERBOSE, "\n");
+                    logprintf (LOG_NOTQUIET, _("No such directory %s.\n\n"),
+                               quote (u->dir));
+                    fd_close (csock);
+                    con->csock = -1;
+                    return err;
+                  case FTPOK:
+                    break;
+                  default:
+                    abort ();
+                }
 
-        } /* for */
+              if (!opt.server_response)
+                logputs (LOG_VERBOSE, _("done.\n"));
+
+            } /* for */
 
           /* 2004-09-20 SMS. */
-          /* End of deviant indenting. */
 
         } /* else */
     }
@@ -1346,7 +1350,6 @@ Error in server response, closing control connection.\n"));
 
   /* Get the server to tell us if everything is retrieved.  */
   err = ftp_response (csock, &respline);
-  *last_expected_bytes = ftp_expected_bytes (respline);
   if (err != FTPOK)
     {
       /* The control connection is decidedly closed.  Print the time
@@ -1362,6 +1365,7 @@ Error in server response, closing control connection.\n"));
       con->csock = -1;
       return FTPRETRINT;
     } /* err != FTPOK */
+  *last_expected_bytes = ftp_expected_bytes (respline);
   /* If retrieval failed for any reason, return FTPRETRINT, but do not
      close socket, since the control connection is still alive.  If
      there is something wrong with the control connection, it will
@@ -1560,6 +1564,7 @@ ftp_loop_internal (struct url *u, struct fileinfo *f, ccon *con, char **local_fi
   else
     {
       /* URL-derived file.  Consider "-O file" name. */
+      xfree (con->target);
       con->target = url_file_name (u, NULL);
       if (!opt.output_document)
         locf = con->target;
@@ -1702,7 +1707,7 @@ ftp_loop_internal (struct url *u, struct fileinfo *f, ccon *con, char **local_fi
           if (err == FOPEN_EXCL_ERR)
             {
               /* Re-determine the file name. */
-              xfree_null (con->target);
+              xfree (con->target);
               con->target = url_file_name (u, NULL);
               locf = con->target;
             }
@@ -2220,9 +2225,9 @@ has_insecure_name_p (const char *s)
 static bool
 is_invalid_entry (struct fileinfo *f)
 {
-  struct fileinfo *cur;
-  cur = f;
+  struct fileinfo *cur = f;
   char *f_name = f->name;
+
   /* If the node we're currently checking has a duplicate later, we eliminate
    * the current node and leave the next one intact. */
   while (cur->next)
@@ -2472,10 +2477,8 @@ ftp_loop (struct url *u, char **local_file, int *dt, struct url *proxy,
   /* If a connection was left, quench it.  */
   if (con.csock != -1)
     fd_close (con.csock);
-  xfree_null (con.id);
-  con.id = NULL;
-  xfree_null (con.target);
-  con.target = NULL;
+  xfree (con.id);
+  xfree (con.target);
   return res;
 }
 
@@ -2489,7 +2492,7 @@ delelement (struct fileinfo *f, struct fileinfo **start)
   struct fileinfo *next = f->next;
 
   xfree (f->name);
-  xfree_null (f->linkto);
+  xfree (f->linkto);
   xfree (f);
 
   if (next)

src/ftp.h

@@ -53,8 +53,6 @@ enum ustype
   UST_OTHER
 };
 
-extern char ftp_last_respline[];
-
 uerr_t ftp_response (int, char **);
 uerr_t ftp_login (int, const char *, const char *);
 uerr_t ftp_port (int, int *);

src/gettext.h

@@ -1,272 +0,0 @@
-/* Convenience header for conditional use of GNU <libintl.h>.
-   Copyright (C) 1995, 1996, 1997, 1998, 2000, 2001, 2002, 2004, 2005,
-   2006, 2009, 2010, 2011 Free Software Foundation, Inc.
-
-   This program is free software; you can redistribute it and/or modify it
-   under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 2, or (at your option)
-   any later version.
-
-   This program is distributed in the hope that it will be useful,
-   but WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Library General Public License for more details.
-
-   You should have received a copy of the GNU General Public
-   License along with this program; if not, write to the Free Software
-   Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301,
-   USA.  */
-
-#ifndef _LIBGETTEXT_H
-#define _LIBGETTEXT_H 1
-
-/* NLS can be disabled through the configure --disable-nls option.  */
-#if ENABLE_NLS
-
-/* Get declarations of GNU message catalog functions.  */
-# include <libintl.h>
-
-/* You can set the DEFAULT_TEXT_DOMAIN macro to specify the domain used by
-   the gettext() and ngettext() macros.  This is an alternative to calling
-   textdomain(), and is useful for libraries.  */
-# ifdef DEFAULT_TEXT_DOMAIN
-#  undef gettext
-#  define gettext(Msgid) \
-     dgettext (DEFAULT_TEXT_DOMAIN, Msgid)
-#  undef ngettext
-#  define ngettext(Msgid1, Msgid2, N) \
-     dngettext (DEFAULT_TEXT_DOMAIN, Msgid1, Msgid2, N)
-# endif
-
-#else
-
-/* Solaris /usr/include/locale.h includes /usr/include/libintl.h, which
-   chokes if dcgettext is defined as a macro.  So include it now, to make
-   later inclusions of <locale.h> a NOP.  We don't include <libintl.h>
-   as well because people using "gettext.h" will not include <libintl.h>,
-   and also including <libintl.h> would fail on SunOS 4, whereas <locale.h>
-   is OK.  */
-#if defined(__sun)
-# include <locale.h>
-#endif
-
-/* Many header files from the libstdc++ coming with g++ 3.3 or newer include
-   <libintl.h>, which chokes if dcgettext is defined as a macro.  So include
-   it now, to make later inclusions of <libintl.h> a NOP.  */
-#if defined(__cplusplus) && defined(__GNUG__) && (__GNUC__ >= 3)
-# include <cstdlib>
-# if (__GLIBC__ >= 2) || _GLIBCXX_HAVE_LIBINTL_H
-#  include <libintl.h>
-# endif
-#endif
-
-/* Disabled NLS.
-   The casts to 'const char *' serve the purpose of producing warnings
-   for invalid uses of the value returned from these functions.
-   On pre-ANSI systems without 'const', the config.h file is supposed to
-   contain "#define const".  */
-# define gettext(Msgid) ((const char *) (Msgid))
-# define dgettext(Domainname, Msgid) ((void) (Domainname), gettext (Msgid))
-# define dcgettext(Domainname, Msgid, Category) \
-    ((void) (Category), dgettext (Domainname, Msgid))
-# define ngettext(Msgid1, Msgid2, N) \
-    ((N) == 1 \
-     ? ((void) (Msgid2), (const char *) (Msgid1)) \
-     : ((void) (Msgid1), (const char *) (Msgid2)))
-# define dngettext(Domainname, Msgid1, Msgid2, N) \
-    ((void) (Domainname), ngettext (Msgid1, Msgid2, N))
-# define dcngettext(Domainname, Msgid1, Msgid2, N, Category) \
-    ((void) (Category), dngettext(Domainname, Msgid1, Msgid2, N))
-# define textdomain(Domainname) ((const char *) (Domainname))
-# define bindtextdomain(Domainname, Dirname) \
-    ((void) (Domainname), (const char *) (Dirname))
-# define bind_textdomain_codeset(Domainname, Codeset) \
-    ((void) (Domainname), (const char *) (Codeset))
-
-#endif
-
-/* A pseudo function call that serves as a marker for the automated
-   extraction of messages, but does not call gettext().  The run-time
-   translation is done at a different place in the code.
-   The argument, String, should be a literal string.  Concatenated strings
-   and other string expressions won't work.
-   The macro's expansion is not parenthesized, so that it is suitable as
-   initializer for static 'char[]' or 'const char[]' variables.  */
-#define gettext_noop(String) String
-
-/* The separator between msgctxt and msgid in a .mo file.  */
-#define GETTEXT_CONTEXT_GLUE "\004"
-
-/* Pseudo function calls, taking a MSGCTXT and a MSGID instead of just a
-   MSGID.  MSGCTXT and MSGID must be string literals.  MSGCTXT should be
-   short and rarely need to change.
-   The letter 'p' stands for 'particular' or 'special'.  */
-#ifdef DEFAULT_TEXT_DOMAIN
-# define pgettext(Msgctxt, Msgid) \
-   pgettext_aux (DEFAULT_TEXT_DOMAIN, Msgctxt GETTEXT_CONTEXT_GLUE Msgid, Msgid, LC_MESSAGES)
-#else
-# define pgettext(Msgctxt, Msgid) \
-   pgettext_aux (NULL, Msgctxt GETTEXT_CONTEXT_GLUE Msgid, Msgid, LC_MESSAGES)
-#endif
-#define dpgettext(Domainname, Msgctxt, Msgid) \
-  pgettext_aux (Domainname, Msgctxt GETTEXT_CONTEXT_GLUE Msgid, Msgid, LC_MESSAGES)
-#define dcpgettext(Domainname, Msgctxt, Msgid, Category) \
-  pgettext_aux (Domainname, Msgctxt GETTEXT_CONTEXT_GLUE Msgid, Msgid, Category)
-#ifdef DEFAULT_TEXT_DOMAIN
-# define npgettext(Msgctxt, Msgid, MsgidPlural, N) \
-   npgettext_aux (DEFAULT_TEXT_DOMAIN, Msgctxt GETTEXT_CONTEXT_GLUE Msgid, Msgid, MsgidPlural, N, LC_MESSAGES)
-#else
-# define npgettext(Msgctxt, Msgid, MsgidPlural, N) \
-   npgettext_aux (NULL, Msgctxt GETTEXT_CONTEXT_GLUE Msgid, Msgid, MsgidPlural, N, LC_MESSAGES)
-#endif
-#define dnpgettext(Domainname, Msgctxt, Msgid, MsgidPlural, N) \
-  npgettext_aux (Domainname, Msgctxt GETTEXT_CONTEXT_GLUE Msgid, Msgid, MsgidPlural, N, LC_MESSAGES)
-#define dcnpgettext(Domainname, Msgctxt, Msgid, MsgidPlural, N, Category) \
-  npgettext_aux (Domainname, Msgctxt GETTEXT_CONTEXT_GLUE Msgid, Msgid, MsgidPlural, N, Category)
-
-#ifdef __GNUC__
-__inline
-#else
-#ifdef __cplusplus
-inline
-#endif
-#endif
-static const char *
-pgettext_aux (const char *domain,
-              const char *msg_ctxt_id, const char *msgid,
-              int category)
-{
-  const char *translation = dcgettext (domain, msg_ctxt_id, category);
-  if (translation == msg_ctxt_id)
-    return msgid;
-  else
-    return translation;
-}
-
-#ifdef __GNUC__
-__inline
-#else
-#ifdef __cplusplus
-inline
-#endif
-#endif
-static const char *
-npgettext_aux (const char *domain,
-               const char *msg_ctxt_id, const char *msgid,
-               const char *msgid_plural, unsigned long int n,
-               int category)
-{
-  const char *translation =
-    dcngettext (domain, msg_ctxt_id, msgid_plural, n, category);
-  if (translation == msg_ctxt_id || translation == msgid_plural)
-    return (n == 1 ? msgid : msgid_plural);
-  else
-    return translation;
-}
-
-/* The same thing extended for non-constant arguments.  Here MSGCTXT and MSGID
-   can be arbitrary expressions.  But for string literals these macros are
-   less efficient than those above.  */
-
-#include <string.h>
-
-#define _LIBGETTEXT_HAVE_VARIABLE_SIZE_ARRAYS \
-  (((__GNUC__ >= 3 || __GNUG__ >= 2) && !__STRICT_ANSI__) \
-   /* || __STDC_VERSION__ >= 199901L */ )
-
-#if !_LIBGETTEXT_HAVE_VARIABLE_SIZE_ARRAYS
-#include <stdlib.h>
-#endif
-
-#define pgettext_expr(Msgctxt, Msgid) \
-  dcpgettext_expr (NULL, Msgctxt, Msgid, LC_MESSAGES)
-#define dpgettext_expr(Domainname, Msgctxt, Msgid) \
-  dcpgettext_expr (Domainname, Msgctxt, Msgid, LC_MESSAGES)
-
-#ifdef __GNUC__
-__inline
-#else
-#ifdef __cplusplus
-inline
-#endif
-#endif
-static const char *
-dcpgettext_expr (const char *domain,
-                 const char *msgctxt, const char *msgid,
-                 int category)
-{
-  size_t msgctxt_len = strlen (msgctxt) + 1;
-  size_t msgid_len = strlen (msgid) + 1;
-  const char *translation;
-#if _LIBGETTEXT_HAVE_VARIABLE_SIZE_ARRAYS
-  char msg_ctxt_id[msgctxt_len + msgid_len];
-#else
-  char buf[1024];
-  char *msg_ctxt_id =
-    (msgctxt_len + msgid_len <= sizeof (buf)
-     ? buf
-     : (char *) malloc (msgctxt_len + msgid_len));
-  if (msg_ctxt_id != NULL)
-#endif
-    {
-      memcpy (msg_ctxt_id, msgctxt, msgctxt_len - 1);
-      msg_ctxt_id[msgctxt_len - 1] = '\004';
-      memcpy (msg_ctxt_id + msgctxt_len, msgid, msgid_len);
-      translation = dcgettext (domain, msg_ctxt_id, category);
-#if !_LIBGETTEXT_HAVE_VARIABLE_SIZE_ARRAYS
-      if (msg_ctxt_id != buf)
-        free (msg_ctxt_id);
-#endif
-      if (translation != msg_ctxt_id)
-        return translation;
-    }
-  return msgid;
-}
-
-#define npgettext_expr(Msgctxt, Msgid, MsgidPlural, N) \
-  dcnpgettext_expr (NULL, Msgctxt, Msgid, MsgidPlural, N, LC_MESSAGES)
-#define dnpgettext_expr(Domainname, Msgctxt, Msgid, MsgidPlural, N) \
-  dcnpgettext_expr (Domainname, Msgctxt, Msgid, MsgidPlural, N, LC_MESSAGES)
-
-#ifdef __GNUC__
-__inline
-#else
-#ifdef __cplusplus
-inline
-#endif
-#endif
-static const char *
-dcnpgettext_expr (const char *domain,
-                  const char *msgctxt, const char *msgid,
-                  const char *msgid_plural, unsigned long int n,
-                  int category)
-{
-  size_t msgctxt_len = strlen (msgctxt) + 1;
-  size_t msgid_len = strlen (msgid) + 1;
-  const char *translation;
-#if _LIBGETTEXT_HAVE_VARIABLE_SIZE_ARRAYS
-  char msg_ctxt_id[msgctxt_len + msgid_len];
-#else
-  char buf[1024];
-  char *msg_ctxt_id =
-    (msgctxt_len + msgid_len <= sizeof (buf)
-     ? buf
-     : (char *) malloc (msgctxt_len + msgid_len));
-  if (msg_ctxt_id != NULL)
-#endif
-    {
-      memcpy (msg_ctxt_id, msgctxt, msgctxt_len - 1);
-      msg_ctxt_id[msgctxt_len - 1] = '\004';
-      memcpy (msg_ctxt_id + msgctxt_len, msgid, msgid_len);
-      translation = dcngettext (domain, msg_ctxt_id, msgid_plural, n, category);
-#if !_LIBGETTEXT_HAVE_VARIABLE_SIZE_ARRAYS
-      if (msg_ctxt_id != buf)
-        free (msg_ctxt_id);
-#endif
-      if (!(translation == msg_ctxt_id || translation == msgid_plural))
-        return translation;
-    }
-  return (n == 1 ? msgid : msgid_plural);
-}
-
-#endif /* _LIBGETTEXT_H */

src/gnutls.c

@@ -122,9 +122,10 @@ ssl_init (void)
           while ((dent = readdir (dir)) != NULL)
             {
               struct stat st;
-              char ca_file[dirlen + strlen(dent->d_name) + 2];
+              size_t ca_file_length = dirlen + strlen(dent->d_name) + 2;
+              char *ca_file = alloca(ca_file_length);
 
-              snprintf (ca_file, sizeof(ca_file), "%s/%s", ca_directory, dent->d_name);
+              snprintf (ca_file, ca_file_length, "%s/%s", ca_directory, dent->d_name);
               if (stat (ca_file, &st) != 0)
                 continue;
 
@@ -138,8 +139,7 @@ ssl_init (void)
               hash_table_put (inode_map, (void *)(intptr_t) st.st_ino, NULL);
               if ((rc = gnutls_certificate_set_x509_trust_file (credentials, ca_file,
                                                                 GNUTLS_X509_FMT_PEM)) <= 0)
-                logprintf (LOG_NOTQUIET, _("ERROR: Failed to open cert %s: (%d).\n"),
-                           ca_file, rc);
+                DEBUGP (("WARNING: Failed to open cert %s: (%d).\n", ca_file, rc));
               else
                 ncerts += rc;
             }
@@ -149,6 +149,36 @@ ssl_init (void)
         }
     }
 
+  if (opt.ca_cert)
+    {
+      int rc;
+
+      ncerts = 0;
+
+      if ((rc = gnutls_certificate_set_x509_trust_file (credentials, opt.ca_cert,
+                                                        GNUTLS_X509_FMT_PEM)) <= 0)
+        logprintf (LOG_NOTQUIET, _ ("ERROR: Failed to open cert %s: (%d).\n"),
+                   opt.ca_cert, rc);
+      else
+        {
+          ncerts += rc;
+          logprintf (LOG_NOTQUIET, _ ("Loaded CA certificate '%s'\n"), opt.ca_cert);
+        }
+    }
+
+  if (opt.crl_file)
+    {
+      int rc;
+
+      if ((rc = gnutls_certificate_set_x509_crl_file (credentials, opt.crl_file, GNUTLS_X509_FMT_PEM)) <= 0)
+        {
+          logprintf (LOG_NOTQUIET, _("ERROR: Failed to load CRL file '%s': (%d)\n"), opt.crl_file, rc);
+          return false;
+        }
+
+      logprintf (LOG_NOTQUIET, _ ("Loaded CRL file '%s'\n"), opt.crl_file);
+    }
+
   DEBUGP (("Certificates loaded: %d\n", ncerts));
 
   /* Use the private key from the cert file unless otherwise specified. */
@@ -181,10 +211,6 @@ cert to be of the same type.\n"));
                                             type);
     }
 
-  if (opt.ca_cert)
-    gnutls_certificate_set_x509_trust_file (credentials, opt.ca_cert,
-                                            GNUTLS_X509_FMT_PEM);
-
   ssl_initialized = true;
 
   return true;
@@ -203,11 +229,6 @@ struct wgnutls_transport_context
   int peeklen;
 };
 
-#ifndef MIN
-# define MIN(i, j) ((i) <= (j) ? (i) : (j))
-#endif
-
-
 static int
 wgnutls_read_timeout (int fd, char *buf, int bufsize, void *arg, double timeout)
 {
@@ -406,10 +427,11 @@ ssl_connect_wget (int fd, const char *hostname)
 #endif
   struct wgnutls_transport_context *ctx;
   gnutls_session_t session;
-  int err,alert;
-  gnutls_init (&session, GNUTLS_CLIENT);
+  int err;
   const char *str;
 
+  gnutls_init (&session, GNUTLS_CLIENT);
+
   /* We set the server name but only if it's not an IP address. */
   if (! is_valid_ip_address (hostname))
     {
@@ -428,27 +450,40 @@ ssl_connect_wget (int fd, const char *hostname)
   gnutls_transport_set_ptr (session, (gnutls_transport_ptr_t) FD_TO_SOCKET (fd));
 #endif
 
-  err = 0;
 #if HAVE_GNUTLS_PRIORITY_SET_DIRECT
   switch (opt.secure_protocol)
     {
     case secure_protocol_auto:
       err = gnutls_priority_set_direct (session, "NORMAL:%COMPAT:-VERS-SSL3.0", NULL);
       break;
+
     case secure_protocol_sslv2:
     case secure_protocol_sslv3:
       err = gnutls_priority_set_direct (session, "NORMAL:-VERS-TLS-ALL:+VERS-SSL3.0", NULL);
       break;
+
     case secure_protocol_tlsv1:
       err = gnutls_priority_set_direct (session, "NORMAL:-VERS-SSL3.0", NULL);
       break;
+
+    case secure_protocol_tlsv1_1:
+      err = gnutls_priority_set_direct (session, "NORMAL:-VERS-SSL3.0:-VERS-TLS1.0", NULL);
+      break;
+
+    case secure_protocol_tlsv1_2:
+      err = gnutls_priority_set_direct (session, "NORMAL:-VERS-SSL3.0:-VERS-TLS1.0:-VERS-TLS1.1", NULL);
+      break;
+
     case secure_protocol_pfs:
       err = gnutls_priority_set_direct (session, "PFS:-VERS-SSL3.0", NULL);
       if (err != GNUTLS_E_SUCCESS)
         /* fallback if PFS is not available */
         err = gnutls_priority_set_direct (session, "NORMAL:-RSA:-VERS-SSL3.0", NULL);
       break;
+
     default:
+      logprintf (LOG_NOTQUIET, _("GnuTLS: unimplemented 'secure-protocol' option value %d\n"), opt.secure_protocol);
+      logprintf (LOG_NOTQUIET, _("Please report this issue to bug-wget@gnu.org\n"));
       abort ();
     }
 #else
@@ -457,6 +492,7 @@ ssl_connect_wget (int fd, const char *hostname)
     {
     case secure_protocol_auto:
       break;
+
     case secure_protocol_sslv2:
     case secure_protocol_sslv3:
       allowed_protocols[0] = GNUTLS_SSL3;
@@ -470,7 +506,20 @@ ssl_connect_wget (int fd, const char *hostname)
       err = gnutls_protocol_set_priority (session, allowed_protocols);
       break;
 
+    case secure_protocol_tlsv1_1:
+      allowed_protocols[0] = GNUTLS_TLS1_1;
+      allowed_protocols[1] = GNUTLS_TLS1_2;
+      err = gnutls_protocol_set_priority (session, allowed_protocols);
+      break;
+
+    case secure_protocol_tlsv1_2:
+      allowed_protocols[0] = GNUTLS_TLS1_2;
+      err = gnutls_protocol_set_priority (session, allowed_protocols);
+      break;
+
     default:
+      logprintf (LOG_NOTQUIET, _("GnuTLS: unimplemented 'secure-protocol' option value %d\n"), opt.secure_protocol);
+      logprintf (LOG_NOTQUIET, _("Please report this issue to bug-wget@gnu.org\n"));
       abort ();
     }
 #endif
@@ -534,7 +583,7 @@ ssl_connect_wget (int fd, const char *hostname)
           if (err == GNUTLS_E_WARNING_ALERT_RECEIVED ||
               err == GNUTLS_E_FATAL_ALERT_RECEIVED)
             {
-              alert = gnutls_alert_get (session);
+              gnutls_alert_description_t alert = gnutls_alert_get (session);
               str = gnutls_alert_get_name (alert);
               if (str == NULL)
                 str = "(unknown)";

src/hash.c

@@ -46,10 +46,11 @@ as that of the covered work.  */
 # include "utils.h"
 #else
 /* Make do without them. */
-# define xnew(x) xmalloc (sizeof (x))
-# define xnew_array(type, x) xmalloc (sizeof (type) * (x))
-# define xmalloc malloc
-# define xfree free
+# define xnew(type) (xmalloc (sizeof (type)))
+# define xnew0(type) (xcalloc (1, sizeof (type)))
+# define xnew_array(type, len) (xmalloc ((len) * sizeof (type)))
+# define xfree(p) do { free ((void *) (p)); p = NULL; } while (0)
+
 # ifndef countof
 #  define countof(x) (sizeof (x) / sizeof ((x)[0]))
 # endif
@@ -585,7 +586,7 @@ hash_table_count (const struct hash_table *ht)
 {
   return ht->count;
 }
-
+
 /* Functions from this point onward are meant for convenience and
    don't strictly belong to this file.  However, this is as good a
    place for them as any.  */
@@ -740,7 +741,7 @@ cmp_pointer (const void *ptr1, const void *ptr2)
 {
   return ptr1 == ptr2;
 }
-
+
 #ifdef TEST
 
 #include <stdio.h>

src/host.c

@@ -57,6 +57,10 @@ as that of the covered work.  */
 
 #include <errno.h>
 
+#ifdef ENABLE_IRI
+#include <idn-free.h>
+#endif
+
 #include "utils.h"
 #include "host.h"
 #include "url.h"
@@ -151,6 +155,13 @@ address_list_set_faulty (struct address_list *al, int index)
      "faulty" attempt is always preceded with all-faulty addresses,
      and this is how Wget uses it.  */
   assert (index == al->faulty);
+  if (index != al->faulty)
+    {
+      logprintf (LOG_ALWAYS, "index: %d\nal->faulty: %d\n", index, al->faulty);
+      logprintf (LOG_ALWAYS, _("Error in handling the address list.\n"));
+      logprintf (LOG_ALWAYS, _("Please report this issue to bug-wget@gnu.org\n"));
+      abort();
+    }
 
   ++al->faulty;
   if (al->faulty >= al->count)
@@ -309,7 +320,7 @@ address_list_release (struct address_list *al)
       address_list_delete (al);
     }
 }
-
+
 /* Versions of gethostbyname and getaddrinfo that support timeout. */
 
 #ifndef ENABLE_IPV6
@@ -408,7 +419,7 @@ getaddrinfo_with_timeout (const char *node, const char *service,
 }
 
 #endif /* ENABLE_IPV6 */
-
+
 /* Return a textual representation of ADDR, i.e. the dotted quad for
    IPv4 addresses, and the colon-separated list of hex words (with all
    zeros omitted, etc.) for IPv6 addresses.  */
@@ -570,7 +581,7 @@ is_valid_ipv6_address (const char *str, const char *end)
 
   return true;
 }
-
+
 /* Simple host cache, used by lookup_host to speed up resolving.  The
    cache doesn't handle TTL because Wget is a fairly short-lived
    application.  Refreshing is attempted when connect fails, though --
@@ -592,7 +603,7 @@ cache_query (const char *host)
   al = hash_table_get (host_name_addresses_map, host);
   if (al)
     {
-      DEBUGP (("Found %s in host_name_addresses_map (%p)\n", host, al));
+      DEBUGP (("Found %s in host_name_addresses_map (%p)\n", host, (void *) al));
       ++al->refcount;
       return al;
     }
@@ -637,7 +648,7 @@ cache_remove (const char *host)
       hash_table_remove (host_name_addresses_map, host);
     }
 }
-
+
 /* Look up HOST in DNS and return a list of IP addresses.
 
    This function caches its result so that, if the same host is passed
@@ -734,14 +745,13 @@ lookup_host (const char *host, int flags)
           str = xmalloc (len);
           snprintf (str, len, "%s (%s)", name, host);
           str[len-1] = '\0';
-          xfree (name);
+          idn_free (name);
         }
 
       logprintf (LOG_VERBOSE, _("Resolving %s... "),
                  quotearg_style (escape_quoting_style, str ? str : host));
 
-      if (str)
-        xfree (str);
+      xfree (str);
     }
 
 #ifdef ENABLE_IPV6
@@ -848,7 +858,7 @@ lookup_host (const char *host, int flags)
 
   return al;
 }
-
+
 /* Determine whether a URL is acceptable to be followed, according to
    a list of domains to accept.  */
 bool

src/html-parse.c

@@ -253,7 +253,7 @@ struct pool {
       (sizevar) = ga_newsize;                                                   \
     }                                                                           \
 } while (0)
-
+
 /* Test whether n+1-sized entity name fits in P.  We don't support
    IE-style non-terminated entities, e.g. "&ltfoo" -> "<foo".
    However, "&lt;foo" will work, as will "&lt!foo", "&lt", etc.  In
@@ -517,7 +517,7 @@ convert_and_copy (struct pool *pool, const char *beg, const char *end, int flags
         *p = c_tolower (*p);
     }
 }
-
+
 /* Originally we used to adhere to rfc 1866 here, and allowed only
    letters, digits, periods, and hyphens as names (of tags or
    attributes).  However, this broke too many pages which used
@@ -770,7 +770,7 @@ find_comment_end (const char *beg, const char *end)
       }
   return NULL;
 }
-
+
 /* Return true if the string containing of characters inside [b, e) is
    present in hash table HT.  */
 
@@ -881,7 +881,7 @@ map_html_tags (const char *text, int size,
     if (*p == '!')
       {
         if (!(flags & MHT_STRICT_COMMENTS)
-            && p < end + 3 && p[1] == '-' && p[2] == '-')
+            && p + 3 < end && p[1] == '-' && p[2] == '-')
           {
             /* If strict comments are not enforced and if we know
                we're looking at a comment, simply look for the
@@ -1167,7 +1167,7 @@ map_html_tags (const char *text, int size,
 #undef ADVANCE
 #undef SKIP_WS
 #undef SKIP_NON_WS
-
+
 #ifdef STANDALONE
 static void
 test_mapper (struct taginfo *taginfo, void *arg)

src/html-url.c

@@ -45,6 +45,7 @@ as that of the covered work.  */
 #include "recur.h"
 #include "html-url.h"
 #include "css-url.h"
+#include "c-strcase.h"
 
 typedef void (*tag_handler_t) (int, struct taginfo *, struct map_context *);
 
@@ -255,7 +256,7 @@ find_attr (struct taginfo *tag, const char *name, int *attrind)
 {
   int i;
   for (i = 0; i < tag->nattrs; i++)
-    if (!strcasecmp (tag->attrs[i].name, name))
+    if (!c_strcasecmp (tag->attrs[i].name, name))
       {
         if (attrind)
           *attrind = i;
@@ -378,7 +379,7 @@ append_url (const char *link_uri, int position, int size,
 
   return newel;
 }
-
+
 static void
 check_style_attr (struct taginfo *tag, struct map_context *ctx)
 {
@@ -487,8 +488,7 @@ tag_handle_base (int tagid _GL_UNUSED, struct taginfo *tag, struct map_context *
   base_urlpos->ignore_when_downloading = 1;
   base_urlpos->link_base_p = 1;
 
-  if (ctx->base)
-    xfree (ctx->base);
+  xfree (ctx->base);
   if (ctx->parent_base)
     ctx->base = uri_merge (ctx->parent_base, newbase);
   else
@@ -536,12 +536,12 @@ tag_handle_link (int tagid _GL_UNUSED, struct taginfo *tag, struct map_context *
           char *rel = find_attr (tag, "rel", NULL);
           if (rel)
             {
-              if (0 == strcasecmp (rel, "stylesheet"))
+              if (0 == c_strcasecmp (rel, "stylesheet"))
                 {
                   up->link_inline_p = 1;
                   up->link_expect_css = 1;
                 }
-              else if (0 == strcasecmp (rel, "shortcut icon"))
+              else if (0 == c_strcasecmp (rel, "shortcut icon"))
                 {
                   up->link_inline_p = 1;
                 }
@@ -553,7 +553,7 @@ tag_handle_link (int tagid _GL_UNUSED, struct taginfo *tag, struct map_context *
                      <link rel="alternate" type="application/rss+xml" href=".../?feed=rss2" />
                   */
                   char *type = find_attr (tag, "type", NULL);
-                  if (!type || strcasecmp (type, "text/html") == 0)
+                  if (!type || c_strcasecmp (type, "text/html") == 0)
                     up->link_expect_html = 1;
                 }
             }
@@ -570,7 +570,7 @@ tag_handle_meta (int tagid _GL_UNUSED, struct taginfo *tag, struct map_context *
   char *name = find_attr (tag, "name", NULL);
   char *http_equiv = find_attr (tag, "http-equiv", NULL);
 
-  if (http_equiv && 0 == strcasecmp (http_equiv, "refresh"))
+  if (http_equiv && 0 == c_strcasecmp (http_equiv, "refresh"))
     {
       /* Some pages use a META tag to specify that the page be
          refreshed by a new page after a given number of seconds.  The
@@ -615,7 +615,7 @@ tag_handle_meta (int tagid _GL_UNUSED, struct taginfo *tag, struct map_context *
           entry->link_expect_html = 1;
         }
     }
-  else if (http_equiv && 0 == strcasecmp (http_equiv, "content-type"))
+  else if (http_equiv && 0 == c_strcasecmp (http_equiv, "content-type"))
     {
       /* Handle stuff like:
          <meta http-equiv="Content-Type" content="text/html; charset=CHARSET"> */
@@ -629,17 +629,17 @@ tag_handle_meta (int tagid _GL_UNUSED, struct taginfo *tag, struct map_context *
       if (!mcharset)
         return;
 
-      xfree_null (meta_charset);
+      xfree (meta_charset);
       meta_charset = mcharset;
     }
-  else if (name && 0 == strcasecmp (name, "robots"))
+  else if (name && 0 == c_strcasecmp (name, "robots"))
     {
       /* Handle stuff like:
          <meta name="robots" content="index,nofollow"> */
       char *content = find_attr (tag, "content", NULL);
       if (!content)
         return;
-      if (!strcasecmp (content, "none"))
+      if (!c_strcasecmp (content, "none"))
         ctx->nofollow = true;
       else
         {
@@ -651,7 +651,7 @@ tag_handle_meta (int tagid _GL_UNUSED, struct taginfo *tag, struct map_context *
               /* Find the next occurrence of ',' or whitespace,
                * or the end of the string.  */
               end = content + strcspn (content, ", \f\n\r\t\v");
-              if (!strncasecmp (content, "nofollow", end - content))
+              if (!c_strncasecmp (content, "nofollow", end - content))
                 ctx->nofollow = true;
               /* Skip past the next comma, if any. */
               if (*end == ',')
@@ -692,7 +692,7 @@ collect_tags_mapper (struct taginfo *tag, void *arg)
 
   check_style_attr (tag, ctx);
 
-  if (tag->end_tag_p && (0 == strcasecmp (tag->name, "style"))
+  if (tag->end_tag_p && (0 == c_strcasecmp (tag->name, "style"))
       && tag->contents_begin && tag->contents_end
       && tag->contents_begin <= tag->contents_end)
   {
@@ -701,7 +701,7 @@ collect_tags_mapper (struct taginfo *tag, void *arg)
                   tag->contents_end - tag->contents_begin);
   }
 }
-
+
 /* Analyze HTML tags FILE and construct a list of URLs referenced from
    it.  It merges relative links in FILE with URL.  It is aware of
    <base href=...> and does the right thing.  */
@@ -757,7 +757,7 @@ get_urls_html (const char *file, const char *url, bool *meta_disallow_follow,
   if (meta_disallow_follow)
     *meta_disallow_follow = ctx.nofollow;
 
-  xfree_null (ctx.base);
+  xfree (ctx.base);
   wget_read_file_free (fm);
   return ctx.head;
 }
@@ -788,6 +788,7 @@ get_urls_file (const char *file)
     {
       int up_error_code;
       char *url_text;
+      char *new_url;
       struct urlpos *entry;
       struct url *url;
 
@@ -822,7 +823,7 @@ get_urls_file (const char *file)
           url_text = merged;
         }
 
-      char *new_url = rewrite_shorthand_url (url_text);
+      new_url = rewrite_shorthand_url (url_text);
       if (new_url)
         {
           xfree (url_text);

src/http-ntlm.c

@@ -74,13 +74,13 @@ as that of the covered work.  */
 /* Define this to make the type-3 message include the NT response message */
 #define USE_NTRESPONSES 1
 
-
+
 /* Flag bits definitions available at on
    http://davenport.sourceforge.net/ntlm.html */
 
 #define NTLMFLAG_NEGOTIATE_OEM                   (1<<1)
 #define NTLMFLAG_NEGOTIATE_NTLM_KEY              (1<<9)
-
+
 /*
   (*) = A "security buffer" is a triplet consisting of two shorts and one
   long:
@@ -474,82 +474,82 @@ ntlm_output (struct ntlmdata *ntlm, const char *user, const char *passwd,
 
     /* Create the big type-3 message binary blob */
 
-    size = (size_t) snprintf (ntlmbuf, sizeof(ntlmbuf),
-                     "NTLMSSP%c"
-                     "\x03%c%c%c" /* type-3, 32 bits */
+    snprintf (ntlmbuf, sizeof (ntlmbuf),
+              "NTLMSSP%c"
+              "\x03%c%c%c" /* type-3, 32 bits */
 
-                     "%c%c%c%c" /* LanManager length + allocated space */
-                     "%c%c" /* LanManager offset */
-                     "%c%c" /* 2 zeroes */
+              "%c%c%c%c" /* LanManager length + allocated space */
+              "%c%c" /* LanManager offset */
+              "%c%c" /* 2 zeroes */
 
-                     "%c%c" /* NT-response length */
-                     "%c%c" /* NT-response allocated space */
-                     "%c%c" /* NT-response offset */
-                     "%c%c" /* 2 zeroes */
+              "%c%c" /* NT-response length */
+              "%c%c" /* NT-response allocated space */
+              "%c%c" /* NT-response offset */
+              "%c%c" /* 2 zeroes */
 
-                     "%c%c"  /* domain length */
-                     "%c%c"  /* domain allocated space */
-                     "%c%c"  /* domain name offset */
-                     "%c%c"  /* 2 zeroes */
+              "%c%c" /* domain length */
+              "%c%c" /* domain allocated space */
+              "%c%c" /* domain name offset */
+              "%c%c" /* 2 zeroes */
 
-                     "%c%c"  /* user length */
-                     "%c%c"  /* user allocated space */
-                     "%c%c"  /* user offset */
-                     "%c%c"  /* 2 zeroes */
+              "%c%c" /* user length */
+              "%c%c" /* user allocated space */
+              "%c%c" /* user offset */
+              "%c%c" /* 2 zeroes */
 
-                     "%c%c"  /* host length */
-                     "%c%c"  /* host allocated space */
-                     "%c%c"  /* host offset */
-                     "%c%c%c%c%c%c"  /* 6 zeroes */
+              "%c%c" /* host length */
+              "%c%c" /* host allocated space */
+              "%c%c" /* host offset */
+              "%c%c%c%c%c%c" /* 6 zeroes */
 
-                     "\xff\xff"  /* message length */
-                     "%c%c"  /* 2 zeroes */
+              "\xff\xff" /* message length */
+              "%c%c" /* 2 zeroes */
 
-                     "\x01\x82" /* flags */
-                     "%c%c"  /* 2 zeroes */
+              "\x01\x82" /* flags */
+              "%c%c" /* 2 zeroes */
 
-                     /* domain string */
-                     /* user string */
-                     /* host string */
-                     /* LanManager response */
-                     /* NT response */
-                     ,
-                     0, /* zero termination */
-                     0,0,0, /* type-3 long, the 24 upper bits */
+              /* domain string */
+              /* user string */
+              /* host string */
+              /* LanManager response */
+              /* NT response */
+              ,
+              0, /* zero termination */
+              0, 0, 0, /* type-3 long, the 24 upper bits */
 
-                     SHORTPAIR(0x18),  /* LanManager response length, twice */
-                     SHORTPAIR(0x18),
-                     SHORTPAIR(lmrespoff),
-                     0x0, 0x0,
+              SHORTPAIR (0x18), /* LanManager response length, twice */
+              SHORTPAIR (0x18),
+              SHORTPAIR (lmrespoff),
+              0x0, 0x0,
 
 #ifdef USE_NTRESPONSES
-                     SHORTPAIR(0x18),  /* NT-response length, twice */
-                     SHORTPAIR(0x18),
+                SHORTPAIR (0x18), /* NT-response length, twice */
+              SHORTPAIR (0x18),
 #else
-                     0x0, 0x0,
-                     0x0, 0x0,
+                0x0, 0x0,
+              0x0, 0x0,
 #endif
-                     SHORTPAIR(ntrespoff),
-                     0x0, 0x0,
+                SHORTPAIR (ntrespoff),
+              0x0, 0x0,
 
-                     SHORTPAIR(domlen),
-                     SHORTPAIR(domlen),
-                     SHORTPAIR(domoff),
-                     0x0, 0x0,
+              SHORTPAIR (domlen),
+              SHORTPAIR (domlen),
+              SHORTPAIR (domoff),
+              0x0, 0x0,
 
-                     SHORTPAIR(userlen),
-                     SHORTPAIR(userlen),
-                     SHORTPAIR(useroff),
-                     0x0, 0x0,
+              SHORTPAIR (userlen),
+              SHORTPAIR (userlen),
+              SHORTPAIR (useroff),
+              0x0, 0x0,
 
-                     SHORTPAIR(hostlen),
-                     SHORTPAIR(hostlen),
-                     SHORTPAIR(hostoff),
-                     0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
+              SHORTPAIR (hostlen),
+              SHORTPAIR (hostlen),
+              SHORTPAIR (hostoff),
+              0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
 
-                     0x0, 0x0,
+              0x0, 0x0,
 
-                     0x0, 0x0);
+              0x0, 0x0);
 
     /* size is now 64 */
     size=64;

src/http.c

@@ -59,6 +59,8 @@ as that of the covered work.  */
 #include "convert.h"
 #include "spider.h"
 #include "warc.h"
+#include "c-strcase.h"
+#include "version.h"
 
 #ifdef TESTING
 #include "test.h"
@@ -68,7 +70,6 @@ as that of the covered work.  */
 # include "vms.h"
 #endif /* def __VMS */
 
-extern char *version_string;
 
 /* Forward decls. */
 struct http_stat;
@@ -80,11 +81,6 @@ static bool known_authentication_scheme_p (const char *, const char *);
 static void ensure_extension (struct http_stat *, const char *, int *);
 static void load_cookies (void);
 
-#ifndef MIN
-# define MIN(x, y) ((x) > (y) ? (y) : (x))
-#endif
-
-
 static bool cookies_loaded_p;
 static struct cookie_jar *wget_cookie_jar;
 
@@ -129,7 +125,8 @@ static struct cookie_jar *wget_cookie_jar;
 #define HTTP_STATUS_NOT_IMPLEMENTED       501
 #define HTTP_STATUS_BAD_GATEWAY           502
 #define HTTP_STATUS_UNAVAILABLE           503
-
+#define HTTP_STATUS_GATEWAY_TIMEOUT       504
+
 enum rp {
   rel_none, rel_name, rel_value, rel_both
 };
@@ -145,7 +142,6 @@ struct request {
   int hcount, hcapacity;
 };
 
-extern int numurls;
 
 /* Create a new, empty request. Set the request's method and its
    arguments.  METHOD should be a literal string (or it should outlive
@@ -235,14 +231,14 @@ request_set_header (struct request *req, const char *name, const char *value,
       /* A NULL value is a no-op; if freeing the name is requested,
          free it now to avoid leaks.  */
       if (release_policy == rel_name || release_policy == rel_both)
-        xfree ((void *)name);
+        xfree (name);
       return;
     }
 
   for (i = 0; i < req->hcount; i++)
     {
       hdr = &req->headers[i];
-      if (0 == strcasecmp (name, hdr->name))
+      if (0 == c_strcasecmp (name, hdr->name))
         {
           /* Replace existing header. */
           release_header (hdr);
@@ -295,7 +291,7 @@ request_remove_header (struct request *req, const char *name)
   for (i = 0; i < req->hcount; i++)
     {
       struct request_header *hdr = &req->headers[i];
-      if (0 == strcasecmp (name, hdr->name))
+      if (0 == c_strcasecmp (name, hdr->name))
         {
           release_header (hdr);
           /* Move the remaining headers by one. */
@@ -386,10 +382,10 @@ static void
 request_free (struct request *req)
 {
   int i;
-  xfree_null (req->arg);
+  xfree (req->arg);
   for (i = 0; i < req->hcount; i++)
     release_header (&req->headers[i]);
-  xfree_null (req->headers);
+  xfree (req->headers);
   xfree (req);
 }
 
@@ -503,7 +499,7 @@ body_file_send (int sock, const char *file_name, wgint promised_size, FILE *warc
   DEBUGP (("done]\n"));
   return 0;
 }
-
+
 /* Determine whether [START, PEEKED + PEEKLEN) contains an empty line.
    If so, return the pointer to the position after the line, otherwise
    return NULL.  This is used as callback to fd_read_hunk.  The data
@@ -680,7 +676,7 @@ resp_header_locate (const struct response *resp, const char *name, int start,
       const char *e = headers[i + 1];
       if (e - b > name_len
           && b[name_len] == ':'
-          && 0 == strncasecmp (b, name, name_len))
+          && 0 == c_strncasecmp (b, name, name_len))
         {
           b += name_len + 1;
           while (b < e && c_isspace (*b))
@@ -817,7 +813,7 @@ resp_status (const struct response *resp, char **message)
 static void
 resp_free (struct response *resp)
 {
-  xfree_null (resp->headers);
+  xfree (resp->headers);
   xfree (resp);
 }
 
@@ -947,7 +943,7 @@ skip_short_body (int fd, wgint contlen, bool chunked)
               if (remaining_chunk_size == 0)
                 {
                   line = fd_read_line (fd);
-                  xfree_null (line);
+                  xfree (line);
                   break;
                 }
             }
@@ -1160,9 +1156,6 @@ append_value_to_filename (char **filename, param_token const * const value,
     url_unescape (*filename + original_length);
 }
 
-#undef MAX
-#define MAX(p, q) ((p) > (q) ? (p) : (q))
-
 /* Parse the contents of the `Content-Disposition' header, extracting
    the information useful to Wget.  Content-Disposition is a header
    borrowed from MIME; when used in HTTP, it typically serves for
@@ -1189,8 +1182,9 @@ static bool
 parse_content_disposition (const char *hdr, char **filename)
 {
   param_token name, value;
-  *filename = NULL;
   bool is_url_encoded = false;
+
+  *filename = NULL;
   for ( ; extract_param (&hdr, &name, &value, ';', &is_url_encoded);
         is_url_encoded = false)
     {
@@ -1224,7 +1218,7 @@ parse_content_disposition (const char *hdr, char **filename)
     return false;
 }
 
-
+
 /* Persistent connections.  Currently, we cache the most recently used
    connection as persistent, provided that the HTTP server agrees to
    make it such.  The persistence data is stored in the variables
@@ -1428,10 +1422,8 @@ persistent_available_p (const char *host, int port, bool ssl,
       if (pconn_active && (fd) == pconn.socket) \
         invalidate_persistent ();               \
       else                                      \
-        {                                       \
           fd_close (fd);                        \
-          fd = -1;                              \
-        }                                       \
+      fd = -1;                                  \
     }                                           \
 } while (0)
 
@@ -1442,7 +1434,7 @@ persistent_available_p (const char *host, int port, bool ssl,
     fd_close (fd);                              \
   fd = -1;                                      \
 } while (0)
-
+
 struct http_stat
 {
   wgint len;                    /* received length */
@@ -1475,18 +1467,13 @@ struct http_stat
 static void
 free_hstat (struct http_stat *hs)
 {
-  xfree_null (hs->newloc);
-  xfree_null (hs->remote_time);
-  xfree_null (hs->error);
-  xfree_null (hs->rderrmsg);
-  xfree_null (hs->local_file);
-  xfree_null (hs->orig_file_name);
-  xfree_null (hs->message);
-
-  /* Guard against being called twice. */
-  hs->newloc = NULL;
-  hs->remote_time = NULL;
-  hs->error = NULL;
+  xfree (hs->newloc);
+  xfree (hs->remote_time);
+  xfree (hs->error);
+  xfree (hs->rderrmsg);
+  xfree (hs->local_file);
+  xfree (hs->orig_file_name);
+  xfree (hs->message);
 }
 
 static void
@@ -1532,6 +1519,7 @@ read_response_body (struct http_stat *hs, int sock, FILE *fp, wgint contlen,
   int warc_payload_offset = 0;
   FILE *warc_tmp = NULL;
   int warcerr = 0;
+  int flags = 0;
 
   if (opt.warc_filename != NULL)
     {
@@ -1568,7 +1556,6 @@ read_response_body (struct http_stat *hs, int sock, FILE *fp, wgint contlen,
     }
 
   /* Read the response body.  */
-  int flags = 0;
   if (contlen != -1)
     /* If content-length is present, read that much; otherwise, read
        until EOF.  The HTTP spec doesn't require the server to
@@ -1633,7 +1620,7 @@ read_response_body (struct http_stat *hs, int sock, FILE *fp, wgint contlen,
 }
 
 #define BEGINS_WITH(line, string_constant)                               \
-  (!strncasecmp (line, string_constant, sizeof (string_constant) - 1)    \
+  (!c_strncasecmp (line, string_constant, sizeof (string_constant) - 1)    \
    && (c_isspace (line[sizeof (string_constant) - 1])                      \
        || !line[sizeof (string_constant) - 1]))
 
@@ -1756,7 +1743,7 @@ gethttp (struct url *u, struct http_stat *hs, int *dt, struct url *proxy,
   hs->res = -1;
   hs->rderrmsg = NULL;
   hs->newloc = NULL;
-  hs->remote_time = NULL;
+  xfree(hs->remote_time);
   hs->error = NULL;
   hs->message = NULL;
 
@@ -1803,6 +1790,7 @@ gethttp (struct url *u, struct http_stat *hs, int *dt, struct url *proxy,
                         rel_value);
   SET_USER_AGENT (req);
   request_set_header (req, "Accept", "*/*", rel_none);
+  request_set_header (req, "Accept-Encoding", "identity", rel_none);
 
   /* Find the username and password for authentication. */
   user = u->user;
@@ -1868,6 +1856,7 @@ gethttp (struct url *u, struct http_stat *hs, int *dt, struct url *proxy,
                 {
                   logprintf (LOG_NOTQUIET, _("BODY data file %s missing: %s\n"),
                              quote (opt.body_file), strerror (errno));
+                  request_free (req);
                   return FILEBADFILE;
                 }
             }
@@ -1875,6 +1864,10 @@ gethttp (struct url *u, struct http_stat *hs, int *dt, struct url *proxy,
                               xstrdup (number_to_static_string (body_data_size)),
                               rel_value);
         }
+      else if (c_strcasecmp (opt.method, "post") == 0
+               || c_strcasecmp (opt.method, "put") == 0
+               || c_strcasecmp (opt.method, "patch") == 0)
+        request_set_header (req, "Content-Length", "0", rel_none);
     }
 
  retry_with_auth:
@@ -2072,6 +2065,7 @@ gethttp (struct url *u, struct http_stat *hs, int *dt, struct url *proxy,
               request_free (req);
               return HERR;
             }
+          xfree(hs->message);
           hs->message = xstrdup (message);
           resp_free (resp);
           xfree (head);
@@ -2080,11 +2074,11 @@ gethttp (struct url *u, struct http_stat *hs, int *dt, struct url *proxy,
             failed_tunnel:
               logprintf (LOG_NOTQUIET, _("Proxy tunneling failed: %s"),
                          message ? quotearg_style (escape_quoting_style, message) : "?");
-              xfree_null (message);
+              xfree (message);
               request_free (req);
               return CONSSLERR;
             }
-          xfree_null (message);
+          xfree (message);
 
           /* SOCK is now *really* connected to u->host, so update CONN
              to reflect this.  That way register_persistent will
@@ -2096,13 +2090,13 @@ gethttp (struct url *u, struct http_stat *hs, int *dt, struct url *proxy,
         {
           if (!ssl_connect_wget (sock, u->host))
             {
-              fd_close (sock);
+              CLOSE_INVALIDATE (sock);
               request_free (req);
               return CONSSLERR;
             }
           else if (!ssl_check_certificate (sock, u->host))
             {
-              fd_close (sock);
+              CLOSE_INVALIDATE (sock);
               request_free (req);
               return VERIFCERTERR;
             }
@@ -2140,11 +2134,13 @@ gethttp (struct url *u, struct http_stat *hs, int *dt, struct url *proxy,
           write_error = fd_write (sock, opt.body_data, body_data_size, -1);
           if (write_error >= 0 && warc_tmp != NULL)
             {
+              int warc_tmp_written;
+
               /* Remember end of headers / start of payload. */
               warc_payload_offset = ftello (warc_tmp);
 
               /* Write a copy of the data to the WARC record. */
-              int warc_tmp_written = fwrite (opt.body_data, 1, body_data_size, warc_tmp);
+              warc_tmp_written = fwrite (opt.body_data, 1, body_data_size, warc_tmp);
               if (warc_tmp_written != body_data_size)
                 write_error = -2;
             }
@@ -2182,8 +2178,9 @@ gethttp (struct url *u, struct http_stat *hs, int *dt, struct url *proxy,
   if (warc_enabled)
     {
       bool warc_result;
+
       /* Generate a timestamp and uuid for this request. */
-      warc_timestamp (warc_timestamp_str);
+      warc_timestamp (warc_timestamp_str, sizeof(warc_timestamp_str));
       warc_uuid_str (warc_request_uuid);
 
       /* Create a request record and store it in the WARC file. */
@@ -2250,6 +2247,7 @@ read_header:
       goto read_header;
     }
 
+  xfree(hs->message);
   hs->message = xstrdup (message);
   if (!opt.server_response)
     logprintf (LOG_VERBOSE, "%2d %s\n", statcode,
@@ -2286,18 +2284,18 @@ read_header:
     }
 
   /* Check for keep-alive related responses. */
-  if (!inhibit_keep_alive && contlen != -1)
+  if (!inhibit_keep_alive)
     {
       if (resp_header_copy (resp, "Connection", hdrval, sizeof (hdrval)))
         {
-          if (0 == strcasecmp (hdrval, "Close"))
+          if (0 == c_strcasecmp (hdrval, "Close"))
             keep_alive = false;
         }
     }
 
   chunked_transfer_encoding = false;
   if (resp_header_copy (resp, "Transfer-Encoding", hdrval, sizeof (hdrval))
-      && 0 == strcasecmp (hdrval, "chunked"))
+      && 0 == c_strcasecmp (hdrval, "chunked"))
     chunked_transfer_encoding = true;
 
   /* Handle (possibly multiple instances of) the Set-Cookie header. */
@@ -2326,6 +2324,7 @@ read_header:
   if (statcode == HTTP_STATUS_UNAUTHORIZED)
     {
       /* Authorization is required.  */
+      uerr_t auth_err = RETROK;
 
       /* Normally we are not interested in the response body.
          But if we are writing a WARC file we are: we like to keep everyting.  */
@@ -2338,13 +2337,13 @@ read_header:
                                     u->url, warc_timestamp_str,
                                     warc_request_uuid, warc_ip, type,
                                     statcode, head);
-          xfree_null (type);
+          xfree (type);
 
           if (_err != RETRFINISHED || hs->res < 0)
             {
               CLOSE_INVALIDATE (sock);
               request_free (req);
-              xfree_null (message);
+              xfree (message);
               resp_free (resp);
               xfree (head);
               return _err;
@@ -2363,33 +2362,70 @@ read_header:
         }
 
       pconn.authorized = false;
-      uerr_t auth_err = RETROK;
       if (!auth_finished && (user && passwd))
         {
           /* IIS sends multiple copies of WWW-Authenticate, one with
              the value "negotiate", and other(s) with data.  Loop over
              all the occurrences and pick the one we recognize.  */
           int wapos;
+          char *buf;
+          const char *www_authenticate = NULL;
           const char *wabeg, *waend;
-          char *www_authenticate = NULL;
-          for (wapos = 0;
-               (wapos = resp_header_locate (resp, "WWW-Authenticate", wapos,
+          const char *digest = NULL, *basic = NULL, *ntlm = NULL;
+          for (wapos = 0; !ntlm
+               && (wapos = resp_header_locate (resp, "WWW-Authenticate", wapos,
                                             &wabeg, &waend)) != -1;
                ++wapos)
-            if (known_authentication_scheme_p (wabeg, waend))
-              {
-                BOUNDED_TO_ALLOCA (wabeg, waend, www_authenticate);
-                break;
-              }
+            {
+              param_token name, value;
 
-          if (!www_authenticate)
+              BOUNDED_TO_ALLOCA (wabeg, waend, buf);
+              www_authenticate = buf;
+
+              for (;!ntlm;)
+                {
+                  /* extract the auth-scheme */
+                  while (c_isspace (*www_authenticate)) www_authenticate++;
+                  name.e = name.b = www_authenticate;
+                  while (*name.e && !c_isspace (*name.e)) name.e++;
+
+                  if (name.b == name.e)
+                    break;
+
+                  DEBUGP (("Auth scheme found '%.*s'\n", (int) (name.e - name.b), name.b));
+
+                  if (known_authentication_scheme_p (name.b, name.e))
+                    {
+                      if (BEGINS_WITH (name.b, "NTLM"))
+                        {
+                          ntlm = name.b;
+                          break; /* this is the most secure challenge, stop here */
+                        }
+                      else if (!digest && BEGINS_WITH (name.b, "Digest"))
+                        digest = name.b;
+                      else if (!basic && BEGINS_WITH (name.b, "Basic"))
+                        basic = name.b;
+                    }
+
+                  /* now advance over the auth-params */
+                  www_authenticate = name.e;
+                  DEBUGP (("Auth param list '%s'\n", www_authenticate));
+                  while (extract_param (&www_authenticate, &name, &value, ',', NULL) && name.b && value.b)
+                    {
+                      DEBUGP (("Auth param %.*s=%.*s\n",
+                               (int) (name.e - name.b), name.b, (int) (value.e - value.b), value.b));
+                    }
+                }
+            }
+
+          if (!basic && !digest && !ntlm)
             {
               /* If the authentication header is missing or
                  unrecognized, there's no sense in retrying.  */
               logputs (LOG_NOTQUIET, _("Unknown authentication scheme.\n"));
             }
           else if (!basic_auth_finished
-                   || !BEGINS_WITH (www_authenticate, "Basic"))
+                   || !basic)
             {
               char *pth = url_full_path (u);
               const char *value;
@@ -2397,6 +2433,15 @@ read_header:
               auth_stat = xmalloc (sizeof (uerr_t));
               *auth_stat = RETROK;
 
+              if (ntlm)
+                www_authenticate = ntlm;
+              else if (digest)
+                www_authenticate = digest;
+              else
+                www_authenticate = basic;
+
+              logprintf (LOG_NOTQUIET, _("Authentication selected: %s\n"), www_authenticate);
+
               value =  create_authorization_line (www_authenticate,
                                                   user, passwd,
                                                   request_method (req),
@@ -2419,7 +2464,7 @@ read_header:
                     }
 
                   xfree (pth);
-                  xfree_null (message);
+                  xfree (message);
                   resp_free (resp);
                   xfree (head);
                   xfree (auth_stat);
@@ -2438,7 +2483,7 @@ read_header:
             }
         }
       request_free (req);
-      xfree_null (message);
+      xfree (message);
       resp_free (resp);
       xfree (head);
       if (auth_err == RETROK)
@@ -2453,6 +2498,23 @@ read_header:
         pconn.authorized = true;
     }
 
+  if (statcode == HTTP_STATUS_GATEWAY_TIMEOUT)
+    {
+      hs->len = 0;
+      hs->res = 0;
+      hs->restval = 0;
+
+      CLOSE_FINISH (sock);
+      request_free (req);
+      xfree (hs->message);
+      xfree (message);
+      resp_free (resp);
+      xfree (head);
+
+      return GATEWAYTIMEOUT;
+    }
+
+
   /* Determine the local filename if needed. Notice that if -O is used
    * hstat.local_file is set by http_loop to the argument of -O. */
   if (!hs->local_file)
@@ -2476,7 +2538,7 @@ read_header:
           hs->local_file = url_file_name (u, local_file);
         }
 
-      xfree_null (local_file);
+      xfree (local_file);
     }
 
   /* TODO: perform this check only once. */
@@ -2491,7 +2553,7 @@ read_header:
           request_free (req);
           resp_free (resp);
           xfree (head);
-          xfree_null (message);
+          xfree (message);
           return RETRUNNEEDED;
         }
       else if (!ALLOW_CLOBBER)
@@ -2574,7 +2636,7 @@ read_header:
     hs->error = xstrdup (_("(no description)"));
   else
     hs->error = xstrdup (message);
-  xfree_null (message);
+  xfree (message);
 
   type = resp_header_strdup (resp, "Content-Type");
   if (type)
@@ -2595,7 +2657,7 @@ read_header:
               tmp = parse_charset (tmp2);
               if (tmp)
                 set_content_encoding (iri, tmp);
-              xfree_null(tmp);
+              xfree(tmp);
             }
         }
     }
@@ -2628,7 +2690,7 @@ read_header:
       hs->restval = 0;
 
       CLOSE_FINISH (sock);
-      xfree_null (type);
+      xfree (type);
       xfree (head);
 
       return RETRFINISHED;
@@ -2669,7 +2731,7 @@ read_header:
               if (_err != RETRFINISHED || hs->res < 0)
                 {
                   CLOSE_INVALIDATE (sock);
-                  xfree_null (type);
+                  xfree (type);
                   xfree (head);
                   return _err;
                 }
@@ -2686,7 +2748,7 @@ read_header:
                 CLOSE_INVALIDATE (sock);
             }
 
-          xfree_null (type);
+          xfree (type);
           xfree (head);
           /* From RFC2616: The status codes 303 and 307 have
              been added for servers that wish to make unambiguously
@@ -2708,11 +2770,11 @@ read_header:
             case HTTP_STATUS_TEMPORARY_REDIRECT:
               return NEWLOCATION_KEEP_POST;
             case HTTP_STATUS_MOVED_PERMANENTLY:
-              if (opt.method && strcasecmp (opt.method, "post") != 0)
+              if (opt.method && c_strcasecmp (opt.method, "post") != 0)
                 return NEWLOCATION_KEEP_POST;
               break;
             case HTTP_STATUS_MOVED_TEMPORARILY:
-              if (opt.method && strcasecmp (opt.method, "post") != 0)
+              if (opt.method && c_strcasecmp (opt.method, "post") != 0)
                 return NEWLOCATION_KEEP_POST;
               break;
             default:
@@ -2769,8 +2831,11 @@ read_header:
       hs->res = 0;
       /* Mark as successfully retrieved. */
       *dt |= RETROKF;
-      xfree_null (type);
-      CLOSE_INVALIDATE (sock);        /* would be CLOSE_FINISH, but there
+      xfree (type);
+      if (statcode == HTTP_STATUS_RANGE_NOT_SATISFIABLE)
+        CLOSE_FINISH (sock);
+      else
+        CLOSE_INVALIDATE (sock);        /* would be CLOSE_FINISH, but there
                                    might be more bytes in the body. */
       xfree (head);
       return RETRUNNEEDED;
@@ -2780,7 +2845,7 @@ read_header:
     {
       /* The Range request was somehow misunderstood by the server.
          Bail out.  */
-      xfree_null (type);
+      xfree (type);
       CLOSE_INVALIDATE (sock);
       xfree (head);
       return RANGEERR;
@@ -2847,7 +2912,7 @@ read_header:
             {
               CLOSE_INVALIDATE (sock);
               xfree (head);
-              xfree_null (type);
+              xfree (type);
               return _err;
             }
           else
@@ -2872,7 +2937,7 @@ read_header:
         }
 
       xfree (head);
-      xfree_null (type);
+      xfree (type);
       return RETRFINISHED;
     }
 
@@ -2913,7 +2978,7 @@ read_header:
                              strerror (errno));
                   CLOSE_INVALIDATE (sock);
                   xfree (head);
-                  xfree_null (type);
+                  xfree (type);
                   return UNLINKERR;
                 }
             }
@@ -2941,7 +3006,7 @@ read_header:
                          hs->local_file);
               CLOSE_INVALIDATE (sock);
               xfree (head);
-              xfree_null (type);
+              xfree (type);
               return FOPEN_EXCL_ERR;
             }
         }
@@ -2950,7 +3015,7 @@ read_header:
           logprintf (LOG_NOTQUIET, "%s: %s\n", hs->local_file, strerror (errno));
           CLOSE_INVALIDATE (sock);
           xfree (head);
-          xfree_null (type);
+          xfree (type);
           return FOPENERR;
         }
     }
@@ -2970,7 +3035,7 @@ read_header:
 
   /* Now we no longer need to store the response header. */
   xfree (head);
-  xfree_null (type);
+  xfree (type);
 
   if (hs->res >= 0)
     CLOSE_FINISH (sock);
@@ -3173,7 +3238,7 @@ Spider mode enabled. Check if remote file exists.\n"));
         {
         case HERR: case HEOF: case CONSOCKERR:
         case CONERROR: case READERR: case WRITEFAILED:
-        case RANGEERR: case FOPEN_EXCL_ERR:
+        case RANGEERR: case FOPEN_EXCL_ERR: case GATEWAYTIMEOUT:
           /* Non-fatal errors continue executing the loop, which will
              bring them to "while" statement at the end, to judge
              whether the number of tries was exceeded.  */
@@ -3290,7 +3355,7 @@ Remote file does not exist -- broken link!!!\n"));
             }
           logputs (LOG_VERBOSE, "\n");
           ret = WRONGCODE;
-          xfree_null (hurl);
+          xfree (hurl);
           goto exit;
         }
 
@@ -3414,8 +3479,8 @@ Remote file exists.\n\n"));
               got_name = true;
               *dt &= ~HEAD_ONLY;
               count = 0;          /* the retrieve count for HEAD is reset */
-              xfree_null (hstat.message);
-              xfree_null (hstat.error);
+              xfree (hstat.message);
+              xfree (hstat.error);
               continue;
             } /* send_head_first */
         } /* !got_head */
@@ -3564,14 +3629,14 @@ Remote file exists.\n\n"));
 exit:
   if (ret == RETROK && local_file)
     {
-      xfree_null (*local_file);
+      xfree (*local_file);
       *local_file = xstrdup (hstat.local_file);
     }
   free_hstat (&hstat);
 
   return ret;
 }
-
+
 /* Check whether the result of strptime() indicates success.
    strptime() returns the pointer to how far it got to in the string.
    The processing has been successful if the string is at `GMT' or
@@ -3682,7 +3747,7 @@ http_atotm (const char *time_string)
 
   return ret;
 }
-
+
 /* Authorization support: We support three authorization schemes:
 
    * `Basic' scheme, consisting of base64-ing USER:PASSWORD string;
@@ -3784,23 +3849,21 @@ digest_authentication_encode (const char *au, const char *user,
   if (qop != NULL && strcmp(qop,"auth"))
     {
       logprintf (LOG_NOTQUIET, _("Unsupported quality of protection '%s'.\n"), qop);
-      xfree_null (qop); /* force freeing mem and return */
-      qop = NULL;
+      xfree (qop); /* force freeing mem and return */
     }
   else if (algorithm != NULL && strcmp (algorithm,"MD5") && strcmp (algorithm,"MD5-sess"))
     {
       logprintf (LOG_NOTQUIET, _("Unsupported algorithm '%s'.\n"), algorithm);
-      xfree_null (qop); /* force freeing mem and return */
-      qop = NULL;
+      xfree (qop); /* force freeing mem and return */
     }
 
   if (!realm || !nonce || !user || !passwd || !path || !method || !qop)
     {
-      xfree_null (realm);
-      xfree_null (opaque);
-      xfree_null (nonce);
-      xfree_null (qop);
-      xfree_null (algorithm);
+      xfree (realm);
+      xfree (opaque);
+      xfree (nonce);
+      xfree (qop);
+      xfree (algorithm);
       if (!qop)
         *auth_err = UNKNOWNATTR;
       else
@@ -3832,7 +3895,7 @@ digest_authentication_encode (const char *au, const char *user,
         snprintf (cnonce, sizeof (cnonce), "%08x", random_number(INT_MAX));
 
         md5_init_ctx (&ctx);
-        // md5_process_bytes (hash, MD5_DIGEST_SIZE, &ctx);
+        /* md5_process_bytes (hash, MD5_DIGEST_SIZE, &ctx); */
         md5_process_bytes (a1buf, MD5_DIGEST_SIZE * 2, &ctx);
         md5_process_bytes ((unsigned char *)":", 1, &ctx);
         md5_process_bytes ((unsigned char *)nonce, strlen (nonce), &ctx);
@@ -3927,11 +3990,11 @@ digest_authentication_encode (const char *au, const char *user,
       }
   }
 
-  xfree_null (realm);
-  xfree_null (opaque);
-  xfree_null (nonce);
-  xfree_null (qop);
-  xfree_null (algorithm);
+  xfree (realm);
+  xfree (opaque);
+  xfree (nonce);
+  xfree (qop);
+  xfree (algorithm);
 
   return res;
 }
@@ -3947,7 +4010,7 @@ digest_authentication_encode (const char *au, const char *user,
 #define STARTS(literal, b, e)                           \
   ((e > b) \
    && ((size_t) ((e) - (b))) >= STRSIZE (literal)   \
-   && 0 == strncasecmp (b, literal, STRSIZE (literal))  \
+   && 0 == c_strncasecmp (b, literal, STRSIZE (literal))  \
    && ((size_t) ((e) - (b)) == STRSIZE (literal)          \
        || c_isspace (b[STRSIZE (literal)])))
 
@@ -4003,7 +4066,7 @@ create_authorization_line (const char *au, const char *user,
       abort ();
     }
 }
-
+
 static void
 load_cookies (void)
 {
@@ -4026,7 +4089,7 @@ save_cookies (void)
 void
 http_cleanup (void)
 {
-  xfree_null (pconn.host);
+  xfree (pconn.host);
   if (wget_cookie_jar)
     cookie_jar_delete (wget_cookie_jar);
 }

src/init.c

@@ -69,6 +69,8 @@ as that of the covered work.  */
 #include "retr.h"               /* for output_stream */
 #include "warc.h"               /* for warc_close */
 #include "spider.h"             /* for spider_cleanup */
+#include "html-url.h"           /* for cleanup_html_url */
+#include "c-strcase.h"
 
 #ifdef TESTING
 #include "test.h"
@@ -90,6 +92,7 @@ CMD_DECLARE (cmd_number_inf);
 CMD_DECLARE (cmd_string);
 CMD_DECLARE (cmd_string_uppercase);
 CMD_DECLARE (cmd_file);
+CMD_DECLARE (cmd_file_once);
 CMD_DECLARE (cmd_directory);
 CMD_DECLARE (cmd_time);
 CMD_DECLARE (cmd_vector);
@@ -101,6 +104,7 @@ CMD_DECLARE (cmd_spec_htmlify);
 CMD_DECLARE (cmd_spec_mirror);
 CMD_DECLARE (cmd_spec_prefer_family);
 CMD_DECLARE (cmd_spec_progress);
+CMD_DECLARE (cmd_spec_progressdisp);
 CMD_DECLARE (cmd_spec_recursive);
 CMD_DECLARE (cmd_spec_regex_type);
 CMD_DECLARE (cmd_spec_restrict_file_names);
@@ -157,6 +161,9 @@ static const struct {
   { "continue",         &opt.always_rest,       cmd_boolean },
   { "convertlinks",     &opt.convert_links,     cmd_boolean },
   { "cookies",          &opt.cookies,           cmd_boolean },
+#ifdef HAVE_SSL
+  { "crlfile",          &opt.crl_file,          cmd_file_once },
+#endif
   { "cutdirs",          &opt.cut_dirs,          cmd_number },
   { "debug",            &opt.debug,             cmd_boolean },
   { "defaultpage",      &opt.default_page,      cmd_string },
@@ -269,7 +276,7 @@ static const struct {
 #endif
   { "serverresponse",   &opt.server_response,   cmd_boolean },
   { "showalldnsentries", &opt.show_all_dns_entries, cmd_boolean },
-  { "showprogress",     &opt.show_progress,      cmd_boolean },
+  { "showprogress",     &opt.show_progress,     cmd_spec_progressdisp },
   { "spanhosts",        &opt.spanhost,          cmd_boolean },
   { "spider",           &opt.spider,            cmd_boolean },
   { "startpos",         &opt.start_pos,         cmd_bytes },
@@ -315,7 +322,7 @@ command_by_name (const char *cmdname)
   while (lo <= hi)
     {
       int mid = (lo + hi) >> 1;
-      int cmp = strcasecmp (cmdname, commands[mid].name);
+      int cmp = c_strcasecmp (cmdname, commands[mid].name);
       if (cmp < 0)
         hi = mid - 1;
       else if (cmp > 0)
@@ -325,7 +332,7 @@ command_by_name (const char *cmdname)
     }
   return -1;
 }
-
+
 /* Reset the variables to default values.  */
 void
 defaults (void)
@@ -428,10 +435,10 @@ defaults (void)
 
   /* Use a negative value to mark the absence of --start-pos option */
   opt.start_pos = -1;
-  opt.show_progress = false;
+  opt.show_progress = -1;
   opt.noscroll = false;
 }
-
+
 /* Return the user's home directory (strdup-ed), or NULL if none is
    found.  */
 char *
@@ -480,7 +487,7 @@ home_dir (void)
     }
 
   ret = home ? xstrdup (home) : NULL;
-  free (buf);
+  xfree (buf);
 
   return ret;
 }
@@ -521,7 +528,7 @@ wgetrc_user_file_name (void)
   home = home_dir ();
   if (home)
     file = aprintf ("%s/.wgetrc", home);
-  xfree_null (home);
+  xfree (home);
 #endif /* def __VMS [else] */
 
   if (!file)
@@ -556,8 +563,7 @@ wgetrc_file_name (void)
   if (!file)
     {
       char *home = home_dir ();
-      xfree_null (file);
-      file = NULL;
+      xfree (file);
       home = ws_mypath ();
       if (home)
         {
@@ -565,7 +571,6 @@ wgetrc_file_name (void)
           if (!file_exists_p (file))
             {
               xfree (file);
-              file = NULL;
             }
           xfree (home);
         }
@@ -639,8 +644,8 @@ run_wgetrc (const char *file)
         default:
           abort ();
         }
-      xfree_null (com);
-      xfree_null (val);
+      xfree (com);
+      xfree (val);
       ++ln;
     }
   xfree (line);
@@ -893,7 +898,7 @@ run_command (const char *cmdopt)
       exit (WGET_EXIT_PARSE_ERROR);
     }
 }
-
+
 /* Generic helper functions, for use with `commands'. */
 
 /* Forward declarations: */
@@ -961,7 +966,7 @@ cmd_number (const char *com, const char *val, void *place)
 static bool
 cmd_number_inf (const char *com, const char *val, void *place)
 {
-  if (!strcasecmp (val, "inf"))
+  if (!c_strcasecmp (val, "inf"))
     {
       *(int *) place = 0;
       return true;
@@ -976,7 +981,7 @@ cmd_string (const char *com _GL_UNUSED, const char *val, void *place)
 {
   char **pstring = (char **)place;
 
-  xfree_null (*pstring);
+  xfree (*pstring);
   *pstring = xstrdup (val);
   return true;
 }
@@ -987,7 +992,7 @@ cmd_string_uppercase (const char *com _GL_UNUSED, const char *val, void *place)
 {
   char *q, **pstring;
   pstring = (char **)place;
-  xfree_null (*pstring);
+  xfree (*pstring);
 
   *pstring = xmalloc (strlen (val) + 1);
 
@@ -1007,7 +1012,7 @@ cmd_file (const char *com _GL_UNUSED, const char *val, void *place)
 {
   char **pstring = (char **)place;
 
-  xfree_null (*pstring);
+  xfree (*pstring);
 
   /* #### If VAL is empty, perhaps should set *PLACE to NULL.  */
 
@@ -1025,6 +1030,20 @@ cmd_file (const char *com _GL_UNUSED, const char *val, void *place)
   return true;
 }
 
+/* like cmd_file, but insist on just a single option usage */
+static bool
+cmd_file_once (const char *com _GL_UNUSED, const char *val, void *place)
+{
+  if (*(char **)place)
+    {
+      fprintf (stderr, _("%s: %s must only be used once\n"),
+               exec_name, com);
+      return false;
+    }
+
+  return cmd_file(com, val, place);
+}
+
 /* Like cmd_file, but strips trailing '/' characters.  */
 static bool
 cmd_directory (const char *com, const char *val, void *place)
@@ -1274,7 +1293,7 @@ cmd_cert_type (const char *com, const char *val, void *place)
   return ok;
 }
 #endif
-
+
 /* Specialized helper functions, used by `commands' to handle some
    options specially.  */
 
@@ -1398,7 +1417,7 @@ cmd_spec_progress (const char *com, const char *val, void *place_ignored _GL_UNU
                exec_name, com, quote (val));
       return false;
     }
-  xfree_null (opt.progress_type);
+  xfree (opt.progress_type);
 
   /* Don't call set_progress_implementation here.  It will be called
      in main when it becomes clear what the log output is.  */
@@ -1499,7 +1518,7 @@ cmd_spec_restrict_file_names (const char *com, const char *val, void *place_igno
 static bool
 cmd_spec_report_speed (const char *com, const char *val, void *place_ignored _GL_UNUSED)
 {
-  opt.report_bps = strcasecmp (val, "bits") == 0;
+  opt.report_bps = c_strcasecmp (val, "bits") == 0;
   if (!opt.report_bps)
     fprintf (stderr, _("%s: %s: Invalid value %s.\n"), exec_name, com, quote (val));
   return opt.report_bps;
@@ -1549,11 +1568,27 @@ cmd_spec_useragent (const char *com, const char *val, void *place_ignored _GL_UN
                exec_name, com, quote (val));
       return false;
     }
-  xfree_null (opt.useragent);
+  xfree (opt.useragent);
   opt.useragent = xstrdup (val);
   return true;
 }
 
+/* The --show-progress option is not a cmd_boolean since we need to keep track
+ * of whether the user explicitly requested the option or not. -1 means
+ * uninitialized. */
+static bool
+cmd_spec_progressdisp (const char *com, const char *val, void *place _GL_UNUSED)
+{
+  bool flag;
+  if (cmd_boolean (com, val, &flag))
+    {
+      opt.show_progress = flag;
+      return true;
+    }
+  return false;
+}
+
+
 /* The "verbose" option cannot be cmd_boolean because the variable is
    not bool -- it's of type int (-1 means uninitialized because of
    some random hackery for disallowing -q -v).  */
@@ -1565,11 +1600,12 @@ cmd_spec_verbose (const char *com, const char *val, void *place_ignored _GL_UNUS
   if (cmd_boolean (com, val, &flag))
     {
       opt.verbose = flag;
+      opt.show_progress = -1;
       return true;
     }
   return false;
 }
-
+
 /* Miscellaneous useful routines.  */
 
 /* A very simple atoi clone, more useful than atoi because it works on
@@ -1704,7 +1740,7 @@ decode_string (const char *val, const struct decode_item *items, int itemcount,
 {
   int i;
   for (i = 0; i < itemcount; i++)
-    if (0 == strcasecmp (val, items[i].name))
+    if (0 == c_strcasecmp (val, items[i].name))
       {
         *place = items[i].code;
         return true;
@@ -1747,55 +1783,57 @@ cleanup (void)
   spider_cleanup ();
   host_cleanup ();
   log_cleanup ();
-  netrc_cleanup (netrc_list);
+  netrc_cleanup ();
 
-  for (i = 0; i < nurl; i++)
-    xfree (url[i]);
-
-  xfree_null (opt.choose_config);
-  xfree_null (opt.lfilename);
-  xfree_null (opt.dir_prefix);
-  xfree_null (opt.input_filename);
-  xfree_null (opt.output_document);
+  xfree (opt.choose_config);
+  xfree (opt.lfilename);
+  xfree (opt.dir_prefix);
+  xfree (opt.input_filename);
+  xfree (opt.output_document);
   free_vec (opt.accepts);
   free_vec (opt.rejects);
-  free_vec (opt.excludes);
-  free_vec (opt.includes);
+  free_vec ((char **)opt.excludes);
+  free_vec ((char **)opt.includes);
   free_vec (opt.domains);
   free_vec (opt.follow_tags);
   free_vec (opt.ignore_tags);
-  xfree_null (opt.progress_type);
-  xfree_null (opt.ftp_user);
-  xfree_null (opt.ftp_passwd);
-  xfree_null (opt.ftp_proxy);
-  xfree_null (opt.https_proxy);
-  xfree_null (opt.http_proxy);
+  xfree (opt.progress_type);
+  xfree (opt.ftp_user);
+  xfree (opt.ftp_passwd);
+  xfree (opt.ftp_proxy);
+  xfree (opt.https_proxy);
+  xfree (opt.http_proxy);
   free_vec (opt.no_proxy);
-  xfree_null (opt.useragent);
-  xfree_null (opt.referer);
-  xfree_null (opt.http_user);
-  xfree_null (opt.http_passwd);
+  xfree (opt.useragent);
+  xfree (opt.referer);
+  xfree (opt.http_user);
+  xfree (opt.http_passwd);
   free_vec (opt.user_headers);
   free_vec (opt.warc_user_headers);
 # ifdef HAVE_SSL
-  xfree_null (opt.cert_file);
-  xfree_null (opt.private_key);
-  xfree_null (opt.ca_directory);
-  xfree_null (opt.ca_cert);
-  xfree_null (opt.random_file);
-  xfree_null (opt.egd_file);
+  xfree (opt.cert_file);
+  xfree (opt.private_key);
+  xfree (opt.ca_directory);
+  xfree (opt.ca_cert);
+  xfree (opt.crl_file);
+  xfree (opt.random_file);
+  xfree (opt.egd_file);
 # endif
-  xfree_null (opt.bind_address);
-  xfree_null (opt.cookies_input);
-  xfree_null (opt.cookies_output);
-  xfree_null (opt.user);
-  xfree_null (opt.passwd);
-  xfree_null (opt.base_href);
-  xfree_null (opt.method);
+  xfree (opt.bind_address);
+  xfree (opt.cookies_input);
+  xfree (opt.cookies_output);
+  xfree (opt.user);
+  xfree (opt.passwd);
+  xfree (opt.base_href);
+  xfree (opt.method);
+  xfree (opt.post_file_name);
+  xfree (opt.post_data);
+  xfree (opt.body_data);
+  xfree (opt.body_file);
 
 #endif /* DEBUG_MALLOC */
 }
-
+
 /* Unit testing routines.  */
 
 #ifdef TESTING
@@ -1807,7 +1845,7 @@ test_commands_sorted(void)
 
   for (i = 1; i < countof(commands); ++i)
     {
-      if (strcasecmp (commands[i - 1].name, commands[i].name) > 0)
+      if (c_strcasecmp (commands[i - 1].name, commands[i].name) > 0)
         {
           mu_assert ("FAILED", false);
           break;
@@ -1823,7 +1861,7 @@ test_cmd_spec_restrict_file_names(void)
   static const struct {
     const char *val;
     int expected_restrict_files_os;
-    int expected_restrict_files_ctrl;
+    bool expected_restrict_files_ctrl;
     int expected_restrict_files_case;
     bool result;
   } test_array[] = {
@@ -1848,9 +1886,9 @@ test_cmd_spec_restrict_file_names(void)
       */
       mu_assert ("test_cmd_spec_restrict_file_names: wrong result",
                  res == test_array[i].result
-                 && opt.restrict_files_os   == test_array[i].expected_restrict_files_os
+                 && (int) opt.restrict_files_os   == test_array[i].expected_restrict_files_os
                  && opt.restrict_files_ctrl == test_array[i].expected_restrict_files_ctrl
-                 && opt.restrict_files_case == test_array[i].expected_restrict_files_case);
+                 && (int) opt.restrict_files_case == test_array[i].expected_restrict_files_case);
     }
 
   return NULL;

src/iri.c

@@ -35,10 +35,14 @@ as that of the covered work.  */
 #include <iconv.h>
 #include <stringprep.h>
 #include <idna.h>
+#include <idn-free.h>
 #include <errno.h>
 
 #include "utils.h"
 #include "url.h"
+#include "c-strcase.h"
+#include "c-strcasestr.h"
+#include "xstrndup.h"
 
 /* RFC3987 section 3.1 mandates STD3 ASCII RULES */
 #define IDNA_FLAGS  IDNA_USE_STD3_ASCII_RULES
@@ -55,7 +59,7 @@ parse_charset (char *str)
   if (!str || !*str)
     return NULL;
 
-  str = strcasestr (str, "charset=");
+  str = c_strcasestr (str, "charset=");
   if (!str)
     return NULL;
 
@@ -123,14 +127,15 @@ do_conversion (const char *tocode, const char *fromcode, char *in, size_t inlen,
   cd = iconv_open (tocode, fromcode);
   if (cd == (iconv_t)(-1))
     {
-      logprintf (LOG_VERBOSE, _("Conversion from %s to %s isn't supported\n"),
-                 quote (opt.locale), quote ("UTF-8"));
+      logprintf (LOG_VERBOSE, _("Conversion from %s to UTF-8 isn't supported\n"),
+                 quote (opt.locale));
+      *out = NULL;
       return false;
     }
 
   /* iconv() has to work on an unescaped string */
   in_org = in;
-  in_save = in = strndup(in, inlen);
+  in_save = in = xstrndup(in, inlen);
   url_unescape(in);
   inlen = strlen(in);
 
@@ -146,7 +151,7 @@ do_conversion (const char *tocode, const char *fromcode, char *in, size_t inlen,
           *(s + len - outlen - done) = '\0';
           xfree(in_save);
           iconv_close(cd);
-          logprintf (LOG_VERBOSE, _("converted '%s' (%s) -> '%s' (%s)\n"), in_org, fromcode, *out, tocode);
+          DEBUGP (("converted '%s' (%s) -> '%s' (%s)\n", in_org, fromcode, *out, tocode));
           return true;
         }
 
@@ -187,7 +192,7 @@ do_conversion (const char *tocode, const char *fromcode, char *in, size_t inlen,
 
     xfree(in_save);
     iconv_close(cd);
-    logprintf (LOG_VERBOSE, _("converted '%s' (%s) -> '%s' (%s)\n"), in_org, fromcode, *out, tocode);
+    DEBUGP (("converted '%s' (%s) -> '%s' (%s)\n", in_org, fromcode, *out, tocode));
     return false;
 }
 
@@ -205,7 +210,7 @@ locale_to_utf8 (const char *str)
       opt.locale = find_locale ();
     }
 
-  if (!opt.locale || !strcasecmp (opt.locale, "utf-8"))
+  if (!opt.locale || !c_strcasecmp (opt.locale, "utf-8"))
     return str;
 
   if (do_conversion ("UTF-8", opt.locale, (char *) str, strlen ((char *) str), &new))
@@ -275,11 +280,11 @@ remote_to_utf8 (struct iri *iri, const char *str, const char **new)
   /* When `i->uri_encoding' == "UTF-8" there is nothing to convert.  But we must
      test for non-ASCII symbols for correct hostname processing in `idn_encode'
      function. */
-  if (!strcasecmp (iri->uri_encoding, "UTF-8"))
+  if (!c_strcasecmp (iri->uri_encoding, "UTF-8"))
     {
-      const char *p = str;
-      for (p = str; *p; p++)
-        if (*p < 0)
+      const unsigned char *p;
+      for (p = (unsigned char *) str; *p; p++)
+        if (*p > 127)
           {
             *new = strdup (str);
             return true;
@@ -291,9 +296,9 @@ remote_to_utf8 (struct iri *iri, const char *str, const char **new)
     ret = true;
 
   /* Test if something was converted */
-  if (!strcmp (str, *new))
+  if (*new && !strcmp (str, *new))
     {
-      xfree ((char *) *new);
+      xfree (*new);
       return false;
     }
 
@@ -327,10 +332,13 @@ struct iri *iri_dup (const struct iri *src)
 void
 iri_free (struct iri *i)
 {
-  xfree_null (i->uri_encoding);
-  xfree_null (i->content_encoding);
-  xfree_null (i->orig_url);
-  xfree (i);
+  if (i)
+    {
+      xfree (i->uri_encoding);
+      xfree (i->content_encoding);
+      xfree (i->orig_url);
+      xfree (i);
+    }
 }
 
 /* Set uri_encoding of struct iri i. If a remote encoding was specified, use
@@ -343,7 +351,7 @@ set_uri_encoding (struct iri *i, char *charset, bool force)
     return;
   if (i->uri_encoding)
     {
-      if (charset && !strcasecmp (i->uri_encoding, charset))
+      if (charset && !c_strcasecmp (i->uri_encoding, charset))
         return;
       xfree (i->uri_encoding);
     }
@@ -360,7 +368,7 @@ set_content_encoding (struct iri *i, char *charset)
     return;
   if (i->content_encoding)
     {
-      if (charset && !strcasecmp (i->content_encoding, charset))
+      if (charset && !c_strcasecmp (i->content_encoding, charset))
         return;
       xfree (i->content_encoding);
     }

src/iri.h

@@ -39,6 +39,9 @@ struct iri {
 
 #ifdef ENABLE_IRI
 
+# include <idna.h>
+# include <idn-free.h>
+
 char *parse_charset (char *str);
 char *find_locale (void);
 bool check_encoding_name (char *encoding);
@@ -62,6 +65,7 @@ extern struct iri dummy_iri;
 #define locale_to_utf8(str)         (str)
 #define idn_encode(a,b)             NULL
 #define idn_decode(str)             NULL
+#define idn_free(str)               ((void)0)
 #define remote_to_utf8(a,b,c)       false
 #define iri_new()                   (&dummy_iri)
 #define iri_dup(a)                  (&dummy_iri)

src/log.c

@@ -73,7 +73,7 @@ as that of the covered work.  */
    - Inhibiting output.  When Wget receives SIGHUP, but redirecting
    the output fails, logging is inhibited.  */
 
-
+
 /* The file descriptor used for logging.  This is NULL before log_init
    is called; logging functions log to stderr then.  log_init sets it
    either to stderr or to a file pointer obtained from fopen().  If
@@ -142,7 +142,7 @@ static int log_line_current = -1;
 static bool trailing_line;
 
 static void check_redirect_output (void);
-
+
 #define ROT_ADVANCE(num) do {                   \
   if (++num >= SAVED_LOG_LINES)                 \
     num = 0;                                    \
@@ -156,11 +156,7 @@ static void
 free_log_line (int num)
 {
   struct log_ln *ln = log_lines + num;
-  if (ln->malloced_line)
-    {
-      xfree (ln->malloced_line);
-      ln->malloced_line = NULL;
-    }
+  xfree (ln->malloced_line);
   ln->content = NULL;
 }
 
@@ -265,7 +261,7 @@ saved_append (const char *s)
       s = end;
     }
 }
-
+
 /* Check X against opt.verbose and opt.quiet.  The semantics is as
    follows:
 
@@ -314,6 +310,14 @@ get_log_fp (void)
   return stderr;
 }
 
+static FILE *
+get_progress_fp (void)
+{
+  if (opt.show_progress == true)
+      return stderr;
+  return get_log_fp();
+}
+
 /* Returns the file descriptor for the secondary log file. This is
    WARCLOGFP, except if called before log_init, in which case it
    returns stderr.  This is useful in case someone calls a logging
@@ -338,7 +342,7 @@ log_set_warc_log_fp (FILE * fp)
 {
   warclogfp = fp;
 }
-
+
 /* Log a literal string S.  The string is logged as-is, without a
    newline appended.  */
 
@@ -349,8 +353,14 @@ logputs (enum log_options o, const char *s)
   FILE *warcfp;
 
   check_redirect_output ();
-  if ((fp = get_log_fp ()) == NULL)
+  if (o == LOG_PROGRESS)
+    fp = get_progress_fp ();
+  else
+    fp = get_log_fp ();
+
+  if (fp == NULL)
     return;
+
   warcfp = get_warc_log_fp ();
   CHECK_VERBOSE (o);
 
@@ -385,7 +395,7 @@ struct logvprintf_state {
    (An alternative approach would be to use va_copy, but that's not
    portable.)  */
 
-static bool
+static bool GCC_FORMAT_ATTR (2, 0)
 log_vprintf_internal (struct logvprintf_state *state, const char *fmt,
                       va_list args)
 {
@@ -451,8 +461,7 @@ log_vprintf_internal (struct logvprintf_state *state, const char *fmt,
   FPUTS (write_ptr, fp);
   if (warcfp != NULL)
     FPUTS (write_ptr, warcfp);
-  if (state->bigmsg)
-    xfree (state->bigmsg);
+  xfree (state->bigmsg);
 
  flush:
   if (flush_log_p)
@@ -541,6 +550,7 @@ logprintf (enum log_options o, const char *fmt, ...)
   CHECK_VERBOSE (o);
 
   xzero (lpstate);
+  errno = 0;
   do
     {
       va_start (args, fmt);
@@ -580,7 +590,7 @@ debug_logprintf (const char *fmt, ...)
     }
 }
 #endif /* ENABLE_DEBUG */
-
+
 /* Open FILE and set up a logging stream.  If FILE cannot be opened,
    exit with status of 1.  */
 void
@@ -675,7 +685,7 @@ log_dump_context (void)
   fflush (fp);
   fflush (warcfp);
 }
-
+
 /* String escape functions. */
 
 /* Return the number of non-printable characters in SOURCE.
@@ -847,9 +857,9 @@ log_cleanup (void)
 {
   size_t i;
   for (i = 0; i < countof (ring); i++)
-    xfree_null (ring[i].buffer);
+    xfree (ring[i].buffer);
 }
-
+
 /* When SIGHUP or SIGUSR1 are received, the output is redirected
    elsewhere.  Such redirection is only allowed once. */
 static enum { RR_NONE, RR_REQUESTED, RR_DONE } redirect_request = RR_NONE;

src/main.c

@@ -54,6 +54,9 @@ as that of the covered work.  */
 #include "http.h"               /* for save_cookies */
 #include "ptimer.h"
 #include "warc.h"
+#include "version.h"
+#include "c-strcase.h"
+#include "dirname.h"
 #include <getopt.h>
 #include <getpass.h>
 #include <quote.h>
@@ -78,30 +81,46 @@ struct iri dummy_iri;
 struct options opt;
 
 /* defined in version.c */
-extern char *version_string;
-extern char *compilation_string;
 extern char *system_getrc;
-extern char *link_string;
-/* defined in build_info.c */
-extern const char *compiled_features[];
 /* Used for --version output in print_version */
 #define MAX_CHARS_PER_LINE      72
 #define TABULATION              4
 
-#if defined(SIGHUP) || defined(SIGUSR1)
-static void redirect_output_signal (int);
-#endif
-
 const char *exec_name;
 
 /* Number of successfully downloaded URLs */
 int numurls = 0;
-
+
 #ifndef TESTING
 /* Initialize I18N/L10N.  That amounts to invoking setlocale, and
    setting up gettext's message catalog using bindtextdomain and
    textdomain.  Does nothing if NLS is disabled or missing.  */
 
+#if defined(SIGHUP) || defined(SIGUSR1)
+/* Hangup signal handler.  When wget receives SIGHUP or SIGUSR1, it
+   will proceed operation as usual, trying to write into a log file.
+   If that is impossible, the output will be turned off.  */
+
+static void
+redirect_output_signal (int sig)
+{
+  const char *signal_name = "WTF?!";
+
+#ifdef SIGHUP
+  if (sig == SIGHUP)
+    signal_name = "SIGHUP";
+#endif
+#ifdef SIGUSR1
+  if (sig == SIGUSR1)
+    signal_name = "SIGUSR1";
+#endif
+
+  log_request_redirect_output (signal_name);
+  progress_schedule_redirect ();
+  signal (sig, redirect_output_signal);
+}
+#endif /* defined(SIGHUP) || defined(SIGUSR1) */
+
 static void
 i18n_initialize (void)
 {
@@ -114,11 +133,11 @@ i18n_initialize (void)
   textdomain ("wget");
 #endif /* ENABLE_NLS */
 }
-
+
 /* Definition of command-line options. */
 
-static void print_help (void);
-static void print_version (void);
+static void _Noreturn print_help (void);
+static void _Noreturn print_version (void);
 
 #ifdef HAVE_SSL
 # define IF_SSL(x) x
@@ -175,6 +194,7 @@ static struct cmdline_option option_data[] =
     { "content-disposition", 0, OPT_BOOLEAN, "contentdisposition", -1 },
     { "content-on-error", 0, OPT_BOOLEAN, "contentonerror", -1 },
     { "cookies", 0, OPT_BOOLEAN, "cookies", -1 },
+    { IF_SSL ("crl-file"), 0, OPT_VALUE, "crlfile", -1 },
     { "cut-dirs", 0, OPT_VALUE, "cutdirs", -1 },
     { "debug", 'd', OPT_BOOLEAN, "debug", -1 },
     { "default-page", 0, OPT_VALUE, "defaultpage", -1 },
@@ -428,206 +448,206 @@ Mandatory arguments to long options are mandatory for short options too.\n\n"),
     N_("\
 Startup:\n"),
     N_("\
-  -V,  --version                   display the version of Wget and exit.\n"),
+  -V,  --version                   display the version of Wget and exit\n"),
     N_("\
-  -h,  --help                      print this help.\n"),
+  -h,  --help                      print this help\n"),
     N_("\
-  -b,  --background                go to background after startup.\n"),
+  -b,  --background                go to background after startup\n"),
     N_("\
-  -e,  --execute=COMMAND           execute a `.wgetrc'-style command.\n"),
+  -e,  --execute=COMMAND           execute a `.wgetrc'-style command\n"),
     "\n",
 
     N_("\
 Logging and input file:\n"),
     N_("\
-  -o,  --output-file=FILE          log messages to FILE.\n"),
+  -o,  --output-file=FILE          log messages to FILE\n"),
     N_("\
-  -a,  --append-output=FILE        append messages to FILE.\n"),
+  -a,  --append-output=FILE        append messages to FILE\n"),
 #ifdef ENABLE_DEBUG
     N_("\
-  -d,  --debug                     print lots of debugging information.\n"),
+  -d,  --debug                     print lots of debugging information\n"),
 #endif
 #ifdef USE_WATT32
     N_("\
-       --wdebug                    print Watt-32 debug output.\n"),
+       --wdebug                    print Watt-32 debug output\n"),
 #endif
     N_("\
-  -q,  --quiet                     quiet (no output).\n"),
+  -q,  --quiet                     quiet (no output)\n"),
     N_("\
-  -v,  --verbose                   be verbose (this is the default).\n"),
+  -v,  --verbose                   be verbose (this is the default)\n"),
     N_("\
-  -nv, --no-verbose                turn off verboseness, without being quiet.\n"),
+  -nv, --no-verbose                turn off verboseness, without being quiet\n"),
     N_("\
-       --report-speed=TYPE         Output bandwidth as TYPE.  TYPE can be bits.\n"),
+       --report-speed=TYPE         output bandwidth as TYPE.  TYPE can be bits\n"),
     N_("\
-  -i,  --input-file=FILE           download URLs found in local or external FILE.\n"),
+  -i,  --input-file=FILE           download URLs found in local or external FILE\n"),
     N_("\
-  -F,  --force-html                treat input file as HTML.\n"),
+  -F,  --force-html                treat input file as HTML\n"),
     N_("\
   -B,  --base=URL                  resolves HTML input-file links (-i -F)\n\
-                                   relative to URL.\n"),
+                                     relative to URL\n"),
     N_("\
-       --config=FILE               Specify config file to use.\n"),
+       --config=FILE               specify config file to use\n"),
     N_("\
-       --no-config                 Do not read any config file.\n"),
+       --no-config                 do not read any config file\n"),
     "\n",
 
     N_("\
 Download:\n"),
     N_("\
-  -t,  --tries=NUMBER              set number of retries to NUMBER (0 unlimits).\n"),
+  -t,  --tries=NUMBER              set number of retries to NUMBER (0 unlimits)\n"),
     N_("\
-       --retry-connrefused         retry even if connection is refused.\n"),
+       --retry-connrefused         retry even if connection is refused\n"),
     N_("\
-  -O,  --output-document=FILE      write documents to FILE.\n"),
+  -O,  --output-document=FILE      write documents to FILE\n"),
     N_("\
   -nc, --no-clobber                skip downloads that would download to\n\
-                                   existing files (overwriting them).\n"),
+                                     existing files (overwriting them)\n"),
     N_("\
-  -c,  --continue                  resume getting a partially-downloaded file.\n"),
+  -c,  --continue                  resume getting a partially-downloaded file\n"),
     N_("\
-       --start-pos=OFFSET          start downloading from zero-based position OFFSET.\n"),
+       --start-pos=OFFSET          start downloading from zero-based position OFFSET\n"),
     N_("\
-       --progress=TYPE             select progress gauge type.\n"),
+       --progress=TYPE             select progress gauge type\n"),
     N_("\
-       --show-progress             display the progress bar in any verbosity mode.\n"),
+       --show-progress             display the progress bar in any verbosity mode\n"),
     N_("\
   -N,  --timestamping              don't re-retrieve files unless newer than\n\
-                                   local.\n"),
+                                     local\n"),
     N_("\
   --no-use-server-timestamps       don't set the local file's timestamp by\n\
-                                   the one on the server.\n"),
+                                     the one on the server\n"),
     N_("\
-  -S,  --server-response           print server response.\n"),
+  -S,  --server-response           print server response\n"),
     N_("\
-       --spider                    don't download anything.\n"),
+       --spider                    don't download anything\n"),
     N_("\
-  -T,  --timeout=SECONDS           set all timeout values to SECONDS.\n"),
+  -T,  --timeout=SECONDS           set all timeout values to SECONDS\n"),
     N_("\
-       --dns-timeout=SECS          set the DNS lookup timeout to SECS.\n"),
+       --dns-timeout=SECS          set the DNS lookup timeout to SECS\n"),
     N_("\
-       --connect-timeout=SECS      set the connect timeout to SECS.\n"),
+       --connect-timeout=SECS      set the connect timeout to SECS\n"),
     N_("\
-       --read-timeout=SECS         set the read timeout to SECS.\n"),
+       --read-timeout=SECS         set the read timeout to SECS\n"),
     N_("\
-  -w,  --wait=SECONDS              wait SECONDS between retrievals.\n"),
+  -w,  --wait=SECONDS              wait SECONDS between retrievals\n"),
     N_("\
-       --waitretry=SECONDS         wait 1..SECONDS between retries of a retrieval.\n"),
+       --waitretry=SECONDS         wait 1..SECONDS between retries of a retrieval\n"),
     N_("\
-       --random-wait               wait from 0.5*WAIT...1.5*WAIT secs between retrievals.\n"),
+       --random-wait               wait from 0.5*WAIT...1.5*WAIT secs between retrievals\n"),
     N_("\
-       --no-proxy                  explicitly turn off proxy.\n"),
+       --no-proxy                  explicitly turn off proxy\n"),
     N_("\
-  -Q,  --quota=NUMBER              set retrieval quota to NUMBER.\n"),
+  -Q,  --quota=NUMBER              set retrieval quota to NUMBER\n"),
     N_("\
-       --bind-address=ADDRESS      bind to ADDRESS (hostname or IP) on local host.\n"),
+       --bind-address=ADDRESS      bind to ADDRESS (hostname or IP) on local host\n"),
     N_("\
-       --limit-rate=RATE           limit download rate to RATE.\n"),
+       --limit-rate=RATE           limit download rate to RATE\n"),
     N_("\
-       --no-dns-cache              disable caching DNS lookups.\n"),
+       --no-dns-cache              disable caching DNS lookups\n"),
     N_("\
-       --restrict-file-names=OS    restrict chars in file names to ones OS allows.\n"),
+       --restrict-file-names=OS    restrict chars in file names to ones OS allows\n"),
     N_("\
-       --ignore-case               ignore case when matching files/directories.\n"),
+       --ignore-case               ignore case when matching files/directories\n"),
 #ifdef ENABLE_IPV6
     N_("\
-  -4,  --inet4-only                connect only to IPv4 addresses.\n"),
+  -4,  --inet4-only                connect only to IPv4 addresses\n"),
     N_("\
-  -6,  --inet6-only                connect only to IPv6 addresses.\n"),
+  -6,  --inet6-only                connect only to IPv6 addresses\n"),
     N_("\
        --prefer-family=FAMILY      connect first to addresses of specified family,\n\
-                                   one of IPv6, IPv4, or none.\n"),
+                                     one of IPv6, IPv4, or none\n"),
 #endif
     N_("\
-       --user=USER                 set both ftp and http user to USER.\n"),
+       --user=USER                 set both ftp and http user to USER\n"),
     N_("\
-       --password=PASS             set both ftp and http password to PASS.\n"),
+       --password=PASS             set both ftp and http password to PASS\n"),
     N_("\
-       --ask-password              prompt for passwords.\n"),
+       --ask-password              prompt for passwords\n"),
     N_("\
-       --no-iri                    turn off IRI support.\n"),
+       --no-iri                    turn off IRI support\n"),
     N_("\
-       --local-encoding=ENC        use ENC as the local encoding for IRIs.\n"),
+       --local-encoding=ENC        use ENC as the local encoding for IRIs\n"),
     N_("\
-       --remote-encoding=ENC       use ENC as the default remote encoding.\n"),
+       --remote-encoding=ENC       use ENC as the default remote encoding\n"),
     N_("\
-       --unlink                    remove file before clobber.\n"),
+       --unlink                    remove file before clobber\n"),
     "\n",
 
     N_("\
 Directories:\n"),
     N_("\
-  -nd, --no-directories            don't create directories.\n"),
+  -nd, --no-directories            don't create directories\n"),
     N_("\
-  -x,  --force-directories         force creation of directories.\n"),
+  -x,  --force-directories         force creation of directories\n"),
     N_("\
-  -nH, --no-host-directories       don't create host directories.\n"),
+  -nH, --no-host-directories       don't create host directories\n"),
     N_("\
-       --protocol-directories      use protocol name in directories.\n"),
+       --protocol-directories      use protocol name in directories\n"),
     N_("\
-  -P,  --directory-prefix=PREFIX   save files to PREFIX/...\n"),
+  -P,  --directory-prefix=PREFIX   save files to PREFIX/..\n"),
     N_("\
-       --cut-dirs=NUMBER           ignore NUMBER remote directory components.\n"),
+       --cut-dirs=NUMBER           ignore NUMBER remote directory components\n"),
     "\n",
 
     N_("\
 HTTP options:\n"),
     N_("\
-       --http-user=USER            set http user to USER.\n"),
+       --http-user=USER            set http user to USER\n"),
     N_("\
-       --http-password=PASS        set http password to PASS.\n"),
+       --http-password=PASS        set http password to PASS\n"),
     N_("\
-       --no-cache                  disallow server-cached data.\n"),
+       --no-cache                  disallow server-cached data\n"),
     N_ ("\
-       --default-page=NAME         Change the default page name (normally\n\
-                                   this is `index.html'.).\n"),
+       --default-page=NAME         change the default page name (normally\n\
+                                     this is 'index.html'.)\n"),
     N_("\
-  -E,  --adjust-extension          save HTML/CSS documents with proper extensions.\n"),
+  -E,  --adjust-extension          save HTML/CSS documents with proper extensions\n"),
     N_("\
-       --ignore-length             ignore `Content-Length' header field.\n"),
+       --ignore-length             ignore 'Content-Length' header field\n"),
     N_("\
-       --header=STRING             insert STRING among the headers.\n"),
+       --header=STRING             insert STRING among the headers\n"),
     N_("\
-       --max-redirect              maximum redirections allowed per page.\n"),
+       --max-redirect              maximum redirections allowed per page\n"),
     N_("\
-       --proxy-user=USER           set USER as proxy username.\n"),
+       --proxy-user=USER           set USER as proxy username\n"),
     N_("\
-       --proxy-password=PASS       set PASS as proxy password.\n"),
+       --proxy-password=PASS       set PASS as proxy password\n"),
     N_("\
-       --referer=URL               include `Referer: URL' header in HTTP request.\n"),
+       --referer=URL               include 'Referer: URL' header in HTTP request\n"),
     N_("\
-       --save-headers              save the HTTP headers to file.\n"),
+       --save-headers              save the HTTP headers to file\n"),
     N_("\
-  -U,  --user-agent=AGENT          identify as AGENT instead of Wget/VERSION.\n"),
+  -U,  --user-agent=AGENT          identify as AGENT instead of Wget/VERSION\n"),
     N_("\
-       --no-http-keep-alive        disable HTTP keep-alive (persistent connections).\n"),
+       --no-http-keep-alive        disable HTTP keep-alive (persistent connections)\n"),
     N_("\
-       --no-cookies                don't use cookies.\n"),
+       --no-cookies                don't use cookies\n"),
     N_("\
-       --load-cookies=FILE         load cookies from FILE before session.\n"),
+       --load-cookies=FILE         load cookies from FILE before session\n"),
     N_("\
-       --save-cookies=FILE         save cookies to FILE after session.\n"),
+       --save-cookies=FILE         save cookies to FILE after session\n"),
     N_("\
-       --keep-session-cookies      load and save session (non-permanent) cookies.\n"),
+       --keep-session-cookies      load and save session (non-permanent) cookies\n"),
     N_("\
-       --post-data=STRING          use the POST method; send STRING as the data.\n"),
+       --post-data=STRING          use the POST method; send STRING as the data\n"),
     N_("\
-       --post-file=FILE            use the POST method; send contents of FILE.\n"),
+       --post-file=FILE            use the POST method; send contents of FILE\n"),
     N_("\
-       --method=HTTPMethod         use method \"HTTPMethod\" in the request.\n"),
+       --method=HTTPMethod         use method \"HTTPMethod\" in the request\n"),
     N_("\
-       --body-data=STRING          Send STRING as data. --method MUST be set.\n"),
+       --body-data=STRING          send STRING as data. --method MUST be set\n"),
     N_("\
-       --body-file=FILE            Send contents of FILE. --method MUST be set.\n"),
+       --body-file=FILE            send contents of FILE. --method MUST be set\n"),
     N_("\
        --content-disposition       honor the Content-Disposition header when\n\
-                                   choosing local file names (EXPERIMENTAL).\n"),
+                                     choosing local file names (EXPERIMENTAL)\n"),
     N_("\
-       --content-on-error          output the received content on server errors.\n"),
+       --content-on-error          output the received content on server errors\n"),
     N_("\
        --auth-no-challenge         send Basic HTTP authentication information\n\
-                                   without first waiting for the server's\n\
-                                   challenge.\n"),
+                                     without first waiting for the server's\n\
+                                     challenge\n"),
     "\n",
 
 #ifdef HAVE_SSL
@@ -635,27 +655,33 @@ HTTP options:\n"),
 HTTPS (SSL/TLS) options:\n"),
     N_("\
        --secure-protocol=PR        choose secure protocol, one of auto, SSLv2,\n\
-                                   SSLv3, TLSv1 and PFS.\n"),
+                                     SSLv3, TLSv1 and PFS\n"),
     N_("\
        --https-only                only follow secure HTTPS links\n"),
     N_("\
-       --no-check-certificate      don't validate the server's certificate.\n"),
+       --no-check-certificate      don't validate the server's certificate\n"),
     N_("\
-       --certificate=FILE          client certificate file.\n"),
+       --certificate=FILE          client certificate file\n"),
     N_("\
-       --certificate-type=TYPE     client certificate type, PEM or DER.\n"),
+       --certificate-type=TYPE     client certificate type, PEM or DER\n"),
     N_("\
-       --private-key=FILE          private key file.\n"),
+       --private-key=FILE          private key file\n"),
     N_("\
-       --private-key-type=TYPE     private key type, PEM or DER.\n"),
+       --private-key-type=TYPE     private key type, PEM or DER\n"),
     N_("\
-       --ca-certificate=FILE       file with the bundle of CA's.\n"),
+       --ca-certificate=FILE       file with the bundle of CAs\n"),
     N_("\
-       --ca-directory=DIR          directory where hash list of CA's is stored.\n"),
+       --ca-directory=DIR          directory where hash list of CAs is stored\n"),
     N_("\
-       --random-file=FILE          file with random data for seeding the SSL PRNG.\n"),
+       --crl-file=FILE             file with bundle of CRLs\n"),
+#if defined(HAVE_LIBSSL) || defined(HAVE_LIBSSL32)
     N_("\
-       --egd-file=FILE             file naming the EGD socket with random data.\n"),
+       --random-file=FILE          file with random data for seeding the SSL PRNG\n"),
+#endif
+#if (defined(HAVE_LIBSSL) || defined(HAVE_LIBSSL32)) && defined(HAVE_RAND_EGD)
+    N_("\
+       --egd-file=FILE             file naming the EGD socket with random data\n"),
+#endif
     "\n",
 #endif /* HAVE_SSL */
 
@@ -663,120 +689,120 @@ HTTPS (SSL/TLS) options:\n"),
 FTP options:\n"),
 #ifdef __VMS
     N_("\
-       --ftp-stmlf                 Use Stream_LF format for all binary FTP files.\n"),
+       --ftp-stmlf                 use Stream_LF format for all binary FTP files\n"),
 #endif /* def __VMS */
     N_("\
-       --ftp-user=USER             set ftp user to USER.\n"),
+       --ftp-user=USER             set ftp user to USER\n"),
     N_("\
-       --ftp-password=PASS         set ftp password to PASS.\n"),
+       --ftp-password=PASS         set ftp password to PASS\n"),
     N_("\
-       --no-remove-listing         don't remove `.listing' files.\n"),
+       --no-remove-listing         don't remove '.listing' files\n"),
     N_("\
-       --no-glob                   turn off FTP file name globbing.\n"),
+       --no-glob                   turn off FTP file name globbing\n"),
     N_("\
-       --no-passive-ftp            disable the \"passive\" transfer mode.\n"),
+       --no-passive-ftp            disable the \"passive\" transfer mode\n"),
     N_("\
-       --preserve-permissions      preserve remote file permissions.\n"),
+       --preserve-permissions      preserve remote file permissions\n"),
     N_("\
-       --retr-symlinks             when recursing, get linked-to files (not dir).\n"),
+       --retr-symlinks             when recursing, get linked-to files (not dir)\n"),
     "\n",
 
     N_("\
 WARC options:\n"),
     N_("\
-       --warc-file=FILENAME        save request/response data to a .warc.gz file.\n"),
+       --warc-file=FILENAME        save request/response data to a .warc.gz file\n"),
     N_("\
-       --warc-header=STRING        insert STRING into the warcinfo record.\n"),
+       --warc-header=STRING        insert STRING into the warcinfo record\n"),
     N_("\
-       --warc-max-size=NUMBER      set maximum size of WARC files to NUMBER.\n"),
+       --warc-max-size=NUMBER      set maximum size of WARC files to NUMBER\n"),
     N_("\
-       --warc-cdx                  write CDX index files.\n"),
+       --warc-cdx                  write CDX index files\n"),
     N_("\
-       --warc-dedup=FILENAME       do not store records listed in this CDX file.\n"),
+       --warc-dedup=FILENAME       do not store records listed in this CDX file\n"),
 #ifdef HAVE_LIBZ
     N_("\
-       --no-warc-compression       do not compress WARC files with GZIP.\n"),
+       --no-warc-compression       do not compress WARC files with GZIP\n"),
 #endif
     N_("\
-       --no-warc-digests           do not calculate SHA1 digests.\n"),
+       --no-warc-digests           do not calculate SHA1 digests\n"),
     N_("\
-       --no-warc-keep-log          do not store the log file in a WARC record.\n"),
+       --no-warc-keep-log          do not store the log file in a WARC record\n"),
     N_("\
        --warc-tempdir=DIRECTORY    location for temporary files created by the\n\
-                                   WARC writer.\n"),
+                                     WARC writer\n"),
     "\n",
 
     N_("\
 Recursive download:\n"),
     N_("\
-  -r,  --recursive                 specify recursive download.\n"),
+  -r,  --recursive                 specify recursive download\n"),
     N_("\
-  -l,  --level=NUMBER              maximum recursion depth (inf or 0 for infinite).\n"),
+  -l,  --level=NUMBER              maximum recursion depth (inf or 0 for infinite)\n"),
     N_("\
-       --delete-after              delete files locally after downloading them.\n"),
+       --delete-after              delete files locally after downloading them\n"),
     N_("\
   -k,  --convert-links             make links in downloaded HTML or CSS point to\n\
-                                   local files.\n"),
+                                     local files\n"),
     N_("\
-       --backups=N                 before writing file X, rotate up to N backup files.\n"),
+       --backups=N                 before writing file X, rotate up to N backup files\n"),
 
 #ifdef __VMS
     N_("\
-  -K,  --backup-converted          before converting file X, back up as X_orig.\n"),
+  -K,  --backup-converted          before converting file X, back up as X_orig\n"),
 #else /* def __VMS */
     N_("\
-  -K,  --backup-converted          before converting file X, back up as X.orig.\n"),
+  -K,  --backup-converted          before converting file X, back up as X.orig\n"),
 #endif /* def __VMS [else] */
     N_("\
-  -m,  --mirror                    shortcut for -N -r -l inf --no-remove-listing.\n"),
+  -m,  --mirror                    shortcut for -N -r -l inf --no-remove-listing\n"),
     N_("\
-  -p,  --page-requisites           get all images, etc. needed to display HTML page.\n"),
+  -p,  --page-requisites           get all images, etc. needed to display HTML page\n"),
     N_("\
-       --strict-comments           turn on strict (SGML) handling of HTML comments.\n"),
+       --strict-comments           turn on strict (SGML) handling of HTML comments\n"),
     "\n",
 
     N_("\
 Recursive accept/reject:\n"),
     N_("\
-  -A,  --accept=LIST               comma-separated list of accepted extensions.\n"),
+  -A,  --accept=LIST               comma-separated list of accepted extensions\n"),
     N_("\
-  -R,  --reject=LIST               comma-separated list of rejected extensions.\n"),
+  -R,  --reject=LIST               comma-separated list of rejected extensions\n"),
     N_("\
-       --accept-regex=REGEX        regex matching accepted URLs.\n"),
+       --accept-regex=REGEX        regex matching accepted URLs\n"),
     N_("\
-       --reject-regex=REGEX        regex matching rejected URLs.\n"),
+       --reject-regex=REGEX        regex matching rejected URLs\n"),
 #ifdef HAVE_LIBPCRE
     N_("\
-       --regex-type=TYPE           regex type (posix|pcre).\n"),
+       --regex-type=TYPE           regex type (posix|pcre)\n"),
 #else
     N_("\
-       --regex-type=TYPE           regex type (posix).\n"),
+       --regex-type=TYPE           regex type (posix)\n"),
 #endif
     N_("\
-  -D,  --domains=LIST              comma-separated list of accepted domains.\n"),
+  -D,  --domains=LIST              comma-separated list of accepted domains\n"),
     N_("\
-       --exclude-domains=LIST      comma-separated list of rejected domains.\n"),
+       --exclude-domains=LIST      comma-separated list of rejected domains\n"),
     N_("\
-       --follow-ftp                follow FTP links from HTML documents.\n"),
+       --follow-ftp                follow FTP links from HTML documents\n"),
     N_("\
-       --follow-tags=LIST          comma-separated list of followed HTML tags.\n"),
+       --follow-tags=LIST          comma-separated list of followed HTML tags\n"),
     N_("\
-       --ignore-tags=LIST          comma-separated list of ignored HTML tags.\n"),
+       --ignore-tags=LIST          comma-separated list of ignored HTML tags\n"),
     N_("\
-  -H,  --span-hosts                go to foreign hosts when recursive.\n"),
+  -H,  --span-hosts                go to foreign hosts when recursive\n"),
     N_("\
-  -L,  --relative                  follow relative links only.\n"),
+  -L,  --relative                  follow relative links only\n"),
     N_("\
-  -I,  --include-directories=LIST  list of allowed directories.\n"),
+  -I,  --include-directories=LIST  list of allowed directories\n"),
     N_("\
        --trust-server-names        use the name specified by the redirection\n\
-                                   url last component.\n"),
+                                     URL's last component\n"),
     N_("\
-  -X,  --exclude-directories=LIST  list of excluded directories.\n"),
+  -X,  --exclude-directories=LIST  list of excluded directories\n"),
     N_("\
-  -np, --no-parent                 don't ascend to the parent directory.\n"),
+  -np, --no-parent                 don't ascend to the parent directory\n"),
     "\n",
-    N_("Mail bug reports and suggestions to <bug-wget@gnu.org>.\n")
+    N_("Mail bug reports and suggestions to <bug-wget@gnu.org>\n")
   };
 
   size_t i;
@@ -867,7 +893,10 @@ format_and_print_line (const char *prefix, const char *line,
           remaining_chars = line_length - TABULATION;
         }
       if (printf ("%s ", token) < 0)
-        return -1;
+        {
+          xfree (line_dup);
+          return -1;
+        }
       remaining_chars -= strlen (token) + 1;  /* account for " " */
       token = strtok (NULL, " ");
     }
@@ -977,24 +1006,27 @@ There is NO WARRANTY, to the extent permitted by law.\n"), stdout) < 0)
   exit (WGET_EXIT_SUCCESS);
 }
 
-char *program_name; /* Needed by lib/error.c. */
-char *program_argstring; /* Needed by wget_warc.c. */
+const char *program_name; /* Needed by lib/error.c. */
+const char *program_argstring; /* Needed by wget_warc.c. */
 
 int
 main (int argc, char **argv)
 {
-  char **url, **t;
+  char **url, **t, *p;
   int i, ret, longindex;
   int nurl;
+  int retconf;
+  int argstring_length;
+  bool use_userconfig = false;
+  bool noconfig = false;
   bool append_to_log = false;
 
-  total_downloaded_bytes = 0;
-
-  program_name = argv[0];
-
   struct ptimer *timer = ptimer_new ();
   double start_time = ptimer_measure (timer);
 
+  total_downloaded_bytes = 0;
+  program_name = argv[0];
+
   i18n_initialize ();
 
   /* Construct the name of the executable, without the directory part.  */
@@ -1002,11 +1034,7 @@ main (int argc, char **argv)
   /* On VMS, lose the "dev:[dir]" prefix and the ".EXE;nnn" suffix. */
   exec_name = vms_basename (argv[0]);
 #else /* def __VMS */
-  exec_name = strrchr (argv[0], PATH_SEPARATOR);
-  if (!exec_name)
-    exec_name = argv[0];
-  else
-    ++exec_name;
+  exec_name = base_name (argv[0]);
 #endif /* def __VMS [else] */
 
 #ifdef WINDOWS
@@ -1015,10 +1043,9 @@ main (int argc, char **argv)
 #endif
 
   /* Construct the arguments string. */
-  int argstring_length = 1;
-  for (i = 1; i < argc; i++)
-    argstring_length += strlen (argv[i]) + 2 + 1;
-  char *p = program_argstring = malloc (argstring_length * sizeof (char));
+  for (argstring_length = 1, i = 1; i < argc; i++)
+    argstring_length += strlen (argv[i]) + 3 + 1;
+  program_argstring = p = malloc (argstring_length);
   if (p == NULL)
     {
       fprintf (stderr, _("Memory allocation problem\n"));
@@ -1026,8 +1053,10 @@ main (int argc, char **argv)
     }
   for (i = 1; i < argc; i++)
     {
+      int arglen;
+
       *p++ = '"';
-      int arglen = strlen (argv[i]);
+      arglen = strlen (argv[i]);
       memcpy (p, argv[i], arglen);
       p += arglen;
       *p++ = '"';
@@ -1043,9 +1072,6 @@ main (int argc, char **argv)
   /* This separate getopt_long is needed to find the user config file
      option ("--config") and parse it before the other user options. */
   longindex = -1;
-  int retconf;
-  bool use_userconfig = false;
-  bool noconfig = false;
 
   while ((retconf = getopt_long (argc, argv,
                                 short_options, long_options, &longindex)) != -1)
@@ -1246,9 +1272,6 @@ main (int argc, char **argv)
   if (opt.verbose == -1)
     opt.verbose = !opt.quiet;
 
-  if (opt.verbose == 1)
-    opt.show_progress = true;
-
   /* Sanity checks.  */
   if (opt.verbose && opt.quiet)
     {
@@ -1438,7 +1461,7 @@ for details.\n\n"));
 
   /* When user specifies HEAD as the method, we do not wish to download any
      files. Hence, set wget to run in spider mode.  */
-  if (opt.method && strcasecmp (opt.method, "HEAD") == 0)
+  if (opt.method && c_strcasecmp (opt.method, "HEAD") == 0)
     setoptval ("spider", "1", "spider");
 
   /* Convert post_data to body-data and post_file_name to body-file options.
@@ -1455,12 +1478,12 @@ for details.\n\n"));
         if (opt.post_data)
           {
             setoptval ("bodydata", opt.post_data, "body-data");
-            opt.post_data = NULL;
+            xfree(opt.post_data);
           }
         else
           {
             setoptval ("bodyfile", opt.post_file_name, "body-file");
-            opt.post_file_name = NULL;
+            xfree(opt.post_file_name);
           }
     }
 
@@ -1671,8 +1694,8 @@ outputting to a regular file.\n"));
               if (unlink (filename))
                 logprintf (LOG_NOTQUIET, "unlink: %s\n", strerror (errno));
             }
-          xfree_null (redirected_URL);
-          xfree_null (filename);
+          xfree (redirected_URL);
+          xfree (filename);
           url_free (url_parsed);
         }
       iri_free (iri);
@@ -1702,10 +1725,11 @@ outputting to a regular file.\n"));
       total_downloaded_bytes != 0)
     {
       double end_time = ptimer_measure (timer);
-      ptimer_destroy (timer);
-
       char *wall_time = xstrdup (secs_to_human_time (end_time - start_time));
       char *download_time = xstrdup (secs_to_human_time (total_download_time));
+
+      ptimer_destroy (timer);
+
       logprintf (LOG_NOTQUIET,
                  _("FINISHED --%s--\nTotal wall clock time: %s\n"
                    "Downloaded: %d files, %s in %s (%s)\n"),
@@ -1735,33 +1759,8 @@ outputting to a regular file.\n"));
 
   exit (get_exit_status ());
 }
+
 #endif /* TESTING */
-
-#if defined(SIGHUP) || defined(SIGUSR1)
-
-/* So the signal_name check doesn't blow when only one is available. */
-#ifndef SIGHUP
-# define SIGHUP -1
-#endif
-#ifndef SIGUSR1
-# define SIGUSR1 -1
-#endif
-
-/* Hangup signal handler.  When wget receives SIGHUP or SIGUSR1, it
-   will proceed operation as usual, trying to write into a log file.
-   If that is impossible, the output will be turned off.  */
-
-static void
-redirect_output_signal (int sig)
-{
-  const char *signal_name = (sig == SIGHUP ? "SIGHUP" :
-                             (sig == SIGUSR1 ? "SIGUSR1" :
-                              "WTF?!"));
-  log_request_redirect_output (signal_name);
-  progress_schedule_redirect ();
-  signal (sig, redirect_output_signal);
-}
-#endif
 
 /*
  * vim: et ts=2 sw=2

src/mswindows.c

@@ -85,11 +85,11 @@ windows_main (char **exec_name)
   if (p)
     *p = '\0';
 }
-
+
 static void
 ws_cleanup (void)
 {
-  xfree ((char*)exec_name);
+  xfree (exec_name);
   WSACleanup ();
 }
 
@@ -367,8 +367,8 @@ static int old_percentage = -1;
 void
 ws_changetitle (const char *url)
 {
-  xfree_null (title_buf);
-  xfree_null (curr_url);
+  xfree (title_buf);
+  xfree (curr_url);
   title_buf = xmalloc (strlen (url) + 20);
   curr_url = xstrdup (url);
   old_percentage = -1;
@@ -477,7 +477,7 @@ ws_startup (void)
   set_sleep_mode ();
   SetConsoleCtrlHandler (ws_handler, TRUE);
 }
-
+
 /* run_with_timeout Windows implementation.  */
 
 /* Stack size 0 uses default thread stack-size (reserve+commit).

src/netrc.c

@@ -163,7 +163,7 @@ search_netrc (const char *host, const char **acc, const char **passwd,
 
 /* Normally, these functions would be defined by your package.  */
 # define xmalloc malloc
-# define xfree free
+# define xfree(p) do { free ((void *) (p)); p = NULL; } while (0)
 # define xstrdup strdup
 
 # define xrealloc realloc
@@ -183,9 +183,9 @@ maybe_add_to_list (acc_t **newentry, acc_t **list)
   if (a && ! a->acc)
     {
       /* Free any allocated space.  */
-      xfree_null (a->host);
-      xfree_null (a->acc);
-      xfree_null (a->passwd);
+      xfree (a->host);
+      xfree (a->acc);
+      xfree (a->passwd);
     }
   else
     {
@@ -424,9 +424,9 @@ free_netrc(acc_t *l)
   while (l)
     {
       t = l->next;
-      xfree_null (l->acc);
-      xfree_null (l->passwd);
-      xfree_null (l->host);
+      xfree (l->acc);
+      xfree (l->passwd);
+      xfree (l->host);
       xfree (l);
       l = t;
     }

src/openssl.c

@@ -40,8 +40,10 @@ as that of the covered work.  */
 #include <openssl/x509v3.h>
 #include <openssl/err.h>
 #include <openssl/rand.h>
+#include <openssl/bio.h>
 #if OPENSSL_VERSION_NUMBER >= 0x00907000
 #include <openssl/conf.h>
+#include <openssl/engine.h>
 #endif
 
 #include "utils.h"
@@ -89,9 +91,11 @@ init_prng (void)
   if (RAND_status ())
     return;
 
+#ifdef HAVE_RAND_EGD
   /* Get random data from EGD if opt.egd_file was used.  */
   if (opt.egd_file && *opt.egd_file)
     RAND_egd (opt.egd_file);
+#endif
 
   if (RAND_status ())
     return;
@@ -167,6 +171,9 @@ static int ssl_true_initialized = 0;
 bool
 ssl_init (void)
 {
+  SSL_METHOD const *meth;
+  long ssl_options = 0;
+
 #if OPENSSL_VERSION_NUMBER >= 0x00907000
   if (ssl_true_initialized == 0)
     {
@@ -175,8 +182,6 @@ ssl_init (void)
     }
 #endif
 
-  SSL_METHOD const *meth;
-
   if (ssl_ctx)
     /* The SSL has already been initialized. */
     return true;
@@ -208,23 +213,43 @@ ssl_init (void)
       meth = SSLv2_client_method ();
       break;
 #endif
+
+#ifndef OPENSSL_NO_SSL3
     case secure_protocol_sslv3:
       meth = SSLv3_client_method ();
       break;
+#endif
+
     case secure_protocol_auto:
     case secure_protocol_pfs:
+      meth = SSLv23_client_method ();
+      ssl_options |= SSL_OP_NO_SSLv2 | SSL_OP_NO_SSLv3;
+      break;
     case secure_protocol_tlsv1:
       meth = TLSv1_client_method ();
       break;
-#if OPENSSL_VERSION_NUMBER >= 0x01001000
+
+#if OPENSSL_VERSION_NUMBER >= 0x10001000
     case secure_protocol_tlsv1_1:
       meth = TLSv1_1_client_method ();
       break;
+
     case secure_protocol_tlsv1_2:
       meth = TLSv1_2_client_method ();
       break;
+#else
+    case secure_protocol_tlsv1_1:
+      logprintf (LOG_NOTQUIET, _("Your OpenSSL version is too old to support TLSv1.1\n"));
+      goto error;
+
+    case secure_protocol_tlsv1_2:
+      logprintf (LOG_NOTQUIET, _("Your OpenSSL version is too old to support TLSv1.2\n"));
+      goto error;
 #endif
+
     default:
+      logprintf (LOG_NOTQUIET, _("OpenSSL: unimplemented 'secure-protocol' option value %d\n"), opt.secure_protocol);
+      logprintf (LOG_NOTQUIET, _("Please report this issue to bug-wget@gnu.org\n"));
       abort ();
     }
 
@@ -234,6 +259,9 @@ ssl_init (void)
   if (!ssl_ctx)
     goto error;
 
+  if (ssl_options)
+    SSL_CTX_set_options (ssl_ctx, ssl_options);
+
   /* OpenSSL ciphers: https://www.openssl.org/docs/apps/ciphers.html
    * Since we want a good protection, we also use HIGH (that excludes MD4 ciphers and some more)
    */
@@ -243,6 +271,18 @@ ssl_init (void)
   SSL_CTX_set_default_verify_paths (ssl_ctx);
   SSL_CTX_load_verify_locations (ssl_ctx, opt.ca_cert, opt.ca_directory);
 
+  if (opt.crl_file)
+    {
+      X509_STORE *store = SSL_CTX_get_cert_store (ssl_ctx);
+      X509_LOOKUP *lookup;
+
+      if (!(lookup = X509_STORE_add_lookup (store, X509_LOOKUP_file ()))
+          || (!X509_load_crl_file (lookup, opt.crl_file, X509_FILETYPE_PEM)))
+        goto error;
+
+      X509_STORE_set_flags (store, X509_V_FLAG_CRL_CHECK | X509_V_FLAG_CRL_CHECK_ALL);
+    }
+
   /* SSL_VERIFY_NONE instructs OpenSSL not to abort SSL_connect if the
      certificate is invalid.  We verify the certificate separately in
      ssl_check_certificate, which provides much better diagnostics
@@ -385,7 +425,7 @@ openssl_errstr (int fd _GL_UNUSED, void *arg)
     return NULL;
 
   /* Get rid of previous contents of ctx->last_error, if any.  */
-  xfree_null (ctx->last_error);
+  xfree (ctx->last_error);
 
   /* Iterate over OpenSSL's error stack and accumulate errors in the
      last_error buffer, separated by "; ".  This is better than using
@@ -429,7 +469,7 @@ openssl_close (int fd, void *arg)
 
   SSL_shutdown (conn);
   SSL_free (conn);
-  xfree_null (ctx->last_error);
+  xfree (ctx->last_error);
   xfree (ctx);
 
   close (fd);
@@ -505,7 +545,7 @@ ssl_connect_wget (int fd, const char *hostname)
     DEBUGP (("SSL handshake timed out.\n"));
     goto timeout;
   }
-  if (scwt_ctx.result <= 0 || conn->state != SSL_ST_OK)
+  if (scwt_ctx.result <= 0 || SSL_state(conn) != SSL_ST_OK)
     goto error;
 
   ctx = xnew0 (struct openssl_transport_context);
@@ -570,6 +610,27 @@ pattern_match (const char *pattern, const char *string)
   return *n == '\0';
 }
 
+static char *_get_rfc2253_formatted (X509_NAME *name)
+{
+  int len;
+  char *out = NULL;
+  BIO* b;
+
+  if ((b = BIO_new (BIO_s_mem ())))
+    {
+      if (X509_NAME_print_ex (b, name, 0, XN_FLAG_RFC2253) >= 0
+          && (len = BIO_number_written (b)) > 0)
+        {
+          out = xmalloc (len + 1);
+          BIO_read (b, out, len);
+          out[len] = 0;
+        }
+      BIO_free (b);
+    }
+
+  return out ? out : xstrdup("");
+}
+
 /* Verify the validity of the certificate presented by the server.
    Also check that the "common name" of the server, as presented by
    its certificate, corresponds to HOST.  (HOST typically comes from
@@ -613,23 +674,25 @@ ssl_check_certificate (int fd, const char *host)
 
   IF_DEBUG
     {
-      char *subject = X509_NAME_oneline (X509_get_subject_name (cert), 0, 0);
-      char *issuer = X509_NAME_oneline (X509_get_issuer_name (cert), 0, 0);
+      char *subject = _get_rfc2253_formatted (X509_get_subject_name (cert));
+      char *issuer = _get_rfc2253_formatted (X509_get_issuer_name (cert));
       DEBUGP (("certificate:\n  subject: %s\n  issuer:  %s\n",
                quotearg_n_style (0, escape_quoting_style, subject),
                quotearg_n_style (1, escape_quoting_style, issuer)));
-      OPENSSL_free (subject);
-      OPENSSL_free (issuer);
+      xfree (subject);
+      xfree (issuer);
     }
 
   vresult = SSL_get_verify_result (conn);
   if (vresult != X509_V_OK)
     {
-      char *issuer = X509_NAME_oneline (X509_get_issuer_name (cert), 0, 0);
+      char *issuer = _get_rfc2253_formatted (X509_get_issuer_name (cert));
       logprintf (LOG_NOTQUIET,
                  _("%s: cannot verify %s's certificate, issued by %s:\n"),
                  severity, quotearg_n_style (0, escape_quoting_style, host),
                  quote_n (1, issuer));
+      xfree(issuer);
+
       /* Try to print more user-friendly (and translated) messages for
          the frequent verification errors.  */
       switch (vresult)
@@ -725,7 +788,7 @@ ssl_check_certificate (int fd, const char *host)
                 }
             }
         }
-      sk_GENERAL_NAME_free (subjectAltNames);
+      sk_GENERAL_NAME_pop_free(subjectAltNames, GENERAL_NAME_free);
       if (host_in_octet_string)
         ASN1_OCTET_STRING_free(host_in_octet_string);
 

src/options.h

@@ -133,7 +133,7 @@ struct options
   char **no_proxy;
   char *base_href;
   char *progress_type;          /* progress indicator type. */
-  bool show_progress;           /* Show only the progress bar */
+  int  show_progress;           /* Show only the progress bar */
   bool noscroll;                /* Don't scroll the filename in the progressbar */
   char *proxy_user; /*oli*/
   char *proxy_passwd;
@@ -218,6 +218,7 @@ struct options
 
   char *ca_directory;           /* CA directory (hash files) */
   char *ca_cert;                /* CA certificate file to use */
+  char *crl_file;               /* file with CRLs */
 
   char *random_file;            /* file with random data to seed the PRNG */
   char *egd_file;               /* file name of the egd daemon socket */

src/progress.c

@@ -42,6 +42,7 @@ as that of the covered work.  */
 #include "progress.h"
 #include "utils.h"
 #include "retr.h"
+#include "c-strcase.h"
 
 struct progress_implementation {
   const char *name;
@@ -195,7 +196,7 @@ progress_finish (void *progress, double dltime)
 {
   current_impl->finish (progress, dltime);
 }
-
+
 /* Dot-printing. */
 
 struct dot_progress {
@@ -427,7 +428,7 @@ dot_set_params (char *params)
     return;
 
   /* We use this to set the retrieval style.  */
-  if (!strcasecmp (params, "default"))
+  if (!c_strcasecmp (params, "default"))
     {
       /* Default style: 1K dots, 10 dots in a cluster, 50 dots in a
          line.  */
@@ -435,7 +436,7 @@ dot_set_params (char *params)
       opt.dot_spacing = 10;
       opt.dots_in_line = 50;
     }
-  else if (!strcasecmp (params, "binary"))
+  else if (!c_strcasecmp (params, "binary"))
     {
       /* "Binary" retrieval: 8K dots, 16 dots in a cluster, 48 dots
          (384K) in a line.  */
@@ -443,7 +444,7 @@ dot_set_params (char *params)
       opt.dot_spacing = 16;
       opt.dots_in_line = 48;
     }
-  else if (!strcasecmp (params, "mega"))
+  else if (!c_strcasecmp (params, "mega"))
     {
       /* "Mega" retrieval, for retrieving very long files; each dot is
          64K, 8 dots in a cluster, 6 clusters (3M) in a line.  */
@@ -451,7 +452,7 @@ dot_set_params (char *params)
       opt.dot_spacing = 8;
       opt.dots_in_line = 48;
     }
-  else if (!strcasecmp (params, "giga"))
+  else if (!c_strcasecmp (params, "giga"))
     {
       /* "Giga" retrieval, for retrieving very very *very* long files;
          each dot is 1M, 8 dots in a cluster, 4 clusters (32M) in a
@@ -465,7 +466,7 @@ dot_set_params (char *params)
              _("Invalid dot style specification %s; leaving unchanged.\n"),
              quote (params));
 }
-
+
 /* "Thermometer" (bar) progress. */
 
 /* Assumed screen width if we can't find the real value.  */
@@ -839,11 +840,13 @@ cols_to_bytes (const char *mbs, const int cols, int *ncols)
   return bytes;
 }
 #else
-# define count_cols(mbs) ((int)(strlen(mbs)))
-# define cols_to_bytes(mbs, cols, *ncols) do {  \
-    *ncols = cols;                              \
-    bytes = cols;                               \
-}while (0)
+static int count_cols (const char *mbs) { return (int) strlen(mbs); }
+static int
+cols_to_bytes (const char *mbs _GL_UNUSED, const int cols, int *ncols)
+{
+  *ncols = cols;
+  return cols;
+}
 #endif
 
 static const char *
@@ -893,13 +896,6 @@ get_eta (int *bcd)
    are confused when they see strchr (s, '\0') in the code.  */
 #define move_to_end(s) s = strchr (s, '\0');
 
-#ifndef MAX
-# define MAX(a, b) ((a) >= (b) ? (a) : (b))
-#endif
-#ifndef MIN
-# define MIN(a, b) ((a) <= (b) ? (a) : (b))
-#endif
-
 static void
 create_image (struct bar_progress *bp, double dl_total_time, bool done)
 {
@@ -907,10 +903,6 @@ create_image (struct bar_progress *bp, double dl_total_time, bool done)
   char *p = bp->buffer;
   wgint size = bp->initial_length + bp->count;
 
-  const char *size_grouped = with_thousand_seps (size);
-  int size_grouped_len = count_cols (size_grouped);
-  /* Difference between num cols and num bytes: */
-  int size_grouped_diff = strlen (size_grouped) - size_grouped_len;
   int size_grouped_pad; /* Used to pad the field width for size_grouped. */
 
   struct bar_progress_hist *hist = &bp->hist;
@@ -950,6 +942,8 @@ create_image (struct bar_progress *bp, double dl_total_time, bool done)
   /* The difference between the number of bytes used,
      and the number of columns used. */
   int bytes_cols_diff = 0;
+  int cols_diff;
+  const char *down_size;
 
   if (progress_size < 5)
     progress_size = 0;
@@ -967,16 +961,22 @@ create_image (struct bar_progress *bp, double dl_total_time, bool done)
       int offset_cols;
       int bytes_in_filename, offset_bytes, col;
       int *cols_ret = &col;
+      int padding;
 
-      if (((orig_filename_cols > MAX_FILENAME_COLS) && !opt.noscroll) && !done)
-        offset_cols = ((int) bp->tick) % (orig_filename_cols - MAX_FILENAME_COLS);
+#define MIN_SCROLL_TEXT 5
+      if ((orig_filename_cols > MAX_FILENAME_COLS + MIN_SCROLL_TEXT) &&
+          !opt.noscroll &&
+          !done)
+        offset_cols = ((int) bp->tick) % (orig_filename_cols - MAX_FILENAME_COLS + 1);
       else
         offset_cols = 0;
       offset_bytes = cols_to_bytes (bp->f_download, offset_cols, cols_ret);
-      bytes_in_filename = cols_to_bytes (bp->f_download + offset_bytes, MAX_FILENAME_COLS, cols_ret);
+      bytes_in_filename = cols_to_bytes (bp->f_download + offset_bytes,
+                                         MAX_FILENAME_COLS,
+                                         cols_ret);
       memcpy (p, bp->f_download + offset_bytes, bytes_in_filename);
       p += bytes_in_filename;
-      int padding = MAX_FILENAME_COLS - *cols_ret;
+      padding = MAX_FILENAME_COLS - *cols_ret;
       for (;padding;padding--)
           *p++ = ' ';
       *p++ = ' ';
@@ -1059,8 +1059,8 @@ create_image (struct bar_progress *bp, double dl_total_time, bool done)
  ++bp->tick;
 
   /* " 234.56M" */
-  const char * down_size = human_readable (size, 1000, 2);
-  int cols_diff = 7 - count_cols (down_size);
+  down_size = human_readable (size, 1000, 2);
+  cols_diff = 7 - count_cols (down_size);
   while (cols_diff > 0)
   {
     *p++=' ';
@@ -1159,9 +1159,18 @@ create_image (struct bar_progress *bp, double dl_total_time, bool done)
       move_to_end (p);
     }
 
-  while (p - bp->buffer - bytes_cols_diff - size_grouped_diff < bp->width)
+  while (p - bp->buffer - bytes_cols_diff < bp->width)
     *p++ = ' ';
   *p = '\0';
+
+  /* 2014-11-14  Darshit Shah  <darnir@gmail.com>
+   * Assert that the length of the progress bar is lesser than the size of the
+   * screen with which we are dealing. This assertion *MUST* always be removed
+   * from the release code since we do not want Wget to crash and burn when the
+   * assertion fails. Instead Wget should continue downloading and display a
+   * horrible and irritating progress bar that spams the screen with newlines.
+   */
+  assert (count_cols (bp->buffer) <= bp->width + 1);
 }
 
 /* Print the contents of the buffer as a one-line ASCII "image" so
@@ -1193,7 +1202,7 @@ bar_set_params (char *params)
         } while ((param = strtok (NULL, ":")) != NULL);
     }
 
-  if ((opt.lfilename
+  if (((opt.lfilename && opt.show_progress != 1)
 #ifdef HAVE_ISATTY
        /* The progress bar doesn't make sense if the output is not a
           TTY -- when logging to file, it is better to review the

src/ptimer.c

@@ -291,7 +291,7 @@ windows_resolution (void)
     return 10;                  /* according to MSDN */
 }
 #endif  /* PTIMER_WINDOWS */
-
+
 /* The code below this point is independent of timer implementation. */
 
 struct ptimer {

src/recur.c

@@ -50,7 +50,7 @@ as that of the covered work.  */
 #include "html-url.h"
 #include "css-url.h"
 #include "spider.h"
-
+
 /* Functions for maintaining the URL queue.  */
 
 struct queue_element {
@@ -159,7 +159,28 @@ url_dequeue (struct url_queue *queue, struct iri **i,
   xfree (qel);
   return true;
 }
-
+
+static void blacklist_add (struct hash_table *blacklist, const char *url)
+{
+  char *url_unescaped = xstrdup (url);
+
+  url_unescape (url_unescaped);
+  string_set_add (blacklist, url_unescaped);
+  xfree (url_unescaped);
+}
+
+static int blacklist_contains (struct hash_table *blacklist, const char *url)
+{
+  char *url_unescaped = xstrdup(url);
+  int ret;
+
+  url_unescape (url_unescaped);
+  ret = string_set_contains (blacklist, url_unescaped);
+  xfree (url_unescaped);
+
+  return ret;
+}
+
 static bool download_child_p (const struct urlpos *, struct url *, int,
                               struct url *, struct hash_table *, struct iri *);
 static bool descend_redirect_p (const char *, struct url *, int,
@@ -220,7 +241,7 @@ retrieve_tree (struct url *start_url_parsed, struct iri *pi)
      just URL so we enqueue the canonical form of the URL.  */
   url_enqueue (queue, i, xstrdup (start_url_parsed->url), NULL, 0, true,
                false);
-  string_set_add (blacklist, start_url_parsed->url);
+  blacklist_add (blacklist, start_url_parsed->url);
 
   while (1)
     {
@@ -311,7 +332,7 @@ retrieve_tree (struct url *start_url_parsed, struct iri *pi)
                   else
                     /* Make sure that the old pre-redirect form gets
                        blacklisted. */
-                    string_set_add (blacklist, url);
+                    blacklist_add (blacklist, url);
                 }
 
               xfree (url);
@@ -322,7 +343,7 @@ retrieve_tree (struct url *start_url_parsed, struct iri *pi)
               xfree (url);
               url = xstrdup (url_parsed->url);
             }
-          url_free(url_parsed);
+          url_free (url_parsed);
         }
 
       if (opt.spider)
@@ -404,7 +425,7 @@ retrieve_tree (struct url *start_url_parsed, struct iri *pi)
                       /* We blacklist the URL we have enqueued, because we
                          don't want to enqueue (and hence download) the
                          same URL twice.  */
-                      string_set_add (blacklist, child->url->url);
+                      blacklist_add (blacklist, child->url->url);
                     }
                 }
 
@@ -440,8 +461,8 @@ retrieve_tree (struct url *start_url_parsed, struct iri *pi)
         }
 
       xfree (url);
-      xfree_null (referer);
-      xfree_null (file);
+      xfree (referer);
+      xfree (file);
       iri_free (i);
     }
 
@@ -457,7 +478,7 @@ retrieve_tree (struct url *start_url_parsed, struct iri *pi)
       {
         iri_free (d6);
         xfree (d1);
-        xfree_null (d2);
+        xfree (d2);
       }
   }
   url_queue_delete (queue);
@@ -491,7 +512,7 @@ download_child_p (const struct urlpos *upos, struct url *parent, int depth,
 
   DEBUGP (("Deciding whether to enqueue \"%s\".\n", url));
 
-  if (string_set_contains (blacklist, url))
+  if (blacklist_contains (blacklist, url))
     {
       if (opt.spider)
         {
@@ -670,7 +691,7 @@ download_child_p (const struct urlpos *upos, struct url *parent, int depth,
       if (!res_match_path (specs, u->path))
         {
           DEBUGP (("Not following %s because robots.txt forbids it.\n", url));
-          string_set_add (blacklist, url);
+          blacklist_add (blacklist, url);
           goto out;
         }
     }
@@ -712,12 +733,14 @@ descend_redirect_p (const char *redirected, struct url *orig_parsed, int depth,
   success = download_child_p (upos, orig_parsed, depth,
                               start_url_parsed, blacklist, iri);
 
+  if (success)
+    blacklist_add (blacklist, upos->url->url);
+  else
+    DEBUGP (("Redirection \"%s\" failed the test.\n", redirected));
+
   url_free (new_parsed);
   xfree (upos);
 
-  if (!success)
-    DEBUGP (("Redirection \"%s\" failed the test.\n", redirected));
-
   return success;
 }
 

src/res.c

@@ -81,6 +81,7 @@ as that of the covered work.  */
 #include "url.h"
 #include "retr.h"
 #include "res.h"
+#include "c-strcase.h"
 
 #ifdef TESTING
 #include "test.h"
@@ -97,7 +98,7 @@ struct robot_specs {
   int size;
   struct path_info *paths;
 };
-
+
 /* Parsing the robot spec. */
 
 /* Check whether AGENT (a string of length LENGTH) equals "wget" or
@@ -403,10 +404,10 @@ free_specs (struct robot_specs *specs)
   int i;
   for (i = 0; i < specs->count; i++)
     xfree (specs->paths[i].path);
-  xfree_null (specs->paths);
+  xfree (specs->paths);
   xfree (specs);
 }
-
+
 /* Matching of a path according to the specs. */
 
 /* If C is '%' and (ptr[1], ptr[2]) form a hexadecimal number, and if
@@ -471,7 +472,7 @@ res_match_path (const struct robot_specs *specs, const char *path)
       }
   return true;
 }
-
+
 /* Registering the specs. */
 
 static struct hash_table *registered_specs;
@@ -521,7 +522,7 @@ res_get_specs (const char *host, int port)
     return NULL;
   return hash_table_get (registered_specs, hp);
 }
-
+
 /* Loading the robots file.  */
 
 #define RES_SPECS_LOCATION "/robots.txt"
@@ -578,11 +579,10 @@ res_retrieve_file (const char *url, char **file, struct iri *iri)
          allocated the file name, deallocate is here so that the
          caller doesn't have to worry about it.  */
       xfree (*file);
-      *file = NULL;
     }
   return err == RETROK;
 }
-
+
 bool
 is_robots_txt_url (const char *url)
 {
@@ -593,7 +593,7 @@ is_robots_txt_url (const char *url)
 
   return ret;
 }
-
+
 void
 res_cleanup (void)
 {
@@ -611,7 +611,7 @@ res_cleanup (void)
       registered_specs = NULL;
     }
 }
-
+
 #ifdef TESTING
 
 const char *

src/retr.c

@@ -70,7 +70,7 @@ FILE *output_stream;
 /* Whether output_document is a regular file we can manipulate,
    i.e. not `-' or a device file. */
 bool output_stream_regular;
-
+
 static struct {
   wgint chunk_bytes;
   double chunk_start;
@@ -136,10 +136,6 @@ limit_bandwidth (wgint bytes, struct ptimer *timer)
   limit_data.chunk_start = ptimer_read (timer);
 }
 
-#ifndef MIN
-# define MIN(i, j) ((i) <= (j) ? (i) : (j))
-#endif
-
 /* Write data in BUF to OUT.  However, if *SKIP is non-zero, skip that
    amount of data and decrease SKIP.  Increment *TOTAL by the amount
    of data written.  If OUT2 is not NULL, also write BUF to OUT2.
@@ -265,11 +261,16 @@ fd_read_body (const char *downloaded_filename, int fd, FILE *out, wgint toread,
 
   if (opt.show_progress)
     {
+      const char *filename_progress;
       /* If we're skipping STARTPOS bytes, pass 0 as the INITIAL
          argument to progress_create because the indicator doesn't
          (yet) know about "skipping" data.  */
       wgint start = skip ? 0 : startpos;
-      progress = progress_create (downloaded_filename, start, start + toread);
+      if (opt.dir_prefix)
+        filename_progress = downloaded_filename + strlen (opt.dir_prefix) + 1;
+      else
+        filename_progress = downloaded_filename;
+      progress = progress_create (filename_progress, start, start + toread);
       progress_interactive = progress_interactive_p (progress);
     }
 
@@ -378,8 +379,10 @@ fd_read_body (const char *downloaded_filename, int fd, FILE *out, wgint toread,
 
       if (ret > 0)
         {
+          int write_res;
+
           sum_read += ret;
-          int write_res = write_data (out, out2, dlbuf, ret, &skip, &sum_written);
+          write_res = write_data (out, out2, dlbuf, ret, &skip, &sum_written);
           if (write_res < 0)
             {
               ret = (write_res == -3) ? -3 : -2;
@@ -434,11 +437,11 @@ fd_read_body (const char *downloaded_filename, int fd, FILE *out, wgint toread,
   if (qtywritten)
     *qtywritten += sum_written;
 
-  free (dlbuf);
+  xfree (dlbuf);
 
   return ret;
 }
-
+
 /* Read a hunk of data from FD, up until a terminator.  The hunk is
    limited by whatever the TERMINATOR callback chooses as its
    terminator.  For example, if terminator stops at newline, the hunk
@@ -543,7 +546,7 @@ fd_read_hunk (int fd, hunk_terminator_t terminator, long sizehint, long maxsize)
       rdlen = fd_read (fd, hunk + tail, remain, 0);
       if (rdlen < 0)
         {
-          xfree_null (hunk);
+          xfree (hunk);
           return NULL;
         }
       tail += rdlen;
@@ -615,7 +618,7 @@ fd_read_line (int fd)
 {
   return fd_read_hunk (fd, line_terminator, 128, FD_READ_LINE_MAX);
 }
-
+
 /* Return a printed representation of the download rate, along with
    the units appropriate for the download speed.  */
 
@@ -679,7 +682,7 @@ calc_rate (wgint bytes, double secs, int *units)
 
   return dlrate;
 }
-
+
 
 #define SUSPEND_METHOD do {                     \
   method_suspended = true;                      \
@@ -761,7 +764,7 @@ retrieve_url (struct url * orig_parsed, const char *origurl, char **file,
       pi->utf8_encode = false;
 
       /* Parse the proxy URL.  */
-      proxy_url = url_parse (proxy, &up_error_code, NULL, true);
+      proxy_url = url_parse (proxy, &up_error_code, pi, true);
       if (!proxy_url)
         {
           char *error = url_error (proxy, up_error_code);
@@ -782,7 +785,8 @@ retrieve_url (struct url * orig_parsed, const char *origurl, char **file,
           result = PROXERR;
           goto bail;
         }
-      free (proxy);
+      iri_free(pi);
+      xfree (proxy);
     }
 
   if (u->scheme == SCHEME_HTTP
@@ -831,8 +835,7 @@ retrieve_url (struct url * orig_parsed, const char *origurl, char **file,
 
       assert (mynewloc != NULL);
 
-      if (local_file)
-        xfree (local_file);
+      xfree (local_file);
 
       /* The HTTP specs only allow absolute URLs to appear in
          redirects, but a ton of boneheaded webservers and CGIs out
@@ -846,8 +849,7 @@ retrieve_url (struct url * orig_parsed, const char *origurl, char **file,
          the content encoding. */
       iri->utf8_encode = opt.enable_iri;
       set_content_encoding (iri, NULL);
-      xfree_null (iri->orig_url);
-      iri->orig_url = NULL;
+      xfree (iri->orig_url);
 
       /* Now, see if this new location makes sense. */
       newloc_parsed = url_parse (mynewloc, &up_error_code, iri, true);
@@ -930,6 +932,7 @@ retrieve_url (struct url * orig_parsed, const char *origurl, char **file,
       if (u)
         {
           DEBUGP (("[IRI fallbacking to non-utf8 for %s\n", quote (url)));
+          xfree (url);
           url = xstrdup (u->url);
           iri_fallbacked = 1;
           goto redirected;
@@ -955,7 +958,7 @@ retrieve_url (struct url * orig_parsed, const char *origurl, char **file,
   if (file)
     *file = local_file ? local_file : NULL;
   else
-    xfree_null (local_file);
+    xfree (local_file);
 
   if (orig_parsed != u)
     {
@@ -1039,8 +1042,7 @@ retrieve_from_file (const char *file, bool html, int *count)
 
       /* Reset UTF-8 encode status */
       iri->utf8_encode = opt.enable_iri;
-      xfree_null (iri->orig_url);
-      iri->orig_url = NULL;
+      xfree (iri->orig_url);
 
       input_file = url_file;
     }
@@ -1050,12 +1052,12 @@ retrieve_from_file (const char *file, bool html, int *count)
   url_list = (html ? get_urls_html (input_file, NULL, NULL, iri)
               : get_urls_file (input_file));
 
-  xfree_null (url_file);
+  xfree (url_file);
 
   for (cur_url = url_list; cur_url; cur_url = cur_url->next, ++*count)
     {
-      char *filename = NULL, *new_file = NULL;
-      int dt;
+      char *filename = NULL, *new_file = NULL, *proxy;
+      int dt = 0;
       struct iri *tmpiri = iri_dup (iri);
       struct url *parsed_url = NULL;
 
@@ -1070,7 +1072,7 @@ retrieve_from_file (const char *file, bool html, int *count)
 
       parsed_url = url_parse (cur_url->url->url, NULL, tmpiri, true);
 
-      char *proxy = getproxy (cur_url->url);
+      proxy = getproxy (cur_url->url);
       if ((opt.recursive || opt.page_requisites)
           && (cur_url->url->scheme != SCHEME_FTP || proxy))
         {
@@ -1090,7 +1092,7 @@ retrieve_from_file (const char *file, bool html, int *count)
                                cur_url->url->url, &filename,
                                &new_file, NULL, &dt, opt.recursive, tmpiri,
                                true);
-      free(proxy);
+      xfree (proxy);
 
       if (parsed_url)
           url_free (parsed_url);
@@ -1105,8 +1107,8 @@ Removing file due to --delete-after in retrieve_from_file():\n"));
           dt &= ~RETROKF;
         }
 
-      xfree_null (new_file);
-      xfree_null (filename);
+      xfree (new_file);
+      xfree (filename);
       iri_free (tmpiri);
     }
 
@@ -1182,7 +1184,7 @@ free_urlpos (struct urlpos *l)
       struct urlpos *next = l->next;
       if (l->url)
         url_free (l->url);
-      xfree_null (l->local_name);
+      xfree (l->local_name);
       xfree (l);
       l = next;
     }
@@ -1283,11 +1285,13 @@ bool
 url_uses_proxy (struct url * u)
 {
   bool ret;
+  char *proxy;
+
   if (!u)
     return false;
-  char *proxy = getproxy (u);
+  proxy = getproxy (u);
   ret = proxy != NULL;
-  free(proxy);
+  xfree (proxy);
   return ret;
 }
 

src/retr.h

@@ -34,6 +34,8 @@ as that of the covered work.  */
 
 #include "url.h"
 
+extern int numurls;
+
 /* These global vars should be made static to retr.c and exported via
    functions! */
 extern SUM_SIZE_INT total_downloaded_bytes;

src/spider.c

@@ -51,7 +51,7 @@ spider_cleanup (void)
   if (nonexisting_urls_set)
     string_set_free (nonexisting_urls_set);
 }
-
+
 /* Remembers broken links.  */
 void
 nonexisting_url (const char *url)

src/test.c

@@ -41,22 +41,22 @@ as that of the covered work.  */
 #error "TESTING not set!!!"
 #endif
 
-const char *test_parse_content_disposition();
-const char *test_subdir_p();
-const char *test_dir_matches_p();
-const char *test_commands_sorted();
-const char *test_cmd_spec_restrict_file_names();
-const char *test_path_simplify ();
-const char *test_append_uri_pathel();
-const char *test_are_urls_equal();
-const char *test_is_robots_txt_url();
+const char *test_parse_content_disposition(void);
+const char *test_subdir_p(void);
+const char *test_dir_matches_p(void);
+const char *test_commands_sorted(void);
+const char *test_cmd_spec_restrict_file_names(void);
+const char *test_path_simplify (void);
+const char *test_append_uri_pathel(void);
+const char *test_are_urls_equal(void);
+const char *test_is_robots_txt_url(void);
 
 const char *program_argstring = "TEST";
 
 int tests_run;
 
 static const char *
-all_tests()
+all_tests(void)
 {
   mu_run_test (test_parse_content_disposition);
   mu_run_test (test_subdir_p);
@@ -71,10 +71,10 @@ all_tests()
   return NULL;
 }
 
-char *program_name; /* Needed by lib/error.c. */
+const char *program_name; /* Needed by lib/error.c. */
 
 int
-main (int argc _GL_UNUSED, char *argv[])
+main (int argc _GL_UNUSED, const char *argv[])
 {
   const char *result;
 

src/url.c

@@ -41,6 +41,7 @@ as that of the covered work.  */
 #include "utils.h"
 #include "url.h"
 #include "host.h"  /* for is_valid_ipv6_address */
+#include "c-strcase.h"
 
 #ifdef __VMS
 #include "vms.h"
@@ -85,7 +86,7 @@ static struct scheme_data supported_schemes[] =
 /* Forward declarations: */
 
 static bool path_simplify (enum url_scheme, char *);
-
+
 /* Support for escaping and unescaping of URL strings.  */
 
 /* Table of "reserved" and "unsafe" characters.  Those terms are
@@ -272,7 +273,7 @@ url_escape_allow_passthrough (const char *s)
 {
   return url_escape_1 (s, urlchr_unsafe, true);
 }
-
+
 /* Decide whether the char at position P needs to be encoded.  (It is
    not enough to pass a single char *P because the function may need
    to inspect the surrounding context.)
@@ -418,7 +419,7 @@ reencode_escapes (const char *s)
   assert (p2 - newstr == newlen);
   return newstr;
 }
-
+
 /* Returns the scheme type if the scheme is supported, or
    SCHEME_INVALID if not.  */
 
@@ -574,8 +575,8 @@ rewrite_shorthand_url (const char *url)
         goto http;
 
       /* Turn "foo.bar.com:path" to "ftp://foo.bar.com/path". */
-      ret = aprintf ("ftp://%s", url);
-      ret[6 + (p - url)] = '/';
+      if ((ret = aprintf ("ftp://%s", url)) != NULL)
+        ret[6 + (p - url)] = '/';
     }
   else
     {
@@ -585,7 +586,7 @@ rewrite_shorthand_url (const char *url)
     }
   return ret;
 }
-
+
 static void split_path (const char *, char **, char **);
 
 /* Like strpbrk, with the exception that it returns the pointer to the
@@ -705,6 +706,7 @@ url_parse (const char *url, int *error, struct iri *iri, bool percent_encode)
         new_url = NULL;
       else
         {
+          xfree (iri->orig_url);
           iri->orig_url = xstrdup (url);
           url_encoded = reencode_escapes (new_url);
           if (url_encoded != new_url)
@@ -895,6 +897,7 @@ url_parse (const char *url, int *error, struct iri *iri, bool percent_encode)
             {
               xfree (u->host);
               u->host = new;
+              u->idn_allocated = true;
               host_modified = true;
             }
         }
@@ -915,7 +918,7 @@ url_parse (const char *url, int *error, struct iri *iri, bool percent_encode)
       u->url = url_string (u, URL_AUTH_SHOW);
 
       if (url_encoded != url)
-        xfree ((char *) url_encoded);
+        xfree (url_encoded);
     }
   else
     {
@@ -930,7 +933,7 @@ url_parse (const char *url, int *error, struct iri *iri, bool percent_encode)
  error:
   /* Cleanup in case of error: */
   if (url_encoded && url_encoded != url)
-    xfree ((char *) url_encoded);
+    xfree (url_encoded);
 
   /* Transmit the error code to the caller, if the caller wants to
      know.  */
@@ -955,7 +958,7 @@ url_error (const char *url, int error_code)
 
       if ((p = strchr (scheme, ':')))
         *p = '\0';
-      if (!strcasecmp (scheme, "https"))
+      if (!c_strcasecmp (scheme, "https"))
         error = aprintf (_("HTTPS support not compiled in"));
       else
         error = aprintf (_(parse_errors[error_code]), quote (scheme));
@@ -1171,22 +1174,31 @@ url_set_file (struct url *url, const char *newfile)
 void
 url_free (struct url *url)
 {
-  xfree (url->host);
-  xfree (url->path);
-  xfree (url->url);
+  if (url)
+    {
+      if (url->idn_allocated) {
+        idn_free (url->host);      /* A dummy if !defined(ENABLE_IRI) */
+        url->host = NULL;
+      }
+      else
+        xfree (url->host);
 
-  xfree_null (url->params);
-  xfree_null (url->query);
-  xfree_null (url->fragment);
-  xfree_null (url->user);
-  xfree_null (url->passwd);
+      xfree (url->path);
+      xfree (url->url);
 
-  xfree (url->dir);
-  xfree (url->file);
+      xfree (url->params);
+      xfree (url->query);
+      xfree (url->fragment);
+      xfree (url->user);
+      xfree (url->passwd);
 
-  xfree (url);
+      xfree (url->dir);
+      xfree (url->file);
+
+      xfree (url);
+    }
 }
-
+
 /* Create all the necessary directories for PATH (a file).  Calls
    make_directory internally.  */
 int
@@ -1238,7 +1250,7 @@ mkalldirs (const char *path)
   xfree (t);
   return res;
 }
-
+
 /* Functions for constructing the file name out of URL components.  */
 
 /* A growable string structure, used by url_file_name and friends.
@@ -1641,7 +1653,7 @@ url_file_name (const struct url *u, char *replaced_filename)
       logprintf (LOG_NOTQUIET, "New name is %s.\n", temp_fnres.base);
     }
 
-  free (fname_len_check);
+  xfree (fname_len_check);
 
   /* The filename has already been 'cleaned' by append_uri_pathel() above.  So,
    * just append it. */
@@ -1654,7 +1666,7 @@ url_file_name (const struct url *u, char *replaced_filename)
   /* Make a final check that the path length is acceptable? */
   /* TODO: check fnres.base for path length problem */
 
-  free (temp_fnres.base);
+  xfree (temp_fnres.base);
 
   /* Check the cases in which the unique extensions are not used:
      1) Clobbering is turned off (-nc).
@@ -1694,7 +1706,7 @@ url_file_name (const struct url *u, char *replaced_filename)
 
   return unique;
 }
-
+
 /* Resolve "." and ".." elements of PATH by destructively modifying
    PATH and return true if PATH has been modified, false otherwise.
 
@@ -1780,7 +1792,7 @@ path_simplify (enum url_scheme scheme, char *path)
 
   return t != h;
 }
-
+
 /* Return the length of URL's path.  Path is considered to be
    terminated by one or more of the ?query or ;params or #fragment,
    depending on the scheme.  */
@@ -1991,7 +2003,7 @@ uri_merge (const char *base, const char *link)
 
   return merge;
 }
-
+
 #define APPEND(p, s) do {                       \
   int len = strlen (s);                         \
   memcpy (p, s, len);                           \
@@ -2108,7 +2120,7 @@ url_string (const struct url *url, enum url_auth_mode auth_mode)
 
   return result;
 }
-
+
 /* Return true if scheme a is similar to scheme b.
 
    Schemes are similar if they are equal.  If SSL is supported, schemes
@@ -2126,7 +2138,7 @@ schemes_are_similar_p (enum url_scheme a, enum url_scheme b)
 #endif
   return false;
 }
-
+
 static int
 getchar_from_escaped_string (const char *str, char *c)
 {
@@ -2187,7 +2199,7 @@ are_urls_equal (const char *u1, const char *u2)
 
   return (*p == 0 && *q == 0 ? true : false);
 }
-
+
 #ifdef TESTING
 /* Debugging and testing support for path_simplify. */
 

src/url.h

@@ -95,6 +95,10 @@ struct url
   /* Username and password (unquoted). */
   char *user;
   char *passwd;
+
+  /* 'host' is allocated by idna_to_ascii_8z() via idn_encode().
+   * Call 'idn_free()' to free this memory. */
+  bool idn_allocated;
 };
 
 /* Function declarations */

src/utils.c

@@ -64,8 +64,10 @@ as that of the covered work.  */
 #include <sys/stat.h>
 
 /* For TIOCGWINSZ and friends: */
-#include <sys/ioctl.h>
-#include <termios.h>
+#ifndef WINDOWS
+# include <sys/ioctl.h>
+# include <termios.h>
+#endif
 
 /* Needed for Unix version of run_with_timeout. */
 #include <signal.h>
@@ -99,6 +101,7 @@ as that of the covered work.  */
 #endif
 
 #include "exits.h"
+#include "c-strcase.h"
 
 static void _Noreturn
 memfatal (const char *context, long attempted_size)
@@ -267,7 +270,7 @@ sepstring (const char *s)
   res[i + 1] = NULL;
   return res;
 }
-
+
 /* Like sprintf, but prints into a string of sufficient size freshly
    allocated with malloc, which is returned.  If unable to print due
    to invalid format, returns NULL.  Inability to allocate needed
@@ -349,6 +352,32 @@ aprintf (const char *fmt, ...)
 #endif /* not HAVE_VASPRINTF */
 }
 
+#ifndef HAVE_STRLCPY
+/* strlcpy() is a BSD function that sometimes is really handy.
+ * It is the same as snprintf(dst,dstsize,"%s",src), but much faster. */
+
+size_t
+strlcpy (char *dst, const char *src, size_t size)
+{
+  const char *old = src;
+
+  /* Copy as many bytes as will fit */
+  if (size)
+    {
+      while (--size)
+        {
+          if (!(*dst++ = *src++))
+            return src - old - 1;
+        }
+
+      *dst = 0;
+    }
+
+  while (*src++);
+  return src - old - 1;
+}
+#endif
+
 /* Concatenate the NULL-terminated list of string arguments into
    freshly allocated space.  */
 
@@ -356,47 +385,30 @@ char *
 concat_strings (const char *str0, ...)
 {
   va_list args;
-  int saved_lengths[5];         /* inspired by Apache's apr_pstrcat */
-  char *ret, *p;
+  const char *arg;
+  size_t length = 0, pos = 0;
+  char *s;
 
-  const char *next_str;
-  int total_length = 0;
-  size_t argcount;
+  if (!str0)
+    return NULL;
 
-  /* Calculate the length of and allocate the resulting string. */
-
-  argcount = 0;
+  /* calculate the length of the resulting string */
   va_start (args, str0);
-  for (next_str = str0; next_str != NULL; next_str = va_arg (args, char *))
-    {
-      int len = strlen (next_str);
-      if (argcount < countof (saved_lengths))
-        saved_lengths[argcount++] = len;
-      total_length += len;
-    }
+  for (arg = str0; arg; arg = va_arg (args, const char *))
+    length += strlen(arg);
   va_end (args);
-  p = ret = xmalloc (total_length + 1);
 
-  /* Copy the strings into the allocated space. */
+  s = xmalloc (length + 1);
 
-  argcount = 0;
+  /* concatenate strings */
   va_start (args, str0);
-  for (next_str = str0; next_str != NULL; next_str = va_arg (args, char *))
-    {
-      int len;
-      if (argcount < countof (saved_lengths))
-        len = saved_lengths[argcount++];
-      else
-        len = strlen (next_str);
-      memcpy (p, next_str, len);
-      p += len;
-    }
+  for (arg = str0; arg; arg = va_arg (args, const char *))
+    pos += strlcpy(s + pos, arg, length - pos + 1);
   va_end (args);
-  *p = '\0';
 
-  return ret;
+  return s;
 }
-
+
 /* Format the provided time according to the specified format.  The
    format is a string with format elements supported by strftime.  */
 
@@ -430,7 +442,7 @@ datetime_str (time_t t)
 {
   return fmttime(t, "%Y-%m-%d %H:%M:%S");
 }
-
+
 /* The Windows versions of the following two functions are defined in
    mswindows.c. On MSDOS this function should never be called. */
 
@@ -495,7 +507,7 @@ fork_to_background (void)
 
 #endif /* def __VMS [else] */
 
-
+
 /* "Touch" FILE, i.e. make its mtime ("modified time") equal the time
    specified with TM.  The atime ("access time") is set to the current
    time.  */
@@ -805,7 +817,7 @@ fopen_excl (const char *fname, int binary)
   return fopen (fname, binary ? "wb" : "w");
 #endif /* not O_EXCL */
 }
-
+
 /* Create DIRECTORY.  If some of the pathname components of DIRECTORY
    are missing, create them first.  In case any mkdir() call fails,
    return its error status.  Returns 0 on successful completion.
@@ -871,7 +883,7 @@ file_merge (const char *base, const char *file)
 
   return result;
 }
-
+
 /* Like fnmatch, but performs a case-insensitive match.  */
 
 int
@@ -1112,11 +1124,11 @@ has_html_suffix_p (const char *fname)
 
   if ((suf = suffix (fname)) == NULL)
     return false;
-  if (!strcasecmp (suf, "html"))
+  if (!c_strcasecmp (suf, "html"))
     return true;
-  if (!strcasecmp (suf, "htm"))
+  if (!c_strcasecmp (suf, "htm"))
     return true;
-  if (suf[0] && !strcasecmp (suf + 1, "html"))
+  if (suf[0] && !c_strcasecmp (suf + 1, "html"))
     return true;
   return false;
 }
@@ -1262,7 +1274,7 @@ wget_read_file_free (struct file_memory *fm)
     }
   xfree (fm);
 }
-
+
 /* Free the pointers in a NULL-terminated vector of pointers, then
    free the pointer itself.  */
 void
@@ -1331,7 +1343,7 @@ vec_append (char **vec, const char *str)
   vec[cnt] = NULL;
   return vec;
 }
-
+
 /* Sometimes it's useful to create "sets" of strings, i.e. special
    hash tables where you want to store strings as keys and merely
    query for their existence.  Here is a set of utility routines that
@@ -1396,7 +1408,7 @@ free_keys_and_values (struct hash_table *ht)
       xfree (iter.value);
     }
 }
-
+
 /* Get digit grouping data for thousand separors by calling
    localeconv().  The data includes separator string and grouping info
    and is cached after the first call to the function.
@@ -1787,7 +1799,7 @@ convert_to_bits (wgint num)
   return num;
 }
 
-
+
 /* Determine the width of the terminal we're running on.  If that's
    not possible, return 0.  */
 
@@ -1800,7 +1812,7 @@ determine_screen_width (void)
   int fd;
   struct winsize wsz;
 
-  if (opt.lfilename != NULL)
+  if (opt.lfilename != NULL && opt.show_progress != 1)
     return 0;
 
   fd = fileno (stderr);
@@ -1817,7 +1829,7 @@ determine_screen_width (void)
   return 0;
 #endif /* neither TIOCGWINSZ nor WINDOWS */
 }
-
+
 /* Whether the rnd system (either rand or [dl]rand48) has been
    seeded.  */
 static int rnd_seeded;
@@ -1837,7 +1849,14 @@ static int rnd_seeded;
 int
 random_number (int max)
 {
-#ifdef HAVE_DRAND48
+#ifdef HAVE_RANDOM
+  if (!rnd_seeded)
+    {
+      srandom ((long) time (NULL) ^ (long) getpid ());
+      rnd_seeded = 1;
+    }
+  return random () % max;
+#elif defined HAVE_DRAND48
   if (!rnd_seeded)
     {
       srand48 ((long) time (NULL) ^ (long) getpid ());
@@ -1872,7 +1891,9 @@ random_number (int max)
 double
 random_float (void)
 {
-#ifdef HAVE_DRAND48
+#ifdef HAVE_RANDOM
+    return ((double) random_number (RAND_MAX)) / RAND_MAX;
+#elif defined HAVE_DRAND48
   if (!rnd_seeded)
     {
       srand48 ((long) time (NULL) ^ (long) getpid ());
@@ -1886,7 +1907,7 @@ random_float (void)
           + random_number (10000) / (10000.0 * 10000.0 * 10000.0 * 10000.0));
 #endif /* not HAVE_DRAND48 */
 }
-
+
 /* Implementation of run_with_timeout, a generic timeout-forcing
    routine for systems with Unix-like signal handling.  */
 
@@ -1897,7 +1918,7 @@ random_float (void)
 static sigjmp_buf run_with_timeout_env;
 
 static void _Noreturn
-abort_run_with_timeout (int sig)
+abort_run_with_timeout (int sig _GL_UNUSED)
 {
   assert (sig == SIGALRM);
   siglongjmp (run_with_timeout_env, -1);
@@ -1907,8 +1928,8 @@ abort_run_with_timeout (int sig)
 
 static jmp_buf run_with_timeout_env;
 
-static void
-abort_run_with_timeout (int sig)
+static void _Noreturn
+abort_run_with_timeout (int sig _GL_UNUSED)
 {
   assert (sig == SIGALRM);
   /* We don't have siglongjmp to preserve the set of blocked signals;
@@ -2045,7 +2066,7 @@ run_with_timeout (double timeout, void (*fun) (void *), void *arg)
 }
 #endif /* not WINDOWS */
 #endif /* not USE_SIGNAL_TIMEOUT */
-
+
 #ifndef WINDOWS
 
 /* Sleep the specified amount of seconds.  On machines without
@@ -2292,6 +2313,7 @@ compile_posix_regex (const char *str)
       fprintf (stderr, _("Invalid regular expression %s, %s\n"),
                quote (str), errbuf);
       xfree (errbuf);
+      xfree (regex);
       return NULL;
     }
 
@@ -2345,7 +2367,7 @@ match_posix_regex (const void *regex, const char *str)
 
 #undef IS_ASCII
 #undef NEXT_CHAR
-
+
 /* Simple merge sort for use by stable_sort.  Implementation courtesy
    Zeljko Vrba with additional debugging by Nenad Barbutov.  */
 
@@ -2391,7 +2413,7 @@ stable_sort (void *base, size_t nmemb, size_t size,
       mergesort_internal (base, temp, size, 0, nmemb - 1, cmpfun);
     }
 }
-
+
 /* Print a decimal number.  If it is equal to or larger than ten, the
    number is rounded.  Otherwise it is printed with one significant
    digit without trailing zeros and with no more than three fractional

src/utils.h

@@ -47,12 +47,7 @@ as that of the covered work.  */
 
 #define alloca_array(type, size) ((type *) alloca ((size) * sizeof (type)))
 
-#define xfree free
-/* Free P if it is non-NULL.  C requires free() to behaves this way by
-   default, but Wget's code is historically careful not to pass NULL
-   to free.  This allows us to assert p!=NULL in xfree to check
-   additional errors.  (But we currently don't do that!)  */
-#define xfree_null(p) if (!(p)) ; else xfree (p)
+#define xfree(p) do { free ((void *) (p)); p = NULL; } while (0)
 
 struct hash_table;
 
@@ -156,6 +151,10 @@ const char *print_decimal (double);
 
 long get_max_length (const char *path, int length, int name);
 
+#ifndef HAVE_STRLCPY
+size_t strlcpy (char *dst, const char *src, size_t size);
+#endif
+
 extern unsigned char char_prop[];
 
 #endif /* UTILS_H */

src/version.h

@@ -0,0 +1,36 @@
+/* Extern declarations for printing version information
+   Copyright (C) 2013 Free Software Foundation, Inc.
+
+This file is part of GNU Wget.
+
+GNU Wget is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 3 of the License, or
+(at your option) any later version.
+
+GNU Wget is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with Wget.  If not, see <http://www.gnu.org/licenses/>.
+
+Additional permission under GNU GPL version 3 section 7
+
+If you modify this program, or any covered work, by linking or
+combining it with the OpenSSL project's OpenSSL library (or a
+modified version of that library), containing parts covered by the
+terms of the OpenSSL or SSLeay licenses, the Free Software Foundation
+grants you additional permission to convey the resulting work.
+Corresponding Source for a non-source form of such a combination
+shall include the source code for the parts of OpenSSL used as well
+as that of the covered work.  */
+
+/* Extern declarations for strings in version.c */
+extern const char *version_string;
+extern const char *compilation_string;
+extern const char *link_string;
+
+/* Extern declaration for string in build_info.c */
+extern const char *compiled_features[];

src/warc.c

@@ -27,11 +27,15 @@ Corresponding Source for a non-source form of such a combination
 shall include the source code for the parts of OpenSSL used as well
 as that of the covered work.  */
 
+#ifndef _GNU_SOURCE
 #define _GNU_SOURCE
+#endif
 
 #include "wget.h"
 #include "hash.h"
 #include "utils.h"
+#include "version.h"
+#include "dirname.h"
 
 #include <stdio.h>
 #include <stdlib.h>
@@ -51,23 +55,21 @@ as that of the covered work.  */
 #include <uuid.h>
 #endif
 
-#ifndef WINDOWS
-#include <libgen.h>
-#else
-#include <fcntl.h>
-#endif
-
 #include "warc.h"
 #include "exits.h"
 
+#ifdef WINDOWS
+/* we need this on Windows to have O_TEMPORARY defined */
+# include <fcntl.h>
+# include <rpc.h>
+#endif
+
 #ifndef O_TEMPORARY
 #define O_TEMPORARY 0
 #endif
 
-extern char *version_string;
-
-/* Set by main in main.c */
-extern char *program_argstring;
+#include "warc.h"
+#include "exits.h"
 
 
 /* The log file (a temporary file that contains a copy
@@ -163,10 +165,12 @@ warc_write_buffer (const char *buffer, size_t size)
 static bool
 warc_write_string (const char *str)
 {
+  size_t n;
+
   if (!warc_write_ok)
     return false;
 
-  size_t n = strlen (str);
+  n = strlen (str);
   if (n != warc_write_buffer (str, n))
     warc_write_ok = false;
 
@@ -255,6 +259,9 @@ warc_write_block_from_file (FILE *data_in)
 {
   /* Add the Content-Length header. */
   char content_length[MAX_INT_TO_STRING_LEN(off_t)];
+  char buffer[BUFSIZ];
+  size_t s;
+
   fseeko (data_in, 0L, SEEK_END);
   number_to_string (content_length, ftello (data_in));
   warc_write_header ("Content-Length", content_length);
@@ -266,8 +273,6 @@ warc_write_block_from_file (FILE *data_in)
     warc_write_ok = false;
 
   /* Copy the data in the file to the WARC record. */
-  char buffer[BUFSIZ];
-  size_t s;
   while (warc_write_ok && (s = fread (buffer, 1, BUFSIZ, data_in)) > 0)
     {
       if (warc_write_buffer (buffer, s) < s)
@@ -292,6 +297,11 @@ warc_write_end_record (void)
   /* We start a new gzip stream for each record.  */
   if (warc_write_ok && warc_current_gzfile)
     {
+      char extra_header[EXTRA_GZIP_HEADER_SIZE];
+      char static_header[GZIP_STATIC_HEADER_SIZE];
+      off_t current_offset, uncompressed_size, compressed_size;
+      size_t result;
+
       if (gzclose (warc_current_gzfile) != Z_OK)
         {
           warc_write_ok = false;
@@ -317,17 +327,16 @@ warc_write_end_record (void)
       */
 
       /* Calculate the uncompressed and compressed sizes. */
-      off_t current_offset = ftello (warc_current_file);
-      off_t uncompressed_size = current_offset - warc_current_gzfile_offset;
-      off_t compressed_size = warc_current_gzfile_uncompressed_size;
+      current_offset = ftello (warc_current_file);
+      uncompressed_size = current_offset - warc_current_gzfile_offset;
+      compressed_size = warc_current_gzfile_uncompressed_size;
 
       /* Go back to the static GZIP header. */
       fseeko (warc_current_file, warc_current_gzfile_offset
               + EXTRA_GZIP_HEADER_SIZE, SEEK_SET);
 
       /* Read the header. */
-      char static_header[GZIP_STATIC_HEADER_SIZE];
-      size_t result = fread (static_header, 1, GZIP_STATIC_HEADER_SIZE,
+      result = fread (static_header, 1, GZIP_STATIC_HEADER_SIZE,
                              warc_current_file);
       if (result != GZIP_STATIC_HEADER_SIZE)
         {
@@ -344,7 +353,6 @@ warc_write_end_record (void)
       fwrite (static_header, 1, GZIP_STATIC_HEADER_SIZE, warc_current_file);
 
       /* Prepare the extra GZIP header. */
-      char extra_header[EXTRA_GZIP_HEADER_SIZE];
       /* XLEN, the length of the extra header fields.  */
       extra_header[0]  = ((EXTRA_GZIP_HEADER_SIZE - 2) & 255);
       extra_header[1]  = ((EXTRA_GZIP_HEADER_SIZE - 2) >> 8) & 255;
@@ -387,12 +395,9 @@ static bool
 warc_write_date_header (const char *timestamp)
 {
   char current_timestamp[21];
-  if (timestamp == NULL)
-    {
-      warc_timestamp (current_timestamp);
-      timestamp = current_timestamp;
-    }
-  return warc_write_header ("WARC-Date", timestamp);
+
+  return warc_write_header ("WARC-Date", timestamp ? timestamp :
+                            warc_timestamp (current_timestamp, sizeof(current_timestamp)));
 }
 
 /* Writes the WARC-IP-Address header for the given IP to
@@ -468,7 +473,7 @@ warc_sha1_stream_with_payload (FILE *stream, void *res_block, void *res_payload,
                  or EWOULDBLOCK.  */
               if (ferror (stream))
                 {
-                  free (buffer);
+                  xfree (buffer);
                   return 1;
                 }
               goto process_partial_block;
@@ -528,7 +533,7 @@ warc_sha1_stream_with_payload (FILE *stream, void *res_block, void *res_payload,
   sha1_finish_ctx (&ctx_block,   res_block);
   if (payload_offset >= 0)
     sha1_finish_ctx (&ctx_payload, res_payload);
-  free (buffer);
+  xfree (buffer);
   return 0;
 
 #undef BLOCKSIZE
@@ -570,13 +575,13 @@ warc_write_digest_headers (FILE *file, long payload_offset)
 
           digest = warc_base32_sha1_digest (sha1_res_block);
           warc_write_header ("WARC-Block-Digest", digest);
-          free (digest);
+          xfree (digest);
 
           if (payload_offset >= 0)
             {
               digest = warc_base32_sha1_digest (sha1_res_payload);
               warc_write_header ("WARC-Payload-Digest", digest);
-              free (digest);
+              xfree (digest);
             }
         }
     }
@@ -587,37 +592,102 @@ warc_write_digest_headers (FILE *file, long payload_offset)
    The UTC time is formatted following ISO 8601, as required
    for use in the WARC-Date header.
    The timestamp will be 21 characters long. */
-void
-warc_timestamp (char *timestamp)
+char *
+warc_timestamp (char *timestamp, size_t timestamp_size)
 {
-  time_t rawtime;
-  struct tm * timeinfo;
-  time ( &rawtime );
-  timeinfo = gmtime (&rawtime);
-  strftime (timestamp, 21, "%Y-%m-%dT%H:%M:%SZ", timeinfo);
+  time_t rawtime = time (NULL);
+  struct tm * timeinfo = gmtime (&rawtime);
+
+  if (strftime (timestamp, timestamp_size, "%Y-%m-%dT%H:%M:%SZ", timeinfo) == 0 && timestamp_size > 0)
+    *timestamp = 0;
+
+  return timestamp;
 }
 
-#if HAVE_LIBUUID || HAVE_UUID_CREATE
 /* Fills urn_str with a UUID in the format required
    for the WARC-Record-Id header.
    The string will be 47 characters long. */
+#if HAVE_LIBUUID
 void
 warc_uuid_str (char *urn_str)
 {
   char uuid_str[37];
-
   uuid_t record_id;
-#if HAVE_UUID_CREATE
-  uuid_create (&record_id, NULL);
-  uuid_to_string (&record_id, &uuid_str, NULL);
-#else
+
   uuid_generate (record_id);
   uuid_unparse (record_id, uuid_str);
-#endif
 
   sprintf (urn_str, "<urn:uuid:%s>", uuid_str);
 }
+#elif HAVE_UUID_CREATE
+void
+warc_uuid_str (char *urn_str)
+{
+  char *uuid_str;
+  uuid_t record_id;
+
+  uuid_create (&record_id, NULL);
+  uuid_to_string (&record_id, &uuid_str, NULL);
+
+  sprintf (urn_str, "<urn:uuid:%s>", uuid_str);
+  xfree (uuid_str);
+}
 #else
+# ifdef WINDOWS
+
+typedef RPC_STATUS (RPC_ENTRY * UuidCreate_proc) (UUID *);
+typedef RPC_STATUS (RPC_ENTRY * UuidToString_proc) (UUID *, unsigned char **);
+typedef RPC_STATUS (RPC_ENTRY * RpcStringFree_proc) (unsigned char **);
+
+static int
+windows_uuid_str (char *urn_str)
+{
+  static UuidCreate_proc pfn_UuidCreate = NULL;
+  static UuidToString_proc pfn_UuidToString = NULL;
+  static RpcStringFree_proc pfn_RpcStringFree = NULL;
+  static int rpc_uuid_avail = -1;
+
+  /* Rpcrt4.dll is not available on older versions of Windows, so we
+     need to test its availability at run time.  */
+  if (rpc_uuid_avail == -1)
+    {
+      HMODULE hm_rpcrt4 = LoadLibrary ("Rpcrt4.dll");
+
+      if (hm_rpcrt4)
+	{
+	  pfn_UuidCreate =
+	    (UuidCreate_proc) GetProcAddress (hm_rpcrt4, "UuidCreate");
+	  pfn_UuidToString =
+	    (UuidToString_proc) GetProcAddress (hm_rpcrt4, "UuidToStringA");
+	  pfn_RpcStringFree =
+	    (RpcStringFree_proc) GetProcAddress (hm_rpcrt4, "RpcStringFreeA");
+	  if (pfn_UuidCreate && pfn_UuidToString && pfn_RpcStringFree)
+	    rpc_uuid_avail = 1;
+	  else
+	    rpc_uuid_avail = 0;
+	}
+      else
+	rpc_uuid_avail = 0;
+    }
+
+  if (rpc_uuid_avail)
+    {
+      BYTE *uuid_str;
+      UUID  uuid;
+
+      if (pfn_UuidCreate (&uuid) == RPC_S_OK)
+	{
+	  if (pfn_UuidToString (&uuid, &uuid_str) == RPC_S_OK)
+	    {
+	      sprintf (urn_str, "<urn:uuid:%s>", uuid_str);
+	      pfn_RpcStringFree (&uuid_str);
+	      return 1;
+	    }
+	}
+    }
+  return 0;
+}
+#endif
 /* Fills urn_str with a UUID based on random numbers in the format
    required for the WARC-Record-Id header.
    (See RFC 4122, UUID version 4.)
@@ -629,19 +699,27 @@ warc_uuid_str (char *urn_str)
 void
 warc_uuid_str (char *urn_str)
 {
-  // RFC 4122, a version 4 UUID with only random numbers
+  /* RFC 4122, a version 4 UUID with only random numbers */
 
   unsigned char uuid_data[16];
   int i;
+
+#ifdef WINDOWS
+  /* If the native method fails (expected on older Windows versions),
+     use the fallback below.  */
+  if (windows_uuid_str (urn_str))
+    return;
+#endif
+
   for (i=0; i<16; i++)
     uuid_data[i] = random_number (255);
 
-  // Set the four most significant bits (bits 12 through 15) of the
-  // time_hi_and_version field to the 4-bit version number
+  /* Set the four most significant bits (bits 12 through 15) of the
+	*  time_hi_and_version field to the 4-bit version number */
   uuid_data[6] = (uuid_data[6] & 0x0F) | 0x40;
 
-  // Set the two most significant bits (bits 6 and 7) of the
-  // clock_seq_hi_and_reserved to zero and one, respectively.
+  /* Set the two most significant bits (bits 6 and 7) of the
+	*  clock_seq_hi_and_reserved to zero and one, respectively. */
   uuid_data[8] = (uuid_data[8] & 0xBF) | 0x80;
 
   sprintf (urn_str,
@@ -658,18 +736,19 @@ warc_uuid_str (char *urn_str)
 static bool
 warc_write_warcinfo_record (char *filename)
 {
+  FILE *warc_tmp;
+  char timestamp[22];
+  char *filename_basename;
+
   /* Write warc-info record as the first record of the file. */
   /* We add the record id of this info record to the other records in the
      file. */
   warc_current_warcinfo_uuid_str = (char *) malloc (48);
   warc_uuid_str (warc_current_warcinfo_uuid_str);
 
-  char timestamp[22];
-  warc_timestamp (timestamp);
+  warc_timestamp (timestamp, sizeof(timestamp));
 
-  char *filename_copy, *filename_basename;
-  filename_copy = strdup (filename);
-  filename_basename = strdup (basename (filename_copy));
+  filename_basename = base_name (filename);
 
   warc_write_start_record ();
   warc_write_header ("WARC-Type", "warcinfo");
@@ -678,12 +757,12 @@ warc_write_warcinfo_record (char *filename)
   warc_write_header ("WARC-Record-ID", warc_current_warcinfo_uuid_str);
   warc_write_header ("WARC-Filename", filename_basename);
 
+  xfree (filename_basename);
+
   /* Create content.  */
-  FILE *warc_tmp = warc_tempfile ();
+  warc_tmp = warc_tempfile ();
   if (warc_tmp == NULL)
     {
-      free (filename_copy);
-      free (filename_basename);
       return false;
     }
 
@@ -709,8 +788,6 @@ warc_write_warcinfo_record (char *filename)
   if (! warc_write_ok)
     logprintf (LOG_NOTQUIET, _("Error writing warcinfo record to WARC file.\n"));
 
-  free (filename_copy);
-  free (filename_basename);
   fclose (warc_tmp);
   return warc_write_ok;
 }
@@ -729,22 +806,6 @@ warc_write_warcinfo_record (char *filename)
 static bool
 warc_start_new_file (bool meta)
 {
-  if (opt.warc_filename == NULL)
-    return false;
-
-  if (warc_current_file != NULL)
-    fclose (warc_current_file);
-
-  free (warc_current_warcinfo_uuid_str);
-  free (warc_current_filename);
-
-  warc_current_file_number++;
-
-  int base_filename_length = strlen (opt.warc_filename);
-  /* filename format:  base + "-" + 5 digit serial number + ".warc.gz" */
-  char *new_filename = malloc (base_filename_length + 1 + 5 + 8 + 1);
-  warc_current_filename = new_filename;
-
 #ifdef __VMS
 # define WARC_GZ "warc-gz"
 #else /* def __VMS */
@@ -757,6 +818,25 @@ warc_start_new_file (bool meta)
   const char *extension = "warc";
 #endif
 
+  int base_filename_length;
+  char *new_filename;
+
+  if (opt.warc_filename == NULL)
+    return false;
+
+  if (warc_current_file != NULL)
+    fclose (warc_current_file);
+
+  xfree (warc_current_warcinfo_uuid_str);
+  xfree (warc_current_filename);
+
+  warc_current_file_number++;
+
+  base_filename_length = strlen (opt.warc_filename);
+  /* filename format:  base + "-" + 5 digit serial number + ".warc.gz" */
+  new_filename = malloc (base_filename_length + 1 + 5 + 8 + 1);
+  warc_current_filename = new_filename;
+
   /* If max size is enabled, we add a serial number to the file names. */
   if (meta)
     sprintf (new_filename, "%s-meta.%s", opt.warc_filename, extension);
@@ -828,12 +908,13 @@ static bool
 warc_parse_cdx_header (char *lineptr, int *field_num_original_url,
                        int *field_num_checksum, int *field_num_record_id)
 {
+  char *token;
+  char *save_ptr;
+
   *field_num_original_url = -1;
   *field_num_checksum = -1;
   *field_num_record_id = -1;
 
-  char *token;
-  char *save_ptr;
   token = strtok_r (lineptr, CDX_FIELDSEP, &save_ptr);
 
   if (token != NULL && strcmp (token, "CDX") == 0)
@@ -874,13 +955,12 @@ warc_process_cdx_line (char *lineptr, int field_num_original_url,
   char *original_url = NULL;
   char *checksum = NULL;
   char *record_id = NULL;
-
   char *token;
   char *save_ptr;
-  token = strtok_r (lineptr, CDX_FIELDSEP, &save_ptr);
+  int field_num = 0;
 
   /* Read this line to get the fields we need. */
-  int field_num = 0;
+  token = strtok_r (lineptr, CDX_FIELDSEP, &save_ptr);
   while (token != NULL)
     {
       char **val;
@@ -909,7 +989,7 @@ warc_process_cdx_line (char *lineptr, int field_num_original_url,
       char * checksum_v;
       base32_decode_alloc (checksum, strlen (checksum), &checksum_v,
                            &checksum_l);
-      free (checksum);
+      xfree (checksum);
 
       if (checksum_v != NULL && checksum_l == SHA1_DIGEST_SIZE)
         {
@@ -920,20 +1000,20 @@ warc_process_cdx_line (char *lineptr, int field_num_original_url,
           rec->uuid = record_id;
           memcpy (rec->digest, checksum_v, SHA1_DIGEST_SIZE);
           hash_table_put (warc_cdx_dedup_table, rec->digest, rec);
-          free (checksum_v);
+          xfree (checksum_v);
         }
       else
         {
-          free (original_url);
-          free (checksum_v);
-          free (record_id);
+          xfree (original_url);
+          xfree (checksum_v);
+          xfree (record_id);
         }
     }
   else
     {
-      xfree_null(checksum);
-      xfree_null(original_url);
-      xfree_null(record_id);
+      xfree(checksum);
+      xfree(original_url);
+      xfree(record_id);
     }
 }
 
@@ -942,17 +1022,17 @@ warc_process_cdx_line (char *lineptr, int field_num_original_url,
 static bool
 warc_load_cdx_dedup_file (void)
 {
-  FILE *f = fopen (opt.warc_cdx_dedup_filename, "r");
-  if (f == NULL)
-    return false;
-
+  FILE *f;
+  char *lineptr = NULL;
+  size_t n = 0;
+  ssize_t line_length;
   int field_num_original_url = -1;
   int field_num_checksum = -1;
   int field_num_record_id = -1;
 
-  char *lineptr = NULL;
-  size_t n = 0;
-  ssize_t line_length;
+  f = fopen (opt.warc_cdx_dedup_filename, "r");
+  if (f == NULL)
+    return false;
 
   /* The first line should contain the CDX header.
      Format:  " CDX x x x x x"
@@ -981,6 +1061,8 @@ _("CDX file does not list record ids. (Missing column 'u'.)\n"));
     }
   else
     {
+      int nrecords;
+
       /* Initialize the table. */
       warc_cdx_dedup_table = hash_table_new (1000, warc_hash_sha1_digest,
                                              warc_cmp_sha1_digest);
@@ -998,14 +1080,14 @@ _("CDX file does not list record ids. (Missing column 'u'.)\n"));
       while (line_length != -1);
 
       /* Print results. */
-      int nrecords = hash_table_count (warc_cdx_dedup_table);
+      nrecords = hash_table_count (warc_cdx_dedup_table);
       logprintf (LOG_VERBOSE, ngettext ("Loaded %d record from CDX.\n\n",
                                         "Loaded %d records from CDX.\n\n",
                                          nrecords),
                               nrecords);
     }
 
-  free (lineptr);
+  xfree (lineptr);
   fclose (f);
 
   return true;
@@ -1018,11 +1100,12 @@ _("CDX file does not list record ids. (Missing column 'u'.)\n"));
 static struct warc_cdx_record *
 warc_find_duplicate_cdx_record (char *url, char *sha1_digest_payload)
 {
+  struct warc_cdx_record *rec_existing;
+
   if (warc_cdx_dedup_table == NULL)
     return NULL;
 
-  struct warc_cdx_record *rec_existing
-    = hash_table_get (warc_cdx_dedup_table, sha1_digest_payload);
+  rec_existing = hash_table_get (warc_cdx_dedup_table, sha1_digest_payload);
 
   if (rec_existing && strcmp (rec_existing->url, url) == 0)
     return rec_existing;
@@ -1093,11 +1176,13 @@ warc_init (void)
 static void
 warc_write_metadata (void)
 {
+  char manifest_uuid[48];
+  FILE *warc_tmp_fp;
+
   /* If there are multiple WARC files, the metadata should be written to a separate file. */
   if (opt.warc_maxsize > 0)
     warc_start_new_file (true);
 
-  char manifest_uuid [48];
   warc_uuid_str (manifest_uuid);
 
   fflush (warc_manifest_fp);
@@ -1107,7 +1192,7 @@ warc_write_metadata (void)
                               warc_manifest_fp, -1);
   /* warc_write_resource_record has closed warc_manifest_fp. */
 
-  FILE * warc_tmp_fp = warc_tempfile ();
+  warc_tmp_fp = warc_tempfile ();
   if (warc_tmp_fp == NULL)
     {
       logprintf (LOG_NOTQUIET, _("Could not open temporary WARC file.\n"));
@@ -1143,7 +1228,7 @@ warc_close (void)
   if (warc_current_file != NULL)
     {
       warc_write_metadata ();
-      free (warc_current_warcinfo_uuid_str);
+      xfree (warc_current_warcinfo_uuid_str);
       fclose (warc_current_file);
     }
   if (warc_current_cdx_file != NULL)
@@ -1162,6 +1247,8 @@ FILE *
 warc_tempfile (void)
 {
   char filename[100];
+  int fd;
+
   if (path_search (filename, 100, opt.warc_tempdir, "wget", true) == -1)
     return NULL;
 
@@ -1180,13 +1267,16 @@ warc_tempfile (void)
     return fopen (tfn, "w+", "fop=tmd");    /* Create auto-delete temp file. */
   }
 #else /* def __VMS */
-  int fd = mkostemp (filename, O_TEMPORARY);
+  fd = mkostemp (filename, O_TEMPORARY);
   if (fd < 0)
     return NULL;
 
 #if !O_TEMPORARY
   if (unlink (filename) < 0)
-    return NULL;
+    {
+      close(fd);
+      return NULL;
+    }
 #endif
 
   return fdopen (fd, "wb+");
@@ -1243,7 +1333,10 @@ warc_write_cdx_record (const char *url, const char *timestamp_str,
                        const char *response_uuid)
 {
   /* Transform the timestamp. */
-  char timestamp_str_cdx [15];
+  char timestamp_str_cdx[15];
+  char offset_string[MAX_INT_TO_STRING_LEN(off_t)];
+  const char *checksum;
+
   memcpy (timestamp_str_cdx     , timestamp_str     , 4); /* "YYYY" "-" */
   memcpy (timestamp_str_cdx +  4, timestamp_str +  5, 2); /* "mm"   "-" */
   memcpy (timestamp_str_cdx +  6, timestamp_str +  8, 2); /* "dd"   "T" */
@@ -1253,7 +1346,6 @@ warc_write_cdx_record (const char *url, const char *timestamp_str,
   timestamp_str_cdx[14] = '\0';
 
   /* Rewrite the checksum. */
-  const char *checksum;
   if (payload_digest != NULL)
     checksum = payload_digest + 5; /* Skip the "sha1:" */
   else
@@ -1264,7 +1356,6 @@ warc_write_cdx_record (const char *url, const char *timestamp_str,
   if (redirect_location == NULL || strlen(redirect_location) == 0)
     redirect_location = "-";
 
-  char offset_string[MAX_INT_TO_STRING_LEN(off_t)];
   number_to_string (offset_string, offset);
 
   /* Print the CDX line. */
@@ -1296,10 +1387,11 @@ warc_write_revisit_record (char *url, char *timestamp_str,
                            char *refers_to, ip_address *ip, FILE *body)
 {
   char revisit_uuid [48];
-  warc_uuid_str (revisit_uuid);
-
   char *block_digest = NULL;
   char sha1_res_block[SHA1_DIGEST_SIZE];
+
+  warc_uuid_str (revisit_uuid);
+
   sha1_stream (body, sha1_res_block);
   block_digest = warc_base32_sha1_digest (sha1_res_block);
 
@@ -1321,7 +1413,7 @@ warc_write_revisit_record (char *url, char *timestamp_str,
   warc_write_end_record ();
 
   fclose (body);
-  free (block_digest);
+  xfree (block_digest);
 
   return warc_write_ok;
 }
@@ -1349,6 +1441,8 @@ warc_write_response_record (char *url, char *timestamp_str,
   char *payload_digest = NULL;
   char sha1_res_block[SHA1_DIGEST_SIZE];
   char sha1_res_payload[SHA1_DIGEST_SIZE];
+  char response_uuid [48];
+  off_t offset;
 
   if (opt.warc_digests_enabled)
     {
@@ -1381,7 +1475,7 @@ warc_write_response_record (char *url, char *timestamp_str,
               result = warc_write_revisit_record (url, timestamp_str,
                          concurrent_to_uuid, payload_digest, rec_existing->uuid,
                          ip, body);
-              free (payload_digest);
+              xfree (payload_digest);
 
               return result;
             }
@@ -1393,11 +1487,10 @@ warc_write_response_record (char *url, char *timestamp_str,
 
   /* Not a revisit, just store the record. */
 
-  char response_uuid [48];
   warc_uuid_str (response_uuid);
 
   fseeko (warc_current_file, 0L, SEEK_END);
-  off_t offset = ftello (warc_current_file);
+  offset = ftello (warc_current_file);
 
   warc_write_start_record ();
   warc_write_header ("WARC-Type", "response");
@@ -1423,8 +1516,8 @@ warc_write_response_record (char *url, char *timestamp_str,
       response_uuid);
     }
 
-  free (block_digest);
-  free (payload_digest);
+  xfree (block_digest);
+  xfree (payload_digest);
 
   return warc_write_ok;
 }

src/warc.h

@@ -6,9 +6,10 @@
 
 void warc_init (void);
 void warc_close (void);
-void warc_timestamp (char *timestamp);
 void warc_uuid_str (char *id_str);
 
+char * warc_timestamp (char *timestamp, size_t timestamp_size);
+
 FILE * warc_tempfile (void);
 
 bool warc_write_request_record (char *url, char *timestamp_str,

src/wget.h

@@ -59,7 +59,7 @@ as that of the covered work.  */
 /* `gettext (FOO)' is long to write, so we use `_(FOO)'.  If NLS is
    unavailable, _(STRING) simply returns STRING.  */
 #include "gettext.h"
-#define _(string)   gettext (string)
+#define _(STRING) gettext(STRING)
 
 /* A pseudo function call that serves as a marker for the automated
    extraction of messages, but does not call gettext().  The run-time
@@ -266,7 +266,7 @@ typedef double SUM_SIZE_INT;
 /* The same as above, except the comparison is case-insensitive. */
 #define BOUNDED_EQUAL_NO_CASE(beg, end, string_literal)         \
   ((end) - (beg) == sizeof (string_literal) - 1                 \
-   && !strncasecmp (beg, string_literal, sizeof (string_literal) - 1))
+   && !c_strncasecmp (beg, string_literal, sizeof (string_literal) - 1))
 
 /* Like ptr=strdup(str), but allocates the space for PTR on the stack.
    This cannot be an expression because this is not portable:
@@ -312,8 +312,15 @@ typedef double SUM_SIZE_INT;
    in base 10. 24082 / 10000 = 8*log_{10}(2).  */
 #define MAX_INT_TO_STRING_LEN(x) ((sizeof(x) * 24082 / 10000) + 2)
 
+/* Find the minimum or maximum of two provided values */
+# define MIN(i, j) ((i) <= (j) ? (i) : (j))
+# define MAX(i, j) ((i) >= (j) ? (i) : (j))
+
+
 extern const char *exec_name;
-
+extern const char *program_name;
+extern const char *program_argstring;
+
 /* Document type ("dt") flags */
 enum
 {
@@ -337,7 +344,7 @@ typedef enum
   FTPOK, FTPLOGINC, FTPLOGREFUSED, FTPPORTERR, FTPSYSERR,
   FTPNSFOD, FTPUNKNOWNTYPE, FTPRERR,
   FTPSRVERR, FTPRETRINT, FTPRESTFAIL, URLERROR, FOPENERR,
-  FOPEN_EXCL_ERR, FWRITEERR, HEOF,
+  FOPEN_EXCL_ERR, FWRITEERR, HEOF, GATEWAYTIMEOUT,
   HERR, RETROK, RECLEVELEXC, WRONGCODE,
   FTPINVPASV, FTPNOPASV, CONTNOTSUPPORTED, RETRUNNEEDED, RETRFINISHED,
   READERR, TRYLIMEXC, FILEBADFILE, RANGEERR,

testenv/ChangeLog

@@ -1,527 +0,0 @@
-2014-10-01  Darshit Shah  <darnir@gmail.com>
-
-	* Makefile.am: Run the tests in Python's Optimizedmode
-	* conf/__init__.py (gen_hook): Use try..except instead of if..else
-	* misc/color_terminal.py: System and check will not change while a test is
-	run. Do not test for them on every invokation of printer()
-	* server/http/http_server.py: The ssl and re modules are required by
-	specific functions. Load them lazily
-	(HTTPSServer.__init__): Lazy load ssl module here
-	(_handler.parse_range_header): Lazy load re module here
-	(_Handler.get_rule_list): get() can return a default value. Use it
-	(_Handler.guess_type): Same
-	(_Handler.is_authorized): Unused function artefact. Remove
-	(_Handler.reject_headers): Unused function artefact. Remove
-
-2014-10-08  Darshit Shah  <darnir@gmail.com>
-
-	* Makefile.am: Fix EXTRA_DIST variable for make distcheck
-	* server/http/http_server.py (HTTPServer.__init__):  Fix how CERTFILE is
-	found when running make dist / make distcheck.
-
-2014-09-30  Tim Ruehsen  <tim.ruehsen@gmx.de>
-
-	* test/base_test.py: Add --track-origins=yes to valgrind testing
-
-2014-08-08  Darshit Shah  <darnir@gmail.com>
-
-	* conf/__init__.py: Add extra newline according to PEP8
-	* conf/{authentication,expect_header,expected_files,expected_ret_code,
-	files_crawled,hook_sample,local_files,reject_header,response,send_header,
-	server_files,urls,wget_commands}.py: Add docstrings explaining the conf file
-	and how it should be used
-	* server/http/http_server (InvalidRangeHeader): Clear TODO and eliminate
-	this exception. Use ServerError for all such purposes.
-	(_Handler): Remove reference to InvalidRangeHeader
-	(_handler.parse_range_header): User ServerError instead of InvalidRangeHeader
-	(_Handler.do_GET):  Add docstring
-	(_Handler.do_POST): Add docstring. Also create an empty dict for rules if
-	no rules are supplied. Send the Location header as suggested in RFC 7231
-	(_Handler.do_PUT): Don't pop the server file already. Push it to later in ..
-	(_Handler.send_put): .. Here. If the file exists respond with a 204 No
-	Content message and pop the file for replacement. Do not send the
-	Content-Length, Content-Type headers since PUT requests should not respond
-	with data.
-	(_Handler.parse_auth_header): Fit line within 80 chars
-	(_Handler.check_response): Better visual indent
-	(_Handler.authorize_digest): Better visual indent.
-	(_Handler.expect_headers): Remove unused function
-	(_Handler.guess_type): Fix indentation
-	(HTTPd): Add newline according to PEP8 guidelines
-	(HTTPSd): Fix indentation
-	(StoppableHTTPServer): Add docstring
-	(HTTPSServer): Fix indentation
-	(WgetHTTPRequestHandler): Merge class into _handler.
-	(_Handler): Add docstring
-	(_Handler.parse_range_header): Fix indentation
-	(ServerError): Split exception into separate file ...
-	* exc/server_error.py: ... Here
-	* misc/colour_terminal.py: Add docstring, fix indentation
-	* test/base_test.py: Fix visual indent
-	* test/http_test.py: Fit within 80 char lines
-
-2014-08-04  Darshit Shah  <darnir@gmail.com>
-
-	* conf/server_conf.py: Delete file. Server configuration is now done via the
-	server_conf() method.
-	* server/http/http_server.py (StppableHTTPServer.server_sett): Delete
-	method required by the above hook
-	(HTTPd.server_sett): Same
-
-2014-07-26  Darshit Shah  <darnir@gmail.com>
-
-	* Test-*.py: Remove the '-d' switch from WGET_OPTIONS.
-	* test/base_test (BaseTest.gen_cmd_line): Add --debug and --no-config to the
-	list of switches passed to wget unconditionally.
-
-2014-07-23  Darshit Shah  <darnir@gmail.com>
-
-	* test/base_test.py (BaseTest.gen_cmd_line): Add support for running all
-	tests through valgrind if the relevant environment variable is set
-	* conf/expected_ret_code (ExpectedRetCode.__call__): Valgrind returns error
-	code 45 when it detects a memory leak.
-	* Readme: Update with details about valgrind tests
-
-2014-07-22  Darshit Shah  <darnir@gmail.com>
-
-	* (README): Remove old TODO and document SERVER_WAIT variable
-
-2014-06-22  Darshit Shah  <darnir@gmail.com>
-
-	* (conf.files_crawled): diff is a set object and needs explicit str
-	conversion.
-
-2014-03-13  Zihang Chen  <chsc4698@gmail.com>
-
-	* base_test.py:
-	(CommonMethods): Rename to BaseTest.
-	(BaseTest): Implement __init__ method where the class-wide variables are
-	initialized. Also variable names like `xxx_list` is renamed to its plural
-	form, e.g. `server_list` => `servers`.
-	(BaseTest.init_test_env): Remove name argument due to its unnecessarity.
-	(BaseTest.get_test_dir): Because the path of the test directory is needed
-	in multiple methods, this method is implemented.
-	(BaseTest.get_domain_addr): Rewrite the return statement utilizing str
-	formatting (which is more Pythonic).
-	(BaseTest.get_cmd_line): Rename to gen_cmd_line. Change the variables with
-	capitcal characters to lower ones. Also, the nested for loop is rewritten
-	to a plain loop using the zip function.
-	(BaseTest.__gen_local_filesys): Rename to gen_local_fs_snapshot. Move to
-	ExpectedFiles in conf/expected_files.py and is marked as a static
-	method. Refactor to a less verbose implementation.
-	(BaseTest._check_downloaded_files): Rename to __call__ to agree with the
-	invocation in test case classes. Move to ExpectedFiles in
-	conf/expected_files.py.
-	(BaseTest.get_server_rules): Refactor to a more Pythonic form utilizing
-	dict.items() and is marked static.
-	(BaseTest.stop_server): (new method) an abstract method which should stop
-	the currently using servers.
-	(BaseTest.instantiate_server_by): (new method) an abstract method which
-	should instantiate a server instance according to the given argument.
-	(BaseTest.__enter__): (new method) method which initialize the context
-	manager
-	(BaseTest.__exit__): (new method) method that finilize the context manager
-	and deal with the exceptions during the execution of the with statement,
-	subclasses can override this method for extensibility
-	* http_test.py:
-	(HTTPTest.__init__): Add call to super.__init__. Default values of
-	pre_hook, test_params, post_hook are set to None to avoid a subtle bug of
-	Python. Argument servers is renamed to protocols.
-	(HTTPTest.Server_setup): Move to BaseTest and rename to server_setup.
-	Calls to pre_hook_call, call_test, post_hook_call are removed.
-	(HTTPTest.hook_call, pre_hook_call, call_test, post_hook_call): Move to
-	BaseTest for that both HTTP test cases and FTP test cases may use these
-	methods.
-	(HTTPTest.init_HTTP_Server, init_HTTPS_Server): Merge and rename to
-	instantiate_server_by to implement the abstract method in BaseTest.
-	(HTTPTest.stop_HTTP_Server): Rename to stop_server to implement the
-	abstract method in BaseTest. Also, pull out the part where remaining
-	requests are gathered into a new method request_remaining.
-	(BaseTest.act_retcode): Rename to ret_code because ExpectedRetCode is
-	moved out from BaseTest, so the name act_retcode is actually a bit
-	verbose.
-	* conf/expected_ret_code.py:
-	(ExpectedRetCode.__call__): Rewrite the str into a more readable form.
-	* conf/files_crawled.py:
-	(FilesCrawled.__call__): Refactor this method into a more Pythonic form
-	utilizing the zip function.
-	* conf/local_files.py:
-	(LocalFiles__call__): Rewrite this method with the recommended with
-	statement.
-	* conf/server_conf.py:
-	(ServerConf.__call__): Rewrite this method due to BaseTest.server_list is
-	renamed to BaseTest.servers.
-	* conf/server_files.py:
-	(ServerFiles.__call__): Refactor the nested for loop into a plain one
-	utilizing the zip function.
-	* conf/urls.py:
-	(URLs): Rename url_list to urls.
-	* conf/wget_commands.py:
-	(WgetCommands): Rename command_list to commands, rename test_obj.options
-	to test_obj.wget_options.
-	* Test--https.py, Test-Proto.py, Test-Parallel-Proto.py: Argument servers
-	is changed to protocols due to change in the signature of
-	HTTPTest.__init__.
-
-2014-03-13  Zihang Chen  <chsc4698@gmail.com>
-
-	* test: (new package) package for test case classes
-	* WgetTest.py: Split into test/base_test.py and test/http_test.py.
-	* Test-*.py: Optimize the imports according to changes of WgetTest.py
-
-2014-03-13  Zihang Chen  <chsc4698@gmail.com>
-
-	* server: (new package) package for the server classes
-	* server.http: (new package) package for HTTP server
-	* server.ftp: (new package) package for FTP server
-	* HTTPServer.py: Move to server/http/http_server.py. Also change the
-	CERTFILE to '../certs/wget-cert.pem'.
-	* FTPServer.py: Move to server/ftp/ftp_server.py.
-	* WgetTest.py: Optimize import respect to the server classes.
-
-2014-03-13  Zihang Chen  <chsc4698@gmail.com>
-
-	* conf: (new package) package for rule classes and hook methods
-	* WgetTest.py:
-	(CommonMethods.Authentication): Move to conf/authentication.py.
-	(CommonMethods.ExpectHeader): Move to conf/expect_header.py.
-	(CommonMethods.RejectHeader): Move to conf/reject_header.py.
-	(CommonMethods.Response): Move to conf/response.py.
-	(CommonMethods.SendHeader): Move to conf/send_header.py.
-	(CommonMethods.ServerFiles): Move to conf/server_files.py.
-	(CommonMethods.LocalFiles): Move to conf/local_files.py.
-	(CommonMethods.ServerConf): Move to conf/server_conf.py.
-	(CommonMethods.WgetCommands): Move to conf/wget_commands.py.
-	(CommonMethods.Urls): Move to conf/urls.py.
-	(CommonMethods.ExpectedRetcode): Move to conf/expected_retcode.py.
-	(CommonMethods.ExpectedFiles): Move to conf/expected_files.py.
-	(CommonMethods.FilesCrawled): Move to conf/files_crawled.py.
-	(CommonMethods.__check_downloaded_files): Rename to
-	_check_downloaded_files, so that the method is callable from outside the
-	class.
-	(CommomMethods.get_server_rules): Modify so that it utilizes the conf
-	package.
-	(HTTPTest): Add a method hook_call(configs, name) to reduce duplications
-	in pre_hook_call, call_test and post_hook_call utilizing the conf package.
-	* conf/hook_sample.py: (new file) sample for hooks
-	* conf/rule_sample.py: (new file) sample for rules
-	* REAMDE: Update sections about customizing rules and hooks.
-
-2014-03-13  Zihang Chen  <chsc4698@gmail.com>
-
-	* exc: (new package) package for miscellaneous exceptions
-	* WgetTest.py: Move TestFailed to exc/test_failed.py.
-
-2014-03-13  Zihang Chen  <chsc4698@gmail.com>
-
-	* Test-Proto.py: Fix a typo (line 71: server to servers).
-
-2014-03-13  Zihang Chen  <chsc4698@gmail.com>
-
-	* WgetTest.py: Move WgetFile to package misc.
-	* README: Modify documentation respect to WgetFile.
-	* Test-*.py: Optimize imports about WgetFile.
-
-2014-03-13  Zihang Chen  <chsc4698@gmail.com>
-
-	* misc: (new package) package for miscellaneous modules
-	* ColourTerm.py: Move to package misc and rename to colour_terminal.py,
-	add print_color functions to reduce the use of string literals like
-	"BLUE", "RED" etc.
-	* WgetTest.py:
-	(CommonMethods.Server_setup): Change invocation to printer to print_blue.
-	(CommonMethods.FilesCrawled): Change invocation to printer to print_red.
-	(HTTPTest.__init__): Change invocations to printer to print_red and
-	print_green respectively.
-
-2014-01-02  Darshit Shah  <darnir@gmail.com>
-	* Makefile.am: Add new Test--https.py to list of tests and EXTRA_DIST.
-	Also replace all tabs with spaces in file for conformity.
-	* Test--https.py: New test to check if Wget works correctly with HTTPS
-	servers
-	* HTTPServer.py: Import new modules for use in HTTPS Servers
-	(HTTPSServer): New class that generates a SSL-wrapped socket for use in a
-	HTTPS Server.
-	(HTTPSd): HTTPS daemon class. Analogous to the HTTPd class
-	* WgetTest.py: Define global variables HTTP and HTTPS to reflect Server
-	types
-	(CommonMethods.exec_wget): Add the protocol information to the URL before
-	passing it to wget
-	(HTTPTest.__init__): Edit syntax. The servers variable now accepts a list of
-	servers defined by their type. E.g. HTTP, HTTPS.
-	(HTTPTest.Server_setup): Reflect change in type of variable servers.
-	However, we maintin the value of self.servers to allow most of the code to
-	remain unchanged.
-	(HTTPTest.init_HTTPS_Server): Initialize a HTTPS Server
-	* Test-Parallel-Proto.py: Edit to reflect slight change in Test Fiel Syntax.
-	* Test-Proto.py: Same
-
-2014-01-02  Darshit Shah  <darnir@gmail.com>
-
-	* WgetTest.py (CommonMentods.exec_wget): Wait for n seconds before calling
-	the Wget executable.
-
-2013-12-27  Darshit Shah  <darnir@gmail.com>
-
-	* WgetTest.py: Add modeline
-	(CommonMethods.ServerConf): New pre-test hook that sets
-	BaseHTTPRequestHandler class variables in all available servers
-	* HTTPServer.py (HTTPd.ServerConf): Call the respective method in the Server
-	to set the class variables
-	(StoppableHTTPServer.server_sett): Set the handler class variables
-
-2013-12-26  Darshit Shah  <darnir@gmail.com>
-
-	* WgetTest.py (HTTPTest.call_test): Correct the call to stop_HTTP_Server.
-
-2013-12-25  Darshit Shah  <darnir@gmail.com>
-
-	* WgetTest.py (CommonMehtods.exec_wget): Catch and handle exception if the
-	Wget executable is not found at src/wget
-	(HTTPTest.call_test): In case of error during execution, remove all existing
-	servers before quitting
-
-2013-12-15  Darshit Shah  <darnir@gmail.com>
-
-	* WgetTest.py (HTTPTest.HTTP_setup): Rename to Server_setup so it can be
-	easily reused for other non-HTTP servers.
-	(HTTPTest.__init__): Call Server_setup instead of HTTP_setup
-	(HTTPTest.Server_setup): Split into three more functions, that handle
-	pre-hooks, test execution and post-hooks respectively.
-	(HTTPTest.pre_hook_call): Set up and execute the pre-test hooks. Code split
-	from HTTPTest.Server_setup
-	(HTTPTest.call_test): Execute wget and log exit code. Code split from
-	HTTPTest.Server_setup
-	(HTTPTest.post_hook_call): Set up and execute post-test hooks. Code split
-	from HTTPTest.Server_setup
-
-2013-10-14  Giuseppe Scrivano  <gscrivan@redhat.com>
-
-	* Makefile.am (XFAIL_TESTS): Remove Test--spider-r.py.
-
-2013-10-06  Giuseppe Scrivano  <gscrivan@redhat.com>
-
-	* Makefile.am (EXTRA_DIST): Distribute test files.
-
-2013-09-16  Darshit Shah  <darnir@gmail.com>
-
-	* README: Update documentation
-
-2013-09-14  Darshit Shah  <darnir@gmail.com>
-
-	* HTTPServer.py (StoppableHTTPServer): Define object variable
-	request_headers which stores a list of requests received by the server
-	(StoppableHTTPServer.get_req_headers): Return the list of Request
-	Headers stored by the server
-	(_Handler.do_HEAD): Send the Request MEthod string for identification
-	(_Handler.do_GET): Same
-	(_Handler.__log_request): Log the request in Request_Headers list
-	(_Handler.send_head): Make a call to __log_request
-	* Test--spider-r.py: Add new list, Request_List, which contains all
-	the requests that Wget is expected to send. This will allow for
-	fine-grained tests on recursive downloading.
-	* WgetTest.py (CommonMethods.FilesCrawled): New Post-Test Hook, that
-	ensures that all the expected Files on the server were accessed as
-	expected.
-	(HTTPTest.stop_HTTP_server): On stopping server, asks it to respond
-	with list of all requests it received.
-
-2013-09-13  Darshit Shah  <darnir@gmail.com>
-
-	* Test--spider-r.py: Test retrieval in recursive spider mode.
-	* Makefile.am: add new file
-
-2013-09-13  Darshit Shah  <darnir@gmail.com>
-
-	* HTTPServer.py (_Handler.do_HEAD): If requested path is /, respond
-	with /index.html
-	(_Handler.do_HEAD): Smartly guess value of Content-Type Header from
-	file extension
-	(_Handler.guess_type): Use a preset list of extensions and
-	Content-Type strings. If the extension matches one in the list, use
-	that string, else default to "text/plain"
-
-2013-09-13  Darshit Shah  <darnir@gmail.com>
-
-	* WgetTest.py (CommonMethods._replace_substring): New method that will
-	replace a substring delimited by {{ }} characters by the value of
-	self.<substring> variable
-	(CommonMethods.WgetCommands): Use the _replace_substring () call to
-	replace the substrings in the the command line.
-	(CommonMethods.ServerFiles): Run the _replace_substring () method on
-	the File contents too.
-
-2013-09-11  Darshit Shah  <darnir@gmail.com>
-
-	* WgetTest.py (CommonMethods.exec_wget): Expect domain_list instead of
-	domain.
-	(CommonMethods.get_cmd_line): Same. Generate command line by
-	prepending to each file it's respective domain string
-	(CommonMethods.ServerFiles): Generate file_list and server_rules for
-	each Server and set the config details
-	(HTTPTest): New named parameter, servers which signifies number of
-	servers to spawn
-	(HTTPTest.HTTP_setup): This method now takes servers as a new
-	parameter. Instead of storing server and domain, we now store
-	server_list and domain_list. Each server must be initialized through a
-	loop.
-	(HTTPTest.stop_HTTP_server): Stop all servers in a loop.
-	* Test-Parallel-Proto.py: Prototype test file for multiple servers.
-
-2013-09-10  Darshit Shah  <darnir@gmail.com>
-
-	* WgetTest.py (HTTPTest.stop_HTTP_server): With the threaded servers,
-	we can simply use the socketserver.shutdown() method to close the
-	server instead of sending a QUIT command
-	* HTTPServer.py (StoppabelHTTPServer.serve_forever): Delete method. No
-	need to override this method anymore.
-	(WgetHTTPRequestHandler.do_QUIT): No longer required
-	(HTTPd): Rename self.server to self.server_inst to reduce ambiguity
-	when referenced from WgetTest
-
-2013-09-08  Darshit Shah  <darnir@gmail.com>
-
-	* README (File Structure): Add explanation about various variables
-	used consistently across all tests.
-
-2013-09-07  Darshit Shah  <darnir@gmail.com>
-
-	* HTTPServer.py: Remove bunch of old code artefacts
-	* WgetTest.py: Same
-
-2013-09-07  Darshit Shah  <darnir@gmail.com>
-
-	* HTTPServer.py (StoppableHTTPServer.server_conf): Change global
-	variable fileSys to an object variable. This is good programming
-	practice and required for parallel-wget support.
-	(StoppableHTTPServer.server_forever): Edit overridden method to remove
-	the global queue variable. No longer required under the new working
-	(WgetHTTPRequestHandler.do_QUIT): Don't push fileSys through the queue
-	(_Handler): Rename class __Handler to _Handler to match Python's
-	encapsulation rules
-	(_Handler.do_POST): fileSys is now an object variable of the server
-	(_Handler.do_PUT): Same
-	(_Handler.send_put): Same
-	(_Handler.send_head): Same
-	(HTTPd): New class that wraps around the server for Threading
-	(create_server): Make new object of HTTPd.
-	(spawn_server): Start the thread created through create_server
-	(ret_fileSys): Removed method. No longer required.
-	* WgetTest.py (HTTPTest.__init__): Don't explicitly set
-	self.act_retcode. Instead toggle tests_passed boolean to set the
-	correct return code.
-	(HTTPTest.HTTP_setup): We no longer call HTTPServer.spawn_server to
-	start a new instance of the server.
-	(HTTPTest.init_HTTP_server): We no longer call the old
-	create_server(), spawn_server() methods. Instead use the new HTTPd
-	class interface to create new instances of the server
-	(HTTPTest.stop_HTTP_server): Don't ask server to return fileSys.
-
-2013-09-07  Darshit Shah  <darnir@gmail.com>
-
-	* Test-Post.py: Test basic functionality for sending HTTP POST
-	requests using the --method command
-	* Makefile.am: Add new test
-
-2013-09-06  Darshit Shah  <darnir@gmail.com>
-
-	* WgetTest.py (CommonMethods.__check_downloaded_files): Print a
-	unified diff in case there is a mismatch in the file contents
-
-2013-09-06  Darshit Shah  <darnir@gmail.com>
-
-	* HTTPServer.py (WgetHTTPRequestHandler.test_cookies): Comment out the
-	old test_cookies code. This is no longer used and was causing problems
-	with expected cookies. The code will soon be removed anyways
-	* Test-cookie.py: Add new test for basic cookie functionality
-	* Test-cookie-401.py: Ensure cookies are saved during a 401 response
-	* Test-cookie-expires.py: Ensure that the Expires field is correctly
-	handled
-	* Test-cookies-domain-mismatch.py: Ensure that mismatched domains are
-	handled by Wget
-	* Makefile.am: Add the new tests
-
-2013-09-06  Darshit Shah  <darnir@gmail.com>
-
-	* README: New section on pending work. Will keep updating this to keep
-	track of work that remains to be done on this implementation
-
-2013-09-05  Darshit Shah  <darnir@gmail.com>
-
-	* Test-auth-with-content-disposition.py: Add test that ensures Content
-	Disposition works alongwith authentication
-	* Makefile.am: Add new test
-
-2013-09-04  Darshit Shah  <darnir@gmail.com>
-
-	* Test-c-full.py: Test Continue options
-	* Makefile.am: Add Test-c-full.py and Test-O
-
-2013-09-02  Darshit Shah  <darnir@gmail.com>
-
-	* Makefile.am:	Add new Test
-	* Test-Head.py: New Test to ensure HEAD requests are handled correctly
-
-2013-08-31  Darshit Shah  <darnir@gmail.com>
-
-	* README: Explain that TEST_NAME needs to be unique
-	* Test-auth-no-challenge.py: Edit non-unique TEST_NAME
-
-2013-08-31  Darshit Shah  <darnir@gmail.com>
-
-	* HTTPTest.py (ServerError): Define new Exception for handling
-	internal control flow.
-	(StoppableHTTPServer.SendHeader): Simply pass. Do nothing. Adding
-	functionality here seems to crash for no apparent reason.
-	(stoppableHTTPServer.send_cust_headers): Minor optimization. No need
-	for extra variable.
-	(__Handler.Response): Handle explicit Response Code Rules
-	(__Handler.Authentication): Handle Authentication rules
-	(__Handler.handle_auth): Actual worker method for authentication
-	(__Handler.ExpectHeader): Ensure Expected Headers are received
-	(__Handler.RejectHeader): Ensure Blacklisted Headers are NOT received
-	(__Handler.send_HEAD): Dynamically call server rule functions based on
-	the self.rules list. This feature will later be added to POST/PUT, etc
-
-2013-08-31  Darshit Shah  <darnir@gmail.com>
-
-	* WgetTest.py: Remove import module defaultdict.
-	(CommonMethods.get_server_rules): server_rules should be a dict, not a
-	defaultdict (list).
-	* HTTPServer.py (WgetHTTPRequestHandler.get_rule_list): If rule does
-	not exist, return None.	Not an emppty list.
-	(WgetHTTPRequestHandler.test_cookies): Rule variable is not a list
-	(__Handler.send_cust_headers): Same
-	(__Handler.custom_response): Same
-	(__Handler.is_authorized): Same
-	(__Handler.expect_headers): Same
-	(__Handler.reject_headers): Same
-
-2013-08-31  Darshit Shah  <darnir@gmail.com>
-
-	* README: (newfile) Simple help / instructions about using the Test
-	Environment.
-	* Makefile.am: (newfile) Makefile for the Test Environment. Uses the
-	Automake Parallel Test Harness
-	* WgetTest.py: (newfile) Base module that executes the Test.
-	* HTTPServer.py: (newfile) Contains the custom HTTP Server for the
-	Test Environment. Creates an instance of http.server in Python3.
-	* FTPServer.py: (newfile) Overrides methods from pyftpdlib for use in
-	the Test Environment. ** Work under progress **.
-	* ColourTerm.py: (newfile) A custom module to output coloured text to
-	the terminal. Known to work on POSIX shells.
-	* Test-Proto.py: (newfile) A prototype Test File. This should be
-	copied when writing a new Test Case.
-	* Test-Content-disposition-2.py: Test Content Disposition clobbering
-	* Test-Content-disposition.py: Test Content Disposition Headers
-	* Test-O.py: Test Output filename command
-	* Test-auth-basic-fail.py: Test returncode on auth failure
-	* Test-auth-basic.py: Test Basic Auth negotiation
-	* Test-auth-both.py: Test handling of Multiple auth providers. This
-	test currently fails.
-	* Test-auth-digest.py: Test Digest Auth Negotiation
-	* Test-auth-no-challenge-url.py: Ensure --auth-no-challenge is handled
-	when auth details are in-URL.
-	* Test-auth-no-challenge.py: Ensure --auth-no-challenge is honoured
-	* Test-auth-retcode.py: Ensure correct return code after 403 Forbidden
-	response.

testenv/Makefile.am

@@ -27,8 +27,10 @@
 
 
 AUTOMAKE_OPTIONS = parallel-tests
-AM_TESTS_ENVIRONMENT = export WGETRC=/dev/null; MAKE_CHECK=True; export MAKE_CHECK; export PYTHONPATH=$$PYTHONPATH:$(srcdir);
-TESTS = Test-auth-basic-fail.py             \
+AM_TESTS_ENVIRONMENT = export WGETRC=/dev/null; MAKE_CHECK=True; export MAKE_CHECK;\
+ export PYTHONPATH=$$PYTHONPATH:$(srcdir); export VALGRIND_TESTS="@VALGRIND_TESTS@";
+if HAVE_PYTHON3
+  TESTS = Test-auth-basic-fail.py           \
     Test-auth-basic.py                      \
     Test-auth-both.py                       \
     Test-auth-digest.py                     \
@@ -45,11 +47,16 @@ TESTS = Test-auth-basic-fail.py             \
     Test-cookie.py                          \
     Test-Head.py                            \
     Test--https.py                          \
+    Test--https-crl.py                      \
     Test-O.py                               \
     Test-Post.py                            \
-    Test--spider-r.py
+    Test-504.py                             \
+    Test--spider-r.py                       \
+    Test-redirect-crash.py
 
-XFAIL_TESTS = Test-auth-both.py
+  # added test cases expected to fail here and under TESTS
+  XFAIL_TESTS =
+endif
 
 EXTRA_DIST = certs conf exc misc server test README $(TESTS) $(XFAIL_TESTS)
 

testenv/README

@@ -93,8 +93,10 @@ Environment Variables:
   valgrind.
 * NO_CLEANUP: Do not remove the temporary files created by the test.
   This will prevent the ${testname}-test directory from being deleted
-* VALGRIND_TESTS: If this variable is set, the test suite will execute all the
-  tests through valgrind's memcheck tool.
+* VALGRIND_TESTS: If this variable is set and contains the valgrind command line,
+  the test suite will execute all the tests via this command.
+  If it is set to "1", valgrind memcheck is enabled with hard coded options.
+  This variable is set by ./configure --enable-valgrind-tests.
 
 
 File Structure:

testenv/Test--https-crl.py

@@ -0,0 +1,50 @@
+#!/usr/bin/env python3
+from sys import exit
+from test.http_test import HTTPTest
+from test.base_test import HTTP, HTTPS
+from misc.wget_file import WgetFile
+import os
+
+"""
+    This test ensures that Wget can download files from HTTPS Servers
+"""
+TEST_NAME = "HTTPS CRL"
+############# File Definitions ###############################################
+File1 = "Would you like some Tea?"
+File2 = "With lemon or cream?"
+
+A_File = WgetFile ("File1", File1)
+B_File = WgetFile ("File2", File2)
+
+CAFILE = os.path.abspath(os.path.join(os.getenv('srcdir', '.'), 'certs', 'ca-cert.pem'))
+CRLFILE = os.path.abspath(os.path.join(os.getenv('srcdir', '.'), 'certs', 'server-crl.pem'))
+WGET_OPTIONS = "--crl-file " + CRLFILE + " --ca-certificate=" + CAFILE
+WGET_URLS = [["File1", "File2"]]
+
+Files = [[A_File, B_File]]
+
+Servers = [HTTPS]
+
+ExpectedReturnCode = 5
+
+################ Pre and Post Test Hooks #####################################
+pre_test = {
+    "ServerFiles"       : Files
+}
+test_options = {
+    "WgetCommands"      : WGET_OPTIONS,
+    "Urls"              : WGET_URLS
+}
+post_test = {
+    "ExpectedRetcode"   : ExpectedReturnCode
+}
+
+err = HTTPTest (
+                name=TEST_NAME,
+                pre_hook=pre_test,
+                test_params=test_options,
+                post_hook=post_test,
+                protocols=Servers
+).begin ()
+
+exit (err)

testenv/Test--https.py

@@ -3,6 +3,7 @@ from sys import exit
 from test.http_test import HTTPTest
 from test.base_test import HTTP, HTTPS
 from misc.wget_file import WgetFile
+import os
 
 """
     This test ensures that Wget can download files from HTTPS Servers
@@ -17,7 +18,8 @@ A_File = WgetFile ("File1", File1)
 B_File = WgetFile ("File2", File2)
 C_File = WgetFile ("File3", File3)
 
-WGET_OPTIONS = "--no-check-certificate"
+CAFILE = os.path.abspath(os.path.join(os.getenv('srcdir', '.'), 'certs', 'ca-cert.pem'))
+WGET_OPTIONS = "--ca-certificate=" + CAFILE
 WGET_URLS = [["File1", "File2"]]
 
 Files = [[A_File, B_File]]

testenv/Test-504.py

@@ -0,0 +1,72 @@
+#!/usr/bin/env python3
+from sys import exit
+from test.http_test import HTTPTest
+from misc.wget_file import WgetFile
+
+"""
+    This test ensures that Wget handles a 504 Gateway Timeout response
+    correctly.
+    Since, we do not have a direct mechanism for conditionally sending responses
+    via the HTTP Server, I've used a workaround.
+    The server will always respond to a request for File1 with a 504 Gateway
+    Timeout. Using the --tries=2 option, we ensure that Wget attempts the file
+    only twice and then move on to the next file. Finally, check the exact
+    requests that the Server received and compare them, in order, to the
+    expected sequence of requests.
+
+    In this case, we expect Wget to attempt File1 twice and File2 once. If Wget
+    considered 504 as a general Server Error, it would be a fatal failure and
+    Wget would request File1 only once.
+"""
+TEST_NAME = "504 Gateway Timeouts"
+############# File Definitions ###############################################
+File1 = """All happy families are alike;
+Each unhappy family is unhappy in its own way"""
+File2 = "Anyone for chocochip cookies?"
+
+File1_rules = {
+    "Response"          : 504
+}
+
+A_File = WgetFile ("File1", File1, rules=File1_rules)
+B_File = WgetFile ("File2", File2)
+
+Request_List = [
+    [
+        "GET /File1",
+        "GET /File1",
+        "GET /File2",
+    ]
+]
+
+
+WGET_OPTIONS = "--tries=2"
+WGET_URLS = [["File1", "File2"]]
+
+Files = [[A_File, B_File]]
+
+ExpectedReturnCode = 4
+ExpectedDownloadedFiles = [B_File]
+
+################ Pre and Post Test Hooks #####################################
+pre_test = {
+    "ServerFiles"       : Files
+}
+test_options = {
+    "WgetCommands"      : WGET_OPTIONS,
+    "Urls"              : WGET_URLS
+}
+post_test = {
+    "ExpectedFiles"     : ExpectedDownloadedFiles,
+    "ExpectedRetcode"   : ExpectedReturnCode,
+    "FilesCrawled"      : Request_List
+}
+
+err = HTTPTest (
+                name=TEST_NAME,
+                pre_hook=pre_test,
+                test_params=test_options,
+                post_hook=post_test
+).begin ()
+
+exit (err)

testenv/Test-Content-disposition-2.py

@@ -12,9 +12,10 @@ TEST_NAME = "Content Disposition Clobber"
 File1 = "Teapot"
 File2 = "The Teapot Protocol"
 
+# use upper case 'I' to provoke Wget failure with turkish locale
 File2_rules = {
     "SendHeader"        : {
-        "Content-Disposition" : "Attachment; filename=HTTP.Teapot"
+        "Content-DIsposition" : "Attachment; FILENAME=HTTP.Teapot"
     }
 }
 A_File = WgetFile ("HTTP.Teapot", File1)

testenv/Test-cookie-domain-mismatch.py

@@ -14,7 +14,8 @@ File2 = "Anyone for chocochip cookies?"
 
 File1_rules = {
     "SendHeader"        : {
-        "Set-Cookie"    : "sess-id=0213; path=/; domain=.example.com"
+        # use upper case 'I' to provoke Wget failure with turkish locale
+        "Set-Cookie"    : "sess-id=0213; path=/; DoMAIn=.example.com"
     }
 }
 File2_rules = {

testenv/Test-cookie-expires.py

@@ -30,7 +30,8 @@ File2_rules = {
 }
 File3_rules = {
     "SendHeader"        : {
-        "Set-Cookie"    : "sess-id=0213; path=/; Expires=Sun, 06 Nov 2001 12:32:43 GMT"
+        # use upper case 'I' to provoke Wget failure with turkish locale
+        "Set-Cookie"    : "sess-id=0213; path=/; ExPIRes=Sun, 06 Nov 2001 12:32:43 GMT"
     },
     "ExpectHeader"      : {
         "Cookie"        : "new-sess=N; sess-id=0213"

testenv/Test-redirect-crash.py

@@ -0,0 +1,74 @@
+#!/usr/bin/env python3
+from sys import exit
+from test.http_test import HTTPTest
+from misc.wget_file import WgetFile
+import os
+
+# This test caused wget up to 1.16 to crash
+#os.environ["LC_ALL"] = "en_US.UTF-8"
+
+urls = [
+    "File%20formats/Images/SVG,%20Scalable%20Vector%20Graphics/html,%20W3C%20v1.2%20rec%20(tiny)/directory",
+    "File formats/Images/SVG, Scalable Vector Graphics/html, W3C v1.2 rec (tiny)/directory/",
+    "File%20formats/Images/SVG,%20Scalable%20Vector%20Graphics/html,%20W3C%20v1.2%20rec%20(tiny)/directory/somefile.rng",
+    "File%20formats/Images/SVG%2C%20Scalable%20Vector%20Graphics/html%2c%20W3C%20v1.2%20rec%20%28tiny%29/directory/somefile.rng",
+    "File%20formats/Images/SVG%2C%20Scalable%20Vector%20Graphics/html%2c%20W3C%20v1.2%20rec%20%28tiny%29/directory/",
+    "File%20formats/Images/SVG%2C%20Scalable%20Vector%20Graphics/html%2C%20W3C%20v1.2%20rec%20%28tiny%29/directory"]
+
+
+redirected = [
+        "File%20formats/Images/SVG,%20Scalable%20Vector%20Graphics/html,%20W3C%20v1.2%20rec%20(tiny)/directory/"
+]
+
+TEST_NAME = "Redirection crash"
+############# File Definitions ###############################################
+Index = ""
+for i in urls:
+    Index = Index + "<a href='/%s'></a>" % i
+
+File1 = ""
+
+def get_redirect(url):
+    data = {
+        "File%20formats/Images/SVG,%20Scalable%20Vector%20Graphics/html,%20W3C%20v1.2%20rec%20(tiny)/directory" :
+           "File%20formats/Images/SVG,%20Scalable%20Vector%20Graphics/html,%20W3C%20v1.2%20rec%20(tiny)/directory/",
+        "File%20formats/Images/SVG%2C%20Scalable%20Vector%20Graphics/html%2C%20W3C%20v1.2%20rec%20%28tiny%29/directory" :
+           "File%20formats/Images/SVG,%20Scalable%20Vector%20Graphics/html,%20W3C%20v1.2%20rec%20(tiny)/directory/"
+    }
+    dest = data.get(url)
+    if dest:
+        return {"Response"          : 301,
+                "SendHeader" : {"Location" : "/%s" % dest}}
+    return None
+
+
+index_url = "File%20formats/Images/SVG,%20Scalable%20Vector%20Graphics/html,%20W3C%20v1.2%20rec%20(tiny)/index.html"
+Index_File = WgetFile (index_url, Index)
+Files = ([Index_File] + [WgetFile(i, File1, rules=get_redirect(i)) for i in (redirected + urls)])
+
+WGET_OPTIONS = "--recursive -e robots=off"
+
+WGET_URLS = [[index_url]]
+
+ExpectedReturnCode = 0
+
+################ Pre and Post Test Hooks #####################################
+pre_test = {
+    "ServerFiles"       : [Files]
+}
+test_options = {
+    "WgetCommands"      : WGET_OPTIONS,
+    "Urls"              : WGET_URLS
+}
+post_test = {
+    "ExpectedRetcode"   : ExpectedReturnCode,
+}
+
+err = HTTPTest (
+                name=TEST_NAME,
+                pre_hook=pre_test,
+                test_params=test_options,
+                post_hook=post_test
+).begin ()
+
+exit (err)

testenv/certs/README

@@ -0,0 +1,77 @@
+To create the server RSA private key:
+$ certtool --generate-privkey --outfile server-key.pem --rsa
+
+
+To create a self signed CA certificate:
+$ certtool --generate-privkey --outfile ca-key.pem
+$ certtool --generate-self-signed --load-privkey ca-key.pem --outfile ca-cert.pem
+Common name: GNU Wget
+UID:
+Organizational unit name: Wget
+Organization name: GNU
+Locality name:
+State or province name:
+Country name (2 chars):
+Enter the subject's domain component (DC):
+This field should not be used in new certificates.
+E-mail:
+Enter the certificate's serial number in decimal (default: 6080487640893163573):
+
+Activation/Expiration time.
+The certificate will expire in (days): -1
+
+Extensions.
+Does the certificate belong to an authority? (y/N): y
+Path length constraint (decimal, -1 for no constraint):
+Is this a TLS web client certificate? (y/N):
+Will the certificate be used for IPsec IKE operations? (y/N):
+Is this a TLS web server certificate? (y/N):
+Enter a dnsName of the subject of the certificate:
+Enter a URI of the subject of the certificate:
+Enter the IP address of the subject of the certificate:
+Enter the e-mail of the subject of the certificate:
+Will the certificate be used to sign other certificates? (y/N): y
+Will the certificate be used to sign CRLs? (y/N): y
+Will the certificate be used to sign code? (y/N):
+Will the certificate be used to sign OCSP requests? (y/N): y
+Will the certificate be used for time stamping? (y/N):
+Enter the URI of the CRL distribution point:
+
+
+To generate a server certificate using the private key only:
+$ certtool --generate-certificate --load-privkey server-key.pem --outfile server-cert.pem --load-ca-certificate ca-cert.pem --load-ca-privkey ca-key.pem
+Common name: 127.0.0.1
+UID:
+Organizational unit name: Wget
+Organization name: GNU
+Locality name:
+State or province name:
+Country name (2 chars):
+Enter the subject's domain component (DC):
+This field should not be used in new certificates.
+E-mail:
+Enter the certificate's serial number in decimal (default: 6080488276853553635):
+
+Activation/Expiration time.
+The certificate will expire in (days): -1
+
+Extensions.
+Does the certificate belong to an authority? (y/N):
+Is this a TLS web client certificate? (y/N):
+Will the certificate be used for IPsec IKE operations? (y/N):
+Is this a TLS web server certificate? (y/N): y
+Enter a dnsName of the subject of the certificate: 127.0.0.1
+Enter a dnsName of the subject of the certificate: localhost
+Enter a dnsName of the subject of the certificate:
+Enter a URI of the subject of the certificate:
+Enter the IP address of the subject of the certificate:
+Will the certificate be used for signing (DHE and RSA-EXPORT ciphersuites)? (Y/n):
+Will the certificate be used for encryption (RSA ciphersuites)? (Y/n):
+
+
+To create a CRL for the server certificate:
+$ certtool --generate-crl --load-ca-privkey ca-key.pem --load-ca-certificate ca-cert.pem --load-certificate server-cert.pem --outfile server-crl.pem
+Generating a signed CRL...
+Update times.
+The certificate will expire in (days): -1
+CRL Number (default: 6080006793650397145):

testenv/certs/ca-cert.pem

@@ -0,0 +1,20 @@
+-----BEGIN CERTIFICATE-----
+MIIDPjCCAiagAwIBAgIIVGI7SyiXCDUwDQYJKoZIhvcNAQELBQAwMDERMA8GA1UE
+AxMIR05VIFdnZXQxDTALBgNVBAsTBFdnZXQxDDAKBgNVBAoTA0dOVTAiGA8yMDE0
+MTExMTE2MzczM1oYDzk5OTkxMjMxMjM1OTU5WjAwMREwDwYDVQQDEwhHTlUgV2dl
+dDENMAsGA1UECxMEV2dldDEMMAoGA1UEChMDR05VMIIBIjANBgkqhkiG9w0BAQEF
+AAOCAQ8AMIIBCgKCAQEAv2IR1/gsaJsn+egtVFbMMMbivK+eVzPY8wEXb2flpe9a
+Kkwz824nSSrWfRigQmf/ODkNlK2x91kppfPmWkClUREQB3I4d/sHFnzqCkFKmNwt
+VT4DsL47VumgZF6ZjSqTYQD3MDH3VhFj1iKrRMM/aCZXwntD+67sKw6UPXfFbfUO
+Recpb7fCZPZFVFYStxgcF7cyH+DADLX5QCEjiLYH1es6FXrdw+ypgwBl+HuLkQ/4
+gE0JLQK5PAKSYeLy0JEBUPovkMXT+r9aPkQBF/9WD6uUPia0ERHbMIT0My84hBrG
+9d5u5gXPnH1ZxKIr8iJOYCydVjYndn21zs2IExwL6wIDAQABo1gwVjAPBgNVHRMB
+Af8EBTADAQH/MBMGA1UdJQQMMAoGCCsGAQUFBwMJMA8GA1UdDwEB/wQFAwMHBgAw
+HQYDVR0OBBYEFPM+TjiESqm+wW/HYaNQ2m4pi+tUMA0GCSqGSIb3DQEBCwUAA4IB
+AQCWKV1Txo3xytRSO1PwWK9ce5FhvQW9mubuuWmbEnQt9JaOhO1Og6ZmdPDlAQeo
+m0CJZZ0VNeodoBXPGopy3VyzF4ZzUR9leGk3sb1qTSoDCvJd/zLrfukHKugtwuK0
+b1fPnEjBjQ6gQI21coc1tmyX8Qsm7of5n+Ji601WG+FiVmmdEsCjmA/eAvwSbu1E
+cAVH476KUt4CrpVPQQVqzFPDz8i4iw/P+Jo+iNmL5S72OG2GfkSJq3XkUYa509W6
+N+8c3p4y0q/PcQ92s1OIHyEPZFRFhm4xw0sHkhD2KVoHseHZjCoYn4/lIzMPnJ6v
+EfZUJQXndeJnffbthaUp8Ea2
+-----END CERTIFICATE-----

testenv/certs/ca-key.pem

@@ -0,0 +1,144 @@
+Public Key Info:
+	Public Key Algorithm: RSA
+	Key Security Level: Medium (2048 bits)
+
+modulus:
+	00:bf:62:11:d7:f8:2c:68:9b:27:f9:e8:2d:54:56:
+	cc:30:c6:e2:bc:af:9e:57:33:d8:f3:01:17:6f:67:
+	e5:a5:ef:5a:2a:4c:33:f3:6e:27:49:2a:d6:7d:18:
+	a0:42:67:ff:38:39:0d:94:ad:b1:f7:59:29:a5:f3:
+	e6:5a:40:a5:51:11:10:07:72:38:77:fb:07:16:7c:
+	ea:0a:41:4a:98:dc:2d:55:3e:03:b0:be:3b:56:e9:
+	a0:64:5e:99:8d:2a:93:61:00:f7:30:31:f7:56:11:
+	63:d6:22:ab:44:c3:3f:68:26:57:c2:7b:43:fb:ae:
+	ec:2b:0e:94:3d:77:c5:6d:f5:0e:45:e7:29:6f:b7:
+	c2:64:f6:45:54:56:12:b7:18:1c:17:b7:32:1f:e0:
+	c0:0c:b5:f9:40:21:23:88:b6:07:d5:eb:3a:15:7a:
+	dd:c3:ec:a9:83:00:65:f8:7b:8b:91:0f:f8:80:4d:
+	09:2d:02:b9:3c:02:92:61:e2:f2:d0:91:01:50:fa:
+	2f:90:c5:d3:fa:bf:5a:3e:44:01:17:ff:56:0f:ab:
+	94:3e:26:b4:11:11:db:30:84:f4:33:2f:38:84:1a:
+	c6:f5:de:6e:e6:05:cf:9c:7d:59:c4:a2:2b:f2:22:
+	4e:60:2c:9d:56:36:27:76:7d:b5:ce:cd:88:13:1c:
+	0b:eb:
+
+public exponent:
+	01:00:01:
+
+private exponent:
+	45:0c:7f:fd:98:a7:85:12:3d:a9:17:90:8b:36:49:
+	b3:6b:7e:50:af:58:04:84:4b:48:d9:62:f8:29:d7:
+	1c:38:30:22:c4:9d:95:bd:6f:65:21:94:83:4b:c8:
+	3e:4d:41:32:aa:ba:f0:a2:7e:6c:0c:7a:4f:4a:a1:
+	18:7c:ec:68:44:2c:b1:53:0f:76:92:56:2b:51:e4:
+	2a:d1:05:b6:02:f2:44:27:fc:b2:de:df:8f:ea:f8:
+	98:5d:dd:2e:a6:66:c7:ff:ce:2f:50:47:b9:80:ca:
+	b1:6e:8e:b6:5f:6f:58:07:45:70:80:82:b5:a2:95:
+	c8:af:18:e2:d8:7c:9d:bf:c5:a9:da:4f:af:08:37:
+	92:27:94:12:c0:94:70:90:ff:e4:05:8b:ed:18:a9:
+	19:3c:47:3a:7c:fe:4f:9c:15:ab:f6:7e:48:2a:58:
+	d7:14:67:96:bd:e6:fa:9f:3a:51:0c:63:49:14:d5:
+	9d:e9:a8:24:19:2a:83:e4:fe:e2:ec:db:f9:13:33:
+	a6:d3:62:d2:6b:7e:a9:5b:93:73:f5:c9:d0:ad:58:
+	11:cb:77:d3:13:3c:bf:37:f9:64:95:c7:4c:69:f2:
+	6e:b8:36:69:57:93:4a:03:06:58:8a:51:3d:d6:97:
+	61:2f:7c:76:33:14:88:51:45:68:4e:29:fe:12:43:
+	69:
+
+prime1:
+	00:e0:e6:81:38:18:3e:c8:98:51:71:2d:5f:22:8c:
+	93:95:37:17:47:00:4f:6a:87:98:73:8d:f3:c3:02:
+	f7:e1:9d:a0:5c:a5:10:a6:0d:88:5d:e0:72:10:93:
+	24:af:6e:a4:0e:55:5c:03:37:5f:1d:90:41:c2:d6:
+	e3:a6:ba:20:08:0b:01:31:eb:fc:7e:97:66:3c:fe:
+	b5:ab:4c:0b:2f:18:16:f3:28:47:70:41:dc:cf:04:
+	9c:7e:28:78:3b:3f:31:cf:b1:77:2c:6d:c9:bf:ad:
+	19:ff:03:1f:c6:98:9a:60:47:a5:1d:c4:52:c5:9e:
+	77:5a:cc:a4:e3:96:81:d4:4d:
+
+prime2:
+	00:d9:d9:0c:6e:81:bb:0e:5d:c6:92:cc:48:70:b8:
+	da:60:e8:56:e7:2a:20:da:29:0f:c9:f0:9f:b8:9f:
+	df:d9:a1:68:7e:ce:3e:7c:f2:00:66:68:79:c4:01:
+	fa:b9:71:3e:73:06:3f:85:5c:83:33:ee:58:77:50:
+	89:aa:90:33:d0:6c:aa:6f:34:b2:30:8b:e9:a9:82:
+	df:e2:7f:04:09:9f:14:9a:db:c7:cb:e5:85:46:b2:
+	42:d0:a7:fe:7a:e3:ff:1e:84:9c:36:50:e3:de:fb:
+	11:1c:34:09:fe:46:db:45:c3:50:19:f1:25:c0:e3:
+	5c:d5:0d:88:13:e1:9a:5d:17:
+
+coefficient:
+	00:ca:79:cb:79:87:91:9f:9a:99:0b:5d:c5:78:21:
+	a7:60:c6:8a:2d:a5:b5:87:a2:d6:df:b0:17:5f:bf:
+	e1:ce:f0:ca:89:18:0e:e0:4a:7f:00:e5:41:2d:04:
+	5b:05:51:e5:08:89:dd:80:82:c7:94:94:1c:f4:0f:
+	1b:9a:d0:72:83:bb:e9:ca:d5:09:0d:4b:c0:b7:6a:
+	a7:b4:c3:df:4e:f1:7f:0f:57:ad:25:ff:e4:d3:ef:
+	05:95:31:ca:00:54:97:4b:2d:56:aa:1a:89:d8:a0:
+	d6:dc:64:88:88:36:26:92:39:57:8b:da:18:23:77:
+	c3:e3:39:0e:95:f7:3c:77:fe:
+
+exp1:
+	00:99:f2:8f:4f:93:a1:1e:74:cd:82:f8:78:df:d0:
+	74:91:b6:a5:53:6f:cd:ec:f1:26:95:2a:fd:4a:67:
+	34:c1:16:c2:17:c8:d1:ed:a8:e3:c8:c7:03:ad:7e:
+	db:a4:ce:ca:b4:19:10:24:0f:7a:27:65:80:ee:5b:
+	64:77:d3:7e:6b:a3:04:cd:64:69:71:4a:37:ac:d6:
+	fa:0a:68:c2:5b:19:55:54:5b:25:13:9d:b2:05:6f:
+	75:a4:12:15:c3:10:8e:0b:4a:c2:76:02:2d:10:ec:
+	f0:17:94:ce:e2:85:c1:5e:d8:8c:19:25:33:37:9d:
+	32:bc:4f:cb:2b:12:f2:8a:1d:
+
+exp2:
+	3e:53:68:c9:1c:f8:a5:6d:92:e8:60:e5:c0:ca:42:
+	40:43:78:c9:7e:36:13:f4:77:7d:f1:07:e1:4c:6c:
+	40:d9:7b:09:fc:7b:c8:47:7c:71:d0:26:36:3b:d2:
+	bd:c7:76:74:76:2f:2a:3a:83:97:11:f3:e1:7e:fb:
+	43:ff:29:b3:d1:c3:19:39:dc:59:23:4e:60:9e:fe:
+	ea:d0:28:19:90:97:d6:8e:56:a5:31:2f:66:40:8d:
+	f9:20:77:20:35:a6:c1:d6:72:d2:df:65:b2:5f:e6:
+	4f:49:5c:2a:91:9f:1e:60:78:c4:53:47:d7:dd:b4:
+	ab:87:c9:8c:d6:98:d1:55:
+
+
+Public Key ID: F3:3E:4E:38:84:4A:A9:BE:C1:6F:C7:61:A3:50:DA:6E:29:8B:EB:54
+Public key's random art:
++--[ RSA 2048]----+
+|                 |
+|                 |
+|                 |
+|    .. .         |
+|   Eo . S        |
+| .+o..+. +       |
+| .+o.= oo o      |
+|.o.o* o  +.      |
+|+o+*..   .o.     |
++-----------------+
+
+-----BEGIN RSA PRIVATE KEY-----
+MIIEpAIBAAKCAQEAv2IR1/gsaJsn+egtVFbMMMbivK+eVzPY8wEXb2flpe9aKkwz
+824nSSrWfRigQmf/ODkNlK2x91kppfPmWkClUREQB3I4d/sHFnzqCkFKmNwtVT4D
+sL47VumgZF6ZjSqTYQD3MDH3VhFj1iKrRMM/aCZXwntD+67sKw6UPXfFbfUORecp
+b7fCZPZFVFYStxgcF7cyH+DADLX5QCEjiLYH1es6FXrdw+ypgwBl+HuLkQ/4gE0J
+LQK5PAKSYeLy0JEBUPovkMXT+r9aPkQBF/9WD6uUPia0ERHbMIT0My84hBrG9d5u
+5gXPnH1ZxKIr8iJOYCydVjYndn21zs2IExwL6wIDAQABAoIBAEUMf/2Yp4USPakX
+kIs2SbNrflCvWASES0jZYvgp1xw4MCLEnZW9b2UhlINLyD5NQTKquvCifmwMek9K
+oRh87GhELLFTD3aSVitR5CrRBbYC8kQn/LLe34/q+Jhd3S6mZsf/zi9QR7mAyrFu
+jrZfb1gHRXCAgrWilcivGOLYfJ2/xanaT68IN5InlBLAlHCQ/+QFi+0YqRk8Rzp8
+/k+cFav2fkgqWNcUZ5a95vqfOlEMY0kU1Z3pqCQZKoPk/uLs2/kTM6bTYtJrfqlb
+k3P1ydCtWBHLd9MTPL83+WSVx0xp8m64NmlXk0oDBliKUT3Wl2EvfHYzFIhRRWhO
+Kf4SQ2kCgYEA4OaBOBg+yJhRcS1fIoyTlTcXRwBPaoeYc43zwwL34Z2gXKUQpg2I
+XeByEJMkr26kDlVcAzdfHZBBwtbjprogCAsBMev8fpdmPP61q0wLLxgW8yhHcEHc
+zwScfih4Oz8xz7F3LG3Jv60Z/wMfxpiaYEelHcRSxZ53Wsyk45aB1E0CgYEA2dkM
+boG7Dl3GksxIcLjaYOhW5yog2ikPyfCfuJ/f2aFofs4+fPIAZmh5xAH6uXE+cwY/
+hVyDM+5Yd1CJqpAz0GyqbzSyMIvpqYLf4n8ECZ8UmtvHy+WFRrJC0Kf+euP/HoSc
+NlDj3vsRHDQJ/kbbRcNQGfElwONc1Q2IE+GaXRcCgYEAmfKPT5OhHnTNgvh439B0
+kbalU2/N7PEmlSr9Smc0wRbCF8jR7ajjyMcDrX7bpM7KtBkQJA96J2WA7ltkd9N+
+a6MEzWRpcUo3rNb6CmjCWxlVVFslE52yBW91pBIVwxCOC0rCdgItEOzwF5TO4oXB
+XtiMGSUzN50yvE/LKxLyih0CgYA+U2jJHPilbZLoYOXAykJAQ3jJfjYT9Hd98Qfh
+TGxA2XsJ/HvIR3xx0CY2O9K9x3Z0di8qOoOXEfPhfvtD/ymz0cMZOdxZI05gnv7q
+0CgZkJfWjlalMS9mQI35IHcgNabB1nLS32WyX+ZPSVwqkZ8eYHjEU0fX3bSrh8mM
+1pjRVQKBgQDKect5h5GfmpkLXcV4IadgxootpbWHotbfsBdfv+HO8MqJGA7gSn8A
+5UEtBFsFUeUIid2AgseUlBz0Dxua0HKDu+nK1QkNS8C3aqe0w99O8X8PV60l/+TT
+7wWVMcoAVJdLLVaqGonYoNbcZIiINiaSOVeL2hgjd8PjOQ6V9zx3/g==
+-----END RSA PRIVATE KEY-----

testenv/certs/server-cert.pem

@@ -0,0 +1,21 @@
+-----BEGIN CERTIFICATE-----
+MIIDgDCCAmigAwIBAgIIVGI73zrIeeMwDQYJKoZIhvcNAQELBQAwMDERMA8GA1UE
+AxMIR05VIFdnZXQxDTALBgNVBAsTBFdnZXQxDDAKBgNVBAoTA0dOVTAiGA8yMDE0
+MTExMTE2NDAwMFoYDzk5OTkxMjMxMjM1OTU5WjAxMRIwEAYDVQQDEwkxMjcuMC4w
+LjExDTALBgNVBAsTBFdnZXQxDDAKBgNVBAoTA0dOVTCCASIwDQYJKoZIhvcNAQEB
+BQADggEPADCCAQoCggEBAMjC3Gt55EfStl6mE371+pD3/cpR5MLxkdbBss5MlIP2
+TDhiPOItLXml8oxs4BjUm3wfn3GV9iJLmbzbIWL+0kbRkQ2LCPKUf+Cln3z2ZE+r
+XwdWlT8gVfv51Opfkp2lLDVUqLfNKRGQgivjSCmLqY2LqeB0SaVNvuaD3EpqZyIH
+0E5SZgjqBHgRRtvGkcy0rOmp5SI2NASLugUioXa9OLWjpYDwodsd3ERlL0DJ1aJW
+8TC8Tqix4i0osWzar+LXBIin0Qvar9/uRHN0p1kq3p0XgNHKqWpiTT54+WYx7Pem
+v4qRXz11swiJzUL+Pw1DurQ9smbzDgAsz7V2FJnUeCcCAwEAAaOBmDCBlTAMBgNV
+HRMBAf8EAjAAMB8GA1UdEQQYMBaCCTEyNy4wLjAuMYIJbG9jYWxob3N0MBMGA1Ud
+JQQMMAoGCCsGAQUFBwMBMA8GA1UdDwEB/wQFAwMHoAAwHQYDVR0OBBYEFJfm323L
+JbKTM/tMKSt0qlUqewbnMB8GA1UdIwQYMBaAFPM+TjiESqm+wW/HYaNQ2m4pi+tU
+MA0GCSqGSIb3DQEBCwUAA4IBAQCDmuSD4IGmn0UQ5jhGQquh92Iu59j64Rrg7EIM
+zoppciyYR8gDUutOI9CEisxJz6umvAdOo5L981gcFaBv6hHWaE/krAZccR+ZXZP6
+fI9btO8My8O63fYcd2KkLEFqvKDF43i01S2LrqXdPo3uELbFZwxCmUKsexFGsoW1
+CbXbRjnS7w/f72myRmvBeDiNMuGfe1lb4IflybH3DMlKC7i0AN1JKglp+IKn5XAE
+neWR03i3UaYJsibIxj0DkTS+hVPu5MXQ9RlF5CkRdFKjGinLE/u70XyAyx0/IeAN
+e7c2MJvpdfRmTXm2ew4sNyK9RXo7Bv0Yqkl65iMscF8LNnxL
+-----END CERTIFICATE-----

testenv/certs/server-crl.pem

@@ -0,0 +1,12 @@
+-----BEGIN X509 CRL-----
+MIIB1DCBvQIBATANBgkqhkiG9w0BAQsFADAwMREwDwYDVQQDEwhHTlUgV2dldDEN
+MAsGA1UECxMEV2dldDEMMAoGA1UEChMDR05VGA8yMDE0MTExMTE2NDU1NFoYDzk5
+OTkxMjMxMjM1OTU5WjAdMBsCCFRiO986yHnjGA8yMDE0MTExMTE2NDU1NFqgNjA0
+MB8GA1UdIwQYMBaAFPM+TjiESqm+wW/HYaNQ2m4pi+tUMBEGA1UdFAQKAghUYj1E
+KHs9ijANBgkqhkiG9w0BAQsFAAOCAQEAZgwqs1VOFG39dFHHMXvBr4eJfhwiG4bC
+cL6IvLhvl9ikcyQMHrpOBtNjkCtgclSbJjjTDdera1+zuCWE0WBOJ4mojYdAIOhR
+QvSwp4NwAtibu2F/fjeXoo+LEpcRKtLvAotB30eCZ1OPrijsa/HxFILOLlayjns8
+wM4RmQC4o43y1G/1jqM8hGDg4Wz0j1URVuyP+pU55JpubV5LlExy3gIRwevD2lam
+q3hiighenJYFO3HGZkYT2SIoSpXZnQqKPJ4HwRBSg/cjOpc1y1lIIvKhmk+Cut6M
++S5HL4pIk8vGYg57nTfOOkj1goqFkfU0DBqvVAZj02ay/VIDu61T1g==
+-----END X509 CRL-----

testenv/certs/server-key.pem

@@ -0,0 +1,144 @@
+Public Key Info:
+	Public Key Algorithm: RSA
+	Key Security Level: Medium (2048 bits)
+
+modulus:
+	00:c8:c2:dc:6b:79:e4:47:d2:b6:5e:a6:13:7e:f5:
+	fa:90:f7:fd:ca:51:e4:c2:f1:91:d6:c1:b2:ce:4c:
+	94:83:f6:4c:38:62:3c:e2:2d:2d:79:a5:f2:8c:6c:
+	e0:18:d4:9b:7c:1f:9f:71:95:f6:22:4b:99:bc:db:
+	21:62:fe:d2:46:d1:91:0d:8b:08:f2:94:7f:e0:a5:
+	9f:7c:f6:64:4f:ab:5f:07:56:95:3f:20:55:fb:f9:
+	d4:ea:5f:92:9d:a5:2c:35:54:a8:b7:cd:29:11:90:
+	82:2b:e3:48:29:8b:a9:8d:8b:a9:e0:74:49:a5:4d:
+	be:e6:83:dc:4a:6a:67:22:07:d0:4e:52:66:08:ea:
+	04:78:11:46:db:c6:91:cc:b4:ac:e9:a9:e5:22:36:
+	34:04:8b:ba:05:22:a1:76:bd:38:b5:a3:a5:80:f0:
+	a1:db:1d:dc:44:65:2f:40:c9:d5:a2:56:f1:30:bc:
+	4e:a8:b1:e2:2d:28:b1:6c:da:af:e2:d7:04:88:a7:
+	d1:0b:da:af:df:ee:44:73:74:a7:59:2a:de:9d:17:
+	80:d1:ca:a9:6a:62:4d:3e:78:f9:66:31:ec:f7:a6:
+	bf:8a:91:5f:3d:75:b3:08:89:cd:42:fe:3f:0d:43:
+	ba:b4:3d:b2:66:f3:0e:00:2c:cf:b5:76:14:99:d4:
+	78:27:
+
+public exponent:
+	01:00:01:
+
+private exponent:
+	00:92:80:1f:f9:0d:e9:d7:bf:9b:f5:55:9b:c4:7a:
+	1b:6e:ce:89:14:aa:ce:14:b3:d3:88:b3:b0:97:7a:
+	aa:a5:e1:85:9d:5f:92:ae:39:e9:85:6b:e3:a3:35:
+	90:12:8e:93:27:f0:ab:99:67:a5:45:41:85:de:9a:
+	c9:b2:43:e1:8e:6c:3f:3d:72:c8:04:bc:f8:d4:26:
+	08:4c:58:40:bb:22:83:26:07:b8:c1:68:07:56:e8:
+	e8:c6:5f:17:ce:92:49:c0:61:16:fd:89:68:fe:b8:
+	45:45:61:85:b7:4b:83:5f:17:1b:cf:ff:0b:fe:e4:
+	cc:f9:ca:1f:66:ee:5e:74:25:94:7a:27:0e:0f:43:
+	50:14:48:ad:c6:8a:e1:ac:ff:8e:10:ed:e6:92:48:
+	c8:94:c1:3a:2c:db:86:71:66:8e:19:93:13:ed:f9:
+	47:06:5e:8b:e2:2e:cb:3a:c2:b3:5e:8d:31:e4:c5:
+	a7:cd:3f:09:70:e4:02:5d:34:2a:4d:b7:f5:06:e2:
+	f5:3b:8f:b6:ad:4a:22:b8:fe:43:a7:4d:67:ef:c3:
+	e1:ed:83:e2:d5:f2:d0:37:0f:56:ab:5b:47:69:0a:
+	14:03:2c:43:a3:73:e9:05:72:5e:df:68:9c:67:4b:
+	08:64:2d:c2:67:23:aa:e5:35:88:56:99:95:17:60:
+	20:01:
+
+prime1:
+	00:ea:ca:12:86:c0:25:b8:ab:fd:44:2c:1a:3f:1b:
+	19:68:d4:26:6e:9c:ad:6d:35:12:29:9f:40:c2:4c:
+	96:ef:8b:08:61:39:08:b7:8a:1f:81:97:71:ff:af:
+	5a:5b:db:9a:2f:2f:29:ab:92:bb:c5:51:a2:84:c5:
+	f4:88:79:ac:a2:b8:17:1e:4a:66:62:be:e5:ab:fd:
+	01:42:6b:16:f9:73:7b:cd:3e:f7:5c:5c:95:dd:79:
+	73:c4:60:a8:cf:95:80:ba:7d:02:14:9c:7e:58:4f:
+	8c:08:2c:b8:46:31:23:b2:1a:c3:38:78:5c:ea:50:
+	9d:42:23:31:30:9a:0f:3f:27:
+
+prime2:
+	00:da:e5:d3:66:0f:34:53:8c:e8:bf:5f:1e:46:93:
+	47:df:30:57:be:1f:30:6a:7e:e9:f0:6b:3f:61:89:
+	51:e2:0b:da:51:09:65:f6:23:3a:61:86:02:46:0a:
+	cf:11:73:7c:2d:65:bd:64:b8:0e:24:d2:b7:51:8f:
+	39:b4:a2:1b:e4:9a:bc:66:31:e2:00:eb:3e:20:06:
+	97:0a:a0:bb:82:da:bf:d5:e9:20:77:a7:55:86:69:
+	ce:eb:38:d3:f4:ad:82:9e:ce:02:05:c5:11:aa:c0:
+	b9:66:6f:e7:f4:26:57:72:fa:50:0b:ad:76:44:86:
+	e0:3e:f7:c0:3e:f3:94:9f:01:
+
+coefficient:
+	00:94:f2:42:a9:1a:62:1c:7a:bf:34:1b:a7:87:ae:
+	bd:3a:d9:f1:8c:4e:f6:f5:27:5a:ae:f1:1e:15:06:
+	a6:d0:e4:e0:ec:3a:40:02:13:b9:31:9a:cd:3a:c6:
+	34:7d:c6:9d:9e:60:5b:ca:03:88:87:56:f0:e1:ea:
+	37:96:2b:53:40:b2:78:4e:80:e2:e0:24:8c:83:0e:
+	f8:77:a4:64:d5:cc:09:6c:d6:52:49:f9:55:61:16:
+	72:b5:d2:ea:e1:61:fb:31:24:f0:30:8c:fe:5c:29:
+	71:06:09:11:4d:ef:51:a6:33:62:54:d2:c7:de:ba:
+	78:17:b1:27:50:f4:ef:c4:3a:
+
+exp1:
+	1f:36:0d:90:6c:2a:97:8a:05:78:f2:83:ea:af:a7:
+	89:0f:ea:ab:f9:97:f4:54:81:bd:96:b5:fd:1e:41:
+	52:46:a1:2e:8b:6e:65:37:af:48:82:e1:5c:a3:ea:
+	d7:1b:32:3b:e3:81:1e:95:ba:f0:58:11:ca:a4:a6:
+	05:1e:67:9c:99:ec:38:d2:9b:19:b5:56:c2:ae:37:
+	64:a4:e7:c0:f1:61:1b:bf:ab:12:54:1c:77:fc:95:
+	2f:1d:ca:53:0e:04:b6:c5:b7:69:16:04:95:a8:bd:
+	6c:b8:c5:26:4f:91:f7:33:27:90:72:2f:a7:d6:5f:
+	91:53:2c:4e:d1:ac:05:31:
+
+exp2:
+	00:83:a4:55:a6:fa:1b:d8:e7:54:0d:ca:f1:55:36:
+	3b:b1:f0:cb:c3:cd:d3:fb:27:ca:1e:c9:10:bb:e2:
+	ae:78:c7:f2:0a:6c:21:82:8e:1b:0d:0d:5f:8e:a9:
+	ef:6f:aa:49:12:b0:2d:df:45:85:54:05:d9:33:56:
+	74:38:ba:89:15:c9:2c:e6:34:b7:9b:1f:de:23:ba:
+	72:d9:74:62:70:46:87:b9:e8:52:9b:42:e9:ff:44:
+	e0:a8:bb:6b:54:a9:88:75:62:a4:fa:bd:52:6b:a3:
+	2d:9c:7a:4e:3f:99:53:5c:15:47:50:4e:88:62:9b:
+	ce:7e:6f:d6:90:c5:42:2b:01:
+
+
+Public Key ID: 97:E6:DF:6D:CB:25:B2:93:33:FB:4C:29:2B:74:AA:55:2A:7B:06:E7
+Public key's random art:
++--[ RSA 2048]----+
+|                 |
+|                 |
+|                 |
+|           .     |
+|        S + .    |
+|        .+oo.  . |
+|        .=+oo.+ .|
+|         +E.=O.oo|
+|        o+ .=*++o|
++-----------------+
+
+-----BEGIN RSA PRIVATE KEY-----
+MIIEpQIBAAKCAQEAyMLca3nkR9K2XqYTfvX6kPf9ylHkwvGR1sGyzkyUg/ZMOGI8
+4i0teaXyjGzgGNSbfB+fcZX2IkuZvNshYv7SRtGRDYsI8pR/4KWffPZkT6tfB1aV
+PyBV+/nU6l+SnaUsNVSot80pEZCCK+NIKYupjYup4HRJpU2+5oPcSmpnIgfQTlJm
+COoEeBFG28aRzLSs6anlIjY0BIu6BSKhdr04taOlgPCh2x3cRGUvQMnVolbxMLxO
+qLHiLSixbNqv4tcEiKfRC9qv3+5Ec3SnWSrenReA0cqpamJNPnj5ZjHs96a/ipFf
+PXWzCInNQv4/DUO6tD2yZvMOACzPtXYUmdR4JwIDAQABAoIBAQCSgB/5DenXv5v1
+VZvEehtuzokUqs4Us9OIs7CXeqql4YWdX5KuOemFa+OjNZASjpMn8KuZZ6VFQYXe
+msmyQ+GObD89csgEvPjUJghMWEC7IoMmB7jBaAdW6OjGXxfOkknAYRb9iWj+uEVF
+YYW3S4NfFxvP/wv+5Mz5yh9m7l50JZR6Jw4PQ1AUSK3GiuGs/44Q7eaSSMiUwTos
+24ZxZo4ZkxPt+UcGXoviLss6wrNejTHkxafNPwlw5AJdNCpNt/UG4vU7j7atSiK4
+/kOnTWfvw+Htg+LV8tA3D1arW0dpChQDLEOjc+kFcl7faJxnSwhkLcJnI6rlNYhW
+mZUXYCABAoGBAOrKEobAJbir/UQsGj8bGWjUJm6crW01EimfQMJMlu+LCGE5CLeK
+H4GXcf+vWlvbmi8vKauSu8VRooTF9Ih5rKK4Fx5KZmK+5av9AUJrFvlze80+91xc
+ld15c8RgqM+VgLp9AhScflhPjAgsuEYxI7Iawzh4XOpQnUIjMTCaDz8nAoGBANrl
+02YPNFOM6L9fHkaTR98wV74fMGp+6fBrP2GJUeIL2lEJZfYjOmGGAkYKzxFzfC1l
+vWS4DiTSt1GPObSiG+SavGYx4gDrPiAGlwqgu4Lav9XpIHenVYZpzus40/Stgp7O
+AgXFEarAuWZv5/QmV3L6UAutdkSG4D73wD7zlJ8BAoGAHzYNkGwql4oFePKD6q+n
+iQ/qq/mX9FSBvZa1/R5BUkahLotuZTevSILhXKPq1xsyO+OBHpW68FgRyqSmBR5n
+nJnsONKbGbVWwq43ZKTnwPFhG7+rElQcd/yVLx3KUw4EtsW3aRYElai9bLjFJk+R
+9zMnkHIvp9ZfkVMsTtGsBTECgYEAg6RVpvob2OdUDcrxVTY7sfDLw83T+yfKHskQ
+u+KueMfyCmwhgo4bDQ1fjqnvb6pJErAt30WFVAXZM1Z0OLqJFcks5jS3mx/eI7py
+2XRicEaHuehSm0Lp/0TgqLtrVKmIdWKk+r1Sa6MtnHpOP5lTXBVHUE6IYpvOfm/W
+kMVCKwECgYEAlPJCqRpiHHq/NBunh669OtnxjE729SdarvEeFQam0OTg7DpAAhO5
+MZrNOsY0fcadnmBbygOIh1bw4eo3litTQLJ4ToDi4CSMgw74d6Rk1cwJbNZSSflV
+YRZytdLq4WH7MSTwMIz+XClxBgkRTe9RpjNiVNLH3rp4F7EnUPTvxDo=
+-----END RSA PRIVATE KEY-----

testenv/certs/wget-cert.pem

@@ -1,30 +0,0 @@
------BEGIN PRIVATE KEY-----
-MIICdQIBADANBgkqhkiG9w0BAQEFAASCAl8wggJbAgEAAoGBAMV8qEpuSVUdWaAY
-F2N1ljGEJ/907Og5B0aZLeDskmLAOohKMWTiiSx+lseXVD/Zf/LaFfy/+q0Rk5+o
-pFEPEEjadvdxogb9HPwjfj48ng74yV1c5ZGRx/aIeIJN9cacfs4J5NlT3ZPiV8/2
-mpBurBYvta5tneUl+lx4NHTEBmjTAgMBAAECgYBHlFlDMRovWYYEuvavPA2GQQpm
-UzETMqhqdFbmsZiVZmtQvuOMV3e0wuVPzo/g3Kq9kUJq7AKl/DrvoaZ9IuKZgkDD
-0QEBYo/lcxEA9qcfgVs5XLp9ED1mXzJSZ3bmpCDqa2NjG7yFdWzPxc1DXmT05MrF
-bZbb0Wao0tvMwoeJYQJBAOql5uOyjDHvLLuS0IFKbYz4LQwAp7Gjs0ZS9qLNhQQn
-m5Vr8xS9QwFID693K6aDl3tqSCIwSnyInacj8M8v18sCQQDXdReE2i4LKOVLcQsP
-XabN96fFLlnoIh9MqFza4skjhXJWqjBLgJuFqyT5CTbU9TmaoIPXdo4454P1CCgR
-KEIZAkAZE7nlQ8Ov4nvJYBtgde/XTP6jdb52QaR7M4qgQ46frwv1oB/Oa5upm2Xx
-vq6vkQiza9xhqv+K557RqgmmWtqZAkASoXJmL4OZvXCOZHkDXCLHXqnoOAjYNNMm
-Csz0tHWWF7z6V38TmExac6Ef07clFQtlHoooAH1t2D8l2g205hlJAkBfeghbZDdY
-16NtVnvtzjjhKqZFqwTSANFV8NSzgb/QiNnX0hsMPt9bbc5VCo77Ly2oP5SvixfZ
-kjrIQqDV8MLu
------END PRIVATE KEY-----
------BEGIN CERTIFICATE-----
-MIICODCCAaGgAwIBAgIJAOiSkPuPcAwqMA0GCSqGSIb3DQEBBQUAMDUxCzAJBgNV
-BAYTAklOMRMwEQYDVQQIDApTb21lLVN0YXRlMREwDwYDVQQKDAhHTlUgV2dldDAe
-Fw0xMzEyMDcwNTA3NTRaFw0xNDEyMDcwNTA3NTRaMDUxCzAJBgNVBAYTAklOMRMw
-EQYDVQQIDApTb21lLVN0YXRlMREwDwYDVQQKDAhHTlUgV2dldDCBnzANBgkqhkiG
-9w0BAQEFAAOBjQAwgYkCgYEAxXyoSm5JVR1ZoBgXY3WWMYQn/3Ts6DkHRpkt4OyS
-YsA6iEoxZOKJLH6Wx5dUP9l/8toV/L/6rRGTn6ikUQ8QSNp293GiBv0c/CN+Pjye
-DvjJXVzlkZHH9oh4gk31xpx+zgnk2VPdk+JXz/aakG6sFi+1rm2d5SX6XHg0dMQG
-aNMCAwEAAaNQME4wHQYDVR0OBBYEFLhtTG9a6v3ihL5DeWKfq6doYI42MB8GA1Ud
-IwQYMBaAFLhtTG9a6v3ihL5DeWKfq6doYI42MAwGA1UdEwQFMAMBAf8wDQYJKoZI
-hvcNAQEFBQADgYEApTEZX3cgmgdXDJsu7wtkejtq3vuyi6NXBUlHzoYzWaS5wn8P
-uDG4G9zd1cwmwrbYA8lS+ANWvkcqjM68gMs1ARMZRS0IrYMCN8bokQw+16sqImZO
-THX50Sb5U+9e1IotDWyRBNO10znsoh569BxhJ5WZdIaoKHOJdXEYV+3Y/hg=
------END CERTIFICATE-----

testenv/server/http/http_server.py

@@ -38,13 +38,15 @@ class HTTPSServer (StoppableHTTPServer):
         import ssl
         BaseServer.__init__ (self, address, handler)
         # step one up because test suite change directory away from $srcdir (don't do that !!!)
-        CERTFILE = os.path.abspath(os.path.join('..', os.getenv('srcdir', '.'), 'certs', 'wget-cert.pem'))
+        CERTFILE = os.path.abspath(os.path.join('..', os.getenv('srcdir', '.'), 'certs', 'server-cert.pem'))
+        KEYFILE = os.path.abspath(os.path.join('..', os.getenv('srcdir', '.'), 'certs', 'server-key.pem'))
         fop = open (CERTFILE)
         print (fop.readline())
         self.socket = ssl.wrap_socket (
             sock = socket.socket (self.address_family, self.socket_type),
             ssl_version = ssl.PROTOCOL_TLSv1,
             certfile = CERTFILE,
+            keyfile = KEYFILE,
             server_side = True
         )
         self.server_bind()
@@ -203,34 +205,37 @@ class _Handler (BaseHTTPRequestHandler):
         return string.decode ('utf-8')
 
     def send_challenge (self, auth_type):
-        if auth_type == "Both":
-            self.send_challenge ("Digest")
-            self.send_challenge ("Basic")
+        auth_type = auth_type.lower()
+        if auth_type == "both":
+            self.send_challenge ("basic")
+            self.send_challenge ("digest")
             return
-        if auth_type == "Basic":
-            challenge_str = 'Basic realm="Wget-Test"'
-        elif auth_type == "Digest" or auth_type == "Both_inline":
+        if auth_type == "basic":
+            challenge_str = 'BasIc realm="Wget-Test"'
+        elif auth_type == "digest" or auth_type == "both_inline":
             self.nonce = md5 (str (random ()).encode ('utf-8')).hexdigest()
             self.opaque = md5 (str (random ()).encode ('utf-8')).hexdigest()
-            challenge_str = 'Digest realm="Test", nonce="%s", opaque="%s"' % (
+            # 'DIgest' to provoke a Wget failure with turkish locales
+            challenge_str = 'DIgest realm="Test", nonce="%s", opaque="%s"' % (
                             self.nonce,
                             self.opaque)
             challenge_str += ', qop="auth"'
-            if auth_type == "Both_inline":
-                challenge_str = 'Basic realm="Wget-Test", ' + challenge_str
+            if auth_type == "both_inline":
+                # 'BasIc' to provoke a Wget failure with turkish locales
+                challenge_str = 'BasIc realm="Wget-Test", ' + challenge_str
         self.send_header ("WWW-Authenticate", challenge_str)
 
-    def authorize_Basic (self, auth_header, auth_rule):
-        if auth_header is None or auth_header.split(' ')[0] != 'Basic':
+    def authorize_basic (self, auth_header, auth_rule):
+        if auth_header is None or auth_header.split(' ')[0].lower() != 'basic':
             return False
         else:
             self.user = auth_rule.auth_user
             self.passw = auth_rule.auth_pass
-            auth_str = "Basic " + self.base64 (self.user + ":" + self.passw)
-            return True if auth_str == auth_header else False
+            auth_str = "basic " + self.base64 (self.user + ":" + self.passw)
+            return True if auth_str.lower() == auth_header.lower() else False
 
     def parse_auth_header (self, auth_header):
-        n = len("Digest ")
+        n = len("digest ")
         auth_header = auth_header[n:].strip()
         items = auth_header.split(", ")
         keyvals = [i.split("=", 1) for i in items]
@@ -262,8 +267,8 @@ class _Handler (BaseHTTPRequestHandler):
 
         return True if resp == params['response'] else False
 
-    def authorize_Digest (self, auth_header, auth_rule):
-        if auth_header is None or auth_header.split(' ')[0] != 'Digest':
+    def authorize_digest (self, auth_header, auth_rule):
+        if auth_header is None or auth_header.split(' ')[0].lower() != 'digest':
             return False
         else:
             self.user = auth_rule.auth_user
@@ -282,10 +287,10 @@ class _Handler (BaseHTTPRequestHandler):
                 pass_auth = False
             return pass_auth
 
-    def authorize_Both (self, auth_header, auth_rule):
+    def authorize_both (self, auth_header, auth_rule):
         return False
 
-    def authorize_Both_inline (self, auth_header, auth_rule):
+    def authorize_both_inline (self, auth_header, auth_rule):
         return False
 
     def Authentication (self, auth_rule):
@@ -300,16 +305,16 @@ class _Handler (BaseHTTPRequestHandler):
     def handle_auth (self, auth_rule):
         is_auth = True
         auth_header = self.headers.get ("Authorization")
-        required_auth = auth_rule.auth_type
-        if required_auth == "Both" or required_auth == "Both_inline":
-            auth_type = auth_header.split(' ')[0] if auth_header else required_auth
+        required_auth = auth_rule.auth_type.lower()
+        if required_auth == "both" or required_auth == "both_inline":
+            auth_type = auth_header.split(' ')[0].lower() if auth_header else required_auth
         else:
             auth_type = required_auth
         try:
             assert hasattr (self, "authorize_" + auth_type)
             is_auth = getattr (self, "authorize_" + auth_type) (auth_header, auth_rule)
         except AssertionError:
-            raise ServerError ("Authentication Mechanism " + auth_rule + " not supported")
+            raise ServerError ("Authentication Mechanism " + auth_type + " not supported")
         except AttributeError as ae:
             raise ServerError (ae.__str__())
         if is_auth is False:
@@ -331,7 +336,7 @@ class _Handler (BaseHTTPRequestHandler):
         for header_line in rej_headers:
             header_recd = self.headers.get (header_line)
             if header_recd is not None and header_recd == rej_headers[header_line]:
-                self.send_error (400, 'Blackisted Header ' + header_line + ' received')
+                self.send_error (400, 'Blacklisted Header ' + header_line + ' received')
                 self.finish_headers ()
                 raise ServerError ("Header " + header_line + ' received')
 

testenv/test/base_test.py

@@ -100,11 +100,14 @@ class BaseTest:
                                                  "..", '..', 'src', "wget"))
         wget_options = '--debug --no-config %s' % self.wget_options
 
-        if os.getenv("VALGRIND_TESTS"):
-            valgrind_test = "valgrind --error-exitcode=301 --leak-check=full --track-origins=yes"
+        valgrind = os.getenv("VALGRIND_TESTS", "")
+        if valgrind in ("", "0"):
+            cmd_line = '%s %s ' % (wget_path, wget_options)
+        elif valgrind == "1":
+            cmd_line = 'valgrind --error-exitcode=301 --leak-check=yes --track-origins=yes %s %s ' % (wget_path, wget_options)
         else:
-            valgrind_test = ""
-        cmd_line = '%s %s %s ' % (valgrind_test, wget_path, wget_options)
+            cmd_line = '%s %s %s ' % (os.getenv("VALGRIND_TESTS", ""), wget_path, wget_options)
+
         for protocol, urls, domain in zip(self.protocols,
                                           self.urls,
                                           self.domains):

tests/FTPTest.pm

@@ -9,9 +9,8 @@ use WgetTests;
 our @ISA = qw(WgetTest);
 my $VERSION = 0.01;
 
-
 {
-    my %_attr_data = ( # DEFAULT
+    my %_attr_data = (    # DEFAULT
     );
 
     sub _default_for
@@ -28,29 +27,32 @@ my $VERSION = 0.01;
     }
 }
 
-
-sub _setup_server {
+sub _setup_server
+{
     my $self = shift;
 
-    $self->{_server} = FTPServer->new (input => $self->{_input},
-                                       server_behavior =>
-                                           $self->{_server_behavior},
-                                       LocalAddr => 'localhost',
-                                       ReuseAddr => 1,
-                                       rootDir => "$self->{_workdir}/$self->{_name}/input") or die "Cannot create server!!!";
+    $self->{_server} = FTPServer->new(
+                             input           => $self->{_input},
+                             server_behavior => $self->{_server_behavior},
+                             LocalAddr       => 'localhost',
+                             ReuseAddr       => 1,
+                             rootDir => "$self->{_workdir}/$self->{_name}/input"
+      )
+      or die "Cannot create server!!!";
 }
 
-
-sub _launch_server {
-    my $self = shift;
+sub _launch_server
+{
+    my $self       = shift;
     my $synch_func = shift;
 
-    $self->{_server}->run ($synch_func);
+    $self->{_server}->run($synch_func);
 }
 
-sub _substitute_port {
+sub _substitute_port
+{
     my $self = shift;
-    my $ret = shift;
+    my $ret  = shift;
     $ret =~ s/{{port}}/$self->{_server}->sockport/eg;
     return $ret;
 }

tests/HTTPServer.pm

@@ -8,47 +8,58 @@ use HTTP::Status;
 use HTTP::Headers;
 use HTTP::Response;
 
-our @ISA=qw(HTTP::Daemon);
+our @ISA = qw(HTTP::Daemon);
 my $VERSION = 0.01;
 
-my $CRLF = "\015\012"; # "\r\n" is not portable
-my $log = undef;
+my $CRLF = "\015\012";    # "\r\n" is not portable
+my $log  = undef;
 
-sub run {
+sub run
+{
     my ($self, $urls, $synch_callback) = @_;
     my $initialized = 0;
 
-    while (1) {
-        if (!$initialized) {
+    while (1)
+    {
+        if (!$initialized)
+        {
             $synch_callback->();
             $initialized = 1;
         }
         my $con = $self->accept();
         print STDERR "Accepted a new connection\n" if $log;
-        while (my $req = $con->get_request) {
+        while (my $req = $con->get_request)
+        {
             #my $url_path = $req->url->path;
             my $url_path = $req->url->as_string;
-            if ($url_path =~ m{/$}) { # append 'index.html'
+            if ($url_path =~ m{/$})
+            {    # append 'index.html'
                 $url_path .= 'index.html';
             }
+
             #if ($url_path =~ m{^/}) { # remove trailing '/'
             #    $url_path = substr ($url_path, 1);
             #}
-            if ($log) {
+            if ($log)
+            {
                 print STDERR "Method: ", $req->method, "\n";
                 print STDERR "Path: ", $url_path, "\n";
                 print STDERR "Available URLs: ", "\n";
-                foreach my $key (keys %$urls) {
+                foreach my $key (keys %$urls)
+                {
                     print STDERR $key, "\n";
                 }
             }
-            if (exists($urls->{$url_path})) {
+            if (exists($urls->{$url_path}))
+            {
                 print STDERR "Serving requested URL: ", $url_path, "\n" if $log;
                 next unless ($req->method eq "HEAD" || $req->method eq "GET");
 
                 my $url_rec = $urls->{$url_path};
                 $self->send_response($req, $url_rec, $con);
-            } else {
+            }
+            else
+            {
                 print STDERR "Requested wrong URL: ", $url_path, "\n" if $log;
                 $con->send_error($HTTP::Status::RC_FORBIDDEN);
                 last;
@@ -59,73 +70,90 @@ sub run {
     }
 }
 
-sub send_response {
+sub send_response
+{
     my ($self, $req, $url_rec, $con) = @_;
 
     # create response
     my ($code, $msg, $headers);
     my $send_content = ($req->method eq "GET");
-    if (exists $url_rec->{'auth_method'}) {
+    if (exists $url_rec->{'auth_method'})
+    {
         ($send_content, $code, $msg, $headers) =
-            $self->handle_auth($req, $url_rec);
-    } elsif (!$self->verify_request_headers ($req, $url_rec)) {
+          $self->handle_auth($req, $url_rec);
+    }
+    elsif (!$self->verify_request_headers($req, $url_rec))
+    {
         ($send_content, $code, $msg, $headers) =
-            ('', 400, 'Mismatch on expected headers', {});
-    } else {
+          ('', 400, 'Mismatch on expected headers', {});
+    }
+    else
+    {
         ($code, $msg) = @{$url_rec}{'code', 'msg'};
         $headers = $url_rec->{headers};
     }
-    my $resp = HTTP::Response->new ($code, $msg);
+    my $resp = HTTP::Response->new($code, $msg);
     print STDERR "HTTP::Response: \n", $resp->as_string if $log;
 
-    while (my ($name, $value) = each %{$headers}) {
+    while (my ($name, $value) = each %{$headers})
+    {
         # print STDERR "setting header: $name = $value\n";
+        $value = $self->_substitute_port($value);
         $resp->header($name => $value);
     }
     print STDERR "HTTP::Response with headers: \n", $resp->as_string if $log;
 
-    if ($send_content) {
+    if ($send_content)
+    {
         my $content = $url_rec->{content};
-        if (exists($url_rec->{headers}{"Content-Length"})) {
+        if (exists($url_rec->{headers}{"Content-Length"}))
+        {
             # Content-Length and length($content) don't match
             # manually prepare the HTTP response
-            $con->send_basic_header($url_rec->{code}, $resp->message, $resp->protocol);
+            $con->send_basic_header($url_rec->{code}, $resp->message,
+                                    $resp->protocol);
             print $con $resp->headers_as_string($CRLF);
             print $con $CRLF;
             print $con $content;
             next;
         }
-        if ($req->header("Range") && !$url_rec->{'force_code'}) {
+        if ($req->header("Range") && !$url_rec->{'force_code'})
+        {
             $req->header("Range") =~ m/bytes=(\d*)-(\d*)/;
             my $content_len = length($content);
-            my $start = $1 ? $1 : 0;
-            my $end = $2 ? $2 : ($content_len - 1);
-            my $len = $2 ? ($2 - $start) : ($content_len - $start);
-            if ($len > 0) {
-                $resp->header("Accept-Ranges" => "bytes");
+            my $start       = $1 ? $1 : 0;
+            my $end         = $2 ? $2 : ($content_len - 1);
+            my $len         = $2 ? ($2 - $start) : ($content_len - $start);
+            if ($len > 0)
+            {
+                $resp->header("Accept-Ranges"  => "bytes");
                 $resp->header("Content-Length" => $len);
-                $resp->header("Content-Range"
-                    => "bytes $start-$end/$content_len");
+                $resp->header(
+                           "Content-Range" => "bytes $start-$end/$content_len");
                 $resp->header("Keep-Alive" => "timeout=15, max=100");
                 $resp->header("Connection" => "Keep-Alive");
                 $con->send_basic_header(206,
-                    "Partial Content", $resp->protocol);
+                                        "Partial Content", $resp->protocol);
                 print $con $resp->headers_as_string($CRLF);
                 print $con $CRLF;
                 print $con substr($content, $start, $len);
-            } else {
+            }
+            else
+            {
                 $con->send_basic_header(416, "Range Not Satisfiable",
-                    $resp->protocol);
+                                        $resp->protocol);
                 $resp->header("Keep-Alive" => "timeout=15, max=100");
                 $resp->header("Connection" => "Keep-Alive");
                 print $con $CRLF;
             }
             next;
         }
+
         # fill in content
         $content = $self->_substitute_port($content) if defined $content;
         $resp->content($content);
-        print STDERR "HTTP::Response with content: \n", $resp->as_string if $log;
+        print STDERR "HTTP::Response with content: \n", $resp->as_string
+          if $log;
     }
 
     $con->send_response($resp);
@@ -134,60 +162,81 @@ sub send_response {
 
 # Generates appropriate response content based on the authentication
 # status of the URL.
-sub handle_auth {
+sub handle_auth
+{
     my ($self, $req, $url_rec) = @_;
     my ($send_content, $code, $msg, $headers);
+
     # Catch failure to set code, msg:
     $code = 500;
     $msg  = "Didn't set response code in handle_auth";
+
     # Most cases, we don't want to send content.
     $send_content = 0;
+
     # Initialize headers
     $headers = {};
     my $authhdr = $req->header('Authorization');
 
     # Have we sent the challenge yet?
-    unless ($url_rec->{auth_challenged} || $url_rec->{auth_no_challenge}) {
+    unless ($url_rec->{auth_challenged} || $url_rec->{auth_no_challenge})
+    {
         # Since we haven't challenged yet, we'd better not
         # have received authentication (for our testing purposes).
-        if ($authhdr) {
+        if ($authhdr)
+        {
             $code = 400;
             $msg  = "You sent auth before I sent challenge";
-        } else {
+        }
+        else
+        {
             # Send challenge
             $code = 401;
             $msg  = "Authorization Required";
-            $headers->{'WWW-Authenticate'} = $url_rec->{'auth_method'}
-                . " realm=\"wget-test\"";
+            $headers->{'WWW-Authenticate'} =
+              $url_rec->{'auth_method'} . " realm=\"wget-test\"";
             $url_rec->{auth_challenged} = 1;
         }
-    } elsif (!defined($authhdr)) {
+    }
+    elsif (!defined($authhdr))
+    {
         # We've sent the challenge; we should have received valid
         # authentication with this one. A normal server would just
         # resend the challenge; but since this is a test, wget just
         # failed it.
         $code = 400;
         $msg  = "You didn't send auth after I sent challenge";
-        if ($url_rec->{auth_no_challenge}) {
-            $msg = "--auth-no-challenge but no auth sent."
+        if ($url_rec->{auth_no_challenge})
+        {
+            $msg = "--auth-no-challenge but no auth sent.";
         }
-    } else {
+    }
+    else
+    {
         my ($sent_method) = ($authhdr =~ /^(\S+)/g);
-        unless ($sent_method eq $url_rec->{'auth_method'}) {
+        unless ($sent_method eq $url_rec->{'auth_method'})
+        {
             # Not the authorization type we were expecting.
             $code = 400;
-            $msg = "Expected auth type $url_rec->{'auth_method'} but got "
-                . "$sent_method";
-        } elsif (($sent_method eq 'Digest'
-                  && &verify_auth_digest($authhdr, $url_rec, \$msg))
-                 ||
-                 ($sent_method eq 'Basic'
-                  && &verify_auth_basic($authhdr, $url_rec, \$msg))) {
+            $msg  = "Expected auth type $url_rec->{'auth_method'} but got "
+              . "$sent_method";
+        }
+        elsif (
+               (
+                   $sent_method eq 'Digest'
+                && &verify_auth_digest($authhdr, $url_rec, \$msg)
+               )
+               || (   $sent_method eq 'Basic'
+                   && &verify_auth_basic($authhdr, $url_rec, \$msg))
+              )
+        {
             # SUCCESSFUL AUTH: send expected message, headers, content.
             ($code, $msg) = @{$url_rec}{'code', 'msg'};
-            $headers = $url_rec->{headers};
+            $headers      = $url_rec->{headers};
             $send_content = 1;
-        } else {
+        }
+        else
+        {
             $code = 400;
         }
     }
@@ -195,43 +244,58 @@ sub handle_auth {
     return ($send_content, $code, $msg, $headers);
 }
 
-sub verify_auth_digest {
-    return undef; # Not yet implemented.
+sub verify_auth_digest
+{
+    return undef;    # Not yet implemented.
 }
 
-sub verify_auth_basic {
+sub verify_auth_basic
+{
     require MIME::Base64;
     my ($authhdr, $url_rec, $msgref) = @_;
-    my $expected = MIME::Base64::encode_base64($url_rec->{'user'} . ':'
-        . $url_rec->{'passwd'}, '');
+    my $expected =
+      MIME::Base64::encode_base64(
+                                $url_rec->{'user'} . ':' . $url_rec->{'passwd'},
+                                '');
     my ($got) = $authhdr =~ /^Basic (.*)$/;
-    if ($got eq $expected) {
+    if ($got eq $expected)
+    {
         return 1;
-    } else {
+    }
+    else
+    {
         $$msgref = "Wanted ${expected} got ${got}";
         return undef;
     }
 }
 
-sub verify_request_headers {
+sub verify_request_headers
+{
     my ($self, $req, $url_rec) = @_;
 
     return 1 unless exists $url_rec->{'request_headers'};
-    for my $hdrname (keys %{$url_rec->{'request_headers'}}) {
+    for my $hdrname (keys %{$url_rec->{'request_headers'}})
+    {
         my $must_not_match;
         my $ehdr = $url_rec->{'request_headers'}{$hdrname};
-        if ($must_not_match = ($hdrname =~ /^!(\w+)/)) {
+        if ($must_not_match = ($hdrname =~ /^!(\w+)/))
+        {
             $hdrname = $1;
         }
-        my $rhdr = $req->header ($hdrname);
-        if ($must_not_match) {
-            if (defined $rhdr && $rhdr =~ $ehdr) {
+        my $rhdr = $req->header($hdrname);
+        if ($must_not_match)
+        {
+            if (defined $rhdr && $rhdr =~ $ehdr)
+            {
                 $rhdr = '' unless defined $rhdr;
                 print STDERR "\n*** Match forbidden $hdrname: $rhdr =~ $ehdr\n";
                 return undef;
             }
-        } else {
-            unless (defined $rhdr && $rhdr =~ $ehdr) {
+        }
+        else
+        {
+            unless (defined $rhdr && $rhdr =~ $ehdr)
+            {
                 $rhdr = '' unless defined $rhdr;
                 print STDERR "\n*** Mismatch on $hdrname: $rhdr =~ $ehdr\n";
                 return undef;
@@ -242,9 +306,10 @@ sub verify_request_headers {
     return 1;
 }
 
-sub _substitute_port {
+sub _substitute_port
+{
     my $self = shift;
-    my $ret = shift;
+    my $ret  = shift;
     $ret =~ s/{{port}}/$self->sockport/eg;
     return $ret;
 }

tests/HTTPTest.pm

@@ -9,9 +9,8 @@ use WgetTests;
 our @ISA = qw(WgetTest);
 my $VERSION = 0.01;
 
-
 {
-    my %_attr_data = ( # DEFAULT
+    my %_attr_data = (    # DEFAULT
     );
 
     sub _default_for
@@ -28,25 +27,26 @@ my $VERSION = 0.01;
     }
 }
 
-
-sub _setup_server {
+sub _setup_server
+{
     my $self = shift;
-    $self->{_server} = HTTPServer->new (LocalAddr => 'localhost',
-                                        ReuseAddr => 1)
-                                    or die "Cannot create server!!!";
+    $self->{_server} = HTTPServer->new(LocalAddr => 'localhost',
+                                       ReuseAddr => 1)
+      or die "Cannot create server!!!";
 }
 
-
-sub _launch_server {
-    my $self = shift;
+sub _launch_server
+{
+    my $self       = shift;
     my $synch_func = shift;
 
-    $self->{_server}->run ($self->{_input}, $synch_func);
+    $self->{_server}->run($self->{_input}, $synch_func);
 }
 
-sub _substitute_port {
+sub _substitute_port
+{
     my $self = shift;
-    my $ret = shift;
+    my $ret  = shift;
     $ret =~ s/{{port}}/$self->{_server}->sockport/eg;
     return $ret;
 }

tests/Makefile.am

@@ -30,10 +30,9 @@
 # see http://www.gnu.org/software/automake/manual/html_node/Parallel-Test-Harness.html#Parallel-Test-Harness
 
 #
-# Version: $(VERSION)
+# Version: @VERSION@
 #
 
-
 ../src/wget$(EXEEXT):
 	cd ../src && $(MAKE) $(AM_MAKEFLAGS)
 
@@ -113,6 +112,7 @@ PX_TESTS = \
              Test-O.px \
              Test--post-file.px \
              Test-proxied-https-auth.px \
+             Test-proxied-https-auth-keepalive.px \
              Test-proxy-auth-basic.px \
              Test-restrict-ascii.px \
              Test-Restrict-Lowercase.px \
@@ -135,12 +135,13 @@ EXTRA_DIST = FTPServer.pm FTPTest.pm HTTPServer.pm HTTPTest.pm \
 
 check_PROGRAMS = unit-tests
 unit_tests_SOURCES =
-LDADD = ../src/libunittest.a ../lib/libgnu.a $(LIBS)
+LDADD = ../src/libunittest.a ../lib/libgnu.a @LIBICONV@ @LIBINTL@ $(LIBS) $(LIB_CLOCK_GETTIME)
 
 CLEANFILES = *~ *.bak core core.[0-9]*
 
 TESTS = ./unit-tests$(EXEEXT) $(PX_TESTS)
 TEST_EXTENSIONS = .px
-AM_TESTS_ENVIRONMENT = export WGETRC=/dev/null; export SYSTEM_WGETRC=/dev/null;
 PX_LOG_COMPILER = $(PERL)
 AM_PX_LOG_FLAGS = -I$(srcdir)
+AM_TESTS_ENVIRONMENT = export WGETRC=/dev/null; export SYSTEM_WGETRC=/dev/null;\
+ export VALGRIND_TESTS="@VALGRIND_TESTS@";

tests/Test-proxied-https-auth-keepalive.px

@@ -0,0 +1,163 @@
+#!/usr/bin/env perl
+# Simulate a tunneling proxy to a HTTPS URL that needs authentication.
+# Use a persistent connection (Connection: keep-alive)
+
+use strict;
+use warnings;
+
+use WgetFeature qw(https);
+use WgetTests;  # For $WGETPATH.
+
+my $cert_path;
+my $key_path;
+my $srcdir;
+
+if (@ARGV) {
+    $srcdir = shift @ARGV;
+} elsif (defined $ENV{srcdir}) {
+    $srcdir = $ENV{srcdir};
+}
+
+if (defined $srcdir) {
+    $key_path = "$srcdir/certs/server-key.pem";
+    $cert_path = "$srcdir/certs/server-cert.pem";
+} else {
+    $key_path = "certs/server-key.pem";
+    $cert_path = "certs/server-cert.pem";
+}
+
+
+use HTTP::Daemon;
+use HTTP::Request;
+use IO::Socket::SSL;
+
+my $SOCKET = HTTP::Daemon->new (LocalAddr => 'localhost',
+    ReuseAddr => 1) or die "Cannot create server!!!";
+
+sub get_request {
+    my $conn = shift;
+    my $content = '';
+    my $line;
+
+    while (defined ($line = <$conn>)) {
+        $content .= $line;
+        last if $line eq "\r\n";
+    }
+
+    my $rqst = HTTP::Request->parse($content)
+        or die "Couldn't parse request:\n$content\n";
+
+    return $rqst;
+}
+
+sub do_server {
+    my ($synch_callback) = @_;
+    my $s = $SOCKET;
+    my $conn;
+    my $rqst;
+    my $rspn;
+
+    # sync with the parent
+    $synch_callback->();
+
+    # Simulate a HTTPS proxy server with tunneling.
+
+    $conn = $s->accept;
+    $rqst = $conn->get_request;
+    die "Method not CONNECT\n" if ($rqst->method ne 'CONNECT');
+    $rspn = HTTP::Response->new(200, 'OK');
+    $conn->send_response($rspn);
+
+    # Now switch from plain to SSL (for simulating a transparent tunnel
+    # to an HTTPS server).
+
+    my %options = (
+        SSL_server => 1,
+        SSL_passwd_cb => sub { return "Hello"; });
+    $options{SSL_cert_file} = $cert_path if ($cert_path);
+    $options{SSL_key_file} = $key_path if ($key_path);
+    my @options = %options;
+    $conn = IO::Socket::SSL->new_from_fd($conn->fileno, @options)
+        or die "Couldn't initiate SSL";
+
+    for my $expect_inner_auth (0, 1) {
+        # TODO: expect no auth the first time, request it, expect it the second
+        #   time.
+
+        $rqst = &get_request($conn)
+            or die "Didn't get proxied request\n";
+
+        unless ($expect_inner_auth) {
+            die "Early proxied auth\n" if $rqst->header('Authorization');
+
+            $rspn = HTTP::Response->new(401, 'Unauthorized', [
+                'WWW-Authenticate' => 'Basic realm="gondor"',
+                Connection => 'keep-alive'
+                ]);
+        } else {
+            die "No proxied auth\n" unless $rqst->header('Authorization');
+
+            $rspn = HTTP::Response->new(200, 'OK', [
+                'Content-Type' => 'text/plain',
+                'Connection' => 'close',
+                ], "foobarbaz\n");
+        }
+
+        $rspn->protocol('HTTP/1.0');
+        print STDERR "=====\n";
+        print STDERR $rspn->as_string;
+        print STDERR "\n=====\n";
+        print $conn $rspn->as_string;
+    }
+
+    $conn->close;
+    undef $conn;
+    undef $s;
+}
+
+sub fork_server {
+    pipe(FROM_CHILD, TO_PARENT) or die "Cannot create pipe!";
+    select((select(TO_PARENT), $| = 1)[0]);
+
+    my $pid = fork();
+    if ($pid < 0) {
+        die "Cannot fork";
+    } elsif ($pid == 0) {
+        # child
+        close FROM_CHILD;
+        do_server(sub { print TO_PARENT "SYNC\n"; close TO_PARENT });
+        exit 0;
+    } else {
+        # parent
+        close TO_PARENT;
+        chomp(my $line = <FROM_CHILD>);
+        close FROM_CHILD;
+    }
+
+    return $pid;
+}
+
+unlink "needs-auth.txt";
+my $pid = &fork_server;
+
+my $cmdline = $WgetTest::WGETPATH . " --user=fiddle-dee-dee"
+    . " --password=Dodgson -e https_proxy=localhost:{{port}}"
+    . " --no-check-certificate"
+    . " https://no.such.domain/needs-auth.txt";
+$cmdline =~ s/{{port}}/$SOCKET->sockport()/e;
+
+my $valgrind = $ENV{VALGRIND_TESTS};
+if (!defined $valgrind || $valgrind eq "" || $valgrind == 0) {
+    # Valgrind not requested - leave $cmdline as it is
+} elsif ($valgrind == 1) {
+    $cmdline = "valgrind --error-exitcode=301 --leak-check=yes --track-origins=yes " . $cmdline;
+} else {
+    $cmdline = $valgrind . " " . $cmdline;
+}
+
+my $code = system($cmdline . " 2>&1") >> 8;
+unlink "needs-auth.txt";
+
+warn "Got code: $code\n" if $code;
+kill ('TERM', $pid);
+exit ($code != 0);

tests/Test-proxied-https-auth.px

@@ -1,4 +1,6 @@
 #!/usr/bin/env perl
+# Simulate a tunneling proxy to a HTTPS URL that needs authentication.
+# Use two connections (Connection: close)
 
 use strict;
 use warnings;
@@ -49,31 +51,33 @@ sub get_request {
 }
 
 sub do_server {
-    my $alrm = alarm 10;
-
+    my ($synch_callback) = @_;
     my $s = $SOCKET;
     my $conn;
     my $rqst;
     my $rspn;
+
+    my %options = (
+        SSL_server => 1,
+        SSL_passwd_cb => sub { return "Hello"; });
+    $options{SSL_cert_file} = $cert_path if ($cert_path);
+    $options{SSL_key_file} = $key_path if ($key_path);
+    my @options = %options;
+
+    # sync with the parent
+    $synch_callback->();
+
+    # Simulate a HTTPS proxy server with tunneling.
+
     for my $expect_inner_auth (0, 1) {
         $conn = $s->accept;
         $rqst = $conn->get_request;
-
-        # TODO: expect no auth the first time, request it, expect it the second
-        #   time.
-
         die "Method not CONNECT\n" if ($rqst->method ne 'CONNECT');
         $rspn = HTTP::Response->new(200, 'OK');
         $conn->send_response($rspn);
 
-        my %options = (
-            SSL_server => 1,
-            SSL_passwd_cb => sub { return "Hello"; });
-
-        $options{SSL_cert_file} = $cert_path if ($cert_path);
-        $options{SSL_key_file} = $key_path if ($key_path);
-
-        my @options = %options;
+        # Now switch from plain to SSL (for simulating a transparent tunnel
+        # to an HTTPS server).
 
         $conn = IO::Socket::SSL->new_from_fd($conn->fileno, @options)
             or die "Couldn't initiate SSL";
@@ -84,14 +88,10 @@ sub do_server {
         unless ($expect_inner_auth) {
             die "Early proxied auth\n" if $rqst->header('Authorization');
 
-            # TODO: handle non-persistent connection here.
             $rspn = HTTP::Response->new(401, 'Unauthorized', [
                 'WWW-Authenticate' => 'Basic realm="gondor"',
                 Connection => 'close'
                 ]);
-            $rspn->protocol('HTTP/1.0');
-            print $rspn->as_string;
-            print $conn $rspn->as_string;
         } else {
             die "No proxied auth\n" unless $rqst->header('Authorization');
 
@@ -99,41 +99,64 @@ sub do_server {
                 'Content-Type' => 'text/plain',
                 'Connection' => 'close',
                 ], "foobarbaz\n");
-            $rspn->protocol('HTTP/1.0');
-            print "=====\n";
-            print $rspn->as_string;
-            print "\n=====\n";
-            print $conn $rspn->as_string;
         }
+
+        $rspn->protocol('HTTP/1.0');
+        print STDERR "=====\n";
+        print STDERR $rspn->as_string;
+        print STDERR "\n=====\n";
+        print $conn $rspn->as_string;
+
         $conn->close;
     }
+
     undef $conn;
     undef $s;
-    alarm $alrm;
 }
 
 sub fork_server {
-    my $pid = fork;
-    die "Couldn't fork" if ($pid < 0);
-    return $pid if $pid;
+    pipe(FROM_CHILD, TO_PARENT) or die "Cannot create pipe!";
+    select((select(TO_PARENT), $| = 1)[0]);
 
-    &do_server;
-    exit;
+    my $pid = fork();
+    if ($pid < 0) {
+        die "Cannot fork";
+    } elsif ($pid == 0) {
+        # child
+        close FROM_CHILD;
+        do_server(sub { print TO_PARENT "SYNC\n"; close TO_PARENT });
+        exit 0;
+    } else {
+        # parent
+        close TO_PARENT;
+        chomp(my $line = <FROM_CHILD>);
+        close FROM_CHILD;
+    }
+
+    return $pid;
 }
 
-system ('rm -f needs-auth.txt');
+unlink "needs-auth.txt";
 my $pid = &fork_server;
 
-sleep 1;
 my $cmdline = $WgetTest::WGETPATH . " --user=fiddle-dee-dee"
     . " --password=Dodgson -e https_proxy=localhost:{{port}}"
     . " --no-check-certificate"
     . " https://no.such.domain/needs-auth.txt";
 $cmdline =~ s/{{port}}/$SOCKET->sockport()/e;
 
-my $code = system($cmdline);
-system ('rm -f needs-auth.txt');
+my $valgrind = $ENV{VALGRIND_TESTS};
+if (!defined $valgrind || $valgrind eq "" || $valgrind == 0) {
+    # Valgrind not requested - leave $cmdline as it is
+} elsif ($valgrind == 1) {
+    $cmdline = "valgrind --error-exitcode=301 --leak-check=yes --track-origins=yes " . $cmdline;
+} else {
+    $cmdline = $valgrind . " " . $cmdline;
+}
+
+my $code = system($cmdline . " 2>&1") >> 8;
+unlink "needs-auth.txt";
 
 warn "Got code: $code\n" if $code;
 kill ('TERM', $pid);
-exit ($code >> 8);
+exit ($code != 0);

tests/Test-stdouterr.px

@@ -21,7 +21,7 @@ my %urls = (
 );
 
 unless(-e "/dev/full") {
-    exit 2; # skip
+    exit 77; # skip
 }
 
 my $cmdline = $WgetTest::WGETPATH . " -c http://localhost:{{port}}/somefile.txt -O /dev/full";

tests/WgetFeature.pm

@@ -3,26 +3,41 @@ package WgetFeature;
 use strict;
 use warnings;
 
+our $VERSION = 0.01;
+
+use Carp;
+use English qw(-no_match_vars);
+use FindBin;
 use WgetTests;
 
-our %skip_messages;
-require 'WgetFeature.cfg';
+our %SKIP_MESSAGES;
+{
+    my $cfgfile = "$FindBin::Bin/WgetFeature.cfg";
+    open my $fh, '<', $cfgfile
+      or croak "Cannot open '$cfgfile': $ERRNO";
+    my @lines = <$fh>;
+    close $fh or carp "Cannot close '$cfgfile': $ERRNO";
+    my $evalstr = join q{}, @lines;
+    eval { $evalstr } or carp "Cannot eval '$cfgfile': $ERRNO";
+}
 
 sub import
 {
     my ($class, $feature) = @_;
 
     my $output = `$WgetTest::WGETPATH --version`;
-    my ($list) = $output =~ /^([\+\-]\S+(?:\s+[\+\-]\S+)+)/m;
-    my %have_features = map {
-        my $feature = $_;
-           $feature =~ s/^.//;
-          ($feature, /^\+/ ? 1 : 0);
-    } split /\s+/, $list;
-
-    unless ($have_features{$feature}) {
-        print $skip_messages{$feature}, "\n";
-        exit 2; # skip
+    my ($list) = $output =~ m/^([+-]\S+(?:\s+[+-]\S+)+)/msx;
+    my %have_features;
+    for my $f (split m/\s+/msx, $list)
+    {
+        my $feat = $f;
+        $feat =~ s/^.//msx;
+        $have_features{$feat} = $f =~ m/^[+]/msx ? 1 : 0;
+    }
+    if (!$have_features{$feature})
+    {
+        print "$SKIP_MESSAGES{$feature}\n";
+        exit 77;    # skip
     }
 }
 

tests/WgetTests.pm

@@ -1,85 +1,103 @@
 package WgetTest;
-$VERSION = 0.01;
 
 use strict;
 use warnings;
 
+our $VERSION = 0.01;
+
+use Carp;
 use Cwd;
+use English qw(-no_match_vars);
 use File::Path;
+use IO::Handle;
 use POSIX qw(locale_h);
 use locale;
 
-our $WGETPATH = "../src/wget";
+our $WGETPATH = '../src/wget';
 
 my @unexpected_downloads = ();
 
 {
-    my %_attr_data = ( # DEFAULT
-        _cmdline      => "",
-        _workdir      => Cwd::getcwd(),
-        _errcode      => 0,
-        _existing     => {},
-        _input        => {},
-        _name         => $0,
-        _output       => {},
-        _server_behavior => {},
-    );
+    my %_attr_data = (    # DEFAULT
+                       _cmdline         => q{},
+                       _workdir         => Cwd::getcwd(),
+                       _errcode         => 0,
+                       _existing        => {},
+                       _input           => {},
+                       _name            => $PROGRAM_NAME,
+                       _output          => {},
+                       _server_behavior => {},
+                     );
 
     sub _default_for
     {
         my ($self, $attr) = @_;
-        $_attr_data{$attr};
+        return $_attr_data{$attr};
     }
 
     sub _standard_keys
     {
-        keys %_attr_data;
+        return keys %_attr_data;
     }
 }
 
-
-sub new {
+sub new
+{
     my ($caller, %args) = @_;
-    my $caller_is_obj = ref($caller);
+    my $caller_is_obj = ref $caller;
     my $class = $caller_is_obj || $caller;
+
     #print STDERR "class = ", $class, "\n";
     #print STDERR "_attr_data {workdir} = ", $WgetTest::_attr_data{_workdir}, "\n";
     my $self = bless {}, $class;
-    foreach my $attrname ($self->_standard_keys()) {
+    for my $attrname ($self->_standard_keys())
+    {
+
         #print STDERR "attrname = ", $attrname, " value = ";
-        my ($argname) = ($attrname =~ /^_(.*)/);
-        if (exists $args{$argname}) {
+        my ($argname) = ($attrname =~ m/^_(.*)/msx);
+        if (exists $args{$argname})
+        {
+
             #printf STDERR "Setting up $attrname\n";
             $self->{$attrname} = $args{$argname};
-        } elsif ($caller_is_obj) {
+        }
+        elsif ($caller_is_obj)
+        {
+
             #printf STDERR "Copying $attrname\n";
             $self->{$attrname} = $caller->{$attrname};
-        } else {
+        }
+        else
+        {
             #printf STDERR "Using default for $attrname\n";
             $self->{$attrname} = $self->_default_for($attrname);
         }
+
         #print STDERR $attrname, '=', $self->{$attrname}, "\n";
     }
+
     #printf STDERR "_workdir default = ", $self->_default_for("_workdir");
     return $self;
 }
 
-
-sub run {
-    my $self = shift;
+sub run
+{
+    my $self           = shift;
     my $result_message = "Test successful.\n";
     my $errcode;
 
-    $self->{_name} =~ s{.*/}{}; # remove path
-    $self->{_name} =~ s{\.[^.]+$}{}; # remove extension
+    $self->{_name} =~ s{.*/}{}msx;          # remove path
+    $self->{_name} =~ s{[.][^.]+$}{}msx;    # remove extension
     printf "Running test $self->{_name}\n";
 
     # Setup
     my $new_result = $self->_setup();
-    chdir ("$self->{_workdir}/$self->{_name}/input");
-    if (defined $new_result) {
+    chdir "$self->{_workdir}/$self->{_name}/input"
+      or carp "Could not chdir to input directory: $ERRNO";
+    if (defined $new_result)
+    {
         $result_message = $new_result;
-        $errcode = 1;
+        $errcode        = 1;
         goto cleanup;
     }
 
@@ -87,131 +105,175 @@ sub run {
     my $pid = $self->_fork_and_launch_server();
 
     # Call wget
-    chdir ("$self->{_workdir}/$self->{_name}/output");
+    chdir "$self->{_workdir}/$self->{_name}/output"
+      or carp "Could not chdir to output directory: $ERRNO";
+
     my $cmdline = $self->{_cmdline};
     $cmdline = $self->_substitute_port($cmdline);
+    $cmdline =
+      ($cmdline =~ m{^/.*}msx) ? $cmdline : "$self->{_workdir}/$cmdline";
+
+    my $valgrind = $ENV{VALGRIND_TESTS};
+    if (!defined $valgrind || $valgrind eq q{} || $valgrind == 0)
+    {
+
+        # Valgrind not requested - leave $cmdline as it is
+    }
+    elsif ($valgrind == 1)
+    {
+        $cmdline =
+          'valgrind --error-exitcode=301 --leak-check=yes --track-origins=yes '
+          . $cmdline;
+    }
+    else
+    {
+        $cmdline = "$valgrind $cmdline";
+    }
+
     print "Calling $cmdline\n";
-    $errcode =
-        ($cmdline =~ m{^/.*})
-            ? system ($cmdline)
-            : system ("$self->{_workdir}/$cmdline");
-    $errcode >>= 8; # XXX: should handle abnormal error codes.
+    $errcode = system $cmdline;
+    $errcode >>= 8;    # XXX: should handle abnormal error codes.
 
     # Shutdown server
     # if we didn't explicitely kill the server, we would have to call
     # waitpid ($pid, 0) here in order to wait for the child process to
     # terminate
-    kill ('TERM', $pid);
+    kill 'TERM', $pid;
 
     # Verify download
-    unless ($errcode == $self->{_errcode}) {
-        $result_message = "Test failed: wrong code returned (was: $errcode, expected: $self->{_errcode})\n";
-        goto cleanup;
+    if ($errcode != $self->{_errcode})
+    {
+        $result_message =
+          "Test failed: wrong code returned (was: $errcode, expected: $self->{_errcode})\n";
+        goto CLEANUP;
     }
     my $error_str;
-    if ($error_str = $self->_verify_download()) {
+    if ($error_str = $self->_verify_download())
+    {
         $result_message = $error_str;
     }
 
-  cleanup:
+  CLEANUP:
     $self->_cleanup();
 
     print $result_message;
     return $errcode != $self->{_errcode} || ($error_str ? 1 : 0);
 }
 
-
-sub _setup {
+sub _setup
+{
     my $self = shift;
 
-    #print $self->{_name}, "\n";
-    chdir ($self->{_workdir});
+    chdir $self->{_workdir}
+      or carp "Could not chdir into $self->{_workdir}: $ERRNO";
 
     # Create temporary directory
-    mkdir ($self->{_name});
-    chdir ($self->{_name});
-    mkdir ("input");
-    mkdir ("output");
+    mkdir $self->{_name} or carp "Could not mkdir '$self->{_name}': $ERRNO";
+    chdir $self->{_name}
+      or carp "Could not chdir into '$self->{_name}': $ERRNO";
+    mkdir 'input'  or carp "Could not mkdir 'input' $ERRNO";
+    mkdir 'output' or carp "Could not mkdir 'output': $ERRNO";
 
     # Setup existing files
-    chdir ("output");
-    foreach my $filename (keys %{$self->{_existing}}) {
-        open (FILE, ">$filename")
-            or return "Test failed: cannot open pre-existing file $filename\n";
+    chdir 'output' or carp "Could not chdir into 'output': $ERRNO";
+    for my $filename (keys %{$self->{_existing}})
+    {
+        open my $fh, '>', $filename
+          or return "Test failed: cannot open pre-existing file $filename\n";
 
         my $file = $self->{_existing}->{$filename};
-        print FILE $file->{content}
-            or return "Test failed: cannot write pre-existing file $filename\n";
+        print {$fh} $file->{content}
+          or return "Test failed: cannot write pre-existing file $filename\n";
 
-        close (FILE);
+        close $fh or carp $ERRNO;
 
-        if (exists($file->{timestamp})) {
+        if (exists($file->{timestamp}))
+        {
             utime $file->{timestamp}, $file->{timestamp}, $filename
-                or return "Test failed: cannot set timestamp on pre-existing file $filename\n";
+              or return
+              "Test failed: cannot set timestamp on pre-existing file $filename\n";
         }
     }
 
-    chdir ("../input");
+    chdir '../input' or carp "Cannot chdir into '../input': $ERRNO";
     $self->_setup_server();
 
-    chdir ($self->{_workdir});
+    chdir $self->{_workdir}
+      or carp "Cannot chdir into '$self->{_workdir}': $ERRNO";
     return;
 }
 
-
-sub _cleanup {
+sub _cleanup
+{
     my $self = shift;
 
-    chdir ($self->{_workdir});
-    File::Path::rmtree ($self->{_name}) unless $ENV{WGET_TEST_NO_CLEANUP};
+    chdir $self->{_workdir}
+      or carp "Could not chdir into '$self->{_workdir}': $ERRNO";
+    if (!$ENV{WGET_TEST_NO_CLEANUP})
+    {
+        File::Path::rmtree($self->{_name});
+    }
+    return 1;
 }
 
 # not a method
-sub quotechar {
-    my $c = ord( shift );
-    if ($c >= 0x7 && $c <= 0xD) {
-       return '\\' . qw(a b t n v f r)[$c - 0x7];
-    } else {
-        return sprintf('\\x%02x', $c);
+sub quotechar
+{
+    my $c = ord shift;
+    if ($c >= 0x7 && $c <= 0xD)
+    {
+        return q{\\} . qw(a b t n v f r) [$c - 0x7];
+    }
+    else
+    {
+        return sprintf '\\x%02x', $c;
     }
 }
 
 # not a method
-sub _show_diff {
+sub _show_diff
+{
+    my ($expected, $actual) = @_;
     my $SNIPPET_SIZE = 10;
 
-    my ($expected, $actual) = @_;
-
-    my $str = '';
+    my $str    = q{};
     my $explen = length $expected;
     my $actlen = length $actual;
 
-    if ($explen != $actlen) {
+    if ($explen != $actlen)
+    {
         $str .= "Sizes don't match: expected = $explen, actual = $actlen\n";
     }
 
-    my $min = $explen <= $actlen? $explen : $actlen;
+    my $min  = $explen <= $actlen ? $explen : $actlen;
     my $line = 1;
-    my $col = 1;
+    my $col  = 1;
     my $i;
-    for ($i=0; $i != $min; ++$i) {
-        last if substr($expected, $i, 1) ne substr($actual, $i, 1);
-        if (substr($expected, $i, 1) eq '\n') {
+
+    # for ($i=0; $i != $min; ++$i) {
+    for my $i (0 .. $min - 1)
+    {
+        last if substr($expected, $i, 1) ne substr $actual, $i, 1;
+        if (substr($expected, $i, 1) eq q{\n})
+        {
             $line++;
             $col = 0;
-        } else {
+        }
+        else
+        {
             $col++;
         }
     }
     my $snip_start = $i - ($SNIPPET_SIZE / 2);
-    if ($snip_start < 0) {
-        $SNIPPET_SIZE += $snip_start; # Take it from the end.
+    if ($snip_start < 0)
+    {
+        $SNIPPET_SIZE += $snip_start;    # Take it from the end.
         $snip_start = 0;
     }
-    my $exp_snip = substr($expected, $snip_start, $SNIPPET_SIZE);
-    my $act_snip = substr($actual, $snip_start, $SNIPPET_SIZE);
-    $exp_snip =~s/[^[:print:]]/ quotechar($&) /ge;
-    $act_snip =~s/[^[:print:]]/ quotechar($&) /ge;
+    my $exp_snip = substr $expected, $snip_start, $SNIPPET_SIZE;
+    my $act_snip = substr $actual,   $snip_start, $SNIPPET_SIZE;
+    $exp_snip =~ s/[^[:print:]]/ quotechar($&) /gemsx;
+    $act_snip =~ s/[^[:print:]]/ quotechar($&) /gemsx;
     $str .= "Mismatch at line $line, col $col:\n";
     $str .= "    $exp_snip\n";
     $str .= "    $act_snip\n";
@@ -219,102 +281,138 @@ sub _show_diff {
     return $str;
 }
 
-sub _verify_download {
+sub _verify_download
+{
     my $self = shift;
 
-    chdir ("$self->{_workdir}/$self->{_name}/output");
+    chdir "$self->{_workdir}/$self->{_name}/output"
+      or carp "Could not chdir into output directory: $ERRNO";
 
     # use slurp mode to read file content
-    my $old_input_record_separator = $/;
-    undef $/;
+    my $old_input_record_separator = $INPUT_RECORD_SEPARATOR;
+    local $INPUT_RECORD_SEPARATOR = undef;
 
-    while (my ($filename, $filedata) = each %{$self->{_output}}) {
-        open (FILE, $filename)
-            or return "Test failed: file $filename not downloaded\n";
+    while (my ($filename, $filedata) = each %{$self->{_output}})
+    {
+        open my $fh, '<', $filename
+          or return "Test failed: file $filename not downloaded\n";
+
+        my $content = <$fh>;
+
+        close $fh or carp $ERRNO;
 
-        my $content = <FILE>;
         my $expected_content = $filedata->{'content'};
         $expected_content = $self->_substitute_port($expected_content);
-        unless ($content eq $expected_content) {
+        if ($content ne $expected_content)
+        {
             return "Test failed: wrong content for file $filename\n"
-                . _show_diff($expected_content, $content);
+              . _show_diff($expected_content, $content);
         }
 
-        if (exists($filedata->{'timestamp'})) {
-            my ($dev, $ino, $mode, $nlink, $uid, $gid, $rdev, $size,
-                $atime, $mtime, $ctime, $blksize, $blocks) = stat FILE;
+        if (exists($filedata->{'timestamp'}))
+        {
+            my (
+                $dev,   $ino,     $mode, $nlink, $uid,
+                $gid,   $rdev,    $size, $atime, $mtime,
+                $ctime, $blksize, $blocks
+               )
+              = stat $filename;
 
             $mtime == $filedata->{'timestamp'}
-                or return "Test failed: wrong timestamp for file $filename\n";
+              or return "Test failed: wrong timestamp for file $filename: expected = $filedata->{'timestamp'}, actual = $mtime\n";
         }
 
-        close (FILE);
     }
 
-    $/ = $old_input_record_separator;
+    local $INPUT_RECORD_SEPARATOR = $old_input_record_separator;
 
     # make sure no unexpected files were downloaded
-    chdir ("$self->{_workdir}/$self->{_name}/output");
+    chdir "$self->{_workdir}/$self->{_name}/output"
+      or carp "Could not change into output directory: $ERRNO";
 
-    __dir_walk('.',
-               sub { push @unexpected_downloads,
-                          $_[0] unless (exists $self->{_output}{$_[0]} || $self->{_existing}{$_[0]}) },
-               sub { shift; return @_ } );
-    if (@unexpected_downloads) {
-        return "Test failed: unexpected downloaded files [" . join(', ', @unexpected_downloads) . "]\n";
+    __dir_walk(
+        q{.},
+        sub {
+            if (!(exists $self->{_output}{$_[0]} || $self->{_existing}{$_[0]}))
+            {
+                push @unexpected_downloads, $_[0];
+            }
+        },
+        sub { shift; return @_ }
+              );
+    if (@unexpected_downloads)
+    {
+        return 'Test failed: unexpected downloaded files [' . join ', ',
+          @unexpected_downloads . "]\n";
     }
 
-    return "";
+    return q{};
 }
 
-
-sub __dir_walk {
+sub __dir_walk
+{
     my ($top, $filefunc, $dirfunc) = @_;
 
     my $DIR;
 
-    if (-d $top) {
+    if (-d $top)
+    {
         my $file;
-        unless (opendir $DIR, $top) {
-            warn "Couldn't open directory $DIR: $!; skipping.\n";
+        if (!opendir $DIR, $top)
+        {
+            warn "Couldn't open directory $DIR: $ERRNO; skipping.\n";
             return;
         }
 
         my @results;
-        while ($file = readdir $DIR) {
-            next if $file eq '.' || $file eq '..';
-            my $nextdir = $top eq '.' ? $file : "$top/$file";
+        while ($file = readdir $DIR)
+        {
+            next if $file eq q{.} || $file eq q{..};
+            my $nextdir = $top eq q{.} ? $file : "$top/$file";
             push @results, __dir_walk($nextdir, $filefunc, $dirfunc);
         }
 
-        return $dirfunc ? $dirfunc->($top, @results) : () ;
-    } else {
-        return $filefunc ? $filefunc->($top) : () ;
+        return $dirfunc ? $dirfunc->($top, @results) : ();
+    }
+    else
+    {
+        return $filefunc ? $filefunc->($top) : ();
     }
 }
 
-
 sub _fork_and_launch_server
 {
     my $self = shift;
 
-    pipe(FROM_CHILD, TO_PARENT) or die "Cannot create pipe!";
-    select((select(TO_PARENT), $| = 1)[0]);
+    pipe FROM_CHILD, TO_PARENT or croak 'Cannot create pipe!';
+    TO_PARENT->autoflush();
+
+    my $pid = fork;
+    if ($pid < 0)
+    {
+        carp 'Cannot fork';
+    }
+    elsif ($pid == 0)
+    {
 
-    my $pid = fork();
-    if ($pid < 0) {
-        die "Cannot fork";
-    } elsif ($pid == 0) {
         # child
-        close FROM_CHILD;
+        close FROM_CHILD or carp $ERRNO;
+
         # FTP Server has to start with english locale due to use of strftime month names in LIST command
-        setlocale(LC_ALL,"C");
-        $self->_launch_server(sub { print TO_PARENT "SYNC\n"; close TO_PARENT });
-    } else {
+        setlocale(LC_ALL, 'C');
+        $self->_launch_server(
+            sub {
+                print {*TO_PARENT} "SYNC\n";
+                close TO_PARENT or carp $ERRNO;
+            }
+        );
+    }
+    else
+    {
         # father
-        close TO_PARENT;
+        close TO_PARENT or carp $ERRNO;
         chomp(my $line = <FROM_CHILD>);
-        close FROM_CHILD;
+        close FROM_CHILD or carp $ERRNO;
     }
 
     return $pid;

util/Makefile.am

@@ -27,7 +27,7 @@
 # as that of the covered work.
 
 #
-# Version: $(VERSION)
+# Version: @VERSION@
 #
 
 EXTRA_DIST = README rmold.pl trunc.c