Tag release 1.17.1

* NEWS: Update.
* gnulib: sync from upstream.

.gitignore

@@ -2,6 +2,7 @@
 /.sc-start-sc_bindtextdomain
 /.sc-start-sc_prohibit_HAVE_MBRTOWC
 /.version
+/ABOUT-NLS
 /GNUmakefile
 /INSTALL
 /Makefile
@@ -19,10 +20,13 @@
 po/*.gmo*
 po/*.po*
 src/*.o
+tests/*.log
+tests/*.trs
 testenv/*.log
 testenv/*.trs
 # build-aux/
 build-aux/.gitignore
+build-aux/ar-lib
 build-aux/compile
 build-aux/config.guess
 build-aux/config.sub
@@ -54,6 +58,7 @@ po/Makefile.in
 po/Makevars
 po/POTFILES
 po/wget.pot
+po/stamp-po
 # src/
 src/.deps/
 src/Makefile
@@ -81,7 +86,6 @@ testenv/Makefile.in
 # tests/
 tests/Makefile
 tests/Makefile.in
-tests/WgetTest.pm
 tests/unit-tests
 # util/
 util/Makefile

.travis.yml

@@ -0,0 +1,50 @@
+sudo: no
+
+language: c
+
+compiler:
+    - gcc
+
+notifications:
+    email:
+        recipients:
+            - darnir@gmail.com
+            - bug-wget@gnu.org
+        on_success: change
+        on_failure: always
+
+env:
+    global:
+        - PERLLIB=$PERLLIB:../../tests
+        - PERL5LIB=$PERL5LIB:../../tests
+    matrix:
+        - SSL=""
+        - SSL="openssl"
+        - SSL="gnutls"
+
+addons:
+    apt:
+        packages:
+            - automake
+            - autoconf
+            - autopoint
+            - flex
+            - texinfo
+            - pkg-config
+            - libgnutls-dev
+            - libssl-dev
+            - make
+            - libhttp-daemon-perl
+            - libio-socket-ssl-perl
+            - libidn11-dev
+            - gettext
+            - texlive
+            - python3
+            - valgrind
+            - language-pack-tr
+            - language-pack-ru
+
+script:
+    - make distclean || true
+    - ./bootstrap
+    - ./contrib/travis-ci $SSL

ChangeLog.README

@@ -1,16 +0,0 @@
-Please note that Wget has more than one ChangeLog file:
-
-    ./ChangeLog: documents changes to files in the top-level directory
-                 and to files in subdirectories like po/ that don't have
-                 their own ChangeLogs
-
-    src/ChangeLog: documents only changes to files in the src directory
-
-    doc/ChangeLog: documents only changes to files in the doc directory
-
-    windows/ChangeLog: documents only changes to files in the windows directory
-
-    msdos/ChangeLog: documents only changes to files in the msdos directory
-
-When checking to see if a patch you sent in has been applied, please
-look in the appropriate ChangeLog(s).

Makefile.am

@@ -27,7 +27,7 @@
 # as that of the covered work.
 
 #
-# Version: $(VERSION)
+# Version: @VERSION@
 #
 
 
@@ -43,8 +43,8 @@ ACLOCAL_AMFLAGS = -I m4
 # subdirectories in the distribution
 SUBDIRS = lib src doc po tests util testenv
 
-EXTRA_DIST = ChangeLog.README MAILING-LIST \
-             msdos/ChangeLog msdos/config.h msdos/Makefile.DJ \
+EXTRA_DIST = MAILING-LIST \
+             msdos/config.h msdos/Makefile.DJ \
              msdos/Makefile.WC ABOUT-NLS \
              build-aux/build_info.pl build-aux/git-version-gen .version
 
@@ -60,5 +60,20 @@ clean-generic:
 
 # Arrange so that .tarball-version appears only in the distribution
 # tarball, and never in a checked-out repository.
-dist-hook:
+dist-hook: gen-ChangeLog
 	$(AM_V_GEN)echo $(VERSION) > $(distdir)/.tarball-version
+
+gen_start_date = 2014-12-10
+.PHONY: gen-ChangeLog
+gen-ChangeLog:
+	$(AM_V_GEN)if test -d .git; then				\
+	  log_fix="$(srcdir)/build-aux/git-log-fix";			\
+	  test -e "$$log_fix"						\
+	    || amend_git_log=;						\
+	  $(top_srcdir)/build-aux/gitlog-to-changelog			\
+	    $$amend_git_log --since=$(gen_start_date) > $(distdir)/cl-t && \
+            echo >>  $(distdir)/cl-t &&                                 \
+            cat ChangeLog-2014-12-10 >>  $(distdir)/cl-t &&             \
+	    { rm -f $(distdir)/ChangeLog &&				\
+	      mv $(distdir)/cl-t $(distdir)/ChangeLog; }		\
+	fi

NEWS

@@ -1,25 +1,106 @@
 GNU Wget NEWS -- history of user-visible changes.
 
 Copyright (C) 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004, 2005,
-2006, 2007, 2008, 2009, 2010, 2011, 2012 Free Software Foundation, Inc.
+2006, 2007, 2008, 2009, 2010, 2011, 2012, 2013, 2014, 2015 Free
+Software Foundation, Inc.
 See the end for copying conditions.
 
 Please send GNU Wget bug reports to <bug-wget@gnu.org>.
-
-* Changes in Wget X.Y.Z
 
-** Use libpsl for verifying cookie domains
+* Changes in Wget 1.17.1
 
-** Default progress bar output changed
+* Fix compile error when IPv6 is disabled or SSL is not present.
 
-** Introduce --show-progress to force display the progress bar
+* Fix HSTS memory leak.
 
-** Introduce --no-config.
+* Fix progress output in non-C locales.
+
+* Fix SIGSEGV when -N and --content-disposition are used together.
+
+* Add --check-certificate=quiet to tell wget to not print any warning about
+  invalid certificates.
+
+* Changes in Wget 1.17
+
+** Remove FTP passive to active fallback due to privacy concerns.
+
+** Add support for --if-modified-since.
+
+** Add support for metalink through --input-metalink and --metalink-over-http.
+
+** Add support for HSTS through --hsts and --hsts-file.
+
+** Add option to restrict filenames under VMS.
+
+** Add support for --rejected-log which logs to a separate file the reasons why
+   URLs are being rejected and some context around it.
+
+** Add support for FTPS.
+
+** Do not download/save file on error when --spider enabled
+
+** Add --convert-file-only option. This option converts only the
+   filename part of the URLs, leaving the rest of the URLs untouched.
+
+* Changes in Wget 1.16.3
+
+** Fix a regression introduced by wget 1.16.2 that --quiet is not
+   really quiet anymore.
+
+* Changes in Wget 1.16.2
+
+** Native uuid generation on Windows
+
+** Fix build on Solaris
+
+** Allow progress bar on stderr when -o is used
+
+** Accept 5-digit port numbers in FTP EPSV responses.
+
+** Support older versions of flex.
+
+** Updated translations.
+
+* Changes in Wget 1.16.1
+
+** Add --enable-assert configure option.
+
+** Use pkg-config to check for libraries presence.
+
+** Do not limit --secure-protocol=auto|pfs to TLSv1.0.
+
+** Add --secure-protocol=TLSv1_1|TLSv1_2 .
+
+** Full C89 source code compliance.
+
+** Select and use the most secure authentication scheme with HTTP connections.
+
+** Fix issues with turkish locales.
+
+** Handle 504 Gateway Timeout.
+
+** New option --crl-file to load Certificate Revocation Lists.
+
+** Add valgrind support to tests suite.
+
+** Fix an off-by-one problem in the progress bar (introduced in 1.16).
+
+* Changes in Wget 1.16
+
+** No longer create local symbolic links by default.  Closes CVE-2014-4877.
+
+** Use libpsl for verifying cookie domains.
+
+** Default progress bar output changed.
+
+** Introduce --show-progress to force display the progress bar.
+
+** Introduce --no-config.  The wgetrc files will not be read.
 
 ** Introduce --start-pos to allow starting downloads from a specified position.
 
 ** Fix a problem with ISA Server Proxy and keep-alive connections.
-
+
 * Changes in Wget 1.15
 
 ** Add support for --method.
@@ -51,7 +132,7 @@ Please send GNU Wget bug reports to <bug-wget@gnu.org>.
 ** Specify Host with the HTTP CONNECT method.
 
 ** Use the correct HTTP method on a redirection.
-
+
 * Changes in Wget 1.14
 
 ** Add support for content-on-error.  It allows to store the HTTP
@@ -73,12 +154,12 @@ Please send GNU Wget bug reports to <bug-wget@gnu.org>.
 
 ** Add support for TLS Server Name Indication.
 
-** Accept the arguments --accept-reject and --reject-regex.
+** Accept the arguments --accept-regex and --reject-regex.
 
 ** The GNU TLS backend honors correctly the timeout value.
 
 ** Add support for RFC 2617 Digest Access Authentication.
-
+
 * Changes in Wget 1.13.4
 
 ** Now --version and --help work again.
@@ -91,7 +172,7 @@ Please send GNU Wget bug reports to <bug-wget@gnu.org>.
    is specified.
 
 ** Fix a segfault on an incomplete STYLE tag.
-
+
 * Changes in Wget 1.13.3
 
 ** Support HTTP/1.1
@@ -157,7 +238,7 @@ Please send GNU Wget bug reports to <bug-wget@gnu.org>.
 
 ** Introduce `show_all_dns_entries' to print all IP addresses corresponding to
    a DNS name when it is resolved.
-
+
 * Changes in Wget 1.12
 
 ** Mailing list MOVED to bug-wget@gnu.org
@@ -211,7 +292,7 @@ forces the percent-encoding of all non-ASCII bytes
 now documented: save_headers, spider, and user_agent,
 auth_no_challenge, and keep_session_cookies. Also added documentation
 for the "lowercase" and "uppercase" values for --restrict-file-names, which had been present since Wget 1.11.
-
+
 * Changes in Wget 1.11.4
 
 ** Fixed an issue (apparently a regression) where -O would refuse to
@@ -225,13 +306,13 @@ locally with content.
 to a pointer-to-time_t.
 
 ** Translation updates for Catalan.
-
+
 * Changes in Wget 1.11.3
 
 ** Downgraded -N with -O to a warning, rather than an error.
 
 ** Translation updates
-
+
 * Changes in Wget 1.11.2
 
 ** Fixed a problem in authenticating over HTTPS through a proxy.
@@ -253,7 +334,7 @@ multiple times. This has been fixed in 1.11.2.
 for improved conformance with RFC 3986. However, this behavior presents
 problems for some FTP setups, and so they are now preserved again, for
 FTP URLs only.
-
+
 * Changes in Wget 1.11.1.
 
 ** Interrupted downloads no longer result in renaming the file
@@ -275,7 +356,7 @@ be surprising, and notes that they may change in the future.
 
 ** Documentation of --no-parents now explains how a trailing slash, or
 lack thereof, in the specified URL, will affect behavior.
-
+
 * Changes in Wget 1.11.
 
 ** Timestamping now uses the value from the most recent HTTP response,
@@ -321,7 +402,7 @@ available at http://hg.addictivecode.org/. Prior to this, the source
 code was hosted on Subversion (migrated from the original CVS); you can
 still get access to older tags and branches for Wget in the Subversion
 repository at http://addictivecode.org/svn/wget/.
-
+
 * Changes in Wget 1.10.
 
 ** Downloading files larger than 2GB, sometimes referred to as "large
@@ -432,9 +513,9 @@ http_password and proxy_password respectively.  The login and passwd
 .wgetrc commands have been deprecated.
 
 * `wget -b' now works correctly under Windows.
-
+
 * Wget 1.9.1 is a bugfix release with no user-visible changes.
-
+
 * Changes in Wget 1.9.
 
 ** It is now possible to specify that POST method be used for HTTP
@@ -486,11 +567,11 @@ values "yes" and "no" along with the traditional "on" and "off".
 ** It is now possible to specify decimal values for timeouts, waiting
 periods, and download rate.  For instance, `--wait=0.5' now works as
 expected, as does `--dns-timeout=0.5' and even `--limit-rate=2.5k'.
-
+
 * Wget 1.8.2 is a bugfix release with no user-visible changes.
-
+
 * Wget 1.8.1 is a bugfix release with no user-visible changes.
-
+
 * Changes in Wget 1.8.
 
 ** A new progress indicator is now available and used by default.
@@ -550,9 +631,9 @@ no longer translated to `?', nor `%2B' to `+'.  Unsafe characters
 which are not reserved are still escaped, of course.
 
 ** No more than 20 successive redirections are allowed.
-
+
 * Wget 1.7.1 is a bugfix release with no user-visible changes.
-
+
 * Changes in Wget 1.7.
 
 ** SSL (`https') pages now work if you compile Wget with SSL support;
@@ -594,7 +675,7 @@ large sites (thousands of documents).
 documentation.  (The last version that shipped with a man page was
 1.4.5).  To get this, you need to have pod2man from the Perl
 distribution installed on your system.
-
+
 * Changes in Wget 1.6
 
 ** Administrative changes.
@@ -645,13 +726,13 @@ passive_ftp is the only .wgetrc command which takes a lockable Boolean.
 bound to multiple IP addresses.
 
 ** wget now accepts (illegal per HTTP spec) relative URLs in HTTP redirects.
-
+
 * Wget 1.5.3 is a bugfix release with no user-visible changes.
-
+
 * Wget 1.5.2 is a bugfix release with no user-visible changes.
-
+
 * Wget 1.5.1 is a bugfix release with no user-visible changes.
-
+
 * Changes in Wget 1.5.0
 
 ** Wget speaks many languages!
@@ -697,11 +778,11 @@ download.
 
 ** The manual page, now hopelessly out of date, is no longer
 distributed with Wget.
-
+
 * Wget 1.4.5 is a bugfix release with no user-visible changes.
-
+
 * Wget 1.4.4 is a bugfix release with no user-visible changes.
-
+
 * Changes in Wget 1.4.3
 
 ** Wget is now a GNU utility.
@@ -717,7 +798,7 @@ distributed with Wget.
 ** Global wgetrc now goes to /usr/local/etc (i.e. $sysconfdir).
 
 ** Lots of bugfixes.
-
+
 * Changes in Wget 1.4.2
 
 ** New mirror site at ftp://sunsite.auc.dk/pub/infosystems/wget/,
@@ -735,7 +816,7 @@ files.
 ** --convert-links should work now.
 
 ** Minor bugfixes.
-
+
 * Changes in Wget 1.4.1
 
 ** Minor bugfixes.
@@ -743,7 +824,7 @@ files.
 ** Added -I (the opposite of -X).
 
 ** Dot tracing is now customizable; try wget --dot-style=binary
-
+
 * Changes in Wget 1.4.0
 
 ** Wget 1.4.0 [formerly known as Geturl] is an extensive rewrite of
@@ -824,7 +905,7 @@ turn on mirroring options).
 ** Lots of small features I can't remember. :-)
 
 ** A host of bugfixes.
-
+
 * Changes in Geturl 1.3
 
 ** Added FTP globbing support (ftp://fly.cc.fer.hr/*)
@@ -842,9 +923,9 @@ turn on mirroring options).
 ** Added support for robots.txt
 
 ** Fixed some minor bugs
-
+
 * Geturl 1.2 is a bugfix release with no user-visible changes.
-
+
 * Changes in Geturl 1.1
 
 ** REST supported in FTP
@@ -862,7 +943,7 @@ geturl -vo log http://fly.cc.fer.hr/
 ** <base href="xxx"> supported
 
 ** autoconf supported
-
+
 ----------------------------------------------------------------------
 Copyright information:
 

README.checkout

@@ -4,8 +4,9 @@ Compiling From Repository Sources
    does not contain automatically-generated files, even when these are
    normally present in the distribution tarballs. Therefore, to build GNU
    Wget from the sources in the repository, you'll need to have one or
-   more of the following (note that gettext, OpenSSL, libidn, libiconv
-   and libpsl are not absolutely required):
+   more of the following (note that gettext, OpenSSL, GnuTLS, libidn,
+   libiconv, libpsl, libpcre, pkg-config, libmetalink and GnuPG are not
+   absolutely required):
 
      * [20]autoconf (currently, GNU Wget requires version 2.61). This is
        needed to generate the configure script from configure.in. This is
@@ -23,12 +24,17 @@ Compiling From Repository Sources
      * [23]Perl, if you wish to generate the wget(1) manpage, or run the
        tests in the tests/ sub directory. Tarball distributions include an
        already-generated wget.1 manual. The command "make check" runs the
-       test suite written in perl. To execute all the tests you need
-       [24]libwww-perl perl library. If "perl -MCPAN -e 'install
-       Bundle::LWP'" fails then you most likely don't have cpan module
-       installed. First download [25]CPAN and [26]install it. Then execute
-       "perl -MCPAN -e 'install Bundle::LWP'". Now "make check" should
-       pass most of the tests in the test suite.
+       test suite written in perl and python (see below). To execute all the tests
+       you need [24]libwww-perl and libio-socket-ssl-perl perl library. If
+       "perl -MCPAN -e 'install Bundle::LWP'" fails then you most likely don't
+       have cpan module installed. First download [25]CPAN and [26]install it.
+       Then execute "perl -MCPAN -e 'install Bundle::LWP'".
+       Now "make check" should pass most of the tests in the test suite.
+
+     * [45]Python, if you want to run the tests in the testenv/ subdirectory.
+       Keep in mind that "make check" will try to run all the perl and python
+       tests. More information about the test suite below in the section
+       "Testing and development".
 
      * [27]texinfo in order to generate Info, PostScript and/or HTML
        documentation. You don't need texinfo in order to generate the
@@ -46,35 +52,58 @@ Compiling From Repository Sources
        to export M4=<new m4 path>, to be sure that autoconf/automake use
        it instead of the old one.
 
-     * [29]OpenSSL to handle encrypted web sessions. You almost certainly
-       want this. It is not enough to have the library installed; you need
-       to have the header files available to compile against as well. For
-       GNU/Linux systems, this usually means installing a "developer"
-       package for openssl (for example, [30]Debian systems require
-       installation of the libssl-dev package). Alternatively, you can
-       compile GNU Wget against the [31]GNUTLS library instead, by
-       specifying --with-ssl=gnutls to the ./configure script; however,
-       building against GNUTLS is currently experimental, and not
-       officially supported. If you have OpenSSL and its development
-       headers installed, but don't want to build Wget with it, specify
-       --without-ssl to the ./configure script.
+     * [29]GnuTLS to allow encrypted data transfer (HTTPS).
+       You need the header files and the library installed.
+       As an alternative, you can use [30]OpenSSL by specifying
+       --with-ssl=openssl to the ./configure script.
+       If you do not want HTTPS support, specify --without-ssl to the
+       ./configure script.
+       If you want to compile+link a non-system library version use
+       --with-libgnutls-prefix (or if having pkg-config: see description
+       below).
 
-     * [32]libidn is required for IDN/IRI support (non-ASCII characters
+     * [30]OpenSSL to allow encrypted data transfer (HTTPS) an
+       alternative to [31]GnuTLS.
+       You need the header files and the library installed.
+       If you want to compile+link a non-system library version use
+       --with-libssl-prefix (or if having pkg-config: see description below).
+
+     * [31]libidn is required for IDN/IRI support (non-ASCII characters
        within what would otherwise be URLs).
 
-     * [33]libiconv is required on non-GNU systems, for IDN/IRI support.
+     * [32]libiconv is required on non-GNU systems, for IDN/IRI support.
        On GNU systems, the functionality provided by libiconv is already
        present in the system libraries.
 
-     * [34]git is used to fetch gnulib files trough the bootstrap.sh script.
+     * [33]git is used to fetch gnulib files trough the bootstrap.sh script.
+
+     * [34]libpsl is required for using a public suffix list to check for valid
+       cookie domains. You need the header files and the library installed.
+
+     * [35]libpcre is required for using Perl-compatible regular expressions
+       with --accept-regex and --reject-regex. You need the header files and
+       the library installed to compile and link Wget with PCRE support.
+
+     * [36]pkg-config helps the ./configure script to find installed libraries.
+       Most libraries provide a pkg-config file (.pc extension) with
+       information about dependencies, header file and library locations.
+       Distributions deliver their specific .pc file to each library.
+       If you want to compile+link against your own library version, make a
+       copy of the appropriate .pc file and amend it to your needs (e.g. edit
+       the line starting with prefix=). Before you execute the ./configure
+       script, set (and export) PKG_CONFIG_PATH to the directory where you
+       saved the .pc file. Example:
+       $ PKG_CONFIG_PATH="." ./configure
+
+     * [46]libmetalink is needed to enable Metalink files support.
+
+     * [47]GnuPG with GPGME is used to verify GPG-signed Metalink resources.
 
-     * [35]libpsl is required for using a public suffix list to check for valid
-       cookie domains.
 
    For those who might be confused as to what to do once they check out
    the source code, considering configure and Makefile do not yet exist at
-   that point, a shell script called bootstrap.sh has been provided. After
-   calling ./bootstrap.sh you're ready to build GNU Wget in the normal
+   that point, a shell script called bootstrap has been provided. After
+   calling ./bootstrap you're ready to build GNU Wget in the normal
    fashion, with ./configure and make.
 
    So, to sum up, after checking out the source code as described above,
@@ -83,15 +112,48 @@ Compiling From Repository Sources
  $  cd wget        # assumes you've cloned a repository to "./wget"
     2. Generate all the automatically-generated files required prior to
        configuring the package:
- $  ./bootstrap.sh
+ $  ./bootstrap
     3. Configure the package and compile it:
- $  ./configure [some_parameters]
+ $  ./configure --enable-assert [some_parameters]
  $  make
     4. Hack, compile, test, hack, compile, test...
  $  src/wget --version
  GNU Wget 1.12-devel (9cb2563197bc)
 
- Copyright © 2008,2010 Free Software Foundation, Inc.
+Testing and development
+
+ All developers are requested to enable the assertions on their development
+ builds to ensure a stable codebase. Assertions are added to state certain
+ assumptions about the code and its data which all developers should be mindful
+ of. To enable assertions, run the configure command with the --enable-assert
+ option, like this:
+
+ $ ./configure --enable-assert [other configure options]
+
+ Both the Perl and Python test suites (test/ and testenv/) include support for GDB and Valgrind.
+ The environment variables GDB_TESTS and VALGRIND_TESTS are available to
+ enable such wrappers. If specified, Wget would be run through either of them
+ during the test. For example:
+
+ $ cd testenv
+ $ VALGRIND_TESTS=1 ./Test-O.py
+
+ That would execute Test-O.py test case, but running Wget through Valgrind.
+
+ GDB has preference over Valgrind. If both variables have been asserted,
+ Wget would be run through GDB.
+
+ If you run a test case through GDB, please bear in mind that it could give
+ a false negative. This is because some tests that expect Wget to fail
+ rely on Wget's return code. However, when run through GDB, its return code
+ will always be zero, causing the test to claim failure. This wrapper for
+ GDB is, however, very useful to tackle bugs, allowing one to write a test case
+ for some specific bug and then using GDB to fix it more easily. Otherwise,
+ a dedicated server would have to be set up and write a custom CGI just to reproduce
+ that bug, which might be tedious. Tests should only be run through GDB for that purpose.
+
+
+ Copyright (C) 2008,2010,2014,2015 Free Software Foundation, Inc.
  License GPLv3+: GNU GPL version 3 or later
  <http://www.gnu.org/licenses/gpl.html>.
  This is free software: you are free to change and redistribute it.
@@ -112,8 +174,8 @@ Compiling From Repository Sources
      * [43]Valid HTML 4.01
      __________________________________________________________________
 
-   All content © 2007 Free Software Foundation. For terms of use,
-   redistribution, and modification, please see the [44]WikiLicense page.
+   All content (C) 2007 Free Software Foundation. For terms of use,
+   redistribution, and modification, please see the [45]WikiLicense page.
 
 References
 
@@ -126,13 +188,13 @@ References
   26. http://apthorpe.cynistar.net/code/configuring_cpan.html
   27. http://www.gnu.org/software/texinfo/
   28. http://www.gnu.org/software/gettext/
-  29. http://www.openssl.org/
-  30. http://www.debian.org/
-  31. http://www.gnu.org/software/gnutls/
-  32. http://www.gnu.org/software/libidn/
-  33. http://www.gnu.org/software/libiconv/
-  34. http://git-scm.com/
-  35. https://github.com/rockdaboot/libpsl
+  29. http://www.gnu.org/software/gnutls/
+  30. http://www.openssl.org/
+  31. http://www.gnu.org/software/libidn/
+  32. http://www.gnu.org/software/libiconv/
+  33. http://git-scm.com/
+  34. https://github.com/rockdaboot/libpsl
+  35. http://www.pcre.org
   36. http://wget.addictivecode.org/CompilingRepoSources?action=edit&editor=text
   37. http://wget.addictivecode.org/CompilingRepoSources
   38. http://wget.addictivecode.org/CompilingRepoSources?action=info
@@ -142,3 +204,6 @@ References
   42. http://moinmo.in/GPL
   43. http://validator.w3.org/check?uri=referer
   44. http://wget.addictivecode.org/WikiLicense
+  45. https://www.python.org/
+  46. https://launchpad.net/libmetalink
+  47. https://www.gnupg.org

bootstrap

@@ -1,10 +1,10 @@
 #! /bin/sh
 # Print a version string.
-scriptversion=2012-07-19.14; # UTC
+scriptversion=2013-12-05.23; # UTC
 
 # Bootstrap this package from checked-out sources.
 
-# Copyright (C) 2003-2012 Free Software Foundation, Inc.
+# Copyright (C) 2003-2014 Free Software Foundation, Inc.
 
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -140,20 +140,21 @@ po_download_command_format2=\
 "wget --mirror -nd -q -np -A.po -P '%s' \
  http://translationproject.org/latest/%s/"
 
+# Prefer a non-empty tarname (4th argument of AC_INIT if given), else
+# fall back to the package name (1st argument with munging)
 extract_package_name='
-  /^AC_INIT(/{
-     /.*,.*,.*, */{
-       s///
-       s/[][]//g
-       s/)$//
+  /^AC_INIT(\[*/{
+     s///
+     /^[^,]*,[^,]*,[^,]*,[ []*\([^][ ,)]\)/{
+       s//\1/
+       s/[],)].*//
        p
        q
      }
-     s/AC_INIT(\[*//
-     s/]*,.*//
+     s/[],)].*//
      s/^GNU //
      y/ABCDEFGHIJKLMNOPQRSTUVWXYZ/abcdefghijklmnopqrstuvwxyz/
-     s/[^A-Za-z0-9_]/-/g
+     s/[^abcdefghijklmnopqrstuvwxyz0123456789_]/-/g
      p
   }
 '
@@ -208,12 +209,26 @@ bootstrap_sync=false
 # Use git to update gnulib sources
 use_git=true
 
+check_exists() {
+  if test "$1" = "--verbose"; then
+    ($2 --version </dev/null) >/dev/null 2>&1
+    if test $? -ge 126; then
+      # If not found, run with diagnostics as one may be
+      # presented with env variables to set to find the right version
+      ($2 --version </dev/null)
+    fi
+  else
+    ($1 --version </dev/null) >/dev/null 2>&1
+  fi
+
+  test $? -lt 126
+}
+
 # find_tool ENVVAR NAMES...
 # -------------------------
 # Search for a required program.  Use the value of ENVVAR, if set,
-# otherwise find the first of the NAMES that can be run (i.e.,
-# supports --version).  If found, set ENVVAR to the program name,
-# die otherwise.
+# otherwise find the first of the NAMES that can be run.
+# If found, set ENVVAR to the program name, die otherwise.
 #
 # FIXME: code duplication, see also gnu-web-doc-update.
 find_tool ()
@@ -223,27 +238,21 @@ find_tool ()
   find_tool_names=$@
   eval "find_tool_res=\$$find_tool_envvar"
   if test x"$find_tool_res" = x; then
-    for i
-    do
-      if ($i --version </dev/null) >/dev/null 2>&1; then
-       find_tool_res=$i
-       break
+    for i; do
+      if check_exists $i; then
+        find_tool_res=$i
+        break
       fi
     done
-  else
-    find_tool_error_prefix="\$$find_tool_envvar: "
   fi
-  test x"$find_tool_res" != x \
-    || die "one of these is required: $find_tool_names"
-  ($find_tool_res --version </dev/null) >/dev/null 2>&1 \
-    || die "${find_tool_error_prefix}cannot run $find_tool_res --version"
+  if test x"$find_tool_res" = x; then
+    warn_ "one of these is required: $find_tool_names;"
+    die   "alternatively set $find_tool_envvar to a compatible tool"
+  fi
   eval "$find_tool_envvar=\$find_tool_res"
   eval "export $find_tool_envvar"
 }
 
-# Find sha1sum, named gsha1sum on MacPorts, and shasum on Mac OS X 10.6.
-find_tool SHA1SUM sha1sum gsha1sum shasum
-
 # Override the default configuration, if necessary.
 # Make sure that bootstrap.conf is sourced from the current directory
 # if we were invoked as "sh bootstrap".
@@ -255,12 +264,12 @@ esac
 # Extra files from gnulib, which override files from other sources.
 test -z "${gnulib_extra_files}" && \
   gnulib_extra_files="
-        $build_aux/install-sh
-        $build_aux/mdate-sh
-        $build_aux/texinfo.tex
-        $build_aux/depcomp
-        $build_aux/config.guess
-        $build_aux/config.sub
+        build-aux/install-sh
+        build-aux/mdate-sh
+        build-aux/texinfo.tex
+        build-aux/depcomp
+        build-aux/config.guess
+        build-aux/config.sub
         doc/INSTALL
 "
 
@@ -306,34 +315,34 @@ if test -n "$checkout_only_file" && test ! -r "$checkout_only_file"; then
   die "Bootstrapping from a non-checked-out distribution is risky."
 fi
 
-# Ensure that lines starting with ! sort last, per gitignore conventions
-# for whitelisting exceptions after a more generic blacklist pattern.
-sort_patterns() {
-  sort -u "$@" | sed '/^!/ {
-    H
-    d
-  }
-  $ {
-    P
-    x
-    s/^\n//
-  }' | sed '/^$/d'
+# Strip blank and comment lines to leave significant entries.
+gitignore_entries() {
+  sed '/^#/d; /^$/d' "$@"
 }
 
-# If $STR is not already on a line by itself in $FILE, insert it,
-# sorting the new contents of the file and replacing $FILE with the result.
-insert_sorted_if_absent() {
+# If $STR is not already on a line by itself in $FILE, insert it at the start.
+# Entries are inserted at the start of the ignore list to ensure existing
+# entries starting with ! are not overridden.  Such entries support
+# whitelisting exceptions after a more generic blacklist pattern.
+insert_if_absent() {
   file=$1
   str=$2
   test -f $file || touch $file
-  echo "$str" | sort_patterns - $file | cmp -s - $file > /dev/null \
-    || { echo "$str" | sort_patterns - $file > $file.bak \
-      && mv $file.bak $file; } \
-    || die "insert_sorted_if_absent $file $str: failed"
+  test -r $file || die "Error: failed to read ignore file: $file"
+  duplicate_entries=$(gitignore_entries $file | sort | uniq -d)
+  if [ "$duplicate_entries" ] ; then
+    die "Error: Duplicate entries in $file: " $duplicate_entries
+  fi
+  linesold=$(gitignore_entries $file | wc -l)
+  linesnew=$( { echo "$str"; cat $file; } | gitignore_entries | sort -u | wc -l)
+  if [ $linesold != $linesnew ] ; then
+    { echo "$str" | cat - $file > $file.bak && mv $file.bak $file; } \
+      || die "insert_if_absent $file $str: failed"
+  fi
 }
 
 # Adjust $PATTERN for $VC_IGNORE_FILE and insert it with
-# insert_sorted_if_absent.
+# insert_if_absent.
 insert_vc_ignore() {
   vc_ignore_file="$1"
   pattern="$2"
@@ -344,7 +353,7 @@ insert_vc_ignore() {
     # .gitignore entry.
     pattern=$(echo "$pattern" | sed s,^,/,);;
   esac
-  insert_sorted_if_absent "$vc_ignore_file" "$pattern"
+  insert_if_absent "$vc_ignore_file" "$pattern"
 }
 
 # Die if there is no AC_CONFIG_AUX_DIR($build_aux) line in configure.ac.
@@ -409,7 +418,7 @@ sort_ver() { # sort -V is not generally available
 get_version() {
   app=$1
 
-  $app --version >/dev/null 2>&1 || return 1
+  $app --version >/dev/null 2>&1 || { $app --version; return 1; }
 
   $app --version 2>&1 |
   sed -n '# Move version to start of line.
@@ -468,8 +477,7 @@ check_versions() {
     if [ "$req_ver" = "-" ]; then
       # Merely require app to exist; not all prereq apps are well-behaved
       # so we have to rely on $? rather than get_version.
-      $app --version >/dev/null 2>&1
-      if [ 126 -le $? ]; then
+      if ! check_exists --verbose $app; then
         warn_ "Error: '$app' not found"
         ret=1
       fi
@@ -502,6 +510,12 @@ print_versions() {
   # can't depend on column -t
 }
 
+# Find sha1sum, named gsha1sum on MacPorts, shasum on Mac OS X 10.6.
+# Also find the compatible sha1 utility on the BSDs
+if test x"$SKIP_PO" = x; then
+  find_tool SHA1SUM sha1sum gsha1sum shasum sha1
+fi
+
 use_libtool=0
 # We'd like to use grep -E, to see if any of LT_INIT,
 # AC_PROG_LIBTOOL, AM_PROG_LIBTOOL is used in configure.ac,
@@ -547,13 +561,21 @@ if ! printf "$buildreq" | check_versions; then
   fi
 fi
 
+# Warn the user if autom4te appears to be broken; this causes known
+# issues with at least gettext 0.18.3.
+probe=$(echo 'm4_quote([hi])' | autom4te -l M4sugar -t 'm4_quote:$%' -)
+if test "x$probe" != xhi; then
+  warn_ "WARNING: your autom4te wrapper eats stdin;"
+  warn_ "if bootstrap fails, consider upgrading your autotools"
+fi
+
 echo "$0: Bootstrapping from checked-out $package sources..."
 
 # See if we can use gnulib's git-merge-changelog merge driver.
-if test -d .git && (git --version) >/dev/null 2>/dev/null ; then
+if $use_git && test -d .git && check_exists git; then
   if git config merge.merge-changelog.driver >/dev/null ; then
     :
-  elif (git-merge-changelog --version) >/dev/null 2>/dev/null ; then
+  elif check_exists git-merge-changelog; then
     echo "$0: initializing git-merge-changelog driver"
     git config merge.merge-changelog.name 'GNU-style ChangeLog merge driver'
     git config merge.merge-changelog.driver 'git-merge-changelog %O %A %B'
@@ -573,17 +595,21 @@ git_modules_config () {
   test -f .gitmodules && git config --file .gitmodules "$@"
 }
 
-gnulib_path=$(git_modules_config submodule.gnulib.path)
-test -z "$gnulib_path" && gnulib_path=gnulib
+if $use_git; then
+  gnulib_path=$(git_modules_config submodule.gnulib.path)
+  test -z "$gnulib_path" && gnulib_path=gnulib
+fi
 
-# Get gnulib files.
+# Get gnulib files.  Populate $GNULIB_SRCDIR, possibly updating a
+# submodule, for use in the rest of the script.
 
 case ${GNULIB_SRCDIR--} in
 -)
+  # Note that $use_git is necessarily true in this case.
   if git_modules_config submodule.gnulib.url >/dev/null; then
     echo "$0: getting gnulib files..."
-    git submodule init || exit $?
-    git submodule update || exit $?
+    git submodule init -- "$gnulib_path" || exit $?
+    git submodule update -- "$gnulib_path" || exit $?
 
   elif [ ! -d "$gnulib_path" ]; then
     echo "$0: getting gnulib files..."
@@ -600,8 +626,8 @@ case ${GNULIB_SRCDIR--} in
   GNULIB_SRCDIR=$gnulib_path
   ;;
 *)
-  # Use GNULIB_SRCDIR as a reference.
-  if test -d "$GNULIB_SRCDIR"/.git && \
+  # Use GNULIB_SRCDIR directly or as a reference.
+  if $use_git && test -d "$GNULIB_SRCDIR"/.git && \
         git_modules_config submodule.gnulib.url >/dev/null; then
     echo "$0: getting gnulib files..."
     if git submodule -h|grep -- --reference > /dev/null; then
@@ -612,13 +638,14 @@ case ${GNULIB_SRCDIR--} in
       # This fallback allows at least git 1.5.5.
       if test -f "$gnulib_path"/gnulib-tool; then
         # Since file already exists, assume submodule init already complete.
-        git submodule update || exit $?
+        git submodule update -- "$gnulib_path" || exit $?
       else
         # Older git can't clone into an empty directory.
         rmdir "$gnulib_path" 2>/dev/null
         git clone --reference "$GNULIB_SRCDIR" \
           "$(git_modules_config submodule.gnulib.url)" "$gnulib_path" \
-          && git submodule init && git submodule update \
+          && git submodule init -- "$gnulib_path" \
+          && git submodule update -- "$gnulib_path" \
           || exit $?
       fi
     fi
@@ -627,12 +654,19 @@ case ${GNULIB_SRCDIR--} in
   ;;
 esac
 
+# $GNULIB_SRCDIR now points to the version of gnulib to use, and
+# we no longer need to use git or $gnulib_path below here.
+
 if $bootstrap_sync; then
   cmp -s "$0" "$GNULIB_SRCDIR/build-aux/bootstrap" || {
     echo "$0: updating bootstrap and restarting..."
+    case $(sh -c 'echo "$1"' -- a) in
+      a) ignored=--;;
+      *) ignored=ignored;;
+    esac
     exec sh -c \
       'cp "$1" "$2" && shift && exec "${CONFIG_SHELL-/bin/sh}" "$@"' \
-      -- "$GNULIB_SRCDIR/build-aux/bootstrap" \
+      $ignored "$GNULIB_SRCDIR/build-aux/bootstrap" \
       "$0" "$@" --no-bootstrap-sync
   }
 fi
@@ -680,11 +714,10 @@ update_po_files() {
     cksum_file="$ref_po_dir/$po.s1"
     if ! test -f "$cksum_file" ||
         ! test -f "$po_dir/$po.po" ||
-        ! $SHA1SUM -c --status "$cksum_file" \
-            < "$new_po" > /dev/null; then
+        ! $SHA1SUM -c "$cksum_file" < "$new_po" > /dev/null 2>&1; then
       echo "$me: updated $po_dir/$po.po..."
       cp "$new_po" "$po_dir/$po.po" \
-          && $SHA1SUM < "$new_po" > "$cksum_file"
+          && $SHA1SUM < "$new_po" > "$cksum_file" || return
     fi
   done
 }
@@ -889,20 +922,21 @@ find "$m4_base" "$source_base" \
   -depth \( -name '*.m4' -o -name '*.[ch]' \) \
   -type l -xtype l -delete > /dev/null 2>&1
 
+# Invoke autoreconf with --force --install to ensure upgrades of tools
+# such as ylwrap.
+AUTORECONFFLAGS="--verbose --install --force -I $m4_base $ACLOCAL_FLAGS"
+
 # Some systems (RHEL 5) are using ancient autotools, for which the
 # --no-recursive option had not been invented.  Detect that lack and
 # omit the option when it's not supported.  FIXME in 2017: remove this
 # hack when RHEL 5 autotools are updated, or when they become irrelevant.
-no_recursive=
 case $($AUTORECONF --help) in
-  *--no-recursive*) no_recursive=--no-recursive;;
+  *--no-recursive*) AUTORECONFFLAGS="$AUTORECONFFLAGS --no-recursive";;
 esac
 
 # Tell autoreconf not to invoke autopoint or libtoolize; they were run above.
-echo "running: AUTOPOINT=true LIBTOOLIZE=true " \
-    "$AUTORECONF --verbose --install $no_recursive -I $m4_base $ACLOCAL_FLAGS"
-AUTOPOINT=true LIBTOOLIZE=true \
-    $AUTORECONF --verbose --install $no_recursive -I $m4_base $ACLOCAL_FLAGS \
+echo "running: AUTOPOINT=true LIBTOOLIZE=true $AUTORECONF $AUTORECONFFLAGS"
+AUTOPOINT=true LIBTOOLIZE=true $AUTORECONF $AUTORECONFFLAGS \
   || die "autoreconf failed"
 
 # Get some extra files from gnulib, overriding existing files.

bootstrap.conf

@@ -31,17 +31,25 @@ announce-gen
 base32
 bind
 c-ctype
+c-strcase
+c-strcasestr
 clock-time
 close
 connect
+dirname
 fcntl
+flock
+fnmatch
 futimens
 ftello
 getaddrinfo
+getline
 getopt-gnu
 getpass-gnu
 getpeername
 getsockname
+gettext
+gitlog-to-changelog
 git-version-gen
 gnupload
 ioctl
@@ -49,13 +57,15 @@ iconv
 iconv-h
 listen
 maintainer-makefile
+mbiter
 mbtowc
+memrchr
 mkdir
 mkstemp
 mkostemp
 crypto/md5
 crypto/sha1
-pipe
+crypto/sha256
 quote
 quotearg
 recv
@@ -67,16 +77,21 @@ sigprocmask
 sigpipe
 snprintf
 socket
+spawn-pipe
 stdbool
-strcasestr
+strcase
 strerror_r-posix
+strptime
 strtok_r
+strtoll
+timegm
 tmpdir
 unlocked-io
 update-copyright
 vasprintf
 vsnprintf
 write
+xstrndup
 "
 
 gnulib_extra_files="
@@ -102,4 +117,5 @@ perl       5.5
 rsync      -
 tar        -
 xz         -
+flex       -
 "

build-aux/build_info.pl

@@ -137,3 +137,4 @@ __DATA__
 
 #include "wget.h"
 #include <stdio.h>
+#include "version.h"

cfg.mk

@@ -19,10 +19,11 @@ local-checks-to-skip = \
     sc_prohibit_S_IS_definition \
     \
     sc_program_name \
-
+    sc_makefile_at_at_check \
+    sc_cast_of_argument_to_free \
 
 #SHELL=bash -x
 show-vc-list-except:
 	@$(VC_LIST_EXCEPT)
 
-VC_LIST_ALWAYS_EXCLUDE_REGEX = ^ABOUT-NLS|((msdos|vms)/.*)$$
+VC_LIST_ALWAYS_EXCLUDE_REGEX = ^ABOUT-NLS|((msdos|vms)/.*)|ChangeLog-2014-12-10$$

configure.ac

@@ -57,48 +57,104 @@ AC_CANONICAL_HOST
 AC_DEFINE_UNQUOTED([OS_TYPE], "$host_os",
                    [Define to be the name of the operating system.])
 
+
 dnl
-dnl Process features.
+dnl Process External Libraries
 dnl
 
-ENABLE_PSL=no
-AC_ARG_WITH(libpsl,
-    AS_HELP_STRING([--without-libpsl],
-                   [disable support for libpsl cookie checking.]),
-    [],
-    [AC_SEARCH_LIBS(psl_builtin, psl,
-                    [ENABLE_PSL=yes; AC_DEFINE([HAVE_LIBPSL], [1], [PSL Support Enabled])],
-                    [AC_MSG_WARN(*** libpsl not found. Falling back to Wget builtin cookie checking.)])
-    ])
+dnl Libpsl: Public Suffix List checking
+AC_ARG_WITH([libpsl],
+  [AS_HELP_STRING([--without-libpsl], [disable support for libpsl cookie checking.])])
 
-AC_ARG_WITH(ssl,
-[[  --without-ssl           disable SSL autodetection
-  --with-ssl={gnutls,openssl} specify the SSL backend.  GNU TLS is the default.]])
+dnl SSL: Configure SSL backend to use
+AC_ARG_WITH([ssl],
+  [AS_HELP_STRING([--with-ssl={gnutls,openssl}], [specify SSL backend. GNU TLS is the default.])])
 
-AC_ARG_WITH(zlib,
-[[  --without-zlib          disable zlib ]])
 
-AC_ARG_ENABLE(opie,
-[  --disable-opie          disable support for opie or s/key FTP login],
-ENABLE_OPIE=$enableval, ENABLE_OPIE=yes)
-test x"${ENABLE_OPIE}" = xyes && AC_DEFINE([ENABLE_OPIE], 1,
-   [Define if you want the Opie support for FTP compiled in.])
+dnl Zlib: Configure use of zlib for compression
+AC_ARG_WITH([zlib],
+  [AS_HELP_STRING([--without-zlib], [disable zlib.])])
 
-AC_ARG_ENABLE(digest,
-[  --disable-digest        disable support for HTTP digest authorization],
-ENABLE_DIGEST=$enableval, ENABLE_DIGEST=yes)
-test x"${ENABLE_DIGEST}" = xyes && AC_DEFINE([ENABLE_DIGEST], 1,
-   [Define if you want the HTTP Digest Authorization compiled in.])
+dnl Metalink: Configure use of the Metalink library
+AC_ARG_WITH([metalink],
+  [AS_HELP_STRING([--with-metalink], [enable support for metalinks.])])
 
-AC_ARG_ENABLE(ntlm,
-[  --disable-ntlm          disable support for NTLM authorization],
-[ENABLE_NTLM=$enableval], [ENABLE_NTLM=auto])
+dnl
+dnl Process features
+dnl
 
-AC_ARG_ENABLE(debug,
-[  --disable-debug         disable support for debugging output],
-ENABLE_DEBUG=$enableval, ENABLE_DEBUG=yes)
-test x"${ENABLE_DEBUG}" = xyes && AC_DEFINE([ENABLE_DEBUG], 1,
-   [Define if you want the debug output support compiled in.])
+dnl Opie: Support for opie s/key FTP logins
+AC_ARG_ENABLE([opie],
+  [AS_HELP_STRING([--disable-opie], [disable support for opie or s/key FTP login])],
+  [ENABLE_OPIE=$enableval],
+  [ENABLE_OPIE=yes])
+
+AS_IF([test "x$ENABLE_OPIE" = xyes],
+  [AC_DEFINE([ENABLE_OPIE], [1], [Define if you want Opie support for FTP compiled in.])],
+  []
+)
+
+
+dnl Digest: Support for HTTP Digest Authentication
+AC_ARG_ENABLE([digest],
+  [AS_HELP_STRING([--disable-digest], [disable support for HTTP digest authorization])],
+  [ENABLE_DIGEST=$enableval],
+  [ENABLE_DIGEST=yes])
+
+AS_IF([test "x$ENABLE_DIGEST" = xyes],
+  [AC_DEFINE([ENABLE_DIGEST], [1], [Define if you want the HTTP Digest Authorization compiled in.])],
+  []
+)
+
+
+dnl NTLM: Support for HTTP NTLM Authentication
+AC_ARG_ENABLE([ntlm],
+  [AS_HELP_STRING([--disable-ntlm], [disable support for NTLM authorization])],
+  [ENABLE_NTLM=$enableval],
+  [ENABLE_NTLM=auto]
+)
+
+
+dnl Debug: Support for printing debugging output
+AC_ARG_ENABLE([debug],
+  [AS_HELP_STRING([--disable-debug], [disable support for debugging output])],
+  [ENABLE_DEBUG=$enableval],
+  [ENABLE_DEBUG=yes])
+
+AS_IF([test "x$ENABLE_DEBUG" = xyes],
+  [AC_DEFINE([ENABLE_DEBUG], [1], [Define if you want the debug output support compiled in.])],
+  []
+)
+
+dnl Valgrind-tests: Should test suite be run under valgrind?
+AC_ARG_ENABLE(valgrind-tests,
+  [AS_HELP_STRING([--enable-valgrind-tests], [enable using Valgrind for tests])],
+  [ENABLE_VALGRIND=$enableval],
+  [ENABLE_VALGRIND=no])
+
+AS_IF([test "x$ENABLE_VALGRIND" != xno], [
+  AC_CHECK_PROG(HAVE_VALGRIND, valgrind, yes, no)
+  AS_IF([test "x$HAVE_VALGRIND" = xyes], [
+    VALGRIND_TESTS="1"
+    AC_SUBST(VALGRIND_TESTS)
+    VALGRIND_INFO="Test suite will be run under Valgrind"
+  ], [
+    VALGRIND_INFO="Valgrind not found"
+  ])
+], [
+  VALGRIND_INFO="Valgrind testing not enabled"
+])
+
+AC_ARG_ENABLE(assert,
+  [AS_HELP_STRING([--enable-assert], [enable assertions in code base])],
+  [ENABLE_ASSERTION=$enableval],
+  [ENABLE_ASSERTION=no]
+)
+
+AS_IF([test "x$ENABLE_ASSERTION" != xyes], [
+  CFLAGS="-DNDEBUG $CFLAGS"
+  ],
+  [])
 
 dnl
 dnl Find the compiler
@@ -114,11 +170,17 @@ AC_AIX
 
 gl_EARLY
 
+dnl
+dnl Find python3
+dnl
+AM_PATH_PYTHON([3.0],,[:])
+AM_CONDITIONAL([HAVE_PYTHON3], [test "$PYTHON" != :])
+
 dnl
 dnl Gettext
 dnl
 AM_GNU_GETTEXT([external],[need-ngettext])
-AM_GNU_GETTEXT_VERSION([0.17])
+AM_GNU_GETTEXT_VERSION([0.18.1])
 
 AC_PROG_RANLIB
 
@@ -131,9 +193,11 @@ dnl   if compiler is gcc, then
 dnl     use -O2 and some warning flags
 dnl   else
 dnl     use os-specific flags or -O
+dnl
+dnl For these tests, the system considers clang to be gcc
 if test -n "$auto_cflags"; then
   if test -n "$GCC"; then
-    CFLAGS="$CFLAGS -O2 -Wall"
+    CFLAGS="$CFLAGS -O2 -Wall -Wextra"
   else
     case "$host_os" in
       *hpux*)  CFLAGS="$CFLAGS +O3"                      ;;
@@ -207,7 +271,7 @@ AC_FUNC_MMAP
 AC_FUNC_FSEEKO
 AC_CHECK_FUNCS(strptime timegm vsnprintf vasprintf drand48 pathconf)
 AC_CHECK_FUNCS(strtoll usleep ftello sigblock sigsetjmp memrchr wcwidth mbtowc)
-AC_CHECK_FUNCS(sleep symlink utime)
+AC_CHECK_FUNCS(sleep symlink utime strlcpy random)
 
 if test x"$ENABLE_OPIE" = xyes; then
   AC_LIBOBJ([ftp-opie])
@@ -244,48 +308,84 @@ dnl
 dnl Checks for libraries.
 dnl
 
+PKG_PROG_PKG_CONFIG
+
+AS_IF([test "x$with_libpsl" != xno], [
+  PKG_CHECK_MODULES([LIBPSL], libpsl, [
+    with_libpsl=yes
+    # correct $LIBPSL_LIBS (in libpsl <= 0.6.0)
+    AS_IF([test "x$LIBPSL_LIBS" = "x-llibpsl "], [LIBPSL_LIBS="-lpsl"])
+    LIBS="$LIBPSL_LIBS $LIBS"
+    CFLAGS="$LIBPSL_CFLAGS $CFLAGS"
+    AC_DEFINE([HAVE_LIBPSL], [1], [PSL support enabled])
+  ], [
+    AC_SEARCH_LIBS(psl_builtin, psl,
+      [with_libpsl=yes; AC_DEFINE([HAVE_LIBPSL], [1], [PSL support enabled])],
+      [with_libpsl=no;  AC_MSG_WARN(*** libpsl was not found. Fallback to builtin cookie checking.)])
+  ])
+])
+
 AS_IF([test x"$with_zlib" != xno], [
   with_zlib=yes
-  AC_CHECK_LIB(z, compress)
+  PKG_CHECK_MODULES([ZLIB], zlib, [
+    LIBS="$ZLIB_LIBS $LIBS"
+    CFLAGS="$ZLIB_CFLAGS $CFLAGS"
+    AC_DEFINE([HAVE_LIBZ], [1], [Define if using zlib.])
+  ], [
+    AC_CHECK_LIB(z, compress)
+  ])
 ])
 
 AS_IF([test x"$with_ssl" = xopenssl], [
-  dnl As of this writing (OpenSSL 0.9.6), the libcrypto shared library
-  dnl doesn't record its dependency on libdl, so we need to make sure
-  dnl -ldl ends up in LIBS on systems that have it.  Most OSes use
-  dnl dlopen(), but HP-UX uses shl_load().
-  AC_CHECK_LIB(dl, dlopen, [], [
-    AC_CHECK_LIB(dl, shl_load)
-  ])
+  if [test x"$with_libssl_prefix" = x]; then
+    PKG_CHECK_MODULES([OPENSSL], [openssl], [
+      AC_MSG_NOTICE([compiling in support for SSL via OpenSSL])
+      AC_LIBOBJ([openssl])
+      LIBS="$OPENSSL_LIBS $LIBS"
+      CFLAGS="$OPENSSL_CFLAGS -DHAVE_LIBSSL $CFLAGS"
+      LIBSSL=" " # ntlm check below wants this
+      AC_CHECK_FUNCS([RAND_egd])
+      AC_DEFINE([HAVE_LIBSSL], [1], [Define if using openssl.])
+      ssl_found=yes
+    ])
+  fi
+  if [test x"$ssl_found" != xyes]; then
+    dnl As of this writing (OpenSSL 0.9.6), the libcrypto shared library
+    dnl doesn't record its dependency on libdl, so we need to make sure
+    dnl -ldl ends up in LIBS on systems that have it.  Most OSes use
+    dnl dlopen(), but HP-UX uses shl_load().
+    AC_CHECK_LIB(dl, dlopen, [], [
+      AC_CHECK_LIB(dl, shl_load)
+    ])
 
-  ssl_found=no
-  case $host_os in
-    *mingw32* )
-      dnl prefer link to openssl dlls if possible. if not then fallback on static libs. if not then error
+    ssl_found=no
+    case $host_os in
+      *mingw32* )
+        dnl prefer link to openssl dlls if possible. if not then fallback on static libs. if not then error
 
-      AC_CHECK_LIB(eay32, EVP_MD_CTX_init)
-      if test x"$ac_cv_lib_eay32_EVP_MD_CTX_init" != xno
-      then
-        AC_CHECK_LIB(ssl32, SSL_connect, [
-          ssl_found=yes
-          AC_MSG_NOTICE([Enabling support for SSL via OpenSSL (shared)])
-          AC_LIBOBJ([openssl])
-          LIBS="${LIBS} -lssl32"
-          AC_DEFINE([HAVE_LIBSSL32], [1], [Define to 1 if you have the `ssl32' library (-lssl32).])
-        ],
-        AC_MSG_ERROR([openssl not found: shared lib eay32 found but ssl32 not found]))
+        AC_CHECK_LIB(eay32, EVP_MD_CTX_init)
+        if test x"$ac_cv_lib_eay32_EVP_MD_CTX_init" != xno
+        then
+          AC_CHECK_LIB(ssl32, SSL_connect, [
+            ssl_found=yes
+            AC_MSG_NOTICE([Enabling support for SSL via OpenSSL (shared)])
+            AC_LIBOBJ([openssl])
+            LIBS="${LIBS} -lssl32"
+            AC_DEFINE([HAVE_LIBSSL32], [1], [Define to 1 if you have the `ssl32' library (-lssl32).])
+          ],
+          AC_MSG_ERROR([openssl not found: shared lib eay32 found but ssl32 not found]))
 
-      else
-        LIBS+=' -lgdi32'
-        dnl fallback and test static libs
-      fi
-      dnl add zdll lib as dep for above tests?
-    ;;
-  esac
+        else
+          LIBS+=' -lgdi32'
+          dnl fallback and test static libs
+        fi
+        dnl add zdll lib as dep for above tests?
+      ;;
+    esac
 
-  AS_IF([test x$ssl_found != xyes], [
-    dnl Now actually check for -lssl if it wasn't already found
-    AC_LIB_HAVE_LINKFLAGS([ssl], [crypto], [
+    AS_IF([test x$ssl_found != xyes], [
+      dnl Now actually check for -lssl if it wasn't already found
+      AC_LIB_HAVE_LINKFLAGS([ssl], [crypto], [
 #include <openssl/ssl.h>
 #include <openssl/x509.h>
 #include <openssl/err.h>
@@ -293,19 +393,20 @@ AS_IF([test x"$with_ssl" = xopenssl], [
 #include <openssl/des.h>
 #include <openssl/md4.h>
 #include <openssl/md5.h>
-    ], [SSL_library_init ()])
-    if test x"$LIBSSL" != x
-    then
-      ssl_found=yes
-      AC_MSG_NOTICE([compiling in support for SSL via OpenSSL])
-      AC_LIBOBJ([openssl])
-      LIBS="$LIBSSL $LIBS"
-    elif test x"$with_ssl" != x
-    then
-      AC_MSG_ERROR([--with-ssl=openssl was given, but SSL is not available.])
-    fi
-  ])
-
+      ], [SSL_library_init ()])
+      if test x"$LIBSSL" != x
+      then
+        ssl_found=yes
+        AC_MSG_NOTICE([compiling in support for SSL via OpenSSL])
+        AC_LIBOBJ([openssl])
+        LIBS="$LIBSSL $LIBS"
+        AC_CHECK_FUNCS([RAND_egd])
+      elif test x"$with_ssl" != x
+      then
+        AC_MSG_ERROR([--with-ssl=openssl was given, but SSL is not available.])
+      fi
+    ])
+  fi
 ], [
   # --with-ssl is not openssl: check if it's no
   AS_IF([test x"$with_ssl" != xno], [
@@ -313,17 +414,29 @@ AS_IF([test x"$with_ssl" = xopenssl], [
     with_ssl=gnutls
 
     dnl Now actually check for -lgnutls
-    AC_LIB_HAVE_LINKFLAGS([gnutls], [], [
+    if [test x"$with_libgnutls_prefix" = x]; then
+      PKG_CHECK_MODULES([GNUTLS], [gnutls], [
+        AC_MSG_NOTICE([compiling in support for SSL via GnuTLS])
+        AC_LIBOBJ([gnutls])
+        LIBS="$GNUTLS_LIBS $LIBS"
+        CFLAGS="$GNUTLS_CFLAGS -DHAVE_LIBGNUTLS $CFLAGS"
+        AC_DEFINE([HAVE_LIBGNUTLS], [1], [Define if using gnutls.])
+        ssl_found=yes
+      ])
+    fi
+    if [test x"$ssl_found" != xyes]; then
+      AC_LIB_HAVE_LINKFLAGS([gnutls], [], [
 #include <gnutls/gnutls.h>
-    ], [gnutls_global_init()])
-    if test x"$LIBGNUTLS" != x
-    then
-      ssl_found=yes
-      AC_MSG_NOTICE([compiling in support for SSL via GnuTLS])
-      AC_LIBOBJ([gnutls])
-      LIBS="$LIBGNUTLS $LIBS"
-    else
-      AC_MSG_ERROR([--with-ssl=gnutls was given, but GNUTLS is not available.])
+      ], [gnutls_global_init()])
+      if test x"$LIBGNUTLS" != x
+      then
+        ssl_found=yes
+        AC_MSG_NOTICE([compiling in support for SSL via GnuTLS])
+        AC_LIBOBJ([gnutls])
+        LIBS="$LIBGNUTLS $LIBS"
+      else
+        AC_MSG_ERROR([GnuTLS has not been found. Use --with-ssl=openssl if you explicitly want OpenSSL.])
+      fi
     fi
 
     AC_CHECK_FUNCS(gnutls_priority_set_direct)
@@ -365,6 +478,34 @@ else
   fi
 fi
 
+dnl
+dnl Check for libmetalink
+dnl
+AS_IF([test x"$with_metalink" != xno], [
+  PKG_CHECK_MODULES([METALINK], libmetalink, [
+      LIBS="$METALINK_LIBS $LIBS"
+      CFLAGS="$METALINK_CFLAGS $CFLAGS"
+      AC_DEFINE([HAVE_METALINK], [1], [Define if using metalink.])
+      with_metalink=yes
+    ], [
+      with_metalink=no
+    ])
+
+  have_gpg=no
+  AS_IF([test x"$with_metalink" = xyes], [
+    dnl
+    dnl Check for GPGME
+    dnl
+    m4_ifdef([AM_PATH_GPGME], [
+      AM_PATH_GPGME([], [
+        LIBS="$GPGME_LIBS $LIBS"
+        CFLAGS="$GPGME_CFLAGS $CFLAGS"
+        AC_DEFINE([HAVE_GPGME], [1], [Define if GPGME is available.])
+        have_gpg=yes
+      ])
+    ])
+  ])
+])
 
 dnl **********************************************************************
 dnl Checks for IPv6
@@ -542,43 +683,76 @@ fi
 dnl
 dnl Check for UUID
 dnl
-AC_CHECK_HEADER(uuid.h,
-                AC_CHECK_FUNC(uuid, uuid_create,
-                  [AC_DEFINE([HAVE_UUID_CREATE], 1,
-                             [Define if uuid_create is available.])
-                  ])
-)
 
-AC_CHECK_HEADER(uuid/uuid.h,
-                AC_CHECK_LIB(uuid, uuid_generate,
-                  [LIBS="${LIBS} -luuid"
-                   AC_DEFINE([HAVE_LIBUUID], 1,
-                             [Define if libuuid is available.])
-                  ])
-)
+AC_ARG_WITH(libuuid, AC_HELP_STRING([--without-libuuid],
+                                    [Generate UUIDs for WARC files via libuuid]))
+
+AS_IF([test "x$with_libuuid" = xyes], [
+  # libuuid was explicitly requested
+  PKG_CHECK_MODULES([UUID], uuid, [
+    LIBS="$UUID_LIBS $LIBS"
+    CFLAGS="$UUID_CFLAGS $CFLAGS"
+    uuid_mode=1
+  ], [
+    AC_SEARCH_LIBS(uuid_generate, uuid,
+      [uuid_mode=1],
+      [AC_MSG_ERROR(*** libuuid was explicitly requested but wasn't found.)])
+  ])
+], [test "x$with_libuuid" = xno], [
+  # libuuid was explicitly *not* requested
+  AC_CHECK_HEADER(uuid.h,
+    AC_CHECK_FUNC(uuid_create, [uuid_mode=2]))
+], [
+  # default:
+  AC_CHECK_HEADER(uuid.h, [
+    AC_CHECK_FUNC(uuid_create, [uuid_mode=2])
+  ], [
+    AC_CHECK_HEADER(uuid/uuid.h,
+      AC_SEARCH_LIBS(uuid_generate, uuid, [uuid_mode=1]))
+  ])
+])
+
+AS_IF([test "x$uuid_mode" = x1], [
+  AC_DEFINE([HAVE_LIBUUID], [1], [Define if using libuuid.])
+], [test "x$uuid_mode" = x2], [
+  AC_DEFINE([HAVE_UUID_CREATE], 1, [Define if uuid_create is available.])
+])
 
 dnl
 dnl Check for PCRE
 dnl
 
-AC_CHECK_HEADER(pcre.h,
-                AC_CHECK_LIB(pcre, pcre_compile,
-                  [LIBS="${LIBS} -lpcre"
-                   AC_DEFINE([HAVE_LIBPCRE], 1,
-                             [Define if libpcre is available.])
-                  ])
-)
+AC_ARG_ENABLE(pcre, AC_HELP_STRING([--disable-pcre],
+                                   [Disable PCRE style regular expressions]))
+
+AS_IF([test "X$enable_pcre" != "Xno"],[
+  PKG_CHECK_MODULES([PCRE], libpcre, [
+    CFLAGS="$PCRE_CFLAGS $CFLAGS"
+    AC_CHECK_HEADER(pcre.h, [
+      LIBS="$PCRE_LIBS $LIBS"
+      AC_DEFINE([HAVE_LIBPCRE], [1], [Define if libpcre is available.])
+    ])
+  ], [
+    AC_CHECK_HEADER(pcre.h, [
+      AC_CHECK_LIB(pcre, pcre_compile, [
+        LIBS="${LIBS} -lpcre"
+        AC_DEFINE([HAVE_LIBPCRE], 1, [Define if libpcre is available.])
+      ])
+    ])
+  ])
+])
 
 
 dnl Needed by src/Makefile.am
 AM_CONDITIONAL([IRI_IS_ENABLED], [test "X$iri" != "Xno"])
-
+AM_CONDITIONAL([WITH_SSL], [test "X$with_ssl" != "Xno"])
+AM_CONDITIONAL([METALINK_IS_ENABLED], [test "X$with_metalink" != "Xno"])
 
 dnl
 dnl Create output
 dnl
 AC_CONFIG_FILES([Makefile src/Makefile doc/Makefile util/Makefile
-                 po/Makefile.in tests/Makefile tests/WgetTest.pm
+                 po/Makefile.in tests/Makefile
                  lib/Makefile testenv/Makefile])
 AC_CONFIG_HEADERS([src/config.h])
 AC_OUTPUT
@@ -594,9 +768,13 @@ AC_MSG_NOTICE([Summary of build options:
   Libs:              $LIBS
   SSL:               $with_ssl
   Zlib:              $with_zlib
-  PSL:               $ENABLE_PSL
+  PSL:               $with_libpsl
   Digest:            $ENABLE_DIGEST
   NTLM:              $ENABLE_NTLM
   OPIE:              $ENABLE_OPIE
   Debugging:         $ENABLE_DEBUG
+  Assertions:        $ENABLE_ASSERTION
+  Valgrind:          $VALGRIND_INFO
+  Metalink:          $with_metalink
+  GPGME:             $have_gpg
 ])

contrib/check-hard

@@ -0,0 +1,54 @@
+#!/bin/sh
+#
+# Do some checking before 'git push'.
+
+# Set a stricter bash mode
+set -e
+set -u
+
+CORES=$(grep -c ^processor /proc/cpuinfo 2> /dev/null || echo 0)
+if [ $CORES -lt 1 ]; then CORES=1; fi
+
+echo "Running: make distclean"
+make distclean > /dev/null || true
+
+CFLAGS="-std=c89 -pedantic -O2 -g -Wall -Wextra -Wstrict-prototypes -Wold-style-definition -Wwrite-strings -Wshadow -Wformat -Wformat-security -Wunreachable-code -Wstrict-prototypes -Wmissing-prototypes -Wold-style-definition"
+
+CACHEFILE=$PWD/config_check.cache
+
+# measure time consumed and print it at the end of the script
+START=$(date +%s.%N)
+
+echo "Running: ./bootstrap"
+./bootstrap > /dev/null
+echo
+
+for CC in gcc clang; do
+  export CC
+  echo "*** Testing with CC=$CC"
+
+  # the compiler changed, so we have to remove the cache file here
+  rm -f $CACHEFILE
+
+  for options in "" "--with-ssl=openssl"; do
+    export DISTCHECK_CONFIGURE_FLAGS="-C --cache-file=$CACHEFILE $options"
+    echo "  ./configure $DISTCHECK_CONFIGURE_FLAGS"
+    ./configure $DISTCHECK_CONFIGURE_FLAGS CFLAGS="$CFLAGS" > /dev/null
+
+    for xVALGRIND in 0 1; do
+      for xLCALL in C tr_TR.utf8; do
+        export TESTS_ENVIRONMENT="LC_ALL=$xLCALL VALGRIND_TESTS=$xVALGRIND"
+        echo "    TESTS_ENVIRONMENT=\"$TESTS_ENVIRONMENT\"" make check -j$CORES
+        make check -j$CORES > /dev/null
+      done
+    done
+
+    unset TESTS_ENVIRONMENT
+    export TESTS_ENVIRONMENT
+    echo "  make distcheck CFLAGS=$CFLAGS -j$CORES"
+    make distcheck CFLAGS="$CFLAGS" -j$CORES > /dev/null
+  done
+done
+
+END=$(date +%s.%N)
+echo "Duration: "$(echo "$END - $START" | bc)

contrib/travis-ci

@@ -0,0 +1,60 @@
+#!/bin/bash
+
+# Set a stricter bash mode
+set -e
+set -u
+
+# Set up the subset of tests this Matrix instance will run.
+
+# One instance is forked for each of the SSL library possibilities
+SSL=${1:-""}
+case $SSL in
+    "") SSL_LIB="--without-ssl";;
+    "openssl") SSL_LIB="--with-ssl=openssl";;
+    "gnutls") SSL_LIB="--with-ssl=gnutls";;
+esac
+
+# For some reason. /proc/cpuinfo reports 16 cores on Travis, while the docs
+# claim that each instance has only 2 cores. We believe the docs and force a
+# value of n+1 here.
+CORES=3
+
+# Define a large number of Warning flags for the compiler. Hopefully, someone
+# will sit and analyze the output to clean the warnings from the codebase.
+CFLAGS="-std=c89 -pedantic -O2 -Wall -Wextra -Wstrict-prototypes -Wold-style-definition -Wwrite-strings -Wshadow -Wformat -Wformat-security -Wunreachable-code -Wstrict-prototypes -Wmissing-prototypes -Wold-style-definition"
+
+# A cachefile for ./configure. Saves significant time when reconfiguring
+CACHEFILE=$PWD/config_check.cache
+
+# measure time consumed and print it at the end of the script
+START=$(date +%s)
+
+# Test different Permutations of Wget. Always under Valgrind
+#
+# Options:
+#   - All Features Enabled
+#     - C       [Default Locale]
+#     - Russian [Multibyte / Multicolumn Locale]
+#     - Turkish [Special Test due to issues with language]
+#   - IPv6 Disabled
+#     - C
+for options in "" "--disable-ipv6"; do
+  export DISTCHECK_CONFIGURE_FLAGS="-C --cache-file=$CACHEFILE --enable-assert --enable-valgrind-tests $SSL_LIB $options"
+  echo "  ./configure $DISTCHECK_CONFIGURE_FLAGS CFLAGS=\"$CFLAGS\""
+  ./configure $DISTCHECK_CONFIGURE_FLAGS CFLAGS="$CFLAGS"
+
+  if [[ -z $options ]]; then
+      LANG_ARR=("ru_RU.UTF-8" "tr_TR.UTF-8" "C")
+  else
+      LANG_ARR=("C")
+  fi
+
+  for xLCALL in "${LANG_ARR[@]}"; do
+    export TESTS_ENVIRONMENT="LC_ALL=$xLCALL VALGRIND_TESTS=1"
+    echo "    TESTS_ENVIRONMENT=\"$TESTS_ENVIRONMENT\" make distcheck CFLAGS=$CFLAGS -j$CORES"
+    make distcheck CFLAGS="$CFLAGS" -j$CORES
+  done
+done
+
+END=$(date +%s)
+echo "Duration: $((END-START))"

contrib/tsocked-wget

@@ -0,0 +1,90 @@
+#!/bin/bash
+
+# Script that executes wget using a socks proxy if the environment variable
+# socks_proxy is set.
+#
+# The socks_proxy variable shall have one of the forms:
+#   socks://username:password@host:port
+#   socks4://username:password@host:port
+#   socks5://username:password@host:port
+# with username, password and port fields being optional
+#
+# As socksification applies to the whole process, domains defined in the
+# no_proxy setting are *not* excluded.
+#
+
+# Requisites: bash (version 3.2.3 or later), tsocks and wget
+
+####################################
+#  Copyright (C) 2014 Ángel González
+#  Permission is hereby granted, free of charge, to any person obtaining a copy
+#  of this software and associated documentation files (the "Software"), to deal
+#  in the Software without restriction, including without limitation the rights
+#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+#  copies of the Software, and to permit persons to whom the Software is
+#  furnished to do so, subject to the following conditions:
+#
+#  The above copyright notice and this permission notice shall be included in
+#  all copies or substantial portions of the Software.
+#
+#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+#  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+#  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+#  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+#  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+#  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+#  THE SOFTWARE.
+#
+
+set -eu
+
+# Binary to be executed. Use an absolute path if installing this script as 'wget'
+WGET=wget
+
+if [ -z "${socks_proxy:-}" ]; then
+exec "$WGET" "$@"
+fi
+
+CONFIG=""
+
+if [[ "${socks_proxy}" =~ ^socks[45]?:// ]]; then
+ if [[ "${socks_proxy:5:1}" != ":" ]]; then
+  CONFIG+="server_type = ${socks_proxy:5:1}"
+  socks_proxy="${socks_proxy:9}"
+ else
+  socks_proxy="${socks_proxy:8}"
+ fi
+elif [[ "${socks_proxy}" =~ ^[[:alnum:]]*:// ]]; then
+ echo "Bad value specified for socks_proxy: $socks_proxy" >&2
+ exit 2
+fi
+
+if [[ "${socks_proxy}" =~ ^([^@:]*)(:([^@]*))?@ ]]; then
+ unset TSOCKS_USERNAME
+ CONFIG+="
+   default_user = ${BASH_REMATCH[1]}"
+
+ if [ ! -z "${BASH_REMATCH[3]}" ]; then
+  unset TSOCKS_PASSWORD
+  CONFIG+="
+   default_pass = ${BASH_REMATCH[3]}"
+ fi
+ socks_proxy="${socks_proxy:${#BASH_REMATCH[0]}}"
+fi
+
+
+# Get rid of trailing slashes
+if [[ "${socks_proxy}" =~ ^([^/]*)/ ]]; then
+ socks_proxy="${socks_proxy:0:${#BASH_REMATCH[1]}}"
+fi
+
+if [[ "${socks_proxy}" =~ :([0-9]+)$ ]]; then
+  CONFIG+="
+   server_port = ${BASH_REMATCH[1]}"
+  socks_proxy=${socks_proxy:0:${#socks_proxy} - ${#BASH_REMATCH[0]}}
+fi
+
+CONFIG+="
+   server = ${socks_proxy}"
+
+TSOCKS_CONF_FILE=<(echo "$CONFIG") exec tsocks "$WGET" --no-proxy "$@"

doc/Makefile.am

@@ -1,6 +1,6 @@
 # Makefile for `wget' utility
-# Copyright (C) 1995, 1996, 1997, 2007, 2008, 2009, 2010, 2011 Free
-# Software Foundation, Inc.
+# Copyright (C) 1995, 1996, 1997, 2007, 2008, 2009, 2010, 2011, 2015
+# Free Software Foundation, Inc.
 
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -16,7 +16,7 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 #
-# Version: $(VERSION)
+# Version: @VERSION@
 #
 
 # Program to convert DVI files to PostScript
@@ -28,6 +28,7 @@ manext     = 1
 RM = rm -f
 
 TEXI2POD     = $(srcdir)/texi2pod.pl
+POD2MAN      = @POD2MAN@
 MAN          = wget.$(manext)
 WGETRC       = $(sysconfdir)/wgetrc
 SAMPLERCTEXI = sample.wgetrc.munged_for_texi_inclusion
@@ -38,7 +39,7 @@ SAMPLERCTEXI = sample.wgetrc.munged_for_texi_inclusion
 
 man_MANS = $(MAN)
 
-all: wget.info $(COMMENT_IF_NO_POD2MAN)$(MAN)
+all: wget.info @COMMENT_IF_NO_POD2MAN@$(MAN)
 
 everything: all wget_us.ps wget_a4.ps wget_toc.html
 
@@ -56,7 +57,7 @@ wget.pod: $(srcdir)/wget.texi version.texi
 	$(TEXI2POD) -D VERSION="$(VERSION)" $(srcdir)/wget.texi $@
 
 $(MAN): wget.pod
-	$(POD2MAN) --center="GNU Wget" --release="GNU Wget $(VERSION)" $? > $@
+	$(POD2MAN) --center="GNU Wget" --release="GNU Wget @VERSION@" --utf8 $? > $@
 
 #wget.cat: $(MAN)
 #	nroff -man $? > $@
@@ -75,10 +76,10 @@ wget_toc.html: $(srcdir)/wget.texi
 #
 
 # install all the documentation
-install-data-local: install.wgetrc $(COMMENT_IF_NO_POD2MAN)install.man
+install-data-local: install.wgetrc @COMMENT_IF_NO_POD2MAN@install.man
 
 # uninstall all the documentation
-uninstall-local: $(COMMENT_IF_NO_POD2MAN)uninstall.man
+uninstall-local: @COMMENT_IF_NO_POD2MAN@uninstall.man
 
 
 # install man page, creating install directory if necessary

doc/fdl.texi

@@ -5,7 +5,8 @@
 @c hence no sectioning command or @node.
 
 @display
-Copyright @copyright{} 2000, 2001, 2002, 2007, 2008 Free Software Foundation, Inc.
+Copyright @copyright{} 2000, 2001, 2002, 2007, 2008, 2015 Free Software
+Foundation, Inc.
 @uref{http://fsf.org/}
 
 Everyone is permitted to copy and distribute verbatim copies

doc/sample.wgetrc

@@ -16,6 +16,9 @@
 ## To use the settings in this file, you will have to uncomment them,
 ## as well as change them, in most cases, as the values on the
 ## commented-out lines are the default values (e.g. "off").
+##
+## Command are case-, underscore- and minus-insensitive.
+## For example ftp_proxy, ftp-proxy and ftpproxy are the same.
 
 
 ##
@@ -122,7 +125,7 @@
 #iri = off
 
 # Force the default system encoding
-#locale = UTF-8
+#localencoding = UTF-8
 
 # Force the default remote server encoding
 #remoteencoding = UTF-8

doc/texi2pod.pl

@@ -1,7 +1,7 @@
 #! /usr/bin/env perl
 
-#   Copyright (C) 1999, 2000, 2001, 2003, 2007, 2009, 2010, 2011 Free
-#   Software Foundation, Inc.
+#   Copyright (C) 1999, 2000, 2001, 2003, 2007, 2009, 2010, 2011, 2015
+#   Free Software Foundation, Inc.
 
 # This file is part of GCC.
 
@@ -315,6 +315,8 @@ die "No filename or title\n" unless defined $fn && defined $tl;
 $sects{NAME} = "$fn \- $tl\n";
 $sects{FOOTNOTES} .= "=back\n" if exists $sects{FOOTNOTES};
 
+print "=encoding utf-8\n\n";
+
 for $sect (qw(NAME SYNOPSIS DESCRIPTION OPTIONS ENVIRONMENT EXITSTATUS
            FILES BUGS NOTES FOOTNOTES SEEALSO AUTHOR COPYRIGHT)) {
     if(exists $sects{$sect}) {

doc/wget.texi

@@ -2,6 +2,7 @@
 
 @c %**start of header
 @setfilename wget.info
+@documentencoding UTF-8
 @include version.texi
 @settitle GNU Wget @value{VERSION} Manual
 @c Disable the monstrous rectangles beside overfull hbox-es.
@@ -31,8 +32,8 @@ data.
 
 @c man begin COPYRIGHT
 Copyright @copyright{} 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003,
-2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011 Free Software Foundation,
-Inc.
+2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2015 Free Software
+Foundation, Inc.
 
 @iftex
 Permission is granted to make and distribute verbatim copies of
@@ -59,11 +60,11 @@ Texts.  A copy of the license is included in the section entitled
 @title GNU Wget @value{VERSION}
 @subtitle The non-interactive download utility
 @subtitle Updated for Wget @value{VERSION}, @value{UPDATED}
-@author by Hrvoje Nik@v{s}i@'{c} and others
+@author by Hrvoje Nikšić and others
 
 @ignore
 @c man begin AUTHOR
-Originally written by Hrvoje Niksic <hniksic@xemacs.org>.
+Originally written by Hrvoje Nikšić <hniksic@xemacs.org>.
 @c man end
 @c man begin SEEALSO
 This is @strong{not} the complete manual for GNU Wget.
@@ -506,6 +507,23 @@ treated as @samp{html} if the Content-Type matches @samp{text/html}.
 Furthermore, the @var{file}'s location will be implicitly used as base
 href if none was specified.
 
+@cindex input-metalink
+@item --input-metalink=@var{file}
+Downloads files covered in local Metalink @var{file}. Metalink version 3
+and 4 are supported.
+
+@cindex metalink-over-http
+@item --metalink-over-http
+Issues HTTP HEAD request instead of GET and extracts Metalink metadata
+from response headers. Then it switches to Metalink download.
+If no valid Metalink metadata is found, it falls back to ordinary HTTP download.
+
+@cindex preferred-location
+@item --preferred-location
+Set preferred location for Metalink resources. This has effect if multiple
+resources with same priority are available.
+
+
 @cindex force html
 @item -F
 @itemx --force-html
@@ -533,6 +551,11 @@ would be resolved to @samp{http://foo/baz/b.html}.
 @cindex specify config 
 @item --config=@var{FILE}
 Specify the location of a startup file you wish to use.
+
+@item --rejected-log=@var{logfile}
+Logs all URL rejections to @var{logfile} as comma separated values.  The values
+include the reason of rejection, the URL and the parent URL it was found in.
+
 @end table
 
 @node Download Options, Directory Options, Logging and Input File Options, Invoking
@@ -585,6 +608,9 @@ content will be placed in @var{file}. This was disabled in version
 1.11, but has been reinstated (with a warning) in 1.11.2, as there are
 some cases where this behavior can actually have some use.
 
+A combination with @samp{-nc} is only accepted if the given output
+file does not exist.
+
 Note that a combination with @samp{-k} is only permitted when
 downloading a single document, as in that case it will just convert
 all relative URIs to external ones; @samp{-k} makes no sense for
@@ -626,6 +652,9 @@ of a file depends on the local and remote timestamp and size of the
 file (@pxref{Time-Stamping}).  @samp{-nc} may not be specified at the
 same time as @samp{-N}.
 
+A combination with @samp{-O}/@samp{--output-document} is only accepted
+if the given output file does not exist.
+
 Note that when @samp{-nc} is specified, files with the suffixes
 @samp{.html} or @samp{.htm} will be loaded from the local disk and
 parsed as if they had been retrieved from the Web.
@@ -774,16 +803,23 @@ use @samp{--progress=bar:force:noscroll}.
 Force wget to display the progress bar in any verbosity.
 
 By default, wget only displays the progress bar in verbose mode.  One may
-however want wget to display the progress bar on screen in conjunction with
+however, want wget to display the progress bar on screen in conjunction with
 any other verbosity modes like @samp{--no-verbose} or @samp{--quiet}.  This
 is often a desired a property when invoking wget to download several small/large
 files.  In such a case, wget could simply be invoked with this parameter to get
 a much cleaner output on the screen.
 
+This option will also force the progress bar to be printed to @file{stderr} when
+used alongside the @samp{--logfile} option.
+
 @item -N
 @itemx --timestamping
 Turn on time-stamping.  @xref{Time-Stamping}, for details.
 
+@item --no-if-modified-since
+Do not send If-Modified-Since header in @samp{-N} mode. Send preliminary HEAD
+request instead. This has only effect in @samp{-N} mode.
+
 @item --no-use-server-timestamps
 Don't set the local file's timestamp by the one on the server.
 
@@ -1518,6 +1554,11 @@ Please note that wget does not require the content to be of the form
 simply transmit whatever data is provided to it. Most servers however expect
 the POST data to be in the above format when processing HTML Forms.
 
+When sending a POST request using the @samp{--post-file} option, Wget treats
+the file as a binary file and will send every character in the POST request
+without stripping trailing newline or formfeed characters. Any other control
+characters in the text will also be sent as-is in the POST request.
+
 Please be aware that Wget needs to know the size of the POST data in
 advance.  Therefore the argument to @code{--post-file} must be a regular
 file; specifying a FIFO or something like @file{/dev/stdin} won't work.
@@ -1634,23 +1675,24 @@ form-based authentication.
 
 @cindex SSL
 To support encrypted HTTP (HTTPS) downloads, Wget must be compiled
-with an external SSL library, currently OpenSSL.  If Wget is compiled
-without SSL support, none of these options are available.
+with an external SSL library. The current default is GnuTLS.
+In addition, Wget also supports HSTS (HTTP Strict Transport Security).
+If Wget is compiled without SSL support, none of these options are available.
 
 @table @samp
 @cindex SSL protocol, choose
 @item --secure-protocol=@var{protocol}
 Choose the secure protocol to be used.  Legal values are @samp{auto},
-@samp{SSLv2}, @samp{SSLv3}, @samp{TLSv1} and @samp{PFS}.  If @samp{auto}
-is used, the SSL library is given the liberty of choosing the appropriate
-protocol automatically, which is achieved by sending an SSLv2 greeting
-and announcing support for SSLv3 and TLSv1.  This is the default.
+@samp{SSLv2}, @samp{SSLv3}, @samp{TLSv1}, @samp{TLSv1_1}, @samp{TLSv1_2}
+and @samp{PFS}.  If @samp{auto} is used, the SSL library is given the
+liberty of choosing the appropriate protocol automatically, which is
+achieved by sending a TLSv1 greeting. This is the default.
 
-Specifying @samp{SSLv2}, @samp{SSLv3}, or @samp{TLSv1} forces the use
-of the corresponding protocol.  This is useful when talking to old and
-buggy SSL server implementations that make it hard for the underlying
-SSL library to choose the correct protocol version.  Fortunately, such
-servers are quite rare.
+Specifying @samp{SSLv2}, @samp{SSLv3}, @samp{TLSv1}, @samp{TLSv1_1} or
+@samp{TLSv1_2} forces the use of the corresponding protocol.  This is
+useful when talking to old and buggy SSL server implementations that
+make it hard for the underlying SSL library to choose the correct
+protocol version.  Fortunately, such servers are quite rare.
 
 Specifying @samp{PFS} enforces the use of the so-called Perfect Forward
 Security cipher suites. In short, PFS adds security by creating a one-time
@@ -1683,6 +1725,12 @@ this option to bypass the verification and proceed with the download.
 site's authenticity, or if you really don't care about the validity of
 its certificate.}  It is almost always a bad idea not to check the
 certificates when transmitting confidential or important data.
+For self-signed/internal certificates, you should download the certificate
+and verify against that instead of forcing this insecure mode.
+If you are really sure of not desiring any certificate verification, you
+can specify --check-certificate=quiet to tell wget to not print any
+warning about invalid certificates, albeit in most cases this is the
+wrong thing to do.
 
 @cindex SSL certificate
 @item --certificate=@var{file}
@@ -1725,19 +1773,23 @@ it allows Wget to fetch certificates on demand.
 Without this option Wget looks for CA certificates at the
 system-specified locations, chosen at OpenSSL installation time.
 
+@cindex SSL CRL, certificate revocation list
+@item --crl-file=@var{file}
+Specifies a CRL file in @var{file}.  This is needed for certificates
+that have been revocated by the CAs.
+
 @cindex entropy, specifying source of
 @cindex randomness, specifying source of
 @item --random-file=@var{file}
+[OpenSSL and LibreSSL only]
 Use @var{file} as the source of random data for seeding the
-pseudo-random number generator on systems without @file{/dev/random}.
+pseudo-random number generator on systems without @file{/dev/urandom}.
 
 On such systems the SSL library needs an external source of randomness
 to initialize.  Randomness may be provided by EGD (see
 @samp{--egd-file} below) or read from an external source specified by
 the user.  If this option is not specified, Wget looks for random data
-in @code{$RANDFILE} or, if that is unset, in @file{$HOME/.rnd}.  If
-none of those are available, it is likely that SSL encryption will not
-be usable.
+in @code{$RANDFILE} or, if that is unset, in @file{$HOME/.rnd}.
 
 If you're getting the ``Could not seed OpenSSL PRNG; disabling SSL.'' 
 error, you should provide random data using some of the methods
@@ -1745,6 +1797,7 @@ described above.
 
 @cindex EGD
 @item --egd-file=@var{file}
+[OpenSSL only]
 Use @var{file} as the EGD socket.  EGD stands for @dfn{Entropy
 Gathering Daemon}, a user-space program that collects data from
 various unpredictable system sources and makes it available to other
@@ -1759,7 +1812,72 @@ read random data from EGD socket specified using this option.
 
 If this option is not specified (and the equivalent startup command is
 not used), EGD is never contacted.  EGD is not needed on modern Unix
-systems that support @file{/dev/random}.
+systems that support @file{/dev/urandom}.
+
+@cindex HSTS
+@item --no-hsts
+Wget supports HSTS (HTTP Strict Transport Security, RFC 6797) by default.
+Use @samp{--no-hsts} to make Wget act as a non-HSTS-compliant UA. As a
+consequence, Wget would ignore all the @code{Strict-Transport-Security}
+headers, and would not enforce any existing HSTS policy.
+
+@item --hsts-file=@var{file}
+By default, Wget stores its HSTS database in @file{~/.wget-hsts}.
+You can use @samp{--hsts-file} to override this. Wget will use
+the supplied file as the HSTS database. Such file must conform to the
+correct HSTS database format used by Wget. If Wget cannot parse the provided
+file, the behaviour is unspecified.
+
+The Wget's HSTS database is a plain text file. Each line contains an HSTS entry
+(ie. a site that has issued a @code{Strict-Transport-Security} header and that
+therefore has specified a concrete HSTS policy to be applied). Lines starting with
+a dash (@code{#}) are ignored by Wget. Please note that in spite of this convenient
+human-readability hand-hacking the HSTS database is generally not a good idea.
+
+An HSTS entry line consists of several fields separated by one or more whitespace:
+
+@code{<hostname> SP [<port>] SP <include subdomains> SP <created> SP <max-age>}
+
+The @var{hostname} and @var{port} fields indicate the hostname and port to which
+the given HSTS policy applies. The @var{port} field may be zero, and it will, in
+most of the cases. That means that the port number will not be taken into account
+when deciding whether such HSTS policy should be applied on a given request (only
+the hostname will be evaluated). When @var{port} is different to zero, both the
+target hostname and the port will be evaluated and the HSTS policy will only be applied
+if both of them match. This feature has been included for testing/development purposes only.
+The Wget testsuite (in @file{testenv/}) creates HSTS databases with explicit ports
+with the purpose of ensuring Wget's correct behaviour. Applying HSTS policies to ports
+other than the default ones is discouraged by RFC 6797 (see Appendix B "Differences
+between HSTS Policy and Same-Origin Policy"). Thus, this functionality should not be used
+in production environments and @var{port} will typically be zero. The last three fields
+do what they are expected to. The field @var{include_subdomains} can either be @code{1}
+or @code{0} and it signals whether the subdomains of the target domain should be
+part of the given HSTS policy as well. The @var{created} and @var{max-age} fields
+hold the timestamp values of when such entry was created (first seen by Wget) and the
+HSTS-defined value 'max-age', which states how long should that HSTS policy remain active,
+measured in seconds elapsed since the timestamp stored in @var{created}. Once that time
+has passed, that HSTS policy will no longer be valid and will eventually be removed
+from the database.
+
+If you supply your own HSTS database via @samp{--hsts-file}, be aware that Wget
+may modify the provided file if any change occurs between the HSTS policies
+requested by the remote servers and those in the file. When Wget exists,
+it effectively updates the HSTS database by rewriting the database file with the new entries.
+
+If the supplied file does not exist, Wget will create one. This file will contain the new HSTS
+entries. If no HSTS entries were generated (no @code{Strict-Transport-Security} headers
+were sent by any of the servers) then no file will be created, not even an empty one. This
+behaviour applies to the default database file (@file{~/.wget-hsts}) as well: it will not be
+created until some server enforces an HSTS policy.
+
+Care is taken not to override possible changes made by other Wget processes at
+the same time over the HSTS database. Before dumping the updated HSTS entries
+on the file, Wget will re-read it and merge the changes.
+
+Using a custom HSTS database and/or modifying an existing one is discouraged.
+For more information about the potential security threats arised from such practice,
+see section 14 "Security Considerations" of RFC 6797, specially section 14.9
+"Creative Manipulation of HSTS Policy Store".
 @end table
 
 @cindex WARC
@@ -1883,17 +2001,18 @@ Preserve remote file permissions instead of permissions set by umask.
 
 @cindex symbolic links, retrieving
 @item --retr-symlinks
-Usually, when retrieving @sc{ftp} directories recursively and a symbolic
-link is encountered, the linked-to file is not downloaded.  Instead, a
-matching symbolic link is created on the local filesystem.  The
-pointed-to file will not be downloaded unless this recursive retrieval
-would have encountered it separately and downloaded it anyway.
+By default, when retrieving @sc{ftp} directories recursively and a symbolic link
+is encountered, the symbolic link is traversed and the pointed-to files are
+retrieved.  Currently, Wget does not traverse symbolic links to directories to
+download them recursively, though this feature may be added in the future.
 
-When @samp{--retr-symlinks} is specified, however, symbolic links are
-traversed and the pointed-to files are retrieved.  At this time, this
-option does not cause Wget to traverse symlinks to directories and
-recurse through them, but in the future it should be enhanced to do
-this.
+When @samp{--retr-symlinks=no} is specified, the linked-to file is not
+downloaded.  Instead, a matching symbolic link is created on the local
+filesystem.  The pointed-to file will not be retrieved unless this recursive
+retrieval would have encountered it separately and downloaded it anyway.  This
+option poses a security risk where a malicious FTP Server may cause Wget to
+write to files outside of the intended directories through a specially crafted
+@sc{.listing} file.
 
 Note that when retrieving a file (not a directory) because it was
 specified on the command-line, rather than because it was recursed to,
@@ -1901,6 +2020,43 @@ this option has no effect.  Symbolic links are always traversed in this
 case.
 @end table
 
+@section FTPS Options
+
+@table @samp
+@item --ftps-implicit
+This option tells Wget to use FTPS implicitly. Implicit FTPS consists of initializing
+SSL/TLS from the very beginning of the control connection. This option does not send
+an @code{AUTH TLS} command: it assumes the server speaks FTPS and directly starts an
+SSL/TLS connection. If the attempt is successful, the session continues just like
+regular FTPS (@code{PBSZ} and @code{PROT} are sent, etc.).
+Implicit FTPS is no longer a requirement for FTPS implementations, and thus
+many servers may not support it. If @samp{--ftps-implicit} is passed and no explicit
+port number specified, the default port for implicit FTPS, 990, will be used, instead
+of the default port for the "normal" (explicit) FTPS which is the same as that of FTP,
+21.
+
+@item --no-ftps-resume-ssl
+Do not resume the SSL/TLS session in the data channel. When starting a data connection,
+Wget tries to resume the SSL/TLS session previously started in the control connection.
+SSL/TLS session resumption avoids performing an entirely new handshake by reusing
+the SSL/TLS parameters of a previous session. Typically, the FTPS servers want it that way,
+so Wget does this by default. Under rare circumstances however, one might want to
+start an entirely new SSL/TLS session in every data connection.
+This is what @samp{--no-ftps-resume-ssl} is for.
+
+@item --ftps-clear-data-connection
+All the data connections will be in plain text. Only the control connection will be
+under SSL/TLS. Wget will send a @code{PROT C} command to achieve this, which must be
+approved by the server.
+
+@item --ftps-fallback-to-ftp
+Fall back to FTP if FTPS is not supported by the target server. For security reasons,
+this option is not asserted by default. The default behaviour is to exit with an error.
+If a server does not successfully reply to the initial @code{AUTH TLS} command, or in the
+case of implicit FTPS, if the initial SSL/TLS connection attempt is rejected, it is
+considered that such server does not support FTPS.
+@end table
+
 @node Recursive Retrieval Options, Recursive Accept/Reject Options, FTP Options, Invoking
 @section Recursive Retrieval Options
 
@@ -1979,6 +2135,23 @@ Note that only at the end of the download can Wget know which links have
 been downloaded.  Because of that, the work done by @samp{-k} will be
 performed at the end of all the downloads.
 
+@item --convert-file-only
+This option converts only the filename part of the URLs, leaving the rest
+of the URLs untouched. This filename part is sometimes referred to as the
+"basename", although we avoid that term here in order not to cause confusion.
+
+It works particularly well in conjunction with @samp{--adjust-extension}, although
+this coupling is not enforced. It proves useful to populate Internet caches
+with files downloaded from different hosts.
+
+Example: if some link points to @file{//foo.com/bar.cgi?xyz} with
+@samp{--adjust-extension} asserted and its local destination is intended to be
+@file{./foo.com/bar.cgi?xyz.css}, then the link would be converted to
+@file{//foo.com/bar.cgi?xyz.css}. Note that only the filename part has been
+modified. The rest of the URL has been left untouched, including the net path
+(@code{//}) which would otherwise be processed by Wget and converted to the
+effective scheme (ie. @code{http://}).
+
 @cindex backing up converted files
 @item -K
 @itemx --backup-converted
@@ -2892,10 +3065,10 @@ variable = value
 The @dfn{variable} will also be called @dfn{command}.  Valid
 @dfn{values} are different for different commands.
 
-The commands are case-insensitive and underscore-insensitive.  Thus
-@samp{DIr__PrefiX} is the same as @samp{dirprefix}.  Empty lines, lines
-beginning with @samp{#} and lines containing white-space only are
-discarded.
+The commands are case-, underscore- and minus-insensitive.  Thus
+@samp{DIr__PrefiX}, @samp{DIr-PrefiX} and @samp{dirprefix} are the same.
+Empty lines, lines beginning with @samp{#} and lines containing white-space
+only are discarded.
 
 Commands that expect a comma-separated list will clear the list on an
 empty command.  So, if you wish to reset the rejection list specified in
@@ -3125,6 +3298,9 @@ environment.
 Set @sc{http} user to @var{string}, equivalent to
 @samp{--http-user=@var{string}}.
 
+@item https_only = on/off
+When in recursive mode, only HTTPS links are followed (defaults to off).
+
 @item https_proxy = @var{string}
 Use @var{string} as @sc{https} proxy, instead of the one specified in
 environment.
@@ -4084,12 +4260,7 @@ me).
 @section Contributors
 @cindex contributors
 
-@iftex
-GNU Wget was written by Hrvoje Nik@v{s}i@'{c} @email{hniksic@@xemacs.org},
-@end iftex
-@ifnottex
-GNU Wget was written by Hrvoje Niksic @email{hniksic@@xemacs.org}.
-@end ifnottex
+GNU Wget was written by Hrvoje Nikšić @email{hniksic@@xemacs.org},
 
 However, the development of Wget could never have gone as far as it has, were
 it not for the help of many people, either with bug reports, feature proposals,
@@ -4129,14 +4300,8 @@ portability fixes.
 Gordon Matzigkeit---@file{.netrc} support.
 
 @item
-@iftex
-Zlatko @v{C}alu@v{s}i@'{c}, Tomislav Vujec and Dra@v{z}en
-Ka@v{c}ar---feature suggestions and ``philosophical'' discussions.
-@end iftex
-@ifnottex
-Zlatko Calusic, Tomislav Vujec and Drazen Kacar---feature suggestions
-and ``philosophical'' discussions.
-@end ifnottex
+Zlatko Čalušić, Tomislav Vujec and Dražen
+Kačar---feature suggestions and ``philosophical'' discussions.
 
 @item
 Darko Budor---initial port to Windows.
@@ -4146,21 +4311,11 @@ Antonio Rosella---help and suggestions, plus the initial Italian
 translation.
 
 @item
-@iftex
-Tomislav Petrovi@'{c}, Mario Miko@v{c}evi@'{c}---many bug reports and
+Tomislav Petrović, Mario Mikočević---many bug reports and
 suggestions.
-@end iftex
-@ifnottex
-Tomislav Petrovic, Mario Mikocevic---many bug reports and suggestions.
-@end ifnottex
 
 @item
-@iftex
-Fran@,{c}ois Pinard---many thorough bug reports and discussions.
-@end iftex
-@ifnottex
-Francois Pinard---many thorough bug reports and discussions.
-@end ifnottex
+François Pinard---many thorough bug reports and discussions.
 
 @item
 Karl Eichwalder---lots of help with internationalization, Makefile
@@ -4223,12 +4378,7 @@ Wanderlei Cavassin,
 Gilles Cedoc,
 Tim Charron,
 Noel Cragg,
-@iftex
-Kristijan @v{C}onka@v{s},
-@end iftex
-@ifnottex
-Kristijan Conkas,
-@end ifnottex
+Kristijan Čonkaš,
 John Daily,
 Andreas Damm,
 Ahmon Dancy,
@@ -4238,27 +4388,12 @@ Alexander Dergachev,
 Andrew Deryabin,
 Ulrich Drepper,
 Marc Duponcheel,
-@iftex
-Damir D@v{z}eko,
-@end iftex
-@ifnottex
-Damir Dzeko,
-@end ifnottex
+Damir Džeko,
 Alan Eldridge,
 Hans-Andreas Engel,
-@iftex
-Aleksandar Erkalovi@'{c},
-@end iftex
-@ifnottex
-Aleksandar Erkalovic,
-@end ifnottex
+Aleksandar Erkalović,
 Andy Eskilsson,
-@iftex
-Jo@~{a}o Ferreira,
-@end iftex
-@ifnottex
-Joao Ferreira,
-@end ifnottex
+João Ferreira,
 Christian Fraenkel,
 David Fritz,
 Mike Frysinger,
@@ -4283,25 +4418,10 @@ Richard Huveneers,
 Jonas Jensen,
 Larry Jones,
 Simon Josefsson,
-@iftex
-Mario Juri@'{c},
-@end iftex
-@ifnottex
-Mario Juric,
-@end ifnottex
-@iftex
-Hack Kampbj@o rn,
-@end iftex
-@ifnottex
-Hack Kampbjorn,
-@end ifnottex
+Mario Jurić,
+Hack Kampbjørn,
 Const Kaplinsky,
-@iftex
-Goran Kezunovi@'{c},
-@end iftex
-@ifnottex
-Goran Kezunovic,
-@end ifnottex
+Goran Kezunović,
 Igor Khristophorov,
 Robert Kleine,
 KOJIMA Haime,
@@ -4310,31 +4430,14 @@ Alexander Kourakos,
 Martin Kraemer,
 Sami Krank,
 Jay Krell,
-@tex
-$\Sigma\acute{\iota}\mu o\varsigma\;
-\Xi\varepsilon\nu\iota\tau\acute{\epsilon}\lambda\lambda\eta\varsigma$
-(Simos KSenitellis),
-@end tex
-@ifnottex
-Simos KSenitellis,
-@end ifnottex
+Σίμος Ξενιτέλλης (Simos KSenitellis),
 Christian Lackas,
 Hrvoje Lacko,
 Daniel S.@: Lewart,
-@iftex
-Nicol@'{a}s Lichtmeier,
-@end iftex
-@ifnottex
-Nicolas Lichtmeier,
-@end ifnottex
+Nicolás Lichtmeier,
 Dave Love,
 Alexander V.@: Lukyanov,
-@iftex
-Thomas Lu@ss{}nig,
-@end iftex
-@ifnottex
-Thomas Lussnig,
-@end ifnottex
+Thomas Lußnig,
 Andre Majorel,
 Aurelien Marchand,
 Matthew J.@: Mellon,
@@ -4355,31 +4458,15 @@ Leonid Petrov,
 Simone Piunno,
 Andrew Pollock,
 Steve Pothier,
-@iftex
-Jan P@v{r}ikryl,
-@end iftex
-@ifnottex
-Jan Prikryl,
-@end ifnottex
+Jan Přikryl,
 Marin Purgar,
-@iftex
-Csaba R@'{a}duly,
-@end iftex
-@ifnottex
-Csaba Raduly,
-@end ifnottex
+Csaba Ráduly,
 Keith Refson,
 Bill Richardson,
 Tyler Riddle,
 Tobias Ringstrom,
 Jochen Roderburg,
-@c Texinfo doesn't grok @'{@i}, so we have to use TeX itself.
-@tex
-Juan Jos\'{e} Rodr\'{\i}guez,
-@end tex
-@ifnottex
-Juan Jose Rodriguez,
-@end ifnottex
+Juan José Rodríguez,
 Maciej W.@: Rozycki,
 Edward J.@: Sabol,
 Heinz Salzmann,
@@ -4406,12 +4493,7 @@ Dave Turner,
 Gisle Vanem,
 Rabin Vincent,
 Russell Vincent,
-@iftex
-@v{Z}eljko Vrba,
-@end iftex
-@ifnottex
-Zeljko Vrba,
-@end ifnottex
+Željko Vrba,
 Charles G Waldman,
 Douglas E.@: Wegscheid,
 Ralf Wildenhues,
@@ -4420,12 +4502,7 @@ Benjamin Wolsey,
 Saint Xavier,
 YAMAZAKI Makoto,
 Jasmin Zainul,
-@iftex
-Bojan @v{Z}drnja,
-@end iftex
-@ifnottex
-Bojan Zdrnja,
-@end ifnottex
+Bojan Ždrnja,
 Kristijan Zimmer,
 Xin Zou.
 

msdos/ChangeLog

@@ -1,61 +0,0 @@
-2014-06-11  Giuseppe Scrivano  <gscrivan@redhat.com>
-
-	* Makefile.DJ: Remove terminating empty lines.
-	* Makefile.WC: Likewise.
-	* msdos.c: Likewise.
-
-2009-09-06  Gisle Vanem  <gvanem@broadpark.no>
-
-	* Makefile.WC: Added compilation of new file msdos.c.
-	Simplified; no need to have explicit rules for files in ./lib and
-	./md5. Just use the suffix search path mechanism in wmake.
-
-2009-09-06  Gisle Vanem  <gvanem@broadpark.no>
-
-	* Makefile.DJ: Added compilation of new file msdos.c.
-
-2009-09-05  Gisle Vanem  <gvanem@broadpark.no>
-
-	* config.h: Added 'HAVE_ALLOCA_H' for Watcom 1.5+.  Added
-	'USE_WATT32' since all DOS-targets use the Watt-32 tcp/ip stack.
-	Added meaningless dummy LOCALEDIR.
-
-	* Makefile.WC: A much needed update. Added rules for many files
-	in ./lib, css.c and version.c.
-
-	* Makefile.DJ: A much needed update. Added rules for css.c and
-	version.c.
-
-2008-01-25  Micah Cowan  <micah@cowan.name>
-
-	* config.h: Updated copyright year.
-
-2007-11-28  Micah Cowan <micah@cowan.name>
-
-	* config.h: Updated license exception for OpenSSL, per the SFLC.
-
-2007-10-15  Gisle Vanem  <gvanem@broadpark.no>
-
-	* config.h: Added some HAVE_* for djgpp 2.04 and Watcom 1.7+.
-
-	* Makefile.DJ: rewritten for including some files from ../lib.
-
-	* Makefile.WC: Ditto. Handling source-files out-of current
-	directory makes compliation a bit more painfull. AFAICS, one
-	must use explicit rules.
-
-2007-10-02  Gisle Vanem  <gvanem@broadpark.no>
-
-	* config.h: Removed unused defines, added needed 'HAVE_*' defines.
-
-	* Makefile.DJ: rewritten to be used from './src' directory.
-	Added '-DOPENSSL_NO_KRB5' for OpenSSL build. Target is
-	now wget.exe.
-
-	* Makefile.WC: Added for building with OpenWatcom targeting
-	32-bit DOS (DOS32A extender). 
-
-2007-09-24  Gisle Vanem  <giva@bgnett.no>
-
-	* Makefile.DJ, config.h: Added to support building on MS-DOS via
-	DJGPP.

msdos/config.h

@@ -1,6 +1,6 @@
 /* Configuration header file for MS-DOS/Watt-32
-   Copyright (C) 2007, 2008, 2009, 2010, 2011 Free Software Foundation,
-   Inc.
+   Copyright (C) 2007, 2008, 2009, 2010, 2011, 2015 Free Software
+   Foundation, Inc.
 
    This file is part of GNU Wget.
 

msdos/msdos.c

@@ -1,5 +1,5 @@
 /* MSDOS utility functions.
-   Copyright (C) 2009, 2010, 2011 Free Software Foundation, Inc.
+   Copyright (C) 2009, 2010, 2011, 2015 Free Software Foundation, Inc.
 
 This file is part of GNU Wget.
 

po/POTFILES.in

@@ -26,6 +26,7 @@ src/init.c
 src/iri.c
 src/log.c
 src/main.c
+src/metalink.c
 src/mswindows.c
 src/netrc.c
 src/openssl.c

po/stamp-po

@@ -1 +0,0 @@
-timestamp

src/Makefile.am

@@ -1,6 +1,6 @@
 # Makefile for `wget' utility
 # Copyright (C) 1995, 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003,
-# 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011 Free Software
+# 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2015 Free Software
 # Foundation, Inc.
 
 # This program is free software; you can redistribute it and/or modify
@@ -28,32 +28,36 @@
 # as that of the covered work.
 
 #
-# Version: $(VERSION)
+# Version: @VERSION@
 #
 
 if IRI_IS_ENABLED
 IRI_OBJ = iri.c
 endif
 
+if METALINK_IS_ENABLED
+METALINK_OBJ = metalink.c
+endif
+
 # The following line is losing on some versions of make!
-DEFS     += -DSYSTEM_WGETRC=\"$(sysconfdir)/wgetrc\" -DLOCALEDIR=\"$(localedir)\"
-LIBS     += $(LIBICONV) $(LIBINTL) $(LIB_CLOCK_GETTIME)
+DEFS     = @DEFS@ -DSYSTEM_WGETRC=\"$(sysconfdir)/wgetrc\" -DLOCALEDIR=\"$(localedir)\"
+LIBS     = @LIBICONV@ @LIBINTL@ @LIBS@ $(LIB_CLOCK_GETTIME)
 
 EXTRA_DIST = css.l css.c css_.c build_info.c.in
 
 bin_PROGRAMS = wget
-wget_SOURCES = cmpt.c connect.c convert.c cookies.c ftp.c	\
+wget_SOURCES = connect.c convert.c cookies.c ftp.c	\
 		css_.c css-url.c	\
-		ftp-basic.c ftp-ls.c hash.c host.c html-parse.c html-url.c	\
+		ftp-basic.c ftp-ls.c hash.c host.c hsts.c html-parse.c html-url.c	\
 		http.c init.c log.c main.c netrc.c progress.c ptimer.c	\
 		recur.c res.c retr.c spider.c url.c warc.c	\
-		utils.c exits.c build_info.c $(IRI_OBJ)	\
+		utils.c exits.c build_info.c $(IRI_OBJ) $(METALINK_OBJ)	\
 		css-url.h css-tokens.h connect.h convert.h cookies.h	\
-		ftp.h hash.h host.h html-parse.h html-url.h	\
+		ftp.h hash.h host.h hsts.h  html-parse.h html-url.h	\
 		http.h http-ntlm.h init.h log.h mswindows.h netrc.h	\
 		options.h progress.h ptimer.h recur.h res.h retr.h	\
 		spider.h ssl.h sysdep.h url.h warc.h utils.h wget.h iri.h	\
-		exits.h gettext.h
+		exits.h version.h metalink.h
 nodist_wget_SOURCES = version.c
 EXTRA_wget_SOURCES = iri.c
 LDADD = $(LIBOBJS) ../lib/libgnu.a
@@ -74,7 +78,8 @@ version.c:  $(wget_SOURCES) ../lib/libgnu.a
 	echo '/* version.c */' > $@
 	echo '/* Autogenerated by Makefile - DO NOT EDIT */' >> $@
 	echo '' >> $@
-	echo 'const char *version_string = "$(VERSION)";' >> $@
+	echo '#include "version.h"' >> $@
+	echo 'const char *version_string = "@VERSION@";' >> $@
 	echo 'const char *compilation_string = "'$(COMPILE)'";' \
 	    | $(ESCAPEQUOTE) >> $@
 	echo 'const char *link_string = "'$(CCLD) $(AM_CFLAGS) $(CFLAGS) \
@@ -82,7 +87,7 @@ version.c:  $(wget_SOURCES) ../lib/libgnu.a
 	    | $(ESCAPEQUOTE) >> $@
 
 css.c: $(srcdir)/css.l
-	$(LEX) $(LFLAGS) -o $@ $^
+	$(LEX) $(LFLAGS) -o$@ $^
 
 css_.c: css.c
 	echo '#include "wget.h"' > $@

src/build_info.c.in

@@ -2,13 +2,16 @@ digest          defined ENABLE_DIGEST
 https           defined HAVE_SSL
 ipv6            defined ENABLE_IPV6
 iri             defined ENABLE_IRI
-large-file      SIZEOF_OFF_T >= 8
+large-file      SIZEOF_OFF_T >= 8 || defined WINDOWS
 
 nls             defined ENABLE_NLS
 ntlm            defined ENABLE_NTLM
 opie            defined ENABLE_OPIE
 psl             defined HAVE_LIBPSL
 
+metalink        defined HAVE_METALINK
+gpgme           defined HAVE_GPGME
+
 ssl choice:
     openssl     defined HAVE_LIBSSL || defined HAVE_LIBSSL32
     gnutls      defined HAVE_LIBGNUTLS

src/connect.c

@@ -1,6 +1,6 @@
 /* Establishing and handling network connections.
    Copyright (C) 1995, 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003,
-   2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011 Free Software
+   2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2015 Free Software
    Foundation, Inc.
 
 This file is part of GNU Wget.
@@ -54,6 +54,11 @@ as that of the covered work.  */
 #include <errno.h>
 #include <string.h>
 #include <sys/time.h>
+
+#ifdef ENABLE_IRI
+#include <idn-free.h>
+#endif
+
 #include "utils.h"
 #include "host.h"
 #include "connect.h"
@@ -168,7 +173,7 @@ sockaddr_size (const struct sockaddr *sa)
       abort ();
     }
 }
-
+
 /* Resolve the bind address specified via --bind-address and store it
    to SA.  The resolved value is stored in a static variable and
    reused after the first invocation of this function.
@@ -213,7 +218,7 @@ resolve_bind_address (struct sockaddr *sa)
   should_bind = true;
   return true;
 }
-
+
 struct cwt_context {
   int fd;
   const struct sockaddr *addr;
@@ -250,7 +255,7 @@ connect_with_timeout (int fd, const struct sockaddr *addr, socklen_t addrlen,
     errno = ETIMEDOUT;
   return ctx.result;
 }
-
+
 /* Connect via TCP to the specified address and port.
 
    If PRINT is non-NULL, it is the host name to print that we're
@@ -278,14 +283,13 @@ connect_to_ip (const ip_address *ip, int port, const char *print)
               str = xmalloc (len);
               snprintf (str, len, "%s (%s)", name, print);
               str[len-1] = '\0';
-              xfree (name);
+              idn_free (name);
             }
 
           logprintf (LOG_VERBOSE, _("Connecting to %s|%s|:%d... "),
                      str ? str : escnonprint_uri (print), txt_addr, port);
 
-          if (str)
-              xfree (str);
+          xfree (str);
         }
       else
         {
@@ -367,7 +371,7 @@ connect_to_ip (const ip_address *ip, int port, const char *print)
     if (sock >= 0)
       fd_close (sock);
     if (print)
-      logprintf (LOG_VERBOSE, _("failed: %s.\n"), strerror (errno));
+      logprintf (LOG_NOTQUIET, _("failed: %s.\n"), strerror (errno));
     errno = save_errno;
     return -1;
   }
@@ -429,7 +433,7 @@ connect_to_host (const char *host, int port)
 
   return -1;
 }
-
+
 /* Create a socket, bind it to local interface BIND_ADDRESS on port
    *PORT, set up a listen backlog, and return the resulting socket, or
    -1 in case of error.
@@ -746,7 +750,7 @@ wget uses blocking sockets so we must convert them back to blocking
        or EOF/error. */
     return false;
 }
-
+
 /* Basic socket operations, mostly EINTR wrappers.  */
 
 static int
@@ -794,7 +798,7 @@ sock_close (int fd)
 #undef read
 #undef write
 #undef close
-
+
 /* Reading and writing from the network.  We build around the socket
    (file descriptor) API, but support "extended" operations for things
    that are not mere file descriptors under the hood, such as SSL
@@ -845,7 +849,7 @@ void *
 fd_transport_context (int fd)
 {
   struct transport_info *info = hash_table_get (transport_map, (void *)(intptr_t) fd);
-  return info->ctx;
+  return info ? info->ctx : NULL;
 }
 
 /* When fd_read/fd_write are called multiple times in a loop, they should

src/connect.h

@@ -1,7 +1,7 @@
 /* Declarations for connect.
    Copyright (C) 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004,
-   2005, 2006, 2007, 2008, 2009, 2010, 2011 Free Software Foundation,
-   Inc.
+   2005, 2006, 2007, 2008, 2009, 2010, 2011, 2015 Free Software
+   Foundation, Inc.
 
 This file is part of GNU Wget.
 

src/convert.c

@@ -1,6 +1,6 @@
 /* Conversion of links to local files.
-   Copyright (C) 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2014
-   Free Software Foundation, Inc.
+   Copyright (C) 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011,
+   2014, 2015 Free Software Foundation, Inc.
 
 This file is part of GNU Wget.
 
@@ -46,6 +46,7 @@ as that of the covered work.  */
 #include "html-url.h"
 #include "css-url.h"
 #include "iri.h"
+#include "xstrndup.h"
 
 static struct hash_table *dl_file_url_map;
 struct hash_table *dl_url_file_map;
@@ -136,8 +137,9 @@ convert_links_in_hashtable (struct hash_table *downloaded_set,
                  form.  We do this even if the URL already is in
                  relative form, because our directory structure may
                  not be identical to that on the server (think `-nd',
-                 `--cut-dirs', etc.)  */
-              cur_url->convert = CO_CONVERT_TO_RELATIVE;
+                 `--cut-dirs', etc.). If --convert-file-only was passed,
+                 we only convert the basename portion of the URL.  */
+              cur_url->convert = (opt.convert_file_only ? CO_CONVERT_BASENAME_ONLY : CO_CONVERT_TO_RELATIVE);
               cur_url->local_name = xstrdup (local_name);
               DEBUGP (("will convert url %s to local %s\n", u->url, local_name));
             }
@@ -193,7 +195,7 @@ convert_all_links (void)
   convert_links_in_hashtable (downloaded_css_set, 1, &file_count);
 
   secs = ptimer_measure (timer);
-  logprintf (LOG_VERBOSE, _("Converted %d files in %s seconds.\n"),
+  logprintf (LOG_VERBOSE, _("Converted links in %d files in %s seconds.\n"),
              file_count, print_decimal (secs));
 
   ptimer_destroy (timer);
@@ -206,6 +208,7 @@ static const char *replace_attr_refresh_hack (const char *, int, FILE *,
                                               const char *, int);
 static char *local_quote_string (const char *, bool);
 static char *construct_relative (const char *, const char *);
+static char *convert_basename (const char *, const struct urlpos *);
 
 /* Change the links in one file.  LINKS is a list of links in the
    document, along with their positions and the desired direction of
@@ -221,7 +224,7 @@ convert_links (const char *file, struct urlpos *links)
   struct urlpos *link;
   int to_url_count = 0, to_file_count = 0;
 
-  logprintf (LOG_VERBOSE, _("Converting %s... "), file);
+  logprintf (LOG_VERBOSE, _("Converting links in %s... "), file);
 
   {
     /* First we do a "dry run": go through the list L and see whether
@@ -315,9 +318,32 @@ convert_links (const char *file, struct urlpos *links)
 
             DEBUGP (("TO_RELATIVE: %s to %s at position %d in %s.\n",
                      link->url->url, newname, link->pos, file));
+
             xfree (newname);
             xfree (quoted_newname);
             ++to_file_count;
+            break;
+          }
+        case CO_CONVERT_BASENAME_ONLY:
+          {
+            char *newname = convert_basename (p, link);
+            char *quoted_newname = local_quote_string (newname, link->link_css_p);
+
+            if (link->link_css_p)
+              p = replace_plain (p, link->size, fp, quoted_newname);
+            else if (!link->link_refresh_p)
+              p = replace_attr (p, link->size, fp, quoted_newname);
+            else
+              p = replace_attr_refresh_hack (p, link->size, fp, quoted_newname,
+                                             link->refresh_timeout);
+
+            DEBUGP (("Converted file part only: %s to %s at position %d in %s.\n",
+                     link->url->url, newname, link->pos, file));
+
+            xfree (newname);
+            xfree (quoted_newname);
+            ++to_file_count;
+
             break;
           }
         case CO_CONVERT_TO_COMPLETE:
@@ -336,6 +362,7 @@ convert_links (const char *file, struct urlpos *links)
 
             DEBUGP (("TO_COMPLETE: <something> to %s at position %d in %s.\n",
                      newlink, link->pos, file));
+
             xfree (quoted_newlink);
             ++to_url_count;
             break;
@@ -414,14 +441,89 @@ construct_relative (const char *basefile, const char *linkfile)
         ++basedirs;
     }
 
-  /* Construct LINK as explained above. */
-  link = xmalloc (3 * basedirs + strlen (linkfile) + 1);
-  for (i = 0; i < basedirs; i++)
-    memcpy (link + 3 * i, "../", 3);
-  strcpy (link + 3 * i, linkfile);
+  if (!basedirs && (b = strpbrk (linkfile, "/:")) && *b == ':')
+    {
+      link = xmalloc (2 + strlen (linkfile) + 1);
+      memcpy (link, "./", 2);
+      strcpy (link + 2, linkfile);
+    }
+  else
+    {
+      /* Construct LINK as explained above. */
+      link = xmalloc (3 * basedirs + strlen (linkfile) + 1);
+      for (i = 0; i < basedirs; i++)
+        memcpy (link + 3 * i, "../", 3);
+      strcpy (link + 3 * i, linkfile);
+    }
+
   return link;
 }
 
+/* Construct and return a "transparent proxy" URL
+   reflecting changes made by --adjust-extension to the file component
+   (i.e., "basename") of the original URL, but leaving the "dirname"
+   of the URL (protocol://hostname... portion) untouched.
+
+   Think: populating a squid cache via a recursive wget scrape, where
+   changing URLs to work locally with "file://..." is NOT desirable.
+
+   Example:
+
+   if
+                     p = "//foo.com/bar.cgi?xyz"
+   and
+      link->local_name = "docroot/foo.com/bar.cgi?xyz.css"
+   then
+
+      new_construct_func(p, link);
+   will return
+      "//foo.com/bar.cgi?xyz.css"
+
+   Essentially, we do s/$(basename orig_url)/$(basename link->local_name)/
+*/
+static char *
+convert_basename (const char *p, const struct urlpos *link)
+{
+  int len = link->size;
+  char *url = NULL;
+  char *org_basename = NULL, *local_basename = NULL;
+  char *result = NULL;
+
+  if (*p == '"' || *p == '\'')
+    {
+      len -= 2;
+      p++;
+    }
+
+  url = xstrndup (p, len);
+
+  org_basename = strrchr (url, '/');
+  if (org_basename)
+    org_basename++;
+  else
+    org_basename = url;
+
+  local_basename = strrchr (link->local_name, '/');
+  if (local_basename)
+    local_basename++;
+  else
+    local_basename = url;
+
+  /*
+   * If the basenames differ, graft the adjusted basename (local_basename)
+   * onto the original URL.
+   */
+  if (strcmp (org_basename, local_basename) == 0)
+    result = url;
+  else
+    {
+      result = uri_merge (url, local_basename);
+      xfree (url);
+    }
+
+  return result;
+}
+
 /* Used by write_backup_file to remember which files have been
    written. */
 static struct hash_table *converted_files;
@@ -661,7 +763,7 @@ local_quote_string (const char *file, bool no_html_quote)
 
   return no_html_quote ? strdup (newname) : html_quote_string (newname);
 }
-
+
 /* Book-keeping code for dl_file_url_map, dl_url_file_map,
    downloaded_html_list, and downloaded_html_set.  Other code calls
    these functions to let us know that a file has been downloaded.  */
@@ -916,7 +1018,7 @@ convert_cleanup (void)
   if (converted_files)
     string_set_free (converted_files);
 }
-
+
 /* Book-keeping code for downloaded files that enables extension
    hacks.  */
 
@@ -1012,7 +1114,7 @@ downloaded_files_free (void)
       downloaded_files_hash = NULL;
     }
 }
-
+
 /* The function returns the pointer to the malloc-ed quoted version of
    string s.  It will recognize and quote numeric and special graphic
    entities, as per RFC1866:

src/convert.h

@@ -1,6 +1,6 @@
 /* Declarations for convert.c
-   Copyright (C) 2003, 2004, 2005, 2006, 2009, 2010, 2011 Free Software
-   Foundation, Inc.
+   Copyright (C) 2003, 2004, 2005, 2006, 2009, 2010, 2011, 2015 Free
+   Software Foundation, Inc.
 
 This file is part of GNU Wget.
 
@@ -40,6 +40,8 @@ enum convert_options {
   CO_NOCONVERT = 0,             /* don't convert this URL */
   CO_CONVERT_TO_RELATIVE,       /* convert to relative, e.g. to
                                    "../../otherdir/foo.gif" */
+  CO_CONVERT_BASENAME_ONLY,     /* convert the file portion only (basename)
+                                   leaving the rest of the URL unchanged */
   CO_CONVERT_TO_COMPLETE,       /* convert to absolute, e.g. to
                                    "http://orighost/somedir/bar.jpg". */
   CO_NULLIFY_BASE               /* change to empty string. */

src/cookies.c

@@ -1,6 +1,6 @@
 /* Support for cookies.
    Copyright (C) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009,
-   2010, 2011 Free Software Foundation, Inc.
+   2010, 2011, 2015 Free Software Foundation, Inc.
 
 This file is part of GNU Wget.
 
@@ -58,7 +58,9 @@ as that of the covered work.  */
 #include "hash.h"
 #include "cookies.h"
 #include "http.h"               /* for http_atotm */
-
+#include "c-strcase.h"
+
+
 /* Declarations of `struct cookie' and the most basic functions. */
 
 /* Cookie jar serves as cookie storage and a means of retrieving
@@ -151,13 +153,13 @@ cookie_expired_p (const struct cookie *c)
 static void
 delete_cookie (struct cookie *cookie)
 {
-  xfree_null (cookie->domain);
-  xfree_null (cookie->path);
-  xfree_null (cookie->attr);
-  xfree_null (cookie->value);
+  xfree (cookie->domain);
+  xfree (cookie->path);
+  xfree (cookie->attr);
+  xfree (cookie->value);
   xfree (cookie);
 }
-
+
 /* Functions for storing cookies.
 
    All cookies can be reached beginning with jar->chains.  The key in
@@ -302,7 +304,12 @@ discard_matching_cookie (struct cookie_jar *jar, struct cookie *cookie)
 
           res = hash_table_get_pair (jar->chains, victim->domain,
                                      &chain_key, NULL);
-          assert (res != 0);
+
+          if (res == 0)
+            {
+              logprintf (LOG_VERBOSE, _("Unable to get cookie for %s\n"),
+                         victim->domain);
+            }
           if (!victim->next)
             {
               /* VICTIM was the only cookie in the chain.  Destroy the
@@ -317,7 +324,7 @@ discard_matching_cookie (struct cookie_jar *jar, struct cookie *cookie)
       DEBUGP (("Discarded old cookie.\n"));
     }
 }
-
+
 /* Functions for parsing the `Set-Cookie' header, and creating new
    cookies from the wire.  */
 
@@ -369,7 +376,7 @@ parse_set_cookie (const char *set_cookie, bool silent)
         {
           if (!TOKEN_NON_EMPTY (value))
             goto error;
-          xfree_null (cookie->domain);
+          xfree (cookie->domain);
           /* Strictly speaking, we should set cookie->domain_exact if the
              domain doesn't begin with a dot.  But many sites set the
              domain to "foo.com" and expect "subhost.foo.com" to get the
@@ -382,7 +389,7 @@ parse_set_cookie (const char *set_cookie, bool silent)
         {
           if (!TOKEN_NON_EMPTY (value))
             goto error;
-          xfree_null (cookie->path);
+          xfree (cookie->path);
           cookie->path = strdupdelim (value.b, value.e);
         }
       else if (TOKEN_IS (name, "expires"))
@@ -456,7 +463,7 @@ parse_set_cookie (const char *set_cookie, bool silent)
 
 #undef TOKEN_IS
 #undef TOKEN_NON_EMPTY
-
+
 /* Sanity checks.  These are important, otherwise it is possible for
    mailcious attackers to destroy important cookie information and/or
    violate your privacy.  */
@@ -518,12 +525,12 @@ check_domain_match (const char *cookie_domain, const char *host)
 {
 
 #ifdef HAVE_LIBPSL
-  DEBUGP (("cdm: 1"));
   char *cookie_domain_lower = NULL;
   char *host_lower = NULL;
   const psl_ctx_t *psl;
   int is_acceptable;
 
+  DEBUGP (("cdm: 1"));
   if (!(psl = psl_builtin()))
     {
       DEBUGP (("\nlibpsl not built with a public suffix list. "
@@ -692,7 +699,7 @@ check_path_match (const char *cookie_path, const char *path)
   s = PS_newstr;                                                \
 } while (0)
 
-
+
 /* Process the HTTP `Set-Cookie' header.  This results in storing the
    cookie or discarding a matching one, or ignoring it completely, all
    depending on the contents.  */
@@ -776,7 +783,7 @@ cookie_handle_set_cookie (struct cookie_jar *jar,
   if (cookie)
     delete_cookie (cookie);
 }
-
+
 /* Support for sending out cookies in HTTP requests, based on
    previously stored cookies.  Entry point is
    `build_cookies_request'.  */
@@ -1107,7 +1114,7 @@ cookie_header (struct cookie_jar *jar, const char *host,
   assert (pos == result_size);
   return result;
 }
-
+
 /* Support for loading and saving cookies.  The format used for
    loading and saving should be the format of the `cookies.txt' file
    used by Netscape and Mozilla, at least the Unix versions.
@@ -1346,7 +1353,7 @@ cookie_jar_save (struct cookie_jar *jar, const char *file)
 
   DEBUGP (("Done saving cookies.\n"));
 }
-
+
 /* Clean up cookie-related data. */
 
 void
@@ -1369,7 +1376,7 @@ cookie_jar_delete (struct cookie_jar *jar)
   hash_table_destroy (jar->chains);
   xfree (jar);
 }
-
+
 /* Test cases.  Currently this is only tests parse_set_cookies.  To
    use, recompile Wget with -DTEST_COOKIES and call test_cookies()
    from main.  */
@@ -1438,8 +1445,8 @@ test_cookies (void)
               printf ("Invalid value %d for '%s' (expected '%s', got '%s')\n",
                       j / 2 + 1, data, expected[j + 1], v);
             j += 2;
-            free (n);
-            free (v);
+            xfree (n);
+            xfree (v);
           }
         if (expected[j])
           printf ("Too few parameters for '%s'\n", data);

src/cookies.h

@@ -1,6 +1,6 @@
 /* Support for cookies.
    Copyright (C) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009,
-   2010, 2011 Free Software Foundation, Inc.
+   2010, 2011, 2015 Free Software Foundation, Inc.
 
 This file is part of GNU Wget.
 

src/css-tokens.h

@@ -1,5 +1,6 @@
 /* Declarations for css.lex
-   Copyright (C) 2006, 2009, 2010, 2011 Free Software Foundation, Inc.
+   Copyright (C) 2006, 2009, 2010, 2011, 2015 Free Software Foundation,
+   Inc.
 
 This file is part of GNU Wget.
 

src/css-url.c

@@ -1,6 +1,6 @@
 /* Collect URLs from CSS source.
-   Copyright (C) 1998, 2000, 2001, 2002, 2003, 2009, 2010, 2011, 2014 Free
-   Software Foundation, Inc.
+   Copyright (C) 1998, 2000, 2001, 2002, 2003, 2009, 2010, 2011, 2014,
+   2015 Free Software Foundation, Inc.
 
 This file is part of GNU Wget.
 
@@ -52,6 +52,7 @@ as that of the covered work.  */
 #include "html-url.h"
 #include "css-tokens.h"
 #include "css-url.h"
+#include "xstrndup.h"
 
 /* from lex.yy.c */
 extern char *yytext;
@@ -97,7 +98,7 @@ get_uri_string (const char *at, int *pos, int *length)
       *length -= 2;
     }
 
-  return xstrdup (at + *pos);
+  return xstrndup (at + *pos, *length);
 }
 
 void

src/css-url.h

@@ -1,5 +1,6 @@
 /* Declarations for css-url.c.
-   Copyright (C) 2006, 2009, 2010, 2011 Free Software Foundation, Inc.
+   Copyright (C) 2006, 2009, 2010, 2011, 2015 Free Software Foundation,
+   Inc.
 
 This file is part of GNU Wget.
 

src/css.l

@@ -6,7 +6,8 @@
 %{
 /* Lex source for CSS tokenizing.
    Taken from http://www.w3.org/TR/CSS21/grammar.html#q2
-   Copyright (C) 2006, 2009, 2010, 2011 Free Software Foundation, Inc.
+   Copyright (C) 2006, 2009, 2010, 2011, 2015 Free Software Foundation,
+   Inc.
 
 This file is part of GNU Wget.
 

src/exits.c

@@ -1,5 +1,6 @@
 /* Exit status handling.
-   Copyright (C) 2009, 2010, 2011, 2012 Free Software Foundation, Inc.
+   Copyright (C) 2009, 2010, 2011, 2012, 2015 Free Software Foundation,
+   Inc.
 
    This file is part of GNU Wget.
 
@@ -56,7 +57,7 @@ get_status_for_err (uerr_t err)
     case FTPNSFOD: case FTPUNKNOWNTYPE: case FTPSRVERR:
     case FTPRETRINT: case FTPRESTFAIL: case FTPNOPASV:
     case CONTNOTSUPPORTED: case RANGEERR: case RETRBADPATTERN:
-    case PROXERR:
+    case PROXERR: case GATEWAYTIMEOUT:
       return WGET_EXIT_SERVER_ERROR;
     case URLERROR: case QUOTEXC: case SSLINITFAILED: case UNKNOWNATTR:
     default:

src/exits.h

@@ -1,5 +1,6 @@
 /* Exit status related declarations.
-   Copyright (C) 2009, 2010, 2011, 2012 Free Software Foundation, Inc.
+   Copyright (C) 2009, 2010, 2011, 2012, 2015 Free Software Foundation,
+   Inc.
 
 This file is part of GNU Wget.
 

src/ftp-basic.c

@@ -1,7 +1,7 @@
 /* Basic FTP routines.
    Copyright (C) 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004,
-   2005, 2006, 2007, 2008, 2009, 2010, 2011, 2014 Free Software Foundation,
-   Inc.
+   2005, 2006, 2007, 2008, 2009, 2010, 2011, 2014, 2015 Free Software
+   Foundation, Inc.
 
 This file is part of GNU Wget.
 
@@ -43,8 +43,9 @@ as that of the covered work.  */
 #include "host.h"
 #include "ftp.h"
 #include "retr.h"
+#include "c-strcase.h"
+
 
-
 /* Get the response of FTP server and allocate enough room to handle
    it.  <CR> and <LF> characters are stripped from the line, and the
    line is 0-terminated.  All the response lines but the last one are
@@ -134,6 +135,23 @@ ftp_request (const char *command, const char *value)
   return res;
 }
 
+uerr_t
+ftp_greeting (int csock)
+{
+  uerr_t err = FTPOK;
+  char *response = NULL;
+
+  err = ftp_response (csock, &response);
+  if (err != FTPOK)
+    goto bail;
+  if (*response != '2')
+    err = FTPSRVERR;
+
+bail:
+  if (response)
+    xfree (response);
+  return err;
+}
 /* Sends the USER and PASS commands to the server, to control
    connection socket csock.  */
 uerr_t
@@ -143,16 +161,6 @@ ftp_login (int csock, const char *acc, const char *pass)
   char *request, *respline;
   int nwritten;
 
-  /* Get greeting.  */
-  err = ftp_response (csock, &respline);
-  if (err != FTPOK)
-    return err;
-  if (*respline != '2')
-    {
-      xfree (respline);
-      return FTPSRVERR;
-    }
-  xfree (respline);
   /* Send USER username.  */
   request = ftp_request ("USER", acc);
   nwritten = fd_write (csock, request, strlen (request), -1);
@@ -190,7 +198,7 @@ ftp_login (int csock, const char *acc, const char *pass)
     for (i = 0; i < countof (skey_head); i++)
       {
         int l = strlen (skey_head[i]);
-        if (0 == strncasecmp (skey_head[i], respline, l))
+        if (0 == c_strncasecmp (skey_head[i], respline, l))
           {
             seed = respline + l;
             break;
@@ -480,6 +488,119 @@ ftp_eprt (int csock, int *local_sock)
 }
 #endif
 
+#ifdef HAVE_SSL
+/*
+ * The following three functions defined into this #ifdef block
+ * wrap the extended FTP commands defined in RFC 2228 (FTP Security Extensions).
+ * Currently, only FTPS is supported, so these functions are only compiled when SSL
+ * support is available, because there's no point in using FTPS when there's no SSL.
+ * Shall someone add new secure FTP protocols in the future, feel free to remove this
+ * #ifdef, or add new constants to it.
+ */
+
+/*
+ * Sends an AUTH command as defined by RFC 2228,
+ * deriving its argument from the scheme. For example, if the provided scheme
+ * is SCHEME_FTPS, the command sent will be "AUTH TLS". Currently, this is the only
+ * scheme supported, so this function will return FTPNOAUTH when supplied a different
+ * one. It will also return FTPNOAUTH if the target server does not support FTPS.
+ */
+uerr_t
+ftp_auth (int csock, enum url_scheme scheme)
+{
+  uerr_t err = 0;
+  int written = 0;
+  char *request = NULL, *response = NULL;
+
+  if (scheme == SCHEME_FTPS)
+    {
+      request = ftp_request ("AUTH", "TLS");
+      written = fd_write (csock, request, strlen (request), -1);
+      if (written < 0)
+        {
+          err = WRITEFAILED;
+          goto bail;
+        }
+      err = ftp_response (csock, &response);
+      if (err != FTPOK)
+        goto bail;
+      if (*response != '2')
+        err = FTPNOAUTH;
+    }
+  else
+    err = FTPNOAUTH;
+
+bail:
+  xfree (request);
+  xfree (response);
+
+  return err;
+}
+
+uerr_t
+ftp_pbsz (int csock, int pbsz)
+{
+  uerr_t err = 0;
+  int written = 0;
+  char spbsz[5];
+  char *request = NULL, *response = NULL;
+
+  snprintf (spbsz, 5, "%d", pbsz);
+  request = ftp_request ("PBSZ", spbsz);
+  written = fd_write (csock, request, strlen (request), -1);
+  if (written < 0)
+    {
+      err = WRITEFAILED;
+      goto bail;
+    }
+
+  err = ftp_response (csock, &response);
+  if (err != FTPOK)
+    goto bail;
+  if (*response != '2')
+    err = FTPNOPBSZ;
+
+bail:
+  xfree (request);
+  xfree (response);
+
+  return err;
+}
+
+uerr_t
+ftp_prot (int csock, enum prot_level prot)
+{
+  uerr_t err = 0;
+  int written = 0;
+  char *request = NULL, *response = NULL;
+  /* value must be a single character value */
+  char value[2];
+
+  value[0] = prot;
+  value[1] = '\0';
+
+  request = ftp_request ("PROT", value);
+  written = fd_write (csock, request, strlen (request), -1);
+  if (written < 0)
+    {
+      err = WRITEFAILED;
+      goto bail;
+    }
+
+  err = ftp_response (csock, &response);
+  if (err != FTPOK)
+    goto bail;
+  if (*response != '2')
+    err = FTPNOPROT;
+
+bail:
+  xfree (request);
+  xfree (response);
+
+  return err;
+}
+#endif /* HAVE_SSL */
+
 /* Similar to ftp_port, but uses `PASV' to initiate the passive FTP
    transfer.  Reads the response from server and parses it.  Reads the
    host and port addresses and returns them.  */
@@ -784,16 +905,8 @@ ftp_epsv (int csock, ip_address *ip, int *port)
     }
 
   /* Finally, get the port number */
-  tport = 0;
-  for (i = 1; c_isdigit (*s); s++)
-    {
-      if (i > 5)
-        {
-          xfree (respline);
-          return FTPINVPASV;
-        }
+  for (tport = 0, i = 0; i < 5 && c_isdigit (*s); i++, s++)
       tport = (*s - '0') + 10 * tport;
-    }
 
   /* Make sure that the response terminates correcty */
   if (*s++ != delim)
@@ -965,16 +1078,18 @@ ftp_list (int csock, const char *file, bool avoid_list_a, bool avoid_list,
   bool ok = false;
   size_t i = 0;
 
-  *list_a_used = false;
-
   /* 2013-10-12 Andrea Urbani (matfanjol)
      For more information about LIST and "LIST -a" please look at ftp.c,
      function getftp, text "__LIST_A_EXPLANATION__".
 
      If somebody changes the following commands, please, checks also the
      later "i" variable.  */
-  const char *list_commands[] = { "LIST -a",
-                                  "LIST" };
+  static const char *list_commands[] = {
+    "LIST -a",
+    "LIST"
+  };
+
+  *list_a_used = false;
 
   if (avoid_list_a)
     {
@@ -1068,25 +1183,25 @@ ftp_syst (int csock, enum stype *server_type, enum ustype *unix_type)
 
   if (request == NULL)
     *server_type = ST_OTHER;
-  else if (!strcasecmp (request, "VMS"))
+  else if (!c_strcasecmp (request, "VMS"))
     *server_type = ST_VMS;
-  else if (!strcasecmp (request, "UNIX"))
+  else if (!c_strcasecmp (request, "UNIX"))
     {
       *server_type = ST_UNIX;
       /* 2013-10-17 Andrea Urbani (matfanjol)
          I check more in depth the system type */
-      if (!strncasecmp (ftp_last_respline, "215 UNIX Type: L8", 17))
+      if (!c_strncasecmp (ftp_last_respline, "215 UNIX Type: L8", 17))
         *unix_type = UST_TYPE_L8;
-      else if (!strncasecmp (ftp_last_respline,
+      else if (!c_strncasecmp (ftp_last_respline,
                              "215 UNIX MultiNet Unix Emulation V5.3(93)", 41))
         *unix_type = UST_MULTINET;
     }
-  else if (!strcasecmp (request, "WINDOWS_NT")
-           || !strcasecmp (request, "WINDOWS2000"))
+  else if (!c_strcasecmp (request, "WINDOWS_NT")
+           || !c_strcasecmp (request, "WINDOWS2000"))
     *server_type = ST_WINNT;
-  else if (!strcasecmp (request, "MACOS"))
+  else if (!c_strcasecmp (request, "MACOS"))
     *server_type = ST_MACOS;
-  else if (!strcasecmp (request, "OS/400"))
+  else if (!c_strcasecmp (request, "OS/400"))
     *server_type = ST_OS400;
   else
     *server_type = ST_OTHER;
@@ -1135,7 +1250,7 @@ ftp_pwd (int csock, char **pwd)
     goto err;
 
   /* Has the `pwd' been already allocated?  Free! */
-  xfree_null (*pwd);
+  xfree (*pwd);
 
   *pwd = xstrdup (request);
 

src/ftp-ls.c

@@ -1,7 +1,7 @@
 /* Parsing FTP `ls' output.
    Copyright (C) 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004,
-   2005, 2006, 2007, 2008, 2009, 2010, 2011 Free Software Foundation,
-   Inc.
+   2005, 2006, 2007, 2008, 2009, 2010, 2011, 2015 Free Software
+   Foundation, Inc.
 
 This file is part of GNU Wget.
 
@@ -42,6 +42,7 @@ as that of the covered work.  */
 #include "url.h"
 #include "convert.h"            /* for html_quote_string prototype */
 #include "retr.h"               /* for output_stream */
+#include "c-strcase.h"
 
 /* Converts symbolic permissions to number-style ones, e.g. string
    rwxr-xr-x to 755.  For now, it knows nothing of
@@ -121,7 +122,7 @@ ftp_parse_unix_ls (const char *file, int ignore_perms)
     {
       len = clean_line (line, len);
       /* Skip if total...  */
-      if (!strncasecmp (line, "total", 5))
+      if (!c_strncasecmp (line, "total", 5))
         continue;
       /* Get the first token (permissions).  */
       tok = strtok (line, " ");
@@ -199,7 +200,7 @@ ftp_parse_unix_ls (const char *file, int ignore_perms)
           if (next < 0)         /* a month name was not encountered */
             {
               for (i = 0; i < 12; i++)
-                if (!strcasecmp (tok, months[i]))
+                if (!c_strcasecmp (tok, months[i]))
                   break;
               /* If we got a month, it means the token before it is the
                  size, and the filename is three tokens away.  */
@@ -362,8 +363,8 @@ ftp_parse_unix_ls (const char *file, int ignore_perms)
       if (error || ignore)
         {
           DEBUGP (("Skipping.\n"));
-          xfree_null (cur.name);
-          xfree_null (cur.linkto);
+          xfree (cur.name);
+          xfree (cur.linkto);
           continue;
         }
 
@@ -775,14 +776,14 @@ ftp_parse_vms_ls (const char *file)
          what will work in a CWD command.
       */
       len = strlen (tok);
-      if (!strncasecmp((tok + (len - 4)), ".DIR", 4))
+      if (!c_strncasecmp((tok + (len - 4)), ".DIR", 4))
         {
           *(tok+ (len - 4)) = '\0'; /* Discard ".DIR". */
           cur.type  = FT_DIRECTORY;
           cur.perms = VMS_DEFAULT_PROT_DIR;
           DEBUGP (("Directory (nv)\n"));
         }
-      else if (!strncasecmp ((tok + (len - 6)), ".DIR;1", 6))
+      else if (!c_strncasecmp ((tok + (len - 6)), ".DIR;1", 6))
         {
           *(tok+ (len - 6)) = '\0'; /* Discard ".DIR;1". */
           cur.type  = FT_DIRECTORY;
@@ -1051,7 +1052,7 @@ Unsupported listing type, trying Unix listing parser.\n"));
       return ftp_parse_unix_ls (file, 0);
     }
 }
-
+
 /* Stuff for creating FTP index. */
 
 /* The function creates an HTML index containing references to given
@@ -1088,7 +1089,7 @@ ftp_index (const char *file, struct url *u, struct fileinfo *f)
       else
         upwd = concat_strings (tmpu, "@", (char *) 0);
       xfree (tmpu);
-      xfree_null (tmpp);
+      xfree (tmpp);
     }
   else
     upwd = xstrdup ("");

src/ftp-opie.c

@@ -1,6 +1,6 @@
 /* Opie (s/key) support for FTP.
    Copyright (C) 1998, 1999, 2000, 2001, 2002, 2003, 2004, 2005, 2006,
-   2007, 2008, 2009, 2010, 2011 Free Software Foundation, Inc.
+   2007, 2008, 2009, 2010, 2011, 2015 Free Software Foundation, Inc.
 
 This file is part of GNU Wget.
 

src/ftp.c

@@ -1,7 +1,7 @@
 /* File Transfer Protocol support.
    Copyright (C) 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004,
-   2005, 2006, 2007, 2008, 2009, 2010, 2011, 2014 Free Software Foundation,
-   Inc.
+   2005, 2006, 2007, 2008, 2009, 2010, 2011, 2014, 2015 Free Software
+   Foundation, Inc.
 
 This file is part of GNU Wget.
 
@@ -44,12 +44,14 @@ as that of the covered work.  */
 #include "url.h"
 #include "retr.h"
 #include "ftp.h"
+#include "ssl.h"
 #include "connect.h"
 #include "host.h"
 #include "netrc.h"
 #include "convert.h"            /* for downloaded_file */
 #include "recur.h"              /* for INFINITE_RECURSION */
 #include "warc.h"
+#include "c-strcase.h"
 
 #ifdef __VMS
 # include "vms.h"
@@ -76,7 +78,6 @@ typedef struct
   struct url *proxy;            /* FTWK-style proxy */
 } ccon;
 
-extern int numurls;
 
 /* Look for regexp "( *[0-9]+ *byte" (literal parenthesis) anywhere in
    the string S, and return the number converted to wgint, if found, 0
@@ -102,7 +103,7 @@ ftp_expected_bytes (const char *s)
         return 0;
       if (c_tolower (*s) != 'b')
         continue;
-      if (strncasecmp (s, "byte", 4))
+      if (c_strncasecmp (s, "byte", 4))
         continue;
       else
         break;
@@ -237,6 +238,78 @@ print_length (wgint size, wgint start, bool authoritative)
 
 static uerr_t ftp_get_listing (struct url *, ccon *, struct fileinfo **);
 
+static uerr_t
+get_ftp_greeting(int csock, ccon *con)
+{
+  uerr_t err = 0;
+
+  /* Get the server's greeting */
+  err = ftp_greeting (csock);
+  if (err != FTPOK)
+    {
+      logputs (LOG_NOTQUIET, "Error in server response. Closing.\n");
+      fd_close (csock);
+      con->csock = -1;
+    }
+
+  return err;
+}
+
+#ifdef HAVE_SSL
+static uerr_t
+init_control_ssl_connection (int csock, struct url *u, bool *using_control_security)
+{
+  bool using_security = false;
+
+  /* If '--ftps-implicit' was passed, perform the SSL handshake directly,
+   * and do not send an AUTH command.
+   * Otherwise send an AUTH sequence before login,
+   * and perform the SSL handshake if accepted by server.
+   */
+  if (!opt.ftps_implicit && !opt.server_response)
+    logputs (LOG_VERBOSE, "==> AUTH TLS ... ");
+  if (opt.ftps_implicit || ftp_auth (csock, SCHEME_FTPS) == FTPOK)
+    {
+      if (!ssl_connect_wget (csock, u->host, NULL))
+        {
+          fd_close (csock);
+          return CONSSLERR;
+        }
+      else if (!ssl_check_certificate (csock, u->host))
+        {
+          fd_close (csock);
+          return VERIFCERTERR;
+        }
+
+      if (!opt.ftps_implicit && !opt.server_response)
+        logputs (LOG_VERBOSE, " done.\n");
+
+      /* If implicit FTPS was requested, we act as "normal" FTP, but over SSL.
+       * We're not using RFC 2228 commands.
+       */
+      using_security = true;
+    }
+  else
+    {
+      /* The server does not support 'AUTH TLS'.
+       * Check if --ftps-fallback-to-ftp was passed. */
+      if (opt.ftps_fallback_to_ftp)
+        {
+          logputs (LOG_NOTQUIET, "Server does not support AUTH TLS. Falling back to FTP.\n");
+          using_security = false;
+        }
+      else
+        {
+          fd_close (csock);
+          return FTPNOAUTH;
+        }
+    }
+
+  *using_control_security = using_security;
+  return NOCONERROR;
+}
+#endif
+
 /* Retrieves a file with denoted parameters through opening an FTP
    connection to the server.  It always closes the data connection,
    and closes the control connection in case of error.  If warc_tmp
@@ -248,11 +321,11 @@ getftp (struct url *u, wgint passed_expected_bytes, wgint *qtyread,
 {
   int csock, dtsock, local_sock, res;
   uerr_t err = RETROK;          /* appease the compiler */
-  FILE *fp;
+  FILE *fp = NULL;
+  struct_fstat st;
   char *respline, *tms;
   const char *user, *passwd, *tmrate;
   int cmd = con->cmd;
-  bool pasv_mode_open = false;
   wgint expected_bytes = 0;
   bool got_expected_bytes = false;
   bool rest_failed = false;
@@ -261,6 +334,15 @@ getftp (struct url *u, wgint passed_expected_bytes, wgint *qtyread,
   char type_char;
   bool try_again;
   bool list_a_used = false;
+#ifdef HAVE_SSL
+  enum prot_level prot = (opt.ftps_clear_data_connection ? PROT_CLEAR : PROT_PRIVATE);
+  /* these variables tell whether the target server
+   * accepts the security extensions (RFC 2228) or not,
+   * and whether we're actually using any of them
+   * (encryption at the control connection only,
+   * or both at control and data connections) */
+  bool using_control_security = false, using_data_security = false;
+#endif
 
   assert (con != NULL);
   assert (con->target != NULL);
@@ -286,8 +368,36 @@ getftp (struct url *u, wgint passed_expected_bytes, wgint *qtyread,
   local_sock = -1;
   con->dltime = 0;
 
+#ifdef HAVE_SSL
+  if (u->scheme == SCHEME_FTPS)
+    {
+      /* Initialize SSL layer first */
+      if (!ssl_init ())
+        {
+          scheme_disable (SCHEME_FTPS);
+          logprintf (LOG_NOTQUIET, _("Could not initialize SSL. It will be disabled."));
+          err = SSLINITFAILED;
+          return err;
+        }
+
+      /* If we're using the default FTP port and implicit FTPS was requested,
+       * rewrite the port to the default *implicit* FTPS port.
+       */
+      if (opt.ftps_implicit && u->port == DEFAULT_FTP_PORT)
+        {
+          DEBUGP (("Implicit FTPS was specified. Rewriting default port to %d.\n", DEFAULT_FTPS_IMPLICIT_PORT));
+          u->port = DEFAULT_FTPS_IMPLICIT_PORT;
+        }
+    }
+#endif
+
   if (!(cmd & DO_LOGIN))
-    csock = con->csock;
+    {
+      csock = con->csock;
+#ifdef HAVE_SSL
+      using_data_security = con->st & DATA_CHANNEL_SECURITY;
+#endif
+    }
   else                          /* cmd & DO_LOGIN */
     {
       char    *host = con->proxy ? con->proxy->host : u->host;
@@ -309,6 +419,43 @@ getftp (struct url *u, wgint passed_expected_bytes, wgint *qtyread,
       else
         con->csock = -1;
 
+#ifdef HAVE_SSL
+      if (u->scheme == SCHEME_FTPS)
+        {
+          /* If we're in implicit FTPS mode, we have to set up SSL/TLS before everything else.
+           * Otherwise we first read the server's greeting, and then send an "AUTH TLS".
+           */
+          if (opt.ftps_implicit)
+            {
+              err = init_control_ssl_connection (csock, u, &using_control_security);
+              if (err != NOCONERROR)
+                return err;
+              err = get_ftp_greeting (csock, con);
+              if (err != FTPOK)
+                return err;
+            }
+          else
+            {
+              err = get_ftp_greeting (csock, con);
+              if (err != FTPOK)
+                return err;
+              err = init_control_ssl_connection (csock, u, &using_control_security);
+              if (err != NOCONERROR)
+                return err;
+            }
+        }
+      else
+        {
+          err = get_ftp_greeting (csock, con);
+          if (err != FTPOK)
+            return err;
+        }
+#else
+      err = get_ftp_greeting (csock, con);
+      if (err != FTPOK)
+        return err;
+#endif
+
       /* Second: Login with proper USER/PASS sequence.  */
       logprintf (LOG_VERBOSE, _("Logging in as %s ... "),
                  quotearg_style (escape_quoting_style, user));
@@ -366,6 +513,46 @@ Error in server response, closing control connection.\n"));
         default:
           abort ();
         }
+
+#ifdef HAVE_SSL
+      if (using_control_security)
+        {
+          /* Send the PBSZ and PROT commands, in that order.
+           * If we are here it means that the server has already accepted
+           * some form of FTPS. Thus, these commands must work.
+           * If they don't work, that's an error. There's no sense in honoring
+           * --ftps-fallback-to-ftp or similar options. */
+          if (u->scheme == SCHEME_FTPS)
+            {
+              if (!opt.server_response)
+                logputs (LOG_VERBOSE, "==> PBSZ 0 ... ");
+              if ((err = ftp_pbsz (csock, 0)) == FTPNOPBSZ)
+                {
+                  logputs (LOG_NOTQUIET, _("Server did not accept the 'PBSZ 0' command.\n"));
+                  return err;
+                }
+              if (!opt.server_response)
+                logputs (LOG_VERBOSE, "done.");
+
+              if (!opt.server_response)
+                logprintf (LOG_VERBOSE, "  ==> PROT %c ... ", prot);
+              if ((err = ftp_prot (csock, prot)) == FTPNOPROT)
+                {
+                  logprintf (LOG_NOTQUIET, _("Server did not accept the 'PROT %c' command.\n"), prot);
+                  return err;
+                }
+              if (!opt.server_response)
+                logputs (LOG_VERBOSE, "done.\n");
+
+              if (prot != PROT_CLEAR)
+                {
+                  using_data_security = true;
+                  con->st |= DATA_CHANNEL_SECURITY;
+                }
+            }
+        }
+#endif
+
       /* Third: Get the system type */
       if (!opt.server_response)
         logprintf (LOG_VERBOSE, "==> SYST ... ");
@@ -449,7 +636,7 @@ Error in server response, closing control connection.\n"));
           return err;
         case FTPSRVERR :
           /* PWD unsupported -- assume "/". */
-          xfree_null (con->id);
+          xfree (con->id);
           con->id = xstrdup ("/");
           break;
         case FTPOK:
@@ -700,74 +887,78 @@ Error in server response, closing control connection.\n"));
 
           for (cwd_count = cwd_start; cwd_count < cwd_end; cwd_count++)
             {
-          switch (cwd_count)
-            {
-              case 0:
-                /* Step one (optional): Go to the initial directory,
-                   exactly as reported by the server.
-                */
-                targ = con->id;
-                break;
+              switch (cwd_count)
+                {
+                  case 0:
+                    /* Step one (optional): Go to the initial directory,
+                       exactly as reported by the server.
+                    */
+                    targ = con->id;
+                    break;
 
-              case 1:
-                /* Step two: Go to the target directory.  (Absolute or
-                   relative will work now.)
-                */
-                targ = target;
-                break;
+                  case 1:
+                    /* Step two: Go to the target directory.  (Absolute or
+                       relative will work now.)
+                    */
+                    targ = target;
+                    break;
 
-              case 2:
-                /* Step three (optional): "CWD []" to restore server
-                   VMS-ness.
-                */
-                targ = "[]";
-                break;
+                  case 2:
+                    /* Step three (optional): "CWD []" to restore server
+                       VMS-ness.
+                    */
+                    targ = "[]";
+                    break;
 
-              default:
-                /* Can't happen. */
-                assert (1);
-            }
+                  default:
+                    logprintf (LOG_ALWAYS, _("Logically impossible section reached in getftp()"));
+                    logprintf (LOG_ALWAYS, _("cwd_count: %d\ncwd_start: %d\ncwd_end: %d\n"),
+                                             cwd_count, cwd_start, cwd_end);
+                    abort ();
+                }
 
-          if (!opt.server_response)
-            logprintf (LOG_VERBOSE, "==> CWD (%d) %s ... ", cwd_count,
-                       quotearg_style (escape_quoting_style, target));
-          err = ftp_cwd (csock, targ);
-          /* FTPRERR, WRITEFAILED, FTPNSFOD */
-          switch (err)
-            {
-            case FTPRERR:
-              logputs (LOG_VERBOSE, "\n");
-              logputs (LOG_NOTQUIET, _("\
+              if (!opt.server_response)
+                logprintf (LOG_VERBOSE, "==> CWD (%d) %s ... ", cwd_count,
+                           quotearg_style (escape_quoting_style, target));
+
+              err = ftp_cwd (csock, targ);
+
+              /* FTPRERR, WRITEFAILED, FTPNSFOD */
+              switch (err)
+                {
+                  case FTPRERR:
+                    logputs (LOG_VERBOSE, "\n");
+                    logputs (LOG_NOTQUIET, _("\
 Error in server response, closing control connection.\n"));
-              fd_close (csock);
-              con->csock = -1;
-              return err;
-            case WRITEFAILED:
-              logputs (LOG_VERBOSE, "\n");
-              logputs (LOG_NOTQUIET,
-                       _("Write failed, closing control connection.\n"));
-              fd_close (csock);
-              con->csock = -1;
-              return err;
-            case FTPNSFOD:
-              logputs (LOG_VERBOSE, "\n");
-              logprintf (LOG_NOTQUIET, _("No such directory %s.\n\n"),
-                         quote (u->dir));
-              fd_close (csock);
-              con->csock = -1;
-              return err;
-            case FTPOK:
-              break;
-            default:
-              abort ();
-            }
-          if (!opt.server_response)
-            logputs (LOG_VERBOSE, _("done.\n"));
+                    fd_close (csock);
+                    con->csock = -1;
+                    return err;
+                  case WRITEFAILED:
+                    logputs (LOG_VERBOSE, "\n");
+                    logputs (LOG_NOTQUIET,
+                             _("Write failed, closing control connection.\n"));
+                    fd_close (csock);
+                    con->csock = -1;
+                    return err;
+                  case FTPNSFOD:
+                    logputs (LOG_VERBOSE, "\n");
+                    logprintf (LOG_NOTQUIET, _("No such directory %s.\n\n"),
+                               quote (u->dir));
+                    fd_close (csock);
+                    con->csock = -1;
+                    return err;
+                  case FTPOK:
+                    break;
+                  default:
+                    abort ();
+                }
 
-        } /* for */
+              if (!opt.server_response)
+                logputs (LOG_VERBOSE, _("done.\n"));
+
+            } /* for */
 
           /* 2004-09-20 SMS. */
-          /* End of deviant indenting. */
 
         } /* else */
     }
@@ -879,13 +1070,19 @@ Error in server response, closing control connection.\n"));
                           ? CONERROR : CONIMPOSSIBLE);
                 }
 
-              pasv_mode_open = true;  /* Flag to avoid accept port */
               if (!opt.server_response)
                 logputs (LOG_VERBOSE, _("done.    "));
-            } /* err==FTP_OK */
-        }
+            }
+          else
+            return err;
 
-      if (!pasv_mode_open)   /* Try to use a port command if PASV failed */
+          /*
+           * We do not want to fall back from PASSIVE mode to ACTIVE mode !
+           * The reason is the PORT command exposes the client's real IP address
+           * to the server. Bad for someone who relies on privacy via a ftp proxy.
+           */
+        }
+      else
         {
           err = ftp_do_port (csock, &local_sock);
           /* FTPRERR, WRITEFAILED, bindport (FTPSYSERR), HOSTERR,
@@ -1144,8 +1341,8 @@ Error in server response, closing control connection.\n"));
     }
 
   /* If no transmission was required, then everything is OK.  */
-  if (!pasv_mode_open)  /* we are not using pasive mode so we need
-                              to accept */
+  if (!opt.ftp_pasv)  /* we are not using passive mode so we need
+                         to accept */
     {
       /* Wait for the server to connect to the address we're waiting
          at.  */
@@ -1304,6 +1501,38 @@ Error in server response, closing control connection.\n"));
   else if (expected_bytes)
     print_length (expected_bytes, restval, false);
 
+#ifdef HAVE_SSL
+  if (u->scheme == SCHEME_FTPS && using_data_security)
+    {
+      /* We should try to restore the existing SSL session in the data connection
+       * and fall back to establishing a new session if the server doesn't want to restore it.
+       */
+      if (!opt.ftps_resume_ssl || !ssl_connect_wget (dtsock, u->host, &csock))
+        {
+          if (opt.ftps_resume_ssl)
+            logputs (LOG_NOTQUIET, "Server does not want to resume the SSL session. Trying with a new one.\n");
+          if (!ssl_connect_wget (dtsock, u->host, NULL))
+            {
+              fd_close (csock);
+              fd_close (dtsock);
+              err = CONERROR;
+              logputs (LOG_NOTQUIET, "Could not perform SSL handshake.\n");
+              goto exit_error;
+            }
+        }
+      else
+        logputs (LOG_NOTQUIET, "Resuming SSL session in data connection.\n");
+
+      if (!ssl_check_certificate (dtsock, u->host))
+        {
+          fd_close (csock);
+          fd_close (dtsock);
+          err = CONERROR;
+          goto exit_error;
+        }
+    }
+#endif
+
   /* Get the contents of the document.  */
   flags = 0;
   if (restval && rest_failed)
@@ -1346,7 +1575,6 @@ Error in server response, closing control connection.\n"));
 
   /* Get the server to tell us if everything is retrieved.  */
   err = ftp_response (csock, &respline);
-  *last_expected_bytes = ftp_expected_bytes (respline);
   if (err != FTPOK)
     {
       /* The control connection is decidedly closed.  Print the time
@@ -1362,16 +1590,25 @@ Error in server response, closing control connection.\n"));
       con->csock = -1;
       return FTPRETRINT;
     } /* err != FTPOK */
+  *last_expected_bytes = ftp_expected_bytes (respline);
   /* If retrieval failed for any reason, return FTPRETRINT, but do not
      close socket, since the control connection is still alive.  If
      there is something wrong with the control connection, it will
      become apparent later.  */
   if (*respline != '2')
     {
-      xfree (respline);
       if (res != -1)
         logprintf (LOG_NOTQUIET, "%s (%s) - ", tms, tmrate);
       logputs (LOG_NOTQUIET, _("Data transfer aborted.\n"));
+#ifdef HAVE_SSL
+      if (!c_strncasecmp (respline, "425", 3) && u->scheme == SCHEME_FTPS)
+        {
+          logputs (LOG_NOTQUIET, "FTPS server rejects new SSL sessions in the data connection.\n");
+          xfree (respline);
+          return FTPRESTFAIL;
+        }
+#endif
+      xfree (respline);
       return FTPRETRINT;
     }
   xfree (respline);
@@ -1528,6 +1765,13 @@ Error in server response, closing control connection.\n"));
     }
   } while (try_again);
   return RETRFINISHED;
+
+exit_error:
+
+  /* If fp is a regular file, close and try to remove it */
+  if (fp && !output_stream)
+    fclose (fp);
+  return err;
 }
 
 /* A one-file FTP loop.  This is the part where FTP retrieval is
@@ -1536,7 +1780,8 @@ Error in server response, closing control connection.\n"));
    This loop either gets commands from con, or (if ON_YOUR_OWN is
    set), makes them up to retrieve the file given by the URL.  */
 static uerr_t
-ftp_loop_internal (struct url *u, struct fileinfo *f, ccon *con, char **local_file)
+ftp_loop_internal (struct url *u, struct fileinfo *f, ccon *con, char **local_file,
+                   bool force_full_retrieve)
 {
   int count, orig_lp;
   wgint restval, len = 0, qtyread = 0;
@@ -1560,6 +1805,7 @@ ftp_loop_internal (struct url *u, struct fileinfo *f, ccon *con, char **local_fi
   else
     {
       /* URL-derived file.  Consider "-O file" name. */
+      xfree (con->target);
       con->target = url_file_name (u, NULL);
       if (!opt.output_document)
         locf = con->target;
@@ -1637,6 +1883,8 @@ ftp_loop_internal (struct url *u, struct fileinfo *f, ccon *con, char **local_fi
       /* Decide whether or not to restart.  */
       if (con->cmd & DO_LIST)
         restval = 0;
+      else if (force_full_retrieve)
+        restval = 0;
       else if (opt.start_pos >= 0)
         restval = opt.start_pos;
       else if (opt.always_rest
@@ -1687,11 +1935,17 @@ ftp_loop_internal (struct url *u, struct fileinfo *f, ccon *con, char **local_fi
       switch (err)
         {
         case HOSTERR: case CONIMPOSSIBLE: case FWRITEERR: case FOPENERR:
-        case FTPNSFOD: case FTPLOGINC: case FTPNOPASV: case CONTNOTSUPPORTED:
-        case UNLINKERR: case WARC_TMP_FWRITEERR:
+        case FTPNSFOD: case FTPLOGINC: case FTPNOPASV: case FTPNOAUTH: case FTPNOPBSZ: case FTPNOPROT:
+        case UNLINKERR: case WARC_TMP_FWRITEERR: case CONSSLERR: case CONTNOTSUPPORTED:
+#ifdef HAVE_SSL
+          if (err == FTPNOAUTH)
+            logputs (LOG_NOTQUIET, "Server does not support AUTH TLS.\n");
+          if (opt.ftps_implicit)
+            logputs (LOG_NOTQUIET, "Server does not like implicit FTPS connections.\n");
+#endif
           /* Fatal errors, give up.  */
           if (warc_tmp != NULL)
-            fclose (warc_tmp);
+              fclose (warc_tmp);
           return err;
         case CONSOCKERR: case CONERROR: case FTPSRVERR: case FTPRERR:
         case WRITEFAILED: case FTPUNKNOWNTYPE: case FTPSYSERR:
@@ -1702,7 +1956,7 @@ ftp_loop_internal (struct url *u, struct fileinfo *f, ccon *con, char **local_fi
           if (err == FOPEN_EXCL_ERR)
             {
               /* Re-determine the file name. */
-              xfree_null (con->target);
+              xfree (con->target);
               con->target = url_file_name (u, NULL);
               locf = con->target;
             }
@@ -1766,10 +2020,12 @@ ftp_loop_internal (struct url *u, struct fileinfo *f, ccon *con, char **local_fi
 
           warc_res = warc_write_resource_record (NULL, u->url, NULL, NULL,
                                                   warc_ip, NULL, warc_tmp, -1);
+
           if (! warc_res)
             return WARC_ERR;
 
           /* warc_write_resource_record has also closed warc_tmp. */
+          warc_tmp = NULL;
         }
 
       if (con->cmd & DO_LIST)
@@ -1816,6 +2072,9 @@ Removing file due to --delete-after in ftp_loop_internal():\n"));
       if (local_file)
         *local_file = xstrdup (locf);
 
+      if (warc_tmp != NULL)
+        fclose (warc_tmp);
+
       return RETROK;
     } while (!opt.ntry || (count < opt.ntry));
 
@@ -1824,6 +2083,10 @@ Removing file due to --delete-after in ftp_loop_internal():\n"));
       fd_close (con->csock);
       con->csock = -1;
     }
+
+  if (warc_tmp != NULL)
+    fclose (warc_tmp);
+
   return TRYLIMEXC;
 }
 
@@ -1851,7 +2114,7 @@ ftp_get_listing (struct url *u, ccon *con, struct fileinfo **f)
 
   con->target = xstrdup (lf);
   xfree (lf);
-  err = ftp_loop_internal (u, NULL, con, NULL);
+  err = ftp_loop_internal (u, NULL, con, NULL, false);
   lf = xstrdup (con->target);
   xfree (con->target);
   con->target = old_target;
@@ -1896,6 +2159,7 @@ ftp_retrieve_list (struct url *u, struct fileinfo *f, ccon *con)
   time_t tml;
   bool dlthis; /* Download this (file). */
   const char *actual_target = NULL;
+  bool force_full_retrieve = false;
 
   /* Increase the depth.  */
   ++depth;
@@ -1975,9 +2239,10 @@ ftp_retrieve_list (struct url *u, struct fileinfo *f, ccon *con)
 Remote file no newer than local file %s -- not retrieving.\n"), quote (con->target));
                   dlthis = false;
                 }
-              else if (eq_size)
+              else if (f->tstamp > tml)
                 {
-                  /* Remote file is newer or sizes cannot be matched */
+                  /* Remote file is newer */
+                  force_full_retrieve = true;
                   logprintf (LOG_VERBOSE, _("\
 Remote file is newer than local file %s -- retrieving.\n\n"),
                              quote (con->target));
@@ -2046,7 +2311,7 @@ Already have correct symlink %s -> %s\n\n"),
           else                /* opt.retr_symlinks */
             {
               if (dlthis)
-                err = ftp_loop_internal (u, f, con, NULL);
+                err = ftp_loop_internal (u, f, con, NULL, force_full_retrieve);
             } /* opt.retr_symlinks */
           break;
         case FT_DIRECTORY:
@@ -2057,7 +2322,7 @@ Already have correct symlink %s -> %s\n\n"),
         case FT_PLAINFILE:
           /* Call the retrieve loop.  */
           if (dlthis)
-            err = ftp_loop_internal (u, f, con, NULL);
+            err = ftp_loop_internal (u, f, con, NULL, force_full_retrieve);
           break;
         case FT_UNKNOWN:
           logprintf (LOG_NOTQUIET, _("%s: unknown/unsupported file type.\n"),
@@ -2211,6 +2476,29 @@ has_insecure_name_p (const char *s)
   return false;
 }
 
+/* Test if the file node is invalid. This can occur due to malformed or
+ * maliciously crafted listing files being returned by the server.
+ *
+ * Currently, this function only tests if there are multiple entries in the
+ * listing file by the same name. However this function can be expanded as more
+ * such illegal listing formats are discovered. */
+static bool
+is_invalid_entry (struct fileinfo *f)
+{
+  struct fileinfo *cur = f;
+  char *f_name = f->name;
+
+  /* If the node we're currently checking has a duplicate later, we eliminate
+   * the current node and leave the next one intact. */
+  while (cur->next)
+    {
+      cur = cur->next;
+      if (strcmp(f_name, cur->name) == 0)
+          return true;
+    }
+  return false;
+}
+
 /* A near-top-level function to retrieve the files in a directory.
    The function calls ftp_get_listing, to get a linked list of files.
    Then it weeds out the file names that do not match the pattern.
@@ -2248,11 +2536,11 @@ ftp_retrieve_glob (struct url *u, ccon *con, int action)
             f = f->next;
         }
     }
-  /* Remove all files with possible harmful names */
+  /* Remove all files with possible harmful names or invalid entries. */
   f = start;
   while (f)
     {
-      if (has_insecure_name_p (f->name))
+      if (has_insecure_name_p (f->name) || is_invalid_entry (f))
         {
           logprintf (LOG_VERBOSE, _("Rejecting %s.\n"),
                      quote (f->name));
@@ -2340,7 +2628,7 @@ ftp_retrieve_glob (struct url *u, ccon *con, int action)
         {
           /* Let's try retrieving it anyway.  */
           con->st |= ON_YOUR_OWN;
-          res = ftp_loop_internal (u, NULL, con, NULL);
+          res = ftp_loop_internal (u, NULL, con, NULL, false);
           return res;
         }
 
@@ -2440,7 +2728,7 @@ ftp_loop (struct url *u, char **local_file, int *dt, struct url *proxy,
                                    ispattern ? GLOB_GLOBALL : GLOB_GETONE);
         }
       else
-        res = ftp_loop_internal (u, NULL, &con, local_file);
+        res = ftp_loop_internal (u, NULL, &con, local_file, false);
     }
   if (res == FTPOK)
     res = RETROK;
@@ -2449,10 +2737,8 @@ ftp_loop (struct url *u, char **local_file, int *dt, struct url *proxy,
   /* If a connection was left, quench it.  */
   if (con.csock != -1)
     fd_close (con.csock);
-  xfree_null (con.id);
-  con.id = NULL;
-  xfree_null (con.target);
-  con.target = NULL;
+  xfree (con.id);
+  xfree (con.target);
   return res;
 }
 
@@ -2466,7 +2752,7 @@ delelement (struct fileinfo *f, struct fileinfo **start)
   struct fileinfo *next = f->next;
 
   xfree (f->name);
-  xfree_null (f->linkto);
+  xfree (f->linkto);
   xfree (f);
 
   if (next)

src/ftp.h

@@ -1,7 +1,7 @@
 /* Declarations for FTP support.
    Copyright (C) 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004,
-   2005, 2006, 2007, 2008, 2009, 2010, 2011 Free Software Foundation,
-   Inc.
+   2005, 2006, 2007, 2008, 2009, 2010, 2011, 2015 Free Software
+   Foundation, Inc.
 
 This file is part of GNU Wget.
 
@@ -33,6 +33,7 @@ as that of the covered work.  */
 #define FTP_H
 
 #include "host.h"
+#include "url.h"
 
 /* System types. */
 enum stype
@@ -53,12 +54,27 @@ enum ustype
   UST_OTHER
 };
 
-extern char ftp_last_respline[];
+#ifdef HAVE_SSL
+/* Data channel protection levels (to be used with PBSZ) */
+enum prot_level
+{
+  PROT_CLEAR = 'C',
+  PROT_SAFE = 'S',
+  PROT_CONFIDENTIAL = 'E',
+  PROT_PRIVATE = 'P'
+};
+#endif
 
 uerr_t ftp_response (int, char **);
+uerr_t ftp_greeting (int);
 uerr_t ftp_login (int, const char *, const char *);
 uerr_t ftp_port (int, int *);
 uerr_t ftp_pasv (int, ip_address *, int *);
+#ifdef HAVE_SSL
+uerr_t ftp_auth (int, enum url_scheme);
+uerr_t ftp_pbsz (int, int);
+uerr_t ftp_prot (int, enum prot_level);
+#endif
 #ifdef ENABLE_IPV6
 uerr_t ftp_lprt (int, int *);
 uerr_t ftp_lpsv (int, ip_address *, int *);
@@ -144,11 +160,12 @@ enum wget_ftp_fstatus
   AVOID_LIST    = 0x0008,   /* It tells us if during this
                                session we have to avoid to use
                                "LIST". */
-  LIST_AFTER_LIST_A_CHECK_DONE  = 0x0010
+  LIST_AFTER_LIST_A_CHECK_DONE  = 0x0010,
                             /* It tells us if we have already
                                checked "LIST" after the first
                                "LIST -a" to handle the case of
                                file/folders named "-a". */
+  DATA_CHANNEL_SECURITY = 0x0020 /* Establish a secure data channel */
 };
 
 struct fileinfo *ftp_parse_ls (const char *, const enum stype);

src/gettext.h

@@ -1,272 +0,0 @@
-/* Convenience header for conditional use of GNU <libintl.h>.
-   Copyright (C) 1995, 1996, 1997, 1998, 2000, 2001, 2002, 2004, 2005,
-   2006, 2009, 2010, 2011 Free Software Foundation, Inc.
-
-   This program is free software; you can redistribute it and/or modify it
-   under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 2, or (at your option)
-   any later version.
-
-   This program is distributed in the hope that it will be useful,
-   but WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Library General Public License for more details.
-
-   You should have received a copy of the GNU General Public
-   License along with this program; if not, write to the Free Software
-   Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301,
-   USA.  */
-
-#ifndef _LIBGETTEXT_H
-#define _LIBGETTEXT_H 1
-
-/* NLS can be disabled through the configure --disable-nls option.  */
-#if ENABLE_NLS
-
-/* Get declarations of GNU message catalog functions.  */
-# include <libintl.h>
-
-/* You can set the DEFAULT_TEXT_DOMAIN macro to specify the domain used by
-   the gettext() and ngettext() macros.  This is an alternative to calling
-   textdomain(), and is useful for libraries.  */
-# ifdef DEFAULT_TEXT_DOMAIN
-#  undef gettext
-#  define gettext(Msgid) \
-     dgettext (DEFAULT_TEXT_DOMAIN, Msgid)
-#  undef ngettext
-#  define ngettext(Msgid1, Msgid2, N) \
-     dngettext (DEFAULT_TEXT_DOMAIN, Msgid1, Msgid2, N)
-# endif
-
-#else
-
-/* Solaris /usr/include/locale.h includes /usr/include/libintl.h, which
-   chokes if dcgettext is defined as a macro.  So include it now, to make
-   later inclusions of <locale.h> a NOP.  We don't include <libintl.h>
-   as well because people using "gettext.h" will not include <libintl.h>,
-   and also including <libintl.h> would fail on SunOS 4, whereas <locale.h>
-   is OK.  */
-#if defined(__sun)
-# include <locale.h>
-#endif
-
-/* Many header files from the libstdc++ coming with g++ 3.3 or newer include
-   <libintl.h>, which chokes if dcgettext is defined as a macro.  So include
-   it now, to make later inclusions of <libintl.h> a NOP.  */
-#if defined(__cplusplus) && defined(__GNUG__) && (__GNUC__ >= 3)
-# include <cstdlib>
-# if (__GLIBC__ >= 2) || _GLIBCXX_HAVE_LIBINTL_H
-#  include <libintl.h>
-# endif
-#endif
-
-/* Disabled NLS.
-   The casts to 'const char *' serve the purpose of producing warnings
-   for invalid uses of the value returned from these functions.
-   On pre-ANSI systems without 'const', the config.h file is supposed to
-   contain "#define const".  */
-# define gettext(Msgid) ((const char *) (Msgid))
-# define dgettext(Domainname, Msgid) ((void) (Domainname), gettext (Msgid))
-# define dcgettext(Domainname, Msgid, Category) \
-    ((void) (Category), dgettext (Domainname, Msgid))
-# define ngettext(Msgid1, Msgid2, N) \
-    ((N) == 1 \
-     ? ((void) (Msgid2), (const char *) (Msgid1)) \
-     : ((void) (Msgid1), (const char *) (Msgid2)))
-# define dngettext(Domainname, Msgid1, Msgid2, N) \
-    ((void) (Domainname), ngettext (Msgid1, Msgid2, N))
-# define dcngettext(Domainname, Msgid1, Msgid2, N, Category) \
-    ((void) (Category), dngettext(Domainname, Msgid1, Msgid2, N))
-# define textdomain(Domainname) ((const char *) (Domainname))
-# define bindtextdomain(Domainname, Dirname) \
-    ((void) (Domainname), (const char *) (Dirname))
-# define bind_textdomain_codeset(Domainname, Codeset) \
-    ((void) (Domainname), (const char *) (Codeset))
-
-#endif
-
-/* A pseudo function call that serves as a marker for the automated
-   extraction of messages, but does not call gettext().  The run-time
-   translation is done at a different place in the code.
-   The argument, String, should be a literal string.  Concatenated strings
-   and other string expressions won't work.
-   The macro's expansion is not parenthesized, so that it is suitable as
-   initializer for static 'char[]' or 'const char[]' variables.  */
-#define gettext_noop(String) String
-
-/* The separator between msgctxt and msgid in a .mo file.  */
-#define GETTEXT_CONTEXT_GLUE "\004"
-
-/* Pseudo function calls, taking a MSGCTXT and a MSGID instead of just a
-   MSGID.  MSGCTXT and MSGID must be string literals.  MSGCTXT should be
-   short and rarely need to change.
-   The letter 'p' stands for 'particular' or 'special'.  */
-#ifdef DEFAULT_TEXT_DOMAIN
-# define pgettext(Msgctxt, Msgid) \
-   pgettext_aux (DEFAULT_TEXT_DOMAIN, Msgctxt GETTEXT_CONTEXT_GLUE Msgid, Msgid, LC_MESSAGES)
-#else
-# define pgettext(Msgctxt, Msgid) \
-   pgettext_aux (NULL, Msgctxt GETTEXT_CONTEXT_GLUE Msgid, Msgid, LC_MESSAGES)
-#endif
-#define dpgettext(Domainname, Msgctxt, Msgid) \
-  pgettext_aux (Domainname, Msgctxt GETTEXT_CONTEXT_GLUE Msgid, Msgid, LC_MESSAGES)
-#define dcpgettext(Domainname, Msgctxt, Msgid, Category) \
-  pgettext_aux (Domainname, Msgctxt GETTEXT_CONTEXT_GLUE Msgid, Msgid, Category)
-#ifdef DEFAULT_TEXT_DOMAIN
-# define npgettext(Msgctxt, Msgid, MsgidPlural, N) \
-   npgettext_aux (DEFAULT_TEXT_DOMAIN, Msgctxt GETTEXT_CONTEXT_GLUE Msgid, Msgid, MsgidPlural, N, LC_MESSAGES)
-#else
-# define npgettext(Msgctxt, Msgid, MsgidPlural, N) \
-   npgettext_aux (NULL, Msgctxt GETTEXT_CONTEXT_GLUE Msgid, Msgid, MsgidPlural, N, LC_MESSAGES)
-#endif
-#define dnpgettext(Domainname, Msgctxt, Msgid, MsgidPlural, N) \
-  npgettext_aux (Domainname, Msgctxt GETTEXT_CONTEXT_GLUE Msgid, Msgid, MsgidPlural, N, LC_MESSAGES)
-#define dcnpgettext(Domainname, Msgctxt, Msgid, MsgidPlural, N, Category) \
-  npgettext_aux (Domainname, Msgctxt GETTEXT_CONTEXT_GLUE Msgid, Msgid, MsgidPlural, N, Category)
-
-#ifdef __GNUC__
-__inline
-#else
-#ifdef __cplusplus
-inline
-#endif
-#endif
-static const char *
-pgettext_aux (const char *domain,
-              const char *msg_ctxt_id, const char *msgid,
-              int category)
-{
-  const char *translation = dcgettext (domain, msg_ctxt_id, category);
-  if (translation == msg_ctxt_id)
-    return msgid;
-  else
-    return translation;
-}
-
-#ifdef __GNUC__
-__inline
-#else
-#ifdef __cplusplus
-inline
-#endif
-#endif
-static const char *
-npgettext_aux (const char *domain,
-               const char *msg_ctxt_id, const char *msgid,
-               const char *msgid_plural, unsigned long int n,
-               int category)
-{
-  const char *translation =
-    dcngettext (domain, msg_ctxt_id, msgid_plural, n, category);
-  if (translation == msg_ctxt_id || translation == msgid_plural)
-    return (n == 1 ? msgid : msgid_plural);
-  else
-    return translation;
-}
-
-/* The same thing extended for non-constant arguments.  Here MSGCTXT and MSGID
-   can be arbitrary expressions.  But for string literals these macros are
-   less efficient than those above.  */
-
-#include <string.h>
-
-#define _LIBGETTEXT_HAVE_VARIABLE_SIZE_ARRAYS \
-  (((__GNUC__ >= 3 || __GNUG__ >= 2) && !__STRICT_ANSI__) \
-   /* || __STDC_VERSION__ >= 199901L */ )
-
-#if !_LIBGETTEXT_HAVE_VARIABLE_SIZE_ARRAYS
-#include <stdlib.h>
-#endif
-
-#define pgettext_expr(Msgctxt, Msgid) \
-  dcpgettext_expr (NULL, Msgctxt, Msgid, LC_MESSAGES)
-#define dpgettext_expr(Domainname, Msgctxt, Msgid) \
-  dcpgettext_expr (Domainname, Msgctxt, Msgid, LC_MESSAGES)
-
-#ifdef __GNUC__
-__inline
-#else
-#ifdef __cplusplus
-inline
-#endif
-#endif
-static const char *
-dcpgettext_expr (const char *domain,
-                 const char *msgctxt, const char *msgid,
-                 int category)
-{
-  size_t msgctxt_len = strlen (msgctxt) + 1;
-  size_t msgid_len = strlen (msgid) + 1;
-  const char *translation;
-#if _LIBGETTEXT_HAVE_VARIABLE_SIZE_ARRAYS
-  char msg_ctxt_id[msgctxt_len + msgid_len];
-#else
-  char buf[1024];
-  char *msg_ctxt_id =
-    (msgctxt_len + msgid_len <= sizeof (buf)
-     ? buf
-     : (char *) malloc (msgctxt_len + msgid_len));
-  if (msg_ctxt_id != NULL)
-#endif
-    {
-      memcpy (msg_ctxt_id, msgctxt, msgctxt_len - 1);
-      msg_ctxt_id[msgctxt_len - 1] = '\004';
-      memcpy (msg_ctxt_id + msgctxt_len, msgid, msgid_len);
-      translation = dcgettext (domain, msg_ctxt_id, category);
-#if !_LIBGETTEXT_HAVE_VARIABLE_SIZE_ARRAYS
-      if (msg_ctxt_id != buf)
-        free (msg_ctxt_id);
-#endif
-      if (translation != msg_ctxt_id)
-        return translation;
-    }
-  return msgid;
-}
-
-#define npgettext_expr(Msgctxt, Msgid, MsgidPlural, N) \
-  dcnpgettext_expr (NULL, Msgctxt, Msgid, MsgidPlural, N, LC_MESSAGES)
-#define dnpgettext_expr(Domainname, Msgctxt, Msgid, MsgidPlural, N) \
-  dcnpgettext_expr (Domainname, Msgctxt, Msgid, MsgidPlural, N, LC_MESSAGES)
-
-#ifdef __GNUC__
-__inline
-#else
-#ifdef __cplusplus
-inline
-#endif
-#endif
-static const char *
-dcnpgettext_expr (const char *domain,
-                  const char *msgctxt, const char *msgid,
-                  const char *msgid_plural, unsigned long int n,
-                  int category)
-{
-  size_t msgctxt_len = strlen (msgctxt) + 1;
-  size_t msgid_len = strlen (msgid) + 1;
-  const char *translation;
-#if _LIBGETTEXT_HAVE_VARIABLE_SIZE_ARRAYS
-  char msg_ctxt_id[msgctxt_len + msgid_len];
-#else
-  char buf[1024];
-  char *msg_ctxt_id =
-    (msgctxt_len + msgid_len <= sizeof (buf)
-     ? buf
-     : (char *) malloc (msgctxt_len + msgid_len));
-  if (msg_ctxt_id != NULL)
-#endif
-    {
-      memcpy (msg_ctxt_id, msgctxt, msgctxt_len - 1);
-      msg_ctxt_id[msgctxt_len - 1] = '\004';
-      memcpy (msg_ctxt_id + msgctxt_len, msgid, msgid_len);
-      translation = dcngettext (domain, msg_ctxt_id, msgid_plural, n, category);
-#if !_LIBGETTEXT_HAVE_VARIABLE_SIZE_ARRAYS
-      if (msg_ctxt_id != buf)
-        free (msg_ctxt_id);
-#endif
-      if (!(translation == msg_ctxt_id || translation == msgid_plural))
-        return translation;
-    }
-  return (n == 1 ? msgid : msgid_plural);
-}
-
-#endif /* _LIBGETTEXT_H */

src/gnutls.c

@@ -1,6 +1,6 @@
 /* SSL support via GnuTLS library.
-   Copyright (C) 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012 Free Software
-   Foundation, Inc.
+   Copyright (C) 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012, 2015
+   Free Software Foundation, Inc.
 
 This file is part of GNU Wget.
 
@@ -56,6 +56,9 @@ as that of the covered work.  */
 
 #include "host.h"
 
+static int
+_do_handshake (gnutls_session_t session, int fd, double timeout);
+
 static int
 key_type_to_gnutls_type (enum keyfile_type type)
 {
@@ -122,9 +125,10 @@ ssl_init (void)
           while ((dent = readdir (dir)) != NULL)
             {
               struct stat st;
-              char ca_file[dirlen + strlen(dent->d_name) + 2];
+              size_t ca_file_length = dirlen + strlen(dent->d_name) + 2;
+              char *ca_file = alloca(ca_file_length);
 
-              snprintf (ca_file, sizeof(ca_file), "%s/%s", ca_directory, dent->d_name);
+              snprintf (ca_file, ca_file_length, "%s/%s", ca_directory, dent->d_name);
               if (stat (ca_file, &st) != 0)
                 continue;
 
@@ -138,8 +142,7 @@ ssl_init (void)
               hash_table_put (inode_map, (void *)(intptr_t) st.st_ino, NULL);
               if ((rc = gnutls_certificate_set_x509_trust_file (credentials, ca_file,
                                                                 GNUTLS_X509_FMT_PEM)) <= 0)
-                logprintf (LOG_NOTQUIET, _("ERROR: Failed to open cert %s: (%d).\n"),
-                           ca_file, rc);
+                DEBUGP (("WARNING: Failed to open cert %s: (%d).\n", ca_file, rc));
               else
                 ncerts += rc;
             }
@@ -149,18 +152,48 @@ ssl_init (void)
         }
     }
 
+  if (opt.ca_cert)
+    {
+      int rc;
+
+      ncerts = 0;
+
+      if ((rc = gnutls_certificate_set_x509_trust_file (credentials, opt.ca_cert,
+                                                        GNUTLS_X509_FMT_PEM)) <= 0)
+        logprintf (LOG_NOTQUIET, _ ("ERROR: Failed to open cert %s: (%d).\n"),
+                   opt.ca_cert, rc);
+      else
+        {
+          ncerts += rc;
+          logprintf (LOG_NOTQUIET, _ ("Loaded CA certificate '%s'\n"), opt.ca_cert);
+        }
+    }
+
+  if (opt.crl_file)
+    {
+      int rc;
+
+      if ((rc = gnutls_certificate_set_x509_crl_file (credentials, opt.crl_file, GNUTLS_X509_FMT_PEM)) <= 0)
+        {
+          logprintf (LOG_NOTQUIET, _("ERROR: Failed to load CRL file '%s': (%d)\n"), opt.crl_file, rc);
+          return false;
+        }
+
+      logprintf (LOG_NOTQUIET, _ ("Loaded CRL file '%s'\n"), opt.crl_file);
+    }
+
   DEBUGP (("Certificates loaded: %d\n", ncerts));
 
   /* Use the private key from the cert file unless otherwise specified. */
   if (opt.cert_file && !opt.private_key)
     {
-      opt.private_key = opt.cert_file;
+      opt.private_key = xstrdup (opt.cert_file);
       opt.private_key_type = opt.cert_type;
     }
   /* Use the cert from the private key file unless otherwise specified. */
   if (!opt.cert_file && opt.private_key)
     {
-      opt.cert_file = opt.private_key;
+      opt.cert_file = xstrdup (opt.private_key);
       opt.cert_type = opt.private_key_type;
     }
 
@@ -181,10 +214,6 @@ cert to be of the same type.\n"));
                                             type);
     }
 
-  if (opt.ca_cert)
-    gnutls_certificate_set_x509_trust_file (credentials, opt.ca_cert,
-                                            GNUTLS_X509_FMT_PEM);
-
   ssl_initialized = true;
 
   return true;
@@ -193,6 +222,7 @@ cert to be of the same type.\n"));
 struct wgnutls_transport_context
 {
   gnutls_session_t session;       /* GnuTLS session handle */
+  gnutls_datum_t *session_data;
   int last_error;               /* last error returned by read/write/... */
 
   /* Since GnuTLS doesn't support the equivalent to recv(...,
@@ -203,11 +233,6 @@ struct wgnutls_transport_context
   int peeklen;
 };
 
-#ifndef MIN
-# define MIN(i, j) ((i) <= (j) ? (i) : (j))
-#endif
-
-
 static int
 wgnutls_read_timeout (int fd, char *buf, int bufsize, void *arg, double timeout)
 {
@@ -255,6 +280,12 @@ wgnutls_read_timeout (int fd, char *buf, int bufsize, void *arg, double timeout)
         {
           ret = gnutls_record_recv (ctx->session, buf, bufsize);
           timed_out = timeout && ptimer_measure (timer) >= timeout;
+          if (!timed_out && ret == GNUTLS_E_REHANDSHAKE)
+            {
+              DEBUGP (("GnuTLS: *** REHANDSHAKE while reading\n"));
+              if ((ret = _do_handshake (ctx->session, fd, timeout)) == 0)
+                ret = GNUTLS_E_AGAIN; /* restart reading */
+            }
         }
     }
   while (ret == GNUTLS_E_INTERRUPTED || (ret == GNUTLS_E_AGAIN && !timed_out));
@@ -384,6 +415,11 @@ wgnutls_close (int fd, void *arg)
 {
   struct wgnutls_transport_context *ctx = arg;
   /*gnutls_bye (ctx->session, GNUTLS_SHUT_RDWR);*/
+  if (ctx->session_data)
+    {
+      gnutls_free (ctx->session_data->data);
+      gnutls_free (ctx->session_data);
+    }
   gnutls_deinit (ctx->session);
   xfree (ctx);
   close (fd);
@@ -398,90 +434,15 @@ static struct transport_implementation wgnutls_transport =
   wgnutls_peek, wgnutls_errstr, wgnutls_close
 };
 
-bool
-ssl_connect_wget (int fd, const char *hostname)
+static int
+_do_handshake (gnutls_session_t session, int fd, double timeout)
 {
 #ifdef F_GETFL
   int flags = 0;
 #endif
-  struct wgnutls_transport_context *ctx;
-  gnutls_session_t session;
-  int err,alert;
-  gnutls_init (&session, GNUTLS_CLIENT);
-  const char *str;
+  int err;
 
-  /* We set the server name but only if it's not an IP address. */
-  if (! is_valid_ip_address (hostname))
-    {
-      gnutls_server_name_set (session, GNUTLS_NAME_DNS, hostname,
-                              strlen (hostname));
-    }
-
-  gnutls_set_default_priority (session);
-  gnutls_credentials_set (session, GNUTLS_CRD_CERTIFICATE, credentials);
-#ifndef FD_TO_SOCKET
-# define FD_TO_SOCKET(X) (X)
-#endif
-#ifdef HAVE_INTPTR_T
-  gnutls_transport_set_ptr (session, (gnutls_transport_ptr_t) (intptr_t) FD_TO_SOCKET (fd));
-#else
-  gnutls_transport_set_ptr (session, (gnutls_transport_ptr_t) FD_TO_SOCKET (fd));
-#endif
-
-  err = 0;
-#if HAVE_GNUTLS_PRIORITY_SET_DIRECT
-  switch (opt.secure_protocol)
-    {
-    case secure_protocol_auto:
-      break;
-    case secure_protocol_sslv2:
-    case secure_protocol_sslv3:
-      err = gnutls_priority_set_direct (session, "NORMAL:-VERS-TLS-ALL:+VERS-SSL3.0", NULL);
-      break;
-    case secure_protocol_tlsv1:
-      err = gnutls_priority_set_direct (session, "NORMAL:-VERS-SSL3.0", NULL);
-      break;
-    case secure_protocol_pfs:
-      err = gnutls_priority_set_direct (session, "PFS", NULL);
-      if (err != GNUTLS_E_SUCCESS)
-        /* fallback if PFS is not available */
-        err = gnutls_priority_set_direct (session, "NORMAL:-RSA", NULL);
-      break;
-    default:
-      abort ();
-    }
-#else
-  int allowed_protocols[4] = {0, 0, 0, 0};
-  switch (opt.secure_protocol)
-    {
-    case secure_protocol_auto:
-      break;
-    case secure_protocol_sslv2:
-    case secure_protocol_sslv3:
-      allowed_protocols[0] = GNUTLS_SSL3;
-      err = gnutls_protocol_set_priority (session, allowed_protocols);
-      break;
-
-    case secure_protocol_tlsv1:
-      allowed_protocols[0] = GNUTLS_TLS1_0;
-      allowed_protocols[1] = GNUTLS_TLS1_1;
-      allowed_protocols[2] = GNUTLS_TLS1_2;
-      err = gnutls_protocol_set_priority (session, allowed_protocols);
-      break;
-
-    default:
-      abort ();
-    }
-#endif
-
-  if (err < 0)
-    {
-      logprintf (LOG_NOTQUIET, "GnuTLS: %s\n", gnutls_strerror (err));
-      gnutls_deinit (session);
-      return false;
-    }
-
-  if (opt.connect_timeout)
+  if (timeout)
     {
 #ifdef F_GETFL
       flags = fcntl (fd, F_GETFL, 0);
@@ -502,17 +463,17 @@ ssl_connect_wget (int fd, const char *hostname)
     {
       err = gnutls_handshake (session);
 
-      if (opt.connect_timeout && err == GNUTLS_E_AGAIN)
+      if (timeout && err == GNUTLS_E_AGAIN)
         {
           if (gnutls_record_get_direction (session))
             {
               /* wait for writeability */
-              err = select_fd (fd, opt.connect_timeout, WAIT_FOR_WRITE);
+              err = select_fd (fd, timeout, WAIT_FOR_WRITE);
             }
           else
             {
               /* wait for readability */
-              err = select_fd (fd, opt.connect_timeout, WAIT_FOR_READ);
+              err = select_fd (fd, timeout, WAIT_FOR_READ);
             }
 
           if (err <= 0)
@@ -533,17 +494,16 @@ ssl_connect_wget (int fd, const char *hostname)
           if (err == GNUTLS_E_WARNING_ALERT_RECEIVED ||
               err == GNUTLS_E_FATAL_ALERT_RECEIVED)
             {
-              alert = gnutls_alert_get (session);
-              str = gnutls_alert_get_name (alert);
-              if (str == NULL)
-                str = "(unknown)";
-              logprintf (LOG_NOTQUIET, "GnuTLS: received alert [%d]: %s\n", alert, str);
+              gnutls_alert_description_t alert = gnutls_alert_get (session);
+              const char *str = gnutls_alert_get_name (alert);
+              logprintf (LOG_NOTQUIET, "GnuTLS: received alert [%d]: %s\n",
+                         alert, str ? str : "(unknown)");
             }
         }
     }
   while (err && gnutls_error_is_fatal (err) == 0);
 
-  if (opt.connect_timeout)
+  if (timeout)
     {
 #ifdef F_GETFL
       if (fcntl (fd, F_SETFL, flags) < 0)
@@ -555,6 +515,144 @@ ssl_connect_wget (int fd, const char *hostname)
 #endif
     }
 
+  return err;
+}
+
+bool
+ssl_connect_wget (int fd, const char *hostname, int *continue_session)
+{
+  struct wgnutls_transport_context *ctx;
+  gnutls_session_t session;
+  int err;
+
+  gnutls_init (&session, GNUTLS_CLIENT);
+
+  /* We set the server name but only if it's not an IP address. */
+  if (! is_valid_ip_address (hostname))
+    {
+      gnutls_server_name_set (session, GNUTLS_NAME_DNS, hostname,
+                              strlen (hostname));
+    }
+
+  gnutls_set_default_priority (session);
+  gnutls_credentials_set (session, GNUTLS_CRD_CERTIFICATE, credentials);
+#ifndef FD_TO_SOCKET
+# define FD_TO_SOCKET(X) (X)
+#endif
+#ifdef HAVE_INTPTR_T
+  gnutls_transport_set_ptr (session, (gnutls_transport_ptr_t) (intptr_t) FD_TO_SOCKET (fd));
+#else
+  gnutls_transport_set_ptr (session, (gnutls_transport_ptr_t) FD_TO_SOCKET (fd));
+#endif
+
+#if HAVE_GNUTLS_PRIORITY_SET_DIRECT
+  switch (opt.secure_protocol)
+    {
+    case secure_protocol_auto:
+      err = gnutls_priority_set_direct (session, "NORMAL:%COMPAT:-VERS-SSL3.0", NULL);
+      break;
+
+    case secure_protocol_sslv2:
+    case secure_protocol_sslv3:
+      err = gnutls_priority_set_direct (session, "NORMAL:-VERS-TLS-ALL:+VERS-SSL3.0", NULL);
+      break;
+
+    case secure_protocol_tlsv1:
+      err = gnutls_priority_set_direct (session, "NORMAL:-VERS-SSL3.0", NULL);
+      break;
+
+    case secure_protocol_tlsv1_1:
+      err = gnutls_priority_set_direct (session, "NORMAL:-VERS-SSL3.0:-VERS-TLS1.0", NULL);
+      break;
+
+    case secure_protocol_tlsv1_2:
+      err = gnutls_priority_set_direct (session, "NORMAL:-VERS-SSL3.0:-VERS-TLS1.0:-VERS-TLS1.1", NULL);
+      break;
+
+    case secure_protocol_pfs:
+      err = gnutls_priority_set_direct (session, "PFS:-VERS-SSL3.0", NULL);
+      if (err != GNUTLS_E_SUCCESS)
+        /* fallback if PFS is not available */
+        err = gnutls_priority_set_direct (session, "NORMAL:-RSA:-VERS-SSL3.0", NULL);
+      break;
+
+    default:
+      logprintf (LOG_NOTQUIET, _("GnuTLS: unimplemented 'secure-protocol' option value %d\n"), opt.secure_protocol);
+      logprintf (LOG_NOTQUIET, _("Please report this issue to bug-wget@gnu.org\n"));
+      abort ();
+    }
+#else
+  int allowed_protocols[4] = {0, 0, 0, 0};
+  switch (opt.secure_protocol)
+    {
+    case secure_protocol_auto:
+      break;
+
+    case secure_protocol_sslv2:
+    case secure_protocol_sslv3:
+      allowed_protocols[0] = GNUTLS_SSL3;
+      err = gnutls_protocol_set_priority (session, allowed_protocols);
+      break;
+
+    case secure_protocol_tlsv1:
+      allowed_protocols[0] = GNUTLS_TLS1_0;
+      allowed_protocols[1] = GNUTLS_TLS1_1;
+      allowed_protocols[2] = GNUTLS_TLS1_2;
+      err = gnutls_protocol_set_priority (session, allowed_protocols);
+      break;
+
+    case secure_protocol_tlsv1_1:
+      allowed_protocols[0] = GNUTLS_TLS1_1;
+      allowed_protocols[1] = GNUTLS_TLS1_2;
+      err = gnutls_protocol_set_priority (session, allowed_protocols);
+      break;
+
+    case secure_protocol_tlsv1_2:
+      allowed_protocols[0] = GNUTLS_TLS1_2;
+      err = gnutls_protocol_set_priority (session, allowed_protocols);
+      break;
+
+    default:
+      logprintf (LOG_NOTQUIET, _("GnuTLS: unimplemented 'secure-protocol' option value %d\n"), opt.secure_protocol);
+      logprintf (LOG_NOTQUIET, _("Please report this issue to bug-wget@gnu.org\n"));
+      abort ();
+    }
+#endif
+
+  if (err < 0)
+    {
+      logprintf (LOG_NOTQUIET, "GnuTLS: %s\n", gnutls_strerror (err));
+      gnutls_deinit (session);
+      return false;
+    }
+
+  if (continue_session)
+    {
+      ctx = (struct wgnutls_transport_context *) fd_transport_context (*continue_session);
+      if (!gnutls_session_is_resumed (session))
+        {
+          if (!ctx || !ctx->session_data || gnutls_session_set_data (session, ctx->session_data->data, ctx->session_data->size))
+            {
+              if (ctx && ctx->session_data)
+                {
+                  /* server does not want to continue the session */
+                  if (ctx->session_data->data)
+                    gnutls_free (ctx->session_data->data);
+                  gnutls_free (ctx->session_data);
+                }
+              gnutls_deinit (session);
+              return false;
+            }
+        }
+      else
+        {
+          logputs (LOG_ALWAYS, "SSL session has already been resumed. Continuing.\n");
+          continue_session = NULL;
+        }
+    }
+
+  err = _do_handshake (session, fd, opt.connect_timeout);
+
   if (err < 0)
     {
       gnutls_deinit (session);
@@ -562,7 +660,13 @@ ssl_connect_wget (int fd, const char *hostname)
     }
 
   ctx = xnew0 (struct wgnutls_transport_context);
+  ctx->session_data = xnew0 (gnutls_datum_t);
   ctx->session = session;
+  if (gnutls_session_get_data2 (session, ctx->session_data))
+    {
+      xfree (ctx->session_data);
+      logprintf (LOG_NOTQUIET, "WARNING: Could not save SSL session data for socket %d\n", fd);
+    }
   fd_register_transport (fd, &wgnutls_transport, ctx);
   return true;
 }
@@ -588,6 +692,10 @@ ssl_check_certificate (int fd, const char *host)
   const char *severity = opt.check_cert ? _("ERROR") : _("WARNING");
   bool success = true;
 
+  /* The user explicitly said to not check for the certificate.  */
+  if (opt.check_cert == CHECK_CERT_QUIET)
+    return success;
+
   err = gnutls_certificate_verify_peers2 (ctx->session, &status);
   if (err < 0)
     {
@@ -662,5 +770,5 @@ ssl_check_certificate (int fd, const char *host)
     }
 
  out:
-  return opt.check_cert ? success : true;
+  return opt.check_cert == CHECK_CERT_ON ? success : true;
 }

src/hash.c

@@ -1,6 +1,6 @@
 /* Hash tables.
    Copyright (C) 2000, 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008,
-   2009, 2010, 2011 Free Software Foundation, Inc.
+   2009, 2010, 2011, 2015 Free Software Foundation, Inc.
 
 This file is part of GNU Wget.
 
@@ -46,10 +46,11 @@ as that of the covered work.  */
 # include "utils.h"
 #else
 /* Make do without them. */
-# define xnew(x) xmalloc (sizeof (x))
-# define xnew_array(type, x) xmalloc (sizeof (type) * (x))
-# define xmalloc malloc
-# define xfree free
+# define xnew(type) (xmalloc (sizeof (type)))
+# define xnew0(type) (xcalloc (1, sizeof (type)))
+# define xnew_array(type, len) (xmalloc ((len) * sizeof (type)))
+# define xfree(p) do { free ((void *) (p)); p = NULL; } while (0)
+
 # ifndef countof
 #  define countof(x) (sizeof (x) / sizeof ((x)[0]))
 # endif
@@ -585,7 +586,7 @@ hash_table_count (const struct hash_table *ht)
 {
   return ht->count;
 }
-
+
 /* Functions from this point onward are meant for convenience and
    don't strictly belong to this file.  However, this is as good a
    place for them as any.  */
@@ -740,7 +741,7 @@ cmp_pointer (const void *ptr1, const void *ptr2)
 {
   return ptr1 == ptr2;
 }
-
+
 #ifdef TEST
 
 #include <stdio.h>

src/hash.h

@@ -1,5 +1,5 @@
 /* Hash table declarations.
-   Copyright (C) 2000, 2007, 2008, 2009, 2010, 2011 Free Software
+   Copyright (C) 2000, 2007, 2008, 2009, 2010, 2011, 2015 Free Software
    Foundation, Inc.
 
 This file is part of GNU Wget.

src/host.c

@@ -1,7 +1,7 @@
 /* Host name resolution and matching.
    Copyright (C) 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004,
-   2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012 Free Software Foundation,
-   Inc.
+   2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012, 2015 Free Software
+   Foundation, Inc.
 
 This file is part of GNU Wget.
 
@@ -57,6 +57,10 @@ as that of the covered work.  */
 
 #include <errno.h>
 
+#ifdef ENABLE_IRI
+#include <idn-free.h>
+#endif
+
 #include "utils.h"
 #include "host.h"
 #include "url.h"
@@ -151,6 +155,13 @@ address_list_set_faulty (struct address_list *al, int index)
      "faulty" attempt is always preceded with all-faulty addresses,
      and this is how Wget uses it.  */
   assert (index == al->faulty);
+  if (index != al->faulty)
+    {
+      logprintf (LOG_ALWAYS, "index: %d\nal->faulty: %d\n", index, al->faulty);
+      logprintf (LOG_ALWAYS, _("Error in handling the address list.\n"));
+      logprintf (LOG_ALWAYS, _("Please report this issue to bug-wget@gnu.org\n"));
+      abort();
+    }
 
   ++al->faulty;
   if (al->faulty >= al->count)
@@ -309,7 +320,7 @@ address_list_release (struct address_list *al)
       address_list_delete (al);
     }
 }
-
+
 /* Versions of gethostbyname and getaddrinfo that support timeout. */
 
 #ifndef ENABLE_IPV6
@@ -408,7 +419,7 @@ getaddrinfo_with_timeout (const char *node, const char *service,
 }
 
 #endif /* ENABLE_IPV6 */
-
+
 /* Return a textual representation of ADDR, i.e. the dotted quad for
    IPv4 addresses, and the colon-separated list of hex words (with all
    zeros omitted, etc.) for IPv6 addresses.  */
@@ -570,7 +581,7 @@ is_valid_ipv6_address (const char *str, const char *end)
 
   return true;
 }
-
+
 /* Simple host cache, used by lookup_host to speed up resolving.  The
    cache doesn't handle TTL because Wget is a fairly short-lived
    application.  Refreshing is attempted when connect fails, though --
@@ -592,7 +603,7 @@ cache_query (const char *host)
   al = hash_table_get (host_name_addresses_map, host);
   if (al)
     {
-      DEBUGP (("Found %s in host_name_addresses_map (%p)\n", host, al));
+      DEBUGP (("Found %s in host_name_addresses_map (%p)\n", host, (void *) al));
       ++al->refcount;
       return al;
     }
@@ -637,7 +648,7 @@ cache_remove (const char *host)
       hash_table_remove (host_name_addresses_map, host);
     }
 }
-
+
 /* Look up HOST in DNS and return a list of IP addresses.
 
    This function caches its result so that, if the same host is passed
@@ -734,14 +745,13 @@ lookup_host (const char *host, int flags)
           str = xmalloc (len);
           snprintf (str, len, "%s (%s)", name, host);
           str[len-1] = '\0';
-          xfree (name);
+          idn_free (name);
         }
 
       logprintf (LOG_VERBOSE, _("Resolving %s... "),
                  quotearg_style (escape_quoting_style, str ? str : host));
 
-      if (str)
-        xfree (str);
+      xfree (str);
     }
 
 #ifdef ENABLE_IPV6
@@ -848,7 +858,7 @@ lookup_host (const char *host, int flags)
 
   return al;
 }
-
+
 /* Determine whether a URL is acceptable to be followed, according to
    a list of domains to accept.  */
 bool

src/host.h

@@ -1,7 +1,7 @@
 /* Declarations for host.c
    Copyright (C) 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004,
-   2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012 Free Software Foundation,
-   Inc.
+   2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012, 2015 Free Software
+   Foundation, Inc.
 
 This file is part of GNU Wget.
 

src/hsts.c

@@ -0,0 +1,768 @@
+/* HTTP Strict Transport Security (HSTS) support.
+   Copyright (C) 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004,
+   2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012, 2015 Free Software
+   Foundation, Inc.
+
+This file is part of GNU Wget.
+
+GNU Wget is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 3 of the License, or
+ (at your option) any later version.
+
+GNU Wget is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with Wget.  If not, see <http://www.gnu.org/licenses/>.
+
+Additional permission under GNU GPL version 3 section 7
+
+If you modify this program, or any covered work, by linking or
+combining it with the OpenSSL project's OpenSSL library (or a
+modified version of that library), containing parts covered by the
+terms of the OpenSSL or SSLeay licenses, the Free Software Foundation
+grants you additional permission to convey the resulting work.
+Corresponding Source for a non-source form of such a combination
+shall include the source code for the parts of OpenSSL used as well
+as that of the covered work.  */
+#include "wget.h"
+
+#ifdef HAVE_HSTS
+#include "hsts.h"
+#include "host.h" /* for is_valid_ip_address() */
+#include "init.h" /* for home_dir() */
+#include "utils.h"
+#include "hash.h"
+#include "c-ctype.h"
+#ifdef TESTING
+#include "test.h"
+#endif
+
+#include <unistd.h>
+#include <sys/types.h>
+#include <stdlib.h>
+#include <time.h>
+#include <sys/stat.h>
+#include <string.h>
+#include <stdio.h>
+#include <sys/file.h>
+
+struct hsts_store {
+  struct hash_table *table;
+  time_t last_mtime;
+};
+
+struct hsts_kh {
+  char *host;
+  int explicit_port;
+};
+
+struct hsts_kh_info {
+  time_t created;
+  time_t max_age;
+  bool include_subdomains;
+};
+
+enum hsts_kh_match {
+  NO_MATCH,
+  SUPERDOMAIN_MATCH,
+  CONGRUENT_MATCH
+};
+
+#define hsts_is_host_name_valid(host) (!is_valid_ip_address (host))
+#define hsts_is_scheme_valid(scheme) (scheme == SCHEME_HTTPS)
+#define hsts_is_host_eligible(scheme, host) \
+    (hsts_is_scheme_valid (scheme) && hsts_is_host_name_valid (host))
+
+#define DEFAULT_HTTP_PORT 80
+#define DEFAULT_SSL_PORT  443
+#define CHECK_EXPLICIT_PORT(p1, p2) (p1 == 0 || p1 == p2)
+#define MAKE_EXPLICIT_PORT(s, p) (s == SCHEME_HTTPS ? (p == DEFAULT_SSL_PORT ? 0 : p) \
+    : (p == DEFAULT_HTTP_PORT ? 0 : p))
+
+/* Hashing and comparison functions for the hash table */
+
+static unsigned long
+hsts_hash_func (const void *key)
+{
+  struct hsts_kh *k = (struct hsts_kh *) key;
+  const char *h = NULL;
+  unsigned int hash = k->explicit_port;
+
+  for (h = k->host; *h; h++)
+    hash = hash * 31 + *h;
+
+  return hash;
+}
+
+static int
+hsts_cmp_func (const void *h1, const void *h2)
+{
+  struct hsts_kh *kh1 = (struct hsts_kh *) h1,
+      *kh2 = (struct hsts_kh *) h2;
+
+  return (!strcmp (kh1->host, kh2->host)) && (kh1->explicit_port == kh2->explicit_port);
+}
+
+/* Private functions. Feel free to make some of these public when needed. */
+
+static struct hsts_kh_info *
+hsts_find_entry (hsts_store_t store,
+                 const char *host, int explicit_port,
+                 enum hsts_kh_match *match_type,
+                 struct hsts_kh *kh)
+{
+  struct hsts_kh *k = NULL;
+  struct hsts_kh_info *khi = NULL;
+  enum hsts_kh_match match = NO_MATCH;
+  char *pos = NULL;
+  char *org_ptr = NULL;
+
+  k = (struct hsts_kh *) xnew (struct hsts_kh);
+  k->host = xstrdup_lower (host);
+  k->explicit_port = explicit_port;
+
+  /* save pointer so that we don't get into trouble later when freeing */
+  org_ptr = k->host;
+
+  khi = (struct hsts_kh_info *) hash_table_get (store->table, k);
+  if (khi)
+    {
+      match = CONGRUENT_MATCH;
+      goto end;
+    }
+
+  while (match == NO_MATCH &&
+      (pos = strchr (k->host, '.')) && pos - k->host > 0 &&
+      strchr (pos + 1, '.'))
+    {
+      k->host += (pos - k->host + 1);
+      khi = (struct hsts_kh_info *) hash_table_get (store->table, k);
+      if (khi)
+        match = SUPERDOMAIN_MATCH;
+    }
+
+end:
+  /* restore pointer or we'll get a SEGV */
+  k->host = org_ptr;
+
+  /* copy parameters to previous frame */
+  if (match_type)
+    *match_type = match;
+  if (kh)
+    memcpy (kh, k, sizeof (struct hsts_kh));
+  else
+    xfree (k->host);
+
+  xfree (k);
+  return khi;
+}
+
+static bool
+hsts_new_entry_internal (hsts_store_t store,
+                         const char *host, int port,
+                         time_t created, time_t max_age,
+                         bool include_subdomains,
+                         bool check_validity,
+                         bool check_expired,
+                         bool check_duplicates)
+{
+  struct hsts_kh *kh = xnew (struct hsts_kh);
+  struct hsts_kh_info *khi = xnew0 (struct hsts_kh_info);
+  bool success = false;
+
+  kh->host = xstrdup_lower (host);
+  kh->explicit_port = MAKE_EXPLICIT_PORT (SCHEME_HTTPS, port);
+
+  khi->created = created;
+  khi->max_age = max_age;
+  khi->include_subdomains = include_subdomains;
+
+  /* Check validity */
+  if (check_validity && !hsts_is_host_name_valid (host))
+    goto bail;
+
+  if (check_expired && ((khi->created + khi->max_age) < khi->created))
+    goto bail;
+
+  if (check_duplicates && hash_table_contains (store->table, kh))
+    goto bail;
+
+  /* Now store the new entry */
+  hash_table_put (store->table, kh, khi);
+  success = true;
+
+bail:
+  if (!success)
+    {
+      /* abort! */
+      xfree (kh->host);
+      xfree (kh);
+      xfree (khi);
+    }
+
+  return success;
+}
+
+/*
+   Creates a new entry, but does not check whether that entry already exists.
+   This function assumes that check has already been done by the caller.
+ */
+static bool
+hsts_add_entry (hsts_store_t store,
+                const char *host, int port,
+                time_t max_age, bool include_subdomains)
+{
+  time_t t = time (NULL);
+
+  /* It might happen time() returned -1 */
+  return (t < 0 ?
+      false :
+      hsts_new_entry_internal (store, host, port, t, max_age, include_subdomains, false, true, false));
+}
+
+/* Creates a new entry, unless an identical one already exists. */
+static bool
+hsts_new_entry (hsts_store_t store,
+                const char *host, int port,
+                time_t created, time_t max_age,
+                bool include_subdomains)
+{
+  return hsts_new_entry_internal (store, host, port, created, max_age, include_subdomains, true, true, true);
+}
+
+static void
+hsts_remove_entry (hsts_store_t store, struct hsts_kh *kh)
+{
+  hash_table_remove (store->table, kh);
+}
+
+static bool
+hsts_store_merge (hsts_store_t store,
+                  const char *host, int port,
+                  time_t created, time_t max_age,
+                  bool include_subdomains)
+{
+  enum hsts_kh_match match_type = NO_MATCH;
+  struct hsts_kh_info *khi = NULL;
+  bool success = false;
+
+  port = MAKE_EXPLICIT_PORT (SCHEME_HTTPS, port);
+  khi = hsts_find_entry (store, host, port, &match_type, NULL);
+  if (khi && match_type == CONGRUENT_MATCH && created > khi->created)
+    {
+      /* update the entry with the new info */
+      khi->created = created;
+      khi->max_age = max_age;
+      khi->include_subdomains = include_subdomains;
+
+      success = true;
+    }
+  else if (!khi)
+    success = hsts_new_entry (store, host, port, created, max_age, include_subdomains);
+
+  return success;
+}
+
+static bool
+hsts_read_database (hsts_store_t store, FILE *fp, bool merge_with_existing_entries)
+{
+  char *line = NULL, *p;
+  size_t len = 0;
+  int items_read;
+  bool result = false;
+  bool (*func)(hsts_store_t, const char *, int, time_t, time_t, bool);
+
+  char host[256];
+  int port;
+  time_t created, max_age;
+  int include_subdomains;
+
+  func = (merge_with_existing_entries ? hsts_store_merge : hsts_new_entry);
+
+  while (getline (&line, &len, fp) > 0)
+    {
+      for (p = line; c_isspace (*p); p++)
+        ;
+
+      if (*p == '#')
+        continue;
+
+      items_read = sscanf (p, "%255s %d %d %lu %lu",
+                           host,
+                           &port,
+                           &include_subdomains,
+                           (unsigned long *) &created,
+                           (unsigned long *) &max_age);
+
+      if (items_read == 5)
+        func (store, host, port, created, max_age, !!include_subdomains);
+    }
+
+  xfree (line);
+  result = true;
+
+  return result;
+}
+
+static void
+hsts_store_dump (hsts_store_t store, FILE *fp)
+{
+  hash_table_iterator it;
+
+  /* Print preliminary comments. We don't care if any of these fail. */
+  fputs ("# HSTS 1.0 Known Hosts database for GNU Wget.\n", fp);
+  fputs ("# Edit at your own risk.\n", fp);
+  fputs ("# <hostname>[:<port>]\t<incl. subdomains>\t<created>\t<max-age>\n", fp);
+
+  /* Now cycle through the HSTS store in memory and dump the entries */
+  for (hash_table_iterate (store->table, &it); hash_table_iter_next (&it);)
+    {
+      struct hsts_kh *kh = (struct hsts_kh *) it.key;
+      struct hsts_kh_info *khi = (struct hsts_kh_info *) it.value;
+
+      if (fprintf (fp, "%s\t%d\t%d\t%lu\t%lu\n",
+                   kh->host, kh->explicit_port, khi->include_subdomains,
+                   khi->created, khi->max_age) < 0)
+        {
+          logprintf (LOG_ALWAYS, "Could not write the HSTS database correctly.\n");
+          break;
+        }
+    }
+}
+
+/* HSTS API */
+
+/*
+   Changes the given URLs according to the HSTS policy.
+
+   If there's no host in the store that either congruently
+   or not, matches the given URL, no changes are made.
+   Returns true if the URL was changed, or false
+   if it was left intact.
+ */
+bool
+hsts_match (hsts_store_t store, struct url *u)
+{
+  bool url_changed = false;
+  struct hsts_kh_info *entry = NULL;
+  struct hsts_kh *kh = xnew(struct hsts_kh);
+  enum hsts_kh_match match = NO_MATCH;
+  int port = MAKE_EXPLICIT_PORT (u->scheme, u->port);
+
+  /* avoid doing any computation if we're already in HTTPS */
+  if (!hsts_is_scheme_valid (u->scheme))
+    {
+      entry = hsts_find_entry (store, u->host, port, &match, kh);
+      if (entry)
+        {
+          if ((entry->created + entry->max_age) >= time(NULL))
+            {
+              if ((match == CONGRUENT_MATCH) ||
+                  (match == SUPERDOMAIN_MATCH && entry->include_subdomains))
+                {
+                  /* we found a matching Known HSTS Host
+                     rewrite the URL */
+                  u->scheme = SCHEME_HTTPS;
+                  if (u->port == 80)
+                    u->port = 443;
+                  url_changed = true;
+                }
+            }
+          else
+            hsts_remove_entry (store, kh);
+        }
+      xfree (kh->host);
+    }
+
+  xfree (kh);
+
+  return url_changed;
+}
+
+/*
+   Add a new HSTS Known Host to the HSTS store.
+
+   If the host already exists, its information is updated,
+   or it'll be removed from the store if max_age is zero.
+
+   Bear in mind that the store is kept in memory, and will not
+   be written to disk until hsts_store_save is called.
+   This function regrows the in-memory HSTS store if necessary.
+
+   Currently, for a host to be taken into consideration,
+   two conditions have to be met:
+     - Connection must be through a secure channel (HTTPS).
+     - The host must not be an IPv4 or IPv6 address.
+
+   The RFC 6797 states that hosts that match IPv4 or IPv6 format
+   should be discarded at URI rewrite time. But we short-circuit
+   that check here, since there's no point in storing a host that
+   will never be matched.
+
+   Returns true if a new entry was actually created, or false
+   if an existing entry was updated/deleted. */
+bool
+hsts_store_entry (hsts_store_t store,
+                  enum url_scheme scheme, const char *host, int port,
+                  time_t max_age, bool include_subdomains)
+{
+  bool result = false;
+  enum hsts_kh_match match = NO_MATCH;
+  struct hsts_kh *kh = xnew(struct hsts_kh);
+  struct hsts_kh_info *entry = NULL;
+  time_t t = 0;
+
+  if (hsts_is_host_eligible (scheme, host))
+    {
+      port = MAKE_EXPLICIT_PORT (scheme, port);
+      entry = hsts_find_entry (store, host, port, &match, kh);
+      if (entry && match == CONGRUENT_MATCH)
+        {
+          if (max_age == 0)
+            hsts_remove_entry (store, kh);
+          else if (max_age > 0)
+            {
+              entry->include_subdomains = include_subdomains;
+
+              if (entry->max_age != max_age)
+                {
+                  /* RFC 6797 states that 'max_age' is a TTL relative to the reception of the STS header
+                     so we have to update the 'created' field too */
+                  t = time (NULL);
+                  if (t != -1)
+                    entry->created = t;
+                  entry->max_age = max_age;
+                }
+            }
+          /* we ignore negative max_ages */
+        }
+      else if (entry == NULL || match == SUPERDOMAIN_MATCH)
+        {
+          /* Either we didn't find a matching host,
+             or we got a superdomain match.
+             In either case, we create a new entry.
+
+             We have to perform an explicit check because it might
+             happen we got a non-existent entry with max_age == 0.
+          */
+          result = hsts_add_entry (store, host, port, max_age, include_subdomains);
+        }
+      /* we ignore new entries with max_age == 0 */
+      xfree (kh->host);
+    }
+
+  xfree (kh);
+
+  return result;
+}
+
+hsts_store_t
+hsts_store_open (const char *filename)
+{
+  hsts_store_t store = NULL;
+  struct_stat st;
+  FILE *fp = NULL;
+
+  store = xnew0 (struct hsts_store);
+  store->table = hash_table_new (0, hsts_hash_func, hsts_cmp_func);
+  store->last_mtime = 0;
+
+  if (file_exists_p (filename))
+    {
+      fp = fopen (filename, "r");
+
+      if (!fp || !hsts_read_database (store, fp, false))
+        {
+          /* abort! */
+          hsts_store_close (store);
+          xfree (store);
+          goto out;
+        }
+
+      if (fstat (fileno (fp), &st) == 0)
+        store->last_mtime = st.st_mtime;
+      fclose (fp);
+    }
+
+out:
+  return store;
+}
+
+void
+hsts_store_save (hsts_store_t store, const char *filename)
+{
+  struct_stat st;
+  FILE *fp = NULL;
+  int fd = 0;
+
+  if (filename && hash_table_count (store->table) > 0)
+    {
+      fp = fopen (filename, "a+");
+      if (fp)
+        {
+          /* Lock the file to avoid potential race conditions */
+          fd = fileno (fp);
+          flock (fd, LOCK_EX);
+
+          /* If the file has changed, merge the changes with our in-memory data
+             before dumping them to the file.
+             Otherwise we could potentially overwrite the data stored by other Wget processes.
+           */
+          if (store->last_mtime && stat (filename, &st) == 0 && st.st_mtime > store->last_mtime)
+            hsts_read_database (store, fp, true);
+
+          /* We've merged the latest changes so we can now truncate the file
+             and dump everything. */
+          fseek (fp, 0, SEEK_SET);
+          ftruncate (fd, 0);
+
+          /* now dump to the file */
+          hsts_store_dump (store, fp);
+
+          /* fclose is expected to unlock the file for us */
+          fclose (fp);
+        }
+    }
+}
+
+void
+hsts_store_close (hsts_store_t store)
+{
+  hash_table_iterator it;
+
+  /* free all the host fields */
+  for (hash_table_iterate (store->table, &it); hash_table_iter_next (&it);)
+    {
+      xfree (((struct hsts_kh *) it.key)->host);
+      xfree (it.key);
+      xfree (it.value);
+    }
+
+  hash_table_destroy (store->table);
+}
+
+#ifdef TESTING
+/* I know I'm really evil because I'm writing macros
+   that change control flow. But we're testing, who will tell? :D
+ */
+#define TEST_URL_RW(s, u, p) do { \
+    if (test_url_rewrite (s, u, p, true)) \
+      return test_url_rewrite (s, u, p, true); \
+  } while (0)
+
+#define TEST_URL_NORW(s, u, p) do { \
+    if (test_url_rewrite (s, u, p, false)) \
+      return test_url_rewrite (s, u, p, false); \
+  } while (0)
+
+static char *
+get_hsts_store_filename (void)
+{
+  char *home = NULL, *filename = NULL;
+  FILE *fp = NULL;
+
+  home = home_dir ();
+  if (home)
+    {
+      filename = aprintf ("%s/.wget-hsts-test", home);
+      fp = fopen (filename, "w");
+      if (fp)
+        fclose (fp);
+    }
+
+  xfree (home);
+  return filename;
+}
+
+static hsts_store_t
+open_hsts_test_store (void)
+{
+  char *filename = NULL;
+  hsts_store_t table = NULL;
+
+  filename = get_hsts_store_filename ();
+  table = hsts_store_open (filename);
+  xfree (filename);
+
+  return table;
+}
+
+static void
+close_hsts_test_store (hsts_store_t store)
+{
+  char *filename = NULL;
+
+  filename = get_hsts_store_filename ();
+  unlink (filename);
+  xfree (filename);
+  xfree (store);
+}
+
+static const char*
+test_url_rewrite (hsts_store_t s, const char *url, int port, bool rewrite)
+{
+  bool result;
+  struct url u;
+
+  u.host = xstrdup (url);
+  u.port = port;
+  u.scheme = SCHEME_HTTP;
+
+  result = hsts_match (s, &u);
+
+  if (rewrite)
+    {
+      if (port == 80)
+        mu_assert("URL: port should've been rewritten to 443", u.port == 443);
+      else
+        mu_assert("URL: port should've been left intact", u.port == port);
+      mu_assert("URL: scheme should've been rewritten to HTTPS", u.scheme == SCHEME_HTTPS);
+      mu_assert("result should've been true", result == true);
+    }
+  else
+    {
+      mu_assert("URL: port should've been left intact", u.port == port);
+      mu_assert("URL: scheme should've been left intact", u.scheme == SCHEME_HTTP);
+      mu_assert("result should've been false", result == false);
+    }
+
+  xfree (u.host);
+  return NULL;
+}
+
+const char *
+test_hsts_new_entry (void)
+{
+  enum hsts_kh_match match = NO_MATCH;
+  struct hsts_kh_info *khi;
+  hsts_store_t s;
+  bool created;
+
+  s = open_hsts_test_store ();
+  mu_assert("Could not open the HSTS store. This could be due to lack of memory.", s != NULL);
+
+  created = hsts_store_entry (s, SCHEME_HTTP, "www.foo.com", 80, 1234, true);
+  mu_assert("No entry should have been created.", created == false);
+
+  created = hsts_store_entry (s, SCHEME_HTTPS, "www.foo.com", 443, 1234, true);
+  mu_assert("A new entry should have been created", created == true);
+
+  khi = hsts_find_entry (s, "www.foo.com", MAKE_EXPLICIT_PORT (SCHEME_HTTPS, 443), &match, NULL);
+  mu_assert("Should've been a congruent match", match == CONGRUENT_MATCH);
+  mu_assert("No valid HSTS info was returned", khi != NULL);
+  mu_assert("Variable 'max_age' should be 1234", khi->max_age == 1234);
+  mu_assert("Variable 'include_subdomains' should be asserted", khi->include_subdomains == true);
+
+  khi = hsts_find_entry (s, "b.www.foo.com", MAKE_EXPLICIT_PORT (SCHEME_HTTPS, 443), &match, NULL);
+  mu_assert("Should've been a superdomain match", match == SUPERDOMAIN_MATCH);
+  mu_assert("No valid HSTS info was returned", khi != NULL);
+  mu_assert("Variable 'max_age' should be 1234", khi->max_age == 1234);
+  mu_assert("Variable 'include_subdomains' should be asserted", khi->include_subdomains == true);
+
+  khi = hsts_find_entry (s, "ww.foo.com", MAKE_EXPLICIT_PORT (SCHEME_HTTPS, 443), &match, NULL);
+  mu_assert("Should've been no match", match == NO_MATCH);
+
+  khi = hsts_find_entry (s, "foo.com", MAKE_EXPLICIT_PORT (SCHEME_HTTPS, 443), &match, NULL);
+  mu_assert("Should've been no match", match == NO_MATCH);
+
+  khi = hsts_find_entry (s, ".foo.com", MAKE_EXPLICIT_PORT (SCHEME_HTTPS, 443), &match, NULL);
+  mu_assert("Should've been no match", match == NO_MATCH);
+
+  khi = hsts_find_entry (s, ".www.foo.com", MAKE_EXPLICIT_PORT (SCHEME_HTTPS, 443), &match, NULL);
+  mu_assert("Should've been no match", match == NO_MATCH);
+
+  hsts_store_close (s);
+  close_hsts_test_store (s);
+
+  return NULL;
+}
+
+const char*
+test_hsts_url_rewrite_superdomain (void)
+{
+  hsts_store_t s;
+  bool created;
+
+  s = open_hsts_test_store ();
+  mu_assert("Could not open the HSTS store", s != NULL);
+
+  created = hsts_store_entry (s, SCHEME_HTTPS, "www.foo.com", 443, 1234, true);
+  mu_assert("A new entry should've been created", created == true);
+
+  TEST_URL_RW (s, "www.foo.com", 80);
+  TEST_URL_RW (s, "bar.www.foo.com", 80);
+
+  hsts_store_close (s);
+  close_hsts_test_store (s);
+
+  return NULL;
+}
+
+const char*
+test_hsts_url_rewrite_congruent (void)
+{
+  hsts_store_t s;
+  bool created;
+
+  s = open_hsts_test_store ();
+  mu_assert("Could not open the HSTS store", s != NULL);
+
+  created = hsts_store_entry (s, SCHEME_HTTPS, "foo.com", 443, 1234, false);
+  mu_assert("A new entry should've been created", created == true);
+
+  TEST_URL_RW (s, "foo.com", 80);
+  TEST_URL_NORW (s, "www.foo.com", 80);
+
+  hsts_store_close (s);
+  close_hsts_test_store (s);
+
+  return NULL;
+}
+
+const char*
+test_hsts_read_database (void)
+{
+  hsts_store_t table;
+  char *home = home_dir();
+  char *file = NULL;
+  FILE *fp = NULL;
+  time_t created = time(NULL) - 10;
+
+  if (home)
+    {
+      file = aprintf ("%s/.wget-hsts-testing", home);
+      fp = fopen (file, "w");
+      if (fp)
+        {
+          fputs ("# dummy comment\n", fp);
+          fprintf (fp, "foo.example.com\t0\t1\t%ld\t123\n",(long) created);
+          fprintf (fp, "bar.example.com\t0\t0\t%ld\t456\n", (long) created);
+          fprintf (fp, "test.example.com\t8080\t0\t%ld\t789\n", (long) created);
+          fclose (fp);
+
+          table = hsts_store_open (file);
+
+          TEST_URL_RW (table, "foo.example.com", 80);
+          TEST_URL_RW (table, "www.foo.example.com", 80);
+          TEST_URL_RW (table, "bar.example.com", 80);
+
+          TEST_URL_NORW(table, "www.bar.example.com", 80);
+
+          TEST_URL_RW (table, "test.example.com", 8080);
+
+          hsts_store_close (table);
+          close_hsts_test_store (table);
+          unlink (file);
+        }
+      xfree (file);
+      xfree (home);
+    }
+
+  return NULL;
+}
+#endif /* TESTING */
+#endif /* HAVE_HSTS */

src/hsts.h

@@ -0,0 +1,53 @@
+/* Declarations for hsts.c
+   Copyright (C) 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004,
+   2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012, 2015 Free Software
+   Foundation, Inc.
+
+This file is part of GNU Wget.
+
+GNU Wget is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 3 of the License, or
+ (at your option) any later version.
+
+GNU Wget is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with Wget.  If not, see <http://www.gnu.org/licenses/>.
+
+Additional permission under GNU GPL version 3 section 7
+
+If you modify this program, or any covered work, by linking or
+combining it with the OpenSSL project's OpenSSL library (or a
+modified version of that library), containing parts covered by the
+terms of the OpenSSL or SSLeay licenses, the Free Software Foundation
+grants you additional permission to convey the resulting work.
+Corresponding Source for a non-source form of such a combination
+shall include the source code for the parts of OpenSSL used as well
+as that of the covered work.  */
+#include "wget.h"
+
+#ifdef HAVE_HSTS
+
+#ifndef HSTS_H
+#define HSTS_H
+
+#include "url.h"
+
+typedef struct hsts_store *hsts_store_t;
+
+hsts_store_t hsts_store_open (const char *);
+
+void hsts_store_save (hsts_store_t, const char *);
+void hsts_store_close (hsts_store_t);
+
+bool hsts_store_entry (hsts_store_t,
+                       enum url_scheme, const char *, int,
+                       time_t, bool);
+bool hsts_match (hsts_store_t, struct url *);
+
+#endif /* HSTS_H */
+#endif /* HAVE_HSTS */

src/html-parse.c

@@ -1,6 +1,6 @@
 /* HTML parser for Wget.
    Copyright (C) 1998, 1999, 2000, 2001, 2002, 2003, 2004, 2005, 2006,
-   2007, 2008, 2009, 2010, 2011 Free Software Foundation, Inc.
+   2007, 2008, 2009, 2010, 2011, 2015 Free Software Foundation, Inc.
 
 This file is part of GNU Wget.
 
@@ -253,7 +253,7 @@ struct pool {
       (sizevar) = ga_newsize;                                                   \
     }                                                                           \
 } while (0)
-
+
 /* Test whether n+1-sized entity name fits in P.  We don't support
    IE-style non-terminated entities, e.g. "&ltfoo" -> "<foo".
    However, "&lt;foo" will work, as will "&lt!foo", "&lt", etc.  In
@@ -517,7 +517,7 @@ convert_and_copy (struct pool *pool, const char *beg, const char *end, int flags
         *p = c_tolower (*p);
     }
 }
-
+
 /* Originally we used to adhere to rfc 1866 here, and allowed only
    letters, digits, periods, and hyphens as names (of tags or
    attributes).  However, this broke too many pages which used
@@ -770,7 +770,7 @@ find_comment_end (const char *beg, const char *end)
       }
   return NULL;
 }
-
+
 /* Return true if the string containing of characters inside [b, e) is
    present in hash table HT.  */
 
@@ -881,7 +881,7 @@ map_html_tags (const char *text, int size,
     if (*p == '!')
       {
         if (!(flags & MHT_STRICT_COMMENTS)
-            && p < end + 3 && p[1] == '-' && p[2] == '-')
+            && p + 3 < end && p[1] == '-' && p[2] == '-')
           {
             /* If strict comments are not enforced and if we know
                we're looking at a comment, simply look for the
@@ -1167,7 +1167,7 @@ map_html_tags (const char *text, int size,
 #undef ADVANCE
 #undef SKIP_WS
 #undef SKIP_NON_WS
-
+
 #ifdef STANDALONE
 static void
 test_mapper (struct taginfo *taginfo, void *arg)

src/html-parse.h

@@ -1,6 +1,6 @@
 /* Declarations for html-parse.c.
    Copyright (C) 1998, 1999, 2000, 2001, 2002, 2003, 2004, 2005, 2006,
-   2007, 2008, 2009, 2010, 2011 Free Software Foundation, Inc.
+   2007, 2008, 2009, 2010, 2011, 2015 Free Software Foundation, Inc.
 
 This file is part of GNU Wget.
 

src/html-url.c

@@ -1,6 +1,7 @@
 /* Collect URLs from HTML source.
    Copyright (C) 1998, 1999, 2000, 2001, 2002, 2003, 2004, 2005, 2006,
-   2007, 2008, 2009, 2010, 2011, 2012 Free Software Foundation, Inc.
+   2007, 2008, 2009, 2010, 2011, 2012, 2015 Free Software Foundation,
+   Inc.
 
 This file is part of GNU Wget.
 
@@ -45,6 +46,7 @@ as that of the covered work.  */
 #include "recur.h"
 #include "html-url.h"
 #include "css-url.h"
+#include "c-strcase.h"
 
 typedef void (*tag_handler_t) (int, struct taginfo *, struct map_context *);
 
@@ -255,7 +257,7 @@ find_attr (struct taginfo *tag, const char *name, int *attrind)
 {
   int i;
   for (i = 0; i < tag->nattrs; i++)
-    if (!strcasecmp (tag->attrs[i].name, name))
+    if (!c_strcasecmp (tag->attrs[i].name, name))
       {
         if (attrind)
           *attrind = i;
@@ -302,6 +304,7 @@ append_url (const char *link_uri, int position, int size,
           logprintf (LOG_NOTQUIET,
                      _("%s: Cannot resolve incomplete link %s.\n"),
                      ctx->document_file, link_uri);
+          iri_free (iri);
           return NULL;
         }
 
@@ -310,6 +313,7 @@ append_url (const char *link_uri, int position, int size,
         {
           DEBUGP (("%s: link \"%s\" doesn't parse.\n",
                    ctx->document_file, link_uri));
+          iri_free (iri);
           return NULL;
         }
     }
@@ -333,6 +337,7 @@ append_url (const char *link_uri, int position, int size,
           DEBUGP (("%s: merged link \"%s\" doesn't parse.\n",
                    ctx->document_file, complete_uri));
           xfree (complete_uri);
+          iri_free (iri);
           return NULL;
         }
       xfree (complete_uri);
@@ -378,7 +383,7 @@ append_url (const char *link_uri, int position, int size,
 
   return newel;
 }
-
+
 static void
 check_style_attr (struct taginfo *tag, struct map_context *ctx)
 {
@@ -487,8 +492,7 @@ tag_handle_base (int tagid _GL_UNUSED, struct taginfo *tag, struct map_context *
   base_urlpos->ignore_when_downloading = 1;
   base_urlpos->link_base_p = 1;
 
-  if (ctx->base)
-    xfree (ctx->base);
+  xfree (ctx->base);
   if (ctx->parent_base)
     ctx->base = uri_merge (ctx->parent_base, newbase);
   else
@@ -536,12 +540,12 @@ tag_handle_link (int tagid _GL_UNUSED, struct taginfo *tag, struct map_context *
           char *rel = find_attr (tag, "rel", NULL);
           if (rel)
             {
-              if (0 == strcasecmp (rel, "stylesheet"))
+              if (0 == c_strcasecmp (rel, "stylesheet"))
                 {
                   up->link_inline_p = 1;
                   up->link_expect_css = 1;
                 }
-              else if (0 == strcasecmp (rel, "shortcut icon"))
+              else if (0 == c_strcasecmp (rel, "shortcut icon"))
                 {
                   up->link_inline_p = 1;
                 }
@@ -553,7 +557,7 @@ tag_handle_link (int tagid _GL_UNUSED, struct taginfo *tag, struct map_context *
                      <link rel="alternate" type="application/rss+xml" href=".../?feed=rss2" />
                   */
                   char *type = find_attr (tag, "type", NULL);
-                  if (!type || strcasecmp (type, "text/html") == 0)
+                  if (!type || c_strcasecmp (type, "text/html") == 0)
                     up->link_expect_html = 1;
                 }
             }
@@ -570,7 +574,7 @@ tag_handle_meta (int tagid _GL_UNUSED, struct taginfo *tag, struct map_context *
   char *name = find_attr (tag, "name", NULL);
   char *http_equiv = find_attr (tag, "http-equiv", NULL);
 
-  if (http_equiv && 0 == strcasecmp (http_equiv, "refresh"))
+  if (http_equiv && 0 == c_strcasecmp (http_equiv, "refresh"))
     {
       /* Some pages use a META tag to specify that the page be
          refreshed by a new page after a given number of seconds.  The
@@ -615,7 +619,7 @@ tag_handle_meta (int tagid _GL_UNUSED, struct taginfo *tag, struct map_context *
           entry->link_expect_html = 1;
         }
     }
-  else if (http_equiv && 0 == strcasecmp (http_equiv, "content-type"))
+  else if (http_equiv && 0 == c_strcasecmp (http_equiv, "content-type"))
     {
       /* Handle stuff like:
          <meta http-equiv="Content-Type" content="text/html; charset=CHARSET"> */
@@ -629,17 +633,17 @@ tag_handle_meta (int tagid _GL_UNUSED, struct taginfo *tag, struct map_context *
       if (!mcharset)
         return;
 
-      xfree_null (meta_charset);
+      xfree (meta_charset);
       meta_charset = mcharset;
     }
-  else if (name && 0 == strcasecmp (name, "robots"))
+  else if (name && 0 == c_strcasecmp (name, "robots"))
     {
       /* Handle stuff like:
          <meta name="robots" content="index,nofollow"> */
       char *content = find_attr (tag, "content", NULL);
       if (!content)
         return;
-      if (!strcasecmp (content, "none"))
+      if (!c_strcasecmp (content, "none"))
         ctx->nofollow = true;
       else
         {
@@ -651,7 +655,7 @@ tag_handle_meta (int tagid _GL_UNUSED, struct taginfo *tag, struct map_context *
               /* Find the next occurrence of ',' or whitespace,
                * or the end of the string.  */
               end = content + strcspn (content, ", \f\n\r\t\v");
-              if (!strncasecmp (content, "nofollow", end - content))
+              if (!c_strncasecmp (content, "nofollow", end - content))
                 ctx->nofollow = true;
               /* Skip past the next comma, if any. */
               if (*end == ',')
@@ -692,7 +696,7 @@ collect_tags_mapper (struct taginfo *tag, void *arg)
 
   check_style_attr (tag, ctx);
 
-  if (tag->end_tag_p && (0 == strcasecmp (tag->name, "style"))
+  if (tag->end_tag_p && (0 == c_strcasecmp (tag->name, "style"))
       && tag->contents_begin && tag->contents_end
       && tag->contents_begin <= tag->contents_end)
   {
@@ -701,7 +705,7 @@ collect_tags_mapper (struct taginfo *tag, void *arg)
                   tag->contents_end - tag->contents_begin);
   }
 }
-
+
 /* Analyze HTML tags FILE and construct a list of URLs referenced from
    it.  It merges relative links in FILE with URL.  It is aware of
    <base href=...> and does the right thing.  */
@@ -748,15 +752,16 @@ get_urls_html (const char *file, const char *url, bool *meta_disallow_follow,
   map_html_tags (fm->content, fm->length, collect_tags_mapper, &ctx, flags,
                  NULL, interesting_attributes);
 
-  /* If meta charset isn't null, override content encoding */
-  if (iri && meta_charset)
+  /* Meta charset is only valid if there was no HTTP header Content-Type charset. */
+  /* This is true for HTTP 1.0 and 1.1. */
+  if (iri && !iri->content_encoding && meta_charset)
     set_content_encoding (iri, meta_charset);
 
   DEBUGP (("no-follow in %s: %d\n", file, ctx.nofollow));
   if (meta_disallow_follow)
     *meta_disallow_follow = ctx.nofollow;
 
-  xfree_null (ctx.base);
+  xfree (ctx.base);
   wget_read_file_free (fm);
   return ctx.head;
 }
@@ -787,6 +792,7 @@ get_urls_file (const char *file)
     {
       int up_error_code;
       char *url_text;
+      char *new_url;
       struct urlpos *entry;
       struct url *url;
 
@@ -821,7 +827,7 @@ get_urls_file (const char *file)
           url_text = merged;
         }
 
-      char *new_url = rewrite_shorthand_url (url_text);
+      new_url = rewrite_shorthand_url (url_text);
       if (new_url)
         {
           xfree (url_text);

src/html-url.h

@@ -1,5 +1,5 @@
 /* Declarations for html-url.c.
-   Copyright (C) 1995, 1996, 1997, 2009, 2010, 2011 Free Software
+   Copyright (C) 1995, 1996, 1997, 2009, 2010, 2011, 2015 Free Software
    Foundation, Inc.
 
 This file is part of GNU Wget.

src/http-ntlm.c

@@ -1,6 +1,6 @@
 /* NTLM code.
-   Copyright (C) 2005, 2006, 2007, 2008, 2009, 2010, 2011 Free Software
-   Foundation, Inc.
+   Copyright (C) 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2015 Free
+   Software Foundation, Inc.
    Contributed by Daniel Stenberg.
 
 This file is part of GNU Wget.
@@ -74,13 +74,13 @@ as that of the covered work.  */
 /* Define this to make the type-3 message include the NT response message */
 #define USE_NTRESPONSES 1
 
-
+
 /* Flag bits definitions available at on
    http://davenport.sourceforge.net/ntlm.html */
 
 #define NTLMFLAG_NEGOTIATE_OEM                   (1<<1)
 #define NTLMFLAG_NEGOTIATE_NTLM_KEY              (1<<9)
-
+
 /*
   (*) = A "security buffer" is a triplet consisting of two shorts and one
   long:
@@ -474,82 +474,82 @@ ntlm_output (struct ntlmdata *ntlm, const char *user, const char *passwd,
 
     /* Create the big type-3 message binary blob */
 
-    size = (size_t) snprintf (ntlmbuf, sizeof(ntlmbuf),
-                     "NTLMSSP%c"
-                     "\x03%c%c%c" /* type-3, 32 bits */
+    snprintf (ntlmbuf, sizeof (ntlmbuf),
+              "NTLMSSP%c"
+              "\x03%c%c%c" /* type-3, 32 bits */
 
-                     "%c%c%c%c" /* LanManager length + allocated space */
-                     "%c%c" /* LanManager offset */
-                     "%c%c" /* 2 zeroes */
+              "%c%c%c%c" /* LanManager length + allocated space */
+              "%c%c" /* LanManager offset */
+              "%c%c" /* 2 zeroes */
 
-                     "%c%c" /* NT-response length */
-                     "%c%c" /* NT-response allocated space */
-                     "%c%c" /* NT-response offset */
-                     "%c%c" /* 2 zeroes */
+              "%c%c" /* NT-response length */
+              "%c%c" /* NT-response allocated space */
+              "%c%c" /* NT-response offset */
+              "%c%c" /* 2 zeroes */
 
-                     "%c%c"  /* domain length */
-                     "%c%c"  /* domain allocated space */
-                     "%c%c"  /* domain name offset */
-                     "%c%c"  /* 2 zeroes */
+              "%c%c" /* domain length */
+              "%c%c" /* domain allocated space */
+              "%c%c" /* domain name offset */
+              "%c%c" /* 2 zeroes */
 
-                     "%c%c"  /* user length */
-                     "%c%c"  /* user allocated space */
-                     "%c%c"  /* user offset */
-                     "%c%c"  /* 2 zeroes */
+              "%c%c" /* user length */
+              "%c%c" /* user allocated space */
+              "%c%c" /* user offset */
+              "%c%c" /* 2 zeroes */
 
-                     "%c%c"  /* host length */
-                     "%c%c"  /* host allocated space */
-                     "%c%c"  /* host offset */
-                     "%c%c%c%c%c%c"  /* 6 zeroes */
+              "%c%c" /* host length */
+              "%c%c" /* host allocated space */
+              "%c%c" /* host offset */
+              "%c%c%c%c%c%c" /* 6 zeroes */
 
-                     "\xff\xff"  /* message length */
-                     "%c%c"  /* 2 zeroes */
+              "\xff\xff" /* message length */
+              "%c%c" /* 2 zeroes */
 
-                     "\x01\x82" /* flags */
-                     "%c%c"  /* 2 zeroes */
+              "\x01\x82" /* flags */
+              "%c%c" /* 2 zeroes */
 
-                     /* domain string */
-                     /* user string */
-                     /* host string */
-                     /* LanManager response */
-                     /* NT response */
-                     ,
-                     0, /* zero termination */
-                     0,0,0, /* type-3 long, the 24 upper bits */
+              /* domain string */
+              /* user string */
+              /* host string */
+              /* LanManager response */
+              /* NT response */
+              ,
+              0, /* zero termination */
+              0, 0, 0, /* type-3 long, the 24 upper bits */
 
-                     SHORTPAIR(0x18),  /* LanManager response length, twice */
-                     SHORTPAIR(0x18),
-                     SHORTPAIR(lmrespoff),
-                     0x0, 0x0,
+              SHORTPAIR (0x18), /* LanManager response length, twice */
+              SHORTPAIR (0x18),
+              SHORTPAIR (lmrespoff),
+              0x0, 0x0,
 
 #ifdef USE_NTRESPONSES
-                     SHORTPAIR(0x18),  /* NT-response length, twice */
-                     SHORTPAIR(0x18),
+              SHORTPAIR (0x18), /* NT-response length, twice */
+              SHORTPAIR (0x18),
 #else
-                     0x0, 0x0,
-                     0x0, 0x0,
+              0x0, 0x0,
+              0x0, 0x0,
 #endif
-                     SHORTPAIR(ntrespoff),
-                     0x0, 0x0,
+              SHORTPAIR (ntrespoff),
+              0x0, 0x0,
 
-                     SHORTPAIR(domlen),
-                     SHORTPAIR(domlen),
-                     SHORTPAIR(domoff),
-                     0x0, 0x0,
+              SHORTPAIR (domlen),
+              SHORTPAIR (domlen),
+              SHORTPAIR (domoff),
+              0x0, 0x0,
 
-                     SHORTPAIR(userlen),
-                     SHORTPAIR(userlen),
-                     SHORTPAIR(useroff),
-                     0x0, 0x0,
+              SHORTPAIR (userlen),
+              SHORTPAIR (userlen),
+              SHORTPAIR (useroff),
+              0x0, 0x0,
 
-                     SHORTPAIR(hostlen),
-                     SHORTPAIR(hostlen),
-                     SHORTPAIR(hostoff),
-                     0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
+              SHORTPAIR (hostlen),
+              SHORTPAIR (hostlen),
+              SHORTPAIR (hostoff),
+              0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
 
-                     0x0, 0x0,
+              0x0, 0x0,
 
-                     0x0, 0x0);
+              0x0, 0x0);
 
     /* size is now 64 */
     size=64;

src/http-ntlm.h

@@ -1,8 +1,8 @@
 #ifndef __HTTP_NTLM_H
 #define __HTTP_NTLM_H
 /* Declarations for http_ntlm.c
-   Copyright (C) 1995, 1996, 1997, 2000, 2007, 2008, 2009, 2010, 2011
-   Free Software Foundation, Inc.
+   Copyright (C) 1995, 1996, 1997, 2000, 2007, 2008, 2009, 2010, 2011,
+   2015 Free Software Foundation, Inc.
    Contributed by Daniel Stenberg.
 
 This file is part of GNU Wget.

src/http.h

@@ -1,6 +1,6 @@
 /* Declarations for HTTP.
-   Copyright (C) 2005, 2006, 2007, 2008, 2009, 2010, 2011 Free Software
-   Foundation, Inc.
+   Copyright (C) 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2015 Free
+   Software Foundation, Inc.
 
 This file is part of GNU Wget.
 
@@ -31,6 +31,8 @@ as that of the covered work.  */
 #ifndef HTTP_H
 #define HTTP_H
 
+#include "hsts.h"
+
 struct url;
 
 uerr_t http_loop (struct url *, struct url *, char **, char **, const char *,

src/init.c

@@ -1,7 +1,7 @@
 /* Reading/parsing the initialization file.
    Copyright (C) 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004,
-   2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012, 2014 Free Software Foundation,
-   Inc.
+   2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012, 2014, 2015 Free
+   Software Foundation, Inc.
 
 This file is part of GNU Wget.
 
@@ -69,6 +69,8 @@ as that of the covered work.  */
 #include "retr.h"               /* for output_stream */
 #include "warc.h"               /* for warc_close */
 #include "spider.h"             /* for spider_cleanup */
+#include "html-url.h"           /* for cleanup_html_url */
+#include "c-strcase.h"
 
 #ifdef TESTING
 #include "test.h"
@@ -90,6 +92,7 @@ CMD_DECLARE (cmd_number_inf);
 CMD_DECLARE (cmd_string);
 CMD_DECLARE (cmd_string_uppercase);
 CMD_DECLARE (cmd_file);
+CMD_DECLARE (cmd_file_once);
 CMD_DECLARE (cmd_directory);
 CMD_DECLARE (cmd_time);
 CMD_DECLARE (cmd_vector);
@@ -101,6 +104,7 @@ CMD_DECLARE (cmd_spec_htmlify);
 CMD_DECLARE (cmd_spec_mirror);
 CMD_DECLARE (cmd_spec_prefer_family);
 CMD_DECLARE (cmd_spec_progress);
+CMD_DECLARE (cmd_spec_progressdisp);
 CMD_DECLARE (cmd_spec_recursive);
 CMD_DECLARE (cmd_spec_regex_type);
 CMD_DECLARE (cmd_spec_restrict_file_names);
@@ -111,6 +115,7 @@ CMD_DECLARE (cmd_spec_secure_protocol);
 CMD_DECLARE (cmd_spec_timeout);
 CMD_DECLARE (cmd_spec_useragent);
 CMD_DECLARE (cmd_spec_verbose);
+CMD_DECLARE (cmd_check_cert);
 
 /* List of recognized commands, each consisting of name, place and
    function.  When adding a new command, simply add it to the list,
@@ -148,15 +153,19 @@ static const struct {
   { "cadirectory",      &opt.ca_directory,      cmd_directory },
   { "certificate",      &opt.cert_file,         cmd_file },
   { "certificatetype",  &opt.cert_type,         cmd_cert_type },
-  { "checkcertificate", &opt.check_cert,        cmd_boolean },
+  { "checkcertificate", &opt.check_cert,        cmd_check_cert },
 #endif
   { "chooseconfig",     &opt.choose_config,     cmd_file },
   { "connecttimeout",   &opt.connect_timeout,   cmd_time },
   { "contentdisposition", &opt.content_disposition, cmd_boolean },
   { "contentonerror",   &opt.content_on_error,  cmd_boolean },
   { "continue",         &opt.always_rest,       cmd_boolean },
+  { "convertfileonly",  &opt.convert_file_only, cmd_boolean },
   { "convertlinks",     &opt.convert_links,     cmd_boolean },
   { "cookies",          &opt.cookies,           cmd_boolean },
+#ifdef HAVE_SSL
+  { "crlfile",          &opt.crl_file,          cmd_file_once },
+#endif
   { "cutdirs",          &opt.cut_dirs,          cmd_number },
   { "debug",            &opt.debug,             cmd_boolean },
   { "defaultpage",      &opt.default_page,      cmd_string },
@@ -181,12 +190,22 @@ static const struct {
   { "ftppasswd",        &opt.ftp_passwd,        cmd_string }, /* deprecated */
   { "ftppassword",      &opt.ftp_passwd,        cmd_string },
   { "ftpproxy",         &opt.ftp_proxy,         cmd_string },
+#ifdef HAVE_SSL
+  { "ftpscleardataconnection", &opt.ftps_clear_data_connection, cmd_boolean },
+  { "ftpsfallbacktoftp", &opt.ftps_fallback_to_ftp, cmd_boolean },
+  { "ftpsimplicit",     &opt.ftps_implicit,     cmd_boolean },
+  { "ftpsresumessl",    &opt.ftps_resume_ssl,   cmd_boolean },
+#endif
 #ifdef __VMS
   { "ftpstmlf",         &opt.ftp_stmlf,         cmd_boolean },
 #endif /* def __VMS */
   { "ftpuser",          &opt.ftp_user,          cmd_string },
   { "glob",             &opt.ftp_glob,          cmd_boolean },
   { "header",           NULL,                   cmd_spec_header },
+#ifdef HAVE_HSTS
+  { "hsts",             &opt.hsts,              cmd_boolean },
+  { "hsts-file",        &opt.hsts_file,         cmd_file },
+#endif
   { "htmlextension",    &opt.adjust_extension,  cmd_boolean }, /* deprecated */
   { "htmlify",          NULL,                   cmd_spec_htmlify },
   { "httpkeepalive",    &opt.http_keep_alive,   cmd_boolean },
@@ -198,6 +217,7 @@ static const struct {
 #endif
   { "httpsproxy",       &opt.https_proxy,       cmd_string },
   { "httpuser",         &opt.http_user,         cmd_string },
+  { "if-modified-since", &opt.if_modified_since, cmd_boolean },
   { "ignorecase",       &opt.ignore_case,       cmd_boolean },
   { "ignorelength",     &opt.ignore_length,     cmd_boolean },
   { "ignoretags",       &opt.ignore_tags,       cmd_vector },
@@ -207,6 +227,9 @@ static const struct {
   { "inet6only",        &opt.ipv6_only,         cmd_boolean },
 #endif
   { "input",            &opt.input_filename,    cmd_file },
+#ifdef HAVE_METALINK
+  { "input-metalink",   &opt.input_metalink,    cmd_file },
+#endif
   { "iri",              &opt.enable_iri,        cmd_boolean },
   { "keepsessioncookies", &opt.keep_session_cookies, cmd_boolean },
   { "limitrate",        &opt.limit_rate,        cmd_bytes },
@@ -215,6 +238,9 @@ static const struct {
   { "logfile",          &opt.lfilename,         cmd_file },
   { "login",            &opt.ftp_user,          cmd_string },/* deprecated*/
   { "maxredirect",      &opt.max_redirect,      cmd_number },
+#ifdef HAVE_METALINK
+  { "metalink-over-http", &opt.metalink_over_http, cmd_boolean },
+#endif
   { "method",           &opt.method,            cmd_string_uppercase },
   { "mirror",           NULL,                   cmd_spec_mirror },
   { "netrc",            &opt.netrc,             cmd_boolean },
@@ -231,6 +257,9 @@ static const struct {
   { "postdata",         &opt.post_data,         cmd_string },
   { "postfile",         &opt.post_file_name,    cmd_file },
   { "preferfamily",     NULL,                   cmd_spec_prefer_family },
+#ifdef HAVE_METALINK
+  { "preferred-location", &opt.preferred_location, cmd_string },
+#endif
   { "preservepermissions", &opt.preserve_perm,  cmd_boolean },
 #ifdef HAVE_SSL
   { "privatekey",       &opt.private_key,       cmd_file },
@@ -253,6 +282,7 @@ static const struct {
   { "referer",          &opt.referer,           cmd_string },
   { "regextype",        &opt.regex_type,        cmd_spec_regex_type },
   { "reject",           &opt.rejects,           cmd_vector },
+  { "rejectedlog",      &opt.rejected_log,      cmd_file },
   { "rejectregex",      &opt.rejectregex_s,     cmd_string },
   { "relativeonly",     &opt.relative_only,     cmd_boolean },
   { "remoteencoding",   &opt.encoding_remote,   cmd_string },
@@ -269,7 +299,7 @@ static const struct {
 #endif
   { "serverresponse",   &opt.server_response,   cmd_boolean },
   { "showalldnsentries", &opt.show_all_dns_entries, cmd_boolean },
-  { "showprogress",     &opt.show_progress,      cmd_boolean },
+  { "showprogress",     &opt.show_progress,     cmd_spec_progressdisp },
   { "spanhosts",        &opt.spanhost,          cmd_boolean },
   { "spider",           &opt.spider,            cmd_boolean },
   { "startpos",         &opt.start_pos,         cmd_bytes },
@@ -315,7 +345,7 @@ command_by_name (const char *cmdname)
   while (lo <= hi)
     {
       int mid = (lo + hi) >> 1;
-      int cmp = strcasecmp (cmdname, commands[mid].name);
+      int cmp = c_strcasecmp (cmdname, commands[mid].name);
       if (cmp < 0)
         hi = mid - 1;
       else if (cmp > 0)
@@ -325,7 +355,7 @@ command_by_name (const char *cmdname)
     }
   return -1;
 }
-
+
 /* Reset the variables to default values.  */
 void
 defaults (void)
@@ -349,11 +379,13 @@ defaults (void)
   opt.htmlify = true;
   opt.http_keep_alive = true;
   opt.use_proxy = true;
+  opt.convert_file_only = false;
   tmp = getenv ("no_proxy");
   if (tmp)
     opt.no_proxy = sepstring (tmp);
   opt.prefer_family = prefer_none;
   opt.allow_cache = true;
+  opt.if_modified_since = true;
 
   opt.read_timeout = 900;
   opt.use_robots = true;
@@ -366,14 +398,36 @@ defaults (void)
 
   opt.dns_cache = true;
   opt.ftp_pasv = true;
+  /* 2014-09-07  Darshit Shah  <darnir@gmail.com>
+   * opt.retr_symlinks is set to true by default. Creating symbolic links on the
+   * local filesystem pose a security threat by malicious FTP Servers that
+   * server a specially crafted .listing file akin to this:
+   *
+   * lrwxrwxrwx   1 root     root           33 Dec 25  2012 JoCxl6d8rFU -> /
+   * drwxrwxr-x  15 1024     106          4096 Aug 28 02:02 JoCxl6d8rFU
+   *
+   * A .listing file in this fashion makes Wget susceptiple to a symlink attack
+   * wherein the attacker is able to create arbitrary files, directories and
+   * symbolic links on the target system and even set permissions.
+   *
+   * Hence, by default Wget attempts to retrieve the pointed-to files and does
+   * not create the symbolic links locally.
+   */
+  opt.retr_symlinks = true;
 
 #ifdef HAVE_SSL
-  opt.check_cert = true;
+  opt.check_cert = CHECK_CERT_ON;
+  opt.ftps_resume_ssl = true;
+  opt.ftps_fallback_to_ftp = false;
+  opt.ftps_implicit = false;
+  opt.ftps_clear_data_connection = false;
 #endif
 
   /* The default for file name restriction defaults to the OS type. */
 #if defined(WINDOWS) || defined(MSDOS) || defined(__CYGWIN__)
   opt.restrict_files_os = restrict_windows;
+#elif defined(__VMS)
+  opt.restrict_files_os = restrict_vms;
 #else
   opt.restrict_files_os = restrict_unix;
 #endif
@@ -412,10 +466,15 @@ defaults (void)
 
   /* Use a negative value to mark the absence of --start-pos option */
   opt.start_pos = -1;
-  opt.show_progress = false;
+  opt.show_progress = -1;
   opt.noscroll = false;
+
+#ifdef HAVE_HSTS
+  /* HSTS is enabled by default */
+  opt.hsts = true;
+#endif
 }
-
+
 /* Return the user's home directory (strdup-ed), or NULL if none is
    found.  */
 char *
@@ -464,7 +523,7 @@ home_dir (void)
     }
 
   ret = home ? xstrdup (home) : NULL;
-  free (buf);
+  xfree (buf);
 
   return ret;
 }
@@ -505,7 +564,7 @@ wgetrc_user_file_name (void)
   home = home_dir ();
   if (home)
     file = aprintf ("%s/.wgetrc", home);
-  xfree_null (home);
+  xfree (home);
 #endif /* def __VMS [else] */
 
   if (!file)
@@ -539,17 +598,13 @@ wgetrc_file_name (void)
      SYSTEM_WGETRC should not be defined under WINDOWS.  */
   if (!file)
     {
-      char *home = home_dir ();
-      xfree_null (file);
-      file = NULL;
-      home = ws_mypath ();
+      char *home = ws_mypath ();
       if (home)
         {
           file = aprintf ("%s/wget.ini", home);
           if (!file_exists_p (file))
             {
               xfree (file);
-              file = NULL;
             }
           xfree (home);
         }
@@ -623,8 +678,8 @@ run_wgetrc (const char *file)
         default:
           abort ();
         }
-      xfree_null (com);
-      xfree_null (val);
+      xfree (com);
+      xfree (val);
       ++ln;
     }
   xfree (line);
@@ -877,7 +932,7 @@ run_command (const char *cmdopt)
       exit (WGET_EXIT_PARSE_ERROR);
     }
 }
-
+
 /* Generic helper functions, for use with `commands'. */
 
 /* Forward declarations: */
@@ -901,6 +956,18 @@ static bool simple_atof (const char *, const char *, double *);
                      && (p)[3] == '\0')
 
 
+static int
+cmd_boolean_internal (const char *com _GL_UNUSED, const char *val, void *place _GL_UNUSED)
+{
+  if (CMP2 (val, 'o', 'n') || CMP3 (val, 'y', 'e', 's') || CMP1 (val, '1'))
+    /* "on", "yes" and "1" mean true. */
+    return 1;
+  else if (CMP3 (val, 'o', 'f', 'f') || CMP2 (val, 'n', 'o') || CMP1 (val, '0'))
+    /* "off", "no" and "0" mean false. */
+    return 0;
+  return -1;
+}
+
 /* Store the boolean value from VAL to PLACE.  COM is ignored,
    except for error messages.  */
 static bool
@@ -908,24 +975,62 @@ cmd_boolean (const char *com, const char *val, void *place)
 {
   bool value;
 
-  if (CMP2 (val, 'o', 'n') || CMP3 (val, 'y', 'e', 's') || CMP1 (val, '1'))
-    /* "on", "yes" and "1" mean true. */
-    value = true;
-  else if (CMP3 (val, 'o', 'f', 'f') || CMP2 (val, 'n', 'o') || CMP1 (val, '0'))
-    /* "off", "no" and "0" mean false. */
-    value = false;
-  else
+  switch (cmd_boolean_internal (com, val, place))
     {
-      fprintf (stderr,
-               _("%s: %s: Invalid boolean %s; use `on' or `off'.\n"),
-               exec_name, com, quote (val));
-      return false;
-    }
+    case 0:
+      value = false;
+      break;
 
+    case 1:
+      value = true;
+      break;
+
+    default:
+      {
+        fprintf (stderr,
+                 _("%s: %s: Invalid boolean %s; use `on' or `off'.\n"),
+                 exec_name, com, quote (val));
+        return false;
+      }
+    }
   *(bool *) place = value;
   return true;
 }
 
+/* Store the check_cert value from VAL to PLACE.  COM is ignored,
+   except for error messages.  */
+static bool
+cmd_check_cert (const char *com, const char *val, void *place)
+{
+  int value;
+
+  switch (cmd_boolean_internal (com, val, place))
+    {
+    case 0:
+      value = CHECK_CERT_OFF;
+      break;
+
+    case 1:
+      value = CHECK_CERT_ON;
+      break;
+
+    default:
+      {
+        if (!c_strcasecmp (val, "quiet"))
+          value = CHECK_CERT_QUIET;
+        else
+          {
+            fprintf (stderr,
+                     _("%s: %s: Invalid %s; use `on', `off' or `quiet'.\n"),
+                     exec_name, com, quote (val));
+            return false;
+          }
+      }
+    }
+  *(int *) place = value;
+  return true;
+}
+
 /* Set the non-negative integer value from VAL to PLACE.  With
    incorrect specification, the number remains unchanged.  */
 static bool
@@ -945,7 +1050,7 @@ cmd_number (const char *com, const char *val, void *place)
 static bool
 cmd_number_inf (const char *com, const char *val, void *place)
 {
-  if (!strcasecmp (val, "inf"))
+  if (!c_strcasecmp (val, "inf"))
     {
       *(int *) place = 0;
       return true;
@@ -960,7 +1065,7 @@ cmd_string (const char *com _GL_UNUSED, const char *val, void *place)
 {
   char **pstring = (char **)place;
 
-  xfree_null (*pstring);
+  xfree (*pstring);
   *pstring = xstrdup (val);
   return true;
 }
@@ -971,7 +1076,7 @@ cmd_string_uppercase (const char *com _GL_UNUSED, const char *val, void *place)
 {
   char *q, **pstring;
   pstring = (char **)place;
-  xfree_null (*pstring);
+  xfree (*pstring);
 
   *pstring = xmalloc (strlen (val) + 1);
 
@@ -991,7 +1096,7 @@ cmd_file (const char *com _GL_UNUSED, const char *val, void *place)
 {
   char **pstring = (char **)place;
 
-  xfree_null (*pstring);
+  xfree (*pstring);
 
   /* #### If VAL is empty, perhaps should set *PLACE to NULL.  */
 
@@ -1009,6 +1114,20 @@ cmd_file (const char *com _GL_UNUSED, const char *val, void *place)
   return true;
 }
 
+/* like cmd_file, but insist on just a single option usage */
+static bool
+cmd_file_once (const char *com _GL_UNUSED, const char *val, void *place)
+{
+  if (*(char **)place)
+    {
+      fprintf (stderr, _("%s: %s must only be used once\n"),
+               exec_name, com);
+      return false;
+    }
+
+  return cmd_file(com, val, place);
+}
+
 /* Like cmd_file, but strips trailing '/' characters.  */
 static bool
 cmd_directory (const char *com, const char *val, void *place)
@@ -1258,7 +1377,7 @@ cmd_cert_type (const char *com, const char *val, void *place)
   return ok;
 }
 #endif
-
+
 /* Specialized helper functions, used by `commands' to handle some
    options specially.  */
 
@@ -1382,7 +1501,7 @@ cmd_spec_progress (const char *com, const char *val, void *place_ignored _GL_UNU
                exec_name, com, quote (val));
       return false;
     }
-  xfree_null (opt.progress_type);
+  xfree (opt.progress_type);
 
   /* Don't call set_progress_implementation here.  It will be called
      in main when it becomes clear what the log output is.  */
@@ -1446,6 +1565,8 @@ cmd_spec_restrict_file_names (const char *com, const char *val, void *place_igno
 
       if (VAL_IS ("unix"))
         restrict_os = restrict_unix;
+      else if (VAL_IS ("vms"))
+        restrict_os = restrict_vms;
       else if (VAL_IS ("windows"))
         restrict_os = restrict_windows;
       else if (VAL_IS ("lowercase"))
@@ -1460,7 +1581,7 @@ cmd_spec_restrict_file_names (const char *com, const char *val, void *place_igno
         {
           fprintf (stderr, _("\
 %s: %s: Invalid restriction %s,\n\
-    use [unix|windows],[lowercase|uppercase],[nocontrol],[ascii].\n"),
+    use [unix|vms|windows],[lowercase|uppercase],[nocontrol],[ascii].\n"),
                    exec_name, com, quote (val));
           return false;
         }
@@ -1483,7 +1604,7 @@ cmd_spec_restrict_file_names (const char *com, const char *val, void *place_igno
 static bool
 cmd_spec_report_speed (const char *com, const char *val, void *place_ignored _GL_UNUSED)
 {
-  opt.report_bps = strcasecmp (val, "bits") == 0;
+  opt.report_bps = c_strcasecmp (val, "bits") == 0;
   if (!opt.report_bps)
     fprintf (stderr, _("%s: %s: Invalid value %s.\n"), exec_name, com, quote (val));
   return opt.report_bps;
@@ -1498,6 +1619,8 @@ cmd_spec_secure_protocol (const char *com, const char *val, void *place)
     { "sslv2", secure_protocol_sslv2 },
     { "sslv3", secure_protocol_sslv3 },
     { "tlsv1", secure_protocol_tlsv1 },
+    { "tlsv1_1", secure_protocol_tlsv1_1 },
+    { "tlsv1_2", secure_protocol_tlsv1_2 },
     { "pfs", secure_protocol_pfs },
   };
   int ok = decode_string (val, choices, countof (choices), place);
@@ -1531,11 +1654,27 @@ cmd_spec_useragent (const char *com, const char *val, void *place_ignored _GL_UN
                exec_name, com, quote (val));
       return false;
     }
-  xfree_null (opt.useragent);
+  xfree (opt.useragent);
   opt.useragent = xstrdup (val);
   return true;
 }
 
+/* The --show-progress option is not a cmd_boolean since we need to keep track
+ * of whether the user explicitly requested the option or not. -1 means
+ * uninitialized. */
+static bool
+cmd_spec_progressdisp (const char *com, const char *val, void *place _GL_UNUSED)
+{
+  bool flag;
+  if (cmd_boolean (com, val, &flag))
+    {
+      opt.show_progress = flag;
+      return true;
+    }
+  return false;
+}
+
+
 /* The "verbose" option cannot be cmd_boolean because the variable is
    not bool -- it's of type int (-1 means uninitialized because of
    some random hackery for disallowing -q -v).  */
@@ -1547,11 +1686,12 @@ cmd_spec_verbose (const char *com, const char *val, void *place_ignored _GL_UNUS
   if (cmd_boolean (com, val, &flag))
     {
       opt.verbose = flag;
+      opt.show_progress = -1;
       return true;
     }
   return false;
 }
-
+
 /* Miscellaneous useful routines.  */
 
 /* A very simple atoi clone, more useful than atoi because it works on
@@ -1686,7 +1826,7 @@ decode_string (const char *val, const struct decode_item *items, int itemcount,
 {
   int i;
   for (i = 0; i < itemcount; i++)
-    if (0 == strcasecmp (val, items[i].name))
+    if (0 == c_strcasecmp (val, items[i].name))
       {
         *place = items[i].code;
         return true;
@@ -1729,55 +1869,62 @@ cleanup (void)
   spider_cleanup ();
   host_cleanup ();
   log_cleanup ();
-  netrc_cleanup (netrc_list);
+  netrc_cleanup ();
 
-  for (i = 0; i < nurl; i++)
-    xfree (url[i]);
-
-  xfree_null (opt.choose_config);
-  xfree_null (opt.lfilename);
-  xfree_null (opt.dir_prefix);
-  xfree_null (opt.input_filename);
-  xfree_null (opt.output_document);
+  xfree (opt.choose_config);
+  xfree (opt.lfilename);
+  xfree (opt.dir_prefix);
+  xfree (opt.input_filename);
+#ifdef HAVE_METALINK
+  xfree (opt.input_metalink);
+  xfree (opt.preferred_location);
+#endif
+  xfree (opt.output_document);
   free_vec (opt.accepts);
   free_vec (opt.rejects);
-  free_vec (opt.excludes);
-  free_vec (opt.includes);
+  free_vec ((char **)opt.excludes);
+  free_vec ((char **)opt.includes);
   free_vec (opt.domains);
   free_vec (opt.follow_tags);
   free_vec (opt.ignore_tags);
-  xfree_null (opt.progress_type);
-  xfree_null (opt.ftp_user);
-  xfree_null (opt.ftp_passwd);
-  xfree_null (opt.ftp_proxy);
-  xfree_null (opt.https_proxy);
-  xfree_null (opt.http_proxy);
+  xfree (opt.progress_type);
+  xfree (opt.ftp_user);
+  xfree (opt.ftp_passwd);
+  xfree (opt.ftp_proxy);
+  xfree (opt.https_proxy);
+  xfree (opt.http_proxy);
   free_vec (opt.no_proxy);
-  xfree_null (opt.useragent);
-  xfree_null (opt.referer);
-  xfree_null (opt.http_user);
-  xfree_null (opt.http_passwd);
+  xfree (opt.useragent);
+  xfree (opt.referer);
+  xfree (opt.http_user);
+  xfree (opt.http_passwd);
   free_vec (opt.user_headers);
   free_vec (opt.warc_user_headers);
 # ifdef HAVE_SSL
-  xfree_null (opt.cert_file);
-  xfree_null (opt.private_key);
-  xfree_null (opt.ca_directory);
-  xfree_null (opt.ca_cert);
-  xfree_null (opt.random_file);
-  xfree_null (opt.egd_file);
+  xfree (opt.cert_file);
+  xfree (opt.private_key);
+  xfree (opt.ca_directory);
+  xfree (opt.ca_cert);
+  xfree (opt.crl_file);
+  xfree (opt.random_file);
+  xfree (opt.egd_file);
 # endif
-  xfree_null (opt.bind_address);
-  xfree_null (opt.cookies_input);
-  xfree_null (opt.cookies_output);
-  xfree_null (opt.user);
-  xfree_null (opt.passwd);
-  xfree_null (opt.base_href);
-  xfree_null (opt.method);
+  xfree (opt.bind_address);
+  xfree (opt.cookies_input);
+  xfree (opt.cookies_output);
+  xfree (opt.user);
+  xfree (opt.passwd);
+  xfree (opt.base_href);
+  xfree (opt.method);
+  xfree (opt.post_file_name);
+  xfree (opt.post_data);
+  xfree (opt.body_data);
+  xfree (opt.body_file);
+  xfree (opt.rejected_log);
 
 #endif /* DEBUG_MALLOC */
 }
-
+
 /* Unit testing routines.  */
 
 #ifdef TESTING
@@ -1789,7 +1936,7 @@ test_commands_sorted(void)
 
   for (i = 1; i < countof(commands); ++i)
     {
-      if (strcasecmp (commands[i - 1].name, commands[i].name) > 0)
+      if (c_strcasecmp (commands[i - 1].name, commands[i].name) > 0)
         {
           mu_assert ("FAILED", false);
           break;
@@ -1805,7 +1952,7 @@ test_cmd_spec_restrict_file_names(void)
   static const struct {
     const char *val;
     int expected_restrict_files_os;
-    int expected_restrict_files_ctrl;
+    bool expected_restrict_files_ctrl;
     int expected_restrict_files_case;
     bool result;
   } test_array[] = {
@@ -1830,9 +1977,9 @@ test_cmd_spec_restrict_file_names(void)
       */
       mu_assert ("test_cmd_spec_restrict_file_names: wrong result",
                  res == test_array[i].result
-                 && opt.restrict_files_os   == test_array[i].expected_restrict_files_os
+                 && (int) opt.restrict_files_os   == test_array[i].expected_restrict_files_os
                  && opt.restrict_files_ctrl == test_array[i].expected_restrict_files_ctrl
-                 && opt.restrict_files_case == test_array[i].expected_restrict_files_case);
+                 && (int) opt.restrict_files_case == test_array[i].expected_restrict_files_case);
     }
 
   return NULL;

src/init.h

@@ -1,7 +1,7 @@
 /* Declarations for init.c.
    Copyright (C) 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004,
-   2005, 2006, 2007, 2008, 2009, 2010, 2011 Free Software Foundation,
-   Inc.
+   2005, 2006, 2007, 2008, 2009, 2010, 2011, 2015 Free Software
+   Foundation, Inc.
 
 This file is part of GNU Wget.
 

src/iri.c

@@ -1,5 +1,6 @@
 /* IRI related functions.
-   Copyright (C) 2008, 2009, 2010, 2011 Free Software Foundation, Inc.
+   Copyright (C) 2008, 2009, 2010, 2011, 2015 Free Software Foundation,
+   Inc.
 
 This file is part of GNU Wget.
 
@@ -35,41 +36,44 @@ as that of the covered work.  */
 #include <iconv.h>
 #include <stringprep.h>
 #include <idna.h>
+#include <idn-free.h>
 #include <errno.h>
 
 #include "utils.h"
+#include "url.h"
+#include "c-strcase.h"
+#include "c-strcasestr.h"
+#include "xstrndup.h"
 
 /* RFC3987 section 3.1 mandates STD3 ASCII RULES */
 #define IDNA_FLAGS  IDNA_USE_STD3_ASCII_RULES
 
 /* Note: locale encoding is kept in options struct (opt.locale) */
 
-static bool do_conversion (iconv_t cd, char *in, size_t inlen, char **out);
-
-
 /* Given a string containing "charset=XXX", return the encoding if found,
    or NULL otherwise */
 char *
-parse_charset (char *str)
+parse_charset (const char *str)
 {
+  const char *end;
   char *charset;
 
   if (!str || !*str)
     return NULL;
 
-  str = strcasestr (str, "charset=");
+  str = c_strcasestr (str, "charset=");
   if (!str)
     return NULL;
 
   str += 8;
-  charset = str;
+  end = str;
 
   /* sXXXav: which chars should be banned ??? */
-  while (*charset && !c_isspace (*charset))
-    charset++;
+  while (*end && !c_isspace (*end))
+    end++;
 
   /* sXXXav: could strdupdelim return NULL ? */
-  charset = strdupdelim (str, charset);
+  charset = strdupdelim (str, end);
 
   /* Do a minimum check on the charset value */
   if (!check_encoding_name (charset))
@@ -92,9 +96,9 @@ find_locale (void)
 
 /* Basic check of an encoding name. */
 bool
-check_encoding_name (char *encoding)
+check_encoding_name (const char *encoding)
 {
-  char *s = encoding;
+  const char *s = encoding;
 
   while (*s)
     {
@@ -110,52 +114,34 @@ check_encoding_name (char *encoding)
   return true;
 }
 
-/* Try converting string str from locale to UTF-8. Return a new string
-   on success, or str on error or if conversion isn't needed. */
-const char *
-locale_to_utf8 (const char *str)
-{
-  iconv_t l2u;
-  char *new;
-
-  /* That shouldn't happen, just in case */
-  if (!opt.locale)
-    {
-      logprintf (LOG_VERBOSE, _("locale_to_utf8: locale is unset\n"));
-      opt.locale = find_locale ();
-    }
-
-  if (!opt.locale || !strcasecmp (opt.locale, "utf-8"))
-    return str;
-
-  l2u = iconv_open ("UTF-8", opt.locale);
-  if (l2u == (iconv_t)(-1))
-    {
-      logprintf (LOG_VERBOSE, _("Conversion from %s to %s isn't supported\n"),
-                 quote (opt.locale), quote ("UTF-8"));
-      return str;
-    }
-
-  if (do_conversion (l2u, (char *) str, strlen ((char *) str), &new))
-    return (const char *) new;
-
-  return str;
-}
-
 /* Do the conversion according to the passed conversion descriptor cd. *out
    will contain the transcoded string on success. *out content is
    unspecified otherwise. */
 static bool
-do_conversion (iconv_t cd, char *in, size_t inlen, char **out)
+do_conversion (const char *tocode, const char *fromcode, char const *in_org, size_t inlen, char **out)
 {
+  iconv_t cd;
   /* sXXXav : hummm hard to guess... */
-  size_t len, done, outlen = inlen * 2;
+  size_t len, done, outlen;
   int invalid = 0, tooshort = 0;
-  char *s;
+  char *s, *in, *in_save;
 
-  s = xmalloc (outlen + 1);
-  *out = s;
-  len = outlen;
+  cd = iconv_open (tocode, fromcode);
+  if (cd == (iconv_t)(-1))
+    {
+      logprintf (LOG_VERBOSE, _("Conversion from %s to UTF-8 isn't supported\n"),
+                 quote (opt.locale));
+      *out = NULL;
+      return false;
+    }
+
+  /* iconv() has to work on an unescaped string */
+  in_save = in = xstrndup (in_org, inlen);
+  url_unescape_except_reserved (in);
+  inlen = strlen(in);
+
+  len = outlen = inlen * 2;
+  *out = s = xmalloc (outlen + 1);
   done = 0;
 
   for (;;)
@@ -164,6 +150,16 @@ do_conversion (iconv_t cd, char *in, size_t inlen, char **out)
         {
           *out = s;
           *(s + len - outlen - done) = '\0';
+          xfree(in_save);
+          iconv_close(cd);
+          IF_DEBUG
+          {
+            /* not not print out embedded passwords, in_org might be an URL */
+            if (!strchr(in_org, '@') && !strchr(*out, '@'))
+              debug_logprintf ("converted '%s' (%s) -> '%s' (%s)\n", in_org, fromcode, *out, tocode);
+            else
+              debug_logprintf ("%s: logging suppressed, strings may contain password\n", __func__);
+          }
           return true;
         }
 
@@ -202,42 +198,129 @@ do_conversion (iconv_t cd, char *in, size_t inlen, char **out)
         }
     }
 
+    xfree(in_save);
+    iconv_close(cd);
+    IF_DEBUG
+    {
+      /* not not print out embedded passwords, in_org might be an URL */
+      if (!strchr(in_org, '@') && !strchr(*out, '@'))
+        debug_logprintf ("converted '%s' (%s) -> '%s' (%s)\n", in_org, fromcode, *out, tocode);
+      else
+        debug_logprintf ("%s: logging suppressed, strings may contain password\n", __func__);
+    }
     return false;
 }
 
+/* Try converting string str from locale to UTF-8. Return a new string
+   on success, or str on error or if conversion isn't needed. */
+const char *
+locale_to_utf8 (const char *str)
+{
+  char *new;
+
+  /* That shouldn't happen, just in case */
+  if (!opt.locale)
+    {
+      logprintf (LOG_VERBOSE, _("locale_to_utf8: locale is unset\n"));
+      opt.locale = find_locale ();
+    }
+
+  if (!opt.locale || !c_strcasecmp (opt.locale, "utf-8"))
+    return str;
+
+  if (do_conversion ("UTF-8", opt.locale, (char *) str, strlen ((char *) str), &new))
+    return (const char *) new;
+
+  xfree (new);
+  return str;
+}
+
+/*
+ * Work around a libidn <= 1.30 vulnerability.
+ *
+ * The function checks for a valid UTF-8 character sequence before
+ * passing it to idna_to_ascii_8z().
+ *
+ * [1] http://lists.gnu.org/archive/html/help-libidn/2015-05/msg00002.html
+ * [2] https://lists.gnu.org/archive/html/bug-wget/2015-06/msg00002.html
+ * [3] http://curl.haxx.se/mail/lib-2015-06/0143.html
+ */
+static bool
+_utf8_is_valid(const char *utf8)
+{
+  const unsigned char *s = (const unsigned char *) utf8;
+
+  while (*s)
+    {
+      if ((*s & 0x80) == 0) /* 0xxxxxxx ASCII char */
+        s++;
+      else if ((*s & 0xE0) == 0xC0) /* 110xxxxx 10xxxxxx */
+        {
+          if ((s[1] & 0xC0) != 0x80)
+            return false;
+          s+=2;
+        }
+      else if ((*s & 0xF0) == 0xE0) /* 1110xxxx 10xxxxxx 10xxxxxx */
+        {
+          if ((s[1] & 0xC0) != 0x80 || (s[2] & 0xC0) != 0x80)
+            return false;
+          s+=3;
+        }
+      else if ((*s & 0xF8) == 0xF0) /* 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx */
+        {
+          if ((s[1] & 0xC0) != 0x80 || (s[2] & 0xC0) != 0x80 || (s[3] & 0xC0) != 0x80)
+            return false;
+          s+=4;
+        }
+      else
+        return false;
+    }
+
+  return true;
+}
+
 /* Try to "ASCII encode" UTF-8 host. Return the new domain on success or NULL
    on error. */
 char *
-idn_encode (struct iri *i, char *host)
+idn_encode (const struct iri *i, const char *host)
 {
-  char *new;
   int ret;
+  char *ascii_encoded;
+  char *utf8_encoded = NULL;
 
   /* Encode to UTF-8 if not done */
   if (!i->utf8_encode)
     {
-      if (!remote_to_utf8 (i, (const char *) host, (const char **) &new))
+      if (!remote_to_utf8 (i, host, &utf8_encoded))
           return NULL;  /* Nothing to encode or an error occured */
-      host = new;
     }
 
-  /* toASCII UTF-8 NULL terminated string */
-  ret = idna_to_ascii_8z (host, &new, IDNA_FLAGS);
+  if (!_utf8_is_valid(utf8_encoded ? utf8_encoded : host))
+    {
+      logprintf (LOG_VERBOSE, _("Invalid UTF-8 sequence: %s\n"),
+                 quote(utf8_encoded ? utf8_encoded : host));
+      xfree (utf8_encoded);
+      return NULL;
+    }
+
+  /* Store in ascii_encoded the ASCII UTF-8 NULL terminated string */
+  ret = idna_to_ascii_8z (utf8_encoded ? utf8_encoded : host, &ascii_encoded, IDNA_FLAGS);
+  xfree (utf8_encoded);
+
   if (ret != IDNA_SUCCESS)
     {
-      /* sXXXav : free new when needed ! */
       logprintf (LOG_VERBOSE, _("idn_encode failed (%d): %s\n"), ret,
                  quote (idna_strerror (ret)));
       return NULL;
     }
 
-  return new;
+  return ascii_encoded;
 }
 
 /* Try to decode an "ASCII encoded" host. Return the new domain in the locale
    on success or NULL on error. */
 char *
-idn_decode (char *host)
+idn_decode (const char *host)
 {
   char *new;
   int ret;
@@ -256,9 +339,8 @@ idn_decode (char *host)
 /* Try to transcode string str from remote encoding to UTF-8. On success, *new
    contains the transcoded string. *new content is unspecified otherwise. */
 bool
-remote_to_utf8 (struct iri *iri, const char *str, const char **new)
+remote_to_utf8 (const struct iri *iri, const char *str, char **new)
 {
-  iconv_t cd;
   bool ret = false;
 
   if (!iri->uri_encoding)
@@ -267,11 +349,11 @@ remote_to_utf8 (struct iri *iri, const char *str, const char **new)
   /* When `i->uri_encoding' == "UTF-8" there is nothing to convert.  But we must
      test for non-ASCII symbols for correct hostname processing in `idn_encode'
      function. */
-  if (!strcmp (iri->uri_encoding, "UTF-8"))
+  if (!c_strcasecmp (iri->uri_encoding, "UTF-8"))
     {
-      const char *p = str;
-      for (p = str; *p; p++)
-        if (*p < 0)
+      const unsigned char *p;
+      for (p = (unsigned char *) str; *p; p++)
+        if (*p > 127)
           {
             *new = strdup (str);
             return true;
@@ -279,19 +361,13 @@ remote_to_utf8 (struct iri *iri, const char *str, const char **new)
       return false;
     }
 
-  cd = iconv_open ("UTF-8", iri->uri_encoding);
-  if (cd == (iconv_t)(-1))
-    return false;
-
-  if (do_conversion (cd, (char *) str, strlen ((char *) str), (char **) new))
+  if (do_conversion ("UTF-8", iri->uri_encoding, str, strlen (str), new))
     ret = true;
 
-  iconv_close (cd);
-
   /* Test if something was converted */
-  if (!strcmp (str, *new))
+  if (*new && !strcmp (str, *new))
     {
-      xfree ((char *) *new);
+      xfree (*new);
       return false;
     }
 
@@ -325,23 +401,26 @@ struct iri *iri_dup (const struct iri *src)
 void
 iri_free (struct iri *i)
 {
-  xfree_null (i->uri_encoding);
-  xfree_null (i->content_encoding);
-  xfree_null (i->orig_url);
-  xfree (i);
+  if (i)
+    {
+      xfree (i->uri_encoding);
+      xfree (i->content_encoding);
+      xfree (i->orig_url);
+      xfree (i);
+    }
 }
 
 /* Set uri_encoding of struct iri i. If a remote encoding was specified, use
    it unless force is true. */
 void
-set_uri_encoding (struct iri *i, char *charset, bool force)
+set_uri_encoding (struct iri *i, const char *charset, bool force)
 {
   DEBUGP (("URI encoding = %s\n", charset ? quote (charset) : "None"));
   if (!force && opt.encoding_remote)
     return;
   if (i->uri_encoding)
     {
-      if (charset && !strcasecmp (i->uri_encoding, charset))
+      if (charset && !c_strcasecmp (i->uri_encoding, charset))
         return;
       xfree (i->uri_encoding);
     }
@@ -351,14 +430,14 @@ set_uri_encoding (struct iri *i, char *charset, bool force)
 
 /* Set content_encoding of struct iri i. */
 void
-set_content_encoding (struct iri *i, char *charset)
+set_content_encoding (struct iri *i, const char *charset)
 {
   DEBUGP (("URI content encoding = %s\n", charset ? quote (charset) : "None"));
   if (opt.encoding_remote)
     return;
   if (i->content_encoding)
     {
-      if (charset && !strcasecmp (i->content_encoding, charset))
+      if (charset && !c_strcasecmp (i->content_encoding, charset))
         return;
       xfree (i->content_encoding);
     }

src/iri.h

@@ -1,5 +1,6 @@
 /* Internationalization related declarations.
-   Copyright (C) 2008, 2009, 2010, 2011 Free Software Foundation, Inc.
+   Copyright (C) 2008, 2009, 2010, 2011, 2015 Free Software Foundation,
+   Inc.
 
 This file is part of GNU Wget.
 
@@ -39,18 +40,21 @@ struct iri {
 
 #ifdef ENABLE_IRI
 
-char *parse_charset (char *str);
+# include <idna.h>
+# include <idn-free.h>
+
+char *parse_charset (const char *str);
 char *find_locale (void);
-bool check_encoding_name (char *encoding);
+bool check_encoding_name (const char *encoding);
 const char *locale_to_utf8 (const char *str);
-char *idn_encode (struct iri *i, char *host);
-char *idn_decode (char *host);
-bool remote_to_utf8 (struct iri *i, const char *str, const char **new);
+char *idn_encode (const struct iri *i, const char *host);
+char *idn_decode (const char *host);
+bool remote_to_utf8 (const struct iri *i, const char *str, char **new);
 struct iri *iri_new (void);
 struct iri *iri_dup (const struct iri *);
 void iri_free (struct iri *i);
-void set_uri_encoding (struct iri *i, char *charset, bool force);
-void set_content_encoding (struct iri *i, char *charset);
+void set_uri_encoding (struct iri *i, const char *charset, bool force);
+void set_content_encoding (struct iri *i, const char *charset);
 
 #else /* ENABLE_IRI */
 
@@ -62,6 +66,7 @@ extern struct iri dummy_iri;
 #define locale_to_utf8(str)         (str)
 #define idn_encode(a,b)             NULL
 #define idn_decode(str)             NULL
+#define idn_free(str)               ((void)0)
 #define remote_to_utf8(a,b,c)       false
 #define iri_new()                   (&dummy_iri)
 #define iri_dup(a)                  (&dummy_iri)

src/log.c

@@ -1,6 +1,6 @@
 /* Messages logging.
    Copyright (C) 1998, 1999, 2000, 2001, 2002, 2003, 2004, 2005, 2006,
-   2007, 2008, 2009, 2010, 2011 Free Software Foundation, Inc.
+   2007, 2008, 2009, 2010, 2011, 2015 Free Software Foundation, Inc.
 
 This file is part of GNU Wget.
 
@@ -73,7 +73,7 @@ as that of the covered work.  */
    - Inhibiting output.  When Wget receives SIGHUP, but redirecting
    the output fails, logging is inhibited.  */
 
-
+
 /* The file descriptor used for logging.  This is NULL before log_init
    is called; logging functions log to stderr then.  log_init sets it
    either to stderr or to a file pointer obtained from fopen().  If
@@ -142,7 +142,7 @@ static int log_line_current = -1;
 static bool trailing_line;
 
 static void check_redirect_output (void);
-
+
 #define ROT_ADVANCE(num) do {                   \
   if (++num >= SAVED_LOG_LINES)                 \
     num = 0;                                    \
@@ -156,11 +156,7 @@ static void
 free_log_line (int num)
 {
   struct log_ln *ln = log_lines + num;
-  if (ln->malloced_line)
-    {
-      xfree (ln->malloced_line);
-      ln->malloced_line = NULL;
-    }
+  xfree (ln->malloced_line);
   ln->content = NULL;
 }
 
@@ -265,7 +261,7 @@ saved_append (const char *s)
       s = end;
     }
 }
-
+
 /* Check X against opt.verbose and opt.quiet.  The semantics is as
    follows:
 
@@ -314,6 +310,14 @@ get_log_fp (void)
   return stderr;
 }
 
+static FILE *
+get_progress_fp (void)
+{
+  if (opt.show_progress == true)
+      return stderr;
+  return get_log_fp();
+}
+
 /* Returns the file descriptor for the secondary log file. This is
    WARCLOGFP, except if called before log_init, in which case it
    returns stderr.  This is useful in case someone calls a logging
@@ -338,7 +342,7 @@ log_set_warc_log_fp (FILE * fp)
 {
   warclogfp = fp;
 }
-
+
 /* Log a literal string S.  The string is logged as-is, without a
    newline appended.  */
 
@@ -349,8 +353,14 @@ logputs (enum log_options o, const char *s)
   FILE *warcfp;
 
   check_redirect_output ();
-  if ((fp = get_log_fp ()) == NULL)
+  if (o == LOG_PROGRESS)
+    fp = get_progress_fp ();
+  else
+    fp = get_log_fp ();
+
+  if (fp == NULL)
     return;
+
   warcfp = get_warc_log_fp ();
   CHECK_VERBOSE (o);
 
@@ -385,7 +395,7 @@ struct logvprintf_state {
    (An alternative approach would be to use va_copy, but that's not
    portable.)  */
 
-static bool
+static bool GCC_FORMAT_ATTR (2, 0)
 log_vprintf_internal (struct logvprintf_state *state, const char *fmt,
                       va_list args)
 {
@@ -451,8 +461,7 @@ log_vprintf_internal (struct logvprintf_state *state, const char *fmt,
   FPUTS (write_ptr, fp);
   if (warcfp != NULL)
     FPUTS (write_ptr, warcfp);
-  if (state->bigmsg)
-    xfree (state->bigmsg);
+  xfree (state->bigmsg);
 
  flush:
   if (flush_log_p)
@@ -541,6 +550,7 @@ logprintf (enum log_options o, const char *fmt, ...)
   CHECK_VERBOSE (o);
 
   xzero (lpstate);
+  errno = 0;
   do
     {
       va_start (args, fmt);
@@ -580,7 +590,7 @@ debug_logprintf (const char *fmt, ...)
     }
 }
 #endif /* ENABLE_DEBUG */
-
+
 /* Open FILE and set up a logging stream.  If FILE cannot be opened,
    exit with status of 1.  */
 void
@@ -588,11 +598,18 @@ log_init (const char *file, bool appendp)
 {
   if (file)
     {
-      logfp = fopen (file, appendp ? "a" : "w");
-      if (!logfp)
+      if (HYPHENP (file))
         {
-          fprintf (stderr, "%s: %s: %s\n", exec_name, file, strerror (errno));
-          exit (WGET_EXIT_GENERIC_ERROR);
+          logfp = stdout;
+        }
+      else
+        {
+          logfp = fopen (file, appendp ? "a" : "w");
+          if (!logfp)
+            {
+              fprintf (stderr, "%s: %s: %s\n", exec_name, file, strerror (errno));
+              exit (WGET_EXIT_GENERIC_ERROR);
+            }
         }
     }
   else
@@ -675,7 +692,7 @@ log_dump_context (void)
   fflush (fp);
   fflush (warcfp);
 }
-
+
 /* String escape functions. */
 
 /* Return the number of non-printable characters in SOURCE.
@@ -847,9 +864,9 @@ log_cleanup (void)
 {
   size_t i;
   for (i = 0; i < countof (ring); i++)
-    xfree_null (ring[i].buffer);
+    xfree (ring[i].buffer);
 }
-
+
 /* When SIGHUP or SIGUSR1 are received, the output is redirected
    elsewhere.  Such redirection is only allowed once. */
 static enum { RR_NONE, RR_REQUESTED, RR_DONE } redirect_request = RR_NONE;

src/log.h

@@ -1,6 +1,6 @@
 /* Declarations for log.c.
    Copyright (C) 1998, 1999, 2000, 2001, 2002, 2003, 2004, 2005, 2006,
-   2007, 2008, 2009, 2010, 2011 Free Software Foundation, Inc.
+   2007, 2008, 2009, 2010, 2011, 2015 Free Software Foundation, Inc.
 
 This file is part of GNU Wget.
 

src/metalink.c

@@ -0,0 +1,785 @@
+/* Metalink module.
+   Copyright (C) 2015 Free Software Foundation, Inc.
+
+This file is part of GNU Wget.
+
+GNU Wget is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 3 of the License, or (at
+your option) any later version.
+
+GNU Wget is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with Wget.  If not, see <http://www.gnu.org/licenses/>.
+
+Additional permission under GNU GPL version 3 section 7
+
+If you modify this program, or any covered work, by linking or
+combining it with the OpenSSL project's OpenSSL library (or a
+modified version of that library), containing parts covered by the
+terms of the OpenSSL or SSLeay licenses, the Free Software Foundation
+grants you additional permission to convey the resulting work.
+Corresponding Source for a non-source form of such a combination
+shall include the source code for the parts of OpenSSL used as well
+as that of the covered work.  */
+
+#include "wget.h"
+#ifdef HAVE_METALINK
+
+#include "metalink.h"
+#include "retr.h"
+#include "exits.h"
+#include "utils.h"
+#include "sha256.h"
+#include "xstrndup.h"
+#include <errno.h>
+#include <unistd.h> /* For unlink.  */
+#include <metalink/metalink_parser.h>
+#ifdef HAVE_GPGME
+#include <gpgme.h>
+#include <fcntl.h> /* For open and close.  */
+#endif
+
+#ifdef TESTING
+#include "test.h"
+#endif
+
+/* Loop through all files in metalink structure and retrieve them.
+   Returns RETROK if all files were downloaded.
+   Returns last retrieval error (from retrieve_url) if some files
+   could not be downloaded.  */
+uerr_t
+retrieve_from_metalink (const metalink_t* metalink)
+{
+  metalink_file_t **mfile_ptr;
+  uerr_t last_retr_err = RETROK; /* Store last encountered retrieve error.  */
+
+  FILE *_output_stream = output_stream;
+  bool _output_stream_regular = output_stream_regular;
+  char *_output_document = opt.output_document;
+
+  DEBUGP (("Retrieving from Metalink\n"));
+
+  /* No files to download.  */
+  if (!metalink->files)
+    return RETROK;
+
+  if (opt.output_document)
+    {
+      /* We cannot support output_document as we need to compute checksum
+         of downloaded file, and to remove it if the checksum is bad.  */
+      logputs (LOG_NOTQUIET,
+               _("-O not supported for metalink download. Ignoring.\n"));
+    }
+
+  for (mfile_ptr = metalink->files; *mfile_ptr; mfile_ptr++)
+    {
+      metalink_file_t *mfile = *mfile_ptr;
+      metalink_resource_t **mres_ptr;
+      char *filename = NULL;
+      bool hash_ok = false;
+
+      uerr_t retr_err = METALINK_MISSING_RESOURCE;
+
+      /* -1 -> file should be rejected
+         0 -> could not verify
+         1 -> verified successfully  */
+      char sig_status = 0;
+
+      output_stream = NULL;
+
+      DEBUGP (("Processing metalink file %s...\n", quote (mfile->name)));
+
+      /* Resources are sorted by priority.  */
+      for (mres_ptr = mfile->resources; *mres_ptr; mres_ptr++)
+        {
+          metalink_resource_t *mres = *mres_ptr;
+          metalink_checksum_t **mchksum_ptr, *mchksum;
+          struct iri *iri;
+          struct url *url;
+          int url_err;
+
+          if (!RES_TYPE_SUPPORTED (mres->type))
+            {
+              logprintf (LOG_VERBOSE,
+                         _("Resource type %s not supported, ignoring...\n"),
+                         quote (mres->type));
+              continue;
+            }
+
+          retr_err = METALINK_RETR_ERROR;
+
+          /* If output_stream is not NULL, then we have failed on
+             previous resource and are retrying. Thus, remove the file.  */
+          if (output_stream)
+            {
+              fclose (output_stream);
+              output_stream = NULL;
+              if (unlink (filename))
+                logprintf (LOG_NOTQUIET, "unlink: %s\n", strerror (errno));
+              xfree (filename);
+            }
+
+          /* Parse our resource URL.  */
+          iri = iri_new ();
+          set_uri_encoding (iri, opt.locale, true);
+          url = url_parse (mres->url, &url_err, iri, false);
+
+          if (!url)
+            {
+              char *error = url_error (mres->url, url_err);
+              logprintf (LOG_NOTQUIET, "%s: %s.\n", mres->url, error);
+              xfree (error);
+              inform_exit_status (URLERROR);
+              iri_free (iri);
+              continue;
+            }
+          else
+            {
+              /* Avoid recursive Metalink from HTTP headers.  */
+              bool _metalink_http = opt.metalink_over_http;
+
+              /* Assure proper local file name regardless of the URL
+                 of particular Metalink resource.
+                 To do that we create the local file here and put
+                 it as output_stream. We restore the original configuration
+                 after we are finished with the file.  */
+              output_stream = unique_create (mfile->name, true, &filename);
+              output_stream_regular = true;
+
+              /* Store the real file name for displaying in messages.  */
+              opt.output_document = filename;
+
+              opt.metalink_over_http = false;
+              DEBUGP (("Storing to %s\n", filename));
+              retr_err = retrieve_url (url, mres->url, NULL, NULL,
+                                       NULL, NULL, opt.recursive, iri, false);
+              opt.metalink_over_http = _metalink_http;
+            }
+          url_free (url);
+          iri_free (iri);
+
+          if (retr_err == RETROK)
+            {
+              FILE *local_file;
+
+              /* Check the digest.  */
+              local_file = fopen (filename, "rb");
+              if (!local_file)
+                {
+                  logprintf (LOG_NOTQUIET, _("Could not open downloaded file.\n"));
+                  continue;
+                }
+
+              for (mchksum_ptr = mfile->checksums; *mchksum_ptr; mchksum_ptr++)
+                {
+                  char sha256[SHA256_DIGEST_SIZE];
+                  char sha256_txt[2 * SHA256_DIGEST_SIZE + 1];
+
+                  mchksum = *mchksum_ptr;
+
+                  /* I have seen both variants...  */
+                  if (strcasecmp (mchksum->type, "sha256")
+                      && strcasecmp (mchksum->type, "sha-256"))
+                    {
+                      DEBUGP (("Ignoring unsupported checksum type %s.\n",
+                               quote (mchksum->type)));
+                      continue;
+                    }
+
+                  logprintf (LOG_VERBOSE, _("Computing checksum for %s\n"),
+                             quote (mfile->name));
+
+                  sha256_stream (local_file, sha256);
+                  wg_hex_to_string (sha256_txt, sha256, SHA256_DIGEST_SIZE);
+                  DEBUGP (("Declared hash: %s\n", mchksum->hash));
+                  DEBUGP (("Computed hash: %s\n", sha256_txt));
+                  if (!strcmp (sha256_txt, mchksum->hash))
+                    {
+                      logputs (LOG_VERBOSE,
+                               _("Checksum matches.\n"));
+                      hash_ok = true;
+                    }
+                  else
+                    {
+                      logprintf (LOG_NOTQUIET,
+                                 _("Checksum mismatch for file %s.\n"),
+                                 quote (mfile->name));
+                      hash_ok = false;
+                    }
+
+                  /* Stop as soon as we checked the supported checksum.  */
+                  break;
+                } /* Iterate over available checksums.  */
+              fclose (local_file);
+              local_file = NULL;
+
+              if (!hash_ok)
+                continue;
+
+              sig_status = 0; /* Not verified.  */
+
+#ifdef HAVE_GPGME
+              /* Check the crypto signature.
+
+                 Note that the signtures from Metalink in XML will not be
+                 parsed when using libmetalink version older than 0.1.3.
+                 Metalink-over-HTTP is not affected by this problem.  */
+              if (mfile->signature)
+                {
+                  metalink_signature_t *msig = mfile->signature;
+                  gpgme_error_t gpgerr;
+                  gpgme_ctx_t gpgctx;
+                  gpgme_data_t gpgsigdata, gpgdata;
+                  gpgme_verify_result_t gpgres;
+                  gpgme_signature_t gpgsig;
+                  gpgme_protocol_t gpgprot = GPGME_PROTOCOL_UNKNOWN;
+                  int fd = -1;
+
+                  /* Initialize the library - as name suggests.  */
+                  gpgme_check_version (NULL);
+
+                  /* Open data file.  */
+                  fd = open (filename, O_RDONLY);
+                  if (fd == -1)
+                    {
+                      logputs (LOG_NOTQUIET,
+                               _("Could not open downloaded file for signature "
+                                 "verification.\n"));
+                      goto gpg_skip_verification;
+                    }
+
+                  /* Assign file descriptor to GPG data structure.  */
+                  gpgerr = gpgme_data_new_from_fd (&gpgdata, fd);
+                  if (gpgerr != GPG_ERR_NO_ERROR)
+                    {
+                      logprintf (LOG_NOTQUIET,
+                                 "GPGME data_new_from_fd: %s\n",
+                                 gpgme_strerror (gpgerr));
+                      goto gpg_skip_verification;
+                    }
+
+                  /* Prepare new GPGME context.  */
+                  gpgerr = gpgme_new (&gpgctx);
+                  if (gpgerr != GPG_ERR_NO_ERROR)
+                    {
+                      logprintf (LOG_NOTQUIET,
+                                 "GPGME new: %s\n",
+                                 gpgme_strerror (gpgerr));
+                      gpgme_data_release (gpgdata);
+                      goto gpg_skip_verification;
+                    }
+
+                  DEBUGP (("Veryfying signature %s:\n%s\n",
+                           quote (msig->mediatype),
+                           msig->signature));
+
+                  /* Check signature type.  */
+                  if (!strcmp (msig->mediatype, "application/pgp-signature"))
+                    gpgprot = GPGME_PROTOCOL_OpenPGP;
+                  else /* Unsupported signature type.  */
+                    {
+                      gpgme_release (gpgctx);
+                      gpgme_data_release (gpgdata);
+                      goto gpg_skip_verification;
+                    }
+
+                  gpgerr = gpgme_set_protocol (gpgctx, gpgprot);
+                  if (gpgerr != GPG_ERR_NO_ERROR)
+                    {
+                      logprintf (LOG_NOTQUIET,
+                                 "GPGME set_protocol: %s\n",
+                                 gpgme_strerror (gpgerr));
+                      gpgme_release (gpgctx);
+                      gpgme_data_release (gpgdata);
+                      goto gpg_skip_verification;
+                    }
+
+                  /* Load the signature.  */
+                  gpgerr = gpgme_data_new_from_mem (&gpgsigdata,
+                                                    msig->signature,
+                                                    strlen (msig->signature),
+                                                    0);
+                  if (gpgerr != GPG_ERR_NO_ERROR)
+                    {
+                      logprintf (LOG_NOTQUIET,
+                                 _("GPGME data_new_from_mem: %s\n"),
+                                 gpgme_strerror (gpgerr));
+                      gpgme_release (gpgctx);
+                      gpgme_data_release (gpgdata);
+                      goto gpg_skip_verification;
+                    }
+
+                  /* Verify the signature.  */
+                  gpgerr = gpgme_op_verify (gpgctx, gpgsigdata, gpgdata, NULL);
+                  if (gpgerr != GPG_ERR_NO_ERROR)
+                    {
+                      logprintf (LOG_NOTQUIET,
+                                 _("GPGME op_verify: %s\n"),
+                                 gpgme_strerror (gpgerr));
+                      gpgme_data_release (gpgsigdata);
+                      gpgme_release (gpgctx);
+                      gpgme_data_release (gpgdata);
+                      goto gpg_skip_verification;
+                    }
+
+                  /* Check the results.  */
+                  gpgres = gpgme_op_verify_result (gpgctx);
+                  if (!gpgres)
+                    {
+                      logputs (LOG_NOTQUIET,
+                               _("GPGME op_verify_result: NULL\n"));
+                      gpgme_data_release (gpgsigdata);
+                      gpgme_release (gpgctx);
+                      gpgme_data_release (gpgdata);
+                      goto gpg_skip_verification;
+                    }
+
+                  /* The list is null-terminated.  */
+                  for (gpgsig = gpgres->signatures; gpgsig; gpgsig = gpgsig->next)
+                    {
+                      DEBUGP (("Checking signature 0x%p\n",
+                               (void *) gpgsig));
+                      DEBUGP (("Summary=0x%x Status=0x%x\n",
+                               gpgsig->summary, gpgsig->status & 0xFFFF));
+
+                      if (gpgsig->summary
+                          & (GPGME_SIGSUM_VALID | GPGME_SIGSUM_GREEN))
+                        {
+                          logputs (LOG_VERBOSE,
+                                   _("Signature validation suceeded.\n"));
+                          sig_status = 1;
+                          break;
+                        }
+
+                      if (gpgsig->summary & GPGME_SIGSUM_RED)
+                        {
+                          logputs (LOG_NOTQUIET,
+                                   _("Invalid signature. Rejecting resource.\n"));
+                          sig_status = -1;
+                          break;
+                        }
+
+                      if (gpgsig->summary == 0
+                          && (gpgsig->status & 0xFFFF) == GPG_ERR_NO_ERROR)
+                        {
+                          logputs (LOG_VERBOSE,
+                                   _("Data matches signature, but signature "
+                                     "is not trusted.\n"));
+                        }
+
+                      if ((gpgsig->status & 0xFFFF) != GPG_ERR_NO_ERROR)
+                        {
+                          logprintf (LOG_NOTQUIET,
+                                     "GPGME: %s\n",
+                                     gpgme_strerror (gpgsig->status & 0xFFFF));
+                        }
+                    }
+                  gpgme_data_release (gpgsigdata);
+                  gpgme_release (gpgctx);
+                  gpgme_data_release (gpgdata);
+gpg_skip_verification:
+                  if (fd != -1)
+                    close (fd);
+                } /* endif (mfile->signature) */
+#endif
+              /* Stop if file was downloaded with success.  */
+              if (sig_status >= 0)
+                break;
+            } /* endif RETR_OK.  */
+        } /* Iterate over resources.  */
+
+      if (retr_err != RETROK)
+        {
+          logprintf (LOG_VERBOSE, _("Failed to download %s. Skipping resource.\n"),
+                     quote (mfile->name));
+        }
+      else if (!hash_ok)
+        {
+          retr_err = METALINK_CHKSUM_ERROR;
+          logprintf (LOG_NOTQUIET,
+                     _("File %s retrieved but checksum does not match. "
+                       "\n"), quote (mfile->name));
+        }
+#ifdef HAVE_GPGME
+        /* Signature will be only validated if hash check was successful.  */
+      else if (sig_status < 0)
+        {
+          retr_err = METALINK_SIG_ERROR;
+          logprintf (LOG_NOTQUIET,
+                     _("File %s retrieved but signature does not match. "
+                       "\n"), quote (mfile->name));
+        }
+#endif
+      last_retr_err = retr_err == RETROK ? last_retr_err : retr_err;
+
+      /* Remove the file if error encountered or if option specified.
+         Note: the file has been downloaded using *_loop. Therefore, it
+         is not necessary to keep the file for continuated download.  */
+      if ((retr_err != RETROK || opt.delete_after)
+           && filename != NULL && file_exists_p (filename))
+        {
+          logprintf (LOG_VERBOSE, _("Removing %s.\n"), quote (filename));
+          if (unlink (filename))
+            logprintf (LOG_NOTQUIET, "unlink: %s\n", strerror (errno));
+        }
+      if (output_stream)
+        {
+          fclose (output_stream);
+          output_stream = NULL;
+        }
+      xfree (filename);
+    } /* Iterate over files.  */
+
+  /* Restore original values.  */
+  opt.output_document = _output_document;
+  output_stream_regular = _output_stream_regular;
+  output_stream = _output_stream;
+
+  return last_retr_err;
+}
+
+int metalink_res_cmp (const void* v1, const void* v2)
+{
+  const metalink_resource_t *res1 = *(metalink_resource_t **) v1,
+                            *res2 = *(metalink_resource_t **) v2;
+  if (res1->preference != res2->preference)
+    return res2->preference - res1->preference;
+  if (res1->priority != res2->priority)
+    return res1->priority - res2->priority;
+  if (opt.preferred_location)
+    {
+      int cmp = 0;
+      if (res1->location &&
+          !strcasecmp (opt.preferred_location, res1->location))
+        cmp -= 1;
+      if (res2->location &&
+          !strcasecmp (opt.preferred_location, res2->location))
+        cmp += 1;
+      return cmp;
+    }
+  return 0;
+}
+
+/*
+  Find value of given key. This is intended for Link header, but will
+  work with any header that uses ';' as field separator and '=' as key-value
+  separator.
+
+  Link           = "Link" ":" #link-value
+  link-value     = "<" URI-Reference ">" *( ";" link-param )
+  link-param     = ( ( "rel" "=" relation-types )
+                 | ( "anchor" "=" <"> URI-Reference <"> )
+                 | ( "rev" "=" relation-types )
+                 | ( "hreflang" "=" Language-Tag )
+                 | ( "media" "=" ( MediaDesc | ( <"> MediaDesc <"> ) ) )
+                 | ( "title" "=" quoted-string )
+                 | ( "title*" "=" ext-value )
+                 | ( "type" "=" ( media-type | quoted-mt ) )
+                 | ( link-extension ) )
+  link-extension = ( parmname [ "=" ( ptoken | quoted-string ) ] )
+                 | ( ext-name-star "=" ext-value )
+  ext-name-star  = parmname "*" ; reserved for RFC2231-profiled
+                                ; extensions.  Whitespace NOT
+                                ; allowed in between.
+  ptoken         = 1*ptokenchar
+  ptokenchar     = "!" | "#" | "$" | "%" | "&" | "'" | "("
+                 | ")" | "*" | "+" | "-" | "." | "/" | DIGIT
+                 | ":" | "<" | "=" | ">" | "?" | "@" | ALPHA
+                 | "[" | "]" | "^" | "_" | "`" | "{" | "|"
+                 | "}" | "~"
+  media-type     = type-name "/" subtype-name
+  quoted-mt      = <"> media-type <">
+  relation-types = relation-type
+                 | <"> relation-type *( 1*SP relation-type ) <">
+  relation-type  = reg-rel-type | ext-rel-type
+  reg-rel-type   = LOALPHA *( LOALPHA | DIGIT | "." | "-" )
+  ext-rel-type   = URI
+
+ See more: rfc5988
+*/
+bool
+find_key_value (const char *start, const char *end, const char *key, char **value)
+{
+  const char *eq;
+  size_t key_len = strlen (key);
+  const char *val_beg, *val_end;
+  const char *key_beg;
+
+  key_beg = start;
+
+  while (key_beg + key_len + 1 < end)
+    {
+      /* Skip whitespaces.  */
+      while (key_beg + key_len + 1 < end && c_isspace (*key_beg))
+        key_beg++;
+      if (strncmp (key_beg, key, key_len))
+        {
+          /* Find next token.  */
+          while (key_beg + key_len + 1 < end && *key_beg != ';')
+            key_beg++;
+          key_beg++;
+          continue;
+        }
+      else
+        {
+          /* Find equals sign.  */
+          eq = key_beg + key_len;
+          while (eq < end && c_isspace (*eq))
+            eq++;
+          if (eq == end)
+            return false;
+          if (*eq != '=')
+            {
+              key_beg++;
+              continue;
+            }
+
+          val_beg = eq + 1;
+          while (val_beg < end && c_isspace (*val_beg))
+            val_beg++;
+          if (val_beg == end)
+            return false;
+          val_end = val_beg + 1;
+          while (val_end < end && *val_end != ';' && !c_isspace (*val_end))
+            val_end++;
+          *value = xstrndup (val_beg, val_end - val_beg);
+          return true;
+        }
+    }
+  *value = NULL;
+  return false;
+}
+
+/* This is to check if given token exists in HTTP header. Tokens are
+   separated by ';'. */
+bool
+has_key (const char *start, const char *end, const char *key)
+{
+  const char *pos; /* Here would the token start.  */
+  size_t key_len = strlen (key);
+
+  pos = start;
+  while (pos + key_len <= end)
+    {
+      /* Skip whitespaces at beginning.  */
+      while (pos + key_len <= end && c_isspace (*pos))
+        pos++;
+
+      /* Does the prefix of pos match our key?  */
+      if (strncmp (key, pos, key_len))
+        {
+          /* This was not a match.
+             Skip all characters until beginning of next token.  */
+          while (pos + key_len <= end && *pos != ';')
+            pos++;
+          pos++;
+          continue;
+        }
+
+      /* key is prefix of pos. Is it the exact token or just a prefix?  */
+      pos += key_len;
+      while (pos < end && c_isspace (*pos))
+        pos++;
+      if (pos == end || *pos == ';')
+        return true;
+
+      /* This was not a match (just a prefix).
+         Skip all characters until beginning of next token.  */
+      while (pos + key_len <= end && *pos != ';')
+        pos++;
+      pos++;
+    }
+  return false;
+}
+
+/* Find all key=value pairs delimited with ';' or ','. This is intended for
+   Digest header parsing.
+   The usage is:
+
+   const char *pos;
+   for (pos = header_beg; pos = find_key_values (pos, header_end, &key, &val); pos++)
+   {
+     ...
+   }
+
+ */
+const char *
+find_key_values (const char *start, const char *end, char **key, char **value)
+{
+  const char *key_start, *key_end;
+  const char *eq;
+  const char *val_start, *val_end;
+
+  eq = start;
+  while (eq < end && *eq != '=')
+    {
+      /* Skip tokens without =value part.  */
+      if (*eq == ';' || *eq == ',')
+        start = eq + 1;
+      eq++;
+    }
+
+  if (eq >= end)
+    return NULL;
+
+  key_start = start;
+  while (key_start < eq && c_isspace (*key_start))
+    key_start++;
+
+  key_end = eq - 1;
+  while (key_end > key_start && c_isspace (*key_end))
+    key_end--;
+  key_end++;
+
+  val_start = eq + 1;
+  while (val_start < end && c_isspace (*val_start))
+    val_start++;
+
+  val_end = val_start;
+
+  while (val_end < end && *val_end != ';' &&
+         *val_end != ',' && !c_isspace (*val_end))
+    val_end++;
+
+  *key = xstrndup (key_start, key_end - key_start);
+  *value = xstrndup (val_start, val_end - val_start);
+
+  /* Skip trailing whitespaces.  */
+  while (val_end < end && c_isspace (*val_end))
+    val_end++;
+
+  return val_end;
+}
+
+#ifdef TESTING
+const char *
+test_find_key_values (void)
+{
+  static const char *header_data = "key1=val1;key2=val2 ;key3=val3; key4=val4"\
+                                   " ; key5=val5;key6 =val6;key7= val7; "\
+                                   "key8 = val8 ;    key9    =   val9       "\
+                                   "    ,key10= val10,key11,key12=val12";
+  static const struct
+  {
+    const char *key;
+    const char *val;
+  } test_array[] =
+  {
+    { "key1", "val1" },
+    { "key2", "val2" },
+    { "key3", "val3" },
+    { "key4", "val4" },
+    { "key5", "val5" },
+    { "key6", "val6" },
+    { "key7", "val7" },
+    { "key8", "val8" },
+    { "key9", "val9" },
+    { "key10", "val10" },
+    { "key12", "val12" }
+  };
+  const char *pos;
+  char *key, *value;
+  size_t i = 0;
+
+  for (pos = header_data; (pos = find_key_values (pos,
+                                                 header_data + strlen (header_data),
+                                                 &key, &value)); pos++)
+    {
+      mu_assert ("test_find_key_values: wrong result",
+                 !strcmp (test_array[i].val, value) &&
+                 !strcmp (test_array[i].key, key));
+      xfree (key);
+      xfree (value);
+      i++;
+    }
+
+  return NULL;
+}
+
+const char *
+test_find_key_value (void)
+{
+  static const char *header_data = "key1=val1;key2=val2 ;key3=val3; key4=val4"\
+                                   " ; key5=val5;key6 =val6;key7= val7; "\
+                                   "key8 = val8 ;    key9    =   val9       ";
+  static const struct
+  {
+    const char *key;
+    const char *val;
+    bool result;
+  } test_array[] =
+  {
+    { "key1",  "val1", true },
+    { "key2",  "val2", true },
+    { "key3",  "val3", true },
+    { "key4",  "val4", true },
+    { "key5",  "val5", true },
+    { "key6",  "val6", true },
+    { "key7",  "val7", true },
+    { "key8",  "val8", true },
+    { "key9",  "val9", true },
+    { "key10", NULL,   false },
+    {  "ey1",  NULL,   false },
+    { "dey1",  NULL,   false }
+  };
+  size_t i;
+
+  for (i=0; i < countof (test_array); ++i)
+    {
+      bool result;
+      char *value;
+
+      result = find_key_value (header_data,
+                               header_data + strlen(header_data),
+                               test_array[i].key, &value);
+
+      mu_assert ("test_find_key_value: wrong result",
+                 result == test_array[i].result &&
+                 ((!test_array[i].result && !value) ||
+                  !strcmp (value, test_array[i].val)));
+
+      xfree (value);
+    }
+
+  return NULL;
+}
+
+const char *
+test_has_key (void)
+{
+  static const char *header_data = "key1=val2;token1;xyz; token2;xyz;token3 ;"\
+                                   "xyz; token4 ;xyz;   token5  ";
+  struct
+  {
+    const char *token;
+    bool result;
+  } test_array[] =
+  {
+    { "key1=val2", true },
+    { "token1", true },
+    { "token2", true },
+    { "token3", true },
+    { "token4", true },
+    { "token5", true },
+    { "token6", false },
+    { "oken1", false },
+    { "poken1", false },
+    { "key1=val2", true }
+  };
+  size_t i;
+
+  for (i = 0; i < countof (test_array); ++i)
+    mu_assert ("test_has_key: wrong result",
+               has_key (header_data, header_data + strlen (header_data),
+                        test_array[i].token) == test_array[i].result);
+
+  return NULL;
+}
+#endif
+
+#endif /* HAVE_METALINK */

src/metalink.h

@@ -0,0 +1,60 @@
+/* Declarations for metalink.c.
+   Copyright (C) 2015 Free Software Foundation, Inc.
+
+This file is part of GNU Wget.
+
+GNU Wget is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 3 of the License, or
+(at your option) any later version.
+
+GNU Wget is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with Wget.  If not, see <http://www.gnu.org/licenses/>.
+
+Additional permission under GNU GPL version 3 section 7
+
+If you modify this program, or any covered work, by linking or
+combining it with the OpenSSL project's OpenSSL library (or a
+modified version of that library), containing parts covered by the
+terms of the OpenSSL or SSLeay licenses, the Free Software Foundation
+grants you additional permission to convey the resulting work.
+Corresponding Source for a non-source form of such a combination
+shall include the source code for the parts of OpenSSL used as well
+as that of the covered work.  */
+#if ! defined METALINK_H && defined HAVE_METALINK
+#define	METALINK_H
+
+#include <metalink/metalink_types.h>
+#include "wget.h"
+
+#ifdef HAVE_SSL
+# define RES_TYPE_SUPPORTED(x)\
+    ((!x) || !strcmp (x, "http") || !strcmp (x, "https") || !strcmp (x, "ftp") || !strcmp (x, "ftps"))
+#else
+# define RES_TYPE_SUPPORTED(x)\
+    ((!x) || !strcmp (x, "ftp") || !strcmp (x, "http"))
+#endif
+
+#define DEFAULT_PRI 999999
+#define VALID_PRI_RANGE(x) ((x) > 0 && (x) < 1000000)
+
+uerr_t retrieve_from_metalink (const metalink_t *metalink);
+
+int metalink_res_cmp (const void *res1, const void *res2);
+
+bool find_key_value (const char *start,
+                     const char *end,
+                     const char *key,
+                     char **value);
+bool has_key (const char *start, const char *end, const char *key);
+const char *find_key_values (const char *start,
+                             const char *end,
+                             char **key,
+                             char **value);
+
+#endif	/* METALINK_H */

src/mswindows.c

@@ -1,7 +1,7 @@
 /* mswindows.c -- Windows-specific support
    Copyright (C) 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004,
-   2005, 2006, 2007, 2008, 2009, 2010, 2011, 2014 Free Software Foundation,
-   Inc.
+   2005, 2006, 2007, 2008, 2009, 2010, 2011, 2014, 2015 Free Software
+   Foundation, Inc.
 
 This file is part of GNU Wget.
 
@@ -85,11 +85,11 @@ windows_main (char **exec_name)
   if (p)
     *p = '\0';
 }
-
+
 static void
 ws_cleanup (void)
 {
-  xfree ((char*)exec_name);
+  xfree (exec_name);
   WSACleanup ();
 }
 
@@ -367,8 +367,8 @@ static int old_percentage = -1;
 void
 ws_changetitle (const char *url)
 {
-  xfree_null (title_buf);
-  xfree_null (curr_url);
+  xfree (title_buf);
+  xfree (curr_url);
   title_buf = xmalloc (strlen (url) + 20);
   curr_url = xstrdup (url);
   old_percentage = -1;
@@ -477,7 +477,7 @@ ws_startup (void)
   set_sleep_mode ();
   SetConsoleCtrlHandler (ws_handler, TRUE);
 }
-
+
 /* run_with_timeout Windows implementation.  */
 
 /* Stack size 0 uses default thread stack-size (reserve+commit).

src/mswindows.h

@@ -1,7 +1,7 @@
 /* Declarations for windows
    Copyright (C) 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004,
-   2005, 2006, 2007, 2008, 2009, 2010, 2011 Free Software Foundation,
-   Inc.
+   2005, 2006, 2007, 2008, 2009, 2010, 2011, 2015 Free Software
+   Foundation, Inc.
 
 This file is part of GNU Wget.
 

src/netrc.c

@@ -1,5 +1,5 @@
 /* Read and parse the .netrc file to get hosts, accounts, and passwords.
-   Copyright (C) 1996, 2007, 2008, 2009, 2010, 2011 Free Software
+   Copyright (C) 1996, 2007, 2008, 2009, 2010, 2011, 2015 Free Software
    Foundation, Inc.
 
 This file is part of GNU Wget.
@@ -163,7 +163,7 @@ search_netrc (const char *host, const char **acc, const char **passwd,
 
 /* Normally, these functions would be defined by your package.  */
 # define xmalloc malloc
-# define xfree free
+# define xfree(p) do { free ((void *) (p)); p = NULL; } while (0)
 # define xstrdup strdup
 
 # define xrealloc realloc
@@ -183,9 +183,9 @@ maybe_add_to_list (acc_t **newentry, acc_t **list)
   if (a && ! a->acc)
     {
       /* Free any allocated space.  */
-      xfree_null (a->host);
-      xfree_null (a->acc);
-      xfree_null (a->passwd);
+      xfree (a->host);
+      xfree (a->acc);
+      xfree (a->passwd);
     }
   else
     {
@@ -424,9 +424,9 @@ free_netrc(acc_t *l)
   while (l)
     {
       t = l->next;
-      xfree_null (l->acc);
-      xfree_null (l->passwd);
-      xfree_null (l->host);
+      xfree (l->acc);
+      xfree (l->passwd);
+      xfree (l->host);
       xfree (l);
       l = t;
     }

src/netrc.h

@@ -1,6 +1,6 @@
 /* Declarations for netrc.c
-   Copyright (C) 1996, 1996, 1997, 2007, 2008, 2009, 2010, 2011 Free
-   Software Foundation, Inc.
+   Copyright (C) 1996, 1996, 1997, 2007, 2008, 2009, 2010, 2011, 2015
+   Free Software Foundation, Inc.
 
 This file is part of GNU Wget.
 

src/openssl.c

@@ -1,6 +1,6 @@
 /* SSL support via OpenSSL library.
    Copyright (C) 2000, 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008,
-   2009, 2010, 2011, 2012 Free Software Foundation, Inc.
+   2009, 2010, 2011, 2012, 2015 Free Software Foundation, Inc.
    Originally contributed by Christian Fraenkel.
 
 This file is part of GNU Wget.
@@ -40,6 +40,11 @@ as that of the covered work.  */
 #include <openssl/x509v3.h>
 #include <openssl/err.h>
 #include <openssl/rand.h>
+#include <openssl/bio.h>
+#if OPENSSL_VERSION_NUMBER >= 0x00907000
+#include <openssl/conf.h>
+#include <openssl/engine.h>
+#endif
 
 #include "utils.h"
 #include "connect.h"
@@ -86,9 +91,11 @@ init_prng (void)
   if (RAND_status ())
     return;
 
+#ifdef HAVE_RAND_EGD
   /* Get random data from EGD if opt.egd_file was used.  */
   if (opt.egd_file && *opt.egd_file)
     RAND_egd (opt.egd_file);
+#endif
 
   if (RAND_status ())
     return;
@@ -164,6 +171,9 @@ static int ssl_true_initialized = 0;
 bool
 ssl_init (void)
 {
+  SSL_METHOD const *meth;
+  long ssl_options = 0;
+
 #if OPENSSL_VERSION_NUMBER >= 0x00907000
   if (ssl_true_initialized == 0)
     {
@@ -172,8 +182,6 @@ ssl_init (void)
     }
 #endif
 
-  SSL_METHOD const *meth;
-
   if (ssl_ctx)
     /* The SSL has already been initialized. */
     return true;
@@ -187,6 +195,12 @@ ssl_init (void)
       goto error;
     }
 
+#if OPENSSL_VERSION_NUMBER >= 0x00907000
+  OPENSSL_load_builtin_modules();
+  ENGINE_load_builtin_engines();
+  CONF_modules_load_file(NULL, NULL,
+      CONF_MFLAGS_DEFAULT_SECTION|CONF_MFLAGS_IGNORE_MISSING_FILE);
+#endif
   SSL_library_init ();
   SSL_load_error_strings ();
   SSLeay_add_all_algorithms ();
@@ -194,22 +208,48 @@ ssl_init (void)
 
   switch (opt.secure_protocol)
     {
-    case secure_protocol_auto:
-      meth = SSLv23_client_method ();
-      break;
 #ifndef OPENSSL_NO_SSL2
     case secure_protocol_sslv2:
       meth = SSLv2_client_method ();
       break;
 #endif
+
+#ifndef OPENSSL_NO_SSL3
     case secure_protocol_sslv3:
       meth = SSLv3_client_method ();
       break;
+#endif
+
+    case secure_protocol_auto:
     case secure_protocol_pfs:
+      meth = SSLv23_client_method ();
+      ssl_options |= SSL_OP_NO_SSLv2 | SSL_OP_NO_SSLv3;
+      break;
     case secure_protocol_tlsv1:
       meth = TLSv1_client_method ();
       break;
+
+#if OPENSSL_VERSION_NUMBER >= 0x10001000
+    case secure_protocol_tlsv1_1:
+      meth = TLSv1_1_client_method ();
+      break;
+
+    case secure_protocol_tlsv1_2:
+      meth = TLSv1_2_client_method ();
+      break;
+#else
+    case secure_protocol_tlsv1_1:
+      logprintf (LOG_NOTQUIET, _("Your OpenSSL version is too old to support TLSv1.1\n"));
+      goto error;
+
+    case secure_protocol_tlsv1_2:
+      logprintf (LOG_NOTQUIET, _("Your OpenSSL version is too old to support TLSv1.2\n"));
+      goto error;
+#endif
+
     default:
+      logprintf (LOG_NOTQUIET, _("OpenSSL: unimplemented 'secure-protocol' option value %d\n"), opt.secure_protocol);
+      logprintf (LOG_NOTQUIET, _("Please report this issue to bug-wget@gnu.org\n"));
       abort ();
     }
 
@@ -219,6 +259,9 @@ ssl_init (void)
   if (!ssl_ctx)
     goto error;
 
+  if (ssl_options)
+    SSL_CTX_set_options (ssl_ctx, ssl_options);
+
   /* OpenSSL ciphers: https://www.openssl.org/docs/apps/ciphers.html
    * Since we want a good protection, we also use HIGH (that excludes MD4 ciphers and some more)
    */
@@ -228,6 +271,18 @@ ssl_init (void)
   SSL_CTX_set_default_verify_paths (ssl_ctx);
   SSL_CTX_load_verify_locations (ssl_ctx, opt.ca_cert, opt.ca_directory);
 
+  if (opt.crl_file)
+    {
+      X509_STORE *store = SSL_CTX_get_cert_store (ssl_ctx);
+      X509_LOOKUP *lookup;
+
+      if (!(lookup = X509_STORE_add_lookup (store, X509_LOOKUP_file ()))
+          || (!X509_load_crl_file (lookup, opt.crl_file, X509_FILETYPE_PEM)))
+        goto error;
+
+      X509_STORE_set_flags (store, X509_V_FLAG_CRL_CHECK | X509_V_FLAG_CRL_CHECK_ALL);
+    }
+
   /* SSL_VERIFY_NONE instructs OpenSSL not to abort SSL_connect if the
      certificate is invalid.  We verify the certificate separately in
      ssl_check_certificate, which provides much better diagnostics
@@ -237,10 +292,17 @@ ssl_init (void)
   /* Use the private key from the cert file unless otherwise specified. */
   if (opt.cert_file && !opt.private_key)
     {
-      opt.private_key = opt.cert_file;
+      opt.private_key = xstrdup (opt.cert_file);
       opt.private_key_type = opt.cert_type;
     }
 
+  /* Use cert from private key file unless otherwise specified. */
+  if (opt.private_key && !opt.cert_file)
+    {
+      opt.cert_file = xstrdup (opt.private_key);
+      opt.cert_type = opt.private_key_type;
+    }
+
   if (opt.cert_file)
     if (SSL_CTX_use_certificate_file (ssl_ctx, opt.cert_file,
                                       key_type_to_ssl_type (opt.cert_type))
@@ -272,6 +334,7 @@ ssl_init (void)
 struct openssl_transport_context
 {
   SSL *conn;                    /* SSL connection handle */
+  SSL_SESSION *sess;            /* SSL session info */
   char *last_error;             /* last error printed with openssl_errstr */
 };
 
@@ -370,7 +433,7 @@ openssl_errstr (int fd _GL_UNUSED, void *arg)
     return NULL;
 
   /* Get rid of previous contents of ctx->last_error, if any.  */
-  xfree_null (ctx->last_error);
+  xfree (ctx->last_error);
 
   /* Iterate over OpenSSL's error stack and accumulate errors in the
      last_error buffer, separated by "; ".  This is better than using
@@ -414,7 +477,7 @@ openssl_close (int fd, void *arg)
 
   SSL_shutdown (conn);
   SSL_free (conn);
-  xfree_null (ctx->last_error);
+  xfree (ctx->last_error);
   xfree (ctx);
 
   close (fd);
@@ -452,7 +515,7 @@ ssl_connect_with_timeout_callback(void *arg)
    Returns true on success, false on failure.  */
 
 bool
-ssl_connect_wget (int fd, const char *hostname)
+ssl_connect_wget (int fd, const char *hostname, int *continue_session)
 {
   SSL *conn;
   struct scwt_context scwt_ctx;
@@ -465,7 +528,7 @@ ssl_connect_wget (int fd, const char *hostname)
   if (!conn)
     goto error;
 #if OPENSSL_VERSION_NUMBER >= 0x0090806fL && !defined(OPENSSL_NO_TLSEXT)
-  /* If the SSL library was build with support for ServerNameIndication
+  /* If the SSL library was built with support for ServerNameIndication
      then use it whenever we have a hostname.  If not, don't, ever. */
   if (! is_valid_ip_address (hostname))
     {
@@ -477,6 +540,14 @@ ssl_connect_wget (int fd, const char *hostname)
     }
 #endif
 
+  if (continue_session)
+    {
+      /* attempt to resume a previous SSL session */
+      ctx = (struct openssl_transport_context *) fd_transport_context (*continue_session);
+      if (!ctx || !ctx->sess || !SSL_set_session (conn, ctx->sess))
+        goto error;
+    }
+
 #ifndef FD_TO_SOCKET
 # define FD_TO_SOCKET(X) (X)
 #endif
@@ -490,11 +561,14 @@ ssl_connect_wget (int fd, const char *hostname)
     DEBUGP (("SSL handshake timed out.\n"));
     goto timeout;
   }
-  if (scwt_ctx.result <= 0 || conn->state != SSL_ST_OK)
+  if (scwt_ctx.result <= 0 || SSL_state(conn) != SSL_ST_OK)
     goto error;
 
   ctx = xnew0 (struct openssl_transport_context);
   ctx->conn = conn;
+  ctx->sess = SSL_get0_session (conn);
+  if (!ctx->sess)
+    logprintf (LOG_NOTQUIET, "WARNING: Could not save SSL session data for socket %d\n", fd);
 
   /* Register FD with Wget's transport layer, i.e. arrange that our
      functions are used for reading, writing, and polling.  */
@@ -555,6 +629,27 @@ pattern_match (const char *pattern, const char *string)
   return *n == '\0';
 }
 
+static char *_get_rfc2253_formatted (X509_NAME *name)
+{
+  int len;
+  char *out = NULL;
+  BIO* b;
+
+  if ((b = BIO_new (BIO_s_mem ())))
+    {
+      if (X509_NAME_print_ex (b, name, 0, XN_FLAG_RFC2253) >= 0
+          && (len = BIO_number_written (b)) > 0)
+        {
+          out = xmalloc (len + 1);
+          BIO_read (b, out, len);
+          out[len] = 0;
+        }
+      BIO_free (b);
+    }
+
+  return out ? out : xstrdup("");
+}
+
 /* Verify the validity of the certificate presented by the server.
    Also check that the "common name" of the server, as presented by
    its certificate, corresponds to HOST.  (HOST typically comes from
@@ -587,6 +682,10 @@ ssl_check_certificate (int fd, const char *host)
   SSL *conn = ctx->conn;
   assert (conn != NULL);
 
+  /* The user explicitly said to not check for the certificate.  */
+  if (opt.check_cert == CHECK_CERT_QUIET)
+    return success;
+
   cert = SSL_get_peer_certificate (conn);
   if (!cert)
     {
@@ -598,23 +697,25 @@ ssl_check_certificate (int fd, const char *host)
 
   IF_DEBUG
     {
-      char *subject = X509_NAME_oneline (X509_get_subject_name (cert), 0, 0);
-      char *issuer = X509_NAME_oneline (X509_get_issuer_name (cert), 0, 0);
+      char *subject = _get_rfc2253_formatted (X509_get_subject_name (cert));
+      char *issuer = _get_rfc2253_formatted (X509_get_issuer_name (cert));
       DEBUGP (("certificate:\n  subject: %s\n  issuer:  %s\n",
                quotearg_n_style (0, escape_quoting_style, subject),
                quotearg_n_style (1, escape_quoting_style, issuer)));
-      OPENSSL_free (subject);
-      OPENSSL_free (issuer);
+      xfree (subject);
+      xfree (issuer);
     }
 
   vresult = SSL_get_verify_result (conn);
   if (vresult != X509_V_OK)
     {
-      char *issuer = X509_NAME_oneline (X509_get_issuer_name (cert), 0, 0);
+      char *issuer = _get_rfc2253_formatted (X509_get_issuer_name (cert));
       logprintf (LOG_NOTQUIET,
                  _("%s: cannot verify %s's certificate, issued by %s:\n"),
                  severity, quotearg_n_style (0, escape_quoting_style, host),
                  quote_n (1, issuer));
+      xfree(issuer);
+
       /* Try to print more user-friendly (and translated) messages for
          the frequent verification errors.  */
       switch (vresult)
@@ -710,7 +811,7 @@ ssl_check_certificate (int fd, const char *host)
                 }
             }
         }
-      sk_GENERAL_NAME_free (subjectAltNames);
+      sk_GENERAL_NAME_pop_free(subjectAltNames, GENERAL_NAME_free);
       if (host_in_octet_string)
         ASN1_OCTET_STRING_free(host_in_octet_string);
 
@@ -783,13 +884,12 @@ ssl_check_certificate (int fd, const char *host)
   X509_free (cert);
 
  no_cert:
-  if (opt.check_cert && !success)
+  if (opt.check_cert == CHECK_CERT_ON && !success)
     logprintf (LOG_NOTQUIET, _("\
 To connect to %s insecurely, use `--no-check-certificate'.\n"),
                quotearg_style (escape_quoting_style, host));
 
-  /* Allow --no-check-cert to disable certificate checking. */
-  return opt.check_cert ? success : true;
+  return opt.check_cert == CHECK_CERT_ON ? success : true;
 }
 
 /*

src/options.h

@@ -1,7 +1,7 @@
 /* struct options.
    Copyright (C) 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004,
-   2005, 2006, 2007, 2008, 2009, 2010, 2011 Free Software Foundation,
-   Inc.
+   2005, 2006, 2007, 2008, 2009, 2010, 2011, 2015 Free Software
+   Foundation, Inc.
 
 This file is part of GNU Wget.
 
@@ -29,6 +29,13 @@ Corresponding Source for a non-source form of such a combination
 shall include the source code for the parts of OpenSSL used as well
 as that of the covered work.  */
 
+enum CHECK_CERT_MODES
+{
+  CHECK_CERT_OFF,
+  CHECK_CERT_ON,
+  CHECK_CERT_QUIET
+};
+
 struct options
 {
   int verbose;                  /* Are we verbose?  (First set to -1,
@@ -58,6 +65,11 @@ struct options
   char *dir_prefix;             /* The top of directory tree */
   char *lfilename;              /* Log filename */
   char *input_filename;         /* Input filename */
+#ifdef HAVE_METALINK
+  char *input_metalink;         /* Input metalink file */
+  bool metalink_over_http;      /* Use Metalink if present in HTTP response */
+  char *preferred_location;     /* Preferred location for Metalink resources */
+#endif
   char *choose_config;          /* Specified config file */
   bool noconfig;                /* Ignore all config files? */
   bool force_html;              /* Is the input file an HTML file? */
@@ -133,7 +145,7 @@ struct options
   char **no_proxy;
   char *base_href;
   char *progress_type;          /* progress indicator type. */
-  bool show_progress;           /* Show only the progress bar */
+  int  show_progress;           /* Show only the progress bar */
   bool noscroll;                /* Don't scroll the filename in the progressbar */
   char *proxy_user; /*oli*/
   char *proxy_passwd;
@@ -165,6 +177,7 @@ struct options
 #endif
 
   bool timestamping;            /* Whether to use time-stamping. */
+  bool if_modified_since;       /* Whether to use conditional get requests.  */
 
   bool backup_converted;        /* Do we save pre-converted files as *.orig? */
   int backups;                  /* Are numeric backups made? */
@@ -176,6 +189,9 @@ struct options
                                    NULL. */
   bool convert_links;           /* Will the links be converted
                                    locally? */
+  bool convert_file_only;       /* Convert only the file portion of the URI (i.e. basename).
+                                   Leave everything else untouched. */
+
   bool remove_listing;          /* Do we remove .listing files
                                    generated by FTP? */
   bool htmlify;                 /* Do we HTML-ify the OS-dependent
@@ -202,9 +218,11 @@ struct options
     secure_protocol_sslv2,
     secure_protocol_sslv3,
     secure_protocol_tlsv1,
+    secure_protocol_tlsv1_1,
+    secure_protocol_tlsv1_2,
     secure_protocol_pfs
   } secure_protocol;            /* type of secure protocol to use. */
-  bool check_cert;              /* whether to validate the server's cert */
+  int check_cert;               /* whether to validate the server's cert */
   char *cert_file;              /* external client certificate to use. */
   char *private_key;            /* private key file (if not internal). */
   enum keyfile_type {
@@ -216,10 +234,15 @@ struct options
 
   char *ca_directory;           /* CA directory (hash files) */
   char *ca_cert;                /* CA certificate file to use */
+  char *crl_file;               /* file with CRLs */
 
   char *random_file;            /* file with random data to seed the PRNG */
   char *egd_file;               /* file name of the egd daemon socket */
   bool https_only;              /* whether to follow HTTPS only */
+  bool ftps_resume_ssl;
+  bool ftps_fallback_to_ftp;
+  bool ftps_implicit;
+  bool ftps_clear_data_connection;
 #endif /* HAVE_SSL */
 
   bool cookies;                 /* whether cookies are used. */
@@ -236,6 +259,7 @@ struct options
 
   enum {
     restrict_unix,
+    restrict_vms,
     restrict_windows
   } restrict_files_os;          /* file name restriction ruleset. */
   bool restrict_files_ctrl;     /* non-zero if control chars in URLs
@@ -285,6 +309,13 @@ struct options
   bool show_all_dns_entries;    /* Show all the DNS entries when resolving a
                                    name. */
   bool report_bps;              /*Output bandwidth in bits format*/
+
+  char *rejected_log;           /* The file to log rejected URLS to. */
+
+#ifdef HAVE_HSTS
+  bool hsts;
+  char *hsts_file;
+#endif
 };
 
 extern struct options opt;

src/progress.c

@@ -1,6 +1,6 @@
 /* Download progress.
    Copyright (C) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009,
-   2010, 2011 Free Software Foundation, Inc.
+   2010, 2011, 2015 Free Software Foundation, Inc.
 
 This file is part of GNU Wget.
 
@@ -37,10 +37,12 @@ as that of the covered work.  */
 #include <unistd.h>
 #include <signal.h>
 #include <wchar.h>
+#include <mbiter.h>
 
 #include "progress.h"
 #include "utils.h"
 #include "retr.h"
+#include "c-strcase.h"
 
 struct progress_implementation {
   const char *name;
@@ -194,7 +196,7 @@ progress_finish (void *progress, double dltime)
 {
   current_impl->finish (progress, dltime);
 }
-
+
 /* Dot-printing. */
 
 struct dot_progress {
@@ -426,7 +428,7 @@ dot_set_params (char *params)
     return;
 
   /* We use this to set the retrieval style.  */
-  if (!strcasecmp (params, "default"))
+  if (!c_strcasecmp (params, "default"))
     {
       /* Default style: 1K dots, 10 dots in a cluster, 50 dots in a
          line.  */
@@ -434,7 +436,7 @@ dot_set_params (char *params)
       opt.dot_spacing = 10;
       opt.dots_in_line = 50;
     }
-  else if (!strcasecmp (params, "binary"))
+  else if (!c_strcasecmp (params, "binary"))
     {
       /* "Binary" retrieval: 8K dots, 16 dots in a cluster, 48 dots
          (384K) in a line.  */
@@ -442,7 +444,7 @@ dot_set_params (char *params)
       opt.dot_spacing = 16;
       opt.dots_in_line = 48;
     }
-  else if (!strcasecmp (params, "mega"))
+  else if (!c_strcasecmp (params, "mega"))
     {
       /* "Mega" retrieval, for retrieving very long files; each dot is
          64K, 8 dots in a cluster, 6 clusters (3M) in a line.  */
@@ -450,7 +452,7 @@ dot_set_params (char *params)
       opt.dot_spacing = 8;
       opt.dots_in_line = 48;
     }
-  else if (!strcasecmp (params, "giga"))
+  else if (!c_strcasecmp (params, "giga"))
     {
       /* "Giga" retrieval, for retrieving very very *very* long files;
          each dot is 1M, 8 dots in a cluster, 4 clusters (32M) in a
@@ -464,7 +466,7 @@ dot_set_params (char *params)
              _("Invalid dot style specification %s; leaving unchanged.\n"),
              quote (params));
 }
-
+
 /* "Thermometer" (bar) progress. */
 
 /* Assumed screen width if we can't find the real value.  */
@@ -592,7 +594,8 @@ bar_create (const char *f_download, wgint initial, wgint total)
   bp->width = screen_width - 1;
   /* + enough space for the terminating zero, and hopefully enough room
    * for multibyte characters. */
-  bp->buffer = xmalloc (bp->width + 100);
+#define BUF_LEN (bp->width + 100)
+  bp->buffer = xmalloc (BUF_LEN);
 
   logputs (LOG_VERBOSE, "\n");
 
@@ -691,7 +694,7 @@ bar_finish (void *progress, double dltime)
    The idea is that for fast downloads, we get the speed over exactly
    the last three seconds.  For slow downloads (where a network read
    takes more than 150ms to complete), we get the speed over a larger
-   time period, as large as it takes to complete thirty reads.  This
+   time period, as large as it takes to complete twenty reads.  This
    is good because slow downloads tend to fluctuate more and a
    3-second average would be too erratic.  */
 
@@ -812,8 +815,39 @@ count_cols (const char *mbs)
     }
   return cols;
 }
+
+static int
+cols_to_bytes (const char *mbs, const int cols, int *ncols)
+{
+  int p_cols = 0, bytes = 0;
+  mbchar_t mbc;
+  mbi_iterator_t iter;
+  mbi_init (iter, mbs, strlen(mbs));
+  while (p_cols < cols && mbi_avail (iter))
+    {
+      mbc = mbi_cur (iter);
+      p_cols += mb_width (mbc);
+      /* The multibyte character has exceeded the total number of columns we
+       * have available. The remaining bytes will be padded with a space. */
+      if (p_cols > cols)
+        {
+          p_cols -= mb_width (mbc);
+          break;
+        }
+      bytes += mb_len (mbc);
+      mbi_advance (iter);
+    }
+  *ncols = p_cols;
+  return bytes;
+}
 #else
-# define count_cols(mbs) ((int)(strlen(mbs)))
+static int count_cols (const char *mbs) { return (int) strlen(mbs); }
+static int
+cols_to_bytes (const char *mbs _GL_UNUSED, const int cols, int *ncols)
+{
+  *ncols = cols;
+  return cols;
+}
 #endif
 
 static const char *
@@ -821,7 +855,7 @@ get_eta (int *bcd)
 {
   /* TRANSLATORS: "ETA" is English-centric, but this must
      be short, ideally 3 chars.  Abbreviate if necessary.  */
-  static const char eta_str[] = N_("   eta %s");
+  static const char eta_str[] = N_("    eta %s");
   static const char *eta_trans;
   static int bytes_cols_diff;
   if (eta_trans == NULL)
@@ -863,28 +897,17 @@ get_eta (int *bcd)
    are confused when they see strchr (s, '\0') in the code.  */
 #define move_to_end(s) s = strchr (s, '\0');
 
-#ifndef MAX
-# define MAX(a, b) ((a) >= (b) ? (a) : (b))
-#endif
-#ifndef MIN
-# define MIN(a, b) ((a) <= (b) ? (a) : (b))
-#endif
-
 static void
 create_image (struct bar_progress *bp, double dl_total_time, bool done)
 {
-  const int MAX_FILENAME_LEN = bp->width / 4;
+  const int MAX_FILENAME_COLS = bp->width / 4;
   char *p = bp->buffer;
   wgint size = bp->initial_length + bp->count;
 
-  const char *size_grouped = with_thousand_seps (size);
-  int size_grouped_len = count_cols (size_grouped);
-  /* Difference between num cols and num bytes: */
-  int size_grouped_diff = strlen (size_grouped) - size_grouped_len;
-  int size_grouped_pad; /* Used to pad the field width for size_grouped. */
-
   struct bar_progress_hist *hist = &bp->hist;
-  int orig_filename_len = strlen (bp->f_download);
+  int orig_filename_cols = count_cols (bp->f_download);
+
+  int padding;
 
   /* The progress bar should look like this:
      file xx% [=======>             ] nnn.nnK 12.34KB/s  eta 36m 51s
@@ -896,7 +919,7 @@ create_image (struct bar_progress *bp, double dl_total_time, bool done)
      It would be especially bad for the progress bar to be resized
      randomly.
 
-     "file "           - Downloaded filename      - MAX_FILENAME_LEN chars + 1
+     "file "           - Downloaded filename      - MAX_FILENAME_COLS chars + 1
      "xx% " or "100%"  - percentage               - 4 chars
      "[]"              - progress bar decorations - 2 chars
      " nnn.nnK"        - downloaded bytes         - 7 chars + 1
@@ -906,12 +929,16 @@ create_image (struct bar_progress *bp, double dl_total_time, bool done)
      "=====>..."       - progress bar             - the rest
   */
 
-#define PROGRESS_FILENAME_LEN  MAX_FILENAME_LEN + 1
+  /* TODO: Ask the Turkish Translators to fix their translation for the "done"
+   * mode of progress bar. Use one less character. Once that is done, redice
+   * PROGRESS_ETA_LEN by 1.
+   */
+#define PROGRESS_FILENAME_LEN  MAX_FILENAME_COLS + 1
 #define PROGRESS_PERCENT_LEN   4
 #define PROGRESS_DECORAT_LEN   2
 #define PROGRESS_FILESIZE_LEN  7 + 1
-#define PROGRESS_DWNLOAD_RATE  8 + 1
-#define PROGRESS_ETA_LEN       14
+#define PROGRESS_DWNLOAD_RATE  8 + 2
+#define PROGRESS_ETA_LEN       15
 
   int progress_size = bp->width - (PROGRESS_FILENAME_LEN + PROGRESS_PERCENT_LEN +
                                    PROGRESS_DECORAT_LEN + PROGRESS_FILESIZE_LEN +
@@ -920,29 +947,58 @@ create_image (struct bar_progress *bp, double dl_total_time, bool done)
   /* The difference between the number of bytes used,
      and the number of columns used. */
   int bytes_cols_diff = 0;
+  int cols_diff;
+  const char *down_size;
+
+  memset (bp->buffer, '\0', BUF_LEN);
 
   if (progress_size < 5)
     progress_size = 0;
 
-  if (orig_filename_len <= MAX_FILENAME_LEN)
+  if (orig_filename_cols <= MAX_FILENAME_COLS)
     {
-      int padding = MAX_FILENAME_LEN - orig_filename_len;
-      sprintf (p, "%s ", bp->f_download);
-      p += orig_filename_len + 1;
-      for (;padding;padding--)
-        *p++ = ' ';
+      padding = MAX_FILENAME_COLS - orig_filename_cols;
+      p += sprintf (p, "%s ", bp->f_download);
+      memset (p, ' ', padding);
+      p += padding;
     }
   else
     {
-      int offset;
+      int offset_cols;
+      int bytes_in_filename, offset_bytes, col;
+      int *cols_ret = &col;
 
-      if (((orig_filename_len > MAX_FILENAME_LEN) && !opt.noscroll) && !done)
-        offset = ((int) bp->tick) % (orig_filename_len - MAX_FILENAME_LEN);
+#define MIN_SCROLL_TEXT 5
+      if ((orig_filename_cols > MAX_FILENAME_COLS + MIN_SCROLL_TEXT) &&
+          !opt.noscroll &&
+          !done)
+        {
+          offset_cols = ((int) bp->tick + orig_filename_cols + MAX_FILENAME_COLS / 2)
+                        % (orig_filename_cols + MAX_FILENAME_COLS);
+          if (offset_cols > orig_filename_cols)
+            {
+              padding = MAX_FILENAME_COLS - (offset_cols - orig_filename_cols);
+              memset(p, ' ', padding);
+              p += padding;
+              offset_cols = 0;
+            }
+          else
+            padding = 0;
+        }
       else
-        offset = 0;
-      memcpy (p, bp->f_download + offset, MAX_FILENAME_LEN);
-      p += MAX_FILENAME_LEN;
-      *p++ = ' ';
+        {
+          padding = 0;
+          offset_cols = 0;
+        }
+      offset_bytes = cols_to_bytes (bp->f_download, offset_cols, cols_ret);
+      bytes_in_filename = cols_to_bytes (bp->f_download + offset_bytes,
+                                         MAX_FILENAME_COLS - padding,
+                                         cols_ret);
+      memcpy (p, bp->f_download + offset_bytes, bytes_in_filename);
+      p += bytes_in_filename;
+      padding = MAX_FILENAME_COLS - (padding + *cols_ret);
+      memset (p, ' ', padding + 1);
+      p += padding + 1;
     }
 
   /* "xx% " */
@@ -950,15 +1006,13 @@ create_image (struct bar_progress *bp, double dl_total_time, bool done)
     {
       int percentage = 100.0 * size / bp->total_length;
       assert (percentage <= 100);
-
-      if (percentage < 100)
-        sprintf (p, "%3d%%", percentage);
-      else
-        strcpy (p, "100%");
-      p += 4;
+      p += sprintf (p, "%3d%%", percentage);
     }
   else
-    APPEND_LITERAL ("    ");
+    {
+      memset (p, ' ', PROGRESS_PERCENT_LEN);
+      p += PROGRESS_PERCENT_LEN;
+    }
 
   /* The progress bar: "[====>      ]" or "[++==>      ]". */
   if (progress_size && bp->total_length > 0)
@@ -970,7 +1024,6 @@ create_image (struct bar_progress *bp, double dl_total_time, bool done)
       int dlsz = (double)size / bp->total_length * progress_size;
 
       char *begin;
-      int i;
 
       assert (dlsz <= progress_size);
       assert (insz <= dlsz);
@@ -980,18 +1033,19 @@ create_image (struct bar_progress *bp, double dl_total_time, bool done)
 
       /* Print the initial portion of the download with '+' chars, the
          rest with '=' and one '>'.  */
-      for (i = 0; i < insz; i++)
-        *p++ = '+';
+      memset (p, '+', insz);
+      p += insz;
+
       dlsz -= insz;
       if (dlsz > 0)
         {
-          for (i = 0; i < dlsz - 1; i++)
-            *p++ = '=';
+          memset (p, '=', dlsz-1);
+          p += dlsz - 1;
           *p++ = '>';
         }
 
-      while (p - begin < progress_size)
-        *p++ = ' ';
+      memset (p, ' ', (progress_size - (p - begin)));
+      p += (progress_size - (p - begin));
       *p++ = ']';
     }
   else if (progress_size)
@@ -1019,27 +1073,14 @@ create_image (struct bar_progress *bp, double dl_total_time, bool done)
       *p++ = ']';
 
     }
- ++bp->tick;
+  ++bp->tick;
 
   /* " 234.56M" */
-  const char * down_size = human_readable (size, 1000, 2);
-  int cols_diff = 7 - count_cols (down_size);
-  while (cols_diff > 0)
-  {
-    *p++=' ';
-    cols_diff--;
-  }
-  sprintf (p, " %s", down_size);
-  move_to_end (p);
-  /* Pad with spaces to 7 chars for the size_grouped field;
-   * couldn't use the field width specifier in sprintf, because
-   * it counts in bytes, not characters. */
-  for (size_grouped_pad = PROGRESS_FILESIZE_LEN - 7;
-       size_grouped_pad > 0;
-       --size_grouped_pad)
-    {
-      *p++ = ' ';
-    }
+  down_size = human_readable (size, 1000, 2);
+  cols_diff = PROGRESS_FILESIZE_LEN - count_cols (down_size);
+  memset (p, ' ', cols_diff);
+  p += cols_diff;
+  p += sprintf (p, "%s", down_size);
 
   /* " 12.52Kb/s or 12.52KB/s" */
   if (hist->total_time > 0 && hist->total_bytes)
@@ -1052,12 +1093,11 @@ create_image (struct bar_progress *bp, double dl_total_time, bool done)
       wgint dlquant = hist->total_bytes + bp->recent_bytes;
       double dltime = hist->total_time + (dl_total_time - bp->recent_start);
       double dlspeed = calc_rate (dlquant, dltime, &units);
-      sprintf (p, " %4.*f%s", dlspeed >= 99.95 ? 0 : dlspeed >= 9.995 ? 1 : 2,
+      p += sprintf (p, "  %4.*f%s", dlspeed >= 99.95 ? 0 : dlspeed >= 9.995 ? 1 : 2,
                dlspeed,  !opt.report_bps ? short_units[units] : short_units_bits[units]);
-      move_to_end (p);
     }
   else
-    APPEND_LITERAL (" --.-KB/s");
+    APPEND_LITERAL ("  --.-KB/s");
 
   if (!done)
     {
@@ -1092,14 +1132,14 @@ create_image (struct bar_progress *bp, double dl_total_time, bool done)
               bp->last_eta_time = dl_total_time;
             }
 
-          sprintf (p, get_eta(&bytes_cols_diff),
+          p += sprintf (p, get_eta(&bytes_cols_diff),
                    eta_to_human_short (eta, false));
-          move_to_end (p);
         }
       else if (bp->total_length > 0)
         {
         skip_eta:
-          APPEND_LITERAL ("             ");
+          memset (p, ' ', PROGRESS_ETA_LEN);
+          p += PROGRESS_ETA_LEN;
         }
     }
   else
@@ -1109,22 +1149,33 @@ create_image (struct bar_progress *bp, double dl_total_time, bool done)
       int ncols;
 
       /* Note to translators: this should not take up more room than
-         available here.  Abbreviate if necessary.  */
-      strcpy (p, _("   in "));
+         available here (6 columns).  Abbreviate if necessary.  */
+      strcpy (p, _("    in "));
       nbytes = strlen (p);
       ncols  = count_cols (p);
       bytes_cols_diff = nbytes - ncols;
-      p += nbytes;
       if (dl_total_time >= 10)
-        strcpy (p, eta_to_human_short ((int) (dl_total_time + 0.5), false));
+        ncols += sprintf (p + nbytes, "%s",  eta_to_human_short ((int) (dl_total_time + 0.5), false));
       else
-        sprintf (p, "%ss", print_decimal (dl_total_time));
-      move_to_end (p);
+        ncols += sprintf (p + nbytes, "%ss", print_decimal (dl_total_time));
+      p += ncols + bytes_cols_diff;
+      memset (p, ' ', PROGRESS_ETA_LEN - ncols);
+      p += PROGRESS_ETA_LEN - ncols;
     }
 
-  while (p - bp->buffer - bytes_cols_diff - size_grouped_diff < bp->width)
-    *p++ = ' ';
+  padding = bp->width - count_cols (bp->buffer);
+  memset (p, ' ', padding);
+  p += padding;
   *p = '\0';
+
+  /* 2014-11-14  Darshit Shah  <darnir@gmail.com>
+   * Assert that the length of the progress bar is lesser than the size of the
+   * screen with which we are dealing. This assertion *MUST* always be removed
+   * from the release code since we do not want Wget to crash and burn when the
+   * assertion fails. Instead Wget should continue downloading and display a
+   * horrible and irritating progress bar that spams the screen with newlines.
+   */
+  assert (count_cols (bp->buffer) == bp->width);
 }
 
 /* Print the contents of the buffer as a one-line ASCII "image" so
@@ -1156,7 +1207,7 @@ bar_set_params (char *params)
         } while ((param = strtok (NULL, ":")) != NULL);
     }
 
-  if ((opt.lfilename
+  if (((opt.lfilename && opt.show_progress != 1)
 #ifdef HAVE_ISATTY
        /* The progress bar doesn't make sense if the output is not a
           TTY -- when logging to file, it is better to review the

src/progress.h

@@ -1,6 +1,6 @@
 /* Download progress.
    Copyright (C) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009,
-   2010, 2011 Free Software Foundation, Inc.
+   2010, 2011, 2015 Free Software Foundation, Inc.
 
 This file is part of GNU Wget.
 

src/ptimer.c

@@ -1,6 +1,6 @@
 /* Portable timers.
-   Copyright (C) 2005, 2006, 2007, 2008, 2009, 2010, 2011 Free Software
-   Foundation, Inc.
+   Copyright (C) 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2015 Free
+   Software Foundation, Inc.
 
 This file is part of GNU Wget.
 
@@ -291,7 +291,7 @@ windows_resolution (void)
     return 10;                  /* according to MSDN */
 }
 #endif  /* PTIMER_WINDOWS */
-
+
 /* The code below this point is independent of timer implementation. */
 
 struct ptimer {

src/ptimer.h

@@ -1,6 +1,6 @@
 /* Declarations for ptimer.c.
-   Copyright (C) 2005, 2006, 2007, 2008, 2009, 2010, 2011 Free Software
-   Foundation, Inc.
+   Copyright (C) 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2015 Free
+   Software Foundation, Inc.
 
 This file is part of GNU Wget.
 

src/recur.c

@@ -1,7 +1,7 @@
 /* Handling of recursive HTTP retrieving.
    Copyright (C) 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004,
-   2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012 Free Software Foundation,
-   Inc.
+   2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012, 2015 Free Software
+   Foundation, Inc.
 
 This file is part of GNU Wget.
 
@@ -50,7 +50,8 @@ as that of the covered work.  */
 #include "html-url.h"
 #include "css-url.h"
 #include "spider.h"
-
+#include "exits.h"
+
 /* Functions for maintaining the URL queue.  */
 
 struct queue_element {
@@ -159,12 +160,42 @@ url_dequeue (struct url_queue *queue, struct iri **i,
   xfree (qel);
   return true;
 }
-
-static bool download_child_p (const struct urlpos *, struct url *, int,
-                              struct url *, struct hash_table *, struct iri *);
-static bool descend_redirect_p (const char *, struct url *, int,
-                                struct url *, struct hash_table *, struct iri *);
 
+static void blacklist_add (struct hash_table *blacklist, const char *url)
+{
+  char *url_unescaped = xstrdup (url);
+
+  url_unescape (url_unescaped);
+  string_set_add (blacklist, url_unescaped);
+  xfree (url_unescaped);
+}
+
+static int blacklist_contains (struct hash_table *blacklist, const char *url)
+{
+  char *url_unescaped = xstrdup(url);
+  int ret;
+
+  url_unescape (url_unescaped);
+  ret = string_set_contains (blacklist, url_unescaped);
+  xfree (url_unescaped);
+
+  return ret;
+}
+
+typedef enum
+{
+  WG_RR_SUCCESS, WG_RR_BLACKLIST, WG_RR_NOTHTTPS, WG_RR_NONHTTP, WG_RR_ABSOLUTE,
+  WG_RR_DOMAIN, WG_RR_PARENT, WG_RR_LIST, WG_RR_REGEX, WG_RR_RULES,
+  WG_RR_SPANNEDHOST, WG_RR_ROBOTS
+} reject_reason;
+
+static reject_reason download_child (const struct urlpos *, struct url *, int,
+                              struct url *, struct hash_table *, struct iri *);
+static reject_reason descend_redirect (const char *, struct url *, int,
+                              struct url *, struct hash_table *, struct iri *);
+static void write_reject_log_header (FILE *);
+static void write_reject_log_reason (FILE *, reject_reason,
+                              const struct url *, const struct url *);
 
 /* Retrieve a part of the web beginning with START_URL.  This used to
    be called "recursive retrieval", because the old function was
@@ -201,6 +232,8 @@ retrieve_tree (struct url *start_url_parsed, struct iri *pi)
 
   struct iri *i = iri_new ();
 
+  FILE *rejectedlog = NULL; /* Don't write a rejected log. */
+
 #define COPYSTR(x)  (x) ? xstrdup(x) : NULL;
   /* Duplicate pi struct if not NULL */
   if (pi)
@@ -220,7 +253,15 @@ retrieve_tree (struct url *start_url_parsed, struct iri *pi)
      just URL so we enqueue the canonical form of the URL.  */
   url_enqueue (queue, i, xstrdup (start_url_parsed->url), NULL, 0, true,
                false);
-  string_set_add (blacklist, start_url_parsed->url);
+  blacklist_add (blacklist, start_url_parsed->url);
+
+  if (opt.rejected_log)
+    {
+      rejectedlog = fopen (opt.rejected_log, "w");
+      write_reject_log_header (rejectedlog);
+      if (!rejectedlog)
+        logprintf (LOG_NOTQUIET, "%s: %s\n", opt.rejected_log, strerror (errno));
+    }
 
   while (1)
     {
@@ -244,9 +285,9 @@ retrieve_tree (struct url *start_url_parsed, struct iri *pi)
         break;
 
       /* ...and download it.  Note that this download is in most cases
-         unconditional, as download_child_p already makes sure a file
+         unconditional, as download_child already makes sure a file
          doesn't get enqueued twice -- and yet this check is here, and
-         not in download_child_p.  This is so that if you run `wget -r
+         not in download_child.  This is so that if you run `wget -r
          URL1 URL2', and a random URL is encountered once under URL1
          and again under URL2, but at a different (possibly smaller)
          depth, we want the URL's children to be taken into account
@@ -277,52 +318,69 @@ retrieve_tree (struct url *start_url_parsed, struct iri *pi)
           char *redirected = NULL;
           struct url *url_parsed = url_parse (url, &url_err, i, true);
 
-          status = retrieve_url (url_parsed, url, &file, &redirected, referer,
-                                 &dt, false, i, true);
-
-          if (html_allowed && file && status == RETROK
-              && (dt & RETROKF) && (dt & TEXTHTML))
+          if (!url_parsed)
             {
-              descend = true;
-              is_css = false;
-            }
-
-          /* a little different, css_allowed can override content type
-             lots of web servers serve css with an incorrect content type
-          */
-          if (file && status == RETROK
-              && (dt & RETROKF) &&
-              ((dt & TEXTCSS) || css_allowed))
-            {
-              descend = true;
-              is_css = true;
-            }
-
-          if (redirected)
-            {
-              /* We have been redirected, possibly to another host, or
-                 different path, or wherever.  Check whether we really
-                 want to follow it.  */
-              if (descend)
-                {
-                  if (!descend_redirect_p (redirected, url_parsed, depth,
-                                           start_url_parsed, blacklist, i))
-                    descend = false;
-                  else
-                    /* Make sure that the old pre-redirect form gets
-                       blacklisted. */
-                    string_set_add (blacklist, url);
-                }
-
-              xfree (url);
-              url = redirected;
+              char *error = url_error (url, url_err);
+              logprintf (LOG_NOTQUIET, "%s: %s.\n",url, error);
+              xfree (error);
+              inform_exit_status (URLERROR);
             }
           else
             {
-              xfree (url);
-              url = xstrdup (url_parsed->url);
+
+              status = retrieve_url (url_parsed, url, &file, &redirected, referer,
+                                     &dt, false, i, true);
+
+              if (html_allowed && file && status == RETROK
+                  && (dt & RETROKF) && (dt & TEXTHTML))
+                {
+                  descend = true;
+                  is_css = false;
+                }
+
+              /* a little different, css_allowed can override content type
+                 lots of web servers serve css with an incorrect content type
+              */
+              if (file && status == RETROK
+                  && (dt & RETROKF) &&
+                  ((dt & TEXTCSS) || css_allowed))
+                {
+                  descend = true;
+                  is_css = true;
+                }
+
+              if (redirected)
+                {
+                  /* We have been redirected, possibly to another host, or
+                     different path, or wherever.  Check whether we really
+                     want to follow it.  */
+                  if (descend)
+                    {
+                      reject_reason r = descend_redirect (redirected, url_parsed,
+                                        depth, start_url_parsed, blacklist, i);
+                      if (r == WG_RR_SUCCESS)
+                        {
+                          /* Make sure that the old pre-redirect form gets
+                             blacklisted. */
+                          blacklist_add (blacklist, url);
+                        }
+                      else
+                        {
+                          write_reject_log_reason (rejectedlog, r, url_parsed, start_url_parsed);
+                          descend = false;
+                        }
+                    }
+
+                  xfree (url);
+                  url = redirected;
+                }
+              else
+                {
+                  xfree (url);
+                  url = xstrdup (url_parsed->url);
+                }
+              url_free (url_parsed);
             }
-          url_free(url_parsed);
         }
 
       if (opt.spider)
@@ -388,12 +446,16 @@ retrieve_tree (struct url *start_url_parsed, struct iri *pi)
 
               for (; child; child = child->next)
                 {
+                  reject_reason r;
+
                   if (child->ignore_when_downloading)
                     continue;
                   if (dash_p_leaf_HTML && !child->link_inline_p)
                     continue;
-                  if (download_child_p (child, url_parsed, depth, start_url_parsed,
-                                        blacklist, i))
+
+                  r = download_child (child, url_parsed, depth,
+                                      start_url_parsed, blacklist, i);
+                  if (r == WG_RR_SUCCESS)
                     {
                       ci = iri_new ();
                       set_uri_encoding (ci, i->content_encoding, false);
@@ -404,7 +466,11 @@ retrieve_tree (struct url *start_url_parsed, struct iri *pi)
                       /* We blacklist the URL we have enqueued, because we
                          don't want to enqueue (and hence download) the
                          same URL twice.  */
-                      string_set_add (blacklist, child->url->url);
+                      blacklist_add (blacklist, child->url->url);
+                    }
+                  else
+                    {
+                      write_reject_log_reason (rejectedlog, r, child->url, url_parsed);
                     }
                 }
 
@@ -440,11 +506,14 @@ retrieve_tree (struct url *start_url_parsed, struct iri *pi)
         }
 
       xfree (url);
-      xfree_null (referer);
-      xfree_null (file);
+      xfree (referer);
+      xfree (file);
       iri_free (i);
     }
 
+  if (rejectedlog)
+    fclose (rejectedlog);
+
   /* If anything is left of the queue due to a premature exit, free it
      now.  */
   {
@@ -457,7 +526,7 @@ retrieve_tree (struct url *start_url_parsed, struct iri *pi)
       {
         iri_free (d6);
         xfree (d1);
-        xfree_null (d2);
+        xfree (d2);
       }
   }
   url_queue_delete (queue);
@@ -480,27 +549,29 @@ retrieve_tree (struct url *start_url_parsed, struct iri *pi)
    by storing these URLs to BLACKLIST.  This may or may not help.  It
    will help if those URLs are encountered many times.  */
 
-static bool
-download_child_p (const struct urlpos *upos, struct url *parent, int depth,
+static reject_reason
+download_child (const struct urlpos *upos, struct url *parent, int depth,
                   struct url *start_url_parsed, struct hash_table *blacklist,
                   struct iri *iri)
 {
   struct url *u = upos->url;
   const char *url = u->url;
   bool u_scheme_like_http;
+  reject_reason reason = WG_RR_SUCCESS;
 
   DEBUGP (("Deciding whether to enqueue \"%s\".\n", url));
 
-  if (string_set_contains (blacklist, url))
+  if (blacklist_contains (blacklist, url))
     {
       if (opt.spider)
         {
           char *referrer = url_string (parent, URL_AUTH_HIDE_PASSWD);
-          DEBUGP (("download_child_p: parent->url is: %s\n", quote (parent->url)));
+          DEBUGP (("download_child: parent->url is: %s\n", quote (parent->url)));
           visited_url (url, referrer);
           xfree (referrer);
         }
       DEBUGP (("Already on the black list.\n"));
+      reason = WG_RR_BLACKLIST;
       goto out;
     }
 
@@ -530,6 +601,7 @@ download_child_p (const struct urlpos *upos, struct url *parent, int depth,
   if (opt.https_only && u->scheme != SCHEME_HTTPS)
     {
       DEBUGP (("Not following non-HTTPS links.\n"));
+      reason = WG_RR_NOTHTTPS;
       goto out;
     }
 #endif
@@ -538,9 +610,14 @@ download_child_p (const struct urlpos *upos, struct url *parent, int depth,
   u_scheme_like_http = schemes_are_similar_p (u->scheme, SCHEME_HTTP);
 
   /* 1. Schemes other than HTTP are normally not recursed into. */
-  if (!u_scheme_like_http && !(u->scheme == SCHEME_FTP && opt.follow_ftp))
+  if (!u_scheme_like_http && !((u->scheme == SCHEME_FTP
+#ifdef HAVE_SSL
+      || u->scheme == SCHEME_FTPS
+#endif
+      ) && opt.follow_ftp))
     {
       DEBUGP (("Not following non-HTTP schemes.\n"));
+      reason = WG_RR_NONHTTP;
       goto out;
     }
 
@@ -550,6 +627,7 @@ download_child_p (const struct urlpos *upos, struct url *parent, int depth,
     if (opt.relative_only && !upos->link_relative_p)
       {
         DEBUGP (("It doesn't really look like a relative link.\n"));
+        reason = WG_RR_ABSOLUTE;
         goto out;
       }
 
@@ -558,6 +636,7 @@ download_child_p (const struct urlpos *upos, struct url *parent, int depth,
   if (!accept_domain (u))
     {
       DEBUGP (("The domain was not accepted.\n"));
+      reason = WG_RR_DOMAIN;
       goto out;
     }
 
@@ -577,6 +656,7 @@ download_child_p (const struct urlpos *upos, struct url *parent, int depth,
         {
           DEBUGP (("Going to \"%s\" would escape \"%s\" with no_parent on.\n",
                    u->dir, start_url_parsed->dir));
+          reason = WG_RR_PARENT;
           goto out;
         }
     }
@@ -589,12 +669,14 @@ download_child_p (const struct urlpos *upos, struct url *parent, int depth,
       if (!accdir (u->dir))
         {
           DEBUGP (("%s (%s) is excluded/not-included.\n", url, u->dir));
+          reason = WG_RR_LIST;
           goto out;
         }
     }
   if (!accept_url (url))
     {
       DEBUGP (("%s is excluded/not-included through regex.\n", url));
+      reason = WG_RR_REGEX;
       goto out;
     }
 
@@ -619,6 +701,7 @@ download_child_p (const struct urlpos *upos, struct url *parent, int depth,
         {
           DEBUGP (("%s (%s) does not match acc/rej rules.\n",
                    url, u->file));
+          reason = WG_RR_RULES;
           goto out;
         }
     }
@@ -629,6 +712,7 @@ download_child_p (const struct urlpos *upos, struct url *parent, int depth,
       {
         DEBUGP (("This is not the same hostname as the parent's (%s and %s).\n",
                  u->host, parent->host));
+        reason = WG_RR_SPANNEDHOST;
         goto out;
       }
 
@@ -670,36 +754,37 @@ download_child_p (const struct urlpos *upos, struct url *parent, int depth,
       if (!res_match_path (specs, u->path))
         {
           DEBUGP (("Not following %s because robots.txt forbids it.\n", url));
-          string_set_add (blacklist, url);
+          blacklist_add (blacklist, url);
+          reason = WG_RR_ROBOTS;
           goto out;
         }
     }
 
-  /* The URL has passed all the tests.  It can be placed in the
-     download queue. */
-  DEBUGP (("Decided to load it.\n"));
+  out:
 
-  return true;
+  if (reason == WG_RR_SUCCESS)
+    /* The URL has passed all the tests.  It can be placed in the
+       download queue. */
+    DEBUGP (("Decided to load it.\n"));
+  else
+    DEBUGP (("Decided NOT to load it.\n"));
 
- out:
-  DEBUGP (("Decided NOT to load it.\n"));
-
-  return false;
+  return reason;
 }
 
 /* This function determines whether we will consider downloading the
    children of a URL whose download resulted in a redirection,
    possibly to another host, etc.  It is needed very rarely, and thus
-   it is merely a simple-minded wrapper around download_child_p.  */
+   it is merely a simple-minded wrapper around download_child.  */
 
-static bool
-descend_redirect_p (const char *redirected, struct url *orig_parsed, int depth,
+static reject_reason
+descend_redirect (const char *redirected, struct url *orig_parsed, int depth,
                     struct url *start_url_parsed, struct hash_table *blacklist,
                     struct iri *iri)
 {
   struct url *new_parsed;
   struct urlpos *upos;
-  bool success;
+  reject_reason reason;
 
   assert (orig_parsed != NULL);
 
@@ -709,16 +794,104 @@ descend_redirect_p (const char *redirected, struct url *orig_parsed, int depth,
   upos = xnew0 (struct urlpos);
   upos->url = new_parsed;
 
-  success = download_child_p (upos, orig_parsed, depth,
+  reason = download_child (upos, orig_parsed, depth,
                               start_url_parsed, blacklist, iri);
 
+  if (reason == WG_RR_SUCCESS)
+    blacklist_add (blacklist, upos->url->url);
+  else
+    DEBUGP (("Redirection \"%s\" failed the test.\n", redirected));
+
   url_free (new_parsed);
   xfree (upos);
 
-  if (!success)
-    DEBUGP (("Redirection \"%s\" failed the test.\n", redirected));
+  return reason;
+}
 
-  return success;
+
+/* This function writes the rejected log header. */
+static void
+write_reject_log_header (FILE *f)
+{
+  if (!f)
+    return;
+
+  /* Note: Update this header when columns change in any way. */
+  fprintf (f, "REASON\t"
+    "U_URL\tU_SCHEME\tU_HOST\tU_PORT\tU_PATH\tU_PARAMS\tU_QUERY\tU_FRAGMENT\t"
+    "P_URL\tP_SCHEME\tP_HOST\tP_PORT\tP_PATH\tP_PARAMS\tP_QUERY\tP_FRAGMENT\n");
+}
+
+/* This function writes a URL to the reject log. Internal use only. */
+static void
+write_reject_log_url (FILE *fp, const struct url *url)
+{
+  const char *escaped_str;
+  const char *scheme_str;
+
+  if (!fp)
+    return;
+
+  escaped_str = url_escape (url->url);
+
+  switch (url->scheme)
+    {
+      case SCHEME_HTTP:  scheme_str = "SCHEME_HTTP";    break;
+#ifdef HAVE_SSL
+      case SCHEME_HTTPS: scheme_str = "SCHEME_HTTPS";   break;
+      case SCHEME_FTPS:  scheme_str = "SCHEME_FTPS";    break;
+#endif
+      case SCHEME_FTP:   scheme_str = "SCHEME_FTP";     break;
+      default:           scheme_str = "SCHEME_INVALID"; break;
+    }
+
+  fprintf (fp, "%s\t%s\t%s\t%i\t%s\t%s\t%s\t%s",
+    escaped_str,
+    scheme_str,
+    url->host,
+    url->port,
+    url->path,
+    url->params ? url->params : "",
+    url->query ? url->query : "",
+    url->fragment ? url->fragment : "");
+
+  xfree (escaped_str);
+}
+
+/* This function writes out information on why a URL was rejected and its
+   context from download_child such as the URL being rejected and it's
+   parent's URL. The format it uses is comma separated values but with tabs. */
+static void
+write_reject_log_reason (FILE *fp, reject_reason reason,
+                         const struct url *url, const struct url *parent)
+{
+  const char *reason_str;
+
+  if (!fp)
+    return;
+
+  switch (reason)
+    {
+      case WG_RR_SUCCESS:     reason_str = "SUCCESS";     break;
+      case WG_RR_BLACKLIST:   reason_str = "BLACKLIST";   break;
+      case WG_RR_NOTHTTPS:    reason_str = "NOTHTTPS";    break;
+      case WG_RR_NONHTTP:     reason_str = "NONHTTP";     break;
+      case WG_RR_ABSOLUTE:    reason_str = "ABSOLUTE";    break;
+      case WG_RR_DOMAIN:      reason_str = "DOMAIN";      break;
+      case WG_RR_PARENT:      reason_str = "PARENT";      break;
+      case WG_RR_LIST:        reason_str = "LIST";        break;
+      case WG_RR_REGEX:       reason_str = "REGEX";       break;
+      case WG_RR_RULES:       reason_str = "RULES";       break;
+      case WG_RR_SPANNEDHOST: reason_str = "SPANNEDHOST"; break;
+      case WG_RR_ROBOTS:      reason_str = "ROBOTS";      break;
+      default:                reason_str = "UNKNOWN";     break;
+    }
+
+  fprintf (fp, "%s\t", reason_str);
+  write_reject_log_url (fp, url);
+  fprintf (fp, "\t");
+  write_reject_log_url (fp, parent);
+  fprintf (fp, "\n");
 }
 
 /* vim:set sts=2 sw=2 cino+={s: */

src/recur.h

@@ -1,7 +1,7 @@
 /* Declarations for recur.c.
    Copyright (C) 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004,
-   2005, 2006, 2007, 2008, 2009, 2010, 2011 Free Software Foundation,
-   Inc.
+   2005, 2006, 2007, 2008, 2009, 2010, 2011, 2015 Free Software
+   Foundation, Inc.
 
 This file is part of GNU Wget.
 

src/res.c

@@ -1,6 +1,6 @@
 /* Support for Robot Exclusion Standard (RES).
-   Copyright (C) 2001, 2006, 2007, 2008, 2009, 2010, 2011 Free Software
-   Foundation, Inc.
+   Copyright (C) 2001, 2006, 2007, 2008, 2009, 2010, 2011, 2015 Free
+   Software Foundation, Inc.
 
 This file is part of Wget.
 
@@ -81,6 +81,7 @@ as that of the covered work.  */
 #include "url.h"
 #include "retr.h"
 #include "res.h"
+#include "c-strcase.h"
 
 #ifdef TESTING
 #include "test.h"
@@ -97,7 +98,7 @@ struct robot_specs {
   int size;
   struct path_info *paths;
 };
-
+
 /* Parsing the robot spec. */
 
 /* Check whether AGENT (a string of length LENGTH) equals "wget" or
@@ -403,10 +404,10 @@ free_specs (struct robot_specs *specs)
   int i;
   for (i = 0; i < specs->count; i++)
     xfree (specs->paths[i].path);
-  xfree_null (specs->paths);
+  xfree (specs->paths);
   xfree (specs);
 }
-
+
 /* Matching of a path according to the specs. */
 
 /* If C is '%' and (ptr[1], ptr[2]) form a hexadecimal number, and if
@@ -471,7 +472,7 @@ res_match_path (const struct robot_specs *specs, const char *path)
       }
   return true;
 }
-
+
 /* Registering the specs. */
 
 static struct hash_table *registered_specs;
@@ -521,7 +522,7 @@ res_get_specs (const char *host, int port)
     return NULL;
   return hash_table_get (registered_specs, hp);
 }
-
+
 /* Loading the robots file.  */
 
 #define RES_SPECS_LOCATION "/robots.txt"
@@ -578,11 +579,10 @@ res_retrieve_file (const char *url, char **file, struct iri *iri)
          allocated the file name, deallocate is here so that the
          caller doesn't have to worry about it.  */
       xfree (*file);
-      *file = NULL;
     }
   return err == RETROK;
 }
-
+
 bool
 is_robots_txt_url (const char *url)
 {
@@ -593,7 +593,7 @@ is_robots_txt_url (const char *url)
 
   return ret;
 }
-
+
 void
 res_cleanup (void)
 {
@@ -611,7 +611,7 @@ res_cleanup (void)
       registered_specs = NULL;
     }
 }
-
+
 #ifdef TESTING
 
 const char *

src/res.h

@@ -1,5 +1,5 @@
 /* Declarations for res.c.
-   Copyright (C) 2001, 2007, 2008, 2009, 2010, 2011 Free Software
+   Copyright (C) 2001, 2007, 2008, 2009, 2010, 2011, 2015 Free Software
    Foundation, Inc.
 
 This file is part of Wget.

src/retr.c

@@ -1,7 +1,7 @@
 /* File retrieval.
    Copyright (C) 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004,
-   2005, 2006, 2007, 2008, 2009, 2010, 2011, 2014 Free Software Foundation,
-   Inc.
+   2005, 2006, 2007, 2008, 2009, 2010, 2011, 2014, 2015 Free Software
+   Foundation, Inc.
 
 This file is part of GNU Wget.
 
@@ -56,6 +56,7 @@ as that of the covered work.  */
 #include "ptimer.h"
 #include "html-url.h"
 #include "iri.h"
+#include "hsts.h"
 
 /* Total size of downloaded files.  Used to enforce quota.  */
 SUM_SIZE_INT total_downloaded_bytes;
@@ -70,7 +71,7 @@ FILE *output_stream;
 /* Whether output_document is a regular file we can manipulate,
    i.e. not `-' or a device file. */
 bool output_stream_regular;
-
+
 static struct {
   wgint chunk_bytes;
   double chunk_start;
@@ -136,10 +137,6 @@ limit_bandwidth (wgint bytes, struct ptimer *timer)
   limit_data.chunk_start = ptimer_read (timer);
 }
 
-#ifndef MIN
-# define MIN(i, j) ((i) <= (j) ? (i) : (j))
-#endif
-
 /* Write data in BUF to OUT.  However, if *SKIP is non-zero, skip that
    amount of data and decrease SKIP.  Increment *TOTAL by the amount
    of data written.  If OUT2 is not NULL, also write BUF to OUT2.
@@ -265,11 +262,16 @@ fd_read_body (const char *downloaded_filename, int fd, FILE *out, wgint toread,
 
   if (opt.show_progress)
     {
+      const char *filename_progress;
       /* If we're skipping STARTPOS bytes, pass 0 as the INITIAL
          argument to progress_create because the indicator doesn't
          (yet) know about "skipping" data.  */
       wgint start = skip ? 0 : startpos;
-      progress = progress_create (downloaded_filename, start, start + toread);
+      if (opt.dir_prefix)
+        filename_progress = downloaded_filename + strlen (opt.dir_prefix) + 1;
+      else
+        filename_progress = downloaded_filename;
+      progress = progress_create (filename_progress, start, start + toread);
       progress_interactive = progress_interactive_p (progress);
     }
 
@@ -378,8 +380,10 @@ fd_read_body (const char *downloaded_filename, int fd, FILE *out, wgint toread,
 
       if (ret > 0)
         {
+          int write_res;
+
           sum_read += ret;
-          int write_res = write_data (out, out2, dlbuf, ret, &skip, &sum_written);
+          write_res = write_data (out, out2, dlbuf, ret, &skip, &sum_written);
           if (write_res < 0)
             {
               ret = (write_res == -3) ? -3 : -2;
@@ -434,11 +438,11 @@ fd_read_body (const char *downloaded_filename, int fd, FILE *out, wgint toread,
   if (qtywritten)
     *qtywritten += sum_written;
 
-  free (dlbuf);
+  xfree (dlbuf);
 
   return ret;
 }
-
+
 /* Read a hunk of data from FD, up until a terminator.  The hunk is
    limited by whatever the TERMINATOR callback chooses as its
    terminator.  For example, if terminator stops at newline, the hunk
@@ -543,7 +547,7 @@ fd_read_hunk (int fd, hunk_terminator_t terminator, long sizehint, long maxsize)
       rdlen = fd_read (fd, hunk + tail, remain, 0);
       if (rdlen < 0)
         {
-          xfree_null (hunk);
+          xfree (hunk);
           return NULL;
         }
       tail += rdlen;
@@ -615,7 +619,7 @@ fd_read_line (int fd)
 {
   return fd_read_hunk (fd, line_terminator, 128, FD_READ_LINE_MAX);
 }
-
+
 /* Return a printed representation of the download rate, along with
    the units appropriate for the download speed.  */
 
@@ -679,7 +683,7 @@ calc_rate (wgint bytes, double secs, int *units)
 
   return dlrate;
 }
-
+
 
 #define SUSPEND_METHOD do {                     \
   method_suspended = true;                      \
@@ -722,7 +726,7 @@ retrieve_url (struct url * orig_parsed, const char *origurl, char **file,
   char *mynewloc, *proxy;
   struct url *u = orig_parsed, *proxy_url;
   int up_error_code;            /* url parse error code */
-  char *local_file;
+  char *local_file = NULL;
   int redirection_count = 0;
 
   bool method_suspended = false;
@@ -750,7 +754,7 @@ retrieve_url (struct url * orig_parsed, const char *origurl, char **file,
 
   result = NOCONERROR;
   mynewloc = NULL;
-  local_file = NULL;
+  xfree(local_file);
   proxy_url = NULL;
 
   proxy = getproxy (u);
@@ -761,7 +765,7 @@ retrieve_url (struct url * orig_parsed, const char *origurl, char **file,
       pi->utf8_encode = false;
 
       /* Parse the proxy URL.  */
-      proxy_url = url_parse (proxy, &up_error_code, NULL, true);
+      proxy_url = url_parse (proxy, &up_error_code, pi, true);
       if (!proxy_url)
         {
           char *error = url_error (proxy, up_error_code);
@@ -769,6 +773,8 @@ retrieve_url (struct url * orig_parsed, const char *origurl, char **file,
                      proxy, error);
           xfree (url);
           xfree (error);
+          xfree (proxy);
+          iri_free (pi);
           RESTORE_METHOD;
           result = PROXERR;
           goto bail;
@@ -778,11 +784,14 @@ retrieve_url (struct url * orig_parsed, const char *origurl, char **file,
           logprintf (LOG_NOTQUIET, _("Error in proxy URL %s: Must be HTTP.\n"), proxy);
           url_free (proxy_url);
           xfree (url);
+          xfree (proxy);
+          iri_free (pi);
           RESTORE_METHOD;
           result = PROXERR;
           goto bail;
         }
-      free (proxy);
+      iri_free(pi);
+      xfree (proxy);
     }
 
   if (u->scheme == SCHEME_HTTP
@@ -791,10 +800,28 @@ retrieve_url (struct url * orig_parsed, const char *origurl, char **file,
 #endif
       || (proxy_url && proxy_url->scheme == SCHEME_HTTP))
     {
+#ifdef HAVE_HSTS
+#ifdef TESTING
+      /* we don't link against main.o when we're testing */
+      hsts_store_t hsts_store = NULL;
+#else
+      extern hsts_store_t hsts_store;
+#endif
+
+      if (opt.hsts && hsts_store)
+	{
+	  if (hsts_match (hsts_store, u))
+	    logprintf (LOG_VERBOSE, "URL transformed to HTTPS due to an HSTS policy\n");
+	}
+#endif
       result = http_loop (u, orig_parsed, &mynewloc, &local_file, refurl, dt,
                           proxy_url, iri);
     }
-  else if (u->scheme == SCHEME_FTP)
+  else if (u->scheme == SCHEME_FTP
+#ifdef HAVE_SSL
+      || u->scheme == SCHEME_FTPS
+#endif
+      )
     {
       /* If this is a redirection, temporarily turn off opt.ftp_glob
          and opt.recursive, both being undesirable when following
@@ -810,7 +837,11 @@ retrieve_url (struct url * orig_parsed, const char *origurl, char **file,
          FTP.  In these cases we must decide whether the text is HTML
          according to the suffix.  The HTML suffixes are `.html',
          `.htm' and a few others, case-insensitive.  */
-      if (redirection_count && local_file && u->scheme == SCHEME_FTP)
+      if (redirection_count && local_file && (u->scheme == SCHEME_FTP
+#ifdef HAVE_SSL
+          || u->scheme == SCHEME_FTPS
+#endif
+          ))
         {
           if (has_html_suffix_p (local_file))
             *dt |= TEXTHTML;
@@ -831,8 +862,7 @@ retrieve_url (struct url * orig_parsed, const char *origurl, char **file,
 
       assert (mynewloc != NULL);
 
-      if (local_file)
-        xfree (local_file);
+      xfree (local_file);
 
       /* The HTTP specs only allow absolute URLs to appear in
          redirects, but a ton of boneheaded webservers and CGIs out
@@ -846,8 +876,7 @@ retrieve_url (struct url * orig_parsed, const char *origurl, char **file,
          the content encoding. */
       iri->utf8_encode = opt.enable_iri;
       set_content_encoding (iri, NULL);
-      xfree_null (iri->orig_url);
-      iri->orig_url = NULL;
+      xfree (iri->orig_url);
 
       /* Now, see if this new location makes sense. */
       newloc_parsed = url_parse (mynewloc, &up_error_code, iri, true);
@@ -913,6 +942,10 @@ retrieve_url (struct url * orig_parsed, const char *origurl, char **file,
 
       goto redirected;
     }
+  else
+    {
+      xfree(mynewloc);
+    }
 
   /* Try to not encode in UTF-8 if fetching failed */
   if (!(*dt & RETROKF) && iri->utf8_encode)
@@ -926,6 +959,7 @@ retrieve_url (struct url * orig_parsed, const char *origurl, char **file,
       if (u)
         {
           DEBUGP (("[IRI fallbacking to non-utf8 for %s\n", quote (url)));
+          xfree (url);
           url = xstrdup (u->url);
           iri_fallbacked = 1;
           goto redirected;
@@ -934,7 +968,7 @@ retrieve_url (struct url * orig_parsed, const char *origurl, char **file,
           DEBUGP (("[Couldn't fallback to non-utf8 for %s\n", quote (url)));
     }
 
-  if (local_file && u && *dt & RETROKF)
+  if (local_file && u && (*dt & RETROKF || opt.content_on_error))
     {
       register_download (u->url, local_file);
 
@@ -951,7 +985,7 @@ retrieve_url (struct url * orig_parsed, const char *origurl, char **file,
   if (file)
     *file = local_file ? local_file : NULL;
   else
-    xfree_null (local_file);
+    xfree (local_file);
 
   if (orig_parsed != u)
     {
@@ -977,6 +1011,7 @@ retrieve_url (struct url * orig_parsed, const char *origurl, char **file,
 bail:
   if (register_status)
     inform_exit_status (result);
+
   return result;
 }
 
@@ -1012,6 +1047,7 @@ retrieve_from_file (const char *file, bool html, int *count)
           char *error = url_error (url, url_err);
           logprintf (LOG_NOTQUIET, "%s: %s.\n", url, error);
           xfree (error);
+          iri_free (iri);
           return URLERROR;
         }
 
@@ -1035,8 +1071,7 @@ retrieve_from_file (const char *file, bool html, int *count)
 
       /* Reset UTF-8 encode status */
       iri->utf8_encode = opt.enable_iri;
-      xfree_null (iri->orig_url);
-      iri->orig_url = NULL;
+      xfree (iri->orig_url);
 
       input_file = url_file;
     }
@@ -1046,12 +1081,12 @@ retrieve_from_file (const char *file, bool html, int *count)
   url_list = (html ? get_urls_html (input_file, NULL, NULL, iri)
               : get_urls_file (input_file));
 
-  xfree_null (url_file);
+  xfree (url_file);
 
   for (cur_url = url_list; cur_url; cur_url = cur_url->next, ++*count)
     {
-      char *filename = NULL, *new_file = NULL;
-      int dt;
+      char *filename = NULL, *new_file = NULL, *proxy;
+      int dt = 0;
       struct iri *tmpiri = iri_dup (iri);
       struct url *parsed_url = NULL;
 
@@ -1066,14 +1101,22 @@ retrieve_from_file (const char *file, bool html, int *count)
 
       parsed_url = url_parse (cur_url->url->url, NULL, tmpiri, true);
 
-      char *proxy = getproxy (cur_url->url);
+      proxy = getproxy (cur_url->url);
       if ((opt.recursive || opt.page_requisites)
-          && (cur_url->url->scheme != SCHEME_FTP || proxy))
+          && ((cur_url->url->scheme != SCHEME_FTP
+#ifdef HAVE_SSL
+          && cur_url->url->scheme != SCHEME_FTPS
+#endif
+          ) || proxy))
         {
           int old_follow_ftp = opt.follow_ftp;
 
           /* Turn opt.follow_ftp on in case of recursive FTP retrieval */
-          if (cur_url->url->scheme == SCHEME_FTP)
+          if (cur_url->url->scheme == SCHEME_FTP
+#ifdef HAVE_SSL
+              || cur_url->url->scheme == SCHEME_FTPS
+#endif
+              )
             opt.follow_ftp = 1;
 
           status = retrieve_tree (parsed_url ? parsed_url : cur_url->url,
@@ -1086,7 +1129,7 @@ retrieve_from_file (const char *file, bool html, int *count)
                                cur_url->url->url, &filename,
                                &new_file, NULL, &dt, opt.recursive, tmpiri,
                                true);
-      free(proxy);
+      xfree (proxy);
 
       if (parsed_url)
           url_free (parsed_url);
@@ -1101,8 +1144,8 @@ Removing file due to --delete-after in retrieve_from_file():\n"));
           dt &= ~RETROKF;
         }
 
-      xfree_null (new_file);
-      xfree_null (filename);
+      xfree (new_file);
+      xfree (filename);
       iri_free (tmpiri);
     }
 
@@ -1178,7 +1221,7 @@ free_urlpos (struct urlpos *l)
       struct urlpos *next = l->next;
       if (l->url)
         url_free (l->url);
-      xfree_null (l->local_name);
+      xfree (l->local_name);
       xfree (l);
       l = next;
     }
@@ -1254,6 +1297,9 @@ getproxy (struct url *u)
     case SCHEME_HTTPS:
       proxy = opt.https_proxy ? opt.https_proxy : getenv ("https_proxy");
       break;
+    case SCHEME_FTPS:
+      proxy = opt.ftp_proxy ? opt.ftp_proxy : getenv ("ftps_proxy");
+      break;
 #endif
     case SCHEME_FTP:
       proxy = opt.ftp_proxy ? opt.ftp_proxy : getenv ("ftp_proxy");
@@ -1279,11 +1325,13 @@ bool
 url_uses_proxy (struct url * u)
 {
   bool ret;
+  char *proxy;
+
   if (!u)
     return false;
-  char *proxy = getproxy (u);
+  proxy = getproxy (u);
   ret = proxy != NULL;
-  free(proxy);
+  xfree (proxy);
   return ret;
 }
 

src/retr.h

@@ -1,7 +1,7 @@
 /* Declarations for retr.c.
    Copyright (C) 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004,
-   2005, 2006, 2007, 2008, 2009, 2010, 2011 Free Software Foundation,
-   Inc.
+   2005, 2006, 2007, 2008, 2009, 2010, 2011, 2015 Free Software
+   Foundation, Inc.
 
 This file is part of GNU Wget.
 
@@ -34,6 +34,8 @@ as that of the covered work.  */
 
 #include "url.h"
 
+extern int numurls;
+
 /* These global vars should be made static to retr.c and exported via
    functions! */
 extern SUM_SIZE_INT total_downloaded_bytes;

src/spider.c

@@ -1,5 +1,5 @@
 /* Keep track of visited URLs in spider mode.
-   Copyright (C) 2006, 2007, 2008, 2009, 2010, 2011 Free Software
+   Copyright (C) 2006, 2007, 2008, 2009, 2010, 2011, 2015 Free Software
    Foundation, Inc.
 
 This file is part of GNU Wget.
@@ -51,7 +51,7 @@ spider_cleanup (void)
   if (nonexisting_urls_set)
     string_set_free (nonexisting_urls_set);
 }
-
+
 /* Remembers broken links.  */
 void
 nonexisting_url (const char *url)

src/spider.h

@@ -1,5 +1,5 @@
 /* Declarations for spider.c
-   Copyright (C) 2006, 2007, 2008, 2009, 2010, 2011 Free Software
+   Copyright (C) 2006, 2007, 2008, 2009, 2010, 2011, 2015 Free Software
    Foundation, Inc.
 
 This file is part of GNU Wget.

src/ssl.h

@@ -1,6 +1,6 @@
 /* SSL support.
    Copyright (C) 2000, 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008,
-   2009, 2010, 2011, 2012 Free Software Foundation, Inc.
+   2009, 2010, 2011, 2012, 2015 Free Software Foundation, Inc.
    Originally contributed by Christian Fraenkel.
 
 This file is part of GNU Wget.
@@ -33,7 +33,7 @@ as that of the covered work.  */
 #define GEN_SSLFUNC_H
 
 bool ssl_init (void);
-bool ssl_connect_wget (int, const char *);
+bool ssl_connect_wget (int, const char *, int *);
 bool ssl_check_certificate (int, const char *);
 
 #endif /* GEN_SSLFUNC_H */

src/sysdep.h

@@ -1,7 +1,7 @@
 /* Dirty system-dependent hacks.
    Copyright (C) 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004,
-   2005, 2006, 2007, 2008, 2009, 2010, 2011 Free Software Foundation,
-   Inc.
+   2005, 2006, 2007, 2008, 2009, 2010, 2011, 2015 Free Software
+   Foundation, Inc.
 
 This file is part of GNU Wget.
 

src/test.c

@@ -1,6 +1,6 @@
 /* Unit testing.
-   Copyright (C) 2005, 2006, 2007, 2008, 2009, 2010, 2011 Free Software
-   Foundation, Inc.
+   Copyright (C) 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2015 Free
+   Software Foundation, Inc.
 
 This file is part of GNU Wget.
 
@@ -41,24 +41,20 @@ as that of the covered work.  */
 #error "TESTING not set!!!"
 #endif
 
-const char *test_parse_content_disposition();
-const char *test_subdir_p();
-const char *test_dir_matches_p();
-const char *test_commands_sorted();
-const char *test_cmd_spec_restrict_file_names();
-const char *test_path_simplify ();
-const char *test_append_uri_pathel();
-const char *test_are_urls_equal();
-const char *test_is_robots_txt_url();
-
 const char *program_argstring = "TEST";
 
-int tests_run;
+static int tests_run;
 
 static const char *
-all_tests()
+all_tests(void)
 {
+#ifdef HAVE_METALINK
+  mu_run_test (test_find_key_value);
+  mu_run_test (test_find_key_values);
+  mu_run_test (test_has_key);
+#endif
   mu_run_test (test_parse_content_disposition);
+  mu_run_test (test_parse_range_header);
   mu_run_test (test_subdir_p);
   mu_run_test (test_dir_matches_p);
   mu_run_test (test_commands_sorted);
@@ -67,17 +63,24 @@ all_tests()
   mu_run_test (test_append_uri_pathel);
   mu_run_test (test_are_urls_equal);
   mu_run_test (test_is_robots_txt_url);
+#ifdef HAVE_HSTS
+  mu_run_test (test_hsts_new_entry);
+  mu_run_test (test_hsts_url_rewrite_superdomain);
+  mu_run_test (test_hsts_url_rewrite_congruent);
+  mu_run_test (test_hsts_read_database);
+#endif
 
   return NULL;
 }
 
-char *program_name; /* Needed by lib/error.c. */
+const char *program_name; /* Needed by lib/error.c. */
 
 int
-main (int argc _GL_UNUSED, char *argv[])
+main (int argc _GL_UNUSED, const char *argv[])
 {
   const char *result;
 
+  printf ("[DEBUG] Testing...\n\n");
 #ifdef ENABLE_NLS
   /* Set the current locale.  */
   setlocale (LC_ALL, "");

src/test.h

@@ -1,6 +1,6 @@
 /* Unit testing declarations.
-   Copyright (C) 2005, 2006, 2007, 2008, 2009, 2010, 2011 Free Software
-   Foundation, Inc.
+   Copyright (C) 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2015 Free
+   Software Foundation, Inc.
 
 This file is part of GNU Wget.
 
@@ -43,9 +43,12 @@ do { \
   puts("PASSED\n"); \
 } while (0)
 
-extern int tests_run;
 
+const char *test_has_key (void);
+const char *test_find_key_value (void);
+const char *test_find_key_values (void);
 const char *test_parse_content_disposition(void);
+const char *test_parse_range_header(void);
 const char *test_commands_sorted(void);
 const char *test_cmd_spec_restrict_file_names(void);
 const char *test_is_robots_txt_url(void);
@@ -54,6 +57,10 @@ const char *test_append_uri_pathel(void);
 const char *test_are_urls_equal(void);
 const char *test_subdir_p(void);
 const char *test_dir_matches_p(void);
+const char *test_hsts_new_entry(void);
+const char *test_hsts_url_rewrite_superdomain(void);
+const char *test_hsts_url_rewrite_congruent(void);
+const char *test_hsts_read_database(void);
 
 #endif /* TEST_H */
 

src/url.c

@@ -1,7 +1,7 @@
 /* URL handling.
    Copyright (C) 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004,
-   2005, 2006, 2007, 2008, 2009, 2010, 2011 Free Software Foundation,
-   Inc.
+   2005, 2006, 2007, 2008, 2009, 2010, 2011, 2015 Free Software
+   Foundation, Inc.
 
 This file is part of GNU Wget.
 
@@ -41,6 +41,7 @@ as that of the covered work.  */
 #include "utils.h"
 #include "url.h"
 #include "host.h"  /* for is_valid_ipv6_address */
+#include "c-strcase.h"
 
 #ifdef __VMS
 #include "vms.h"
@@ -77,6 +78,13 @@ static struct scheme_data supported_schemes[] =
   { "https",    "https://", DEFAULT_HTTPS_PORT, scm_has_query|scm_has_fragment },
 #endif
   { "ftp",      "ftp://",   DEFAULT_FTP_PORT,   scm_has_params|scm_has_fragment },
+#ifdef HAVE_SSL
+  /*
+   * Explicit FTPS uses the same port as FTP.
+   * Implicit FTPS has its own port (990), but it is disabled by default.
+   */
+  { "ftps",     "ftps://",  DEFAULT_FTP_PORT,  scm_has_params|scm_has_fragment },
+#endif
 
   /* SCHEME_INVALID */
   { NULL,       NULL,       -1,                 0 }
@@ -85,7 +93,7 @@ static struct scheme_data supported_schemes[] =
 /* Forward declarations: */
 
 static bool path_simplify (enum url_scheme, char *);
-
+
 /* Support for escaping and unescaping of URL strings.  */
 
 /* Table of "reserved" and "unsafe" characters.  Those terms are
@@ -160,17 +168,8 @@ static const unsigned char urlchr_table[256] =
 #undef U
 #undef RU
 
-/* URL-unescape the string S.
-
-   This is done by transforming the sequences "%HH" to the character
-   represented by the hexadecimal digits HH.  If % is not followed by
-   two hexadecimal digits, it is inserted literally.
-
-   The transformation is done in place.  If you need the original
-   string intact, make a copy before calling this function.  */
-
-void
-url_unescape (char *s)
+static void
+url_unescape_1 (char *s, unsigned char mask)
 {
   char *t = s;                  /* t - tortoise */
   char *h = s;                  /* h - hare     */
@@ -189,6 +188,8 @@ url_unescape (char *s)
           if (!h[1] || !h[2] || !(c_isxdigit (h[1]) && c_isxdigit (h[2])))
             goto copychar;
           c = X2DIGITS_TO_NUM (h[1], h[2]);
+          if (urlchr_test(c, mask))
+            goto copychar;
           /* Don't unescape %00 because there is no way to insert it
              into a C string without effectively truncating it. */
           if (c == '\0')
@@ -200,6 +201,31 @@ url_unescape (char *s)
   *t = '\0';
 }
 
+/* URL-unescape the string S.
+
+   This is done by transforming the sequences "%HH" to the character
+   represented by the hexadecimal digits HH.  If % is not followed by
+   two hexadecimal digits, it is inserted literally.
+
+   The transformation is done in place.  If you need the original
+   string intact, make a copy before calling this function.  */
+void
+url_unescape (char *s)
+{
+  url_unescape_1 (s, 0);
+}
+
+/* URL-unescape the string S.
+
+   This functions behaves identically as url_unescape(), but does not
+   convert characters from "reserved". In other words, it only converts
+   "unsafe" characters.  */
+void
+url_unescape_except_reserved (char *s)
+{
+  url_unescape_1 (s, urlchr_reserved);
+}
+
 /* The core of url_escape_* functions.  Escapes the characters that
    match the provided mask in urlchr_table.
 
@@ -272,7 +298,7 @@ url_escape_allow_passthrough (const char *s)
 {
   return url_escape_1 (s, urlchr_unsafe, true);
 }
-
+
 /* Decide whether the char at position P needs to be encoded.  (It is
    not enough to pass a single char *P because the function may need
    to inspect the surrounding context.)
@@ -418,7 +444,7 @@ reencode_escapes (const char *s)
   assert (p2 - newstr == newlen);
   return newstr;
 }
-
+
 /* Returns the scheme type if the scheme is supported, or
    SCHEME_INVALID if not.  */
 
@@ -574,8 +600,8 @@ rewrite_shorthand_url (const char *url)
         goto http;
 
       /* Turn "foo.bar.com:path" to "ftp://foo.bar.com/path". */
-      ret = aprintf ("ftp://%s", url);
-      ret[6 + (p - url)] = '/';
+      if ((ret = aprintf ("ftp://%s", url)) != NULL)
+        ret[6 + (p - url)] = '/';
     }
   else
     {
@@ -585,7 +611,7 @@ rewrite_shorthand_url (const char *url)
     }
   return ret;
 }
-
+
 static void split_path (const char *, char **, char **);
 
 /* Like strpbrk, with the exception that it returns the pointer to the
@@ -681,7 +707,6 @@ url_parse (const char *url, int *error, struct iri *iri, bool percent_encode)
   char *user = NULL, *passwd = NULL;
 
   const char *url_encoded = NULL;
-  char *new_url = NULL;
 
   int error_code;
 
@@ -695,29 +720,30 @@ url_parse (const char *url, int *error, struct iri *iri, bool percent_encode)
       goto error;
     }
 
+  url_encoded = url;
+
   if (iri && iri->utf8_encode)
     {
-      iri->utf8_encode = remote_to_utf8 (iri, iri->orig_url ? iri->orig_url : url, (const char **) &new_url);
+      char *new_url = NULL;
+
+      iri->utf8_encode = remote_to_utf8 (iri, iri->orig_url ? iri->orig_url : url, &new_url);
       if (!iri->utf8_encode)
         new_url = NULL;
       else
         {
+          xfree (iri->orig_url);
           iri->orig_url = xstrdup (url);
-          percent_encode = true;
+          url_encoded = reencode_escapes (new_url);
+          if (url_encoded != new_url)
+            xfree (new_url);
+          percent_encode = false;
         }
     }
 
-  /* XXX XXX Could that change introduce (security) bugs ???  XXX XXX*/
   if (percent_encode)
-    url_encoded = reencode_escapes (new_url ? new_url : url);
-  else
-    url_encoded = new_url ? new_url : url;
+    url_encoded = reencode_escapes (url);
 
   p = url_encoded;
-
-  if (new_url && url_encoded != new_url)
-    xfree (new_url);
-
   p += strlen (supported_schemes[scheme].leading_string);
   uname_b = p;
   p = url_skip_credentials (p);
@@ -896,6 +922,7 @@ url_parse (const char *url, int *error, struct iri *iri, bool percent_encode)
             {
               xfree (u->host);
               u->host = new;
+              u->idn_allocated = true;
               host_modified = true;
             }
         }
@@ -916,7 +943,7 @@ url_parse (const char *url, int *error, struct iri *iri, bool percent_encode)
       u->url = url_string (u, URL_AUTH_SHOW);
 
       if (url_encoded != url)
-        xfree ((char *) url_encoded);
+        xfree (url_encoded);
     }
   else
     {
@@ -931,7 +958,7 @@ url_parse (const char *url, int *error, struct iri *iri, bool percent_encode)
  error:
   /* Cleanup in case of error: */
   if (url_encoded && url_encoded != url)
-    xfree ((char *) url_encoded);
+    xfree (url_encoded);
 
   /* Transmit the error code to the caller, if the caller wants to
      know.  */
@@ -956,7 +983,7 @@ url_error (const char *url, int error_code)
 
       if ((p = strchr (scheme, ':')))
         *p = '\0';
-      if (!strcasecmp (scheme, "https"))
+      if (!c_strcasecmp (scheme, "https"))
         error = aprintf (_("HTTPS support not compiled in"));
       else
         error = aprintf (_(parse_errors[error_code]), quote (scheme));
@@ -1172,22 +1199,31 @@ url_set_file (struct url *url, const char *newfile)
 void
 url_free (struct url *url)
 {
-  xfree (url->host);
-  xfree (url->path);
-  xfree (url->url);
+  if (url)
+    {
+      if (url->idn_allocated) {
+        idn_free (url->host);      /* A dummy if !defined(ENABLE_IRI) */
+        url->host = NULL;
+      }
+      else
+        xfree (url->host);
 
-  xfree_null (url->params);
-  xfree_null (url->query);
-  xfree_null (url->fragment);
-  xfree_null (url->user);
-  xfree_null (url->passwd);
+      xfree (url->path);
+      xfree (url->url);
 
-  xfree (url->dir);
-  xfree (url->file);
+      xfree (url->params);
+      xfree (url->query);
+      xfree (url->fragment);
+      xfree (url->user);
+      xfree (url->passwd);
 
-  xfree (url);
+      xfree (url->dir);
+      xfree (url->file);
+
+      xfree (url);
+    }
 }
-
+
 /* Create all the necessary directories for PATH (a file).  Calls
    make_directory internally.  */
 int
@@ -1239,7 +1275,7 @@ mkalldirs (const char *path)
   xfree (t);
   return res;
 }
-
+
 /* Functions for constructing the file name out of URL components.  */
 
 /* A growable string structure, used by url_file_name and friends.
@@ -1317,8 +1353,9 @@ append_string (const char *str, struct growable *dest)
 
 enum {
   filechr_not_unix    = 1,      /* unusable on Unix, / and \0 */
-  filechr_not_windows = 2,      /* unusable on Windows, one of \|/<>?:*" */
-  filechr_control     = 4       /* a control character, e.g. 0-31 */
+  filechr_not_vms     = 2,      /* unusable on VMS (ODS5), 0x00-0x1F * ? */
+  filechr_not_windows = 4,      /* unusable on Windows, one of \|/<>?:*" */
+  filechr_control     = 8       /* a control character, e.g. 0-31 */
 };
 
 #define FILE_CHAR_TEST(c, mask) \
@@ -1327,11 +1364,14 @@ enum {
 
 /* Shorthands for the table: */
 #define U filechr_not_unix
+#define V filechr_not_vms
 #define W filechr_not_windows
 #define C filechr_control
 
+#define UVWC U|V|W|C
 #define UW U|W
-#define UWC U|W|C
+#define VC V|C
+#define VW V|W
 
 /* Table of characters unsafe under various conditions (see above).
 
@@ -1342,22 +1382,22 @@ enum {
 
 static const unsigned char filechr_table[256] =
 {
-UWC,  C,  C,  C,   C,  C,  C,  C,   /* NUL SOH STX ETX  EOT ENQ ACK BEL */
-  C,  C,  C,  C,   C,  C,  C,  C,   /* BS  HT  LF  VT   FF  CR  SO  SI  */
-  C,  C,  C,  C,   C,  C,  C,  C,   /* DLE DC1 DC2 DC3  DC4 NAK SYN ETB */
-  C,  C,  C,  C,   C,  C,  C,  C,   /* CAN EM  SUB ESC  FS  GS  RS  US  */
-  0,  0,  W,  0,   0,  0,  0,  0,   /* SP  !   "   #    $   %   &   '   */
-  0,  0,  W,  0,   0,  0,  0, UW,   /* (   )   *   +    ,   -   .   /   */
-  0,  0,  0,  0,   0,  0,  0,  0,   /* 0   1   2   3    4   5   6   7   */
-  0,  0,  W,  0,   W,  0,  W,  W,   /* 8   9   :   ;    <   =   >   ?   */
-  0,  0,  0,  0,   0,  0,  0,  0,   /* @   A   B   C    D   E   F   G   */
-  0,  0,  0,  0,   0,  0,  0,  0,   /* H   I   J   K    L   M   N   O   */
-  0,  0,  0,  0,   0,  0,  0,  0,   /* P   Q   R   S    T   U   V   W   */
-  0,  0,  0,  0,   W,  0,  0,  0,   /* X   Y   Z   [    \   ]   ^   _   */
-  0,  0,  0,  0,   0,  0,  0,  0,   /* `   a   b   c    d   e   f   g   */
-  0,  0,  0,  0,   0,  0,  0,  0,   /* h   i   j   k    l   m   n   o   */
-  0,  0,  0,  0,   0,  0,  0,  0,   /* p   q   r   s    t   u   v   w   */
-  0,  0,  0,  0,   W,  0,  0,  C,   /* x   y   z   {    |   }   ~   DEL */
+UVWC, VC, VC, VC,  VC, VC, VC, VC,   /* NUL SOH STX ETX  EOT ENQ ACK BEL */
+  VC, VC, VC, VC,  VC, VC, VC, VC,   /* BS  HT  LF  VT   FF  CR  SO  SI  */
+  VC, VC, VC, VC,  VC, VC, VC, VC,   /* DLE DC1 DC2 DC3  DC4 NAK SYN ETB */
+  VC, VC, VC, VC,  VC, VC, VC, VC,   /* CAN EM  SUB ESC  FS  GS  RS  US  */
+   0,  0,  W,  0,   0,  0,  0,  0,   /* SP  !   "   #    $   %   &   '   */
+   0,  0, VW,  0,   0,  0,  0, UW,   /* (   )   *   +    ,   -   .   /   */
+   0,  0,  0,  0,   0,  0,  0,  0,   /* 0   1   2   3    4   5   6   7   */
+   0,  0,  W,  0,   W,  0,  W, VW,   /* 8   9   :   ;    <   =   >   ?   */
+   0,  0,  0,  0,   0,  0,  0,  0,   /* @   A   B   C    D   E   F   G   */
+   0,  0,  0,  0,   0,  0,  0,  0,   /* H   I   J   K    L   M   N   O   */
+   0,  0,  0,  0,   0,  0,  0,  0,   /* P   Q   R   S    T   U   V   W   */
+   0,  0,  0,  0,   W,  0,  0,  0,   /* X   Y   Z   [    \   ]   ^   _   */
+   0,  0,  0,  0,   0,  0,  0,  0,   /* `   a   b   c    d   e   f   g   */
+   0,  0,  0,  0,   0,  0,  0,  0,   /* h   i   j   k    l   m   n   o   */
+   0,  0,  0,  0,   0,  0,  0,  0,   /* p   q   r   s    t   u   v   w   */
+   0,  0,  0,  0,   W,  0,  0,  C,   /* x   y   z   {    |   }   ~   DEL */
 
   C, C, C, C,  C, C, C, C,  C, C, C, C,  C, C, C, C, /* 128-143 */
   C, C, C, C,  C, C, C, C,  C, C, C, C,  C, C, C, C, /* 144-159 */
@@ -1370,10 +1410,13 @@ UWC,  C,  C,  C,   C,  C,  C,  C,   /* NUL SOH STX ETX  EOT ENQ ACK BEL */
   0, 0, 0, 0,  0, 0, 0, 0,  0, 0, 0, 0,  0, 0, 0, 0,
 };
 #undef U
+#undef V
 #undef W
 #undef C
 #undef UW
-#undef UWC
+#undef UVWC
+#undef VC
+#undef VW
 
 /* FN_PORT_SEP is the separator between host and port in file names
    for non-standard port numbers.  On Unix this is normally ':', as in
@@ -1382,10 +1425,14 @@ UWC,  C,  C,  C,   C,  C,  C,  C,   /* NUL SOH STX ETX  EOT ENQ ACK BEL */
 #define FN_PORT_SEP  (opt.restrict_files_os != restrict_windows ? ':' : '+')
 
 /* FN_QUERY_SEP is the separator between the file name and the URL
-   query, normally '?'.  Since Windows cannot handle '?' as part of
+   query, normally '?'.  Because VMS and Windows cannot handle '?' in a
    file name, we use '@' instead there.  */
-#define FN_QUERY_SEP (opt.restrict_files_os != restrict_windows ? '?' : '@')
-#define FN_QUERY_SEP_STR (opt.restrict_files_os != restrict_windows ? "?" : "@")
+#define FN_QUERY_SEP \
+ (((opt.restrict_files_os != restrict_vms) && \
+   (opt.restrict_files_os != restrict_windows)) ? '?' : '@')
+#define FN_QUERY_SEP_STR \
+ (((opt.restrict_files_os != restrict_vms) && \
+   (opt.restrict_files_os != restrict_windows)) ? "?" : "@")
 
 /* Quote path element, characters in [b, e), as file name, and append
    the quoted string to DEST.  Each character is quoted as per
@@ -1404,6 +1451,8 @@ append_uri_pathel (const char *b, const char *e, bool escaped,
   int mask;
   if (opt.restrict_files_os == restrict_unix)
     mask = filechr_not_unix;
+  else if (opt.restrict_files_os == restrict_vms)
+    mask = filechr_not_vms;
   else
     mask = filechr_not_windows;
   if (opt.restrict_files_ctrl)
@@ -1642,7 +1691,7 @@ url_file_name (const struct url *u, char *replaced_filename)
       logprintf (LOG_NOTQUIET, "New name is %s.\n", temp_fnres.base);
     }
 
-  free (fname_len_check);
+  xfree (fname_len_check);
 
   /* The filename has already been 'cleaned' by append_uri_pathel() above.  So,
    * just append it. */
@@ -1655,7 +1704,7 @@ url_file_name (const struct url *u, char *replaced_filename)
   /* Make a final check that the path length is acceptable? */
   /* TODO: check fnres.base for path length problem */
 
-  free (temp_fnres.base);
+  xfree (temp_fnres.base);
 
   /* Check the cases in which the unique extensions are not used:
      1) Clobbering is turned off (-nc).
@@ -1695,7 +1744,7 @@ url_file_name (const struct url *u, char *replaced_filename)
 
   return unique;
 }
-
+
 /* Resolve "." and ".." elements of PATH by destructively modifying
    PATH and return true if PATH has been modified, false otherwise.
 
@@ -1730,7 +1779,7 @@ path_simplify (enum url_scheme scheme, char *path)
       else if (h[0] == '.' && h[1] == '.' && (h[2] == '/' || h[2] == '\0'))
         {
           /* Handle "../" by retreating the tortoise by one path
-             element -- but not past beggining.  */
+             element -- but not past beginning.  */
           if (t > beg)
             {
               /* Move backwards until T hits the beginning of the
@@ -1738,7 +1787,11 @@ path_simplify (enum url_scheme scheme, char *path)
               for (--t; t > beg && t[-1] != '/'; t--)
                 ;
             }
-          else if (scheme == SCHEME_FTP)
+          else if (scheme == SCHEME_FTP
+#ifdef HAVE_SSL
+              || scheme == SCHEME_FTPS
+#endif
+              )
             {
               /* If we're at the beginning, copy the "../" literally
                  and move the beginning so a later ".." doesn't remove
@@ -1781,7 +1834,7 @@ path_simplify (enum url_scheme scheme, char *path)
 
   return t != h;
 }
-
+
 /* Return the length of URL's path.  Path is considered to be
    terminated by one or more of the ?query or ;params or #fragment,
    depending on the scheme.  */
@@ -1992,7 +2045,7 @@ uri_merge (const char *base, const char *link)
 
   return merge;
 }
-
+
 #define APPEND(p, s) do {                       \
   int len = strlen (s);                         \
   memcpy (p, s, len);                           \
@@ -2109,7 +2162,7 @@ url_string (const struct url *url, enum url_auth_mode auth_mode)
 
   return result;
 }
-
+
 /* Return true if scheme a is similar to scheme b.
 
    Schemes are similar if they are equal.  If SSL is supported, schemes
@@ -2127,7 +2180,7 @@ schemes_are_similar_p (enum url_scheme a, enum url_scheme b)
 #endif
   return false;
 }
-
+
 static int
 getchar_from_escaped_string (const char *str, char *c)
 {
@@ -2188,7 +2241,7 @@ are_urls_equal (const char *u1, const char *u2)
 
   return (*p == 0 && *q == 0 ? true : false);
 }
-
+
 #ifdef TESTING
 /* Debugging and testing support for path_simplify. */
 
@@ -2308,6 +2361,7 @@ test_append_uri_pathel(void)
 
       mu_assert ("test_append_uri_pathel: wrong result",
                  strcmp (dest.base, test_array[i].expected_result) == 0);
+      xfree (dest.base);
     }
 
   return NULL;

src/url.h

@@ -1,7 +1,7 @@
 /* Declarations for url.c.
    Copyright (C) 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004,
-   2005, 2006, 2007, 2008, 2009, 2010, 2011 Free Software Foundation,
-   Inc.
+   2005, 2006, 2007, 2008, 2009, 2010, 2011, 2015 Free Software
+   Foundation, Inc.
 
 This file is part of GNU Wget.
 
@@ -36,6 +36,7 @@ as that of the covered work.  */
 #define DEFAULT_HTTP_PORT 80
 #define DEFAULT_FTP_PORT 21
 #define DEFAULT_HTTPS_PORT 443
+#define DEFAULT_FTPS_IMPLICIT_PORT 990
 
 /* This represents how many characters less than the OS max name length a file
  * should be.  More precisely, a file name should be at most
@@ -70,6 +71,9 @@ enum url_scheme {
   SCHEME_HTTPS,
 #endif
   SCHEME_FTP,
+#ifdef HAVE_SSL
+  SCHEME_FTPS,
+#endif
   SCHEME_INVALID
 };
 
@@ -95,6 +99,10 @@ struct url
   /* Username and password (unquoted). */
   char *user;
   char *passwd;
+
+  /* 'host' is allocated by idna_to_ascii_8z() via idn_encode().
+   * Call 'idn_free()' to free this memory. */
+  bool idn_allocated;
 };
 
 /* Function declarations */
@@ -102,6 +110,7 @@ struct url
 char *url_escape (const char *);
 char *url_escape_unsafe_and_reserved (const char *);
 void url_unescape (char *);
+void url_unescape_except_reserved (char *);
 
 struct url *url_parse (const char *, int *, struct iri *iri, bool percent_encode);
 char *url_error (const char *, int);

src/utils.c

@@ -1,7 +1,7 @@
 /* Various utility functions.
    Copyright (C) 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004,
-   2005, 2006, 2007, 2008, 2009, 2010, 2011 Free Software Foundation,
-   Inc.
+   2005, 2006, 2007, 2008, 2009, 2010, 2011, 2015 Free Software
+   Foundation, Inc.
 
 This file is part of GNU Wget.
 
@@ -64,8 +64,10 @@ as that of the covered work.  */
 #include <sys/stat.h>
 
 /* For TIOCGWINSZ and friends: */
-#include <sys/ioctl.h>
-#include <termios.h>
+#ifndef WINDOWS
+# include <sys/ioctl.h>
+# include <termios.h>
+#endif
 
 /* Needed for Unix version of run_with_timeout. */
 #include <signal.h>
@@ -99,6 +101,7 @@ as that of the covered work.  */
 #endif
 
 #include "exits.h"
+#include "c-strcase.h"
 
 static void _Noreturn
 memfatal (const char *context, long attempted_size)
@@ -267,7 +270,7 @@ sepstring (const char *s)
   res[i + 1] = NULL;
   return res;
 }
-
+
 /* Like sprintf, but prints into a string of sufficient size freshly
    allocated with malloc, which is returned.  If unable to print due
    to invalid format, returns NULL.  Inability to allocate needed
@@ -333,7 +336,7 @@ aprintf (const char *fmt, ...)
         {                               /* maybe we have some wrong
                                            format string? */
           logprintf (LOG_ALWAYS,
-                     _("%s: aprintf: text buffer is too big (%ld bytes), "
+                     _("%s: aprintf: text buffer is too big (%d bytes), "
                        "aborting.\n"),
                      exec_name, size);  /* printout a log message */
           abort ();                     /* and abort... */
@@ -349,6 +352,32 @@ aprintf (const char *fmt, ...)
 #endif /* not HAVE_VASPRINTF */
 }
 
+#ifndef HAVE_STRLCPY
+/* strlcpy() is a BSD function that sometimes is really handy.
+ * It is the same as snprintf(dst,dstsize,"%s",src), but much faster. */
+
+size_t
+strlcpy (char *dst, const char *src, size_t size)
+{
+  const char *old = src;
+
+  /* Copy as many bytes as will fit */
+  if (size)
+    {
+      while (--size)
+        {
+          if (!(*dst++ = *src++))
+            return src - old - 1;
+        }
+
+      *dst = 0;
+    }
+
+  while (*src++);
+  return src - old - 1;
+}
+#endif
+
 /* Concatenate the NULL-terminated list of string arguments into
    freshly allocated space.  */
 
@@ -356,47 +385,30 @@ char *
 concat_strings (const char *str0, ...)
 {
   va_list args;
-  int saved_lengths[5];         /* inspired by Apache's apr_pstrcat */
-  char *ret, *p;
+  const char *arg;
+  size_t length = 0, pos = 0;
+  char *s;
 
-  const char *next_str;
-  int total_length = 0;
-  size_t argcount;
+  if (!str0)
+    return NULL;
 
-  /* Calculate the length of and allocate the resulting string. */
-
-  argcount = 0;
+  /* calculate the length of the resulting string */
   va_start (args, str0);
-  for (next_str = str0; next_str != NULL; next_str = va_arg (args, char *))
-    {
-      int len = strlen (next_str);
-      if (argcount < countof (saved_lengths))
-        saved_lengths[argcount++] = len;
-      total_length += len;
-    }
+  for (arg = str0; arg; arg = va_arg (args, const char *))
+    length += strlen(arg);
   va_end (args);
-  p = ret = xmalloc (total_length + 1);
 
-  /* Copy the strings into the allocated space. */
+  s = xmalloc (length + 1);
 
-  argcount = 0;
+  /* concatenate strings */
   va_start (args, str0);
-  for (next_str = str0; next_str != NULL; next_str = va_arg (args, char *))
-    {
-      int len;
-      if (argcount < countof (saved_lengths))
-        len = saved_lengths[argcount++];
-      else
-        len = strlen (next_str);
-      memcpy (p, next_str, len);
-      p += len;
-    }
+  for (arg = str0; arg; arg = va_arg (args, const char *))
+    pos += strlcpy(s + pos, arg, length - pos + 1);
   va_end (args);
-  *p = '\0';
 
-  return ret;
+  return s;
 }
-
+
 /* Format the provided time according to the specified format.  The
    format is a string with format elements supported by strftime.  */
 
@@ -430,7 +442,7 @@ datetime_str (time_t t)
 {
   return fmttime(t, "%Y-%m-%d %H:%M:%S");
 }
-
+
 /* The Windows versions of the following two functions are defined in
    mswindows.c. On MSDOS this function should never be called. */
 
@@ -495,7 +507,7 @@ fork_to_background (void)
 
 #endif /* def __VMS [else] */
 
-
+
 /* "Touch" FILE, i.e. make its mtime ("modified time") equal the time
    specified with TM.  The atime ("access time") is set to the current
    time.  */
@@ -805,7 +817,7 @@ fopen_excl (const char *fname, int binary)
   return fopen (fname, binary ? "wb" : "w");
 #endif /* not O_EXCL */
 }
-
+
 /* Create DIRECTORY.  If some of the pathname components of DIRECTORY
    are missing, create them first.  In case any mkdir() call fails,
    return its error status.  Returns 0 on successful completion.
@@ -871,7 +883,7 @@ file_merge (const char *base, const char *file)
 
   return result;
 }
-
+
 /* Like fnmatch, but performs a case-insensitive match.  */
 
 int
@@ -1112,11 +1124,11 @@ has_html_suffix_p (const char *fname)
 
   if ((suf = suffix (fname)) == NULL)
     return false;
-  if (!strcasecmp (suf, "html"))
+  if (!c_strcasecmp (suf, "html"))
     return true;
-  if (!strcasecmp (suf, "htm"))
+  if (!c_strcasecmp (suf, "htm"))
     return true;
-  if (suf[0] && !strcasecmp (suf + 1, "html"))
+  if (suf[0] && !c_strcasecmp (suf + 1, "html"))
     return true;
   return false;
 }
@@ -1262,7 +1274,7 @@ wget_read_file_free (struct file_memory *fm)
     }
   xfree (fm);
 }
-
+
 /* Free the pointers in a NULL-terminated vector of pointers, then
    free the pointer itself.  */
 void
@@ -1272,7 +1284,10 @@ free_vec (char **vec)
     {
       char **p = vec;
       while (*p)
-        xfree (*p++);
+        {
+          xfree (*p);
+          p++;
+        }
       xfree (vec);
     }
 }
@@ -1331,7 +1346,7 @@ vec_append (char **vec, const char *str)
   vec[cnt] = NULL;
   return vec;
 }
-
+
 /* Sometimes it's useful to create "sets" of strings, i.e. special
    hash tables where you want to store strings as keys and merely
    query for their existence.  Here is a set of utility routines that
@@ -1396,7 +1411,7 @@ free_keys_and_values (struct hash_table *ht)
       xfree (iter.value);
     }
 }
-
+
 /* Get digit grouping data for thousand separors by calling
    localeconv().  The data includes separator string and grouping info
    and is cached after the first call to the function.
@@ -1787,7 +1802,7 @@ convert_to_bits (wgint num)
   return num;
 }
 
-
+
 /* Determine the width of the terminal we're running on.  If that's
    not possible, return 0.  */
 
@@ -1800,7 +1815,7 @@ determine_screen_width (void)
   int fd;
   struct winsize wsz;
 
-  if (opt.lfilename != NULL)
+  if (opt.lfilename != NULL && opt.show_progress != 1)
     return 0;
 
   fd = fileno (stderr);
@@ -1817,7 +1832,7 @@ determine_screen_width (void)
   return 0;
 #endif /* neither TIOCGWINSZ nor WINDOWS */
 }
-
+
 /* Whether the rnd system (either rand or [dl]rand48) has been
    seeded.  */
 static int rnd_seeded;
@@ -1837,7 +1852,14 @@ static int rnd_seeded;
 int
 random_number (int max)
 {
-#ifdef HAVE_DRAND48
+#ifdef HAVE_RANDOM
+  if (!rnd_seeded)
+    {
+      srandom ((long) time (NULL) ^ (long) getpid ());
+      rnd_seeded = 1;
+    }
+  return random () % max;
+#elif defined HAVE_DRAND48
   if (!rnd_seeded)
     {
       srand48 ((long) time (NULL) ^ (long) getpid ());
@@ -1872,7 +1894,9 @@ random_number (int max)
 double
 random_float (void)
 {
-#ifdef HAVE_DRAND48
+#ifdef HAVE_RANDOM
+    return ((double) random_number (RAND_MAX)) / RAND_MAX;
+#elif defined HAVE_DRAND48
   if (!rnd_seeded)
     {
       srand48 ((long) time (NULL) ^ (long) getpid ());
@@ -1886,7 +1910,7 @@ random_float (void)
           + random_number (10000) / (10000.0 * 10000.0 * 10000.0 * 10000.0));
 #endif /* not HAVE_DRAND48 */
 }
-
+
 /* Implementation of run_with_timeout, a generic timeout-forcing
    routine for systems with Unix-like signal handling.  */
 
@@ -1897,7 +1921,7 @@ random_float (void)
 static sigjmp_buf run_with_timeout_env;
 
 static void _Noreturn
-abort_run_with_timeout (int sig)
+abort_run_with_timeout (int sig _GL_UNUSED)
 {
   assert (sig == SIGALRM);
   siglongjmp (run_with_timeout_env, -1);
@@ -1907,8 +1931,8 @@ abort_run_with_timeout (int sig)
 
 static jmp_buf run_with_timeout_env;
 
-static void
-abort_run_with_timeout (int sig)
+static void _Noreturn
+abort_run_with_timeout (int sig _GL_UNUSED)
 {
   assert (sig == SIGALRM);
   /* We don't have siglongjmp to preserve the set of blocked signals;
@@ -2045,7 +2069,7 @@ run_with_timeout (double timeout, void (*fun) (void *), void *arg)
 }
 #endif /* not WINDOWS */
 #endif /* not USE_SIGNAL_TIMEOUT */
-
+
 #ifndef WINDOWS
 
 /* Sleep the specified amount of seconds.  On machines without
@@ -2292,6 +2316,7 @@ compile_posix_regex (const char *str)
       fprintf (stderr, _("Invalid regular expression %s, %s\n"),
                quote (str), errbuf);
       xfree (errbuf);
+      xfree (regex);
       return NULL;
     }
 
@@ -2345,7 +2370,7 @@ match_posix_regex (const void *regex, const char *str)
 
 #undef IS_ASCII
 #undef NEXT_CHAR
-
+
 /* Simple merge sort for use by stable_sort.  Implementation courtesy
    Zeljko Vrba with additional debugging by Nenad Barbutov.  */
 
@@ -2391,7 +2416,7 @@ stable_sort (void *base, size_t nmemb, size_t size,
       mergesort_internal (base, temp, size, 0, nmemb - 1, cmpfun);
     }
 }
-
+
 /* Print a decimal number.  If it is equal to or larger than ten, the
    number is rounded.  Otherwise it is printed with one significant
    digit without trailing zeros and with no more than three fractional
@@ -2481,6 +2506,21 @@ get_max_length (const char *path, int length, int name)
   return ret;
 }
 
+void
+wg_hex_to_string (char *str_buffer, const char *hex_buffer, size_t hex_len)
+{
+  size_t i;
+
+  for (i = 0; i < hex_len; i++)
+    {
+      /* Each byte takes 2 characters.  */
+      sprintf (str_buffer + 2 * i, "%02x", hex_buffer[i] & 0xFF);
+    }
+
+  /* Null-terminate result.  */
+  str_buffer[2 * i] = '\0';
+}
+
 #ifdef TESTING
 
 const char *