Update NEWS file for new release

Gzip compression has a number of bugs which need to be ironed out before
we can support it by default. Some of these stem from a misunderstanding
of the HTTP spec, but a lot of them are also due to many web servers not
being compliant with RFC 7231.

With this commit, I am marking GZip compression support as experimental
in GNU Wget pending further investigation and the addition of tests.

* src/init.c (defaults): Switch of compression support by default
* docs/wget.texi: State that compression is experimental

.gitignore

@@ -2,6 +2,7 @@
 /.sc-start-sc_bindtextdomain
 /.sc-start-sc_prohibit_HAVE_MBRTOWC
 /.version
+/ABOUT-NLS
 /GNUmakefile
 /INSTALL
 /Makefile
@@ -25,6 +26,7 @@ testenv/*.log
 testenv/*.trs
 # build-aux/
 build-aux/.gitignore
+build-aux/ar-lib
 build-aux/compile
 build-aux/config.guess
 build-aux/config.sub

.gitlab-ci.yml

@@ -0,0 +1,124 @@
+variables:
+  BUILD_IMAGES_PROJECT: gnuwget/build-images
+  DEBIAN_BUILD: buildenv-debian-stretch
+  FEDORA_BUILD: buildenv-f25
+  CENTOS7_BUILD: buildenv-centos7
+  GET_SOURCES_ATTEMPTS: "3"
+
+# In this build we combine
+#  * syntax-check
+#  * abi-check
+#  * build/valgrind-check
+#  * build/asan-check
+#  * build w/Werror and ubsan-check
+Debian GNU/Linux build:
+  image: $CI_REGISTRY/$BUILD_IMAGES_PROJECT:$DEBIAN_BUILD
+  script:
+  - echo "127.0.0.1 wgettestingserver" >>/etc/hosts
+  - cat /etc/hosts
+  - touch .manywarnings
+  - alias make="make -j$(nproc)"
+  - ./bootstrap
+  - ./configure -C
+  - make syntax-check
+  - make check
+#  - make abi-check
+  - make distcheck
+  - make check-valgrind
+  - ./configure -C --with-ssl=openssl
+  - make clean
+  - make check
+#  - ./configure -C --enable-fsanitize-asan
+#  - make clean
+#  - make check
+#  - ./configure -C --enable-fsanitize-ubsan
+#  - make clean
+#  - make check
+#  - ./configure -C --enable-fsanitize-msan
+#  - make clean
+#  - make check
+  tags:
+  - shared
+  artifacts:
+    expire_in: 2 weeks
+    when: on_failure
+    paths:
+      - tests/*.log
+#      - compat_reports/
+
+#Centos7 build:
+#  image: $CI_REGISTRY/$BUILD_IMAGES_PROJECT:$CENTOS7_BUILD
+#  script:
+#  - sed -i 's/AM_GNU_GETTEXT_VERSION(0.19.3)/AM_GNU_GETTEXT_VERSION(0.18.2)/g' configure.ac
+#  - ./bootstrap && ./configure --enable-gcc-warnings --disable-doc && make -j$(nproc) check
+#  tags:
+#  - shared
+#  artifacts:
+#    expire_in: 2 weeks
+#    when: on_failure
+#    paths:
+#      - tests/*.log
+#      - compat_reports/
+
+#clang-analyzer/Fedora:
+#  image: $CI_REGISTRY/$BUILD_IMAGES_PROJECT:$FEDORA_BUILD
+#  script:
+#  - ./bootstrap
+#  - scan-build ./configure --disable-doc --enable-gcc-warnings
+#  - make -j$(nproc) -C gl
+#  - scan-build --status-bugs -o scan-build-lib make -j$(nproc) -C lib
+#  tags:
+#  - shared
+#  except:
+#  - tags
+#  artifacts:
+#    expire_in: 2 weeks
+#    when: on_failure
+#    paths:
+#      - scan-build-lib/*
+
+#pages:
+#  image: $CI_REGISTRY/$BUILD_IMAGES_PROJECT:$DEBIAN_BUILD
+#  script:
+#  - ./bootstrap
+#  - CFLAGS="-g -O0" ./configure --enable-gtk-doc --enable-code-coverage --disable-gcc-warnings
+#  - make -j$(nproc) check
+#  - make local-code-coverage-output
+#  - mkdir -p public
+#  - rm -rf public/coverage
+#  - mv libidn2-*-coverage public/coverage
+#  - rm -rf public/reference
+#  - mv doc/reference/html public/reference
+#  - make -C doc manual
+#  - rm -rf public/manual
+#  - mv doc/manual public/manual
+#  tags:
+#  - shared
+#  artifacts:
+#    when: on_success
+#    paths:
+#      - public
+#  only:
+#    - master
+
+#MinGW64:
+#  image: $CI_REGISTRY/$BUILD_IMAGES_PROJECT:$FEDORA_BUILD
+#  before_script:
+#  - mount -t binfmt_misc binfmt_misc /proc/sys/fs/binfmt_misc &&
+#    echo ':DOSWin:M::MZ::/usr/bin/wine64:' > /proc/sys/fs/binfmt_misc/register
+#  script:
+#  - ./bootstrap
+#  - mingw64-configure --disable-valgrind-tests --disable-doc
+#  - mingw64-make -j$(nproc)
+#  - mingw64-make -j$(nproc) check
+#  tags:
+#  - shared
+#  - docker
+#  except:
+#  - tags
+#  artifacts:
+#    expire_in: 2 weeks
+#    when: on_failure
+#    paths:
+#      - ./*.log
+#      - tests/*.log

.travis.yml

@@ -0,0 +1,51 @@
+sudo: no
+dist: trusty
+
+language: c
+
+compiler:
+    - gcc
+
+notifications:
+    email:
+        recipients:
+            - darnir@gmail.com
+            - bug-wget@gnu.org
+        on_success: change
+        on_failure: always
+
+env:
+    global:
+        - PERLLIB=$PERLLIB:../../tests
+        - PERL5LIB=$PERL5LIB:../../tests
+    matrix:
+        - SSL=""
+        - SSL="openssl"
+        - SSL="gnutls"
+
+addons:
+    apt:
+        packages:
+            - automake
+            - autoconf
+            - autopoint
+            - flex
+            - texinfo
+            - pkg-config
+            - libgnutls-dev
+            - libssl-dev
+            - make
+            - libhttp-daemon-perl
+            - libio-socket-ssl-perl
+            - libidn2-dev
+            - gettext
+            - texlive
+            - python3
+            - valgrind
+            - language-pack-tr
+            - language-pack-ru
+
+script:
+    - make distclean || true
+    - ./bootstrap
+    - ./contrib/travis-ci $SSL

COPYING

@@ -1,7 +1,7 @@
                     GNU GENERAL PUBLIC LICENSE
                        Version 3, 29 June 2007
 
- Copyright (C) 2007 Free Software Foundation, Inc. <http://fsf.org/>
+ Copyright (C) 2007 Free Software Foundation, Inc. <https://fsf.org/>
  Everyone is permitted to copy and distribute verbatim copies
  of this license document, but changing it is not allowed.
 
@@ -645,7 +645,7 @@ the "copyright" line and a pointer to where the full notice is found.
     GNU General Public License for more details.
 
     You should have received a copy of the GNU General Public License
-    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
 Also add information on how to contact you by electronic and paper mail.
 
@@ -664,11 +664,11 @@ might be different; for a GUI interface, you would use an "about box".
   You should also get your employer (if you work as a programmer) or school,
 if any, to sign a "copyright disclaimer" for the program, if necessary.
 For more information on this, and how to apply and follow the GNU GPL, see
-<http://www.gnu.org/licenses/>.
+<https://www.gnu.org/licenses/>.
 
   The GNU General Public License does not permit incorporating your program
 into proprietary programs.  If your program is a subroutine library, you
 may consider it more useful to permit linking proprietary applications with
 the library.  If this is what you want to do, use the GNU Lesser General
 Public License instead of this License.  But first, please read
-<http://www.gnu.org/philosophy/why-not-lgpl.html>.
+<https://www.gnu.org/philosophy/why-not-lgpl.html>.

ChangeLog-2014-12-10

@@ -729,7 +729,7 @@
 	* testenv/Makefile.am: Run the tests in Python's Optimizedmode
 	* testenv/conf/__init__.py (gen_hook): Use try..except instead of if..else
 	* testenv/misc/color_terminal.py: System and check will not change while a test is
-	run. Do not test for them on every invokation of printer()
+	run. Do not test for them on every invocation of printer()
 	* testenv/server/http/http_server.py: The ssl and re modules are required by
 	specific functions. Load them lazily
 	(HTTPSServer.__init__): Lazy load ssl module here

MAILING-LIST

@@ -7,30 +7,14 @@ Primary List
 The primary mailinglist for discussion, bug-reports, or questions about
 GNU Wget is at <bug-wget@gnu.org>. To subscribe, send an email to
 <bug-wget-join@gnu.org>, or visit
-`http://lists.gnu.org/mailman/listinfo/bug-wget'.
+`https://lists.gnu.org/mailman/listinfo/bug-wget'.
 
    You do not need to subscribe to send a message to the list; however,
 please note that unsubscribed messages are moderated, and may take a
 while before they hit the list--*usually around a day*.  If you want
 your message to show up immediately, please subscribe to the list
 before posting. Archives for the list may be found at
-`http://lists.gnu.org/pipermail/bug-wget/'.
-
-   An NNTP/Usenettish gateway is also available via Gmane
-(http://gmane.org/about.php). You can see the Gmane archives at
-`http://news.gmane.org/gmane.comp.web.wget.general'. Note that the
-Gmane archives conveniently include messages from both the current
-list, and the previous one. Messages also show up in the Gmane archives
-sooner than they do at `lists.gnu.org'.
-
-Bug Notices List
-----------------
-
-Additionally, there is the <wget-notify@addictivecode.org> mailing
-list. This is a non-discussion list that receives bug report
-notifications from the bug-tracker. To subscribe to this list, send an
-email to <wget-notify-join@addictivecode.org>, or visit
-`http://addictivecode.org/mailman/listinfo/wget-notify'.
+`https://lists.gnu.org/pipermail/bug-wget/'.
 
 Obsolete Lists
 --------------
@@ -40,10 +24,7 @@ discussion list, and another list, <wget-patches@sunsite.dk> was used
 for submitting and discussing patches to GNU Wget.
 
    Messages from <wget@sunsite.dk> are archived at
-     `http://www.mail-archive.com/wget%40sunsite.dk/' and at
-
-     `http://news.gmane.org/gmane.comp.web.wget.general' (which also
-     continues to archive the current list, <bug-wget@gnu.org>).
+     `https://www.mail-archive.com/wget%40sunsite.dk/' and at
 
    Messages from <wget-patches@sunsite.dk> are archived at
-     `http://news.gmane.org/gmane.comp.web.wget.patches'.
+     `https://news.gmane.org/gmane.comp.web.wget.patches'.

Makefile.am

@@ -1,6 +1,6 @@
 # Makefile for `Wget' utility
-# Copyright (C) 1995, 1996, 1997, 2006, 2007, 2008, 2009, 2010, 2011
-# Free Software Foundation, Inc.
+# Copyright (C) 1995-1997, 2006-2018 Free Software Foundation,
+# Inc.
 
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -13,7 +13,7 @@
 # GNU General Public License for more details.
 
 # You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
 # Additional permission under GNU GPL version 3 section 7
 
@@ -77,3 +77,6 @@ gen-ChangeLog:
 	    { rm -f $(distdir)/ChangeLog &&				\
 	      mv $(distdir)/cl-t $(distdir)/ChangeLog; }		\
 	fi
+
+check-valgrind:
+	TESTS_ENVIRONMENT="VALGRIND_TESTS=1" $(MAKE) check

NEWS

@@ -1,12 +1,146 @@
 GNU Wget NEWS -- history of user-visible changes.
 
-Copyright (C) 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004, 2005,
-2006, 2007, 2008, 2009, 2010, 2011, 2012, 2013, 2014, 2015 Free
-Software Foundation, Inc.
+Copyright (C) 1997-2018 Free Software Foundation, Inc.
 See the end for copying conditions.
 
 Please send GNU Wget bug reports to <bug-wget@gnu.org>.
 
+* Changes in Wget 1.19.4
+
+* A major bug that caused GZip'ed pages to never be decompressed has been fixed
+
+* Support for Content-Encoding and Transfer-Encoding have been marked as
+  experimental and disabled by default
+
+
+* Changes in Wget 1.19.3
+
+* Prevent erroneous decompression of .gz and .tgz files with broken servers
+
+* Added support for HTTP 308 Permanent Redirect response
+
+* Fix a segfault in some cases where the Content-Type header is not sent
+
+* Support OpenSSL 1.1 builds without using deprecated features
+
+* Fix netrc file detection on Windows
+
+* Several minor bug fixes
+
+
+* Changes in Wget 1.19.2
+
+* Fix CVE-2017-13089 (Stack overflow in HTTP protocol handling)
+
+* Fix CVE-2017-13090 (Heap overflow in HTTP protocol handling)
+
+* New option --compression for gzip Content-Encoding
+
+* New option --[no]-netrc to control .netrc parsing
+
+* Added GNU extensions to .netrc parsing
+
+* Improved IDNA 2003 compatibility
+
+* Fix VPATH issues
+
+* Improved and extended the test suite
+
+* Support Wayback Machine's X-Archive-Orig-last-modified
+
+* Several bug fixes
+
+
+* Changes in Wget 1.19.1
+
+* Fix bugs, a regression, portability/build issues
+
+* Add new option --retry-on-http-error
+
+
+* Changes in Wget 1.19
+
+* New option --use-askpass=COMMAND. Fetch user/password by calling
+  an external program.
+
+* Use IDNA2008 (+ TR46 if available) through libidn2
+
+* When processing a Metalink header, --metalink-index=<number> allows
+  to process the header's application/metalink4+xml files.
+
+* When processing a Metalink file, --trust-server-names enables the
+  use of the destination file names specified in the Metalink file,
+  otherwise a safe destination file name is computed.
+
+* When processing a Metalink file, enforce a safe destination path.
+  Remove any drive letter prefix under w32, i.e. 'C:D:file'.  Call
+  libmetalink's metalink_check_safe_path() to prevent absolute,
+  relative, or home paths:
+  https://tools.ietf.org/html/rfc5854#section-4.1.2.1
+  https://tools.ietf.org/html/rfc5854#section-4.2.8.3
+
+* When processing a Metalink file, --directory-prefix=<prefix> sets
+  the top of the retrieval tree to prefix for Metalink downloads.
+
+* When processing a Metalink file, reject downloaded files which don't
+  agree with their own metalink:size value:
+  https://tools.ietf.org/html/rfc5854#section-4.2.16
+
+* When processing a Metalink file, with --continue resume partially
+  downloaded files and keep fully downloaded files even if they fail
+  the verification.
+
+* When processing a Metalink file, create the parent directories of a
+  "path/file" destination file name:
+  https://tools.ietf.org/html/rfc5854#section-4.1.2.1
+  https://tools.ietf.org/html/rfc5854#section-4.2.8.3
+
+* On a recursive download, append a .tmp suffix to temporary files
+  that will be deleted after being parsed, and create them
+  readable/writable only by the owner.
+
+* New make target 'check-valgrind'
+
+* Fix several bugs
+
+* Fix compatibility issues
+
+* Changes in Wget 1.18
+
+* By default, on server redirects to a FTP resource, use the original
+  URL to get the local file name. Close CVE-2016-4971.  This
+  introduces a backward-incompatibility for HTTP->FTP redirects and
+  any script that relies on the old  behaviour must use
+  --trust-server-names.
+
+* Check the HSTS file is not world-writable before using it.
+
+* Parse <img srcset> attributes on a recursive download.
+
+* Fix problem with SNI server names having trailing dot(s)
+
+* New options --bind-dns-address and --dns-servers.
+
+* When Wget is built with libiconv, it now converts non-ASCII URIs to
+  the locale's codeset when it creates files.  The encoding of the
+  remote files and URIs is taken from --remote-encoding, defaulting to
+  UTF-8.  The result is that non-ASCII URIs and files downloaded via
+  HTTP/HTTPS and FTP will have names on the local filesystem that
+  correspond to their remote names.
+
+* Changes in Wget 1.17.1
+
+* Fix compile error when IPv6 is disabled or SSL is not present.
+
+* Fix HSTS memory leak.
+
+* Fix progress output in non-C locales.
+
+* Fix SIGSEGV when -N and --content-disposition are used together.
+
+* Add --check-certificate=quiet to tell wget to not print any warning about
+  invalid certificates.
+
 * Changes in Wget 1.17
 
 ** Remove FTP passive to active fallback due to privacy concerns.
@@ -610,7 +744,7 @@ addresses when accessing the first one fails.
 non-standard port.
 
 ** Wget now supports the robots.txt directives specified in
-<http://www.robotstxt.org/wc/norobots-rfc.txt>.
+<http://www.robotstxt.org/norobots-rfc.txt>.
 
 ** URL parser has been fixed, especially the infamous overzealous
 quoting.  Wget no longer dequotes reserved characters, e.g. `%3F' is

README

@@ -1,7 +1,7 @@
                                                           -*- text -*-
 GNU Wget
 ========
-                  Current Web home: http://www.gnu.org/software/wget/
+                  Current Web home: https://www.gnu.org/software/wget/
 
 GNU Wget is a free utility for non-interactive download of files from
 the Web.  It supports HTTP, HTTPS, and FTP protocols, as well as
@@ -50,10 +50,10 @@ resides at <ftp://ftp.gnu.org/pub/gnu/wget/>.
 Please report bugs in Wget to <bug-wget@gnu.org>.
 
 See the file `MAILING-LIST' for information about Wget mailing lists.
-Wget's home page is at <http://www.gnu.org/software/wget/>.
+Wget's home page is at <https://www.gnu.org/software/wget/>.
 
 If you would like to contribute code for Wget, please read
-http://wget.addictivecode.org/PatchGuidelines.
+https://wget.addictivecode.org/PatchGuidelines.
 
 Wget was originally written and mainained by Hrvoje Niksic.  Please see
 the file AUTHORS for a list of major contributors, and the ChangeLogs

README.checkout

@@ -4,7 +4,7 @@ Compiling From Repository Sources
    does not contain automatically-generated files, even when these are
    normally present in the distribution tarballs. Therefore, to build GNU
    Wget from the sources in the repository, you'll need to have one or
-   more of the following (note that gettext, OpenSSL, GnuTLS, libidn,
+   more of the following (note that gettext, OpenSSL, GnuTLS, libidn2,
    libiconv, libpsl, libpcre, pkg-config, libmetalink and GnuPG are not
    absolutely required):
 
@@ -25,14 +25,15 @@ Compiling From Repository Sources
        tests in the tests/ sub directory. Tarball distributions include an
        already-generated wget.1 manual. The command "make check" runs the
        test suite written in perl and python (see below). To execute all the tests
-       you need [24]libwww-perl and libio-socket-ssl-perl perl library. If
-       "perl -MCPAN -e 'install Bundle::LWP'" fails then you most likely don't
-       have cpan module installed. First download [25]CPAN and [26]install it.
-       Then execute "perl -MCPAN -e 'install Bundle::LWP'".
+       you need libwww-perl and libio-socket-ssl-perl perl library.
+       GNU/Linux distros likely have these two libraries packaged.
+       If not, install them directly with "perl -MCPAN -e 'install Bundle::LWP'"[24].
+       If that fails then you most likely don't have the CPAN module installed.
+       First download [25]CPAN and [26]install it, then try again the above command.
        Now "make check" should pass most of the tests in the test suite.
 
-     * [45]Python, if you want to run the tests in the testenv/ subdirectory.
-       Keep in mind that "make check" will try to run all the perl and python
+     * [45]Python3, if you want to run the tests in the testenv/ subdirectory.
+       Keep in mind that "make check" will try to run all the Perl and Python3
        tests. More information about the test suite below in the section
        "Testing and development".
 
@@ -68,7 +69,7 @@ Compiling From Repository Sources
        If you want to compile+link a non-system library version use
        --with-libssl-prefix (or if having pkg-config: see description below).
 
-     * [31]libidn is required for IDN/IRI support (non-ASCII characters
+     * [31]libidn2 is required for IDN/IRI support (non-ASCII characters
        within what would otherwise be URLs).
 
      * [32]libiconv is required on non-GNU systems, for IDN/IRI support.
@@ -99,6 +100,13 @@ Compiling From Repository Sources
 
      * [47]GnuPG with GPGME is used to verify GPG-signed Metalink resources.
 
+     * [48]libcares is needed to bind DNS resolving to a given IP address.
+       The command line options --dns-servers and --bind-dns-address are
+       only available when configured with --with-cares.
+
+     * [49]Gpef is required by some gnulib modules, most notably "unicase". You
+	   need the application installed on the machine to compile Wget from git.
+
 
    For those who might be confused as to what to do once they check out
    the source code, considering configure and Makefile do not yet exist at
@@ -112,7 +120,7 @@ Compiling From Repository Sources
  $  cd wget        # assumes you've cloned a repository to "./wget"
     2. Generate all the automatically-generated files required prior to
        configuring the package:
- $  ./bootstrap
+ $  ./bootstrap (or: bash ./bootstrap)
     3. Configure the package and compile it:
  $  ./configure --enable-assert [some_parameters]
  $  make
@@ -153,9 +161,10 @@ Testing and development
  that bug, which might be tedious. Tests should only be run through GDB for that purpose.
 
 
- Copyright (C) 2008,2010,2014,2015 Free Software Foundation, Inc.
+ Copyright (C) 2008, 2010, 2014-2015, 2018 Free Software Foundation,
+ Inc.
  License GPLv3+: GNU GPL version 3 or later
- <http://www.gnu.org/licenses/gpl.html>.
+ <https://www.gnu.org/licenses/gpl.html>.
  This is free software: you are free to change and redistribute it.
  There is NO WARRANTY, to the extent permitted by law.
 
@@ -179,31 +188,33 @@ Testing and development
 
 References
 
-  20. http://www.gnu.org/software/autoconf/
-  21. http://www.gnu.org/software/automake/
-  22. http://flex.sourceforge.net/
-  23. http://www.perl.org/
+  20. https://www.gnu.org/software/autoconf/
+  21. https://www.gnu.org/software/automake/
+  22. https://github.com/westes/flex
+  23. https://www.perl.org/
   24. http://search.cpan.org/dist/libwww-perl/lib/Bundle/LWP.pm
   25. http://search.cpan.org/CPAN/authors/id/A/AN/ANDK/CPAN-1.9402.tar.gz
   26. http://apthorpe.cynistar.net/code/configuring_cpan.html
-  27. http://www.gnu.org/software/texinfo/
-  28. http://www.gnu.org/software/gettext/
-  29. http://www.gnu.org/software/gnutls/
-  30. http://www.openssl.org/
-  31. http://www.gnu.org/software/libidn/
-  32. http://www.gnu.org/software/libiconv/
-  33. http://git-scm.com/
+  27. https://www.gnu.org/software/texinfo/
+  28. https://www.gnu.org/software/gettext/
+  29. https://www.gnu.org/software/gnutls/
+  30. https://www.openssl.org/
+  31. https://www.gnu.org/software/libidn/#libidn2
+  32. https://www.gnu.org/software/libiconv/
+  33. https://git-scm.com/
   34. https://github.com/rockdaboot/libpsl
-  35. http://www.pcre.org
+  35. https://www.pcre.org
   36. http://wget.addictivecode.org/CompilingRepoSources?action=edit&editor=text
   37. http://wget.addictivecode.org/CompilingRepoSources
   38. http://wget.addictivecode.org/CompilingRepoSources?action=info
   39. http://wget.addictivecode.org/CompilingRepoSources?action=AttachFile
-  40. http://moinmo.in/
-  41. http://moinmo.in/Python
-  42. http://moinmo.in/GPL
-  43. http://validator.w3.org/check?uri=referer
+  40. https://moinmo.in/
+  41. https://moinmo.in/Python
+  42. https://moinmo.in/GPL
+  43. https://validator.w3.org/check?uri=referer
   44. http://wget.addictivecode.org/WikiLicense
   45. https://www.python.org/
   46. https://launchpad.net/libmetalink
   47. https://www.gnupg.org
+  48. https://c-ares.haxx.se/
+  49. https://www.gnu.org/software/gperf/

bootstrap

@@ -1,10 +1,10 @@
 #! /bin/sh
 # Print a version string.
-scriptversion=2013-12-05.23; # UTC
+scriptversion=2017-09-19.08; # UTC
 
 # Bootstrap this package from checked-out sources.
 
-# Copyright (C) 2003-2014 Free Software Foundation, Inc.
+# Copyright (C) 2003-2018 Free Software Foundation, Inc.
 
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -17,7 +17,7 @@ scriptversion=2013-12-05.23; # UTC
 # GNU General Public License for more details.
 
 # You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
 # Originally written by Paul Eggert.  The canonical version of this
 # script is maintained as build-aux/bootstrap in gnulib, however, to
@@ -42,6 +42,9 @@ export LC_ALL
 
 local_gl_dir=gl
 
+# Honor $PERL, but work even if there is none.
+PERL="${PERL-perl}"
+
 me=$0
 
 usage() {
@@ -138,7 +141,7 @@ po_download_command_format=\
 # Fallback for downloading .po files (if rsync fails).
 po_download_command_format2=\
 "wget --mirror -nd -q -np -A.po -P '%s' \
- http://translationproject.org/latest/%s/"
+ https://translationproject.org/latest/%s/"
 
 # Prefer a non-empty tarname (4th argument of AC_INIT if given), else
 # fall back to the package name (1st argument with munging)
@@ -415,28 +418,30 @@ sort_ver() { # sort -V is not generally available
   done
 }
 
+get_version_sed='
+# Move version to start of line.
+s/.*[v ]\([0-9]\)/\1/
+
+# Skip lines that do not start with version.
+/^[0-9]/!d
+
+# Remove characters after the version.
+s/[^.a-z0-9-].*//
+
+# The first component must be digits only.
+s/^\([0-9]*\)[a-z-].*/\1/
+
+#the following essentially does s/5.005/5.5/
+s/\.0*\([1-9]\)/.\1/g
+p
+q'
+
 get_version() {
   app=$1
 
   $app --version >/dev/null 2>&1 || { $app --version; return 1; }
 
-  $app --version 2>&1 |
-  sed -n '# Move version to start of line.
-          s/.*[v ]\([0-9]\)/\1/
-
-          # Skip lines that do not start with version.
-          /^[0-9]/!d
-
-          # Remove characters after the version.
-          s/[^.a-z0-9-].*//
-
-          # The first component must be digits only.
-          s/^\([0-9]*\)[a-z-].*/\1/
-
-          #the following essentially does s/5.005/5.5/
-          s/\.0*\([1-9]\)/.\1/g
-          p
-          q'
+  $app --version 2>&1 | sed -n "$get_version_sed"
 }
 
 check_versions() {
@@ -456,6 +461,7 @@ check_versions() {
     test "$appvar" = TAR && appvar=AMTAR
     case $appvar in
         GZIP) ;; # Do not use $GZIP:  it contains gzip options.
+        PERL::*) ;; # Keep perl modules as-is
         *) eval "app=\${$appvar-$app}" ;;
     esac
 
@@ -473,6 +479,17 @@ check_versions() {
           ret=1
           continue
         } ;;
+      # Another check is for perl modules.  These can be written as
+      # e.g. perl::XML::XPath in case of XML::XPath module, etc.
+      perl::*)
+        # Extract module name
+        app="${app#perl::}"
+        if ! $PERL -m"$app" -e 'exit 0' >/dev/null 2>&1; then
+          warn_ "Error: perl module '$app' not found"
+          ret=1
+        fi
+        continue
+        ;;
     esac
     if [ "$req_ver" = "-" ]; then
       # Merely require app to exist; not all prereq apps are well-behaved
@@ -671,8 +688,11 @@ if $bootstrap_sync; then
   }
 fi
 
-gnulib_tool=$GNULIB_SRCDIR/gnulib-tool
-<$gnulib_tool || exit $?
+gnulib_tool=$GNULIB_SRCDIR/gnulib-tool.py
+if ! <$gnulib_tool; then
+  gnulib_tool=$GNULIB_SRCDIR/gnulib-tool
+  <$gnulib_tool || exit $?
+fi
 
 # Get translations.
 
@@ -682,7 +702,7 @@ download_po_files() {
   echo "$me: getting translations into $subdir for $domain..."
   cmd=$(printf "$po_download_command_format" "$domain" "$subdir")
   eval "$cmd" && return
-  # Fallback to HTTP.
+  # Fallback to HTTPS.
   cmd=$(printf "$po_download_command_format2" "$subdir" "$domain")
   eval "$cmd"
 }
@@ -773,9 +793,9 @@ symlink_to_dir()
       # Leave any existing symlink alone, if it already points to the source,
       # so that broken build tools that care about symlink times
       # aren't confused into doing unnecessary builds.  Conversely, if the
-      # existing symlink's time stamp is older than the source, make it afresh,
+      # existing symlink's timestamp is older than the source, make it afresh,
       # so that broken tools aren't confused into skipping needed builds.  See
-      # <http://lists.gnu.org/archive/html/bug-gnulib/2011-05/msg00326.html>.
+      # <https://lists.gnu.org/r/bug-gnulib/2011-05/msg00326.html>.
       test -h "$dst" &&
       src_ls=$(ls -diL "$src" 2>/dev/null) && set $src_ls && src_i=$1 &&
       dst_ls=$(ls -diL "$dst" 2>/dev/null) && set $dst_ls && dst_i=$1 &&
@@ -900,7 +920,8 @@ if test $use_libtool = 1; then
   esac
 fi
 echo "$0: $gnulib_tool $gnulib_tool_options --import ..."
-$gnulib_tool $gnulib_tool_options --import $gnulib_modules &&
+$gnulib_tool $gnulib_tool_options --import $gnulib_modules \
+  || die "gnulib-tool failed"
 
 for file in $gnulib_files; do
   symlink_to_dir "$GNULIB_SRCDIR" $file \
@@ -1005,6 +1026,6 @@ echo "$0: done.  Now you can run './configure'."
 # eval: (add-hook 'write-file-hooks 'time-stamp)
 # time-stamp-start: "scriptversion="
 # time-stamp-format: "%:y-%02m-%02d.%02H"
-# time-stamp-time-zone: "UTC"
+# time-stamp-time-zone: "UTC0"
 # time-stamp-end: "; # UTC"
 # End:

bootstrap.conf

@@ -1,6 +1,5 @@
 # bootstrap.conf - Bootstrap configuration.
-# Copyright (C) 2007, 2008, 2009, 2010, 2011, 2012 Free Software Foundation,
-# Inc.
+# Copyright (C) 2007-2012, 2018 Free Software Foundation, Inc.
 #
 # This file is part of GNU Wget.
 #
@@ -9,13 +8,13 @@
 # the Free Software Foundation, either version 3 of the License, or (at
 # your option) any later version.
 #
-# GNU Wgt is distributed in the hope that it will be useful, but
+# GNU Wget is distributed in the hope that it will be useful, but
 # WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 # General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
-# along with this program.  If not, see `http://www.gnu.org/licenses/'.
+# along with this program.  If not, see `https://www.gnu.org/licenses/'.
 
 # gnulib library name
 gnulib_name=libgnu
@@ -38,7 +37,9 @@ close
 connect
 dirname
 fcntl
+flock
 fnmatch
+fopen
 futimens
 ftello
 getaddrinfo
@@ -51,9 +52,17 @@ gettext
 gitlog-to-changelog
 git-version-gen
 gnupload
+group-member
+hostent
 ioctl
 iconv
 iconv-h
+inet_ntop
+intprops
+inttypes
+langinfo
+limits-h
+link
 listen
 maintainer-makefile
 mbiter
@@ -62,9 +71,16 @@ memrchr
 mkdir
 mkstemp
 mkostemp
+nanosleep
+crypto/md2
+crypto/md4
 crypto/md5
 crypto/sha1
 crypto/sha256
+crypto/sha512
+open
+pipe-posix
+posix_spawn
 quote
 quotearg
 recv
@@ -72,24 +88,37 @@ regex
 select
 send
 setsockopt
+signal-h
 sigprocmask
 sigpipe
 snprintf
 socket
 spawn-pipe
+stat
 stdbool
+stdint
 strcase
 strerror_r-posix
+strpbrk
 strptime
 strtok_r
 strtoll
+symlink
+sys_types
 timegm
 tmpdir
+unlink
 unlocked-io
 update-copyright
+libunistring-optional
+unistr/u8-strlen
+unicase/u8-tolower
+utimens
 vasprintf
 vsnprintf
+wcwidth
 write
+xmemdup0
 xstrndup
 "
 
@@ -117,4 +146,21 @@ rsync      -
 tar        -
 xz         -
 flex       -
+gperf      -
 "
+
+bootstrap_post_import_hook ()
+{
+  # We need to patch unicase/special-casing-table.h for gperf < 3.1
+  GPERF_VER=$(gperf --version | {
+    IFS=' .' read x y major minor bugfix _
+    printf "%d%02d%02d" $major $minor $bugfix
+  })
+
+  if [ $GPERF_VER -lt 30100 ]; then
+    echo "Creating lib/unicase/special-casing-table.h for gperf < 3.1"
+     gperf -m 10 lib/unicase/special-casing-table.gperf | \
+       sed -e "s/gl_unicase_special_lookup.*/gl_unicase_special_lookup\ \(const char \*str, size_t len\)/g" \
+         > lib/unicase/special-casing-table.h
+  fi
+}

build-aux/build_info.pl

@@ -2,7 +2,7 @@
 
 # Generate build_info.c.
 
-# Copyright (C) 2009, 2010, 2011 Free Software Foundation, Inc.
+# Copyright (C) 2009-2011, 2018 Free Software Foundation, Inc.
 
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by

cfg.mk

@@ -1,4 +1,7 @@
-export VC_LIST_EXCEPT_DEFAULT=^(lib/.*|m4/.*|md5/.*|build-aux/.*|src/gettext\.h|.*ChangeLog)$$
+# Explicit syntax-check exceptions.
+exclude_file_name_regexp--sc_trailing_blank = ^doc/annou.*$$
+
+export VC_LIST_EXCEPT_DEFAULT=^(lib/.*|m4/.*|md5/.*|build-aux/.*|src/gettext\.h|.*ChangeLog|tests/certs/.*)$$
 
 local-checks-to-skip = \
     sc_const_long_option \
@@ -21,9 +24,10 @@ local-checks-to-skip = \
     sc_program_name \
     sc_makefile_at_at_check \
     sc_cast_of_argument_to_free \
+    sc_prohibit_undesirable_word_seq
 
 #SHELL=bash -x
 show-vc-list-except:
 	@$(VC_LIST_EXCEPT)
 
-VC_LIST_ALWAYS_EXCLUDE_REGEX = ^ABOUT-NLS|((msdos|vms)/.*)|ChangeLog-2014-12-10$$
+VC_LIST_ALWAYS_EXCLUDE_REGEX = ^ABOUT-NLS|((msdos|vms)/.*)|ChangeLog-2014-12-10|/.*.der|^tests/certs/.*$$

configure.ac

@@ -1,6 +1,6 @@
 dnl Template file for GNU Autoconf
-dnl Copyright (C) 1995, 1996, 1997, 2001, 2007, 2008, 2009, 2010, 2011, 2012,
-dnl 2013, 2014 Free Software Foundation, Inc.
+dnl Copyright (C) 1995-1997, 2001, 2007-2014, 2018 Free Software
+dnl Foundation, Inc.
 
 dnl This program is free software; you can redistribute it and/or modify
 dnl it under the terms of the GNU General Public License as published by
@@ -57,6 +57,8 @@ AC_CANONICAL_HOST
 AC_DEFINE_UNQUOTED([OS_TYPE], "$host_os",
                    [Define to be the name of the operating system.])
 
+dnl Non-verbose make
+m4_ifdef([AM_SILENT_RULES], [AM_SILENT_RULES([yes])])
 
 dnl
 dnl Process External Libraries
@@ -70,7 +72,6 @@ dnl SSL: Configure SSL backend to use
 AC_ARG_WITH([ssl],
   [AS_HELP_STRING([--with-ssl={gnutls,openssl}], [specify SSL backend. GNU TLS is the default.])])
 
-
 dnl Zlib: Configure use of zlib for compression
 AC_ARG_WITH([zlib],
   [AS_HELP_STRING([--without-zlib], [disable zlib.])])
@@ -79,6 +80,9 @@ dnl Metalink: Configure use of the Metalink library
 AC_ARG_WITH([metalink],
   [AS_HELP_STRING([--with-metalink], [enable support for metalinks.])])
 
+dnl C-Ares: Configure use of the c-ares library for DNS lookup
+AC_ARG_WITH(cares, AS_HELP_STRING([--with-cares], [enable support for C-Ares DNS lookup.]), with_cares=$withval, with_cares=no)
+
 dnl
 dnl Process features
 dnl
@@ -180,7 +184,7 @@ dnl
 dnl Gettext
 dnl
 AM_GNU_GETTEXT([external],[need-ngettext])
-AM_GNU_GETTEXT_VERSION([0.17])
+AM_GNU_GETTEXT_VERSION([0.18.1])
 
 AC_PROG_RANLIB
 
@@ -286,15 +290,6 @@ AC_DEFINE([HAVE_STRNCASECMP], 1, [Define to 1 if you have the `strncasecmp' func
 AC_DEFINE([HAVE_STRDUP], 1, [Define to 1 if you have the `strdup' function.])
 AC_DEFINE([HAVE_ISATTY], 1, [Define to 1 if you have the `isatty' function.])
 
-dnl
-dnl Call Wget-specific macros defined in aclocal.
-dnl
-WGET_STRUCT_UTIMBUF
-WGET_FNMATCH
-WGET_NANOSLEEP
-WGET_POSIX_CLOCK
-WGET_NSL_SOCKET
-
 dnl Deal with specific hosts
 case $host_os in
   *mingw32* )
@@ -325,6 +320,10 @@ AS_IF([test "x$with_libpsl" != xno], [
   ])
 ])
 
+AS_IF([test x"$with_libpsl" = xyes], [
+  AC_CHECK_FUNCS(psl_latest)
+])
+
 AS_IF([test x"$with_zlib" != xno], [
   with_zlib=yes
   PKG_CHECK_MODULES([ZLIB], zlib, [
@@ -631,9 +630,6 @@ AC_ARG_ENABLE(iri,
   ]
 )
 
-AC_ARG_WITH(libidn, AC_HELP_STRING([--with-libidn=[DIR]],
-                                   [Support IDN/IRIs (needs GNU Libidn)]),
-                                   libidn=$withval, libidn="")
 AS_IF([test "X$iri" != "Xno"],[
   AM_ICONV
 
@@ -645,39 +641,23 @@ AS_IF([test "X$iri" != "Xno"],[
       AC_MSG_NOTICE([disabling IRIs because libiconv wasn't found])
     fi
   fi
-],[  # else
-  # For some reason, this seems to be set even when we don't check.
-  # Explicitly unset.
-  LIBICONV=
 ])
 
+AC_ARG_WITH(libidn, AC_HELP_STRING([--with-libidn=[DIR]],
+                                   [Support IDN2008/IRIs (needs GNU libidn2 + libunicode)]),
+                                   libidn=$withval, libidn="")
 if test "X$iri" != "Xno"; then
-  if test "$libidn" != ""; then
-    LDFLAGS="${LDFLAGS} -L$libidn/lib"
-    CPPFLAGS="${CPPFLAGS} -I$libidn/include"
-  fi
+  AS_IF([test "x$with_libidn2" != xno], [
+    AC_SEARCH_LIBS(idn2_lookup_u8, idn2,
+      [with_libidn2=yes; AC_DEFINE([ENABLE_IRI], 1, [Define if IRI support is enabled.])],
+      [with_libidn2=no; iri=no; AC_MSG_WARN(*** LIBIDN2 was not found. You will not be able to use IDN2008 support)])
 
-  # If idna.h can't be found, check to see if it was installed under
-  # /usr/include/idn (OpenSolaris, at least, places it there).
-  # Check for idn-int.h in that case, because idna.h won't find
-  # idn-int.h until we've decided to add -I/usr/include/idn.
-  AC_CHECK_HEADER(idna.h, ,
-    [AC_CHECK_HEADER(idn/idn-int.h,
-                    [CPPFLAGS="${CPPFLAGS} -I/usr/include/idn"],
-                    [iri=no])]
-  )
-
-  if test "X$iri" != "Xno"; then
-    AC_CHECK_LIB(idn, stringprep_check_version,
-      [iri=yes LIBS="${LIBS} -lidn"], iri=no)
-  fi
-
-  if test "X$iri" != "Xno" ; then
-    AC_DEFINE([ENABLE_IRI], 1, [Define if IRI support is enabled.])
-    AC_MSG_NOTICE([Enabling support for IRI.])
-  else
-    AC_MSG_WARN([Libidn not found])
-  fi
+#    AS_IF([test "x$with_libidn2" = xyes], [
+#      AC_SEARCH_LIBS(u8_tolower, unistring,
+#        [AC_DEFINE([ENABLE_IRI], 1, [Define if IRI support is enabled.])],
+#        [iri=no; AC_MSG_WARN(*** LIBUNISTRING was not found. You will not be able to use IDN2008 support)])
+#    ])
+  ])
 fi
 
 dnl
@@ -742,18 +722,70 @@ AS_IF([test "X$enable_pcre" != "Xno"],[
   ])
 ])
 
+dnl
+dnl Check for libcares (resolver library)
+dnl
+
+AS_IF([test "X$with_cares" = "Xyes"],[
+  PKG_CHECK_MODULES([CARES], libcares, [
+    CFLAGS="$CARES_CFLAGS $CFLAGS"
+    AC_CHECK_HEADER(ares.h, [
+      LIBS="$CARES_LIBS $LIBS"
+      AC_DEFINE([HAVE_LIBCARES], [1], [Define if libcares is available.])
+      RESOLVER_INFO="libcares, --bind-dns-address and --dns-servers available"
+    ])
+  ], [
+    AC_CHECK_HEADER(ares.h, [
+      AC_CHECK_LIB(cares, ares_set_local_ip4, [
+        LIBS="-lcares ${LIBS}"
+        AC_DEFINE([HAVE_LIBCARES], 1, [Define if libcares is available.])
+        RESOLVER_INFO="libcares, --bind-dns-address and --dns-servers available"
+      ])
+    ])
+  ])
+], [
+  RESOLVER_INFO="libc, --bind-dns-address and --dns-servers not available"
+])
+
+dnl
+dnl Extended Attribute support
+dnl
+
+AC_ARG_ENABLE([xattr],
+  [AS_HELP_STRING([--disable-xattr], [disable support for POSIX Extended Attributes])],
+  [ENABLE_XATTR=$enableval],
+  [ENABLE_XATTR=yes])
+
+case "$host_os" in
+  *linux* | *darwin*) xattr_syscalls="fsetxattr" ;;
+  freebsd*)           xattr_syscalls="extattr_set_fd" ;;
+  *)  AC_MSG_NOTICE([Disabling Extended Attribute support: your system is not known to support extended attributes.])
+      ENABLE_XATTR=no
+esac
+
+if test "X${ENABLE_XATTR}" = "Xyes"; then
+  AC_CHECK_FUNCS([$xattr_syscalls], [], [
+    AC_MSG_NOTICE([Disabling Extended Attribute support: your system does not support $xattr_syscalls])
+    ENABLE_XATTR=no
+  ])
+fi
+
+test "X${ENABLE_XATTR}" = "Xyes" && AC_DEFINE([ENABLE_XATTR], 1,
+    [Define if you want file meta-data storing into POSIX Extended Attributes compiled in.])
 
 dnl Needed by src/Makefile.am
 AM_CONDITIONAL([IRI_IS_ENABLED], [test "X$iri" != "Xno"])
 AM_CONDITIONAL([WITH_SSL], [test "X$with_ssl" != "Xno"])
 AM_CONDITIONAL([METALINK_IS_ENABLED], [test "X$with_metalink" != "Xno"])
+AM_CONDITIONAL([WITH_XATTR], [test "X$ENABLE_XATTR" != "Xno"])
 
 dnl
 dnl Create output
 dnl
 AC_CONFIG_FILES([Makefile src/Makefile doc/Makefile util/Makefile
                  po/Makefile.in tests/Makefile
-                 lib/Makefile testenv/Makefile])
+                 lib/Makefile testenv/Makefile
+                 tests/certs/interca.conf tests/certs/rootca.conf])
 AC_CONFIG_HEADERS([src/config.h])
 AC_OUTPUT
 
@@ -772,9 +804,12 @@ AC_MSG_NOTICE([Summary of build options:
   Digest:            $ENABLE_DIGEST
   NTLM:              $ENABLE_NTLM
   OPIE:              $ENABLE_OPIE
+  POSIX xattr:       $ENABLE_XATTR
   Debugging:         $ENABLE_DEBUG
   Assertions:        $ENABLE_ASSERTION
   Valgrind:          $VALGRIND_INFO
   Metalink:          $with_metalink
+  Resolver:          $RESOLVER_INFO
   GPGME:             $have_gpg
+  IRI:               $iri
 ])

contrib/check-hard

@@ -12,7 +12,8 @@ if [ $CORES -lt 1 ]; then CORES=1; fi
 echo "Running: make distclean"
 make distclean > /dev/null || true
 
-CFLAGS="-std=c89 -pedantic -O2 -g -Wall -Wextra -Wstrict-prototypes -Wold-style-definition -Wwrite-strings -Wshadow -Wformat -Wformat-security -Wunreachable-code -Wstrict-prototypes -Wmissing-prototypes -Wold-style-definition"
+#CFLAGS="-std=c89 -pedantic -O0 -g -Wall -Wextra -Wstrict-prototypes -Wold-style-definition -Wwrite-strings -Wshadow -Wformat -Wformat-security -Wunreachable-code -Wstrict-prototypes -Wmissing-prototypes -Wold-style-definition"
+CFLAGS="-Wall -Wextra -O2 -g"
 
 CACHEFILE=$PWD/config_check.cache
 
@@ -30,7 +31,7 @@ for CC in gcc clang; do
   # the compiler changed, so we have to remove the cache file here
   rm -f $CACHEFILE
 
-  for options in "" "--with-ssl=openssl"; do
+  for options in "" "--with-ssl=openssl" "--disable-iri"; do
     export DISTCHECK_CONFIGURE_FLAGS="-C --cache-file=$CACHEFILE $options"
     echo "  ./configure $DISTCHECK_CONFIGURE_FLAGS"
     ./configure $DISTCHECK_CONFIGURE_FLAGS CFLAGS="$CFLAGS" > /dev/null

contrib/release

@@ -0,0 +1,123 @@
+#!/bin/bash -e
+#
+# Written 2016 by Tim Ruehsen
+#
+# To get the version number right: add a signed tag and
+#   make sure the working dir is clean (git diff).
+# If that fails, put the version into '.tarball-version'.
+
+export CFLAGS="-g -O2 -Wall"
+
+echo
+echo "* Bootstrap"
+#./bootstrap >/dev/null
+
+echo
+echo "* Configure"
+#./configure -q
+
+extensions="gz lz"
+app_name=$(grep "^PACKAGE_NAME='" configure|cut -d "'" -f2)
+app_ver=$(grep "^PACKAGE_VERSION='" configure|cut -d "'" -f2)
+
+echo "Current version of $app_name is $app_ver"
+echo
+echo -n "Is that OK ? [y/N] "
+read x
+[ "$x" != "y" -a "$x" != "Y" ] && exit 0
+
+echo
+grep -i "Changes in $app_name" NEWS|head -1
+echo -n "Did you edit the NEWS file ? [y/N] "
+read x
+[ "$x" != "y" -a "$x" != "Y" ] && exit 0
+
+echo
+echo "* Make"
+#make -s clean
+#make -s
+
+echo
+echo "* Make distcheck"
+#make -s distcheck >/dev/null
+
+echo
+echo "* Make dist"
+for ext in $extensions; do
+  [[ $ext = 'gz' ]] && ext="gzip"
+  [[ $ext = 'lz' ]] && ext="lzip"
+  make -s dist-$ext >/dev/null
+done
+
+while true; do
+  echo "Alpha or Regular release ? [R/A] "
+  read x
+  if test "$x" = "r" || test "$x" = "R"; then
+    directory="/incoming/ftp"
+    break
+  elif test "$x" = "a" || test "$x" = "A"; then
+    directory="/incoming/alpha"
+    break
+  fi
+done
+
+x=$(gpg --list-secret-keys 2>/dev/null|grep ^sec|awk '{ print $2 }')
+echo $x
+if [ "$x" = "" ]; then
+  echo No signing key found - please check with 'gpg --list-secret-keys'
+  exit 1
+fi
+
+# arrays just work in Bash
+keys=($x)
+if [ ${#keys[@]} -eq 1 ]; then
+  IFS='/' arr=($x)
+  signing_key=${arr[1]}
+else
+  for ((i=0; i < ${#keys[@]}; i++)); do
+    echo "$((i+1))) ${keys[i]}"
+  done
+  while true; do
+    echo -n "Enter 1-${#keys[@]}: "
+    read x
+
+    # check for digits only
+    if [ -n "${i//[0-9]/}" ]; then continue; fi
+
+    if test $x -ge 1 || test $x -le ${#keys[@]}; then
+      IFS='/' arr=(${keys[$((x-1))]})
+      signing_key=${arr[1]}
+      break
+    fi
+  done
+fi
+unset IFS
+
+echo
+echo -n "Really upload to ftp-upload.gnu.org/$directory ? "
+read x
+[ "$x" != "y" -a "$x" != "Y" ] && exit 0
+
+for ext in $extensions; do
+  app_tarball="${app_name}-${app_ver}.tar.$ext"
+  gpg --default-key $signing_key --yes -b $app_tarball
+
+  echo -e "version: 1.2\nfilename: ${app_tarball}\ndirectory: ${app_name}\n" >${app_tarball}.directive
+  gpg --default-key $signing_key --yes --clearsign ${app_tarball}.directive
+
+ftp -v -n -i -p ftp-upload.gnu.org <<EOF
+user anonymous
+cd $directory
+put ${app_tarball}
+put ${app_tarball}.sig
+put ${app_tarball}.directive.asc
+EOF
+
+done
+
+prev_tag=`git tag|tail -2|head -1`
+echo "Don't forget to mention the following contributors in the announcement:"
+(
+  git log ${prev_tag}.. --format="%aN" --reverse
+  git log ${prev_tag}.. --reverse|grep -i reported|cut -d':' -f2|cut -d'<' -f1|cut -d' ' -f2-|tr -d '"'
+) | sort -u

contrib/travis-ci

@@ -0,0 +1,60 @@
+#!/bin/bash
+
+# Set a stricter bash mode
+set -e
+set -u
+
+# Set up the subset of tests this Matrix instance will run.
+
+# One instance is forked for each of the SSL library possibilities
+SSL=${1:-""}
+case $SSL in
+    "") SSL_LIB="--without-ssl";;
+    "openssl") SSL_LIB="--with-ssl=openssl";;
+    "gnutls") SSL_LIB="--with-ssl=gnutls";;
+esac
+
+# For some reason. /proc/cpuinfo reports 16 cores on Travis, while the docs
+# claim that each instance has only 2 cores. We believe the docs and force a
+# value of n+1 here.
+CORES=3
+
+# Define a large number of Warning flags for the compiler. Hopefully, someone
+# will sit and analyze the output to clean the warnings from the codebase.
+CFLAGS="-std=c89 -pedantic -O0 -Wall -Wextra -Wstrict-prototypes -Wold-style-definition -Wwrite-strings -Wshadow -Wformat -Wformat-security -Wunreachable-code -Wstrict-prototypes -Wmissing-prototypes -Wold-style-definition"
+
+# A cachefile for ./configure. Saves significant time when reconfiguring
+CACHEFILE=$PWD/config_check.cache
+
+# measure time consumed and print it at the end of the script
+START=$(date +%s)
+
+# Test different Permutations of Wget. Always under Valgrind
+#
+# Options:
+#   - All Features Enabled
+#     - C       [Default Locale]
+#     - Russian [Multibyte / Multicolumn Locale]
+#     - Turkish [Special Test due to issues with language]
+#   - IPv6 Disabled
+#     - C
+for options in "" "--disable-ipv6"; do
+  export DISTCHECK_CONFIGURE_FLAGS="-C --cache-file=$CACHEFILE --enable-assert --enable-valgrind-tests $SSL_LIB $options"
+  echo "  ./configure $DISTCHECK_CONFIGURE_FLAGS CFLAGS=\"$CFLAGS\""
+  ./configure $DISTCHECK_CONFIGURE_FLAGS CFLAGS="$CFLAGS"
+
+  if [[ -z $options ]]; then
+      LANG_ARR=("ru_RU.UTF-8" "tr_TR.UTF-8" "C")
+  else
+      LANG_ARR=("C")
+  fi
+
+  for xLCALL in "${LANG_ARR[@]}"; do
+    export TESTS_ENVIRONMENT="LC_ALL=$xLCALL VALGRIND_TESTS=1"
+    echo "    TESTS_ENVIRONMENT=\"$TESTS_ENVIRONMENT\" make distcheck CFLAGS=$CFLAGS -j$CORES"
+    make distcheck CFLAGS="$CFLAGS" -j$CORES
+  done
+done
+
+END=$(date +%s)
+echo "Duration: $((END-START))"

doc/Makefile.am

@@ -1,6 +1,6 @@
 # Makefile for `wget' utility
-# Copyright (C) 1995, 1996, 1997, 2007, 2008, 2009, 2010, 2011, 2015
-# Free Software Foundation, Inc.
+# Copyright (C) 1995-1997, 2007-2011, 2015, 2018 Free Software
+# Foundation, Inc.
 
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -57,7 +57,8 @@ wget.pod: $(srcdir)/wget.texi version.texi
 	$(TEXI2POD) -D VERSION="$(VERSION)" $(srcdir)/wget.texi $@
 
 $(MAN): wget.pod
-	$(POD2MAN) --center="GNU Wget" --release="GNU Wget @VERSION@" --utf8 $? > $@
+	$(POD2MAN) --center="GNU Wget" --release="GNU Wget @VERSION@" --utf8 $? > $@ || \
+	$(POD2MAN) --center="GNU Wget" --release="GNU Wget @VERSION@" $? > $@
 
 #wget.cat: $(MAN)
 #	nroff -man $? > $@

doc/announcement_1.19.txt

@@ -0,0 +1,104 @@
+Hello,
+
+we are pleased to announce the new version of GNU wget 1.19.
+
+It comes with major improvements for Metalink, IDNA2008 for
+international domain names, an option to call external tools
+for fetching user/password, several bugfixes and improvements.
+
+Many thanks go to all the contributors and list activists !
+
+Contributors (from the git log):
+Ander Juaristi
+Dale R. Worley 
+Eli Zaretskii 
+Gisle Vanem 
+Giuseppe Scrivano
+Göran Uddeborg 
+Jann Horn 
+Jeffery To
+Jérémie Courrèges-Anglas 
+Liam R. Howlett
+losgrandes
+Matthew White
+Misra, Deapesh 
+Nikos Mavrogiannopoulos
+Noël Köthe
+Piotr Wajda
+Rahul Bedarkar
+Sean Burford
+Tim Rühsen
+Tobias Stoeckmann
+vijeth-aradhya
+
+
+The new version is available for download here:
+
+https://ftp.gnu.org/gnu/wget/wget-1.19.tar.gz
+https://ftp.gnu.org/gnu/wget/wget-1.19.tar.xz
+
+and the GPG detached signatures using the key 0x08302DB6A2670428:
+
+https://ftp.gnu.org/gnu/wget/wget-1.19.tar.gz.sig
+https://ftp.gnu.org/gnu/wget/wget-1.19.tar.xz.sig
+
+To reduce load on the main server, you can use this redirector service
+which automatically redirects you to a mirror:
+
+http://ftpmirror.gnu.org/wget/wget-1.19.tar.gz
+http://ftpmirror.gnu.org/wget/wget-1.19.tar.xz
+
+
+Noteworthy changes:
+
+* New option --use-askpass=COMMAND. Fetch user/password by calling
+  an external program.
+
+* Use IDNA2008 (+ TR46 if available) through libidn2
+
+* When processing a Metalink header, --metalink-index=<number> allows
+  to process the header's application/metalink4+xml files.
+
+* When processing a Metalink file, --trust-server-names enables the
+  use of the destination file names specified in the Metalink file,
+  otherwise a safe destination file name is computed.
+
+* When processing a Metalink file, enforce a safe destination path.
+  Remove any drive letter prefix under w32, i.e. 'C:D:file'.  Call
+  libmetalink's metalink_check_safe_path() to prevent absolute,
+  relative, or home paths:
+  https://tools.ietf.org/html/rfc5854#section-4.1.2.1
+  https://tools.ietf.org/html/rfc5854#section-4.2.8.3
+
+* When processing a Metalink file, --directory-prefix=<prefix> sets
+  the top of the retrieval tree to prefix for Metalink downloads.
+
+* When processing a Metalink file, reject downloaded files which don't
+  agree with their own metalink:size value:
+  https://tools.ietf.org/html/rfc5854#section-4.2.16
+
+* When processing a Metalink file, with --continue resume partially
+  downloaded files and keep fully downloaded files even if they fail
+  the verification.
+
+* When processing a Metalink file, create the parent directories of a
+  "path/file" destination file name:
+  https://tools.ietf.org/html/rfc5854#section-4.1.2.1
+  https://tools.ietf.org/html/rfc5854#section-4.2.8.3
+
+* On a recursive download, append a .tmp suffix to temporary files
+  that will be deleted after being parsed, and create them
+  readable/writable only by the owner.
+
+* New make target 'check-valgrind'
+
+* Fix several bugs
+
+* Fix compatibility issues
+
+
+Please report any problem you may experience to the bug-wget@gnu.org
+mailing list.
+
+For the maintainers of Wget,
+Tim

doc/fdl.texi

@@ -5,7 +5,7 @@
 @c hence no sectioning command or @node.
 
 @display
-Copyright @copyright{} 2000, 2001, 2002, 2007, 2008, 2015 Free Software
+Copyright @copyright{} 2000-2002, 2007-2008, 2015, 2018 Free Software
 Foundation, Inc.
 @uref{http://fsf.org/}
 

doc/metalink-standard.txt

@@ -0,0 +1,233 @@
+GNU Wget Metalink recommended behaviour
+
+  Metalink/XML and Metalink/HTTP standard reference
+
+
+1. Security features
+********************
+
+Only metalink:file elements with safe "name" fields shall be accepted
+[1 #section-4.1.2.1]. If unsafe metalink:file elements are saved, any
+related test shall fail (see '2. Tests').
+
+By design, libmetalink rejects unsafe metalink:file elements [3]:
+* lib/metalink_helper.c (metalink_check_safe_path): Verify path
+
+1.1 Exceptions
+==============
+
+The option --directory-prefix could allow to use an absolute, relative
+or home path.
+
+2. Tests
+********
+
+Saving a file to an unexpected path poses a security problem. We must
+ensure that Wget's automated tests never modify the root and the home
+paths or descend/escalate to a relative path unexpectedly.
+
+2.1 Metalink/XML implemented tests
+==================================
+
+See testenv/Makefile.am (METALINK_TESTS).
+
+2.2 Metalink/HTTP implemented tests
+===================================
+
+See testenv/Makefile.am (METALINK_TESTS).
+
+3. Download file name
+*********************
+
+The download file name shall be decided by precise rules which prevent
+any naming uncertainty and security issues.
+
+3.1 Naming rules
+================
+
+The final name of downloaded files is computed starting from a trusted
+name, which is then combined with the "Directory Options".  The result
+is verified and eventually made safer following security rules. If the
+final name isn't found safe enough, then the file isn't downloaded.
+
+Depending on the options used, a suffix could be appended to the final
+name to not overwrite existing files.
+
+3.1.1 The trusted name
+======================
+
+The option --trust-server-names decides what is the trusted name.
+
+Any Metalink/XML element with an unsafe metalink:file "name" field is
+ignored, see '1. Security features'.
+
+3.1.1.1 Without --trust-server-names
+====================================
+
+When --trust-server-names is off, the basename of the --input-metalink
+file, if available, or of the mother URL is trusted. This trusted name
+is the radix of any subsequent file name.
+
+When a Metalink/HTTP in encountered, any fetched Metalink/XML file has
+its own ordinal number appended as suffix to the trusted name. In this
+case scenario, an unique Metalink/XML file is saved each time applying
+an additional suffix to the currently computed name when necessary.
+
+The files described by a Metalink/XML file will be named sequentially
+applying an additional suffix to the currently trusted/computed name.
+
+3.1.1.2 With --trust-server-names
+=================================
+
+When --trust-server-names is on, the metalink:file "name" field parsed
+from Metalink/XML files is trusted. When no Metalink/XML is available,
+the mother URL is trusted.
+
+Any Metalink/HTTP application/metalink4+xml file is saved using the
+basename of its own Link header "name" field, if available.
+
+In conjunction with the option --content-disposition, a 'Content-Type:
+application/metalink4+xml' file is saved using the basename of its own
+Content-Disposition header "filename" field, if available.
+
+3.1.2 The final name
+====================
+
+The "Directory Options" are combined with the trusted name. The result
+is evaluated again by the '1. Security features'. If the path is found
+unsafe, only the basename of the final name is considered.  If this is
+found unsafe too, the file is not downloaded.
+
+4. Metalink/XML
+***************
+
+4.1 Example files
+=================
+
+See [1 #section-1.1].
+
+cat > bugus.meta4 << EOF
+<?xml version="1.0" encoding="UTF-8"?>
+<metalink xmlns="urn:ietf:params:xml:ns:metalink">
+  <file name="/dir/A/File1">
+    <size>1617</size>
+    <hash type="sha256">ecb3dff2648667513e31554b3ad054ccd89fce38e33367c9459ac3a285153742</hash>
+    <url>http://another.url/common_name</url>
+    <url>http://ftpmirror.gnu.org/bash/bash-4.3-patches/bash43-001</url>
+  </file>
+  <file name="dir/B/File2">
+    <size>1594</size>
+    <hash type="sha256">eee7cd7062ab29a9e4f02924d9c367264dcb8b162703f74ff6eb8f175a91502b</hash>
+    <url>http://another.url/again/common_name</url>
+    <url>http://ftpmirror.gnu.org/bash/bash-4.3-patches/bash43-002</url>
+  </file>
+</metalink>
+EOF
+
+4.2 Command line example
+========================
+
+$ wget --input-metalink=bogus.meta4
+
+4.3 Metalink/XML file parsing
+=============================
+
+The metalink xml file is parsed by one of the following libmetalink's
+functions [3], depending upon the library configured to use:
+* lib/libexpat_metalink_parser.c (metalink_parse_file): Expat [4]
+* lib/libxml2_metalink_parser.c (metalink_parse_file): Libxml2 [5]
+
+The result returned doesn't include unsafe metalink:file elements, as
+stated at point '1. Security features'.
+
+An empty result shall not be considered an error.  Parsing errors will
+be informed to the caller of libmetalink's metalink_parse_file().
+
+4.4 Saving files
+================
+
+Fetched metalink:file elements shall be wrote using the unique "name"
+field as file name [1 #section-4.1.2.1].
+
+A metalink:file url's file name shall not substitute the "name" field.
+
+Security exceptions are explained in '3. Download file name'.
+
+4.5 Multi-Source download
+=========================
+
+Parallel range requests are allowed [1 #section-1].
+
+5. Metalink/HTTP
+****************
+
+5.1 HTTP server
+===============
+
+The local server http://127.0.0.1 is used as reference in the course
+of this chapter. Any server service capable of sending Metalink/HTTP
+header answers may be used.
+
+5.2 Command line example
+========================
+
+$ wget --metalink-over-http http://127.0.0.1/dir/file.ext
+
+5.3 Metalink/HTTP header answer
+===============================
+
+See [2 #section-1.1].
+
+Etag: "thvDyvhfIqlvFe+A9MYgxAfm1q5="
+Link: <http://www2.example.com/example.ext>; rel=duplicate
+Link: <ftp://ftp.example.com/example.ext>; rel=duplicate
+Link: <http://example.com/example.ext.torrent>; rel=describedby;
+type="application/x-bittorrent"
+Link: <http://example.com/example.ext.meta4>; rel=describedby;
+type="application/metalink4+xml"
+Link: <http://example.com/example.ext.asc>; rel=describedby;
+type="application/pgp-signature"
+Digest: SHA-256=MWVkMWQxYTRiMzk5MDQ0MzI3NGU5NDEyZTk5OWY1ZGFmNzgyZTJlO
+DYzYjRjYzFhOTlmNTQwYzI2M2QwM2U2MQ==
+
+See [2 #section-4].
+
+Link: <http://example.com/example.ext.torrent>; rel=describedby;
+type="application/x-bittorrent"; name="differentname.ext"
+Link: <http://example.com/example.ext.meta4>; rel=describedby;
+type="application/metalink4+xml"
+
+5.4 Saving files
+================
+
+When none of --output-document and/or --content-disposition is used,
+the file name to wrote is computed from the cli's url hierarchy. The
+purpose of the "Directory Options" is as usual, and the file name is
+the cli's url file name, see wget(1).
+
+The url followed to download the file shall not substitute the cli's
+url to compute the file name to wrote, except when it redirects to a
+Metalink/XML file, following the rules in '3. Download file name'.
+
+5.5 Multi-Source download
+=========================
+
+Parallel range requests are allowed [2 #section-7].
+
+4. References
+*************
+
+[1] The Metalink Download Description Format
+https://tools.ietf.org/html/rfc5854
+
+[2] Metalink/HTTP: Mirrors and Hashes
+https://tools.ietf.org/html/rfc6249
+
+[3] Libmetalink
+https://github.com/metalink-dev/libmetalink
+
+[4] Expat
+http://www.libexpat.org
+
+[5] Libxml2
+http://xmlsoft.org

doc/metalink.txt

@@ -0,0 +1,165 @@
+GNU Wget Metalink module
+
+  Evaluation of the Metalink/XML and Metalink/HTTP implementations
+
+
+1. Introduction
+***************
+
+This document, and the results contained in it, is focused over the
+evaluation of the Metalink/XML and Metalink/HTTP implementations.
+
+The "Directory Options" mentioned here are used on the command line in
+conjunction with the option '--input-metalink=file' for Metalink/XML,
+and '--metalink-over-http' for Metalink/HTTP.
+
+$ wget --input-metalink=<file> [directory options]
+$ wget --metalink-over-http [directory options] <url>
+
+2. Notes
+********
+
+Tests for metalink:file names beginning with '/', '~/', './', or '../'
+(e.g. "/path/file") shall be run manually due to security concerns.
+
+3. Metalink files used as reference
+***********************************
+
+3.1 Test: metalink:file with "path/file" name format
+====================================================
+
+cat > test.meta4 << EOF
+<?xml version="1.0" encoding="UTF-8"?>
+<metalink xmlns="urn:ietf:params:xml:ns:metalink">
+  <file name="path/file">
+    <size>543</size>
+    <hash type="sha256">d37d3965f8e1a7b16504b4273b09c392776b7e4dd17e601256c7b2fd9ce5f56e</hash>
+    <hash type="md5">0f6ff5cdc15603f1b81227b5a296f001</hash>
+    <url>http://wrongurl.really/gnu/wget/wget-1.18.tar.xz.sig</url>
+    <url>http://ftpmirror.gnu.org/wget/wget-1.18.tar.xz.sig</url>
+    <url>http://ftp.gnu.org/gnu/wget/wget-1.18.tar.xz.sig</url>
+    <url>http://nl.mirror.babylon.network/gnu/wget/wget-1.18.tar.xz.sig</url>
+  </file>
+</metalink>
+EOF
+
+4. `wget --input-metalink=test.meta4`
+*************************************
+
+4.1 Implemented safety features
+===============================
+
+Any metalink:file name containing an absolute, relative, or home path
+(see '2. Notes') parsed from Metalink/XML files is rejected.
+
+This is a libmetalink's design decision implemented in the function
+metalink_check_safe_path().  This feature shall not be modified.
+
+All the above conform to the RFC5854 standard.
+
+References:
+ https://tools.ietf.org/html/rfc5854#section-4.1.2.1
+ https://tools.ietf.org/html/rfc5854#section-4.2.8.3
+
+4.2 File download behaviour
+===========================
+
+When a Metalink/XML file is parsed:
+1. create the metalink:file "path/file" tree;
+2. download the metalink:url file as "path/file";
+3. verify the "path/file" size, if declared;
+4. verify the "path/file" checksum.
+
+All the above conform to the RFC5854 standard.
+
+References:
+ https://tools.ietf.org/html/rfc5854
+
+4.3 Questionable behaviours
+===========================
+
+If more metalink:file elements are the same, wget downloads them all.
+
+5. `wget --metalink-over-http`
+******************************
+
+5.1 Implemented safety features
+===============================
+
+The function url_file_name() is responsible of parsing the url's file
+name and mixing in the "Directory Options" wrote on the command line.
+
+The use of libmetalink's metalink_check_safe_path() shouldn't be
+necessary (see '4.1 Implemented safety features').
+
+All the above comform to the usual Wget's download behaviour.
+
+References:
+ wget(1)
+
+5.2 File download behaviour
+===========================
+
+When a Metalink/HTTP header is parsed:
+1. extract metalink metadata from the header;
+2. download the file from the mirror with the highest priority;
+3. verify the file's size, if declared;
+4. verify the file's checksum.
+
+All the above comform to the usual Wget's download behaviour and to
+the RFC6249 standard.
+
+References:
+ wget(1)
+ https://tools.ietf.org/html/rfc6249
+
+6. Directory Options
+********************
+
+'-nd'
+'--no-directories'
+
+    Do not apply to Metalink/XML files (aka --input-metalink=<file>).
+
+    Apply to Metalink/HTTP urls as described in the Wget's manual, see
+    wget(1).  The target url is the url wrote on the command line.
+
+'-x'
+'--force-directories'
+
+    Do not apply to Metalink/XML files (aka --input-metalink=<file>).
+
+    Apply to Metalink/HTTP urls as described in the Wget's manual, see
+    wget(1).  The target url is the url wrote on the command line.
+
+'-nH'
+'--no-host-directories'
+
+    Do not apply to Metalink/XML files (aka --input-metalink=<file>).
+
+    Apply to Metalink/HTTP urls as described in the Wget's manual, see
+    wget(1).  The target url is the url wrote on the command line.
+
+'--protocol-directories'
+
+    Do not apply to Metalink/XML files (aka --input-metalink=<file>).
+
+    Apply to Metalink/HTTP urls as described in the Wget's manual, see
+    wget(1).  The target url is the url wrote on the command line.
+
+'--cut-dirs=number'
+
+    Do not apply to Metalink/XML files (aka --input-metalink=<file>).
+
+    Apply to Metalink/HTTP urls as described in the Wget's manual, see
+    wget(1).  The target url is the url wrote on the command line.
+
+'-P prefix'
+'--directory-prefix=prefix'
+
+    Set the top of the retrieval tree to prefix for both Metalink/XML
+    and Metalink/HTTP downloads, see wget(1).
+
+    If combining the prefix with the file name results in an absolute,
+    relative, or home path, the directory components are stripped and
+    only the basename is used. See '4.1 Implemented safety features'.

doc/texi2pod.pl

@@ -1,7 +1,7 @@
 #! /usr/bin/env perl
 
-#   Copyright (C) 1999, 2000, 2001, 2003, 2007, 2009, 2010, 2011, 2015
-#   Free Software Foundation, Inc.
+#   Copyright (C) 1999-2001, 2003, 2007, 2009-2011, 2015, 2018 Free
+#   Software Foundation, Inc.
 
 # This file is part of GCC.
 

doc/wget.texi

@@ -31,9 +31,8 @@ This file documents the GNU Wget utility for downloading network
 data.
 
 @c man begin COPYRIGHT
-Copyright @copyright{} 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003,
-2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2015 Free Software
-Foundation, Inc.
+Copyright @copyright{} 1996-2011, 2015, 2018 Free Software Foundation,
+Inc.
 
 @iftex
 Permission is granted to make and distribute verbatim copies of
@@ -193,6 +192,8 @@ options, or via the initialization file @file{.wgetrc} (@pxref{Startup
 File}).  Wget allows you to define @dfn{global} startup files
 (@file{/usr/local/etc/wgetrc} by default) for site settings. You can also
 specify the location of a startup file with the --config option.
+To disable the reading of config files, use --no-config.
+If both --config and --no-config are given, --no-config is ignored.
  
 
 @ignore
@@ -512,11 +513,27 @@ href if none was specified.
 Downloads files covered in local Metalink @var{file}. Metalink version 3
 and 4 are supported.
 
+@cindex keep-badhash
+@item --keep-badhash
+Keeps downloaded Metalink's files with a bad hash. It appends .badhash
+to the name of Metalink's files which have a checksum mismatch, except
+without overwriting existing files.
+
 @cindex metalink-over-http
 @item --metalink-over-http
 Issues HTTP HEAD request instead of GET and extracts Metalink metadata
 from response headers. Then it switches to Metalink download.
 If no valid Metalink metadata is found, it falls back to ordinary HTTP download.
+Enables @samp{Content-Type: application/metalink4+xml} files download/processing.
+
+@cindex metalink-index
+@item --metalink-index=@var{number}
+Set the Metalink @samp{application/metalink4+xml} metaurl ordinal
+NUMBER. From 1 to the total number of ``application/metalink4+xml''
+available.  Specify 0 or @samp{inf} to choose the first good one.
+Metaurls, such as those from a @samp{--metalink-over-http}, may have
+been sorted by priority key's value; keep this in mind to choose the
+right NUMBER.
 
 @cindex preferred-location
 @item --preferred-location
@@ -550,7 +567,10 @@ would be resolved to @samp{http://foo/baz/b.html}.
 
 @cindex specify config 
 @item --config=@var{FILE}
-Specify the location of a startup file you wish to use.
+Specify the location of a startup file you wish to use instead of the
+default one(s). Use --no-config to disable reading of config files.
+If both --config and --no-config are given, --no-config is ignored.
+
 
 @item --rejected-log=@var{logfile}
 Logs all URL rejections to @var{logfile} as comma separated values.  The values
@@ -571,6 +591,27 @@ the local machine.  @var{ADDRESS} may be specified as a hostname or IP
 address.  This option can be useful if your machine is bound to multiple
 IPs.
 
+@cindex bind DNS address
+@cindex client DNS address
+@cindex DNS IP address, client, DNS
+@item --bind-dns-address=@var{ADDRESS}
+[libcares only]
+This address overrides the route for DNS requests. If you ever need to
+circumvent the standard settings from /etc/resolv.conf, this option together
+with @samp{--dns-servers} is your friend.
+@var{ADDRESS} must be specified either as IPv4 or IPv6 address.
+Wget needs to be built with libcares for this option to be available.
+
+@cindex DNS server
+@cindex DNS IP address, client, DNS
+@item --dns-servers=@var{ADDRESSES}
+[libcares only]
+The given address(es) override the standard nameserver
+addresses,  e.g. as configured in /etc/resolv.conf.
+@var{ADDRESSES} may be specified either as IPv4 or IPv6 addresses,
+comma-separated.
+Wget needs to be built with libcares for this option to be available.
+
 @cindex retries
 @cindex tries
 @cindex number of tries
@@ -666,6 +707,12 @@ Before (over)writing a file, back up an existing file by adding a
 files are rotated to @samp{.2}, @samp{.3}, and so on, up to
 @var{backups} (and lost beyond that).
 
+@cindex authentication credentials
+@item --no-netrc
+Do not try to obtain credentials from @file{.netrc} file. By default
+@file{.netrc} file is searched for credentials in case none have been
+passed on command line and authentication is required.
+
 @cindex continue retrieval
 @cindex incomplete downloads
 @cindex resume download
@@ -694,13 +741,11 @@ Without @samp{-c}, the previous example would just download the remote
 file to @file{ls-lR.Z.1}, leaving the truncated @file{ls-lR.Z} file
 alone.
 
-Beginning with Wget 1.7, if you use @samp{-c} on a non-empty file, and
-it turns out that the server does not support continued downloading,
-Wget will refuse to start the download from scratch, which would
-effectively ruin existing contents.  If you really want the download to
-start from scratch, remove the file.
+If you use @samp{-c} on a non-empty file, and the server does not support
+continued downloading, Wget will restart the download from scratch and overwrite
+the existing file entirely.
 
-Also beginning with Wget 1.7, if you use @samp{-c} on a file which is of
+Beginning with Wget 1.7, if you use @samp{-c} on a file which is of
 equal size as the one on the server, Wget will refuse to download the
 file and print an explanatory message.  The same happens when the file
 is smaller on the server than locally (presumably because it was changed
@@ -783,7 +828,7 @@ With @samp{--progress=bar}, there are currently two possible parameters,
 @var{force} and @var{noscroll}.
 
 When the output is not a TTY, the progress bar always falls back to ``dot'',
-even if @samp{--progress=bar} was passed to Wget during invokation. This
+even if @samp{--progress=bar} was passed to Wget during invocation. This
 behaviour can be overridden and the ``bar'' output forced by using the ``force''
 parameter as @samp{--progress=bar:force}.
 
@@ -983,7 +1028,7 @@ specified in bytes (default), kilobytes (with @samp{k} suffix), or
 megabytes (with @samp{m} suffix).
 
 Note that quota will never affect downloading a single file.  So if you
-specify @samp{wget -Q10k ftp://wuarchive.wustl.edu/ls-lR.gz}, all of the
+specify @samp{wget -Q10k https://example.com/ls-lR.gz}, all of the
 @file{ls-lR.gz} will be downloaded.  The same goes even when several
 @sc{url}s are specified on the command-line.  However, quota is
 respected when retrieving either recursively, or from an input file.
@@ -1135,6 +1180,15 @@ options for @sc{http} connections.
 Prompt for a password for each connection established. Cannot be specified
 when @samp{--password} is being used, because they are mutually exclusive.
 
+@item --use-askpass=@var{command}
+Prompt for a user and password using the specified command.  If no command is
+specified then the command in the environment variable WGET_ASKPASS is used.
+If WGET_ASKPASS is not set then the command in the environment variable
+SSH_ASKPASS is used.
+
+You can set the default command for use-askpass in the @file{.wgetrc}.  That
+setting may be overridden from the command line.
+
 @cindex iri support
 @cindex idn support
 @item --no-iri
@@ -1291,6 +1345,11 @@ renamed from @samp{--html-extension}, to better reflect its new
 behavior. The old option name is still acceptable, but should now be
 considered deprecated.
 
+As of version 1.19.2, Wget will also ensure that any downloaded files with
+a @code{Content-Encoding} of @samp{br}, @samp{compress}, @samp{deflate}
+or @samp{gzip} end in the suffix @samp{.br}, @samp{.Z}, @samp{.zlib}
+and @samp{.gz} respectively.
+
 At some point in the future, this option may well be expanded to
 include suffixes for other types of content, including content types
 that are not parsed by Wget.
@@ -1308,10 +1367,10 @@ the @code{digest}, or the Windows @code{NTLM} authentication scheme.
 Another way to specify username and password is in the @sc{url} itself
 (@pxref{URL Format}).  Either method reveals your password to anyone who
 bothers to run @code{ps}.  To prevent the passwords from being seen,
-store them in @file{.wgetrc} or @file{.netrc}, and make sure to protect
-those files from other users with @code{chmod}.  If the passwords are
-really important, do not leave them lying in those files either---edit
-the files and delete them after Wget has started the download.
+use the @samp{--use-askpass} or store them in @file{.wgetrc} or @file{.netrc},
+and make sure to protect those files from other users with @code{chmod}.  If
+the passwords are really important, do not leave them lying in those files
+either---edit the files and delete them after Wget has started the download.
 
 @iftex
 @xref{Security Considerations}, for more information about security
@@ -1475,6 +1534,23 @@ wget --header="Host: foo.bar" http://localhost/
 In versions of Wget prior to 1.10 such use of @samp{--header} caused
 sending of duplicate headers.
 
+@cindex Content-Encoding, choose
+@item --compression=@var{type}
+Choose the type of compression to be used.  Legal values are
+@samp{auto}, @samp{gzip} and @samp{none}.
+
+If @samp{auto} or @samp{gzip} are specified, Wget asks the server to
+compress the file using the gzip compression format. If the server
+compresses the file and responds with the @code{Content-Encoding}
+header field set appropriately, the file will be decompressed
+automatically.
+
+If @samp{none} is specified, wget will not ask the server to compress
+the file and will not decompress any server responses. This is the default.
+
+Compression support is currently experimental. In case it is turned on,
+please report any bugs to @code{bug-wget@@gnu.org}.
+
 @cindex redirect
 @item --max-redirect=@var{number}
 Specifies the maximum number of redirections to follow for a resource.
@@ -1586,11 +1662,11 @@ users:
 # @r{Log in to the server.  This can be done only once.}
 wget --save-cookies cookies.txt \
      --post-data 'user=foo&password=bar' \
-     http://server.com/auth.php
+     http://example.com/auth.php
 
 # @r{Now grab the page or pages we care about.}
 wget --load-cookies cookies.txt \
-     -p http://server.com/interesting/article.php
+     -p http://example.com/interesting/article.php
 @end group
 @end example
 
@@ -1643,6 +1719,10 @@ This option is useful for some file-downloading CGI programs that use
 @code{Content-Disposition} headers to describe what the name of a
 downloaded file should be.
 
+When combined with @samp{--metalink-over-http} and @samp{--trust-server-names},
+a @samp{Content-Type: application/metalink4+xml} file is named using the
+@code{Content-Disposition} filename field, if available.
+
 @cindex Content On Error
 @item --content-on-error
 
@@ -1652,9 +1732,11 @@ with a http status code that indicates error.
 @cindex Trust server names
 @item --trust-server-names
 
-If this is set to on, on a redirect the last component of the
-redirection URL will be used as the local file name.  By default it is
-used the last component in the original URL.
+If this is set, on a redirect, the local file name will be based
+on the redirection URL.  By default the local file name is based on
+the original URL.  When doing recursive retrieving this can be helpful
+because in many web sites redirected URLs correspond to an underlying
+file structure, while link URLs do not.
 
 @cindex authentication
 @item --auth-no-challenge
@@ -1668,6 +1750,21 @@ some few obscure servers, which never send HTTP authentication
 challenges, but accept unsolicited auth info, say, in addition to
 form-based authentication.
 
+@item --retry-on-http-error=@var{code[,code,...]}
+Consider given HTTP response codes as non-fatal, transient errors.
+Supply a comma-separated list of 3-digit HTTP response codes as
+argument. Useful to work around special circumstances where retries
+are required, but the server responds with an error code normally not
+retried by Wget. Such errors might be 503 (Service Unavailable) and
+429 (Too Many Requests). Retries enabled by this option are performed
+subject to the normal retry timing and retry count limitations of
+Wget.
+
+Using this option is intended to support special use cases only and is
+generally not recommended, as it can force retries even in cases where
+the server is actually trying to decrease its load. Please use wisely
+and only if you know what you are doing.
+
 @end table
 
 @node HTTPS (SSL/TLS) Options, FTP Options, HTTP Options, Invoking
@@ -1725,6 +1822,12 @@ this option to bypass the verification and proceed with the download.
 site's authenticity, or if you really don't care about the validity of
 its certificate.}  It is almost always a bad idea not to check the
 certificates when transmitting confidential or important data.
+For self-signed/internal certificates, you should download the certificate
+and verify against that instead of forcing this insecure mode.
+If you are really sure of not desiring any certificate verification, you
+can specify --check-certificate=quiet to tell wget to not print any
+warning about invalid certificates, albeit in most cases this is the
+wrong thing to do.
 
 @cindex SSL certificate
 @item --certificate=@var{file}
@@ -1772,6 +1875,18 @@ system-specified locations, chosen at OpenSSL installation time.
 Specifies a CRL file in @var{file}.  This is needed for certificates
 that have been revocated by the CAs.
 
+@cindex SSL Public Key Pin
+@item --pinnedpubkey=file/hashes
+Tells wget to use the specified public key file (or hashes) to verify the peer.
+This can be a path to a file which contains a single public key in PEM or DER
+format, or any number of base64 encoded sha256 hashes preceded by ``sha256//''
+and separated by ``;''
+
+When negotiating a TLS or SSL connection, the server sends a certificate
+indicating its identity. A public key is extracted from this certificate and if
+it does not exactly match the public key(s) provided to this option, wget will
+abort the connection before sending or receiving any data.
+
 @cindex entropy, specifying source of
 @cindex randomness, specifying source of
 @item --random-file=@var{file}
@@ -2555,11 +2670,11 @@ The @samp{-D} option allows you to specify the domains that will be
 followed, thus limiting the recursion only to the hosts that belong to
 these domains.  Obviously, this makes sense only in conjunction with
 @samp{-H}.  A typical example would be downloading the contents of
-@samp{www.server.com}, but allowing downloads from
-@samp{images.server.com}, etc.:
+@samp{www.example.com}, but allowing downloads from
+@samp{images.example.com}, etc.:
 
 @example
-wget -rH -Dserver.com http://www.server.com/
+wget -rH -Dexample.com http://www.example.com/
 @end example
 
 You can specify more than one address by separating them with a comma,
@@ -2799,7 +2914,7 @@ These links are not relative:
 @example
 <a href="/foo.gif">
 <a href="/foo/bar.gif">
-<a href="http://www.server.com/foo/bar.gif">
+<a href="http://www.example.com/foo/bar.gif">
 @end example
 
 Using this option guarantees that recursive retrieval will not span
@@ -3162,8 +3277,8 @@ Turn on recognition of the (non-standard) @samp{Content-Disposition}
 HTTP header---if set to @samp{on}, the same as @samp{--content-disposition}.
 
 @item trust_server_names = on/off
-If set to on, use the last component of a redirection URL for the local
-file name.
+If set to on, construct the local file name from redirection URLs
+rather than original URLs.
 
 @item continue = on/off
 If set to on, force continuation of preexistent partially retrieved
@@ -3269,10 +3384,16 @@ Turn globbing on/off---the same as @samp{--glob} and @samp{--no-glob}.
 Define a header for HTTP downloads, like using
 @samp{--header=@var{string}}.
 
+@item compression = @var{string}
+Choose the compression type to be used.  Legal values are @samp{auto}
+(the default), @samp{gzip}, and @samp{none}.  The same as
+@samp{--compression=@var{string}}.
+
 @item adjust_extension = on/off
 Add a @samp{.html} extension to @samp{text/html} or
-@samp{application/xhtml+xml} files that lack one, or a @samp{.css}
-extension to @samp{text/css} files that lack one, like
+@samp{application/xhtml+xml} files that lack one, a @samp{.css}
+extension to @samp{text/css} files that lack one, and a @samp{.br},
+@samp{.Z}, @samp{.zlib} or @samp{.gz} to compressed files like
 @samp{-E}. Previously named @samp{html_extension} (still acceptable,
 but deprecated).
 
@@ -3669,7 +3790,7 @@ same directory structure the original has, with only one try per
 document, saving the log of the activities to @file{gnulog}:
 
 @example
-wget -r http://www.gnu.org/ -o gnulog
+wget -r https://www.gnu.org/ -o gnulog
 @end example
 
 @item
@@ -3677,7 +3798,7 @@ The same as the above, but convert the links in the downloaded files to
 point to local files, so you can view the documents off-line:
 
 @example
-wget --convert-links -r http://www.gnu.org/ -o gnulog
+wget --convert-links -r https://www.gnu.org/ -o gnulog
 @end example
 
 @item
@@ -3687,22 +3808,22 @@ sheets, are also downloaded.  Also make sure the downloaded page
 references the downloaded links.
 
 @example
-wget -p --convert-links http://www.server.com/dir/page.html
+wget -p --convert-links http://www.example.com/dir/page.html
 @end example
 
-The @sc{html} page will be saved to @file{www.server.com/dir/page.html}, and
-the images, stylesheets, etc., somewhere under @file{www.server.com/},
+The @sc{html} page will be saved to @file{www.example.com/dir/page.html}, and
+the images, stylesheets, etc., somewhere under @file{www.example.com/},
 depending on where they were on the remote server.
 
 @item
-The same as the above, but without the @file{www.server.com/} directory.
+The same as the above, but without the @file{www.example.com/} directory.
 In fact, I don't want to have all those random server directories
 anyway---just save @emph{all} those files under a @file{download/}
 subdirectory of the current directory.
 
 @example
 wget -p --convert-links -nH -nd -Pdownload \
-     http://www.server.com/dir/page.html
+     http://www.example.com/dir/page.html
 @end example
 
 @item
@@ -3731,12 +3852,12 @@ wget -r -l2 -P/tmp ftp://wuarchive.wustl.edu/
 
 @item
 You want to download all the @sc{gif}s from a directory on an @sc{http}
-server.  You tried @samp{wget http://www.server.com/dir/*.gif}, but that
+server.  You tried @samp{wget http://www.example.com/dir/*.gif}, but that
 didn't work because @sc{http} retrieval does not support globbing.  In
 that case, use:
 
 @example
-wget -r -l1 --no-parent -A.gif http://www.server.com/dir/
+wget -r -l1 --no-parent -A.gif http://www.example.com/dir/
 @end example
 
 More verbose, but the effect is the same.  @samp{-r -l1} means to
@@ -3752,7 +3873,7 @@ interrupted.  Now you do not want to clobber the files already present.
 It would be:
 
 @example
-wget -nc -r http://www.gnu.org/
+wget -nc -r https://www.gnu.org/
 @end example
 
 @item
@@ -3760,7 +3881,7 @@ If you want to encode your own username and password to @sc{http} or
 @sc{ftp}, use the appropriate @sc{url} syntax (@pxref{URL Format}).
 
 @example
-wget ftp://hniksic:mypassword@@unix.server.com/.emacs
+wget ftp://hniksic:mypassword@@unix.example.com/.emacs
 @end example
 
 Note, however, that this usage is not advisable on multi-user systems
@@ -3797,7 +3918,7 @@ to recheck a site each Sunday:
 
 @example
 crontab
-0 0 * * 0 wget --mirror http://www.gnu.org/ -o /home/me/weeklog
+0 0 * * 0 wget --mirror https://www.gnu.org/ -o /home/me/weeklog
 @end example
 
 @item
@@ -3809,7 +3930,7 @@ would look like this:
 
 @example
 wget --mirror --convert-links --backup-converted  \
-     http://www.gnu.org/ -o /home/me/weeklog
+     https://www.gnu.org/ -o /home/me/weeklog
 @end example
 
 @item
@@ -3822,13 +3943,13 @@ or @samp{application/xhtml+xml} to @file{@var{name}.html}.
 @example
 wget --mirror --convert-links --backup-converted \
      --html-extension -o /home/me/weeklog        \
-     http://www.gnu.org/
+     https://www.gnu.org/
 @end example
 
 Or, with less typing:
 
 @example
-wget -m -k -K -E http://www.gnu.org/ -o /home/me/weeklog
+wget -m -k -K -E https://www.gnu.org/ -o /home/me/weeklog
 @end example
 @end itemize
 @c man end
@@ -3935,14 +4056,14 @@ username and password.
 Like all GNU utilities, the latest version of Wget can be found at the
 master GNU archive site ftp.gnu.org, and its mirrors.  For example,
 Wget @value{VERSION} can be found at
-@url{ftp://ftp.gnu.org/pub/gnu/wget/wget-@value{VERSION}.tar.gz}
+@url{https://ftp.gnu.org/pub/gnu/wget/wget-@value{VERSION}.tar.gz}
 
 @node Web Site, Mailing Lists, Distribution, Various
 @section Web Site
 @cindex web site
 
 The official web site for GNU Wget is at
-@url{http://www.gnu.org/software/wget/}. However, most useful
+@url{https//www.gnu.org/software/wget/}. However, most useful
 information resides at ``The Wget Wgiki'',
 @url{http://wget.addictivecode.org/}.
 
@@ -3956,14 +4077,14 @@ information resides at ``The Wget Wgiki'',
 The primary mailinglist for discussion, bug-reports, or questions
 about GNU Wget is at @email{bug-wget@@gnu.org}. To subscribe, send an
 email to @email{bug-wget-join@@gnu.org}, or visit
-@url{http://lists.gnu.org/mailman/listinfo/bug-wget}.
+@url{https://lists.gnu.org/mailman/listinfo/bug-wget}.
 
 You do not need to subscribe to send a message to the list; however,
 please note that unsubscribed messages are moderated, and may take a
 while before they hit the list---@strong{usually around a day}.  If
 you want your message to show up immediately, please subscribe to the
 list before posting. Archives for the list may be found at
-@url{http://lists.gnu.org/pipermail/bug-wget/}.
+@url{https://lists.gnu.org/archive/html/bug-wget/}.
 
 An NNTP/Usenettish gateway is also available via
 @uref{http://gmane.org/about.php,Gmane}. You can see the Gmane
@@ -3971,15 +4092,7 @@ archives at
 @url{http://news.gmane.org/gmane.comp.web.wget.general}. Note that the
 Gmane archives conveniently include messages from both the current
 list, and the previous one. Messages also show up in the Gmane
-archives sooner than they do at @url{lists.gnu.org}.
-
-@unnumberedsubsec Bug Notices List
-
-Additionally, there is the @email{wget-notify@@addictivecode.org} mailing
-list. This is a non-discussion list that receives bug report
-notifications from the bug-tracker. To subscribe to this list,
-send an email to @email{wget-notify-join@@addictivecode.org},
-or visit @url{http://addictivecode.org/mailman/listinfo/wget-notify}.
+archives sooner than they do at @url{https://lists.gnu.org}.
 
 @unnumberedsubsec Obsolete Lists
 
@@ -3991,7 +4104,7 @@ discussing patches to GNU Wget.
 Messages from @email{wget@@sunsite.dk} are archived at
 @itemize @tie{}
 @item
-@url{http://www.mail-archive.com/wget%40sunsite.dk/} and at
+@url{https://www.mail-archive.com/wget%40sunsite.dk/} and at
 @item
 @url{http://news.gmane.org/gmane.comp.web.wget.general} (which also
 continues to archive the current list, @email{bug-wget@@gnu.org}).
@@ -4020,7 +4133,7 @@ via IRC at @code{irc.freenode.org}, @code{#wget}. Come check it out!
 
 @c man begin BUGS
 You are welcome to submit bug reports via the GNU Wget bug tracker (see
-@url{http://wget.addictivecode.org/BugTracker}).
+@url{https://savannah.gnu.org/bugs/?func=additem&group=wget}).
 
 Before actually submitting a bug report, please try to follow a few
 simple guidelines.
@@ -4037,7 +4150,7 @@ Lists}).
 @item
 Try to repeat the bug in as simple circumstances as possible.  E.g. if
 Wget crashes while downloading @samp{wget -rl0 -kKE -t5 --no-proxy
-http://yoyodyne.com -o /tmp/log}, you should try to see if the crash is
+http://example.com -o /tmp/log}, you should try to see if the crash is
 repeatable, and if will occur with a simpler set of options.  You might
 even try to start the download at the page where the crash occurred to
 see if that page somehow triggered the crash.
@@ -4102,7 +4215,7 @@ Windows-related features might look at them.
 
 Support for building on MS-DOS via DJGPP has been contributed by Gisle
 Vanem; a port to VMS is maintained by Steven Schweda, and is available
-at @url{http://antinode.org/}.
+at @url{https://antinode.info/dec/sw/wget.html}.
 
 @node Signals,  , Portability, Various
 @section Signals
@@ -4180,23 +4293,23 @@ download an individual page.  Because of that, Wget honors RES when
 downloading recursively.  For instance, when you issue:
 
 @example
-wget -r http://www.server.com/
+wget -r http://www.example.com/
 @end example
 
-First the index of @samp{www.server.com} will be downloaded.  If Wget
+First the index of @samp{www.example.com} will be downloaded.  If Wget
 finds that it wants to download more documents from that server, it will
-request @samp{http://www.server.com/robots.txt} and, if found, use it
+request @samp{http://www.example.com/robots.txt} and, if found, use it
 for further downloads.  @file{robots.txt} is loaded only once per each
 server.
 
 Until version 1.8, Wget supported the first version of the standard,
 written by Martijn Koster in 1994 and available at
-@url{http://www.robotstxt.org/wc/norobots.html}.  As of version 1.8,
+@url{http://www.robotstxt.org/orig.html}.  As of version 1.8,
 Wget has supported the additional directives specified in the internet
 draft @samp{<draft-koster-robots-00.txt>} titled ``A Method for Web
 Robots Control''.  The draft, which has as far as I know never made to
 an @sc{rfc}, is available at
-@url{http://www.robotstxt.org/wc/norobots-rfc.txt}.
+@url{http://www.robotstxt.org/norobots-rfc.txt}.
 
 This manual no longer includes the text of the Robot Exclusion Standard.
 
@@ -4210,7 +4323,7 @@ this:
 @end example
 
 This is explained in some detail at
-@url{http://www.robotstxt.org/wc/meta-user.html}.  Wget supports this
+@url{http://www.robotstxt.org/meta.html}.  Wget supports this
 method of robot exclusion in addition to the usual @file{/robots.txt}
 exclusion.
 

m4/exitfail.m4

@@ -1,5 +1,5 @@
 # exitfail.m4 serial 6
-dnl Copyright (C) 2002, 2003, 2005, 2006, 2010, 2011 Free Software
+dnl Copyright (C) 2002-2003, 2005-2006, 2010-2011, 2018 Free Software
 dnl Foundation, Inc.
 dnl This file is free software; the Free Software Foundation
 dnl gives unlimited permission to copy and/or distribute it,

m4/getpagesize.m4

@@ -1,5 +1,5 @@
 # getpagesize.m4 serial 7
-dnl Copyright (C) 2002, 2004, 2005, 2007, 2010, 2011 Free Software
+dnl Copyright (C) 2002, 2004-2005, 2007, 2010-2011, 2018 Free Software
 dnl Foundation, Inc.
 dnl This file is free software; the Free Software Foundation
 dnl gives unlimited permission to copy and/or distribute it,

m4/wchar.m4

@@ -1,7 +1,6 @@
 dnl A placeholder for ISO C99 <wchar.h>, for platforms that have issues.
 
-dnl Copyright (C) 2007, 2008, 2009, 2010, 2011 Free Software Foundation,
-dnl Inc.
+dnl Copyright (C) 2007-2011, 2018 Free Software Foundation, Inc.
 dnl This file is free software; the Free Software Foundation
 dnl gives unlimited permission to copy and/or distribute it,
 dnl with or without modifications, as long as this notice is preserved.

m4/wctype.m4

@@ -2,7 +2,7 @@
 
 dnl A placeholder for ISO C99 <wctype.h>, for platforms that lack it.
 
-dnl Copyright (C) 2006, 2007, 2008, 2010, 2011 Free Software Foundation,
+dnl Copyright (C) 2006-2008, 2010-2011, 2018 Free Software Foundation,
 dnl Inc.
 dnl This file is free software; the Free Software Foundation
 dnl gives unlimited permission to copy and/or distribute it,

m4/wget.m4

@@ -1,7 +1,5 @@
 dnl Wget-specific Autoconf macros.
-dnl Copyright (C) 1995, 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003,
-dnl 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011 Free Software
-dnl Foundation, Inc.
+dnl Copyright (C) 1995-2011, 2018 Free Software Foundation, Inc.
 
 dnl This program is free software; you can redistribute it and/or modify
 dnl it under the terms of the GNU General Public License as published by
@@ -27,89 +25,6 @@ dnl Corresponding Source for a non-source form of such a combination
 dnl shall include the source code for the parts of OpenSSL used as well
 dnl as that of the covered work.
 
-dnl
-dnl Check for `struct utimbuf'.
-dnl
-
-AC_DEFUN([WGET_STRUCT_UTIMBUF], [
-  AC_CHECK_TYPES([struct utimbuf], [], [], [
-#include <stdio.h>
-#if HAVE_SYS_TYPES_H
-# include <sys/types.h>
-#endif
-#if HAVE_UTIME_H
-# include <utime.h>
-#endif
-  ])
-])
-
-dnl Check whether fnmatch.h can be included.  This doesn't use
-dnl AC_FUNC_FNMATCH because Wget is already careful to only use
-dnl fnmatch on certain OS'es.  However, fnmatch.h is sometimes broken
-dnl even on those because Apache installs its own fnmatch.h to
-dnl /usr/local/include (!), which GCC uses before /usr/include.
-
-AC_DEFUN([WGET_FNMATCH], [
-  AC_MSG_CHECKING([for working fnmatch.h])
-  AC_COMPILE_IFELSE([AC_LANG_SOURCE([#include <fnmatch.h>
-                    ])], [
-    AC_MSG_RESULT(yes)
-    AC_DEFINE([HAVE_WORKING_FNMATCH_H], 1,
-              [Define if fnmatch.h can be included.])
-  ], [
-    AC_MSG_RESULT(no)
-  ])
-])
-
-dnl Check for nanosleep.  For nanosleep to work on Solaris, we must
-dnl link with -lrt (recently) or with -lposix4 (older releases).
-
-AC_DEFUN([WGET_NANOSLEEP], [
-  AC_CHECK_FUNCS(nanosleep, [], [
-    AC_CHECK_LIB(rt, nanosleep, [
-      AC_DEFINE([HAVE_NANOSLEEP], 1,
-                [Define if you have the nanosleep function.])
-      LIBS="-lrt $LIBS"
-    ], [
-      AC_CHECK_LIB(posix4, nanosleep, [
-	AC_DEFINE([HAVE_NANOSLEEP], 1,
-		  [Define if you have the nanosleep function.])
-	LIBS="-lposix4 $LIBS"
-      ])
-    ])
-  ])
-])
-
-AC_DEFUN([WGET_POSIX_CLOCK], [
-  AC_CHECK_FUNCS(clock_gettime, [], [
-    AC_CHECK_LIB(rt, clock_gettime)
-  ])
-])
-
-dnl Check whether we need to link with -lnsl and -lsocket, as is the
-dnl case on e.g. Solaris.
-
-AC_DEFUN([WGET_NSL_SOCKET], [
-  dnl On Solaris, -lnsl is needed to use gethostbyname.  But checking
-  dnl for gethostbyname is not enough because on "NCR MP-RAS 3.0"
-  dnl gethostbyname is in libc, but -lnsl is still needed to use
-  dnl -lsocket, as well as for functions such as inet_ntoa.  We look
-  dnl for such known offenders and if one of them is not found, we
-  dnl check if -lnsl is needed.
-  wget_check_in_nsl=NONE
-  AC_CHECK_FUNCS(gethostbyname, [], [
-    wget_check_in_nsl=gethostbyname
-  ])
-  AC_CHECK_FUNCS(inet_ntoa, [], [
-    wget_check_in_nsl=inet_ntoa
-  ])
-  if test $wget_check_in_nsl != NONE; then
-    AC_CHECK_LIB(nsl, $wget_check_in_nsl)
-  fi
-  AC_CHECK_LIB(socket, socket)
-])
-
-
 dnl ************************************************************
 dnl START OF IPv6 AUTOCONFIGURATION SUPPORT MACROS
 dnl ************************************************************

po/POTFILES.in

@@ -1,7 +1,5 @@
 # List of files which containing translatable strings.
-# Copyright (C) 1995, 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003,
-# 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011 Free Software
-# Foundation, Inc.
+# Copyright (C) 1995-2011, 2018 Free Software Foundation, Inc.
 
 # Package source files
 lib/error.c
@@ -35,7 +33,6 @@ src/ptimer.c
 src/recur.c
 src/res.c
 src/retr.c
-src/spider.c
 src/url.c
 src/utils.c
 src/warc.c

src/Makefile.am

@@ -1,7 +1,5 @@
 # Makefile for `wget' utility
-# Copyright (C) 1995, 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003,
-# 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2015 Free Software
-# Foundation, Inc.
+# Copyright (C) 1995-2011, 2015, 2018 Free Software Foundation, Inc.
 
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -39,9 +37,12 @@ if METALINK_IS_ENABLED
 METALINK_OBJ = metalink.c
 endif
 
+if WITH_XATTR
+XATTR_OBJ = xattr.c
+endif
+
 # The following line is losing on some versions of make!
 DEFS     = @DEFS@ -DSYSTEM_WGETRC=\"$(sysconfdir)/wgetrc\" -DLOCALEDIR=\"$(localedir)\"
-LIBS     = @LIBICONV@ @LIBINTL@ @LIBS@ $(LIB_CLOCK_GETTIME)
 
 EXTRA_DIST = css.l css.c css_.c build_info.c.in
 
@@ -50,17 +51,20 @@ wget_SOURCES = connect.c convert.c cookies.c ftp.c	\
 		css_.c css-url.c	\
 		ftp-basic.c ftp-ls.c hash.c host.c hsts.c html-parse.c html-url.c	\
 		http.c init.c log.c main.c netrc.c progress.c ptimer.c	\
-		recur.c res.c retr.c spider.c url.c warc.c	\
+		recur.c res.c retr.c spider.c url.c warc.c $(XATTR_OBJ) \
 		utils.c exits.c build_info.c $(IRI_OBJ) $(METALINK_OBJ)	\
 		css-url.h css-tokens.h connect.h convert.h cookies.h	\
 		ftp.h hash.h host.h hsts.h  html-parse.h html-url.h	\
 		http.h http-ntlm.h init.h log.h mswindows.h netrc.h	\
 		options.h progress.h ptimer.h recur.h res.h retr.h	\
 		spider.h ssl.h sysdep.h url.h warc.h utils.h wget.h iri.h	\
-		exits.h version.h metalink.h
+		exits.h version.h metalink.h xattr.h
 nodist_wget_SOURCES = version.c
 EXTRA_wget_SOURCES = iri.c
-LDADD = $(LIBOBJS) ../lib/libgnu.a
+LDADD = $(LIBOBJS) ../lib/libgnu.a $(GETADDRINFO_LIB) $(HOSTENT_LIB)\
+ $(INET_NTOP_LIB) $(LIBSOCKET) $(LIB_CLOCK_GETTIME) $(LIB_CRYPTO)\
+ $(LIB_NANOSLEEP) $(LIB_POSIX_SPAWN) $(LIB_SELECT) $(LIBICONV) $(LIBINTL)\
+ $(LIBTHREAD) $(LIBUNISTRING) $(SERVENT_LIB)
 AM_CPPFLAGS = -I$(top_builddir)/lib -I$(top_srcdir)/lib
 
 

src/build_info.c.in

@@ -8,6 +8,10 @@ nls             defined ENABLE_NLS
 ntlm            defined ENABLE_NTLM
 opie            defined ENABLE_OPIE
 psl             defined HAVE_LIBPSL
+cares            defined HAVE_LIBCARES
+
+metalink        defined HAVE_METALINK
+gpgme           defined HAVE_GPGME
 
 ssl choice:
     openssl     defined HAVE_LIBSSL || defined HAVE_LIBSSL32

src/connect.c

@@ -1,7 +1,5 @@
 /* Establishing and handling network connections.
-   Copyright (C) 1995, 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003,
-   2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2015 Free Software
-   Foundation, Inc.
+   Copyright (C) 1995-2011, 2015, 2018 Free Software Foundation, Inc.
 
 This file is part of GNU Wget.
 
@@ -31,6 +29,7 @@ as that of the covered work.  */
 
 #include "wget.h"
 
+#include "exits.h"
 #include <stdio.h>
 #include <stdlib.h>
 #include <unistd.h>
@@ -55,10 +54,6 @@ as that of the covered work.  */
 #include <string.h>
 #include <sys/time.h>
 
-#ifdef ENABLE_IRI
-#include <idn-free.h>
-#endif
-
 #include "utils.h"
 #include "host.h"
 #include "connect.h"
@@ -279,11 +274,8 @@ connect_to_ip (const ip_address *ip, int port, const char *print)
 
           if (opt.enable_iri && (name = idn_decode ((char *) print)) != NULL)
             {
-              int len = strlen (print) + strlen (name) + 4;
-              str = xmalloc (len);
-              snprintf (str, len, "%s (%s)", name, print);
-              str[len-1] = '\0';
-              idn_free (name);
+              str = aprintf ("%s (%s)", name, print);
+              xfree (name);
             }
 
           logprintf (LOG_VERBOSE, _("Connecting to %s|%s|:%d... "),
@@ -331,8 +323,10 @@ connect_to_ip (const ip_address *ip, int port, const char *print)
       if (bufsize < 512)
         bufsize = 512;          /* avoid pathologically small values */
 #ifdef SO_RCVBUF
-      setsockopt (sock, SOL_SOCKET, SO_RCVBUF,
-                  (void *)&bufsize, (socklen_t)sizeof (bufsize));
+      if (setsockopt (sock, SOL_SOCKET, SO_RCVBUF,
+                  (void *) &bufsize, (socklen_t) sizeof (bufsize)))
+        logprintf (LOG_NOTQUIET, _("setsockopt SO_RCVBUF failed: %s\n"),
+                   strerror (errno));
 #endif
       /* When we add limit_rate support for writing, which is useful
          for POST, we should also set SO_SNDBUF here.  */
@@ -369,7 +363,14 @@ connect_to_ip (const ip_address *ip, int port, const char *print)
        logprintf.  */
     int save_errno = errno;
     if (sock >= 0)
-      fd_close (sock);
+      {
+#ifdef WIN32
+	/* If the connection timed out, fd_close will hang in Gnulib's
+	   close_fd_maybe_socket, inside the call to WSAEnumNetworkEvents.  */
+	if (errno != ETIMEDOUT)
+#endif
+	  fd_close (sock);
+      }
     if (print)
       logprintf (LOG_NOTQUIET, _("failed: %s.\n"), strerror (errno));
     errno = save_errno;
@@ -464,7 +465,9 @@ bind_local (const ip_address *bind_address, int *port)
     return -1;
 
 #ifdef SO_REUSEADDR
-  setsockopt (sock, SOL_SOCKET, SO_REUSEADDR, setopt_ptr, setopt_size);
+  if (setsockopt (sock, SOL_SOCKET, SO_REUSEADDR, setopt_ptr, setopt_size))
+    logprintf (LOG_NOTQUIET, _("setsockopt SO_REUSEADDR failed: %s\n"),
+               strerror (errno));
 #endif
 
   xzero (ss);
@@ -682,6 +685,11 @@ select_fd (int fd, double maxtime, int wait_for)
   struct timeval tmout;
   int result;
 
+  if (fd >= FD_SETSIZE)
+    {
+      logprintf (LOG_NOTQUIET, _("Too many fds open.  Cannot use select on a fd >= %d\n"), FD_SETSIZE);
+      exit (WGET_EXIT_GENERIC_ERROR);
+    }
   FD_ZERO (&fdset);
   FD_SET (fd, &fdset);
   if (wait_for & WAIT_FOR_READ)
@@ -724,6 +732,11 @@ test_socket_open (int sock)
   struct timeval to;
   int ret = 0;
 
+  if (sock >= FD_SETSIZE)
+    {
+      logprintf (LOG_NOTQUIET, _("Too many fds open.  Cannot use select on a fd >= %d\n"), FD_SETSIZE);
+      exit (WGET_EXIT_GENERIC_ERROR);
+    }
   /* Check if we still have a valid (non-EOF) connection.  From Andrew
    * Maholski's code in the Unix Socket FAQ.  */
 

src/connect.h

@@ -1,7 +1,5 @@
 /* Declarations for connect.
-   Copyright (C) 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004,
-   2005, 2006, 2007, 2008, 2009, 2010, 2011, 2015 Free Software
-   Foundation, Inc.
+   Copyright (C) 1996-2011, 2015, 2018 Free Software Foundation, Inc.
 
 This file is part of GNU Wget.
 

src/convert.c

@@ -1,6 +1,6 @@
 /* Conversion of links to local files.
-   Copyright (C) 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011,
-   2014, 2015 Free Software Foundation, Inc.
+   Copyright (C) 2003-2011, 2014-2015, 2018 Free Software Foundation,
+   Inc.
 
 This file is part of GNU Wget.
 
@@ -308,7 +308,7 @@ convert_links (const char *file, struct urlpos *links)
             char *quoted_newname = local_quote_string (newname,
                                                        link->link_css_p);
 
-            if (link->link_css_p)
+            if (link->link_css_p || link->link_noquote_html_p)
               p = replace_plain (p, link->size, fp, quoted_newname);
             else if (!link->link_refresh_p)
               p = replace_attr (p, link->size, fp, quoted_newname);
@@ -329,7 +329,7 @@ convert_links (const char *file, struct urlpos *links)
             char *newname = convert_basename (p, link);
             char *quoted_newname = local_quote_string (newname, link->link_css_p);
 
-            if (link->link_css_p)
+            if (link->link_css_p || link->link_noquote_html_p)
               p = replace_plain (p, link->size, fp, quoted_newname);
             else if (!link->link_refresh_p)
               p = replace_attr (p, link->size, fp, quoted_newname);
@@ -352,7 +352,7 @@ convert_links (const char *file, struct urlpos *links)
             char *newlink = link->url->url;
             char *quoted_newlink = html_quote_string (newlink);
 
-            if (link->link_css_p)
+            if (link->link_css_p || link->link_noquote_html_p)
               p = replace_plain (p, link->size, fp, newlink);
             else if (!link->link_refresh_p)
               p = replace_attr (p, link->size, fp, quoted_newlink);
@@ -1024,7 +1024,7 @@ convert_cleanup (void)
 
 /* This table should really be merged with dl_file_url_map and
    downloaded_html_files.  This was originally a list, but I changed
-   it to a hash table beause it was actually taking a lot of time to
+   it to a hash table because it was actually taking a lot of time to
    find things in it.  */
 
 static struct hash_table *downloaded_files_hash;

src/convert.h

@@ -1,6 +1,6 @@
 /* Declarations for convert.c
-   Copyright (C) 2003, 2004, 2005, 2006, 2009, 2010, 2011, 2015 Free
-   Software Foundation, Inc.
+   Copyright (C) 2003-2006, 2009-2011, 2015, 2018 Free Software
+   Foundation, Inc.
 
 This file is part of GNU Wget.
 
@@ -69,6 +69,7 @@ struct urlpos {
   unsigned int link_base_p  :1;     /* the url came from <base href=...> */
   unsigned int link_inline_p    :1; /* needed to render the page */
   unsigned int link_css_p   :1;     /* the url came from CSS */
+  unsigned int link_noquote_html_p :1; /* from HTML, but doesn't need " */
   unsigned int link_expect_html :1; /* expected to contain HTML */
   unsigned int link_expect_css  :1; /* expected to contain CSS */
 

src/cookies.c

@@ -1,6 +1,5 @@
 /* Support for cookies.
-   Copyright (C) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009,
-   2010, 2011, 2015 Free Software Foundation, Inc.
+   Copyright (C) 2001-2011, 2015, 2018 Free Software Foundation, Inc.
 
 This file is part of GNU Wget.
 
@@ -45,6 +44,7 @@ as that of the covered work.  */
 
 #include "wget.h"
 
+#include <stdint.h>
 #include <stdio.h>
 #include <string.h>
 #include <stdlib.h>
@@ -484,7 +484,7 @@ parse_set_cookie (const char *set_cookie, bool silent)
 /* Check whether ADDR matches <digits>.<digits>.<digits>.<digits>.
 
    We don't want to call network functions like inet_addr() because
-   all we need is a check, preferrably one that is small, fast, and
+   all we need is a check, preferably one that is small, fast, and
    well-defined.  */
 
 static bool
@@ -515,7 +515,7 @@ numeric_address_p (const char *addr)
    psl on our own, if libpsl is compiled without a public suffix list,
    fall back to using the original "tail matching" heuristic. Also if
    libpsl is unable to convert the domain to lowercase, which means that
-   it doesnt have any runtime conversion support, we again fall back to
+   it doesn't have any runtime conversion support, we again fall back to
    "tail matching" since libpsl states the results are unpredictable with
    upper case strings.
    */
@@ -525,19 +525,51 @@ check_domain_match (const char *cookie_domain, const char *host)
 {
 
 #ifdef HAVE_LIBPSL
+  static int init_psl;
+  static const psl_ctx_t *psl;
+
   char *cookie_domain_lower = NULL;
   char *host_lower = NULL;
-  const psl_ctx_t *psl;
   int is_acceptable;
 
-  DEBUGP (("cdm: 1"));
-  if (!(psl = psl_builtin()))
+  DEBUGP (("cdm: 1\n"));
+  if (!init_psl)
     {
-      DEBUGP (("\nlibpsl not built with a public suffix list. "
-               "Falling back to simple heuristics.\n"));
-      goto no_psl;
-    }
+      init_psl = 1;
 
+#ifdef HAVE_PSL_LATEST
+      if ((psl = psl_latest (NULL)))
+        goto have_psl;
+
+      DEBUGP (("\nPSL: Failed to load any PSL data. "
+               "Falling back to insecure heuristics.\n"));
+#else
+      if ((psl = psl_builtin ()) && !psl_builtin_outdated ())
+        goto have_psl;
+
+      DEBUGP (("\nPSL: built-in data outdated. "
+               "Trying to load data from %s.\n",
+              quote (psl_builtin_filename ())));
+
+      if ((psl = psl_load_file (psl_builtin_filename ())))
+        goto have_psl;
+
+      DEBUGP (("\nPSL: %s not found or not readable. "
+               "Falling back to built-in data.\n",
+              quote (psl_builtin_filename ())));
+
+      if (!(psl = psl_builtin ()))
+        {
+          DEBUGP (("\nPSL: libpsl not built with a public suffix list. "
+                   "Falling back to insecure heuristics.\n"));
+          goto no_psl;
+        }
+#endif
+    }
+  else if (!psl)
+    goto no_psl;
+
+have_psl:
   if (psl_str_to_utf8lower (cookie_domain, NULL, NULL, &cookie_domain_lower) == PSL_SUCCESS &&
       psl_str_to_utf8lower (host, NULL, NULL, &host_lower) == PSL_SUCCESS)
     {
@@ -562,13 +594,13 @@ no_psl:
 #endif
 
   /* For efficiency make some elementary checks first */
-  DEBUGP (("cdm: 2"));
+  DEBUGP (("cdm: 2\n"));
 
   /* For the sake of efficiency, check for exact match first. */
   if (0 == strcasecmp (cookie_domain, host))
     return true;
 
-  DEBUGP ((" 3"));
+  DEBUGP (("cdm: 3\n"));
 
   /* HOST must match the tail of cookie_domain. */
   if (!match_tail (host, cookie_domain, true))
@@ -608,7 +640,7 @@ no_psl:
     if (*p == '.')
       /* Ignore leading period in this calculation. */
       ++p;
-    DEBUGP ((" 4"));
+    DEBUGP (("cdm: 4\n"));
     for (out = 0; !out; p++)
       switch (*p)
         {
@@ -634,12 +666,12 @@ no_psl:
           ++ldcl;
         }
 
-    DEBUGP ((" 5"));
+    DEBUGP (("cdm: 5\n"));
 
     if (dccount < 2)
       return false;
 
-    DEBUGP ((" 6"));
+    DEBUGP (("cdm: 6\n"));
 
     if (dccount == 2)
       {
@@ -659,7 +691,7 @@ no_psl:
       }
   }
 
-  DEBUGP ((" 7"));
+  DEBUGP (("cdm: 7\n"));
 
   /* Don't allow the host "foobar.com" to set a cookie for domain
      "bar.com".  */
@@ -674,7 +706,7 @@ no_psl:
         return false;
     }
 
-  DEBUGP ((" 8"));
+  DEBUGP (("cdm: 8\n"));
 
   return true;
 }
@@ -1018,7 +1050,7 @@ cookie_header (struct cookie_jar *jar, const char *host,
 
   struct cookie *cookie;
   struct weighed_cookie *outgoing;
-  int count, i, ocnt;
+  size_t count, i, ocnt;
   char *result;
   int result_size, pos;
   PREPEND_SLASH (path);         /* see cookie_handle_set_cookie */
@@ -1032,7 +1064,7 @@ cookie_header (struct cookie_jar *jar, const char *host,
   chain_count = find_chains_of_host (jar, host, chains);
 
   /* No cookies for this host. */
-  if (!chain_count)
+  if (chain_count <= 0)
     return NULL;
 
   cookies_now = time (NULL);
@@ -1043,7 +1075,7 @@ cookie_header (struct cookie_jar *jar, const char *host,
 
   /* Count the number of matching cookies. */
   count = 0;
-  for (i = 0; i < chain_count; i++)
+  for (i = 0; i < (unsigned) chain_count; i++)
     for (cookie = chains[i]; cookie; cookie = cookie->next)
       if (cookie_matches_url (cookie, host, port, path, secflag, NULL))
         ++count;
@@ -1051,12 +1083,14 @@ cookie_header (struct cookie_jar *jar, const char *host,
     return NULL;                /* no cookies matched */
 
   /* Allocate the array. */
-  outgoing = alloca_array (struct weighed_cookie, count);
+  if (count > SIZE_MAX / sizeof (struct weighed_cookie))
+    return NULL;                /* unable to process so many cookies */
+  outgoing = xmalloc (count * sizeof (struct weighed_cookie));
 
   /* Fill the array with all the matching cookies from the chains that
      match HOST. */
   ocnt = 0;
-  for (i = 0; i < chain_count; i++)
+  for (i = 0; i < (unsigned) chain_count; i++)
     for (cookie = chains[i]; cookie; cookie = cookie->next)
       {
         int pg;
@@ -1111,6 +1145,7 @@ cookie_header (struct cookie_jar *jar, const char *host,
         }
     }
   result[pos++] = '\0';
+  xfree (outgoing);
   assert (pos == result_size);
   return result;
 }

src/cookies.h

@@ -1,6 +1,5 @@
 /* Support for cookies.
-   Copyright (C) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009,
-   2010, 2011, 2015 Free Software Foundation, Inc.
+   Copyright (C) 2001-2011, 2015, 2018 Free Software Foundation, Inc.
 
 This file is part of GNU Wget.
 

src/css-tokens.h

@@ -1,5 +1,5 @@
 /* Declarations for css.lex
-   Copyright (C) 2006, 2009, 2010, 2011, 2015 Free Software Foundation,
+   Copyright (C) 2006, 2009-2011, 2015, 2018 Free Software Foundation,
    Inc.
 
 This file is part of GNU Wget.

src/css-url.c

@@ -1,6 +1,6 @@
 /* Collect URLs from CSS source.
-   Copyright (C) 1998, 2000, 2001, 2002, 2003, 2009, 2010, 2011, 2014,
-   2015 Free Software Foundation, Inc.
+   Copyright (C) 1998, 2000-2003, 2009-2011, 2014-2015, 2018 Free
+   Software Foundation, Inc.
 
 This file is part of GNU Wget.
 
@@ -59,6 +59,7 @@ extern char *yytext;
 extern int yyleng;
 typedef struct yy_buffer_state *YY_BUFFER_STATE;
 extern YY_BUFFER_STATE yy_scan_bytes (const char *bytes,int len  );
+extern void yy_delete_buffer (YY_BUFFER_STATE  b);
 extern int yylex (void);
 
 /*
@@ -109,9 +110,10 @@ get_urls_css (struct map_context *ctx, int offset, int buf_length)
   int buffer_pos = 0;
   int pos, length;
   char *uri;
+  YY_BUFFER_STATE b;
 
   /* tell flex to scan from this buffer */
-  yy_scan_bytes (ctx->text + offset, buf_length);
+  b = yy_scan_bytes (ctx->text + offset, buf_length);
 
   while((token = yylex()) != CSSEOF)
     {
@@ -188,6 +190,9 @@ get_urls_css (struct map_context *ctx, int offset, int buf_length)
         }
       buffer_pos += yyleng;
     }
+
+  yy_delete_buffer(b);
+
   DEBUGP (("\n"));
 }
 

src/css-url.h

@@ -1,5 +1,5 @@
 /* Declarations for css-url.c.
-   Copyright (C) 2006, 2009, 2010, 2011, 2015 Free Software Foundation,
+   Copyright (C) 2006, 2009-2011, 2015, 2018 Free Software Foundation,
    Inc.
 
 This file is part of GNU Wget.
@@ -31,7 +31,6 @@ as that of the covered work.  */
 #ifndef CSS_URL_H
 #define CSS_URL_H
 
-void get_urls_css (struct map_context *, int, int);
 void get_urls_css (struct map_context *, int, int);
 struct urlpos *get_urls_css_file (const char *, const char *);
 

src/css.l

@@ -6,7 +6,7 @@
 %{
 /* Lex source for CSS tokenizing.
    Taken from http://www.w3.org/TR/CSS21/grammar.html#q2
-   Copyright (C) 2006, 2009, 2010, 2011, 2015 Free Software Foundation,
+   Copyright (C) 2006, 2009-2011, 2015, 2018 Free Software Foundation,
    Inc.
 
 This file is part of GNU Wget.

src/exits.c

@@ -1,6 +1,5 @@
 /* Exit status handling.
-   Copyright (C) 2009, 2010, 2011, 2012, 2015 Free Software Foundation,
-   Inc.
+   Copyright (C) 2009-2012, 2015, 2018 Free Software Foundation, Inc.
 
    This file is part of GNU Wget.
 

src/exits.h

@@ -1,6 +1,5 @@
 /* Exit status related declarations.
-   Copyright (C) 2009, 2010, 2011, 2012, 2015 Free Software Foundation,
-   Inc.
+   Copyright (C) 2009-2012, 2015, 2018 Free Software Foundation, Inc.
 
 This file is part of GNU Wget.
 

src/ftp-basic.c

@@ -1,7 +1,6 @@
 /* Basic FTP routines.
-   Copyright (C) 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004,
-   2005, 2006, 2007, 2008, 2009, 2010, 2011, 2014, 2015 Free Software
-   Foundation, Inc.
+   Copyright (C) 1996-2011, 2014-2015, 2018 Free Software Foundation,
+   Inc.
 
 This file is part of GNU Wget.
 
@@ -429,6 +428,65 @@ ip_address_to_eprt_repr (const ip_address *addr, int port, char *buf,
   buf[buflen - 1] = '\0';
 }
 
+/* Bind a port and send the appropriate PORT command to the FTP
+   server.  Use acceptport after RETR, to get the socket of data
+   connection.  */
+uerr_t
+ftp_eprt (int csock, int *local_sock)
+{
+  uerr_t err;
+  char *request, *respline;
+  ip_address addr;
+  int nwritten;
+  int port;
+  /* Must contain the argument of EPRT (of the form |af|addr|port|).
+   * 4 chars for the | separators, INET6_ADDRSTRLEN chars for addr
+   * 1 char for af (1-2) and 5 chars for port (0-65535) */
+  char bytes[4 + INET6_ADDRSTRLEN + 1 + 5 + 1];
+
+  /* Get the address of this side of the connection. */
+  if (!socket_ip_address (csock, &addr, ENDPOINT_LOCAL))
+    return FTPSYSERR;
+
+  /* Setting port to 0 lets the system choose a free port.  */
+  port = 0;
+
+  /* Bind the port.  */
+  *local_sock = bind_local (&addr, &port);
+  if (*local_sock < 0)
+    return FTPSYSERR;
+
+  /* Construct the argument of EPRT (of the form |af|addr|port|). */
+  ip_address_to_eprt_repr (&addr, port, bytes, sizeof (bytes));
+
+  /* Send PORT request.  */
+  request = ftp_request ("EPRT", bytes);
+  nwritten = fd_write (csock, request, strlen (request), -1);
+  if (nwritten < 0)
+    {
+      xfree (request);
+      fd_close (*local_sock);
+      return WRITEFAILED;
+    }
+  xfree (request);
+  /* Get appropriate response.  */
+  err = ftp_response (csock, &respline);
+  if (err != FTPOK)
+    {
+      fd_close (*local_sock);
+      return err;
+    }
+  if (*respline != '2')
+    {
+      xfree (respline);
+      fd_close (*local_sock);
+      return FTPPORTERR;
+    }
+  xfree (respline);
+  return FTPOK;
+}
+#endif
+
 #ifdef HAVE_SSL
 /*
  * The following three functions defined into this #ifdef block
@@ -542,65 +600,6 @@ bail:
 }
 #endif /* HAVE_SSL */
 
-/* Bind a port and send the appropriate PORT command to the FTP
-   server.  Use acceptport after RETR, to get the socket of data
-   connection.  */
-uerr_t
-ftp_eprt (int csock, int *local_sock)
-{
-  uerr_t err;
-  char *request, *respline;
-  ip_address addr;
-  int nwritten;
-  int port;
-  /* Must contain the argument of EPRT (of the form |af|addr|port|).
-   * 4 chars for the | separators, INET6_ADDRSTRLEN chars for addr
-   * 1 char for af (1-2) and 5 chars for port (0-65535) */
-  char bytes[4 + INET6_ADDRSTRLEN + 1 + 5 + 1];
-
-  /* Get the address of this side of the connection. */
-  if (!socket_ip_address (csock, &addr, ENDPOINT_LOCAL))
-    return FTPSYSERR;
-
-  /* Setting port to 0 lets the system choose a free port.  */
-  port = 0;
-
-  /* Bind the port.  */
-  *local_sock = bind_local (&addr, &port);
-  if (*local_sock < 0)
-    return FTPSYSERR;
-
-  /* Construct the argument of EPRT (of the form |af|addr|port|). */
-  ip_address_to_eprt_repr (&addr, port, bytes, sizeof (bytes));
-
-  /* Send PORT request.  */
-  request = ftp_request ("EPRT", bytes);
-  nwritten = fd_write (csock, request, strlen (request), -1);
-  if (nwritten < 0)
-    {
-      xfree (request);
-      fd_close (*local_sock);
-      return WRITEFAILED;
-    }
-  xfree (request);
-  /* Get appropriate response.  */
-  err = ftp_response (csock, &respline);
-  if (err != FTPOK)
-    {
-      fd_close (*local_sock);
-      return err;
-    }
-  if (*respline != '2')
-    {
-      xfree (respline);
-      fd_close (*local_sock);
-      return FTPPORTERR;
-    }
-  xfree (respline);
-  return FTPOK;
-}
-#endif
-
 /* Similar to ftp_port, but uses `PASV' to initiate the passive FTP
    transfer.  Reads the response from server and parses it.  Reads the
    host and port addresses and returns them.  */

src/ftp-ls.c

@@ -1,7 +1,5 @@
 /* Parsing FTP `ls' output.
-   Copyright (C) 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004,
-   2005, 2006, 2007, 2008, 2009, 2010, 2011, 2015 Free Software
-   Foundation, Inc.
+   Copyright (C) 1996-2011, 2015, 2018 Free Software Foundation, Inc.
 
 This file is part of GNU Wget.
 
@@ -167,12 +165,12 @@ ftp_parse_unix_ls (const char *file, int ignore_perms)
               /*cur.perms = 1023;*/     /* #### What is this?  --hniksic */
               cur.perms = 0644;
             }
-          DEBUGP (("implicit perms %0o; ", cur.perms));
+          DEBUGP (("implicit perms %0o; ", (unsigned) cur.perms));
         }
        else
          {
            cur.perms = symperms (tok + 1);
-           DEBUGP (("perms %0o; ", cur.perms));
+           DEBUGP (("perms %0o; ", (unsigned) cur.perms));
          }
 
       error = ignore = 0;       /* Erroneous and ignoring entries are
@@ -478,6 +476,7 @@ ftp_parse_winnt_ls (const char *file)
         }
       /* Now it is possible to determine the position of the first symbol in
          filename. */
+      memset(&cur, 0, sizeof (cur));
       cur.name = xstrdup(filename);
       DEBUGP (("Name: '%s'\n", cur.name));
 
@@ -931,7 +930,7 @@ ftp_parse_vms_ls (const char *file)
                     }
                 }
               cur.perms = perms;
-              DEBUGP (("Prot.  perms = %0o.\n", cur.perms));
+              DEBUGP (("Prot.  perms = %0o.\n", (unsigned) cur.perms));
             }
           else
             {

src/ftp-opie.c

@@ -1,6 +1,5 @@
 /* Opie (s/key) support for FTP.
-   Copyright (C) 1998, 1999, 2000, 2001, 2002, 2003, 2004, 2005, 2006,
-   2007, 2008, 2009, 2010, 2011, 2015 Free Software Foundation, Inc.
+   Copyright (C) 1998-2011, 2015, 2018 Free Software Foundation, Inc.
 
 This file is part of GNU Wget.
 

src/ftp.c

@@ -1,7 +1,6 @@
 /* File Transfer Protocol support.
-   Copyright (C) 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004,
-   2005, 2006, 2007, 2008, 2009, 2010, 2011, 2014, 2015 Free Software
-   Foundation, Inc.
+   Copyright (C) 1996-2011, 2014-2015, 2018 Free Software Foundation,
+   Inc.
 
 This file is part of GNU Wget.
 
@@ -52,6 +51,9 @@ as that of the covered work.  */
 #include "recur.h"              /* for INFINITE_RECURSION */
 #include "warc.h"
 #include "c-strcase.h"
+#ifdef ENABLE_XATTR
+#include "xattr.h"
+#endif
 
 #ifdef __VMS
 # include "vms.h"
@@ -236,10 +238,10 @@ print_length (wgint size, wgint start, bool authoritative)
   logputs (LOG_VERBOSE, !authoritative ? _(" (unauthoritative)\n") : "\n");
 }
 
-static uerr_t ftp_get_listing (struct url *, ccon *, struct fileinfo **);
+static uerr_t ftp_get_listing (struct url *, struct url *, ccon *, struct fileinfo **);
 
 static uerr_t
-get_ftp_greeting(int csock, ccon *con)
+get_ftp_greeting (int csock, ccon *con)
 {
   uerr_t err = 0;
 
@@ -315,13 +317,14 @@ init_control_ssl_connection (int csock, struct url *u, bool *using_control_secur
    and closes the control connection in case of error.  If warc_tmp
    is non-NULL, the downloaded data will be written there as well.  */
 static uerr_t
-getftp (struct url *u, wgint passed_expected_bytes, wgint *qtyread,
+getftp (struct url *u, struct url *original_url,
+        wgint passed_expected_bytes, wgint *qtyread,
         wgint restval, ccon *con, int count, wgint *last_expected_bytes,
         FILE *warc_tmp)
 {
   int csock, dtsock, local_sock, res;
   uerr_t err = RETROK;          /* appease the compiler */
-  FILE *fp;
+  FILE *fp = NULL;
   char *respline, *tms;
   const char *user, *passwd, *tmrate;
   int cmd = con->cmd;
@@ -355,12 +358,35 @@ getftp (struct url *u, wgint passed_expected_bytes, wgint *qtyread,
 
   *qtyread = restval;
 
-  user = u->user;
-  passwd = u->passwd;
-  search_netrc (u->host, (const char **)&user, (const char **)&passwd, 1);
-  user = user ? user : (opt.ftp_user ? opt.ftp_user : opt.user);
+  /* Find the username with priority */
+  if (u->user)
+    user = u->user;
+  else if (opt.user && (opt.use_askpass || opt.ask_passwd))
+    user = opt.user;
+  else if (opt.ftp_user)
+    user = opt.ftp_user;
+  else if (opt.user)
+    user = opt.user;
+  else
+    user = NULL;
+
+  /* Find the password with priority */
+  if (u->passwd)
+    passwd = u->passwd;
+  else if (opt.passwd && (opt.use_askpass || opt.ask_passwd))
+    passwd = opt.passwd;
+  else if (opt.ftp_passwd)
+    passwd = opt.ftp_passwd;
+  else if (opt.passwd)
+    passwd = opt.passwd;
+  else
+    passwd = NULL;
+
+  /* Check for ~/.netrc if none of the above match */
+  if (opt.netrc && (!user || !passwd))
+    search_netrc (u->host, (const char **) &user, (const char **) &passwd, 1);
+
   if (!user) user = "anonymous";
-  passwd = passwd ? passwd : (opt.ftp_passwd ? opt.ftp_passwd : opt.passwd);
   if (!passwd) passwd = "-wget@";
 
   dtsock = -1;
@@ -393,7 +419,9 @@ getftp (struct url *u, wgint passed_expected_bytes, wgint *qtyread,
   if (!(cmd & DO_LOGIN))
     {
       csock = con->csock;
+#ifdef HAVE_SSL
       using_data_security = con->st & DATA_CHANNEL_SECURITY;
+#endif
     }
   else                          /* cmd & DO_LOGIN */
     {
@@ -532,10 +560,10 @@ Error in server response, closing control connection.\n"));
                 logputs (LOG_VERBOSE, "done.");
 
               if (!opt.server_response)
-                logprintf (LOG_VERBOSE, "  ==> PROT %c ... ", prot);
+                logprintf (LOG_VERBOSE, "  ==> PROT %c ... ", (int) prot);
               if ((err = ftp_prot (csock, prot)) == FTPNOPROT)
                 {
-                  logprintf (LOG_NOTQUIET, _("Server did not accept the 'PROT %c' command.\n"), prot);
+                  logprintf (LOG_NOTQUIET, _("Server did not accept the 'PROT %c' command.\n"), (int) prot);
                   return err;
                 }
               if (!opt.server_response)
@@ -869,7 +897,7 @@ Error in server response, closing control connection.\n"));
              Unlike the rest of this block, this particular behavior
              _is_ VMS-specific, so it gets its own VMS test.
           */
-          if ((con->rs == ST_VMS) && (strchr( target, '/') != NULL))
+          if ((con->rs == ST_VMS) && (strchr (target, '/') != NULL))
             {
               cwd_end = 3;
               DEBUGP (("Using extra \"CWD []\" step for VMS server.\n"));
@@ -1185,8 +1213,9 @@ Error in server response, closing control connection.\n"));
       if (opt.spider)
         {
           bool exists = false;
+          bool all_exist = true;
           struct fileinfo *f;
-          uerr_t _res = ftp_get_listing (u, con, &f);
+          uerr_t _res = ftp_get_listing (u, original_url, con, &f);
           /* Set the DO_RETR command flag again, because it gets unset when
              calling ftp_get_listing() and would otherwise cause an assertion
              failure earlier on when this function gets repeatedly called
@@ -1200,6 +1229,8 @@ Error in server response, closing control connection.\n"));
                     {
                       exists = true;
                       break;
+                    } else {
+                      all_exist = false;
                     }
                   f = f->next;
                 }
@@ -1220,7 +1251,11 @@ Error in server response, closing control connection.\n"));
           con->csock = -1;
           fd_close (dtsock);
           fd_close (local_sock);
-          return RETRFINISHED;
+          if (all_exist) {
+              return RETRFINISHED;
+          } else {
+              return FTPNSFOD;
+          }
         }
 
       if (opt.verbose)
@@ -1427,7 +1462,7 @@ Error in server response, closing control connection.\n"));
       else if (opt.noclobber || opt.always_rest || opt.timestamping || opt.dirstruct
                || opt.output_document || count > 0)
         {
-          if (opt.unlink && file_exists_p (con->target))
+          if (opt.unlink_requested && file_exists_p (con->target, NULL))
             {
               if (unlink (con->target) < 0)
                 {
@@ -1512,8 +1547,9 @@ Error in server response, closing control connection.\n"));
             {
               fd_close (csock);
               fd_close (dtsock);
+              err = CONERROR;
               logputs (LOG_NOTQUIET, "Could not perform SSL handshake.\n");
-              return CONERROR;
+              goto exit_error;
             }
         }
       else
@@ -1523,7 +1559,8 @@ Error in server response, closing control connection.\n"));
         {
           fd_close (csock);
           fd_close (dtsock);
-          return CONERROR;
+          err = CONERROR;
+          goto exit_error;
         }
     }
 #endif
@@ -1541,6 +1578,11 @@ Error in server response, closing control connection.\n"));
   tmrate = retr_rate (rd_size, con->dltime);
   total_download_time += con->dltime;
 
+#ifdef ENABLE_XATTR
+  if (opt.enable_xattr)
+    set_file_metadata (u->url, NULL, fp);
+#endif
+
   fd_close (local_sock);
   /* Close the local file.  */
   if (!output_stream || con->cmd & DO_LIST)
@@ -1636,10 +1678,10 @@ Error in server response, closing control connection.\n"));
 #ifdef __VMS
       char *targ;
 
-      targ = ods_conform( con->target);
+      targ = ods_conform (con->target);
       if (targ != con->target)
         {
-          xfree( con->target);
+          xfree (con->target);
           con->target = targ;
         }
 #endif /* def __VMS */
@@ -1760,6 +1802,13 @@ Error in server response, closing control connection.\n"));
     }
   } while (try_again);
   return RETRFINISHED;
+
+exit_error:
+
+  /* If fp is a regular file, close and try to remove it */
+  if (fp && !output_stream)
+    fclose (fp);
+  return err;
 }
 
 /* A one-file FTP loop.  This is the part where FTP retrieval is
@@ -1768,15 +1817,15 @@ Error in server response, closing control connection.\n"));
    This loop either gets commands from con, or (if ON_YOUR_OWN is
    set), makes them up to retrieve the file given by the URL.  */
 static uerr_t
-ftp_loop_internal (struct url *u, struct fileinfo *f, ccon *con, char **local_file,
-                   bool force_full_retrieve)
+ftp_loop_internal (struct url *u, struct url *original_url, struct fileinfo *f,
+                   ccon *con, char **local_file, bool force_full_retrieve)
 {
   int count, orig_lp;
   wgint restval, len = 0, qtyread = 0;
   char *tms, *locf;
   const char *tmrate = NULL;
   uerr_t err;
-  struct_stat st;
+  struct stat st;
 
   /* Declare WARC variables. */
   bool warc_enabled = (opt.warc_filename != NULL);
@@ -1794,7 +1843,7 @@ ftp_loop_internal (struct url *u, struct fileinfo *f, ccon *con, char **local_fi
     {
       /* URL-derived file.  Consider "-O file" name. */
       xfree (con->target);
-      con->target = url_file_name (u, NULL);
+      con->target = url_file_name (opt.trustservernames || !original_url ? u : original_url, NULL);
       if (!opt.output_document)
         locf = con->target;
       else
@@ -1807,7 +1856,7 @@ ftp_loop_internal (struct url *u, struct fileinfo *f, ccon *con, char **local_fi
   /* If we receive .listing file it is necessary to determine system type of the ftp
      server even if opn.noclobber is given. Thus we must ignore opt.noclobber in
      order to establish connection with the server and get system type. */
-  if (opt.noclobber && !opt.output_document && file_exists_p (con->target)
+  if (opt.noclobber && !opt.output_document && file_exists_p (con->target, NULL)
       && !((con->cmd & DO_LIST) && !(con->cmd & DO_RETR)))
     {
       logprintf (LOG_VERBOSE,
@@ -1912,8 +1961,8 @@ ftp_loop_internal (struct url *u, struct fileinfo *f, ccon *con, char **local_fi
 
       /* If we are working on a WARC record, getftp should also write
          to the warc_tmp file. */
-      err = getftp (u, len, &qtyread, restval, con, count, &last_expected_bytes,
-                    warc_tmp);
+      err = getftp (u, original_url, len, &qtyread, restval, con, count,
+                    &last_expected_bytes, warc_tmp);
 
       if (con->csock == -1)
         con->st &= ~DONE_CWD;
@@ -2079,9 +2128,10 @@ Removing file due to --delete-after in ftp_loop_internal():\n"));
 }
 
 /* Return the directory listing in a reusable format.  The directory
-   is specifed in u->dir.  */
+   is specified in u->dir.  */
 static uerr_t
-ftp_get_listing (struct url *u, ccon *con, struct fileinfo **f)
+ftp_get_listing (struct url *u, struct url *original_url, ccon *con,
+                 struct fileinfo **f)
 {
   uerr_t err;
   char *uf;                     /* url file name */
@@ -2102,7 +2152,7 @@ ftp_get_listing (struct url *u, ccon *con, struct fileinfo **f)
 
   con->target = xstrdup (lf);
   xfree (lf);
-  err = ftp_loop_internal (u, NULL, con, NULL, false);
+  err = ftp_loop_internal (u, original_url, NULL, con, NULL, false);
   lf = xstrdup (con->target);
   xfree (con->target);
   con->target = old_target;
@@ -2125,8 +2175,9 @@ ftp_get_listing (struct url *u, ccon *con, struct fileinfo **f)
   return err;
 }
 
-static uerr_t ftp_retrieve_dirs (struct url *, struct fileinfo *, ccon *);
-static uerr_t ftp_retrieve_glob (struct url *, ccon *, int);
+static uerr_t ftp_retrieve_dirs (struct url *, struct url *,
+                                 struct fileinfo *, ccon *);
+static uerr_t ftp_retrieve_glob (struct url *, struct url *, ccon *, int);
 static struct fileinfo *delelement (struct fileinfo *, struct fileinfo **);
 static void freefileinfo (struct fileinfo *f);
 
@@ -2138,7 +2189,8 @@ static void freefileinfo (struct fileinfo *f);
    If opt.recursive is set, after all files have been retrieved,
    ftp_retrieve_dirs will be called to retrieve the directories.  */
 static uerr_t
-ftp_retrieve_list (struct url *u, struct fileinfo *f, ccon *con)
+ftp_retrieve_list (struct url *u, struct url *original_url,
+                   struct fileinfo *f, ccon *con)
 {
   static int depth = 0;
   uerr_t err;
@@ -2196,7 +2248,7 @@ ftp_retrieve_list (struct url *u, struct fileinfo *f, ccon *con)
       dlthis = true;
       if (opt.timestamping && f->type == FT_PLAINFILE)
         {
-          struct_stat st;
+          struct stat st;
           /* If conversion of HTML files retrieved via FTP is ever implemented,
              we'll need to stat() <file>.orig here when -K has been specified.
              I'm not implementing it now since files on an FTP server are much
@@ -2259,7 +2311,7 @@ The sizes do not match (local %s) -- retrieving.\n\n"),
                          _("Invalid name of the symlink, skipping.\n"));
               else
                 {
-                  struct_stat st;
+                  struct stat st;
                   /* Check whether we already have the correct
                      symbolic link.  */
                   int rc = lstat (con->target, &st);
@@ -2299,7 +2351,10 @@ Already have correct symlink %s -> %s\n\n"),
           else                /* opt.retr_symlinks */
             {
               if (dlthis)
-                err = ftp_loop_internal (u, f, con, NULL, force_full_retrieve);
+                {
+                  err = ftp_loop_internal (u, original_url, f, con, NULL,
+                                           force_full_retrieve);
+                }
             } /* opt.retr_symlinks */
           break;
         case FT_DIRECTORY:
@@ -2310,9 +2365,13 @@ Already have correct symlink %s -> %s\n\n"),
         case FT_PLAINFILE:
           /* Call the retrieve loop.  */
           if (dlthis)
-            err = ftp_loop_internal (u, f, con, NULL, force_full_retrieve);
+            {
+              err = ftp_loop_internal (u, original_url, f, con, NULL,
+                                       force_full_retrieve);
+            }
           break;
         case FT_UNKNOWN:
+        default:
           logprintf (LOG_NOTQUIET, _("%s: unknown/unsupported file type.\n"),
                      quote (f->name));
           break;
@@ -2334,7 +2393,12 @@ Already have correct symlink %s -> %s\n\n"),
        (f->type == FT_PLAINFILE) && opt.preserve_perm)
         {
           if (f->perms)
-            chmod (actual_target, f->perms);
+            {
+              if (chmod (actual_target, f->perms))
+                logprintf (LOG_NOTQUIET,
+                           _("Failed to set permissions for %s.\n"),
+                           actual_target);
+            }
           else
             DEBUGP (("Unrecognized permissions for %s.\n", actual_target));
         }
@@ -2348,7 +2412,7 @@ Already have correct symlink %s -> %s\n\n"),
               && !(f->type == FT_SYMLINK && !opt.retr_symlinks)
               && f->tstamp != -1
               && dlthis
-              && file_exists_p (con->target))
+              && file_exists_p (con->target, NULL))
             {
               touch (actual_target, f->tstamp);
             }
@@ -2375,7 +2439,7 @@ Already have correct symlink %s -> %s\n\n"),
   /* We do not want to call ftp_retrieve_dirs here */
   if (opt.recursive &&
       !(opt.reclevel != INFINITE_RECURSION && depth >= opt.reclevel))
-    err = ftp_retrieve_dirs (u, orig, con);
+    err = ftp_retrieve_dirs (u, original_url, orig, con);
   else if (opt.recursive)
     DEBUGP ((_("Will not retrieve dirs since depth is %d (max %d).\n"),
              depth, opt.reclevel));
@@ -2388,7 +2452,8 @@ Already have correct symlink %s -> %s\n\n"),
    ftp_retrieve_glob on each directory entry.  The function knows
    about excluded directories.  */
 static uerr_t
-ftp_retrieve_dirs (struct url *u, struct fileinfo *f, ccon *con)
+ftp_retrieve_dirs (struct url *u, struct url *original_url,
+                   struct fileinfo *f, ccon *con)
 {
   char *container = NULL;
   int container_size = 0;
@@ -2438,7 +2503,7 @@ Not descending to %s as it is excluded/not-included.\n"),
       odir = xstrdup (u->dir);  /* because url_set_dir will free
                                    u->dir. */
       url_set_dir (u, newdir);
-      ftp_retrieve_glob (u, con, GLOB_GETALL);
+      ftp_retrieve_glob (u, original_url, con, GLOB_GETALL);
       url_set_dir (u, odir);
       xfree (odir);
 
@@ -2481,7 +2546,7 @@ is_invalid_entry (struct fileinfo *f)
   while (cur->next)
     {
       cur = cur->next;
-      if (strcmp(f_name, cur->name) == 0)
+      if (strcmp (f_name, cur->name) == 0)
           return true;
     }
   return false;
@@ -2497,14 +2562,15 @@ is_invalid_entry (struct fileinfo *f)
    GLOB_GLOBALL, use globbing; if it's GLOB_GETALL, download the whole
    directory.  */
 static uerr_t
-ftp_retrieve_glob (struct url *u, ccon *con, int action)
+ftp_retrieve_glob (struct url *u, struct url *original_url,
+                   ccon *con, int action)
 {
   struct fileinfo *f, *start;
   uerr_t res;
 
   con->cmd |= LEAVE_PENDING;
 
-  res = ftp_get_listing (u, con, &start);
+  res = ftp_get_listing (u, original_url, con, &start);
   if (res != RETROK)
     return res;
   /* First: weed out that do not conform the global rules given in
@@ -2600,7 +2666,7 @@ ftp_retrieve_glob (struct url *u, ccon *con, int action)
   if (start)
     {
       /* Just get everything.  */
-      res = ftp_retrieve_list (u, start, con);
+      res = ftp_retrieve_list (u, original_url, start, con);
     }
   else
     {
@@ -2616,7 +2682,7 @@ ftp_retrieve_glob (struct url *u, ccon *con, int action)
         {
           /* Let's try retrieving it anyway.  */
           con->st |= ON_YOUR_OWN;
-          res = ftp_loop_internal (u, NULL, con, NULL, false);
+          res = ftp_loop_internal (u, original_url, NULL, con, NULL, false);
           return res;
         }
 
@@ -2636,8 +2702,8 @@ ftp_retrieve_glob (struct url *u, ccon *con, int action)
    of URL.  Inherently, its capabilities are limited on what can be
    encoded into a URL.  */
 uerr_t
-ftp_loop (struct url *u, char **local_file, int *dt, struct url *proxy,
-          bool recursive, bool glob)
+ftp_loop (struct url *u, struct url *original_url, char **local_file, int *dt,
+          struct url *proxy, bool recursive, bool glob)
 {
   ccon con;                     /* FTP connection */
   uerr_t res;
@@ -2658,22 +2724,23 @@ ftp_loop (struct url *u, char **local_file, int *dt, struct url *proxy,
   if (!*u->file && !recursive)
     {
       struct fileinfo *f;
-      res = ftp_get_listing (u, &con, &f);
+      res = ftp_get_listing (u, original_url, &con, &f);
 
       if (res == RETROK)
         {
           if (opt.htmlify && !opt.spider)
             {
+              struct url *url_file = opt.trustservernames ? u : original_url;
               char *filename = (opt.output_document
                                 ? xstrdup (opt.output_document)
                                 : (con.target ? xstrdup (con.target)
-                                   : url_file_name (u, NULL)));
+                                   : url_file_name (url_file, NULL)));
               res = ftp_index (filename, u, f);
               if (res == FTPOK && opt.verbose)
                 {
                   if (!opt.output_document)
                     {
-                      struct_stat st;
+                      struct stat st;
                       wgint sz;
                       if (stat (filename, &st) == 0)
                         sz = st.st_size;
@@ -2712,11 +2779,13 @@ ftp_loop (struct url *u, char **local_file, int *dt, struct url *proxy,
           /* ftp_retrieve_glob is a catch-all function that gets called
              if we need globbing, time-stamping, recursion or preserve
              permissions.  Its third argument is just what we really need.  */
-          res = ftp_retrieve_glob (u, &con,
+          res = ftp_retrieve_glob (u, original_url, &con,
                                    ispattern ? GLOB_GLOBALL : GLOB_GETONE);
         }
       else
-        res = ftp_loop_internal (u, NULL, &con, local_file, false);
+        {
+          res = ftp_loop_internal (u, original_url, NULL, &con, local_file, false);
+        }
     }
   if (res == FTPOK)
     res = RETROK;

src/ftp.h

@@ -1,7 +1,5 @@
 /* Declarations for FTP support.
-   Copyright (C) 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004,
-   2005, 2006, 2007, 2008, 2009, 2010, 2011, 2015 Free Software
-   Foundation, Inc.
+   Copyright (C) 1996-2011, 2015, 2018 Free Software Foundation, Inc.
 
 This file is part of GNU Wget.
 
@@ -169,7 +167,8 @@ enum wget_ftp_fstatus
 };
 
 struct fileinfo *ftp_parse_ls (const char *, const enum stype);
-uerr_t ftp_loop (struct url *, char **, int *, struct url *, bool, bool);
+uerr_t ftp_loop (struct url *, struct url *, char **, int *, struct url *,
+                 bool, bool);
 
 uerr_t ftp_index (const char *, struct url *, struct fileinfo *);
 

src/gnutls.c

@@ -1,6 +1,5 @@
 /* SSL support via GnuTLS library.
-   Copyright (C) 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012, 2015
-   Free Software Foundation, Inc.
+   Copyright (C) 2005-2012, 2015, 2018 Free Software Foundation, Inc.
 
 This file is part of GNU Wget.
 
@@ -36,7 +35,9 @@ as that of the covered work.  */
 #include <stdio.h>
 #include <dirent.h>
 #include <stdlib.h>
+#include <xalloc.h>
 
+#include <gnutls/abstract.h>
 #include <gnutls/gnutls.h>
 #include <gnutls/x509.h>
 #include <sys/ioctl.h>
@@ -48,7 +49,7 @@ as that of the covered work.  */
 #include "hash.h"
 #include "ssl.h"
 
-#include <sys/fcntl.h>
+#include <fcntl.h>
 
 #ifdef WIN32
 # include "w32sock.h"
@@ -496,7 +497,7 @@ _do_handshake (gnutls_session_t session, int fd, double timeout)
             {
               gnutls_alert_description_t alert = gnutls_alert_get (session);
               const char *str = gnutls_alert_get_name (alert);
-              logprintf (LOG_NOTQUIET, "GnuTLS: received alert [%d]: %s\n",
+              logprintf (LOG_NOTQUIET, "GnuTLS: received alert [%u]: %s\n",
                          alert, str ? str : "(unknown)");
             }
         }
@@ -518,6 +519,22 @@ _do_handshake (gnutls_session_t session, int fd, double timeout)
   return err;
 }
 
+static const char *
+_sni_hostname(const char *hostname)
+{
+  size_t len = strlen(hostname);
+
+  char *sni_hostname = xmemdup(hostname, len + 1);
+
+  /* Remove trailing dot(s) to fix #47408.
+   * Regarding RFC 6066 (SNI): The hostname is represented as a byte
+   * string using ASCII encoding without a trailing dot. */
+  while (len && sni_hostname[--len] == '.')
+    sni_hostname[len] = 0;
+
+  return sni_hostname;
+}
+
 bool
 ssl_connect_wget (int fd, const char *hostname, int *continue_session)
 {
@@ -530,11 +547,14 @@ ssl_connect_wget (int fd, const char *hostname, int *continue_session)
   /* We set the server name but only if it's not an IP address. */
   if (! is_valid_ip_address (hostname))
     {
-      gnutls_server_name_set (session, GNUTLS_NAME_DNS, hostname,
-                              strlen (hostname));
+      /* GnuTLS 3.4.x (x<=10) disrespects the length parameter, we have to construct a new string */
+      /* see https://gitlab.com/gnutls/gnutls/issues/78 */
+      const char *sni_hostname = _sni_hostname(hostname);
+
+      gnutls_server_name_set (session, GNUTLS_NAME_DNS, sni_hostname, strlen(sni_hostname));
+      xfree(sni_hostname);
     }
 
-  gnutls_set_default_priority (session);
   gnutls_credentials_set (session, GNUTLS_CRD_CERTIFICATE, credentials);
 #ifndef FD_TO_SOCKET
 # define FD_TO_SOCKET(X) (X)
@@ -549,7 +569,8 @@ ssl_connect_wget (int fd, const char *hostname, int *continue_session)
   switch (opt.secure_protocol)
     {
     case secure_protocol_auto:
-      err = gnutls_priority_set_direct (session, "NORMAL:%COMPAT:-VERS-SSL3.0", NULL);
+      err = gnutls_set_default_priority (session);
+      gnutls_session_enable_compatibility_mode(session);
       break;
 
     case secure_protocol_sslv2:
@@ -577,7 +598,8 @@ ssl_connect_wget (int fd, const char *hostname, int *continue_session)
       break;
 
     default:
-      logprintf (LOG_NOTQUIET, _("GnuTLS: unimplemented 'secure-protocol' option value %d\n"), opt.secure_protocol);
+      logprintf (LOG_NOTQUIET, _("GnuTLS: unimplemented 'secure-protocol' option value %u\n"),
+                 (unsigned) opt.secure_protocol);
       logprintf (LOG_NOTQUIET, _("Please report this issue to bug-wget@gnu.org\n"));
       abort ();
     }
@@ -586,6 +608,7 @@ ssl_connect_wget (int fd, const char *hostname, int *continue_session)
   switch (opt.secure_protocol)
     {
     case secure_protocol_auto:
+      err = gnutls_set_default_priority (session);
       break;
 
     case secure_protocol_sslv2:
@@ -633,9 +656,13 @@ ssl_connect_wget (int fd, const char *hostname, int *continue_session)
         {
           if (!ctx || !ctx->session_data || gnutls_session_set_data (session, ctx->session_data->data, ctx->session_data->size))
             {
-              /* server does not want to continue the session */
-              gnutls_free (ctx->session_data->data);
-              gnutls_free (ctx->session_data);
+              if (ctx && ctx->session_data)
+                {
+                  /* server does not want to continue the session */
+                  if (ctx->session_data->data)
+                    gnutls_free (ctx->session_data->data);
+                  gnutls_free (ctx->session_data);
+                }
               gnutls_deinit (session);
               return false;
             }
@@ -667,6 +694,59 @@ ssl_connect_wget (int fd, const char *hostname, int *continue_session)
   return true;
 }
 
+static bool
+pkp_pin_peer_pubkey (gnutls_x509_crt_t cert, const char *pinnedpubkey)
+{
+  /* Scratch */
+  size_t len1 = 0, len2 = 0;
+  char *buff1 = NULL;
+
+  gnutls_pubkey_t key = NULL;
+
+  /* Result is returned to caller */
+  int ret = 0;
+  bool result = false;
+
+  /* if a path wasn't specified, don't pin */
+  if (NULL == pinnedpubkey)
+    return true;
+
+  if (NULL == cert)
+    return result;
+
+  /* Begin Gyrations to get the public key     */
+  gnutls_pubkey_init (&key);
+
+  ret = gnutls_pubkey_import_x509 (key, cert, 0);
+  if (ret < 0)
+    goto cleanup; /* failed */
+
+  ret = gnutls_pubkey_export (key, GNUTLS_X509_FMT_DER, NULL, &len1);
+  if (ret != GNUTLS_E_SHORT_MEMORY_BUFFER || len1 == 0)
+    goto cleanup; /* failed */
+
+  buff1 = xmalloc (len1);
+
+  len2 = len1;
+
+  ret = gnutls_pubkey_export (key, GNUTLS_X509_FMT_DER, buff1, &len2);
+  if (ret < 0 || len1 != len2)
+    goto cleanup; /* failed */
+
+  /* End Gyrations */
+
+  /* The one good exit point */
+  result = wg_pin_peer_pubkey (pinnedpubkey, buff1, len1);
+
+ cleanup:
+  if (NULL != key)
+    gnutls_pubkey_deinit (key);
+
+  xfree (buff1);
+
+  return result;
+}
+
 #define _CHECK_CERT(flag,msg) \
   if (status & (flag))\
     {\
@@ -687,6 +767,11 @@ ssl_check_certificate (int fd, const char *host)
      him about problems with the server's certificate.  */
   const char *severity = opt.check_cert ? _("ERROR") : _("WARNING");
   bool success = true;
+  bool pinsuccess = opt.pinnedpubkey == NULL;
+
+  /* The user explicitly said to not check for the certificate.  */
+  if (opt.check_cert == CHECK_CERT_QUIET && pinsuccess)
+    return success;
 
   err = gnutls_certificate_verify_peers2 (ctx->session, &status);
   if (err < 0)
@@ -711,6 +796,7 @@ ssl_check_certificate (int fd, const char *host)
       gnutls_x509_crt_t cert;
       const gnutls_datum_t *cert_list;
       unsigned int cert_list_size;
+      const char *sni_hostname;
 
       if ((err = gnutls_x509_crt_init (&cert)) < 0)
         {
@@ -745,13 +831,23 @@ ssl_check_certificate (int fd, const char *host)
           logprintf (LOG_NOTQUIET, _("The certificate has expired\n"));
           success = false;
         }
-      if (!gnutls_x509_crt_check_hostname (cert, host))
+      sni_hostname = _sni_hostname(host);
+      if (!gnutls_x509_crt_check_hostname (cert, sni_hostname))
         {
           logprintf (LOG_NOTQUIET,
                      _("The certificate's owner does not match hostname %s\n"),
-                     quote (host));
+                     quote (sni_hostname));
           success = false;
         }
+      xfree(sni_hostname);
+
+      pinsuccess = pkp_pin_peer_pubkey (cert, opt.pinnedpubkey);
+      if (!pinsuccess)
+        {
+          logprintf (LOG_ALWAYS, _("The public key does not match pinned public key!\n"));
+          success = false;
+        }
+
  crt_deinit:
       gnutls_x509_crt_deinit (cert);
     }
@@ -762,5 +858,6 @@ ssl_check_certificate (int fd, const char *host)
     }
 
  out:
-  return opt.check_cert ? success : true;
+  /* never return true if pinsuccess fails */
+  return !pinsuccess ? false : (opt.check_cert == CHECK_CERT_ON ? success : true);
 }

src/hash.c

@@ -1,6 +1,5 @@
 /* Hash tables.
-   Copyright (C) 2000, 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008,
-   2009, 2010, 2011, 2015 Free Software Foundation, Inc.
+   Copyright (C) 2000-2011, 2015, 2018 Free Software Foundation, Inc.
 
 This file is part of GNU Wget.
 
@@ -283,10 +282,10 @@ hash_table_new (int items,
 
   /* Calculate the size that ensures that the table will store at
      least ITEMS keys without the need to resize.  */
-  size = 1 + items / HASH_MAX_FULLNESS;
+  size = (int) (1 + items / HASH_MAX_FULLNESS);
   size = prime_size (size, &ht->prime_offset);
   ht->size = size;
-  ht->resize_threshold = size * HASH_MAX_FULLNESS;
+  ht->resize_threshold = (int) (size * HASH_MAX_FULLNESS);
   /*assert (ht->resize_threshold >= items);*/
 
   ht->cells = xnew_array (struct cell, ht->size);
@@ -394,7 +393,7 @@ grow_hash_table (struct hash_table *ht)
 #endif
 
   ht->size = newsize;
-  ht->resize_threshold = newsize * HASH_MAX_FULLNESS;
+  ht->resize_threshold = (int) (newsize * HASH_MAX_FULLNESS);
 
   cells = xnew_array (struct cell, newsize);
   memset (cells, INVALID_PTR_CHAR, newsize * sizeof (struct cell));

src/hash.h

@@ -1,6 +1,6 @@
 /* Hash table declarations.
-   Copyright (C) 2000, 2007, 2008, 2009, 2010, 2011, 2015 Free Software
-   Foundation, Inc.
+   Copyright (C) 2000, 2007-2011, 2015, 2018 Free Software Foundation,
+   Inc.
 
 This file is part of GNU Wget.
 

src/host.c

@@ -1,7 +1,5 @@
 /* Host name resolution and matching.
-   Copyright (C) 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004,
-   2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012, 2015 Free Software
-   Foundation, Inc.
+   Copyright (C) 1996-2012, 2015, 2018 Free Software Foundation, Inc.
 
 This file is part of GNU Wget.
 
@@ -57,14 +55,11 @@ as that of the covered work.  */
 
 #include <errno.h>
 
-#ifdef ENABLE_IRI
-#include <idn-free.h>
-#endif
-
 #include "utils.h"
 #include "host.h"
 #include "url.h"
 #include "hash.h"
+#include "ptimer.h"
 
 #ifndef NO_ADDRESS
 # define NO_ADDRESS NO_DATA
@@ -427,14 +422,12 @@ getaddrinfo_with_timeout (const char *node, const char *service,
 const char *
 print_address (const ip_address *addr)
 {
-#ifdef ENABLE_IPV6
   static char buf[64];
+
   if (!inet_ntop (addr->family, IP_INADDR_DATA (addr), buf, sizeof buf))
     snprintf (buf, sizeof buf, "<error: %s>", strerror (errno));
+
   return buf;
-#else
-  return inet_ntoa (addr->data.d4);
-#endif
 }
 
 /* The following two functions were adapted from glibc's
@@ -649,6 +642,112 @@ cache_remove (const char *host)
     }
 }
 
+#ifdef HAVE_LIBCARES
+#include <sys/select.h>
+#include <ares.h>
+extern ares_channel ares;
+
+static struct address_list *
+merge_address_lists (struct address_list *al1, struct address_list *al2)
+{
+  int count = al1->count + al2->count;
+
+  /* merge al2 into al1 */
+  al1->addresses = xrealloc (al1->addresses, sizeof (ip_address) * count);
+  memcpy (al1->addresses + al1->count, al2->addresses, sizeof (ip_address) * al2->count);
+  al1->count = count;
+
+  address_list_delete (al2);
+
+  return al1;
+}
+
+static struct address_list *
+address_list_from_hostent (struct hostent *host)
+{
+  int count, i;
+  struct address_list *al = xnew0 (struct address_list);
+
+  for (count = 0; host->h_addr_list[count]; count++)
+    ;
+
+  assert (count > 0);
+
+  al->addresses = xnew_array (ip_address, count);
+  al->count     = count;
+  al->refcount  = 1;
+
+  for (i = 0; i < count; i++)
+    {
+      ip_address *ip = &al->addresses[i];
+      ip->family = host->h_addrtype;
+      memcpy (IP_INADDR_DATA (ip), host->h_addr_list[i], ip->family == AF_INET ? 4 : 16);
+    }
+
+  return al;
+}
+
+/* Since GnuLib's select() (i.e. rpl_select()) cannot handle socket-numbers
+ * returned from C-ares, we must use the original select() from Winsock.
+ */
+#ifdef WINDOWS
+#undef select
+#endif
+
+static void
+wait_ares (ares_channel channel)
+{
+  struct ptimer *timer = NULL;
+
+  if (opt.dns_timeout)
+    timer = ptimer_new ();
+
+  for (;;)
+    {
+      struct timeval *tvp, tv;
+      fd_set read_fds, write_fds;
+      int nfds, rc;
+
+      FD_ZERO (&read_fds);
+      FD_ZERO (&write_fds);
+      nfds = ares_fds (channel, &read_fds, &write_fds);
+      if (nfds == 0)
+        break;
+
+      if (timer)
+        {
+          double max = opt.dns_timeout - ptimer_measure (timer);
+
+          tv.tv_sec = (long) max;
+          tv.tv_usec = 1000000 * (max - (long) max);
+          tvp = ares_timeout (channel, &tv, &tv);
+        }
+      else
+        tvp = ares_timeout (channel, NULL, &tv);
+
+      rc = select (nfds, &read_fds, &write_fds, NULL, tvp);
+      if (rc == 0 && timer && ptimer_measure (timer) >= opt.dns_timeout)
+        ares_cancel (channel);
+      else
+        ares_process (channel, &read_fds, &write_fds);
+    }
+}
+
+static void
+callback (void *arg, int status, int timeouts _GL_UNUSED, struct hostent *host)
+{
+  struct address_list **al = (struct address_list **) arg;
+
+  if (!host || status != ARES_SUCCESS)
+    {
+      *al = NULL;
+      return;
+    }
+
+  *al = address_list_from_hostent (host);
+}
+#endif
+
 /* Look up HOST in DNS and return a list of IP addresses.
 
    This function caches its result so that, if the same host is passed
@@ -741,11 +840,8 @@ lookup_host (const char *host, int flags)
 
       if (opt.enable_iri && (name = idn_decode ((char *) host)) != NULL)
         {
-          int len = strlen (host) + strlen (name) + 4;
-          str = xmalloc (len);
-          snprintf (str, len, "%s (%s)", name, host);
-          str[len-1] = '\0';
-          idn_free (name);
+          str = aprintf ("%s (%s)", name, host);
+          xfree (name);
         }
 
       logprintf (LOG_VERBOSE, _("Resolving %s... "),
@@ -755,80 +851,112 @@ lookup_host (const char *host, int flags)
     }
 
 #ifdef ENABLE_IPV6
-  {
-    int err;
-    struct addrinfo hints, *res;
+#ifdef HAVE_LIBCARES
+  if (ares)
+    {
+      struct address_list *al4;
+      struct address_list *al6;
 
-    xzero (hints);
-    hints.ai_socktype = SOCK_STREAM;
-    if (opt.ipv4_only)
-      hints.ai_family = AF_INET;
-    else if (opt.ipv6_only)
-      hints.ai_family = AF_INET6;
-    else
-      /* We tried using AI_ADDRCONFIG, but removed it because: it
-         misinterprets IPv6 loopbacks, it is broken on AIX 5.1, and
-         it's unneeded since we sort the addresses anyway.  */
+      if (opt.ipv4_only || !opt.ipv6_only)
+        ares_gethostbyname (ares, host, AF_INET, callback, &al4);
+      if (opt.ipv6_only || !opt.ipv4_only)
+        ares_gethostbyname (ares, host, AF_INET6, callback, &al6);
+
+      wait_ares (ares);
+
+      if (al4 && al6)
+        al = merge_address_lists (al4, al6);
+      else if (al4)
+        al = al4;
+      else
+        al = al6;
+    }
+  else
+#endif
+    {
+      int err;
+      struct addrinfo hints, *res;
+
+      xzero (hints);
+      hints.ai_socktype = SOCK_STREAM;
+      if (opt.ipv4_only)
+        hints.ai_family = AF_INET;
+      else if (opt.ipv6_only)
+        hints.ai_family = AF_INET6;
+      else
+        /* We tried using AI_ADDRCONFIG, but removed it because: it
+           misinterprets IPv6 loopbacks, it is broken on AIX 5.1, and
+           it's unneeded since we sort the addresses anyway.  */
         hints.ai_family = AF_UNSPEC;
 
-    if (flags & LH_BIND)
-      hints.ai_flags |= AI_PASSIVE;
+      if (flags & LH_BIND)
+        hints.ai_flags |= AI_PASSIVE;
 
 #ifdef AI_NUMERICHOST
-    if (numeric_address)
-      {
-        /* Where available, the AI_NUMERICHOST hint can prevent costly
-           access to DNS servers.  */
-        hints.ai_flags |= AI_NUMERICHOST;
-        timeout = 0;            /* no timeout needed when "resolving"
+      if (numeric_address)
+        {
+          /* Where available, the AI_NUMERICHOST hint can prevent costly
+             access to DNS servers.  */
+          hints.ai_flags |= AI_NUMERICHOST;
+          timeout = 0; /* no timeout needed when "resolving"
                                    numeric hosts -- avoid setting up
                                    signal handlers and such. */
-      }
+        }
 #endif
 
-    err = getaddrinfo_with_timeout (host, NULL, &hints, &res, timeout);
-    if (err != 0 || res == NULL)
-      {
-        if (!silent)
-          logprintf (LOG_VERBOSE, _("failed: %s.\n"),
-                     err != EAI_SYSTEM ? gai_strerror (err) : strerror (errno));
-        return NULL;
-      }
-    al = address_list_from_addrinfo (res);
-    freeaddrinfo (res);
-    if (!al)
-      {
-        logprintf (LOG_VERBOSE,
-                   _("failed: No IPv4/IPv6 addresses for host.\n"));
-        return NULL;
-      }
+      err = getaddrinfo_with_timeout (host, NULL, &hints, &res, timeout);
 
-    /* Reorder addresses so that IPv4 ones (or IPv6 ones, as per
-       --prefer-family) come first.  Sorting is stable so the order of
-       the addresses with the same family is undisturbed.  */
-    if (al->count > 1 && opt.prefer_family != prefer_none)
-      stable_sort (al->addresses, al->count, sizeof (ip_address),
-                   opt.prefer_family == prefer_ipv4
-                   ? cmp_prefer_ipv4 : cmp_prefer_ipv6);
-  }
+      if (err != 0 || res == NULL)
+        {
+          if (!silent)
+            logprintf (LOG_VERBOSE, _ ("failed: %s.\n"),
+                       err != EAI_SYSTEM ? gai_strerror (err) : strerror (errno));
+          return NULL;
+        }
+      al = address_list_from_addrinfo (res);
+      freeaddrinfo (res);
+    }
+
+  if (!al)
+    {
+      logprintf (LOG_VERBOSE,
+                 _ ("failed: No IPv4/IPv6 addresses for host.\n"));
+      return NULL;
+    }
+
+  /* Reorder addresses so that IPv4 ones (or IPv6 ones, as per
+     --prefer-family) come first.  Sorting is stable so the order of
+     the addresses with the same family is undisturbed.  */
+  if (al->count > 1 && opt.prefer_family != prefer_none)
+    stable_sort (al->addresses, al->count, sizeof (ip_address),
+                 opt.prefer_family == prefer_ipv4
+                 ? cmp_prefer_ipv4 : cmp_prefer_ipv6);
 #else  /* not ENABLE_IPV6 */
-  {
-    struct hostent *hptr = gethostbyname_with_timeout (host, timeout);
-    if (!hptr)
-      {
-        if (!silent)
-          {
-            if (errno != ETIMEDOUT)
-              logprintf (LOG_VERBOSE, _("failed: %s.\n"),
-                         host_errstr (h_errno));
-            else
-              logputs (LOG_VERBOSE, _("failed: timed out.\n"));
-          }
-        return NULL;
-      }
-    /* Do older systems have h_addr_list?  */
-    al = address_list_from_ipv4_addresses (hptr->h_addr_list);
-  }
+#ifdef HAVE_LIBCARES
+  if (ares)
+    {
+      ares_gethostbyname (ares, host, AF_INET, callback, &al);
+      wait_ares (ares);
+    }
+  else
+#endif
+    {
+      struct hostent *hptr = gethostbyname_with_timeout (host, timeout);
+      if (!hptr)
+        {
+          if (!silent)
+            {
+              if (errno != ETIMEDOUT)
+                logprintf (LOG_VERBOSE, _ ("failed: %s.\n"),
+                           host_errstr (h_errno));
+              else
+                logputs (LOG_VERBOSE, _ ("failed: timed out.\n"));
+            }
+          return NULL;
+        }
+      /* Do older systems have h_addr_list?  */
+      al = address_list_from_ipv4_addresses (hptr->h_addr_list);
+    }
 #endif /* not ENABLE_IPV6 */
 
   /* Print the addresses determined by DNS lookup, but no more than
@@ -889,18 +1017,25 @@ sufmatch (const char **list, const char *what)
   int i, j, k, lw;
 
   lw = strlen (what);
+
   for (i = 0; list[i]; i++)
     {
-      if (list[i][0] == '\0')
-        continue;
+      j = strlen (list[i]);
+      if (lw < j)
+        continue; /* what is no (sub)domain of list[i] */
 
-      for (j = strlen (list[i]), k = lw; j >= 0 && k >= 0; j--, k--)
+      for (k = lw; j >= 0 && k >= 0; j--, k--)
         if (c_tolower (list[i][j]) != c_tolower (what[k]))
           break;
-      /* The domain must be first to reach to beginning.  */
-      if (j == -1)
+
+      /* Domain or subdomain match
+       * k == -1: exact match
+       * k >= 0 && what[k] == '.': subdomain match
+       */
+      if (j == -1 && (k == -1 || what[k] == '.'))
         return true;
     }
+
   return false;
 }
 

src/host.h

@@ -1,7 +1,5 @@
 /* Declarations for host.c
-   Copyright (C) 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004,
-   2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012, 2015 Free Software
-   Foundation, Inc.
+   Copyright (C) 1996-2012, 2015, 2018 Free Software Foundation, Inc.
 
 This file is part of GNU Wget.
 

src/hsts.c

@@ -1,7 +1,5 @@
 /* HTTP Strict Transport Security (HSTS) support.
-   Copyright (C) 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004,
-   2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012, 2015 Free Software
-   Foundation, Inc.
+   Copyright (C) 1996-2012, 2015, 2018 Free Software Foundation, Inc.
 
 This file is part of GNU Wget.
 
@@ -32,9 +30,9 @@ as that of the covered work.  */
 
 #ifdef HAVE_HSTS
 #include "hsts.h"
+#include "utils.h"
 #include "host.h" /* for is_valid_ip_address() */
 #include "init.h" /* for home_dir() */
-#include "utils.h"
 #include "hash.h"
 #include "c-ctype.h"
 #ifdef TESTING
@@ -53,6 +51,7 @@ as that of the covered work.  */
 struct hsts_store {
   struct hash_table *table;
   time_t last_mtime;
+  bool changed;
 };
 
 struct hsts_kh {
@@ -79,7 +78,6 @@ enum hsts_kh_match {
 
 #define DEFAULT_HTTP_PORT 80
 #define DEFAULT_SSL_PORT  443
-#define CHECK_EXPLICIT_PORT(p1, p2) (p1 == 0 || p1 == p2)
 #define MAKE_EXPLICIT_PORT(s, p) (s == SCHEME_HTTPS ? (p == DEFAULT_SSL_PORT ? 0 : p) \
     : (p == DEFAULT_HTTP_PORT ? 0 : p))
 
@@ -148,13 +146,14 @@ hsts_find_entry (hsts_store_t store,
 end:
   /* restore pointer or we'll get a SEGV */
   k->host = org_ptr;
-  xfree (k->host);
 
   /* copy parameters to previous frame */
   if (match_type)
     *match_type = match;
   if (kh)
     memcpy (kh, k, sizeof (struct hsts_kh));
+  else
+    xfree (k->host);
 
   xfree (k);
   return khi;
@@ -236,8 +235,7 @@ hsts_new_entry (hsts_store_t store,
 static void
 hsts_remove_entry (hsts_store_t store, struct hsts_kh *kh)
 {
-  if (hash_table_remove (store->table, kh))
-    xfree (kh->host);
+  hash_table_remove (store->table, kh);
 }
 
 static bool
@@ -316,7 +314,7 @@ hsts_store_dump (hsts_store_t store, FILE *fp)
   /* Print preliminary comments. We don't care if any of these fail. */
   fputs ("# HSTS 1.0 Known Hosts database for GNU Wget.\n", fp);
   fputs ("# Edit at your own risk.\n", fp);
-  fputs ("# <hostname>[:<port>]\t<incl. subdomains>\t<created>\t<max-age>\n", fp);
+  fputs ("# <hostname>\t<port>\t<incl. subdomains>\t<created>\t<max-age>\n", fp);
 
   /* Now cycle through the HSTS store in memory and dump the entries */
   for (hash_table_iterate (store->table, &it); hash_table_iter_next (&it);)
@@ -326,7 +324,8 @@ hsts_store_dump (hsts_store_t store, FILE *fp)
 
       if (fprintf (fp, "%s\t%d\t%d\t%lu\t%lu\n",
                    kh->host, kh->explicit_port, khi->include_subdomains,
-                   khi->created, khi->max_age) < 0)
+                   (unsigned long) khi->created,
+                   (unsigned long) khi->max_age) < 0)
         {
           logprintf (LOG_ALWAYS, "Could not write the HSTS database correctly.\n");
           break;
@@ -334,6 +333,30 @@ hsts_store_dump (hsts_store_t store, FILE *fp)
     }
 }
 
+/*
+ * Test:
+ *  - The file is a regular file (ie. not a symlink), and
+ *  - The file is not world-writable.
+ */
+static bool
+hsts_file_access_valid (const char *filename)
+{
+  struct stat st;
+
+  if (stat (filename, &st) == -1)
+    return false;
+
+  return
+#ifndef WINDOWS
+      /*
+       * The world-writable concept is a Unix-centric notion.
+       * We bypass this test on Windows.
+       */
+      !(st.st_mode & S_IWOTH) &&
+#endif
+      S_ISREG (st.st_mode);
+}
+
 /* HSTS API */
 
 /*
@@ -370,14 +393,19 @@ hsts_match (hsts_store_t store, struct url *u)
                   if (u->port == 80)
                     u->port = 443;
                   url_changed = true;
+                  store->changed = true;
                 }
             }
           else
-            hsts_remove_entry (store, kh);
+            {
+              hsts_remove_entry (store, kh);
+              store->changed = true;
+            }
         }
+      xfree (kh->host);
     }
 
-  xfree(kh);
+  xfree (kh);
 
   return url_changed;
 }
@@ -413,7 +441,6 @@ hsts_store_entry (hsts_store_t store,
   enum hsts_kh_match match = NO_MATCH;
   struct hsts_kh *kh = xnew(struct hsts_kh);
   struct hsts_kh_info *entry = NULL;
-  time_t t = 0;
 
   if (hsts_is_host_eligible (scheme, host))
     {
@@ -422,19 +449,25 @@ hsts_store_entry (hsts_store_t store,
       if (entry && match == CONGRUENT_MATCH)
         {
           if (max_age == 0)
-            hsts_remove_entry (store, kh);
+            {
+              hsts_remove_entry (store, kh);
+              store->changed = true;
+            }
           else if (max_age > 0)
             {
-              entry->include_subdomains = include_subdomains;
+              /* RFC 6797 states that 'max_age' is a TTL relative to the
+               * reception of the STS header so we have to update the
+               * 'created' field too. The RFC also states that we have to
+               * update the entry each time we see HSTS header.
+               * See also Section 11.2. */
+              time_t t = time (NULL);
 
-              if (entry->max_age != max_age)
+              if (t != -1 && t != entry->created)
                 {
-                  /* RFC 6797 states that 'max_age' is a TTL relative to the reception of the STS header
-                     so we have to update the 'created' field too */
-                  t = time (NULL);
-                  if (t != -1)
-                    entry->created = t;
+                  entry->created = t;
                   entry->max_age = max_age;
+                  entry->include_subdomains = include_subdomains;
+                  store->changed = true;
                 }
             }
           /* we ignore negative max_ages */
@@ -449,11 +482,14 @@ hsts_store_entry (hsts_store_t store,
              happen we got a non-existent entry with max_age == 0.
           */
           result = hsts_add_entry (store, host, port, max_age, include_subdomains);
+          if (result)
+            store->changed = true;
         }
       /* we ignore new entries with max_age == 0 */
+      xfree (kh->host);
     }
 
-  xfree(kh);
+  xfree (kh);
 
   return result;
 }
@@ -462,29 +498,50 @@ hsts_store_t
 hsts_store_open (const char *filename)
 {
   hsts_store_t store = NULL;
-  struct stat st;
-  FILE *fp = NULL;
+  file_stats_t fstats;
 
   store = xnew0 (struct hsts_store);
   store->table = hash_table_new (0, hsts_hash_func, hsts_cmp_func);
   store->last_mtime = 0;
+  store->changed = false;
 
-  if (file_exists_p (filename))
+  if (file_exists_p (filename, &fstats))
     {
-      if (stat (filename, &st) == 0)
-        store->last_mtime = st.st_mtime;
-
-      fp = fopen (filename, "r");
-      if (!fp || !hsts_read_database (store, fp, false))
+      if (hsts_file_access_valid (filename))
         {
-          /* abort! */
+          struct stat st;
+          FILE *fp = fopen_stat (filename, "r", &fstats);
+
+          if (!fp || !hsts_read_database (store, fp, false))
+            {
+              /* abort! */
+              hsts_store_close (store);
+              xfree (store);
+              if (fp)
+                fclose (fp);
+              goto out;
+            }
+
+          if (fstat (fileno (fp), &st) == 0)
+            store->last_mtime = st.st_mtime;
+
+          fclose (fp);
+        }
+      else
+        {
+          /*
+           * If we're not reading the HSTS database,
+           * then by all means act as if HSTS was disabled.
+           */
           hsts_store_close (store);
           xfree (store);
+
+          logprintf (LOG_NOTQUIET, "Will not apply HSTS. "
+                     "The HSTS database must be a regular and non-world-writable file.\n");
         }
-      if (fp)
-        fclose (fp);
     }
 
+out:
   return store;
 }
 
@@ -525,6 +582,12 @@ hsts_store_save (hsts_store_t store, const char *filename)
     }
 }
 
+bool
+hsts_store_has_changed (hsts_store_t store)
+{
+  return (store ? store->changed : false);
+}
+
 void
 hsts_store_close (hsts_store_t store)
 {
@@ -613,7 +676,7 @@ test_url_rewrite (hsts_store_t s, const char *url, int port, bool rewrite)
   if (rewrite)
     {
       if (port == 80)
-       mu_assert("URL: port should've been rewritten to 443", u.port == 443);
+        mu_assert("URL: port should've been rewritten to 443", u.port == 443);
       else
         mu_assert("URL: port should've been left intact", u.port == port);
       mu_assert("URL: scheme should've been rewritten to HTTPS", u.scheme == SCHEME_HTTPS);
@@ -686,7 +749,7 @@ test_hsts_url_rewrite_superdomain (void)
   s = open_hsts_test_store ();
   mu_assert("Could not open the HSTS store", s != NULL);
 
-  created = hsts_store_entry (s, SCHEME_HTTPS, "www.foo.com", 443, time(NULL) + 1234, true);
+  created = hsts_store_entry (s, SCHEME_HTTPS, "www.foo.com", 443, 1234, true);
   mu_assert("A new entry should've been created", created == true);
 
   TEST_URL_RW (s, "www.foo.com", 80);
@@ -707,7 +770,7 @@ test_hsts_url_rewrite_congruent (void)
   s = open_hsts_test_store ();
   mu_assert("Could not open the HSTS store", s != NULL);
 
-  created = hsts_store_entry (s, SCHEME_HTTPS, "foo.com", 443, time(NULL) + 1234, false);
+  created = hsts_store_entry (s, SCHEME_HTTPS, "foo.com", 443, 1234, false);
   mu_assert("A new entry should've been created", created == true);
 
   TEST_URL_RW (s, "foo.com", 80);
@@ -726,6 +789,7 @@ test_hsts_read_database (void)
   char *home = home_dir();
   char *file = NULL;
   FILE *fp = NULL;
+  time_t created = time(NULL) - 10;
 
   if (home)
     {
@@ -734,9 +798,9 @@ test_hsts_read_database (void)
       if (fp)
         {
           fputs ("# dummy comment\n", fp);
-          fputs ("foo.example.com\t0\t1\t1434224817\t123123123\n", fp);
-          fputs ("bar.example.com\t0\t0\t1434224817\t456456456\n", fp);
-          fputs ("test.example.com\t8080\t0\t1434224817\t789789789\n", fp);
+          fprintf (fp, "foo.example.com\t0\t1\t%lu\t123\n",(unsigned long) created);
+          fprintf (fp, "bar.example.com\t0\t0\t%lu\t456\n", (unsigned long) created);
+          fprintf (fp, "test.example.com\t8080\t0\t%lu\t789\n", (unsigned long) created);
           fclose (fp);
 
           table = hsts_store_open (file);

src/hsts.h

@@ -1,7 +1,5 @@
 /* Declarations for hsts.c
-   Copyright (C) 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004,
-   2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012, 2015 Free Software
-   Foundation, Inc.
+   Copyright (C) 1996-2012, 2015, 2018 Free Software Foundation, Inc.
 
 This file is part of GNU Wget.
 
@@ -43,6 +41,7 @@ hsts_store_t hsts_store_open (const char *);
 
 void hsts_store_save (hsts_store_t, const char *);
 void hsts_store_close (hsts_store_t);
+bool hsts_store_has_changed (hsts_store_t);
 
 bool hsts_store_entry (hsts_store_t,
                        enum url_scheme, const char *, int,

src/html-parse.c

@@ -1,6 +1,5 @@
 /* HTML parser for Wget.
-   Copyright (C) 1998, 1999, 2000, 2001, 2002, 2003, 2004, 2005, 2006,
-   2007, 2008, 2009, 2010, 2011, 2015 Free Software Foundation, Inc.
+   Copyright (C) 1998-2011, 2015, 2018 Free Software Foundation, Inc.
 
 This file is part of GNU Wget.
 
@@ -176,8 +175,8 @@ struct pool {
   P->orig_size = P->size;                                       \
 } while (0)
 
-/* Grow the pool to accomodate at least SIZE new bytes.  If the pool
-   already has room to accomodate SIZE bytes of data, this is a no-op.  */
+/* Grow the pool to accommodate at least SIZE new bytes.  If the pool
+   already has room to accommodate SIZE bytes of data, this is a no-op.  */
 
 #define POOL_GROW(p, increase)                                  \
   GROW_ARRAY ((p)->contents, (p)->size, (p)->tail + (increase), \
@@ -801,14 +800,6 @@ name_allowed (const struct hash_table *ht, const char *b, const char *e)
   }                                             \
 } while (0)
 
-/* Skip non-whitespace, if any. */
-
-#define SKIP_NON_WS(p) do {                     \
-  while (!c_isspace (*p)) {                       \
-    ADVANCE (p);                                \
-  }                                             \
-} while (0)
-
 #ifdef STANDALONE
 static int tag_backout_count;
 #endif

src/html-parse.h

@@ -1,6 +1,5 @@
 /* Declarations for html-parse.c.
-   Copyright (C) 1998, 1999, 2000, 2001, 2002, 2003, 2004, 2005, 2006,
-   2007, 2008, 2009, 2010, 2011, 2015 Free Software Foundation, Inc.
+   Copyright (C) 1998-2011, 2015, 2018 Free Software Foundation, Inc.
 
 This file is part of GNU Wget.
 

src/html-url.c

@@ -1,7 +1,5 @@
 /* Collect URLs from HTML source.
-   Copyright (C) 1998, 1999, 2000, 2001, 2002, 2003, 2004, 2005, 2006,
-   2007, 2008, 2009, 2010, 2011, 2012, 2015 Free Software Foundation,
-   Inc.
+   Copyright (C) 1998-2012, 2015, 2018 Free Software Foundation, Inc.
 
 This file is part of GNU Wget.
 
@@ -56,6 +54,7 @@ typedef void (*tag_handler_t) (int, struct taginfo *, struct map_context *);
 DECLARE_TAG_HANDLER (tag_find_urls);
 DECLARE_TAG_HANDLER (tag_handle_base);
 DECLARE_TAG_HANDLER (tag_handle_form);
+DECLARE_TAG_HANDLER (tag_handle_img);
 DECLARE_TAG_HANDLER (tag_handle_link);
 DECLARE_TAG_HANDLER (tag_handle_meta);
 
@@ -105,7 +104,7 @@ static struct known_tag {
   { TAG_FORM,    "form",        tag_handle_form },
   { TAG_FRAME,   "frame",       tag_find_urls },
   { TAG_IFRAME,  "iframe",      tag_find_urls },
-  { TAG_IMG,     "img",         tag_find_urls },
+  { TAG_IMG,     "img",         tag_handle_img },
   { TAG_INPUT,   "input",       tag_find_urls },
   { TAG_LAYER,   "layer",       tag_find_urls },
   { TAG_LINK,    "link",        tag_handle_link },
@@ -183,7 +182,8 @@ static const char *additional_attributes[] = {
   "name",                       /* used by tag_handle_meta  */
   "content",                    /* used by tag_handle_meta  */
   "action",                     /* used by tag_handle_form  */
-  "style"                       /* used by check_style_attr */
+  "style",                      /* used by check_style_attr */
+  "srcset",                     /* used by tag_handle_img */
 };
 
 static struct hash_table *interesting_tags;
@@ -674,6 +674,91 @@ tag_handle_meta (int tagid _GL_UNUSED, struct taginfo *tag, struct map_context *
     }
 }
 
+/* Handle the IMG tag.  This requires special handling for the srcset attr,
+   while the traditional src/lowsrc/href attributes can be handled generically.
+*/
+
+static void
+tag_handle_img (int tagid, struct taginfo *tag, struct map_context *ctx) {
+  int attrind;
+  char *srcset;
+
+  /* Use the generic approach for the attributes without special syntax. */
+  tag_find_urls(tagid, tag, ctx);
+
+  srcset = find_attr (tag, "srcset", &attrind);
+  if (srcset)
+    {
+      /* These are relative to the input text. */
+      int base_ind = ATTR_POS (tag,attrind,ctx);
+      int size = strlen (srcset);
+
+      /* These are relative to srcset. */
+      int offset, url_start, url_end;
+
+      /* Make sure to line up base_ind with srcset[0], not outside quotes. */
+      if (ctx->text[base_ind] == '"' || ctx->text[base_ind] == '\'')
+        ++base_ind;
+
+      offset = 0;
+      while (offset < size)
+        {
+          bool has_descriptor = true;
+
+          /* Skip over initial whitespace and commas. Note there is no \v
+            in HTML5 whitespace. */
+          url_start = offset + strspn (srcset + offset, " \f\n\r\t,");
+
+          if (url_start == size)
+            return;
+
+          /* URL is any non-whitespace chars (including commas) - but with
+             trailing commas removed. */
+          url_end = url_start + strcspn (srcset + url_start, " \f\n\r\t");
+          while ((url_end - 1) > url_start && srcset[url_end - 1] == ',')
+            {
+              has_descriptor = false;
+              --url_end;
+            }
+
+          if (url_end > url_start)
+            {
+              char *url_text = strdupdelim (srcset + url_start,
+                                            srcset + url_end);
+              struct urlpos *up = append_url (url_text, base_ind + url_start,
+                                              url_end - url_start, ctx);
+              if (up)
+                {
+                  up->link_inline_p = 1;
+                  up->link_noquote_html_p = 1;
+                }
+              xfree (url_text);
+            }
+
+          /* If the URL wasn't terminated by a , there may also be a descriptor
+             which we just skip. */
+          if (has_descriptor)
+            {
+              /* This is comma-terminated, except there may be one level of
+                 parentheses escaping that. */
+              bool in_paren = false;
+              for (offset = url_end; offset < size; ++offset)
+                {
+                  char c = srcset[offset];
+                  if (c == '(')
+                    in_paren = true;
+                  else if (c == ')' && in_paren)
+                    in_paren = false;
+                  else if (c == ',' && !in_paren)
+                    break;
+                }
+            }
+          else
+            offset = url_end;
+        }
+    }
+}
+
 /* Dispatch the tag handler appropriate for the tag we're mapping
    over.  See known_tags[] for definition of tag handlers.  */
 
@@ -752,10 +837,13 @@ get_urls_html (const char *file, const char *url, bool *meta_disallow_follow,
   map_html_tags (fm->content, fm->length, collect_tags_mapper, &ctx, flags,
                  NULL, interesting_attributes);
 
+#ifdef ENABLE_IRI
   /* Meta charset is only valid if there was no HTTP header Content-Type charset. */
   /* This is true for HTTP 1.0 and 1.1. */
   if (iri && !iri->content_encoding && meta_charset)
     set_content_encoding (iri, meta_charset);
+#endif
+  xfree (meta_charset);
 
   DEBUGP (("no-follow in %s: %d\n", file, ctx.nofollow));
   if (meta_disallow_follow)

src/html-url.h

@@ -1,5 +1,5 @@
 /* Declarations for html-url.c.
-   Copyright (C) 1995, 1996, 1997, 2009, 2010, 2011, 2015 Free Software
+   Copyright (C) 1995-1997, 2009-2011, 2015, 2018 Free Software
    Foundation, Inc.
 
 This file is part of GNU Wget.

src/http-ntlm.c

@@ -1,6 +1,5 @@
 /* NTLM code.
-   Copyright (C) 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2015 Free
-   Software Foundation, Inc.
+   Copyright (C) 2005-2011, 2015, 2018 Free Software Foundation, Inc.
    Contributed by Daniel Stenberg.
 
 This file is part of GNU Wget.
@@ -122,7 +121,7 @@ ntlm_input (struct ntlmdata *ntlm, const char *header)
 
       DEBUGP (("Received a type-2 NTLM message.\n"));
 
-      size = base64_decode (header, buffer);
+      size = wget_base64_decode (header, buffer, strlen (header));
       if (size < 0)
         return false;           /* malformed base64 from server */
 
@@ -411,7 +410,7 @@ ntlm_output (struct ntlmdata *ntlm, const char *user, const char *passwd,
     size = 32 + hostlen + domlen;
 
     base64 = (char *) alloca (BASE64_LENGTH (size) + 1);
-    base64_encode (ntlmbuf, size, base64);
+    wget_base64_encode (ntlmbuf, size, base64);
 
     output = concat_strings ("NTLM ", base64, (char *) 0);
     break;
@@ -584,7 +583,7 @@ ntlm_output (struct ntlmdata *ntlm, const char *user, const char *passwd,
 
     /* convert the binary blob into base64 */
     base64 = (char *) alloca (BASE64_LENGTH (size) + 1);
-    base64_encode (ntlmbuf, size, base64);
+    wget_base64_encode (ntlmbuf, size, base64);
 
     output = concat_strings ("NTLM ", base64, (char *) 0);
 

src/http-ntlm.h

@@ -1,8 +1,8 @@
 #ifndef __HTTP_NTLM_H
 #define __HTTP_NTLM_H
 /* Declarations for http_ntlm.c
-   Copyright (C) 1995, 1996, 1997, 2000, 2007, 2008, 2009, 2010, 2011,
-   2015 Free Software Foundation, Inc.
+   Copyright (C) 1995-1997, 2000, 2007-2011, 2015, 2018 Free Software
+   Foundation, Inc.
    Contributed by Daniel Stenberg.
 
 This file is part of GNU Wget.

src/http.h

@@ -1,6 +1,5 @@
 /* Declarations for HTTP.
-   Copyright (C) 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2015 Free
-   Software Foundation, Inc.
+   Copyright (C) 2005-2011, 2015, 2018 Free Software Foundation, Inc.
 
 This file is part of GNU Wget.
 
@@ -35,7 +34,7 @@ as that of the covered work.  */
 
 struct url;
 
-uerr_t http_loop (struct url *, struct url *, char **, char **, const char *,
+uerr_t http_loop (const struct url *, struct url *, char **, char **, const char *,
                   int *, struct url *, struct iri *);
 void save_cookies (void);
 void http_cleanup (void);

src/init.c

@@ -1,7 +1,6 @@
 /* Reading/parsing the initialization file.
-   Copyright (C) 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004,
-   2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012, 2014, 2015 Free
-   Software Foundation, Inc.
+   Copyright (C) 1996-2012, 2014-2015, 2018 Free Software Foundation,
+   Inc.
 
 This file is part of GNU Wget.
 
@@ -97,6 +96,11 @@ CMD_DECLARE (cmd_directory);
 CMD_DECLARE (cmd_time);
 CMD_DECLARE (cmd_vector);
 
+CMD_DECLARE (cmd_use_askpass);
+
+#ifdef HAVE_LIBZ
+CMD_DECLARE (cmd_spec_compression);
+#endif
 CMD_DECLARE (cmd_spec_dirstruct);
 CMD_DECLARE (cmd_spec_header);
 CMD_DECLARE (cmd_spec_warc_header);
@@ -115,6 +119,7 @@ CMD_DECLARE (cmd_spec_secure_protocol);
 CMD_DECLARE (cmd_spec_timeout);
 CMD_DECLARE (cmd_spec_useragent);
 CMD_DECLARE (cmd_spec_verbose);
+CMD_DECLARE (cmd_check_cert);
 
 /* List of recognized commands, each consisting of name, place and
    function.  When adding a new command, simply add it to the list,
@@ -142,6 +147,9 @@ static const struct {
   { "backups",          &opt.backups,           cmd_number },
   { "base",             &opt.base_href,         cmd_string },
   { "bindaddress",      &opt.bind_address,      cmd_string },
+#ifdef HAVE_LIBCARES
+  { "binddnsaddress",   &opt.bind_dns_address,  cmd_string },
+#endif
   { "bodydata",         &opt.body_data,         cmd_string },
   { "bodyfile",         &opt.body_file,         cmd_string },
 #ifdef HAVE_SSL
@@ -152,9 +160,12 @@ static const struct {
   { "cadirectory",      &opt.ca_directory,      cmd_directory },
   { "certificate",      &opt.cert_file,         cmd_file },
   { "certificatetype",  &opt.cert_type,         cmd_cert_type },
-  { "checkcertificate", &opt.check_cert,        cmd_boolean },
+  { "checkcertificate", &opt.check_cert,        cmd_check_cert },
 #endif
   { "chooseconfig",     &opt.choose_config,     cmd_file },
+#ifdef HAVE_LIBZ
+  { "compression",      &opt.compression,       cmd_spec_compression },
+#endif
   { "connecttimeout",   &opt.connect_timeout,   cmd_time },
   { "contentdisposition", &opt.content_disposition, cmd_boolean },
   { "contentonerror",   &opt.content_on_error,  cmd_boolean },
@@ -172,6 +183,9 @@ static const struct {
   { "dirprefix",        &opt.dir_prefix,        cmd_directory },
   { "dirstruct",        NULL,                   cmd_spec_dirstruct },
   { "dnscache",         &opt.dns_cache,         cmd_boolean },
+#ifdef HAVE_LIBCARES
+  { "dnsservers",       &opt.dns_servers,       cmd_string },
+#endif
   { "dnstimeout",       &opt.dns_timeout,       cmd_time },
   { "domains",          &opt.domains,           cmd_vector },
   { "dotbytes",         &opt.dot_bytes,         cmd_bytes },
@@ -203,7 +217,7 @@ static const struct {
   { "header",           NULL,                   cmd_spec_header },
 #ifdef HAVE_HSTS
   { "hsts",             &opt.hsts,              cmd_boolean },
-  { "hsts-file",        &opt.hsts_file,         cmd_file },
+  { "hstsfile",         &opt.hsts_file,         cmd_file },
 #endif
   { "htmlextension",    &opt.adjust_extension,  cmd_boolean }, /* deprecated */
   { "htmlify",          NULL,                   cmd_spec_htmlify },
@@ -216,7 +230,7 @@ static const struct {
 #endif
   { "httpsproxy",       &opt.https_proxy,       cmd_string },
   { "httpuser",         &opt.http_user,         cmd_string },
-  { "if-modified-since", &opt.if_modified_since, cmd_boolean },
+  { "ifmodifiedsince",  &opt.if_modified_since, cmd_boolean },
   { "ignorecase",       &opt.ignore_case,       cmd_boolean },
   { "ignorelength",     &opt.ignore_length,     cmd_boolean },
   { "ignoretags",       &opt.ignore_tags,       cmd_vector },
@@ -227,9 +241,10 @@ static const struct {
 #endif
   { "input",            &opt.input_filename,    cmd_file },
 #ifdef HAVE_METALINK
-  { "input-metalink",   &opt.input_metalink,    cmd_file },
+  { "inputmetalink",    &opt.input_metalink,    cmd_file },
 #endif
   { "iri",              &opt.enable_iri,        cmd_boolean },
+  { "keepbadhash",      &opt.keep_badhash,      cmd_boolean },
   { "keepsessioncookies", &opt.keep_session_cookies, cmd_boolean },
   { "limitrate",        &opt.limit_rate,        cmd_bytes },
   { "loadcookies",      &opt.cookies_input,     cmd_file },
@@ -238,7 +253,8 @@ static const struct {
   { "login",            &opt.ftp_user,          cmd_string },/* deprecated*/
   { "maxredirect",      &opt.max_redirect,      cmd_number },
 #ifdef HAVE_METALINK
-  { "metalink-over-http", &opt.metalink_over_http, cmd_boolean },
+  { "metalinkindex",    &opt.metalink_index,     cmd_number_inf },
+  { "metalinkoverhttp", &opt.metalink_over_http, cmd_boolean },
 #endif
   { "method",           &opt.method,            cmd_string_uppercase },
   { "mirror",           NULL,                   cmd_spec_mirror },
@@ -253,11 +269,14 @@ static const struct {
   { "passiveftp",       &opt.ftp_pasv,          cmd_boolean },
   { "passwd",           &opt.ftp_passwd,        cmd_string },/* deprecated*/
   { "password",         &opt.passwd,            cmd_string },
+#ifdef HAVE_SSL
+  { "pinnedpubkey",     &opt.pinnedpubkey,      cmd_string },
+#endif
   { "postdata",         &opt.post_data,         cmd_string },
   { "postfile",         &opt.post_file_name,    cmd_file },
   { "preferfamily",     NULL,                   cmd_spec_prefer_family },
 #ifdef HAVE_METALINK
-  { "preferred-location", &opt.preferred_location, cmd_string },
+  { "preferredlocation", &opt.preferred_location, cmd_string },
 #endif
   { "preservepermissions", &opt.preserve_perm,  cmd_boolean },
 #ifdef HAVE_SSL
@@ -290,6 +309,7 @@ static const struct {
   { "restrictfilenames", NULL,                  cmd_spec_restrict_file_names },
   { "retrsymlinks",     &opt.retr_symlinks,     cmd_boolean },
   { "retryconnrefused", &opt.retry_connrefused, cmd_boolean },
+  { "retryonhttperror", &opt.retry_on_http_error, cmd_string },
   { "robots",           &opt.use_robots,        cmd_boolean },
   { "savecookies",      &opt.cookies_output,    cmd_file },
   { "saveheaders",      &opt.save_headers,      cmd_boolean },
@@ -307,7 +327,8 @@ static const struct {
   { "timestamping",     &opt.timestamping,      cmd_boolean },
   { "tries",            &opt.ntry,              cmd_number_inf },
   { "trustservernames", &opt.trustservernames,  cmd_boolean },
-  { "unlink",           &opt.unlink,            cmd_boolean },
+  { "unlink",           &opt.unlink_requested,  cmd_boolean },
+  { "useaskpass" ,      &opt.use_askpass,       cmd_use_askpass },
   { "useproxy",         &opt.use_proxy,         cmd_boolean },
   { "user",             &opt.user,              cmd_string },
   { "useragent",        NULL,                   cmd_spec_useragent },
@@ -329,6 +350,9 @@ static const struct {
 #ifdef USE_WATT32
   { "wdebug",           &opt.wdebug,            cmd_boolean },
 #endif
+#ifdef ENABLE_XATTR
+  { "xattr",            &opt.enable_xattr,      cmd_boolean },
+#endif
 };
 
 /* Look up CMDNAME in the commands[] and return its position in the
@@ -368,6 +392,10 @@ defaults (void)
      bit pattern will be the least of the implementors' worries.  */
   xzero (opt);
 
+#ifdef HAVE_METALINK
+  opt.metalink_index = -1;
+#endif
+
   opt.cookies = true;
   opt.verbose = -1;
   opt.ntry = 20;
@@ -415,13 +443,17 @@ defaults (void)
   opt.retr_symlinks = true;
 
 #ifdef HAVE_SSL
-  opt.check_cert = true;
+  opt.check_cert = CHECK_CERT_ON;
   opt.ftps_resume_ssl = true;
   opt.ftps_fallback_to_ftp = false;
   opt.ftps_implicit = false;
   opt.ftps_clear_data_connection = false;
 #endif
 
+#ifdef HAVE_LIBZ
+  opt.compression = compression_none;
+#endif
+
   /* The default for file name restriction defaults to the OS type. */
 #if defined(WINDOWS) || defined(MSDOS) || defined(__CYGWIN__)
   opt.restrict_files_os = restrict_windows;
@@ -472,6 +504,12 @@ defaults (void)
   /* HSTS is enabled by default */
   opt.hsts = true;
 #endif
+
+#ifdef ENABLE_XATTR
+  opt.enable_xattr = true;
+#else
+  opt.enable_xattr = false;
+#endif
 }
 
 /* Return the user's home directory (strdup-ed), or NULL if none is
@@ -537,10 +575,11 @@ wgetrc_env_file_name (void)
   char *env = getenv ("WGETRC");
   if (env && *env)
     {
-      if (!file_exists_p (env))
+      file_stats_t flstat;
+      if (!file_exists_p (env, &flstat))
         {
-          fprintf (stderr, _("%s: WGETRC points to %s, which doesn't exist.\n"),
-                   exec_name, env);
+          fprintf (stderr, _("%s: WGETRC points to %s, which couldn't be accessed because of error: %s.\n"),
+                   exec_name, env, strerror(flstat.access_err));
           exit (WGET_EXIT_GENERIC_ERROR);
         }
       return xstrdup (env);
@@ -548,7 +587,7 @@ wgetrc_env_file_name (void)
   return NULL;
 }
 
-/* Check for the existance of '$HOME/.wgetrc' and return its path
+/* Check for the existence of '$HOME/.wgetrc' and return its path
    if it exists and is set.  */
 char *
 wgetrc_user_file_name (void)
@@ -568,7 +607,7 @@ wgetrc_user_file_name (void)
 
   if (!file)
     return NULL;
-  if (!file_exists_p (file))
+  if (!file_exists_p (file, NULL))
     {
       xfree (file);
       return NULL;
@@ -601,7 +640,7 @@ wgetrc_file_name (void)
       if (home)
         {
           file = aprintf ("%s/wget.ini", home);
-          if (!file_exists_p (file))
+          if (!file_exists_p (file, NULL))
             {
               xfree (file);
             }
@@ -629,7 +668,7 @@ static bool setval_internal_tilde (int, const char *, const char *);
    there were errors in the file.  */
 
 bool
-run_wgetrc (const char *file)
+run_wgetrc (const char *file, file_stats_t *flstats)
 {
   FILE *fp;
   char *line = NULL;
@@ -637,7 +676,7 @@ run_wgetrc (const char *file)
   int ln;
   int errcnt = 0;
 
-  fp = fopen (file, "r");
+  fp = fopen_stat (file, "r", flstats);
   if (!fp)
     {
       fprintf (stderr, _("%s: Cannot read %s (%s).\n"), exec_name,
@@ -693,14 +732,16 @@ void
 initialize (void)
 {
   char *file, *env_sysrc;
+  file_stats_t flstats;
   bool ok = true;
 
+  memset(&flstats, 0, sizeof(flstats));
   /* Run a non-standard system rc file when the according environment
      variable has been set. For internal testing purposes only!  */
   env_sysrc = getenv ("SYSTEM_WGETRC");
-  if (env_sysrc && file_exists_p (env_sysrc))
+  if (env_sysrc && file_exists_p (env_sysrc, &flstats))
     {
-      ok &= run_wgetrc (env_sysrc);
+      ok &= run_wgetrc (env_sysrc, &flstats);
       /* If there are any problems parsing the system wgetrc file, tell
          the user and exit */
       if (! ok)
@@ -714,8 +755,8 @@ or specify a different file using --config.\n"), env_sysrc);
     }
   /* Otherwise, if SYSTEM_WGETRC is defined, use it.  */
 #ifdef SYSTEM_WGETRC
-  else if (file_exists_p (SYSTEM_WGETRC))
-    ok &= run_wgetrc (SYSTEM_WGETRC);
+  else if (file_exists_p (SYSTEM_WGETRC, &flstats))
+    ok &= run_wgetrc (SYSTEM_WGETRC, &flstats);
   /* If there are any problems parsing the system wgetrc file, tell
      the user and exit */
   if (! ok)
@@ -742,7 +783,8 @@ or specify a different file using --config.\n"), SYSTEM_WGETRC);
     }
   else
 #endif
-    ok &= run_wgetrc (file);
+	if (file_exists_p (file, &flstats))
+        ok &= run_wgetrc (file, &flstats);
 
   /* If there were errors processing either `.wgetrc', abort. */
   if (!ok)
@@ -849,6 +891,10 @@ static bool
 setval_internal (int comind, const char *com, const char *val)
 {
   assert (0 <= comind && ((size_t) comind) < countof (commands));
+
+  if ((unsigned) comind >= countof (commands))
+    return NULL;
+
   DEBUGP (("Setting %s (%s) to %s\n", com, commands[comind].name, val));
   return commands[comind].action (com, val, commands[comind].place);
 }
@@ -955,6 +1001,18 @@ static bool simple_atof (const char *, const char *, double *);
                      && (p)[3] == '\0')
 
 
+static int
+cmd_boolean_internal (const char *com _GL_UNUSED, const char *val, void *place _GL_UNUSED)
+{
+  if (CMP2 (val, 'o', 'n') || CMP3 (val, 'y', 'e', 's') || CMP1 (val, '1'))
+    /* "on", "yes" and "1" mean true. */
+    return 1;
+  else if (CMP3 (val, 'o', 'f', 'f') || CMP2 (val, 'n', 'o') || CMP1 (val, '0'))
+    /* "off", "no" and "0" mean false. */
+    return 0;
+  return -1;
+}
+
 /* Store the boolean value from VAL to PLACE.  COM is ignored,
    except for error messages.  */
 static bool
@@ -962,24 +1020,62 @@ cmd_boolean (const char *com, const char *val, void *place)
 {
   bool value;
 
-  if (CMP2 (val, 'o', 'n') || CMP3 (val, 'y', 'e', 's') || CMP1 (val, '1'))
-    /* "on", "yes" and "1" mean true. */
-    value = true;
-  else if (CMP3 (val, 'o', 'f', 'f') || CMP2 (val, 'n', 'o') || CMP1 (val, '0'))
-    /* "off", "no" and "0" mean false. */
-    value = false;
-  else
+  switch (cmd_boolean_internal (com, val, place))
     {
-      fprintf (stderr,
-               _("%s: %s: Invalid boolean %s; use `on' or `off'.\n"),
-               exec_name, com, quote (val));
-      return false;
-    }
+    case 0:
+      value = false;
+      break;
 
+    case 1:
+      value = true;
+      break;
+
+    default:
+      {
+        fprintf (stderr,
+                 _("%s: %s: Invalid boolean %s; use `on' or `off'.\n"),
+                 exec_name, com, quote (val));
+        return false;
+      }
+    }
   *(bool *) place = value;
   return true;
 }
 
+/* Store the check_cert value from VAL to PLACE.  COM is ignored,
+   except for error messages.  */
+static bool
+cmd_check_cert (const char *com, const char *val, void *place)
+{
+  int value;
+
+  switch (cmd_boolean_internal (com, val, place))
+    {
+    case 0:
+      value = CHECK_CERT_OFF;
+      break;
+
+    case 1:
+      value = CHECK_CERT_ON;
+      break;
+
+    default:
+      {
+        if (!c_strcasecmp (val, "quiet"))
+          value = CHECK_CERT_QUIET;
+        else
+          {
+            fprintf (stderr,
+                     _("%s: %s: Invalid %s; use `on', `off' or `quiet'.\n"),
+                     exec_name, com, quote (val));
+            return false;
+          }
+      }
+    }
+  *(int *) place = value;
+  return true;
+}
+
 /* Set the non-negative integer value from VAL to PLACE.  With
    incorrect specification, the number remains unchanged.  */
 static bool
@@ -1311,6 +1407,32 @@ cmd_time (const char *com, const char *val, void *place)
   return true;
 }
 
+
+static bool
+cmd_use_askpass (const char *com _GL_UNUSED, const char *val, void *place)
+{
+  const char *env_name = "WGET_ASKPASS";
+  const char *env;
+
+  if (val && *val)
+    return cmd_string (com, val, place);
+
+  env = getenv (env_name);
+  if (!(env && *env))
+    {
+      env_name = "SSH_ASKPASS";
+      env = getenv (env_name);
+    }
+
+  if (!(env && *env))
+    {
+      fprintf (stderr, _("use-askpass requires a string or either environment variable WGET_ASKPASS or SSH_ASKPASS to be set.\n"));
+      exit (WGET_EXIT_GENERIC_ERROR);
+    }
+
+  return cmd_string (com, env, place);
+}
+
 #ifdef HAVE_SSL
 static bool
 cmd_cert_type (const char *com, const char *val, void *place)
@@ -1332,6 +1454,25 @@ cmd_cert_type (const char *com, const char *val, void *place)
 
 static bool check_user_specified_header (const char *);
 
+#ifdef HAVE_LIBZ
+static bool
+cmd_spec_compression (const char *com, const char *val, void *place)
+{
+  static const struct decode_item choices[] = {
+    { "auto", compression_auto },
+    { "gzip", compression_gzip },
+    { "none", compression_none },
+  };
+  int ok = decode_string (val, choices, countof (choices), place);
+  if (!ok)
+    {
+      fprintf (stderr, _("%s: %s: Invalid value %s.\n"), exec_name, com,
+               quote (val));
+    }
+  return ok;
+}
+#endif
+
 static bool
 cmd_spec_dirstruct (const char *com, const char *val, void *place_ignored _GL_UNUSED)
 {
@@ -1870,6 +2011,20 @@ cleanup (void)
   xfree (opt.body_data);
   xfree (opt.body_file);
   xfree (opt.rejected_log);
+  xfree (opt.use_askpass);
+  xfree (opt.retry_on_http_error);
+
+#ifdef HAVE_LIBCARES
+#include <ares.h>
+  {
+    extern ares_channel ares;
+
+    xfree (opt.bind_dns_address);
+    xfree (opt.dns_servers);
+    ares_destroy (ares);
+    ares_library_cleanup ();
+  }
+#endif
 
 #endif /* DEBUG_MALLOC */
 }

src/init.h

@@ -1,7 +1,5 @@
 /* Declarations for init.c.
-   Copyright (C) 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004,
-   2005, 2006, 2007, 2008, 2009, 2010, 2011, 2015 Free Software
-   Foundation, Inc.
+   Copyright (C) 1996-2011, 2015, 2018 Free Software Foundation, Inc.
 
 This file is part of GNU Wget.
 
@@ -41,6 +39,6 @@ void setoptval (const char *, const char *, const char *);
 char *home_dir (void);
 void cleanup (void);
 void defaults (void);
-bool run_wgetrc (const char *file);
+bool run_wgetrc (const char *file, file_stats_t *);
 
 #endif /* INIT_H */

src/iri.c

@@ -1,6 +1,5 @@
 /* IRI related functions.
-   Copyright (C) 2008, 2009, 2010, 2011, 2015 Free Software Foundation,
-   Inc.
+   Copyright (C) 2008-2011, 2015, 2018 Free Software Foundation, Inc.
 
 This file is part of GNU Wget.
 
@@ -33,11 +32,16 @@ as that of the covered work.  */
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
-#include <iconv.h>
-#include <stringprep.h>
-#include <idna.h>
-#include <idn-free.h>
+#include <langinfo.h>
 #include <errno.h>
+#ifdef HAVE_ICONV
+# include <iconv.h>
+#endif
+#include <idn2.h>
+#if IDN2_VERSION_NUMBER < 0x00140000
+# include <unicase.h>
+# include <unistr.h>
+#endif
 
 #include "utils.h"
 #include "url.h"
@@ -45,9 +49,6 @@ as that of the covered work.  */
 #include "c-strcasestr.h"
 #include "xstrndup.h"
 
-/* RFC3987 section 3.1 mandates STD3 ASCII RULES */
-#define IDNA_FLAGS  IDNA_USE_STD3_ASCII_RULES
-
 /* Note: locale encoding is kept in options struct (opt.locale) */
 
 /* Given a string containing "charset=XXX", return the encoding if found,
@@ -88,10 +89,15 @@ parse_charset (const char *str)
 }
 
 /* Find the locale used, or fall back on a default value */
-char *
+const char *
 find_locale (void)
 {
-  return (char *) stringprep_locale_charset ();
+	const char *encoding = nl_langinfo(CODESET);
+
+	if (!encoding || !*encoding)
+		return "ASCII";
+
+   return encoding;
 }
 
 /* Basic check of an encoding name. */
@@ -114,6 +120,7 @@ check_encoding_name (const char *encoding)
   return true;
 }
 
+#ifdef HAVE_ICONV
 /* Do the conversion according to the passed conversion descriptor cd. *out
    will contain the transcoded string on success. *out content is
    unspecified otherwise. */
@@ -129,8 +136,8 @@ do_conversion (const char *tocode, const char *fromcode, char const *in_org, siz
   cd = iconv_open (tocode, fromcode);
   if (cd == (iconv_t)(-1))
     {
-      logprintf (LOG_VERBOSE, _("Conversion from %s to UTF-8 isn't supported\n"),
-                 quote (opt.locale));
+      logprintf (LOG_VERBOSE, _("Conversion from %s to %s isn't supported\n"),
+                 quote (fromcode), quote (tocode));
       *out = NULL;
       return false;
     }
@@ -146,7 +153,8 @@ do_conversion (const char *tocode, const char *fromcode, char const *in_org, siz
 
   for (;;)
     {
-      if (iconv (cd, &in, &inlen, out, &outlen) != (size_t)(-1))
+      if (iconv (cd, (ICONV_CONST char **) &in, &inlen, out, &outlen) != (size_t)(-1) &&
+          iconv (cd, NULL, NULL, out, &outlen) != (size_t)(-1))
         {
           *out = s;
           *(s + len - outlen - done) = '\0';
@@ -158,7 +166,7 @@ do_conversion (const char *tocode, const char *fromcode, char const *in_org, siz
             if (!strchr(in_org, '@') && !strchr(*out, '@'))
               debug_logprintf ("converted '%s' (%s) -> '%s' (%s)\n", in_org, fromcode, *out, tocode);
             else
-              debug_logprintf ("%s: logging suppressed, strings may contain password\n", __func__);
+              debug_logprintf ("logging suppressed, strings may contain password\n");
           }
           return true;
         }
@@ -179,16 +187,10 @@ do_conversion (const char *tocode, const char *fromcode, char const *in_org, siz
         }
       else if (errno == E2BIG) /* Output buffer full */
         {
-          char *new;
-
           tooshort++;
           done = len;
-          outlen = done + inlen * 2;
-          new = xmalloc (outlen + 1);
-          memcpy (new, s, done);
-          xfree (s);
-          s = new;
-          len = outlen;
+          len = outlen = done + inlen * 2;
+          s = xrealloc (s, outlen + 1);
           *out = s + done;
         }
       else /* Weird, we got an unspecified error */
@@ -206,10 +208,19 @@ do_conversion (const char *tocode, const char *fromcode, char const *in_org, siz
       if (!strchr(in_org, '@') && !strchr(*out, '@'))
         debug_logprintf ("converted '%s' (%s) -> '%s' (%s)\n", in_org, fromcode, *out, tocode);
       else
-        debug_logprintf ("%s: logging suppressed, strings may contain password\n", __func__);
+        debug_logprintf ("logging suppressed, strings may contain password\n");
     }
     return false;
 }
+#else
+static bool
+do_conversion (const char *tocode _GL_UNUSED, const char *fromcode _GL_UNUSED,
+               char const *in_org _GL_UNUSED, size_t inlen _GL_UNUSED, char **out)
+{
+  *out = NULL;
+  return false;
+}
+#endif
 
 /* Try converting string str from locale to UTF-8. Return a new string
    on success, or str on error or if conversion isn't needed. */
@@ -235,50 +246,6 @@ locale_to_utf8 (const char *str)
   return str;
 }
 
-/*
- * Work around a libidn <= 1.30 vulnerability.
- *
- * The function checks for a valid UTF-8 character sequence before
- * passing it to idna_to_ascii_8z().
- *
- * [1] http://lists.gnu.org/archive/html/help-libidn/2015-05/msg00002.html
- * [2] https://lists.gnu.org/archive/html/bug-wget/2015-06/msg00002.html
- * [3] http://curl.haxx.se/mail/lib-2015-06/0143.html
- */
-static bool
-_utf8_is_valid(const char *utf8)
-{
-  const unsigned char *s = (const unsigned char *) utf8;
-
-  while (*s)
-    {
-      if ((*s & 0x80) == 0) /* 0xxxxxxx ASCII char */
-        s++;
-      else if ((*s & 0xE0) == 0xC0) /* 110xxxxx 10xxxxxx */
-        {
-          if ((s[1] & 0xC0) != 0x80)
-            return false;
-          s+=2;
-        }
-      else if ((*s & 0xF0) == 0xE0) /* 1110xxxx 10xxxxxx 10xxxxxx */
-        {
-          if ((s[1] & 0xC0) != 0x80 || (s[2] & 0xC0) != 0x80)
-            return false;
-          s+=3;
-        }
-      else if ((*s & 0xF8) == 0xF0) /* 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx */
-        {
-          if ((s[1] & 0xC0) != 0x80 || (s[2] & 0xC0) != 0x80 || (s[3] & 0xC0) != 0x80)
-            return false;
-          s+=4;
-        }
-      else
-        return false;
-    }
-
-  return true;
-}
-
 /* Try to "ASCII encode" UTF-8 host. Return the new domain on success or NULL
    on error. */
 char *
@@ -287,34 +254,62 @@ idn_encode (const struct iri *i, const char *host)
   int ret;
   char *ascii_encoded;
   char *utf8_encoded = NULL;
+  const char *src;
+#if IDN2_VERSION_NUMBER < 0x00140000
+  uint8_t *lower;
+  size_t len = 0;
+#endif
 
   /* Encode to UTF-8 if not done */
   if (!i->utf8_encode)
     {
       if (!remote_to_utf8 (i, host, &utf8_encoded))
-          return NULL;  /* Nothing to encode or an error occured */
+          return NULL;  /* Nothing to encode or an error occurred */
+      src = utf8_encoded;
     }
+  else
+    src = host;
 
-  if (!_utf8_is_valid(utf8_encoded ? utf8_encoded : host))
+#if IDN2_VERSION_NUMBER >= 0x00140000
+  /* IDN2_TRANSITIONAL implies input NFC encoding */
+  ret = idn2_lookup_u8 ((uint8_t *) src, (uint8_t **) &ascii_encoded, IDN2_NONTRANSITIONAL);
+  if (ret != IDN2_OK)
+    /* fall back to TR46 Transitional mode, max IDNA2003 compatibility */
+    ret = idn2_lookup_u8 ((uint8_t *) src, (uint8_t **) &ascii_encoded, IDN2_TRANSITIONAL);
+
+  if (ret != IDN2_OK)
+    logprintf (LOG_VERBOSE, _("idn_encode failed (%d): %s\n"), ret,
+               quote (idn2_strerror (ret)));
+#else
+  /* we need a conversion to lowercase */
+  lower = u8_tolower ((uint8_t *) src, u8_strlen ((uint8_t *) src) + 1, 0, UNINORM_NFKC, NULL, &len);
+  if (!lower)
     {
-      logprintf (LOG_VERBOSE, _("Invalid UTF-8 sequence: %s\n"),
-                 quote(utf8_encoded ? utf8_encoded : host));
+      logprintf (LOG_VERBOSE, _("Failed to convert to lower: %d: %s\n"),
+                 errno, quote (src));
       xfree (utf8_encoded);
       return NULL;
     }
 
-  /* Store in ascii_encoded the ASCII UTF-8 NULL terminated string */
-  ret = idna_to_ascii_8z (utf8_encoded ? utf8_encoded : host, &ascii_encoded, IDNA_FLAGS);
-  xfree (utf8_encoded);
-
-  if (ret != IDNA_SUCCESS)
+  if ((ret = idn2_lookup_u8 (lower, (uint8_t **) &ascii_encoded, IDN2_NFC_INPUT)) != IDN2_OK)
     {
       logprintf (LOG_VERBOSE, _("idn_encode failed (%d): %s\n"), ret,
-                 quote (idna_strerror (ret)));
-      return NULL;
+                 quote (idn2_strerror (ret)));
     }
 
-  return ascii_encoded;
+  xfree (lower);
+#endif
+
+  xfree (utf8_encoded);
+
+  if (ret == IDN2_OK && ascii_encoded)
+    {
+      char *tmp = xstrdup (ascii_encoded);
+      idn2_free (ascii_encoded);
+      ascii_encoded = tmp;
+    }
+
+  return ret == IDN2_OK ? ascii_encoded : NULL;
 }
 
 /* Try to decode an "ASCII encoded" host. Return the new domain in the locale
@@ -322,18 +317,24 @@ idn_encode (const struct iri *i, const char *host)
 char *
 idn_decode (const char *host)
 {
+/*
   char *new;
   int ret;
 
-  ret = idna_to_unicode_8zlz (host, &new, IDNA_FLAGS);
-  if (ret != IDNA_SUCCESS)
+  ret = idn2_register_u8 (NULL, host, (uint8_t **) &new, 0);
+  if (ret != IDN2_OK)
     {
-      logprintf (LOG_VERBOSE, _("idn_decode failed (%d): %s\n"), ret,
-                 quote (idna_strerror (ret)));
+      logprintf (LOG_VERBOSE, _("idn2_register_u8 failed (%d): %s: %s\n"), ret,
+                 quote (idn2_strerror (ret)), host);
       return NULL;
     }
 
   return new;
+*/
+  /* idn2_register_u8() just works label by label.
+   * That is pretty much overhead for just displaying the original ulabels.
+   * To keep at least the debug output format, return a cloned host. */
+  return xstrdup(host);
 }
 
 /* Try to transcode string str from remote encoding to UTF-8. On success, *new

src/iri.h

@@ -1,6 +1,5 @@
 /* Internationalization related declarations.
-   Copyright (C) 2008, 2009, 2010, 2011, 2015 Free Software Foundation,
-   Inc.
+   Copyright (C) 2008-2011, 2015, 2018 Free Software Foundation, Inc.
 
 This file is part of GNU Wget.
 
@@ -40,11 +39,8 @@ struct iri {
 
 #ifdef ENABLE_IRI
 
-# include <idna.h>
-# include <idn-free.h>
-
 char *parse_charset (const char *str);
-char *find_locale (void);
+const char *find_locale (void);
 bool check_encoding_name (const char *encoding);
 const char *locale_to_utf8 (const char *str);
 char *idn_encode (const struct iri *i, const char *host);
@@ -60,13 +56,13 @@ void set_content_encoding (struct iri *i, const char *charset);
 
 extern struct iri dummy_iri;
 
-#define parse_charset(str)          (str, NULL)
+#define parse_charset(str)          NULL
 #define find_locale()               NULL
 #define check_encoding_name(str)    false
 #define locale_to_utf8(str)         (str)
 #define idn_encode(a,b)             NULL
 #define idn_decode(str)             NULL
-#define idn_free(str)               ((void)0)
+#define idn2_free(str)              ((void)0)
 #define remote_to_utf8(a,b,c)       false
 #define iri_new()                   (&dummy_iri)
 #define iri_dup(a)                  (&dummy_iri)

src/log.c

@@ -1,6 +1,5 @@
 /* Messages logging.
-   Copyright (C) 1998, 1999, 2000, 2001, 2002, 2003, 2004, 2005, 2006,
-   2007, 2008, 2009, 2010, 2011, 2015 Free Software Foundation, Inc.
+   Copyright (C) 1998-2011, 2015, 2018 Free Software Foundation, Inc.
 
 This file is part of GNU Wget.
 
@@ -80,6 +79,18 @@ as that of the covered work.  */
    logging is inhibited, logfp is set back to NULL. */
 static FILE *logfp;
 
+/* Descriptor of the stdout|stderr */
+static FILE *stdlogfp;
+
+/* Descriptor of the wget.log* file (if created) */
+static FILE *filelogfp;
+
+/* Name of log file */
+static char *logfile;
+
+/* Is interactive shell ? */
+static int shell_is_interactive;
+
 /* A second file descriptor pointing to the temporary log file for the
    WARC writer.  If WARC writing is disabled, this is NULL.  */
 static FILE *warclogfp;
@@ -332,7 +343,9 @@ get_warc_log_fp (void)
     return NULL;
   if (warclogfp)
     return warclogfp;
-  return NULL;
+  if (logfp)
+    return NULL;
+  return stderr;
 }
 
 /* Sets the file descriptor for the secondary log file.  */
@@ -351,6 +364,7 @@ logputs (enum log_options o, const char *s)
 {
   FILE *fp;
   FILE *warcfp;
+  int errno_save = errno;
 
   check_redirect_output ();
   if (o == LOG_PROGRESS)
@@ -358,10 +372,14 @@ logputs (enum log_options o, const char *s)
   else
     fp = get_log_fp ();
 
+  errno = errno_save;
+
   if (fp == NULL)
     return;
 
   warcfp = get_warc_log_fp ();
+  errno = errno_save;
+
   CHECK_VERBOSE (o);
 
   FPUTS (s, fp);
@@ -373,6 +391,8 @@ logputs (enum log_options o, const char *s)
     logflush ();
   else
     needs_flushing = true;
+
+  errno = errno_save;
 }
 
 struct logvprintf_state {
@@ -513,7 +533,7 @@ log_set_flush (bool flush)
     }
   else
     {
-      /* Reenable flushing.  If anything was printed in no-flush mode,
+      /* Re-enable flushing.  If anything was printed in no-flush mode,
          flush the log now.  */
       if (needs_flushing)
         logflush ();
@@ -543,8 +563,10 @@ logprintf (enum log_options o, const char *fmt, ...)
   va_list args;
   struct logvprintf_state lpstate;
   bool done;
+  int errno_saved = errno;
 
   check_redirect_output ();
+  errno = errno_saved;
   if (inhibit_logging)
     return;
   CHECK_VERBOSE (o);
@@ -561,6 +583,8 @@ logprintf (enum log_options o, const char *fmt, ...)
         exit (WGET_EXIT_GENERIC_ERROR);
     }
   while (!done);
+
+  errno = errno_saved;
 }
 
 #ifdef ENABLE_DEBUG
@@ -600,16 +624,18 @@ log_init (const char *file, bool appendp)
     {
       if (HYPHENP (file))
         {
-          logfp = stdout;
+          stdlogfp = stdout;
+          logfp = stdlogfp;
         }
       else
         {
-          logfp = fopen (file, appendp ? "a" : "w");
-          if (!logfp)
+          filelogfp = fopen (file, appendp ? "a" : "w");
+          if (!filelogfp)
             {
               fprintf (stderr, "%s: %s: %s\n", exec_name, file, strerror (errno));
               exit (WGET_EXIT_GENERIC_ERROR);
             }
+          logfp = filelogfp;
         }
     }
   else
@@ -620,7 +646,8 @@ log_init (const char *file, bool appendp)
          stderr only if the user actually specifies `-O -'.  He says
          this inconsistency is harder to document, but is overall
          easier on the user.  */
-      logfp = stderr;
+      stdlogfp = stderr;
+      logfp = stdlogfp;
 
       if (1
 #ifdef HAVE_ISATTY
@@ -635,6 +662,11 @@ log_init (const char *file, bool appendp)
           save_context_p = true;
         }
     }
+
+#ifndef WINDOWS
+  /* Initialize this values so we don't have to ask every time we print line */
+  shell_is_interactive = isatty (STDIN_FILENO);
+#endif
 }
 
 /* Close LOGFP (only if we opened it, not if it's stderr), inhibit
@@ -869,59 +901,80 @@ log_cleanup (void)
 
 /* When SIGHUP or SIGUSR1 are received, the output is redirected
    elsewhere.  Such redirection is only allowed once. */
-static enum { RR_NONE, RR_REQUESTED, RR_DONE } redirect_request = RR_NONE;
 static const char *redirect_request_signal_name;
 
-/* Redirect output to `wget-log'.  */
+/* Redirect output to `wget-log' or back to stdout/stderr.  */
 
-static void
-redirect_output (void)
+void
+redirect_output (bool to_file, const char *signal_name)
 {
-  char *logfile;
-  logfp = unique_create (DEFAULT_LOGFILE, false, &logfile);
-  if (logfp)
+  if (to_file && logfp != filelogfp)
     {
-      fprintf (stderr, _("\n%s received, redirecting output to %s.\n"),
-               redirect_request_signal_name, quote (logfile));
-      xfree (logfile);
-      /* Dump the context output to the newly opened log.  */
+      if (signal_name)
+        {
+          fprintf (stderr, "\n%s received.", signal_name);
+        }
+      if (!filelogfp)
+        {
+          filelogfp = unique_create (DEFAULT_LOGFILE, false, &logfile);
+          if (filelogfp)
+            {
+              fprintf (stderr, _("\nRedirecting output to %s.\n"),
+                  quote (logfile));
+              /* Store signal name to tell wget it's permanent redirect to log file */
+              redirect_request_signal_name = signal_name;
+              logfp = filelogfp;
+              /* Dump the context output to the newly opened log.  */
+              log_dump_context ();
+            }
+          else
+            {
+              /* Eek!  Opening the alternate log file has failed.  Nothing we
+                can do but disable printing completely. */
+              fprintf (stderr, _("%s: %s; disabling logging.\n"),
+                      (logfile) ? logfile : DEFAULT_LOGFILE, strerror (errno));
+              inhibit_logging = true;
+            }
+        }
+      else
+        {
+          fprintf (stderr, _("\nRedirecting output to %s.\n"),
+              quote (logfile));
+          logfp = filelogfp;
+          log_dump_context ();
+        }
+    }
+  else if (!to_file && logfp != stdlogfp)
+    {
+      logfp = stdlogfp;
       log_dump_context ();
     }
-  else
-    {
-      /* Eek!  Opening the alternate log file has failed.  Nothing we
-         can do but disable printing completely. */
-      fprintf (stderr, _("\n%s received.\n"), redirect_request_signal_name);
-      fprintf (stderr, _("%s: %s; disabling logging.\n"),
-               (logfile) ? logfile : DEFAULT_LOGFILE, strerror (errno));
-      inhibit_logging = true;
-    }
-  save_context_p = false;
 }
 
-/* Check whether a signal handler requested the output to be
-   redirected. */
+/* Check whether there's a need to redirect output. */
 
 static void
 check_redirect_output (void)
 {
-  if (redirect_request == RR_REQUESTED)
+#ifndef WINDOWS
+  /* If it was redirected already to log file by SIGHUP, SIGUSR1 or -o parameter,
+   * it was permanent.
+   * If there was no SIGHUP or SIGUSR1 and shell is interactive
+   * we check if process is fg or bg before every line is printed.*/
+  if (!redirect_request_signal_name && shell_is_interactive && !opt.lfilename)
     {
-      redirect_request = RR_DONE;
-      redirect_output ();
+      pid_t foreground_pgrp = tcgetpgrp (STDIN_FILENO);
+
+      if (foreground_pgrp != -1 && foreground_pgrp != getpgrp ())
+        {
+          /* Process backgrounded */
+          redirect_output (true,NULL);
+        }
+      else
+        {
+          /* Process foregrounded */
+          redirect_output (false,NULL);
+        }
     }
-}
-
-/* Request redirection at a convenient time.  This may be called from
-   a signal handler. */
-
-void
-log_request_redirect_output (const char *signal_name)
-{
-  if (redirect_request == RR_NONE && save_context_p)
-    /* Request output redirection.  The request will be processed by
-       check_redirect_output(), which is called from entry point log
-       functions. */
-    redirect_request = RR_REQUESTED;
-  redirect_request_signal_name = signal_name;
+#endif /* WINDOWS */
 }

src/log.h

@@ -1,6 +1,5 @@
 /* Declarations for log.c.
-   Copyright (C) 1998, 1999, 2000, 2001, 2002, 2003, 2004, 2005, 2006,
-   2007, 2008, 2009, 2010, 2011, 2015 Free Software Foundation, Inc.
+   Copyright (C) 1998-2011, 2015, 2018 Free Software Foundation, Inc.
 
 This file is part of GNU Wget.
 
@@ -52,6 +51,7 @@ void log_init (const char *, bool);
 void log_close (void);
 void log_cleanup (void);
 void log_request_redirect_output (const char *);
+void redirect_output (bool, const char *);
 
 const char *escnonprint (const char *);
 const char *escnonprint_uri (const char *);

src/main.c

@@ -1,7 +1,5 @@
 /* Command line parsing.
-   Copyright (C) 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004,
-   2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012, 2013, 2014, 2015 Free
-   Software Foundation, Inc.
+   Copyright (C) 1996-2015, 2018 Free Software Foundation, Inc.
 
 This file is part of GNU Wget.
 
@@ -36,7 +34,8 @@ as that of the covered work.  */
 #include <unistd.h>
 #include <string.h>
 #include <signal.h>
-#ifdef ENABLE_NLS
+#include <spawn.h>
+#if defined(ENABLE_NLS) || defined(WINDOWS)
 # include <locale.h>
 #endif
 #include <assert.h>
@@ -60,6 +59,7 @@ as that of the covered work.  */
 #include "version.h"
 #include "c-strcase.h"
 #include "dirname.h"
+#include "xmemdup0.h"
 #include <getopt.h>
 #include <getpass.h>
 #include <quote.h>
@@ -72,6 +72,9 @@ as that of the covered work.  */
 #ifdef WINDOWS
 # include <io.h>
 # include <fcntl.h>
+#ifndef ENABLE_NLS
+# include <mbctype.h>
+#endif
 #endif
 
 #ifdef __VMS
@@ -86,6 +89,13 @@ as that of the covered work.  */
 struct iri dummy_iri;
 #endif
 
+#ifdef HAVE_LIBCARES
+#include <ares.h>
+ares_channel ares;
+#else
+void *ares;
+#endif
+
 struct options opt;
 
 /* defined in version.c */
@@ -123,7 +133,7 @@ redirect_output_signal (int sig)
     signal_name = "SIGUSR1";
 #endif
 
-  log_request_redirect_output (signal_name);
+  redirect_output (true,signal_name);
   progress_schedule_redirect ();
   signal (sig, redirect_output_signal);
 }
@@ -139,6 +149,14 @@ i18n_initialize (void)
   /* Set the text message domain.  */
   bindtextdomain ("wget", LOCALEDIR);
   textdomain ("wget");
+#elif defined WINDOWS
+  char MBCP[16] = "";
+  int CP;
+
+  CP = _getmbcp(); /* Consider it's different from default. */
+  if (CP > 0)
+    snprintf(MBCP, sizeof(MBCP), ".%d", CP);
+  setlocale(LC_ALL, MBCP);
 #endif /* ENABLE_NLS */
 }
 
@@ -197,10 +215,12 @@ save_hsts (void)
     {
       char *filename = get_hsts_database ();
 
-      if (filename)
-        DEBUGP (("Saving HSTS entries to %s\n", filename));
+      if (filename && hsts_store_has_changed (hsts_store))
+        {
+          DEBUGP (("Saving HSTS entries to %s\n", filename));
+          hsts_store_save (hsts_store, filename);
+        }
 
-      hsts_store_save (hsts_store, filename);
       hsts_store_close (hsts_store);
 
       xfree (filename);
@@ -210,8 +230,8 @@ save_hsts (void)
 
 /* Definition of command-line options. */
 
-static void _Noreturn print_help (void);
-static void _Noreturn print_version (void);
+_Noreturn static void print_help (void);
+_Noreturn static void print_version (void);
 
 #ifdef HAVE_SSL
 # define IF_SSL(x) x
@@ -252,6 +272,9 @@ static struct cmdline_option option_data[] =
     { "backups", 0, OPT_BOOLEAN, "backups", -1 },
     { "base", 'B', OPT_VALUE, "base", -1 },
     { "bind-address", 0, OPT_VALUE, "bindaddress", -1 },
+#ifdef HAVE_LIBCARES
+    { "bind-dns-address", 0, OPT_VALUE, "binddnsaddress", -1 },
+#endif
     { "body-data", 0, OPT_VALUE, "bodydata", -1 },
     { "body-file", 0, OPT_VALUE, "bodyfile", -1 },
     { IF_SSL ("ca-certificate"), 0, OPT_VALUE, "cacertificate", -1 },
@@ -261,6 +284,9 @@ static struct cmdline_option option_data[] =
     { IF_SSL ("certificate-type"), 0, OPT_VALUE, "certificatetype", -1 },
     { IF_SSL ("check-certificate"), 0, OPT_BOOLEAN, "checkcertificate", -1 },
     { "clobber", 0, OPT__CLOBBER, NULL, optional_argument },
+#ifdef HAVE_LIBZ
+    { "compression", 0, OPT_VALUE, "compression", -1 },
+#endif
     { "config", 0, OPT_VALUE, "chooseconfig", -1 },
     { "connect-timeout", 0, OPT_VALUE, "connecttimeout", -1 },
     { "continue", 'c', OPT_BOOLEAN, "continue", -1 },
@@ -277,6 +303,9 @@ static struct cmdline_option option_data[] =
     { "directories", 0, OPT_BOOLEAN, "dirstruct", -1 },
     { "directory-prefix", 'P', OPT_VALUE, "dirprefix", -1 },
     { "dns-cache", 0, OPT_BOOLEAN, "dnscache", -1 },
+#ifdef HAVE_LIBCARES
+    { "dns-servers", 0, OPT_VALUE, "dnsservers", -1 },
+#endif
     { "dns-timeout", 0, OPT_VALUE, "dnstimeout", -1 },
     { "domains", 'D', OPT_VALUE, "domains", -1 },
     { "dont-remove-listing", 0, OPT__DONT_REMOVE_LISTING, NULL, no_argument },
@@ -306,7 +335,7 @@ static struct cmdline_option option_data[] =
     { "host-directories", 0, OPT_BOOLEAN, "addhostdir", -1 },
 #ifdef HAVE_HSTS
     { "hsts", 0, OPT_BOOLEAN, "hsts", -1},
-    { "hsts-file", 0, OPT_VALUE, "hsts-file", -1 },
+    { "hsts-file", 0, OPT_VALUE, "hstsfile", -1 },
 #endif
     { "html-extension", 'E', OPT_BOOLEAN, "adjustextension", -1 }, /* deprecated */
     { "htmlify", 0, OPT_BOOLEAN, "htmlify", -1 },
@@ -325,9 +354,10 @@ static struct cmdline_option option_data[] =
 #endif
     { "input-file", 'i', OPT_VALUE, "input", -1 },
 #ifdef HAVE_METALINK
-    { "input-metalink", 0, OPT_VALUE, "input-metalink", -1 },
+    { "input-metalink", 0, OPT_VALUE, "inputmetalink", -1 },
 #endif
     { "iri", 0, OPT_BOOLEAN, "iri", -1 },
+    { "keep-badhash", 0, OPT_BOOLEAN, "keepbadhash", -1 },
     { "keep-session-cookies", 0, OPT_BOOLEAN, "keepsessioncookies", -1 },
     { "level", 'l', OPT_VALUE, "reclevel", -1 },
     { "limit-rate", 0, OPT_VALUE, "limitrate", -1 },
@@ -336,10 +366,12 @@ static struct cmdline_option option_data[] =
     { "rejected-log", 0, OPT_VALUE, "rejectedlog", -1 },
     { "max-redirect", 0, OPT_VALUE, "maxredirect", -1 },
 #ifdef HAVE_METALINK
-    { "metalink-over-http", 0, OPT_BOOLEAN, "metalink-over-http", -1 },
+    { "metalink-index", 0, OPT_VALUE, "metalinkindex", -1 },
+    { "metalink-over-http", 0, OPT_BOOLEAN, "metalinkoverhttp", -1 },
 #endif
     { "method", 0, OPT_VALUE, "method", -1 },
     { "mirror", 'm', OPT_BOOLEAN, "mirror", -1 },
+    { "netrc", 0, OPT_BOOLEAN, "netrc", -1 },
     { "no", 'n', OPT__NO, NULL, required_argument },
     { "no-clobber", 0, OPT_BOOLEAN, "noclobber", -1 },
     { "no-config", 0, OPT_BOOLEAN, "noconfig", -1},
@@ -350,11 +382,12 @@ static struct cmdline_option option_data[] =
     { "parent", 0, OPT__PARENT, NULL, optional_argument },
     { "passive-ftp", 0, OPT_BOOLEAN, "passiveftp", -1 },
     { "password", 0, OPT_VALUE, "password", -1 },
+    { IF_SSL ("pinnedpubkey"), 0, OPT_VALUE, "pinnedpubkey", -1 },
     { "post-data", 0, OPT_VALUE, "postdata", -1 },
     { "post-file", 0, OPT_VALUE, "postfile", -1 },
     { "prefer-family", 0, OPT_VALUE, "preferfamily", -1 },
 #ifdef HAVE_METALINK
-    { "preferred-location", 0, OPT_VALUE, "preferred-location", -1 },
+    { "preferred-location", 0, OPT_VALUE, "preferredlocation", -1 },
 #endif
     { "preserve-permissions", 0, OPT_BOOLEAN, "preservepermissions", -1 },
     { IF_SSL ("private-key"), 0, OPT_VALUE, "privatekey", -1 },
@@ -384,6 +417,7 @@ static struct cmdline_option option_data[] =
     { "restrict-file-names", 0, OPT_BOOLEAN, "restrictfilenames", -1 },
     { "retr-symlinks", 0, OPT_BOOLEAN, "retrsymlinks", -1 },
     { "retry-connrefused", 0, OPT_BOOLEAN, "retryconnrefused", -1 },
+    { "retry-on-http-error", 0, OPT_VALUE, "retryonhttperror", -1 },
     { "save-cookies", 0, OPT_VALUE, "savecookies", -1 },
     { "save-headers", 0, OPT_BOOLEAN, "saveheaders", -1 },
     { IF_SSL ("secure-protocol"), 0, OPT_VALUE, "secureprotocol", -1 },
@@ -394,15 +428,15 @@ static struct cmdline_option option_data[] =
     { "strict-comments", 0, OPT_BOOLEAN, "strictcomments", -1 },
     { "timeout", 'T', OPT_VALUE, "timeout", -1 },
     { "timestamping", 'N', OPT_BOOLEAN, "timestamping", -1 },
-    { "if-modified-since", 0, OPT_BOOLEAN, "if-modified-since", -1 },
+    { "if-modified-since", 0, OPT_BOOLEAN, "ifmodifiedsince", -1 },
     { "tries", 't', OPT_VALUE, "tries", -1 },
     { "unlink", 0, OPT_BOOLEAN, "unlink", -1 },
     { "trust-server-names", 0, OPT_BOOLEAN, "trustservernames", -1 },
+    { "use-askpass", 0, OPT_VALUE, "useaskpass", -1},
     { "use-server-timestamps", 0, OPT_BOOLEAN, "useservertimestamps", -1 },
     { "user", 0, OPT_VALUE, "user", -1 },
     { "user-agent", 'U', OPT_VALUE, "useragent", -1 },
     { "verbose", 'v', OPT_BOOLEAN, "verbose", -1 },
-    { "verbose", 0, OPT_BOOLEAN, "verbose", -1 },
     { "version", 'V', OPT_FUNCALL, (void *) print_version, no_argument },
     { "wait", 'w', OPT_VALUE, "wait", -1 },
     { "waitretry", 0, OPT_VALUE, "waitretry", -1 },
@@ -419,6 +453,9 @@ static struct cmdline_option option_data[] =
     { "warc-tempdir", 0, OPT_VALUE, "warctempdir", -1 },
 #ifdef USE_WATT32
     { "wdebug", 0, OPT_BOOLEAN, "wdebug", -1 },
+#endif
+#ifdef ENABLE_XATTR
+    { "xattr", 0, OPT_BOOLEAN, "xattr", -1 },
 #endif
   };
 
@@ -531,7 +568,7 @@ print_usage (int error)
 
 /* Print the help message, describing all the available options.  If
    you add an option, be sure to update this list.  */
-static void _Noreturn
+_Noreturn static void
 print_help (void)
 {
   /* We split the help text this way to ease translation of individual
@@ -604,6 +641,8 @@ Download:\n"),
     N_("\
   -nc, --no-clobber                skip downloads that would download to\n\
                                      existing files (overwriting them)\n"),
+    N_("\
+       --no-netrc                  don't try to obtain credentials from .netrc\n"),
     N_("\
   -c,  --continue                  resume getting a partially-downloaded file\n"),
     N_("\
@@ -627,6 +666,12 @@ Download:\n"),
        --spider                    don't download anything\n"),
     N_("\
   -T,  --timeout=SECONDS           set all timeout values to SECONDS\n"),
+#ifdef HAVE_LIBCARES
+    N_("\
+       --dns-servers=ADDRESSES     list of DNS servers to query (comma separated)\n"),
+    N_("\
+       --bind-dns-address=ADDRESS  bind DNS resolver to ADDRESS (hostname or IP) on local host\n"),
+#endif
     N_("\
        --dns-timeout=SECS          set the DNS lookup timeout to SECS\n"),
     N_("\
@@ -668,6 +713,11 @@ Download:\n"),
        --password=PASS             set both ftp and http password to PASS\n"),
     N_("\
        --ask-password              prompt for passwords\n"),
+    N_("\
+       --use-askpass=COMMAND       specify credential handler for requesting \n\
+                                     username and password.  If no COMMAND is \n\
+                                     specified the WGET_ASKPASS or the SSH_ASKPASS \n\
+                                     environment variable is used.\n"),
     N_("\
        --no-iri                    turn off IRI support\n"),
     N_("\
@@ -677,10 +727,18 @@ Download:\n"),
     N_("\
        --unlink                    remove file before clobber\n"),
 #ifdef HAVE_METALINK
+    N_("\
+       --keep-badhash              keep files with checksum mismatch (append .badhash)\n"),
+    N_("\
+       --metalink-index=NUMBER     Metalink application/metalink4+xml metaurl ordinal NUMBER\n"),
     N_("\
        --metalink-over-http        use Metalink metadata from HTTP response headers\n"),
     N_("\
        --preferred-location        preferred location for Metalink resources\n"),
+#endif
+#ifdef ENABLE_XATTR
+    N_("\
+       --no-xattr                  turn off storage of metadata in extended file attributes\n"),
 #endif
     "\n",
 
@@ -717,6 +775,10 @@ HTTP options:\n"),
        --ignore-length             ignore 'Content-Length' header field\n"),
     N_("\
        --header=STRING             insert STRING among the headers\n"),
+#ifdef HAVE_LIBZ
+    N_("\
+       --compression=TYPE          choose compression, one of auto, gzip and none. (default: none)\n"),
+#endif
     N_("\
        --max-redirect              maximum redirections allowed per page\n"),
     N_("\
@@ -765,7 +827,7 @@ HTTP options:\n"),
 HTTPS (SSL/TLS) options:\n"),
     N_("\
        --secure-protocol=PR        choose secure protocol, one of auto, SSLv2,\n\
-                                     SSLv3, TLSv1 and PFS\n"),
+                                     SSLv3, TLSv1, TLSv1_1, TLSv1_2 and PFS\n"),
     N_("\
        --https-only                only follow secure HTTPS links\n"),
     N_("\
@@ -784,6 +846,11 @@ HTTPS (SSL/TLS) options:\n"),
        --ca-directory=DIR          directory where hash list of CAs is stored\n"),
     N_("\
        --crl-file=FILE             file with bundle of CRLs\n"),
+    N_("\
+       --pinnedpubkey=FILE/HASHES  Public key (PEM/DER) file, or any number\n\
+                                   of base64 encoded sha256 hashes preceded by\n\
+                                   \'sha256//\' and separated by \';\', to verify\n\
+                                   peer against\n"),
 #if defined(HAVE_LIBSSL) || defined(HAVE_LIBSSL32)
     N_("\
        --random-file=FILE          file with random data for seeding the SSL PRNG\n"),
@@ -992,6 +1059,103 @@ prompt_for_password (void)
   return getpass("");
 }
 
+
+/* Execute external application opt.use_askpass */
+static void
+run_use_askpass (char *question, char **answer)
+{
+  char tmp[1024];
+  pid_t pid;
+  int status;
+  int com[2];
+  ssize_t bytes = 0;
+  char *argv[3], *p;
+  posix_spawn_file_actions_t fa;
+
+  if (pipe (com) == -1)
+    {
+      fprintf (stderr, _("Cannot create pipe\n"));
+      exit (WGET_EXIT_GENERIC_ERROR);
+    }
+
+  status = posix_spawn_file_actions_init (&fa);
+  if (status)
+    {
+      fprintf (stderr,
+              _("Error initializing spawn file actions for use-askpass: %d\n"),
+              status);
+      exit (WGET_EXIT_GENERIC_ERROR);
+    }
+
+  status = posix_spawn_file_actions_adddup2 (&fa, com[1], STDOUT_FILENO);
+  if (status)
+    {
+      fprintf (stderr,
+              _("Error setting spawn file actions for use-askpass: %d\n"),
+              status);
+      exit (WGET_EXIT_GENERIC_ERROR);
+    }
+
+  /* C89 initializer lists must be computable at load time,
+   * thus this explicit initialization. */
+  argv[0] = opt.use_askpass;
+  argv[1] = question;
+  argv[2] = NULL;
+
+  status = posix_spawnp (&pid, opt.use_askpass, &fa, NULL, argv, environ);
+  if (status)
+    {
+      fprintf (stderr, "Error spawning %s: %d\n", opt.use_askpass, status);
+      exit (WGET_EXIT_GENERIC_ERROR);
+    }
+
+  /* Parent process reads from child. */
+  close (com[1]);
+  bytes = read (com[0], tmp, sizeof (tmp) - 1);
+  if (bytes <= 0)
+    {
+      fprintf (stderr,
+              _("Error reading response from command \"%s %s\": %s\n"),
+              opt.use_askpass, question, strerror (errno));
+      exit (WGET_EXIT_GENERIC_ERROR);
+    }
+
+  /* Make sure there is a trailing 0 */
+  tmp[bytes] = '\0';
+
+  /* Remove a possible new line */
+  if ((p = strpbrk (tmp, "\r\n")))
+    bytes = p - tmp;
+
+  *answer = xmemdup0 (tmp, bytes);
+}
+
+/* set the user name and password*/
+static void
+use_askpass (struct url *u)
+{
+  static char question[1024];
+
+  if (u->user == NULL || u->user[0] == '\0')
+    {
+      snprintf (question, sizeof (question),  _("Username for '%s%s': "),
+                scheme_leading_string(u->scheme), u->host);
+      /* Prompt for username */
+      run_use_askpass (question, &u->user);
+      if (opt.recursive)
+        opt.user = xstrdup (u->user);
+    }
+
+  if (u->passwd == NULL || u->passwd[0] == '\0')
+    {
+      snprintf(question, sizeof (question), _("Password for '%s%s@%s': "),
+               scheme_leading_string (u->scheme), u->user, u->host);
+      /* Prompt for password */
+      run_use_askpass (question, &u->passwd);
+      if (opt.recursive)
+        opt.passwd = xstrdup (u->passwd);
+    }
+}
 /* Function that prints the line argument while limiting it
    to at most line_length. prefix is printed on the first line
    and an appropriate number of spaces are added on subsequent
@@ -1053,7 +1217,7 @@ format_and_print_line (const char *prefix, const char *line,
   return 0;
 }
 
-static void _Noreturn
+_Noreturn static void
 print_version (void)
 {
   const char *wgetrc_title  = _("Wgetrc: ");
@@ -1236,10 +1400,10 @@ main (int argc, char **argv)
             }
           else if (strcmp (config_opt->long_name, "config") == 0)
             {
-              bool userrc_ret = true;
-              userrc_ret &= run_wgetrc (optarg);
+              file_stats_t flstats;
               use_userconfig = true;
-              if (userrc_ret)
+              memset(&flstats, 0, sizeof(flstats));
+              if (file_exists_p(optarg, &flstats) && run_wgetrc (optarg, &flstats))
                 break;
               else
                 {
@@ -1312,7 +1476,8 @@ main (int argc, char **argv)
           append_to_log = true;
           break;
         case OPT__EXECUTE:
-          run_command (optarg);
+          if (optarg) /* check silences static analyzer */
+            run_command (optarg);
           break;
         case OPT__NO:
           {
@@ -1373,6 +1538,9 @@ main (int argc, char **argv)
 
   nurl = argc - optind;
 
+  /* Initialize logging ASAP.  */
+  log_init (opt.lfilename, append_to_log);
+
   /* If we do not have Debug support compiled in AND Wget is invoked with the
    * --debug switch, instead of failing, we silently turn it into a no-op. For
    *  this no-op, we explicitly set opt.debug to false and hence none of the
@@ -1474,12 +1642,12 @@ WARNING: timestamping does nothing in combination with -O. See the manual\n\
 for details.\n\n"));
           opt.timestamping = false;
         }
-      if (opt.noclobber && file_exists_p(opt.output_document))
+      if (opt.noclobber && file_exists_p(opt.output_document, NULL))
            {
               /* Check if output file exists; if it does, exit. */
               logprintf (LOG_VERBOSE,
-                         _("File `%s' already there; not retrieving.\n"),
-                         opt.output_document);
+                         _("File %s already there; not retrieving.\n"),
+                         quote (opt.output_document));
               exit (WGET_EXIT_GENERIC_ERROR);
            }
     }
@@ -1526,6 +1694,26 @@ for details.\n\n"));
         }
     }
 
+#ifdef HAVE_LIBZ
+  if (opt.always_rest || opt.start_pos >= 0)
+    {
+      if (opt.compression == compression_auto)
+        {
+          /* Compression does not work with --continue or --start-pos.
+             Since compression was not explicitly set, it will be disabled. */
+          opt.compression = compression_none;
+        }
+      else if (opt.compression != compression_none)
+        {
+          fprintf (stderr,
+                   _("Compression does not work with --continue or"
+                     " --start-pos, they will be disabled.\n"));
+          opt.always_rest = false;
+          opt.start_pos = -1;
+        }
+    }
+#endif
+
   if (opt.ask_passwd && opt.passwd)
     {
       fprintf (stderr,
@@ -1596,7 +1784,7 @@ for details.\n\n"));
       else if (opt.method)
         {
           fprintf (stderr, _("You cannot use --post-data or --post-file along with --method. "
-                             "--method expects data through --body-data and --body-file options"));
+                             "--method expects data through --body-data and --body-file options\n"));
           exit (WGET_EXIT_GENERIC_ERROR);
         }
     }
@@ -1675,6 +1863,16 @@ for details.\n\n"));
         exit (WGET_EXIT_GENERIC_ERROR);
     }
 
+  if (opt.use_askpass)
+  {
+    if (opt.use_askpass[0] == '\0')
+      {
+        fprintf (stderr,
+                 _("use-askpass requires a string or either environment variable WGET_ASKPASS or SSH_ASKPASS to be set.\n"));
+        exit (WGET_EXIT_GENERIC_ERROR);
+      }
+  }
+
 #ifdef USE_WATT32
   if (opt.wdebug)
      dbug_init();
@@ -1706,9 +1904,6 @@ for details.\n\n"));
     }
   url[i] = NULL;
 
-  /* Initialize logging.  */
-  log_init (opt.lfilename, append_to_log);
-
   /* Open WARC file. */
   if (opt.warc_filename != 0)
     warc_init ();
@@ -1737,7 +1932,7 @@ for details.\n\n"));
         }
       else
         {
-          struct_fstat st;
+          struct stat st;
 
 #ifdef __VMS
 /* Common fopen() optional arguments:
@@ -1774,6 +1969,58 @@ only if outputting to a regular file.\n"));
         }
     }
 
+#ifdef HAVE_LIBCARES
+  if (opt.bind_dns_address || opt.dns_servers)
+    {
+      if (ares_library_init (ARES_LIB_INIT_ALL))
+        {
+          fprintf (stderr, _("Failed to init libcares\n"));
+          exit (WGET_EXIT_GENERIC_ERROR);
+        }
+
+      if (ares_init (&ares) != ARES_SUCCESS)
+        {
+          fprintf (stderr, _("Failed to init c-ares channel\n"));
+          exit (WGET_EXIT_GENERIC_ERROR);
+        }
+
+      if (opt.bind_dns_address)
+        {
+          struct in_addr a4;
+#ifdef ENABLE_IPV6
+          struct in6_addr a6;
+#endif
+
+          if (inet_pton (AF_INET, opt.bind_dns_address, &a4) == 1)
+            {
+              ares_set_local_ip4 (ares, ntohl (a4.s_addr));
+            }
+#ifdef ENABLE_IPV6
+          else if (inet_pton (AF_INET6, opt.bind_dns_address, &a6) == 1)
+            {
+              ares_set_local_ip6 (ares, (unsigned char *) &a6);
+            }
+#endif
+          else
+            {
+              fprintf (stderr, _("Failed to parse IP address '%s'\n"), opt.bind_dns_address);
+              exit (WGET_EXIT_GENERIC_ERROR);
+            }
+        }
+
+      if (opt.dns_servers)
+        {
+          int result;
+
+          if ((result = ares_set_servers_csv (ares, opt.dns_servers)) != ARES_SUCCESS)
+            {
+              fprintf (stderr, _("Failed to set DNS server(s) '%s' (%d)\n"), opt.dns_servers, result);
+              exit (WGET_EXIT_GENERIC_ERROR);
+            }
+        }
+    }
+#endif
+
 #ifdef __VMS
   /* Set global ODS5 flag according to the specified destination (if
      any), otherwise according to the current default device.
@@ -1841,14 +2088,26 @@ only if outputting to a regular file.\n"));
         }
       else
         {
+          /* Request credentials if use_askpass is set. */
+          if (opt.use_askpass)
+            use_askpass (url_parsed);
+
           if ((opt.recursive || opt.page_requisites)
-              && ((url_scheme (*t) != SCHEME_FTP && url_scheme (*t) != SCHEME_FTPS)
+              && ((url_scheme (*t) != SCHEME_FTP
+#ifdef HAVE_SSL
+              && url_scheme (*t) != SCHEME_FTPS
+#endif
+              )
                   || url_uses_proxy (url_parsed)))
             {
               int old_follow_ftp = opt.follow_ftp;
 
               /* Turn opt.follow_ftp on in case of recursive FTP retrieval */
-              if (url_scheme (*t) == SCHEME_FTP || url_scheme (*t) == SCHEME_FTPS)
+              if (url_scheme (*t) == SCHEME_FTP
+#ifdef HAVE_SSL
+                  || url_scheme (*t) == SCHEME_FTPS
+#endif
+                  )
                 opt.follow_ftp = 1;
 
               retrieve_tree (url_parsed, NULL);
@@ -1861,7 +2120,7 @@ only if outputting to a regular file.\n"));
                             &dt, opt.recursive, iri, true);
             }
 
-          if (opt.delete_after && filename != NULL && file_exists_p (filename))
+          if (opt.delete_after && filename != NULL && file_exists_p (filename, NULL))
             {
               DEBUGP (("Removing file due to --delete-after in main():\n"));
               logprintf (LOG_VERBOSE, _("Removing %s.\n"), filename);

src/metalink.h

@@ -1,5 +1,5 @@
 /* Declarations for metalink.c.
-   Copyright (C) 2015 Free Software Foundation, Inc.
+   Copyright (C) 2015, 2018 Free Software Foundation, Inc.
 
 This file is part of GNU Wget.
 
@@ -46,6 +46,21 @@ as that of the covered work.  */
 uerr_t retrieve_from_metalink (const metalink_t *metalink);
 
 int metalink_res_cmp (const void *res1, const void *res2);
+int metalink_meta_cmp (const void* meta1, const void* meta2);
+
+int metalink_check_safe_path (const char *path);
+
+char *last_component (char const *name);
+void replace_metalink_basename (char **name, char *ref);
+char *get_metalink_basename (char *name);
+void append_suffix_number (char **str, const char *sep, wgint num);
+void clean_metalink_string (char **str);
+void dequote_metalink_string (char **str);
+void badhash_suffix (char *name);
+void badhash_or_remove (char *name);
+uerr_t fetch_metalink_file (const char *url_str,
+                            bool resume, bool metalink_http,
+                            const char *filename, char **destname);
 
 bool find_key_value (const char *start,
                      const char *end,

src/mswindows.c

@@ -1,7 +1,6 @@
 /* mswindows.c -- Windows-specific support
-   Copyright (C) 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004,
-   2005, 2006, 2007, 2008, 2009, 2010, 2011, 2014, 2015 Free Software
-   Foundation, Inc.
+   Copyright (C) 1996-2011, 2014-2015, 2018 Free Software Foundation,
+   Inc.
 
 This file is part of GNU Wget.
 
@@ -53,16 +52,13 @@ as that of the covered work.  */
 #endif
 
 
-/* Defined in log.c.  */
-void log_request_redirect_output (const char *);
-
 /* Windows version of xsleep in utils.c.  */
 
 void
 xsleep (double seconds)
 {
 #if defined(HAVE_USLEEP) && defined(HAVE_SLEEP)
-  if (seconds > 1000)
+  if (seconds >= 1)
     {
       /* Explained in utils.c. */
       sleep (seconds);
@@ -98,7 +94,7 @@ static void
 ws_hangup (const char *reason)
 {
   fprintf (stderr, _("Continuing in background.\n"));
-  log_request_redirect_output (reason);
+  redirect_output (true, reason);
 
   /* Detach process from the current console.  Under Windows 9x, if we
      were launched from a 16-bit process (which is usually the case;
@@ -583,7 +579,7 @@ run_with_timeout (double seconds, void (*fun) (void *), void *arg)
 const char *
 inet_ntop (int af, const void *src, char *dst, socklen_t cnt)
 {
-  /* struct sockaddr can't accomodate struct sockaddr_in6. */
+  /* struct sockaddr can't accommodate struct sockaddr_in6. */
   union {
     struct sockaddr_in6 sin6;
     struct sockaddr_in sin;

src/mswindows.h

@@ -1,7 +1,5 @@
 /* Declarations for windows
-   Copyright (C) 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004,
-   2005, 2006, 2007, 2008, 2009, 2010, 2011, 2015 Free Software
-   Foundation, Inc.
+   Copyright (C) 1996-2011, 2015, 2018 Free Software Foundation, Inc.
 
 This file is part of GNU Wget.
 
@@ -57,6 +55,9 @@ as that of the covered work.  */
 /* Declares getpid(). */
 #include <process.h>
 
+/* Declares inet_ntop() and inet_pton(). */
+#include <arpa/inet.h>
+
 /* We have strcasecmp and strncasecmp, just under different names.  */
 #ifndef HAVE_STRCASECMP
 # define strcasecmp stricmp
@@ -85,11 +86,6 @@ typedef __int64 wgint;
 
 #define PATH_SEPARATOR '\\'
 
-/* Additional declarations needed for IPv6: */
-#ifdef ENABLE_IPV6
-const char *inet_ntop (int, const void *, char *, socklen_t);
-#endif
-
 /* ioctl needed by set_windows_fd_as_blocking_socket() */
 #include <sys/ioctl.h>
 

src/netrc.c

@@ -1,6 +1,6 @@
 /* Read and parse the .netrc file to get hosts, accounts, and passwords.
-   Copyright (C) 1996, 2007, 2008, 2009, 2010, 2011, 2015 Free Software
-   Foundation, Inc.
+   Copyright (C) 1996, 2007-2011, 2015, 2018 Free Software Foundation,
+   Inc.
 
 This file is part of GNU Wget.
 
@@ -42,7 +42,11 @@ as that of the covered work.  */
 #include "netrc.h"
 #include "init.h"
 
-#define NETRC_FILE_NAME ".netrc"
+#ifdef WINDOWS
+#  define NETRC_FILE_NAME "_netrc"
+#else
+#  define NETRC_FILE_NAME ".netrc"
+#endif
 
 static acc_t *netrc_list;
 
@@ -75,7 +79,7 @@ search_netrc (const char *host, const char **acc, const char **passwd,
 #ifdef __VMS
 
       int err;
-      struct_stat buf;
+      struct stat buf;
       char *path = "SYS$LOGIN:.netrc";
 
       netrc_list = NULL;
@@ -94,7 +98,7 @@ search_netrc (const char *host, const char **acc, const char **passwd,
       if (home)
         {
           int err;
-          struct_stat buf;
+          struct stat buf;
           char *path = (char *)alloca (strlen (home) + 1
                                        + strlen (NETRC_FILE_NAME) + 1);
           sprintf (path, "%s/%s", home, NETRC_FILE_NAME);
@@ -235,7 +239,7 @@ parse_netrc (const char *path)
   /* The latest token we've seen in the file.  */
   enum
   {
-    tok_nothing, tok_account, tok_login, tok_macdef, tok_machine, tok_password
+    tok_nothing, tok_account, tok_login, tok_macdef, tok_machine, tok_password, tok_port, tok_force
   } last_token = tok_nothing;
 
   current = retval = NULL;
@@ -344,6 +348,18 @@ parse_netrc (const char *path)
                 premature_token = "account";
               break;
 
+              /* We don't handle the port keyword at all.  */
+            case tok_port:
+              if (!current)
+                premature_token = "port";
+              break;
+
+              /* We don't handle the force keyword at all.  */
+            case tok_force:
+              if (!current)
+                premature_token = "force";
+              break;
+
               /* We handle tok_nothing below this switch.  */
             case tok_nothing:
               break;
@@ -365,10 +381,10 @@ parse_netrc (const char *path)
               /* Fetch the next token.  */
               if (!strcmp (tok, "account"))
                 last_token = tok_account;
+
               else if (!strcmp (tok, "default"))
-                {
                   maybe_add_to_list (&current, &retval);
-                }
+
               else if (!strcmp (tok, "login"))
                 last_token = tok_login;
 
@@ -381,6 +397,16 @@ parse_netrc (const char *path)
               else if (!strcmp (tok, "password"))
                 last_token = tok_password;
 
+				  /* GNU extensions 'port' and 'force', not operational
+					* see https://www.gnu.org/software/emacs/manual/html_node/gnus/NNTP.html#index-nntp_002dauthinfo_002dfunction-2003
+					* see https://savannah.gnu.org/bugs/index.php?52066
+					*/
+              else if (!strcmp (tok, "port"))
+                last_token = tok_port;
+
+              else if (!strcmp (tok, "force"))
+                last_token = tok_force;
+
               else
                 fprintf (stderr, _("%s: %s:%d: unknown token \"%s\"\n"),
                          exec_name, path, ln, tok);
@@ -435,11 +461,14 @@ free_netrc(acc_t *l)
 #ifdef STANDALONE
 #include <sys/types.h>
 #include <sys/stat.h>
+#include "exits.h"
+
+const char *program_argstring = NULL; /* Needed by warc.c */
 
 int
 main (int argc, char **argv)
 {
-  struct_stat sb;
+  struct stat sb;
   char *program_name, *file, *target;
   acc_t *head, *a;
 

src/netrc.h

@@ -1,6 +1,6 @@
 /* Declarations for netrc.c
-   Copyright (C) 1996, 1996, 1997, 2007, 2008, 2009, 2010, 2011, 2015
-   Free Software Foundation, Inc.
+   Copyright (C) 1996, 1996-1997, 2007-2011, 2015, 2018 Free Software
+   Foundation, Inc.
 
 This file is part of GNU Wget.
 

src/openssl.c

@@ -1,6 +1,5 @@
 /* SSL support via OpenSSL library.
-   Copyright (C) 2000, 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008,
-   2009, 2010, 2011, 2012, 2015 Free Software Foundation, Inc.
+   Copyright (C) 2000-2012, 2015, 2018 Free Software Foundation, Inc.
    Originally contributed by Christian Fraenkel.
 
 This file is part of GNU Wget.
@@ -35,6 +34,7 @@ as that of the covered work.  */
 #include <errno.h>
 #include <unistd.h>
 #include <string.h>
+#include <xalloc.h>
 
 #include <openssl/ssl.h>
 #include <openssl/x509v3.h>
@@ -173,11 +173,16 @@ ssl_init (void)
 {
   SSL_METHOD const *meth;
   long ssl_options = 0;
+#if !defined(LIBRESSL_VERSION_NUMBER) && (OPENSSL_VERSION_NUMBER >= 0x10100000L)
+  int ssl_proto_version = 0;
+#endif
 
 #if OPENSSL_VERSION_NUMBER >= 0x00907000
   if (ssl_true_initialized == 0)
     {
+#if OPENSSL_API_COMPAT < 0x10100000L
       OPENSSL_config (NULL);
+#endif
       ssl_true_initialized = 1;
     }
 #endif
@@ -201,20 +206,26 @@ ssl_init (void)
   CONF_modules_load_file(NULL, NULL,
       CONF_MFLAGS_DEFAULT_SECTION|CONF_MFLAGS_IGNORE_MISSING_FILE);
 #endif
+#if OPENSSL_API_COMPAT >= 0x10100000L
+  OPENSSL_init_ssl(0, NULL);
+#else
   SSL_library_init ();
   SSL_load_error_strings ();
+#endif
+#if OPENSSL_VERSION_NUMBER < 0x10100000L
   SSLeay_add_all_algorithms ();
   SSLeay_add_ssl_algorithms ();
+#endif
 
   switch (opt.secure_protocol)
     {
-#ifndef OPENSSL_NO_SSL2
+#if !defined OPENSSL_NO_SSL2 && OPENSSL_VERSION_NUMBER < 0x10100000L
     case secure_protocol_sslv2:
       meth = SSLv2_client_method ();
       break;
 #endif
 
-#ifndef OPENSSL_NO_SSL3
+#ifndef OPENSSL_NO_SSL3_METHOD
     case secure_protocol_sslv3:
       meth = SSLv3_client_method ();
       break;
@@ -226,16 +237,31 @@ ssl_init (void)
       ssl_options |= SSL_OP_NO_SSLv2 | SSL_OP_NO_SSLv3;
       break;
     case secure_protocol_tlsv1:
+#if !defined(LIBRESSL_VERSION_NUMBER) && (OPENSSL_VERSION_NUMBER >= 0x10100000L)
+      meth = TLS_client_method();
+      ssl_proto_version = TLS1_VERSION;
+#else
       meth = TLSv1_client_method ();
+#endif
       break;
 
 #if OPENSSL_VERSION_NUMBER >= 0x10001000
     case secure_protocol_tlsv1_1:
+#if !defined(LIBRESSL_VERSION_NUMBER) && (OPENSSL_VERSION_NUMBER >= 0x10100000L)
+      meth = TLS_client_method();
+      ssl_proto_version = TLS1_1_VERSION;
+#else
       meth = TLSv1_1_client_method ();
+#endif
       break;
 
     case secure_protocol_tlsv1_2:
+#if !defined(LIBRESSL_VERSION_NUMBER) && (OPENSSL_VERSION_NUMBER >= 0x10100000L)
+      meth = TLS_client_method();
+      ssl_proto_version = TLS1_2_VERSION;
+#else
       meth = TLSv1_2_client_method ();
+#endif
       break;
 #else
     case secure_protocol_tlsv1_1:
@@ -262,6 +288,11 @@ ssl_init (void)
   if (ssl_options)
     SSL_CTX_set_options (ssl_ctx, ssl_options);
 
+#if !defined(LIBRESSL_VERSION_NUMBER) && (OPENSSL_VERSION_NUMBER >= 0x10100000L)
+  if (ssl_proto_version)
+    SSL_CTX_set_min_proto_version(ssl_ctx, ssl_proto_version);
+#endif
+
   /* OpenSSL ciphers: https://www.openssl.org/docs/apps/ciphers.html
    * Since we want a good protection, we also use HIGH (that excludes MD4 ciphers and some more)
    */
@@ -506,6 +537,22 @@ ssl_connect_with_timeout_callback(void *arg)
   ctx->result = SSL_connect(ctx->ssl);
 }
 
+static const char *
+_sni_hostname(const char *hostname)
+{
+  size_t len = strlen(hostname);
+
+  char *sni_hostname = xmemdup(hostname, len + 1);
+
+  /* Remove trailing dot(s) to fix #47408.
+   * Regarding RFC 6066 (SNI): The hostname is represented as a byte
+   * string using ASCII encoding without a trailing dot. */
+  while (len && sni_hostname[--len] == '.')
+    sni_hostname[len] = 0;
+
+  return sni_hostname;
+}
+
 /* Perform the SSL handshake on file descriptor FD, which is assumed
    to be connected to an SSL server.  The SSL handle provided by
    OpenSSL is registered with the file descriptor FD using
@@ -532,7 +579,12 @@ ssl_connect_wget (int fd, const char *hostname, int *continue_session)
      then use it whenever we have a hostname.  If not, don't, ever. */
   if (! is_valid_ip_address (hostname))
     {
-      if (! SSL_set_tlsext_host_name (conn, hostname))
+      const char *sni_hostname = _sni_hostname(hostname);
+
+      long rc = SSL_set_tlsext_host_name (conn, sni_hostname);
+      xfree(sni_hostname);
+
+      if (rc == 0)
         {
           DEBUGP (("Failed to set TLS server-name indication."));
           goto error;
@@ -561,7 +613,7 @@ ssl_connect_wget (int fd, const char *hostname, int *continue_session)
     DEBUGP (("SSL handshake timed out.\n"));
     goto timeout;
   }
-  if (scwt_ctx.result <= 0 || SSL_state(conn) != SSL_ST_OK)
+  if (scwt_ctx.result <= 0 || !SSL_is_init_finished(conn))
     goto error;
 
   ctx = xnew0 (struct openssl_transport_context);
@@ -650,6 +702,65 @@ static char *_get_rfc2253_formatted (X509_NAME *name)
   return out ? out : xstrdup("");
 }
 
+/*
+ * Heavily modified from:
+ * https://www.owasp.org/index.php/Certificate_and_Public_Key_Pinning#OpenSSL
+ */
+static bool
+pkp_pin_peer_pubkey (X509* cert, const char *pinnedpubkey)
+{
+  /* Scratch */
+  int len1 = 0, len2 = 0;
+  char *buff1 = NULL, *temp = NULL;
+
+  /* Result is returned to caller */
+  bool result = false;
+
+  /* if a path wasn't specified, don't pin */
+  if (!pinnedpubkey)
+    return true;
+
+  if (!cert)
+    return result;
+
+  /* Begin Gyrations to get the subjectPublicKeyInfo     */
+  /* Thanks to Viktor Dukhovni on the OpenSSL mailing list */
+
+  /* https://groups.google.com/group/mailing.openssl.users/browse_thread
+   /thread/d61858dae102c6c7 */
+  len1 = i2d_X509_PUBKEY (X509_get_X509_PUBKEY (cert), NULL);
+  if (len1 < 1)
+    goto cleanup; /* failed */
+
+  /* https://www.openssl.org/docs/crypto/buffer.html */
+  buff1 = temp = OPENSSL_malloc (len1);
+  if (!buff1)
+    goto cleanup; /* failed */
+
+  /* https://www.openssl.org/docs/crypto/d2i_X509.html */
+  len2 = i2d_X509_PUBKEY (X509_get_X509_PUBKEY (cert), (unsigned char **) &temp);
+
+  /*
+   * These checks are verifying we got back the same values as when we
+   * sized the buffer. It's pretty weak since they should always be the
+   * same. But it gives us something to test.
+   */
+  if ((len1 != len2) || !temp || ((temp - buff1) != len1))
+    goto cleanup; /* failed */
+
+  /* End Gyrations */
+
+  /* The one good exit point */
+  result = wg_pin_peer_pubkey (pinnedpubkey, buff1, len1);
+
+ cleanup:
+  /* https://www.openssl.org/docs/crypto/buffer.html */
+  if (NULL != buff1)
+    OPENSSL_free (buff1);
+
+  return result;
+}
+
 /* Verify the validity of the certificate presented by the server.
    Also check that the "common name" of the server, as presented by
    its certificate, corresponds to HOST.  (HOST typically comes from
@@ -673,6 +784,7 @@ ssl_check_certificate (int fd, const char *host)
   long vresult;
   bool success = true;
   bool alt_name_checked = false;
+  bool pinsuccess = opt.pinnedpubkey == NULL;
 
   /* If the user has specified --no-check-cert, we still want to warn
      him about problems with the server's certificate.  */
@@ -682,6 +794,10 @@ ssl_check_certificate (int fd, const char *host)
   SSL *conn = ctx->conn;
   assert (conn != NULL);
 
+  /* The user explicitly said to not check for the certificate.  */
+  if (opt.check_cert == CHECK_CERT_QUIET && pinsuccess)
+    return success;
+
   cert = SSL_get_peer_certificate (conn);
   if (!cert)
     {
@@ -758,9 +874,12 @@ ssl_check_certificate (int fd, const char *host)
     {
       /* Test subject alternative names */
 
+      /* SNI hostname must not have a trailing dot */
+      const char *sni_hostname = _sni_hostname(host);
+
       /* Do we want to check for dNSNAmes or ipAddresses (see RFC 2818)?
        * Signal it by host_in_octet_string. */
-      ASN1_OCTET_STRING *host_in_octet_string = a2i_IPADDRESS (host);
+      ASN1_OCTET_STRING *host_in_octet_string = a2i_IPADDRESS (sni_hostname);
 
       int numaltnames = sk_GENERAL_NAME_num (subjectAltNames);
       int i;
@@ -795,7 +914,7 @@ ssl_check_certificate (int fd, const char *host)
                   if (0 <= ASN1_STRING_to_UTF8 (&name_in_utf8, name->d.dNSName))
                     {
                       /* Compare and check for NULL attack in ASN1_STRING */
-                      if (pattern_match ((char *)name_in_utf8, host) &&
+                      if (pattern_match ((char *)name_in_utf8, sni_hostname) &&
                             (strlen ((char *)name_in_utf8) ==
                                 (size_t) ASN1_STRING_length (name->d.dNSName)))
                         {
@@ -816,9 +935,11 @@ ssl_check_certificate (int fd, const char *host)
           logprintf (LOG_NOTQUIET,
               _("%s: no certificate subject alternative name matches\n"
                 "\trequested host name %s.\n"),
-                     severity, quote_n (1, host));
+                     severity, quote_n (1, sni_hostname));
           success = false;
         }
+
+      xfree(sni_hostname);
     }
 
   if (alt_name_checked == false)
@@ -873,6 +994,13 @@ ssl_check_certificate (int fd, const char *host)
         }
     }
 
+    pinsuccess = pkp_pin_peer_pubkey (cert, opt.pinnedpubkey);
+    if (!pinsuccess)
+      {
+        logprintf (LOG_ALWAYS, _("The public key does not match pinned public key!\n"));
+        success = false;
+      }
+
 
   if (success)
     DEBUGP (("X509 certificate successfully verified and matches host %s\n",
@@ -880,13 +1008,13 @@ ssl_check_certificate (int fd, const char *host)
   X509_free (cert);
 
  no_cert:
-  if (opt.check_cert && !success)
+  if (opt.check_cert == CHECK_CERT_ON && !success)
     logprintf (LOG_NOTQUIET, _("\
 To connect to %s insecurely, use `--no-check-certificate'.\n"),
                quotearg_style (escape_quoting_style, host));
 
-  /* Allow --no-check-cert to disable certificate checking. */
-  return opt.check_cert ? success : true;
+  /* never return true if pinsuccess fails */
+  return !pinsuccess ? false : (opt.check_cert == CHECK_CERT_ON ? success : true);
 }
 
 /*

src/options.h

@@ -1,7 +1,5 @@
 /* struct options.
-   Copyright (C) 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004,
-   2005, 2006, 2007, 2008, 2009, 2010, 2011, 2015 Free Software
-   Foundation, Inc.
+   Copyright (C) 1996-2011, 2015, 2018 Free Software Foundation, Inc.
 
 This file is part of GNU Wget.
 
@@ -29,6 +27,13 @@ Corresponding Source for a non-source form of such a combination
 shall include the source code for the parts of OpenSSL used as well
 as that of the covered work.  */
 
+enum CHECK_CERT_MODES
+{
+  CHECK_CERT_OFF,
+  CHECK_CERT_ON,
+  CHECK_CERT_QUIET
+};
+
 struct options
 {
   int verbose;                  /* Are we verbose?  (First set to -1,
@@ -36,6 +41,7 @@ struct options
   bool quiet;                   /* Are we quiet? */
   int ntry;                     /* Number of tries per URL */
   bool retry_connrefused;       /* Treat CONNREFUSED as non-fatal. */
+  char *retry_on_http_error;    /* Treat given HTTP errors as non-fatal. */
   bool background;              /* Whether we should work in background. */
   bool ignore_length;           /* Do we heed content-length at all?  */
   bool recursive;               /* Are we recursive? */
@@ -54,12 +60,13 @@ struct options
   bool add_hostdir;             /* Do we add hostname directory? */
   bool protocol_directories;    /* Whether to prepend "http"/"ftp" to dirs. */
   bool noclobber;               /* Disables clobbering of existing data. */
-  bool unlink;                  /* remove file before clobbering */
+  bool unlink_requested;        /* remove file before clobbering */
   char *dir_prefix;             /* The top of directory tree */
   char *lfilename;              /* Log filename */
   char *input_filename;         /* Input filename */
 #ifdef HAVE_METALINK
   char *input_metalink;         /* Input metalink file */
+  int metalink_index;           /* Metalink application/metalink4+xml metaurl ordinal number. */
   bool metalink_over_http;      /* Use Metalink if present in HTTP response */
   char *preferred_location;     /* Preferred location for Metalink resources */
 #endif
@@ -92,6 +99,11 @@ struct options
   void *(*regex_compile_fun)(const char *);             /* Function to compile a regex. */
   bool (*regex_match_fun)(const void *, const char *);  /* Function to match a string to a regex. */
 
+#ifdef HAVE_LIBCARES
+  char *bind_dns_address;
+  char *dns_servers;
+#endif
+
   char **domains;               /* See host.c */
   char **exclude_domains;
   bool dns_cache;               /* whether we cache DNS lookups. */
@@ -115,9 +127,12 @@ struct options
   bool warc_keep_log;           /* Store the log file in a WARC record. */
   char **warc_user_headers;     /* User-defined WARC header(s). */
 
+  bool enable_xattr;            /* Store metadata in POSIX extended attributes. */
+
   char *user;                   /* Generic username */
   char *passwd;                 /* Generic password */
   bool ask_passwd;              /* Ask for password? */
+  char *use_askpass;           /* value to use for use-askpass if WGET_ASKPASS is not set */
 
   bool always_rest;             /* Always use REST. */
   wgint start_pos;              /* Start position of a download. */
@@ -215,7 +230,7 @@ struct options
     secure_protocol_tlsv1_2,
     secure_protocol_pfs
   } secure_protocol;            /* type of secure protocol to use. */
-  bool check_cert;              /* whether to validate the server's cert */
+  int check_cert;               /* whether to validate the server's cert */
   char *cert_file;              /* external client certificate to use. */
   char *private_key;            /* private key file (if not internal). */
   enum keyfile_type {
@@ -229,6 +244,11 @@ struct options
   char *ca_cert;                /* CA certificate file to use */
   char *crl_file;               /* file with CRLs */
 
+  char *pinnedpubkey;           /* Public key (PEM/DER) file, or any number
+                                   of base64 encoded sha256 hashes preceded by
+                                   \'sha256//\' and separated by \';\', to verify
+                                   peer against */
+
   char *random_file;            /* file with random data to seed the PRNG */
   char *egd_file;               /* file name of the egd daemon socket */
   bool https_only;              /* whether to follow HTTPS only */
@@ -241,6 +261,7 @@ struct options
   bool cookies;                 /* whether cookies are used. */
   char *cookies_input;          /* file we're loading the cookies from. */
   char *cookies_output;         /* file we're saving the cookies to. */
+  bool keep_badhash;            /* Keep files with checksum mismatch. */
   bool keep_session_cookies;    /* whether session cookies should be
                                    saved and loaded. */
 
@@ -289,7 +310,7 @@ struct options
 
   bool enable_iri;
   char *encoding_remote;
-  char *locale;
+  const char *locale;
 
   bool trustservernames;
 #ifdef __VMS
@@ -303,6 +324,14 @@ struct options
                                    name. */
   bool report_bps;              /*Output bandwidth in bits format*/
 
+#ifdef HAVE_LIBZ
+  enum compression_options {
+    compression_auto,
+    compression_gzip,
+    compression_none
+  } compression;                /* type of HTTP compression to use */
+#endif
+
   char *rejected_log;           /* The file to log rejected URLS to. */
 
 #ifdef HAVE_HSTS

src/progress.c

@@ -1,6 +1,5 @@
 /* Download progress.
-   Copyright (C) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009,
-   2010, 2011, 2015 Free Software Foundation, Inc.
+   Copyright (C) 2001-2011, 2015, 2018 Free Software Foundation, Inc.
 
 This file is part of GNU Wget.
 
@@ -75,7 +74,7 @@ static struct progress_implementation implementations[] = {
 static struct progress_implementation *current_impl;
 static int current_impl_locked;
 
-/* Progress implementation used by default.  Can be overriden in
+/* Progress implementation used by default.  Can be overridden in
    wgetrc or by the fallback one.  */
 
 #define DEFAULT_PROGRESS_IMPLEMENTATION "bar"
@@ -594,7 +593,8 @@ bar_create (const char *f_download, wgint initial, wgint total)
   bp->width = screen_width - 1;
   /* + enough space for the terminating zero, and hopefully enough room
    * for multibyte characters. */
-  bp->buffer = xmalloc (bp->width + 100);
+#define BUF_LEN (bp->width + 100)
+  bp->buffer = xmalloc (BUF_LEN);
 
   logputs (LOG_VERBOSE, "\n");
 
@@ -854,7 +854,7 @@ get_eta (int *bcd)
 {
   /* TRANSLATORS: "ETA" is English-centric, but this must
      be short, ideally 3 chars.  Abbreviate if necessary.  */
-  static const char eta_str[] = N_("   eta %s");
+  static const char eta_str[] = N_("    eta %s");
   static const char *eta_trans;
   static int bytes_cols_diff;
   if (eta_trans == NULL)
@@ -891,11 +891,6 @@ get_eta (int *bcd)
   p += sizeof (s) - 1;                          \
 } while (0)
 
-/* Use move_to_end (s) to get S to point the end of the string (the
-   terminating \0).  This is faster than s+=strlen(s), but some people
-   are confused when they see strchr (s, '\0') in the code.  */
-#define move_to_end(s) s = strchr (s, '\0');
-
 static void
 create_image (struct bar_progress *bp, double dl_total_time, bool done)
 {
@@ -903,11 +898,11 @@ create_image (struct bar_progress *bp, double dl_total_time, bool done)
   char *p = bp->buffer;
   wgint size = bp->initial_length + bp->count;
 
-  int size_grouped_pad; /* Used to pad the field width for size_grouped. */
-
   struct bar_progress_hist *hist = &bp->hist;
   int orig_filename_cols = count_cols (bp->f_download);
 
+  int padding;
+
   /* The progress bar should look like this:
      file xx% [=======>             ] nnn.nnK 12.34KB/s  eta 36m 51s
 
@@ -928,12 +923,16 @@ create_image (struct bar_progress *bp, double dl_total_time, bool done)
      "=====>..."       - progress bar             - the rest
   */
 
+  /* TODO: Ask the Turkish Translators to fix their translation for the "done"
+   * mode of progress bar. Use one less character. Once that is done, redice
+   * PROGRESS_ETA_LEN by 1.
+   */
 #define PROGRESS_FILENAME_LEN  MAX_FILENAME_COLS + 1
 #define PROGRESS_PERCENT_LEN   4
 #define PROGRESS_DECORAT_LEN   2
 #define PROGRESS_FILESIZE_LEN  7 + 1
-#define PROGRESS_DWNLOAD_RATE  8 + 1
-#define PROGRESS_ETA_LEN       14
+#define PROGRESS_DWNLOAD_RATE  8 + 2
+#define PROGRESS_ETA_LEN       15
 
   int progress_size = bp->width - (PROGRESS_FILENAME_LEN + PROGRESS_PERCENT_LEN +
                                    PROGRESS_DECORAT_LEN + PROGRESS_FILESIZE_LEN +
@@ -945,23 +944,23 @@ create_image (struct bar_progress *bp, double dl_total_time, bool done)
   int cols_diff;
   const char *down_size;
 
+  memset (bp->buffer, '\0', BUF_LEN);
+
   if (progress_size < 5)
     progress_size = 0;
 
   if (orig_filename_cols <= MAX_FILENAME_COLS)
     {
-      int padding = MAX_FILENAME_COLS - orig_filename_cols;
-      sprintf (p, "%s ", bp->f_download);
-      p += orig_filename_cols + 1;
-      for (;padding;padding--)
-        *p++ = ' ';
+      padding = MAX_FILENAME_COLS - orig_filename_cols;
+      p += sprintf (p, "%s ", bp->f_download);
+      memset (p, ' ', padding);
+      p += padding;
     }
   else
     {
       int offset_cols;
       int bytes_in_filename, offset_bytes, col;
       int *cols_ret = &col;
-      int padding;
 
 #define MIN_SCROLL_TEXT 5
       if ((orig_filename_cols > MAX_FILENAME_COLS + MIN_SCROLL_TEXT) &&
@@ -992,9 +991,8 @@ create_image (struct bar_progress *bp, double dl_total_time, bool done)
       memcpy (p, bp->f_download + offset_bytes, bytes_in_filename);
       p += bytes_in_filename;
       padding = MAX_FILENAME_COLS - (padding + *cols_ret);
-      for (;padding;padding--)
-          *p++ = ' ';
-      *p++ = ' ';
+      memset (p, ' ', padding + 1);
+      p += padding + 1;
     }
 
   /* "xx% " */
@@ -1002,15 +1000,13 @@ create_image (struct bar_progress *bp, double dl_total_time, bool done)
     {
       int percentage = 100.0 * size / bp->total_length;
       assert (percentage <= 100);
-
-      if (percentage < 100)
-        sprintf (p, "%3d%%", percentage);
-      else
-        strcpy (p, "100%");
-      p += 4;
+      p += sprintf (p, "%3d%%", percentage);
     }
   else
-    APPEND_LITERAL ("    ");
+    {
+      memset (p, ' ', PROGRESS_PERCENT_LEN);
+      p += PROGRESS_PERCENT_LEN;
+    }
 
   /* The progress bar: "[====>      ]" or "[++==>      ]". */
   if (progress_size && bp->total_length > 0)
@@ -1022,7 +1018,6 @@ create_image (struct bar_progress *bp, double dl_total_time, bool done)
       int dlsz = (double)size / bp->total_length * progress_size;
 
       char *begin;
-      int i;
 
       assert (dlsz <= progress_size);
       assert (insz <= dlsz);
@@ -1032,18 +1027,19 @@ create_image (struct bar_progress *bp, double dl_total_time, bool done)
 
       /* Print the initial portion of the download with '+' chars, the
          rest with '=' and one '>'.  */
-      for (i = 0; i < insz; i++)
-        *p++ = '+';
+      memset (p, '+', insz);
+      p += insz;
+
       dlsz -= insz;
       if (dlsz > 0)
         {
-          for (i = 0; i < dlsz - 1; i++)
-            *p++ = '=';
+          memset (p, '=', dlsz-1);
+          p += dlsz - 1;
           *p++ = '>';
         }
 
-      while (p - begin < progress_size)
-        *p++ = ' ';
+      memset (p, ' ', (progress_size - (p - begin)));
+      p += (progress_size - (p - begin));
       *p++ = ']';
     }
   else if (progress_size)
@@ -1071,27 +1067,14 @@ create_image (struct bar_progress *bp, double dl_total_time, bool done)
       *p++ = ']';
 
     }
- ++bp->tick;
+  ++bp->tick;
 
   /* " 234.56M" */
   down_size = human_readable (size, 1000, 2);
-  cols_diff = 7 - count_cols (down_size);
-  while (cols_diff > 0)
-  {
-    *p++=' ';
-    cols_diff--;
-  }
-  sprintf (p, " %s", down_size);
-  move_to_end (p);
-  /* Pad with spaces to 7 chars for the size_grouped field;
-   * couldn't use the field width specifier in sprintf, because
-   * it counts in bytes, not characters. */
-  for (size_grouped_pad = PROGRESS_FILESIZE_LEN - 7;
-       size_grouped_pad > 0;
-       --size_grouped_pad)
-    {
-      *p++ = ' ';
-    }
+  cols_diff = PROGRESS_FILESIZE_LEN - count_cols (down_size);
+  memset (p, ' ', cols_diff);
+  p += cols_diff;
+  p += sprintf (p, "%s", down_size);
 
   /* " 12.52Kb/s or 12.52KB/s" */
   if (hist->total_time > 0 && hist->total_bytes)
@@ -1104,12 +1087,11 @@ create_image (struct bar_progress *bp, double dl_total_time, bool done)
       wgint dlquant = hist->total_bytes + bp->recent_bytes;
       double dltime = hist->total_time + (dl_total_time - bp->recent_start);
       double dlspeed = calc_rate (dlquant, dltime, &units);
-      sprintf (p, " %4.*f%s", dlspeed >= 99.95 ? 0 : dlspeed >= 9.995 ? 1 : 2,
+      p += sprintf (p, "  %4.*f%s", dlspeed >= 99.95 ? 0 : dlspeed >= 9.995 ? 1 : 2,
                dlspeed,  !opt.report_bps ? short_units[units] : short_units_bits[units]);
-      move_to_end (p);
     }
   else
-    APPEND_LITERAL (" --.-KB/s");
+    APPEND_LITERAL ("  --.-KB/s");
 
   if (!done)
     {
@@ -1144,14 +1126,14 @@ create_image (struct bar_progress *bp, double dl_total_time, bool done)
               bp->last_eta_time = dl_total_time;
             }
 
-          sprintf (p, get_eta(&bytes_cols_diff),
+          p += sprintf (p, get_eta(&bytes_cols_diff),
                    eta_to_human_short (eta, false));
-          move_to_end (p);
         }
       else if (bp->total_length > 0)
         {
         skip_eta:
-          APPEND_LITERAL ("             ");
+          memset (p, ' ', PROGRESS_ETA_LEN);
+          p += PROGRESS_ETA_LEN;
         }
     }
   else
@@ -1161,21 +1143,25 @@ create_image (struct bar_progress *bp, double dl_total_time, bool done)
       int ncols;
 
       /* Note to translators: this should not take up more room than
-         available here.  Abbreviate if necessary.  */
-      strcpy (p, _("   in "));
+         available here (6 columns).  Abbreviate if necessary.  */
+      strcpy (p, _("    in "));
       nbytes = strlen (p);
       ncols  = count_cols (p);
       bytes_cols_diff = nbytes - ncols;
-      p += nbytes;
       if (dl_total_time >= 10)
-        strcpy (p, eta_to_human_short ((int) (dl_total_time + 0.5), false));
+        ncols += sprintf (p + nbytes, "%s",  eta_to_human_short ((int) (dl_total_time + 0.5), false));
       else
-        sprintf (p, "%ss", print_decimal (dl_total_time));
-      move_to_end (p);
+        ncols += sprintf (p + nbytes, "%ss", print_decimal (dl_total_time));
+      p += ncols + bytes_cols_diff;
+      memset (p, ' ', PROGRESS_ETA_LEN - ncols);
+      p += PROGRESS_ETA_LEN - ncols;
     }
 
-  while (p - bp->buffer - bytes_cols_diff < bp->width)
-    *p++ = ' ';
+  padding = bp->width - count_cols (bp->buffer);
+  assert (padding >= 0 && "Padding length became non-positive!");
+  padding = padding > 0 ? padding : 0;
+  memset (p, ' ', padding);
+  p += padding;
   *p = '\0';
 
   /* 2014-11-14  Darshit Shah  <darnir@gmail.com>
@@ -1184,8 +1170,11 @@ create_image (struct bar_progress *bp, double dl_total_time, bool done)
    * from the release code since we do not want Wget to crash and burn when the
    * assertion fails. Instead Wget should continue downloading and display a
    * horrible and irritating progress bar that spams the screen with newlines.
+   *
+   * By default, all assertions are disabled in a Wget build and are enabled
+   * only with the --enable-assert configure option.
    */
-  assert (count_cols (bp->buffer) <= bp->width + 1);
+  assert (count_cols (bp->buffer) == bp->width);
 }
 
 /* Print the contents of the buffer as a one-line ASCII "image" so
@@ -1203,8 +1192,6 @@ display_image (char *buf)
 static void
 bar_set_params (char *params)
 {
-  char *term = getenv ("TERM");
-
   if (params)
     {
       char *param = strtok (params, ":");
@@ -1224,12 +1211,6 @@ bar_set_params (char *params)
           dots.  */
        || !isatty (fileno (stderr))
 #endif
-       /* Normally we don't depend on terminal type because the
-          progress bar only uses ^M to move the cursor to the
-          beginning of line, which works even on dumb terminals.  But
-          Jamie Zawinski reports that ^M and ^H tricks don't work in
-          Emacs shell buffers, and only make a mess.  */
-       || (term && 0 == strcmp (term, "emacs"))
        )
       && !current_impl_locked)
     {

src/progress.h

@@ -1,6 +1,5 @@
 /* Download progress.
-   Copyright (C) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009,
-   2010, 2011, 2015 Free Software Foundation, Inc.
+   Copyright (C) 2001-2011, 2015, 2018 Free Software Foundation, Inc.
 
 This file is part of GNU Wget.
 

src/ptimer.c

@@ -1,6 +1,5 @@
 /* Portable timers.
-   Copyright (C) 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2015 Free
-   Software Foundation, Inc.
+   Copyright (C) 2005-2011, 2015, 2018 Free Software Foundation, Inc.
 
 This file is part of GNU Wget.
 

src/ptimer.h

@@ -1,6 +1,5 @@
 /* Declarations for ptimer.c.
-   Copyright (C) 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2015 Free
-   Software Foundation, Inc.
+   Copyright (C) 2005-2011, 2015, 2018 Free Software Foundation, Inc.
 
 This file is part of GNU Wget.
 

src/recur.c

@@ -1,7 +1,5 @@
 /* Handling of recursive HTTP retrieving.
-   Copyright (C) 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004,
-   2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012, 2015 Free Software
-   Foundation, Inc.
+   Copyright (C) 1996-2012, 2015, 2018 Free Software Foundation, Inc.
 
 This file is part of GNU Wget.
 
@@ -234,17 +232,19 @@ retrieve_tree (struct url *start_url_parsed, struct iri *pi)
 
   FILE *rejectedlog = NULL; /* Don't write a rejected log. */
 
-#define COPYSTR(x)  (x) ? xstrdup(x) : NULL;
   /* Duplicate pi struct if not NULL */
   if (pi)
     {
+#define COPYSTR(x)  (x) ? xstrdup(x) : NULL;
       i->uri_encoding = COPYSTR (pi->uri_encoding);
       i->content_encoding = COPYSTR (pi->content_encoding);
       i->utf8_encode = pi->utf8_encode;
+#undef COPYSTR
     }
+#ifdef ENABLE_IRI
   else
     set_uri_encoding (i, opt.locale, true);
-#undef COPYSTR
+#endif
 
   queue = url_queue_new ();
   blacklist = make_string_hash_table (0);
@@ -436,10 +436,15 @@ retrieve_tree (struct url *start_url_parsed, struct iri *pi)
               struct url *url_parsed = url_parse (url, NULL, i, true);
               struct iri *ci;
               char *referer_url = url;
-              bool strip_auth = (url_parsed != NULL
-                                 && url_parsed->user != NULL);
+              bool strip_auth;
+
               assert (url_parsed != NULL);
 
+              if (!url_parsed)
+                continue;
+
+              strip_auth = (url_parsed && url_parsed->user);
+
               /* Strip auth info if present */
               if (strip_auth)
                 referer_url = url_string (url_parsed, URL_AUTH_HIDE);
@@ -449,9 +454,16 @@ retrieve_tree (struct url *start_url_parsed, struct iri *pi)
                   reject_reason r;
 
                   if (child->ignore_when_downloading)
-                    continue;
+                    {
+                      DEBUGP (("Not following due to 'ignore' flag: %s\n", child->url->url));
+                      continue;
+                    }
+
                   if (dash_p_leaf_HTML && !child->link_inline_p)
-                    continue;
+                    {
+                      DEBUGP (("Not following due to 'link inline' flag: %s\n", child->url->url));
+                      continue;
+                    }
 
                   r = download_child (child, url_parsed, depth,
                                       start_url_parsed, blacklist, i);
@@ -483,7 +495,7 @@ retrieve_tree (struct url *start_url_parsed, struct iri *pi)
 
       if (file
           && (opt.delete_after
-              || opt.spider /* opt.recursive is implicitely true */
+              || opt.spider /* opt.recursive is implicitly true */
               || !acceptable (file)))
         {
           /* Either --delete-after was specified, or we loaded this
@@ -610,7 +622,11 @@ download_child (const struct urlpos *upos, struct url *parent, int depth,
   u_scheme_like_http = schemes_are_similar_p (u->scheme, SCHEME_HTTP);
 
   /* 1. Schemes other than HTTP are normally not recursed into. */
-  if (!u_scheme_like_http && !((u->scheme == SCHEME_FTP || u->scheme == SCHEME_FTPS) && opt.follow_ftp))
+  if (!u_scheme_like_http && !((u->scheme == SCHEME_FTP
+#ifdef HAVE_SSL
+      || u->scheme == SCHEME_FTPS
+#endif
+      ) && opt.follow_ftp))
     {
       DEBUGP (("Not following non-HTTP schemes.\n"));
       reason = WG_RR_NONHTTP;
@@ -680,7 +696,7 @@ download_child (const struct urlpos *upos, struct url *parent, int depth,
      for directories (no file name to match) and for non-leaf HTMLs,
      which can lead to other files that do need to be downloaded.  (-p
      automatically implies non-leaf because with -p we can, if
-     necesary, overstep the maximum depth to get the page requisites.)  */
+     necessary, overstep the maximum depth to get the page requisites.)  */
   if (u->file[0] != '\0'
       && !(has_html_suffix_p (u->file)
            /* The exception only applies to non-leaf HTMLs (but -p
@@ -795,6 +811,12 @@ descend_redirect (const char *redirected, struct url *orig_parsed, int depth,
 
   if (reason == WG_RR_SUCCESS)
     blacklist_add (blacklist, upos->url->url);
+  else if (reason == WG_RR_LIST || reason == WG_RR_REGEX)
+    {
+      DEBUGP (("Ignoring decision for redirects, decided to load it.\n"));
+      blacklist_add (blacklist, upos->url->url);
+      reason = WG_RR_SUCCESS;
+    }
   else
     DEBUGP (("Redirection \"%s\" failed the test.\n", redirected));
 

src/recur.h

@@ -1,7 +1,5 @@
 /* Declarations for recur.c.
-   Copyright (C) 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004,
-   2005, 2006, 2007, 2008, 2009, 2010, 2011, 2015 Free Software
-   Foundation, Inc.
+   Copyright (C) 1996-2011, 2015, 2018 Free Software Foundation, Inc.
 
 This file is part of GNU Wget.
 

src/res.c

@@ -1,6 +1,6 @@
 /* Support for Robot Exclusion Standard (RES).
-   Copyright (C) 2001, 2006, 2007, 2008, 2009, 2010, 2011, 2015 Free
-   Software Foundation, Inc.
+   Copyright (C) 2001, 2006-2011, 2015, 2018 Free Software Foundation,
+   Inc.
 
 This file is part of Wget.
 
@@ -37,12 +37,12 @@ as that of the covered work.  */
    disallow access to certain parts of the site.
 
    The first specification was written by Martijn Koster in 1994, and
-   is still available at <http://www.robotstxt.org/wc/norobots.html>.
+   is still available at <http://www.robotstxt.org/orig.html>.
    In 1996, Martijn wrote an Internet Draft specifying an improved RES
    specification; however, that work was apparently abandoned since
    the draft has expired in 1997 and hasn't been replaced since.  The
    draft is available at
-   <http://www.robotstxt.org/wc/norobots-rfc.html>.
+   <http://www.robotstxt.org/norobots-rfc.txt>.
 
    This file implements RES as specified by the draft.  Note that this
    only handles the "robots.txt" support.  The META tag that controls
@@ -428,7 +428,7 @@ free_specs (struct robot_specs *specs)
 
 /* The inner matching engine: return true if RECORD_PATH matches
    URL_PATH.  The rules for matching are described at
-   <http://www.robotstxt.org/wc/norobots-rfc.txt>, section 3.2.2.  */
+   <http://www.robotstxt.org/norobots-rfc.txt>, section 3.2.2.  */
 
 static bool
 matches (const char *record_path, const char *url_path)

src/res.h

@@ -1,6 +1,6 @@
 /* Declarations for res.c.
-   Copyright (C) 2001, 2007, 2008, 2009, 2010, 2011, 2015 Free Software
-   Foundation, Inc.
+   Copyright (C) 2001, 2007-2011, 2015, 2018 Free Software Foundation,
+   Inc.
 
 This file is part of Wget.
 

src/retr.c

@@ -1,7 +1,6 @@
 /* File retrieval.
-   Copyright (C) 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004,
-   2005, 2006, 2007, 2008, 2009, 2010, 2011, 2014, 2015 Free Software
-   Foundation, Inc.
+   Copyright (C) 1996-2011, 2014-2015, 2018 Free Software Foundation,
+   Inc.
 
 This file is part of GNU Wget.
 
@@ -41,6 +40,10 @@ as that of the covered work.  */
 # include <unixio.h>            /* For delete(). */
 #endif
 
+#ifdef HAVE_LIBZ
+# include <zlib.h>
+#endif
+
 #include "exits.h"
 #include "utils.h"
 #include "retr.h"
@@ -84,6 +87,22 @@ limit_bandwidth_reset (void)
   xzero (limit_data);
 }
 
+#ifdef HAVE_LIBZ
+static voidpf
+zalloc (voidpf opaque, unsigned int items, unsigned int size)
+{
+  (void) opaque;
+  return (voidpf) xcalloc (items, size);
+}
+
+static void
+zfree (voidpf opaque, voidpf address)
+{
+  (void) opaque;
+  xfree (address);
+}
+#endif
+
 /* Limit the bandwidth by pausing the download for an amount of time.
    BYTES is the number of bytes received from the network, and TIMER
    is the timer that started at the beginning of download.  */
@@ -257,6 +276,44 @@ fd_read_body (const char *downloaded_filename, int fd, FILE *out, wgint toread,
   wgint sum_written = 0;
   wgint remaining_chunk_size = 0;
 
+#ifdef HAVE_LIBZ
+  /* try to minimize the number of calls to inflate() and write_data() per
+     call to fd_read() */
+  unsigned int gzbufsize = dlbufsize * 4;
+  char *gzbuf = NULL;
+  z_stream gzstream;
+
+  if (flags & rb_compressed_gzip)
+    {
+      gzbuf = xmalloc (gzbufsize);
+      if (gzbuf != NULL)
+        {
+          gzstream.zalloc = zalloc;
+          gzstream.zfree = zfree;
+          gzstream.opaque = Z_NULL;
+          gzstream.next_in = Z_NULL;
+          gzstream.avail_in = 0;
+
+          #define GZIP_DETECT 32 /* gzip format detection */
+          #define GZIP_WINDOW 15 /* logarithmic window size (default: 15) */
+          ret = inflateInit2 (&gzstream, GZIP_DETECT | GZIP_WINDOW);
+          if (ret != Z_OK)
+            {
+              xfree (gzbuf);
+              errno = (ret == Z_MEM_ERROR) ? ENOMEM : EINVAL;
+              ret = -1;
+              goto out;
+            }
+        }
+      else
+        {
+          errno = ENOMEM;
+          ret = -1;
+          goto out;
+        }
+    }
+#endif
+
   if (flags & rb_skip_startpos)
     skip = startpos;
 
@@ -320,6 +377,12 @@ fd_read_body (const char *downloaded_filename, int fd, FILE *out, wgint toread,
               remaining_chunk_size = strtol (line, &endl, 16);
               xfree (line);
 
+              if (remaining_chunk_size < 0)
+                {
+                  ret = -1;
+                  break;
+                }
+
               if (remaining_chunk_size == 0)
                 {
                   ret = 0;
@@ -383,12 +446,64 @@ fd_read_body (const char *downloaded_filename, int fd, FILE *out, wgint toread,
           int write_res;
 
           sum_read += ret;
-          write_res = write_data (out, out2, dlbuf, ret, &skip, &sum_written);
-          if (write_res < 0)
+
+#ifdef HAVE_LIBZ
+          if (gzbuf != NULL)
             {
-              ret = (write_res == -3) ? -3 : -2;
-              goto out;
+              int err;
+              int towrite;
+              gzstream.avail_in = ret;
+              gzstream.next_in = (unsigned char *) dlbuf;
+
+              do
+                {
+                  gzstream.avail_out = gzbufsize;
+                  gzstream.next_out = (unsigned char *) gzbuf;
+
+                  err = inflate (&gzstream, Z_NO_FLUSH);
+
+                  switch (err)
+                    {
+                    case Z_MEM_ERROR:
+                      errno = ENOMEM;
+                      ret = -1;
+                      goto out;
+                    case Z_NEED_DICT:
+                    case Z_DATA_ERROR:
+                      errno = EINVAL;
+                      ret = -1;
+                      goto out;
+                    case Z_STREAM_END:
+                      if (exact && sum_read != toread)
+                        {
+                          DEBUGP(("zlib stream ended unexpectedly after "
+                                  "%ld/%ld bytes\n", sum_read, toread));
+                        }
+                    }
+
+                  towrite = gzbufsize - gzstream.avail_out;
+                  write_res = write_data (out, out2, gzbuf, towrite, &skip,
+                                          &sum_written);
+                  if (write_res < 0)
+                    {
+                      ret = (write_res == -3) ? -3 : -2;
+                      goto out;
+                    }
+                }
+              while (gzstream.avail_out == 0);
             }
+          else
+#endif
+            {
+              write_res = write_data (out, out2, dlbuf, ret, &skip,
+                                      &sum_written);
+              if (write_res < 0)
+                {
+                  ret = (write_res == -3) ? -3 : -2;
+                  goto out;
+                }
+            }
+
           if (chunked)
             {
               remaining_chunk_size -= ret;
@@ -433,6 +548,31 @@ fd_read_body (const char *downloaded_filename, int fd, FILE *out, wgint toread,
   if (timer)
     ptimer_destroy (timer);
 
+#ifdef HAVE_LIBZ
+  if (gzbuf != NULL)
+    {
+      int err = inflateEnd (&gzstream);
+      if (ret >= 0)
+        {
+          /* with compression enabled, ret must be 0 if successful */
+          if (err == Z_OK)
+            ret = 0;
+          else
+            {
+              errno = EINVAL;
+              ret = -1;
+            }
+        }
+      xfree (gzbuf);
+
+      if (gzstream.total_in != sum_read)
+        {
+          DEBUGP(("zlib read size differs from raw read size (%lu/%lu)\n",
+                  gzstream.total_in, sum_read));
+        }
+    }
+#endif
+
   if (qtyread)
     *qtyread += sum_read;
   if (qtywritten)
@@ -634,7 +774,7 @@ retr_rate (wgint bytes, double secs)
   double dlrate = calc_rate (bytes, secs, &units);
   /* Use more digits for smaller numbers (regardless of unit used),
      e.g. "1022", "247", "12.5", "2.38".  */
-  sprintf (res, "%.*f %s",
+  snprintf (res, sizeof(res), "%.*f %s",
            dlrate >= 99.95 ? 0 : dlrate >= 9.995 ? 1 : 2,
            dlrate, !opt.report_bps ? rate_names[units]: rate_names_bits[units]);
 
@@ -830,14 +970,19 @@ retrieve_url (struct url * orig_parsed, const char *origurl, char **file,
       if (redirection_count)
         oldrec = glob = false;
 
-      result = ftp_loop (u, &local_file, dt, proxy_url, recursive, glob);
+      result = ftp_loop (u, orig_parsed, &local_file, dt, proxy_url,
+                         recursive, glob);
       recursive = oldrec;
 
       /* There is a possibility of having HTTP being redirected to
          FTP.  In these cases we must decide whether the text is HTML
          according to the suffix.  The HTML suffixes are `.html',
          `.htm' and a few others, case-insensitive.  */
-      if (redirection_count && local_file && (u->scheme == SCHEME_FTP || u->scheme == SCHEME_FTPS))
+      if (redirection_count && local_file && (u->scheme == SCHEME_FTP
+#ifdef HAVE_SSL
+          || u->scheme == SCHEME_FTPS
+#endif
+          ))
         {
           if (has_html_suffix_p (local_file))
             *dt |= TEXTHTML;
@@ -864,15 +1009,19 @@ retrieve_url (struct url * orig_parsed, const char *origurl, char **file,
          redirects, but a ton of boneheaded webservers and CGIs out
          there break the rules and use relative URLs, and popular
          browsers are lenient about this, so wget should be too. */
-      construced_newloc = uri_merge (url, mynewloc);
+      construced_newloc = uri_merge (url, mynewloc ? mynewloc : "");
       xfree (mynewloc);
       mynewloc = construced_newloc;
 
-      /* Reset UTF-8 encoding state, keep the URI encoding and reset
+#ifdef ENABLE_IRI
+      /* Reset UTF-8 encoding state, set the URI encoding and reset
          the content encoding. */
       iri->utf8_encode = opt.enable_iri;
+      if (opt.encoding_remote)
+       set_uri_encoding (iri, opt.encoding_remote, true);
       set_content_encoding (iri, NULL);
       xfree (iri->orig_url);
+#endif
 
       /* Now, see if this new location makes sense. */
       newloc_parsed = url_parse (mynewloc, &up_error_code, iri, true);
@@ -954,11 +1103,16 @@ retrieve_url (struct url * orig_parsed, const char *origurl, char **file,
       u = url_parse (origurl, NULL, iri, true);
       if (u)
         {
-          DEBUGP (("[IRI fallbacking to non-utf8 for %s\n", quote (url)));
-          xfree (url);
-          url = xstrdup (u->url);
-          iri_fallbacked = 1;
-          goto redirected;
+          if (strcmp(u->url, orig_parsed->url))
+            {
+              DEBUGP (("[IRI fallbacking to non-utf8 for %s\n", quote (url)));
+              xfree (url);
+              url = xstrdup (u->url);
+              iri_fallbacked = 1;
+              goto redirected;
+            }
+          else
+              DEBUGP (("[Needn't fallback to non-utf8 for %s\n", quote (url)));
         }
       else
           DEBUGP (("[Couldn't fallback to non-utf8 for %s\n", quote (url)));
@@ -1060,10 +1214,12 @@ retrieve_from_file (const char *file, bool html, int *count)
       if (dt & TEXTHTML)
         html = true;
 
+#ifdef ENABLE_IRI
       /* If we have a found a content encoding, use it.
        * ( == is okay, because we're checking for identical object) */
       if (iri->content_encoding != opt.locale)
           set_uri_encoding (iri, iri->content_encoding, false);
+#endif
 
       /* Reset UTF-8 encode status */
       iri->utf8_encode = opt.enable_iri;
@@ -1099,12 +1255,20 @@ retrieve_from_file (const char *file, bool html, int *count)
 
       proxy = getproxy (cur_url->url);
       if ((opt.recursive || opt.page_requisites)
-          && ((cur_url->url->scheme != SCHEME_FTP && cur_url->url->scheme != SCHEME_FTPS) || proxy))
+          && ((cur_url->url->scheme != SCHEME_FTP
+#ifdef HAVE_SSL
+          && cur_url->url->scheme != SCHEME_FTPS
+#endif
+          ) || proxy))
         {
           int old_follow_ftp = opt.follow_ftp;
 
           /* Turn opt.follow_ftp on in case of recursive FTP retrieval */
-          if (cur_url->url->scheme == SCHEME_FTP || cur_url->url->scheme == SCHEME_FTPS)
+          if (cur_url->url->scheme == SCHEME_FTP
+#ifdef HAVE_SSL
+              || cur_url->url->scheme == SCHEME_FTPS
+#endif
+              )
             opt.follow_ftp = 1;
 
           status = retrieve_tree (parsed_url ? parsed_url : cur_url->url,
@@ -1122,13 +1286,13 @@ retrieve_from_file (const char *file, bool html, int *count)
       if (parsed_url)
           url_free (parsed_url);
 
-      if (filename && opt.delete_after && file_exists_p (filename))
+      if (filename && opt.delete_after && file_exists_p (filename, NULL))
         {
           DEBUGP (("\
 Removing file due to --delete-after in retrieve_from_file():\n"));
           logprintf (LOG_VERBOSE, _("Removing %s.\n"), filename);
           if (unlink (filename))
-            logprintf (LOG_NOTQUIET, "unlink: %s\n", strerror (errno));
+            logprintf (LOG_NOTQUIET, "Failed to unlink %s: (%d) %s\n", filename, errno, strerror (errno));
           dt &= ~RETROKF;
         }
 
@@ -1229,9 +1393,9 @@ rotate_backups(const char *fname)
 #endif
 
   int maxlen = strlen (fname) + sizeof (SEP) + numdigit (opt.backups) + AVSL;
-  char *from = (char *)alloca (maxlen);
-  char *to = (char *)alloca (maxlen);
-  struct_stat sb;
+  char *from = alloca (maxlen);
+  char *to = alloca (maxlen);
+  struct stat sb;
   int i;
 
   if (stat (fname, &sb) == 0)
@@ -1248,17 +1412,21 @@ rotate_backups(const char *fname)
        */
       if (i == opt.backups)
         {
-          sprintf (to, "%s%s%d%s", fname, SEP, i, AVS);
+          snprintf (to, sizeof(to), "%s%s%d%s", fname, SEP, i, AVS);
           delete (to);
         }
 #endif
-      sprintf (to, "%s%s%d", fname, SEP, i);
-      sprintf (from, "%s%s%d", fname, SEP, i - 1);
-      rename (from, to);
+      snprintf (to, maxlen, "%s%s%d", fname, SEP, i);
+      snprintf (from, maxlen, "%s%s%d", fname, SEP, i - 1);
+      if (rename (from, to))
+        logprintf (LOG_NOTQUIET, "Failed to rename %s to %s: (%d) %s\n",
+                   from, to, errno, strerror (errno));
     }
 
-  sprintf (to, "%s%s%d", fname, SEP, 1);
-  rename(fname, to);
+  snprintf (to, maxlen, "%s%s%d", fname, SEP, 1);
+  if (rename(fname, to))
+    logprintf (LOG_NOTQUIET, "Failed to rename %s to %s: (%d) %s\n",
+               fname, to, errno, strerror (errno));
 }
 
 static bool no_proxy_match (const char *, const char **);

src/retr.h

@@ -1,7 +1,5 @@
 /* Declarations for retr.c.
-   Copyright (C) 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004,
-   2005, 2006, 2007, 2008, 2009, 2010, 2011, 2015 Free Software
-   Foundation, Inc.
+   Copyright (C) 1996-2011, 2015, 2018 Free Software Foundation, Inc.
 
 This file is part of GNU Wget.
 
@@ -49,7 +47,9 @@ enum {
   rb_skip_startpos = 2,
 
   /* Used by HTTP/HTTPS*/
-  rb_chunked_transfer_encoding = 4
+  rb_chunked_transfer_encoding = 4,
+
+  rb_compressed_gzip = 8
 };
 
 int fd_read_body (const char *, int, FILE *, wgint, wgint, wgint *, wgint *, double *, int, FILE *);

src/ssl.h

@@ -1,6 +1,5 @@
 /* SSL support.
-   Copyright (C) 2000, 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008,
-   2009, 2010, 2011, 2012, 2015 Free Software Foundation, Inc.
+   Copyright (C) 2000-2012, 2015, 2018 Free Software Foundation, Inc.
    Originally contributed by Christian Fraenkel.
 
 This file is part of GNU Wget.

src/sysdep.h

@@ -1,7 +1,5 @@
 /* Dirty system-dependent hacks.
-   Copyright (C) 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004,
-   2005, 2006, 2007, 2008, 2009, 2010, 2011, 2015 Free Software
-   Foundation, Inc.
+   Copyright (C) 1996-2011, 2015, 2018 Free Software Foundation, Inc.
 
 This file is part of GNU Wget.
 
@@ -35,56 +33,11 @@ as that of the covered work.  */
 #ifndef SYSDEP_H
 #define SYSDEP_H
 
-/* Testing for __sun is not enough because it's also defined on SunOS.  */
-#ifdef __sun
-# ifdef __SVR4
-#  define solaris
-# endif
-#endif
-
-#if defined(__INTERIX) && !defined(_ALL_SOURCE)
-# define _ALL_SOURCE
-#endif
-
-/* The "namespace tweaks" below attempt to set a friendly "compilation
-   environment" under popular operating systems.  Default compilation
-   environment often means that some functions that are "extensions"
-   are not declared -- `strptime' is one example.
-
-   But non-default environments can expose bugs in the system header
-   files, crippling compilation in _very_ non-obvious ways.  Because
-   of that, we define them only on well-tested architectures where we
-   know they will work.  */
-
-#undef NAMESPACE_TWEAKS
-
-#ifdef solaris
-# define NAMESPACE_TWEAKS
-#endif
-
-#if defined(__linux__) || defined(__GLIBC__)
-# define NAMESPACE_TWEAKS
-#endif
-
-#ifdef NAMESPACE_TWEAKS
-
-/* Request the "Unix 98 compilation environment". */
-#define _XOPEN_SOURCE 500
-
-#endif /* NAMESPACE_TWEAKS */
-
-
-/* Alloca declaration, based on recommendation in the Autoconf manual.
-   These have to be after the above namespace tweaks, but before any
-   non-preprocessor code.  */
+/* Provided by gnulib on systems that don't have it: */
 
 #include <alloca.h>
-
-/* Must include these, so we can test for the missing stat macros and
-   define them as necessary.  */
 #include <sys/types.h>
 #include <sys/stat.h>
-
 #include <stdint.h>
 #include <inttypes.h>
 
@@ -98,108 +51,9 @@ as that of the covered work.  */
 # include "mswindows.h"
 #endif
 
-/* Provided by gnulib on systems that don't have it: */
-# include <stdbool.h>
-
-#ifndef struct_stat
-# define struct_stat struct stat
-#endif
-#ifndef struct_fstat
-# define struct_fstat struct stat
-#endif
-
-#include <intprops.h>
-
-/* For CHAR_BIT, LONG_MAX, etc. */
+#include <stdbool.h>
 #include <limits.h>
-
-#ifndef CHAR_BIT
-# define CHAR_BIT 8
-#endif
-
-/* These are defined in cmpt.c if missing, so we must declare
-   them.  */
-#ifndef HAVE_STRCASECMP
-int strcasecmp ();
-#endif
-#ifndef HAVE_STRNCASECMP
-int strncasecmp ();
-#endif
-#ifndef HAVE_STRPTIME
-char *strptime ();
-#endif
-#ifndef HAVE_TIMEGM
-# include <time.h>
-time_t timegm (struct tm *);
-#endif
-#ifndef HAVE_MEMRCHR
-void *memrchr (const void *, int, size_t);
-#endif
-
-/* These are defined in snprintf.c.  It would be nice to have an
-   snprintf.h, though.  */
-#ifndef HAVE_SNPRINTF
-int snprintf (char *str, size_t count, const char *fmt, ...);
-#endif
-#ifndef HAVE_VSNPRINTF
-#include <stdarg.h>
-int vsnprintf (char *str, size_t count, const char *fmt, va_list arg);
-#endif
-
-/* Some systems (Linux libc5, "NCR MP-RAS 3.0", and others) don't
-   provide MAP_FAILED, a symbolic constant for the value returned by
-   mmap() when it doesn't work.  Usually, this constant should be -1.
-   This only makes sense for files that use mmap() and include
-   sys/mman.h *before* sysdep.h, but doesn't hurt others.  */
-
-#ifndef MAP_FAILED
-# define MAP_FAILED ((void *) -1)
-#endif
-
-/* Enable system fnmatch only on systems where fnmatch.h is usable.
-   If the fnmatch on your system is buggy, undef this symbol and a
-   replacement implementation will be used instead.  */
-#ifdef HAVE_WORKING_FNMATCH_H
-# define SYSTEM_FNMATCH
-#endif
-
 #include <fnmatch.h>
-
-/* Provide sig_atomic_t if the system doesn't.  */
-#ifndef HAVE_SIG_ATOMIC_T
-typedef int sig_atomic_t;
-#endif
-
-/* Provide uint32_t on the platforms that don't define it.  Although
-   most code should be agnostic about integer sizes, some code really
-   does need a 32-bit integral type.  Such code should use uint32_t.
-   (The exception is gnu-md5.[ch], which uses its own detection for
-   portability across platforms.)  */
-
-#ifndef HAVE_UINT32_T
-# if SIZEOF_INT == 4
-typedef unsigned int uint32_t;
-# else
-#  if SIZEOF_LONG == 4
-typedef unsigned long uint32_t;
-#  else
-#   if SIZEOF_SHORT == 4
-typedef unsigned short uint32_t;
-#   else
- #error "Cannot determine a 32-bit unsigned integer type"
-#   endif
-#  endif
-# endif
-#endif
-
-/* If uintptr_t isn't defined, simply typedef it to unsigned long. */
-#ifndef HAVE_UINTPTR_T
-typedef unsigned long uintptr_t;
-#endif
-
-/* If intptr_t isn't defined, simply typedef it to long. */
-#ifndef HAVE_INTPTR_T
-typedef long intptr_t;
-#endif
+#include "intprops.h"
 
 #endif /* SYSDEP_H */

src/test.c

@@ -1,6 +1,5 @@
 /* Unit testing.
-   Copyright (C) 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2015 Free
-   Software Foundation, Inc.
+   Copyright (C) 2005-2011, 2015, 2018 Free Software Foundation, Inc.
 
 This file is part of GNU Wget.
 

src/test.h

@@ -1,6 +1,5 @@
 /* Unit testing declarations.
-   Copyright (C) 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2015 Free
-   Software Foundation, Inc.
+   Copyright (C) 2005-2011, 2015, 2018 Free Software Foundation, Inc.
 
 This file is part of GNU Wget.
 

src/url.c

@@ -1,7 +1,5 @@
 /* URL handling.
-   Copyright (C) 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004,
-   2005, 2006, 2007, 2008, 2009, 2010, 2011, 2015 Free Software
-   Foundation, Inc.
+   Copyright (C) 1996-2011, 2015, 2018 Free Software Foundation, Inc.
 
 This file is part of GNU Wget.
 
@@ -43,6 +41,11 @@ as that of the covered work.  */
 #include "host.h"  /* for is_valid_ipv6_address */
 #include "c-strcase.h"
 
+#ifdef HAVE_ICONV
+# include <iconv.h>
+#endif
+#include <langinfo.h>
+
 #ifdef __VMS
 #include "vms.h"
 #endif /* def __VMS */
@@ -454,7 +457,7 @@ url_scheme (const char *url)
   int i;
 
   for (i = 0; supported_schemes[i].leading_string; i++)
-    if (0 == strncasecmp (url, supported_schemes[i].leading_string,
+    if (0 == c_strncasecmp (url, supported_schemes[i].leading_string,
                           strlen (supported_schemes[i].leading_string)))
       {
         if (!(supported_schemes[i].flags & scm_disabled))
@@ -507,6 +510,12 @@ scheme_disable (enum url_scheme scheme)
   supported_schemes[scheme].flags |= scm_disabled;
 }
 
+const char *
+scheme_leading_string (enum url_scheme scheme)
+{
+  return supported_schemes[scheme].leading_string;
+}
+
 /* Skip the username and password, if present in the URL.  The
    function should *not* be called with the complete URL, but with the
    portion after the scheme.
@@ -914,6 +923,17 @@ url_parse (const char *url, int *error, struct iri *iri, bool percent_encode)
       url_unescape (u->host);
       host_modified = true;
 
+      /* check for invalid control characters in host name */
+      for (p = u->host; *p; p++)
+        {
+          if (c_iscntrl(*p))
+            {
+              url_free(u);
+              error_code = PE_INVALID_HOST_NAME;
+              goto error;
+            }
+        }
+
       /* Apply IDNA regardless of iri->utf8_encode status */
       if (opt.enable_iri && iri)
         {
@@ -922,7 +942,6 @@ url_parse (const char *url, int *error, struct iri *iri, bool percent_encode)
             {
               xfree (u->host);
               u->host = new;
-              u->idn_allocated = true;
               host_modified = true;
             }
         }
@@ -1201,12 +1220,7 @@ url_free (struct url *url)
 {
   if (url)
     {
-      if (url->idn_allocated) {
-        idn_free (url->host);      /* A dummy if !defined(ENABLE_IRI) */
-        url->host = NULL;
-      }
-      else
-        xfree (url->host);
+      xfree (url->host);
 
       xfree (url->path);
       xfree (url->url);
@@ -1231,7 +1245,7 @@ mkalldirs (const char *path)
 {
   const char *p;
   char *t;
-  struct_stat st;
+  struct stat st;
   int res;
 
   p = path + strlen (path);
@@ -1266,12 +1280,14 @@ mkalldirs (const char *path)
              name exists, we just remove it and create the directory
              anyway.  */
           DEBUGP (("Removing %s because of directory danger!\n", t));
-          unlink (t);
+          if (unlink (t))
+            logprintf (LOG_NOTQUIET, "Failed to unlink %s (%d): %s\n",
+                       t, errno, strerror(errno));
         }
     }
   res = make_directory (t);
   if (res != 0)
-    logprintf (LOG_NOTQUIET, "%s: %s", t, strerror (errno));
+    logprintf (LOG_NOTQUIET, "%s: %s\n", t, strerror (errno));
   xfree (t);
   return res;
 }
@@ -1399,8 +1415,8 @@ UVWC, VC, VC, VC,  VC, VC, VC, VC,   /* NUL SOH STX ETX  EOT ENQ ACK BEL */
    0,  0,  0,  0,   0,  0,  0,  0,   /* p   q   r   s    t   u   v   w   */
    0,  0,  0,  0,   W,  0,  0,  C,   /* x   y   z   {    |   }   ~   DEL */
 
-  C, C, C, C,  C, C, C, C,  C, C, C, C,  C, C, C, C, /* 128-143 */
-  C, C, C, C,  C, C, C, C,  C, C, C, C,  C, C, C, C, /* 144-159 */
+  0, 0, 0, 0,  0, 0, 0, 0,  0, 0, 0, 0,  0, 0, 0, 0, /* 128-143 */
+  0, 0, 0, 0,  0, 0, 0, 0,  0, 0, 0, 0,  0, 0, 0, 0, /* 144-159 */
   0, 0, 0, 0,  0, 0, 0, 0,  0, 0, 0, 0,  0, 0, 0, 0,
   0, 0, 0, 0,  0, 0, 0, 0,  0, 0, 0, 0,  0, 0, 0, 0,
 
@@ -1531,6 +1547,93 @@ append_uri_pathel (const char *b, const char *e, bool escaped,
   append_null (dest);
 }
 
+#ifdef HAVE_ICONV
+static char *
+convert_fname (char *fname)
+{
+  char *converted_fname = fname;
+  const char *from_encoding = opt.encoding_remote;
+  const char *to_encoding = opt.locale;
+  iconv_t cd;
+  size_t len, done, inlen, outlen;
+  char *s;
+  const char *orig_fname = fname;
+
+  /* Defaults for remote and local encodings.  */
+  if (!from_encoding)
+    from_encoding = "UTF-8";
+  if (!to_encoding)
+    to_encoding = nl_langinfo (CODESET);
+
+  cd = iconv_open (to_encoding, from_encoding);
+  if (cd == (iconv_t) (-1))
+    logprintf (LOG_VERBOSE, _ ("Conversion from %s to %s isn't supported\n"),
+               quote (from_encoding), quote (to_encoding));
+  else
+    {
+      inlen = strlen (fname);
+      len = outlen = inlen * 2;
+      converted_fname = s = xmalloc (outlen + 1);
+      done = 0;
+
+      for (;;)
+        {
+          errno = 0;
+          if (iconv (cd, (ICONV_CONST char **) &fname, &inlen, &s, &outlen) == 0
+              && iconv (cd, NULL, NULL, &s, &outlen) == 0)
+            {
+              *(converted_fname + len - outlen - done) = '\0';
+              iconv_close (cd);
+              DEBUGP (("Converted file name '%s' (%s) -> '%s' (%s)\n",
+                       orig_fname, from_encoding, converted_fname, to_encoding));
+              xfree (orig_fname);
+              return converted_fname;
+            }
+
+          /* Incomplete or invalid multibyte sequence */
+          if (errno == EINVAL || errno == EILSEQ || errno == 0)
+            {
+              if (errno)
+                logprintf (LOG_VERBOSE,
+                           _ ("Incomplete or invalid multibyte sequence encountered\n"));
+              else
+                logprintf (LOG_VERBOSE,
+                           _ ("Unconvertable multibyte sequence encountered\n"));
+              xfree (converted_fname);
+              converted_fname = (char *) orig_fname;
+              break;
+            }
+          else if (errno == E2BIG) /* Output buffer full */
+            {
+              done = len;
+              len = outlen = done + inlen * 2;
+              converted_fname = xrealloc (converted_fname, outlen + 1);
+              s = converted_fname + done;
+            }
+          else /* Weird, we got an unspecified error */
+            {
+              logprintf (LOG_VERBOSE, _ ("Unhandled errno %d\n"), errno);
+              xfree (converted_fname);
+              converted_fname = (char *) orig_fname;
+              break;
+            }
+        }
+      DEBUGP (("Failed to convert file name '%s' (%s) -> '?' (%s)\n",
+               orig_fname, from_encoding, to_encoding));
+    }
+
+    iconv_close(cd);
+
+  return converted_fname;
+}
+#else
+static char *
+convert_fname (char *fname)
+{
+  return fname;
+}
+#endif
+
 /* Append to DEST the directory structure that corresponds the
    directory part of URL's path.  For example, if the URL is
    http://server/dir1/dir2/file, this appends "/dir1/dir2".
@@ -1603,35 +1706,43 @@ url_file_name (const struct url *u, char *replaced_filename)
   /* If "dirstruct" is turned on (typically the case with -r), add
      the host and port (unless those have been turned off) and
      directory structure.  */
+  /* All safe remote chars are unescaped and stored in temp_fnres,
+     then converted to local and appended to fnres.
+     Internationalized URL/IDN will produce punycode to lookup IP from DNS:
+     https://en.wikipedia.org/wiki/URL
+     https://en.wikipedia.org/wiki/Internationalized_domain_name
+     Non-ASCII code chars in the path:
+     https://en.wikipedia.org/wiki/List_of_Unicode_characters
+     https://en.wikipedia.org/wiki/List_of_writing_systems */
   if (opt.dirstruct)
     {
       if (opt.protocol_directories)
         {
-          if (fnres.tail)
-            append_char ('/', &fnres);
-          append_string (supported_schemes[u->scheme].name, &fnres);
+          if (temp_fnres.tail)
+            append_char ('/', &temp_fnres);
+          append_string (supported_schemes[u->scheme].name, &temp_fnres);
         }
       if (opt.add_hostdir)
         {
-          if (fnres.tail)
-            append_char ('/', &fnres);
+          if (temp_fnres.tail)
+            append_char ('/', &temp_fnres);
           if (0 != strcmp (u->host, ".."))
-            append_string (u->host, &fnres);
+            append_string (u->host, &temp_fnres);
           else
             /* Host name can come from the network; malicious DNS may
                allow ".." to be resolved, causing us to write to
                "../<file>".  Defang such host names.  */
-            append_string ("%2E%2E", &fnres);
+            append_string ("%2E%2E", &temp_fnres);
           if (u->port != scheme_default_port (u->scheme))
             {
               char portstr[24];
               number_to_string (portstr, u->port);
-              append_char (FN_PORT_SEP, &fnres);
-              append_string (portstr, &fnres);
+              append_char (FN_PORT_SEP, &temp_fnres);
+              append_string (portstr, &temp_fnres);
             }
         }
 
-      append_dir_structure (u, &fnres);
+      append_dir_structure (u, &temp_fnres);
     }
 
   if (!replaced_filename)
@@ -1652,12 +1763,23 @@ url_file_name (const struct url *u, char *replaced_filename)
       fname_len_check = strdupdelim (u_file, u_file + strlen (u_file));
     }
 
+  if (temp_fnres.tail)
+    append_char ('/', &temp_fnres);
+
   append_uri_pathel (fname_len_check,
-    fname_len_check + strlen (fname_len_check), false, &temp_fnres);
+    fname_len_check + strlen (fname_len_check), true, &temp_fnres);
 
   /* Zero-terminate the temporary file name. */
   append_char ('\0', &temp_fnres);
 
+  /* convert all remote chars before length check and appending to local path */
+  fname = convert_fname (temp_fnres.base);
+  temp_fnres.base = NULL;
+  temp_fnres.size = 0;
+  temp_fnres.tail = 0;
+  append_string (fname, &temp_fnres);
+  xfree (fname);
+
   /* Check that the length of the file name is acceptable. */
 #ifdef WINDOWS
   if (MAX_PATH > (fnres.tail + CHOMP_BUFFER + 2))
@@ -1717,7 +1839,7 @@ url_file_name (const struct url *u, char *replaced_filename)
      directory (see `mkalldirs' for explanation).  */
 
   if (ALLOW_CLOBBER
-      && !(file_exists_p (fname) && !file_non_directory_p (fname)))
+      && !(file_exists_p (fname, NULL) && !file_non_directory_p (fname)))
     {
       unique = fname;
     }
@@ -1787,7 +1909,11 @@ path_simplify (enum url_scheme scheme, char *path)
               for (--t; t > beg && t[-1] != '/'; t--)
                 ;
             }
-          else if (scheme == SCHEME_FTP || scheme == SCHEME_FTPS)
+          else if (scheme == SCHEME_FTP
+#ifdef HAVE_SSL
+              || scheme == SCHEME_FTPS
+#endif
+              )
             {
               /* If we're at the beginning, copy the "../" literally
                  and move the beginning so a later ".." doesn't remove

src/url.h

@@ -1,7 +1,5 @@
 /* Declarations for url.c.
-   Copyright (C) 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004,
-   2005, 2006, 2007, 2008, 2009, 2010, 2011, 2015 Free Software
-   Foundation, Inc.
+   Copyright (C) 1996-2011, 2015, 2018 Free Software Foundation, Inc.
 
 This file is part of GNU Wget.
 
@@ -99,10 +97,6 @@ struct url
   /* Username and password (unquoted). */
   char *user;
   char *passwd;
-
-  /* 'host' is allocated by idna_to_ascii_8z() via idn_encode().
-   * Call 'idn_free()' to free this memory. */
-  bool idn_allocated;
 };
 
 /* Function declarations */
@@ -124,6 +118,7 @@ bool url_has_scheme (const char *);
 bool url_valid_scheme (const char *);
 int scheme_default_port (enum url_scheme);
 void scheme_disable (enum url_scheme);
+const char *scheme_leading_string (enum url_scheme);
 
 char *url_string (const struct url *, enum url_auth_mode);
 char *url_file_name (const struct url *, char *);

src/utils.c

@@ -1,7 +1,5 @@
 /* Various utility functions.
-   Copyright (C) 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004,
-   2005, 2006, 2007, 2008, 2009, 2010, 2011, 2015 Free Software
-   Foundation, Inc.
+   Copyright (C) 1996-2011, 2015, 2018 Free Software Foundation, Inc.
 
 This file is part of GNU Wget.
 
@@ -31,14 +29,12 @@ as that of the covered work.  */
 
 #include "wget.h"
 
+#include "sha256.h"
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
 #include <time.h>
 #include <unistd.h>
-#ifdef HAVE_MMAP
-# include <sys/mman.h>
-#endif
 #ifdef HAVE_PROCESS_H
 # include <process.h>  /* getpid() */
 #endif
@@ -47,13 +43,11 @@ as that of the covered work.  */
 #include <assert.h>
 #include <stdarg.h>
 #include <locale.h>
+#include <errno.h>
 
 #if HAVE_UTIME
 # include <sys/types.h>
-# ifdef HAVE_UTIME_H
-#  include <utime.h>
-# endif
-
+# include <utime.h>
 # ifdef HAVE_SYS_UTIME_H
 #  include <sys/utime.h>
 # endif
@@ -89,6 +83,18 @@ as that of the covered work.  */
 # define USE_SIGNAL_TIMEOUT
 #endif
 
+/* Some systems (Linux libc5, "NCR MP-RAS 3.0", and others) don't
+   provide MAP_FAILED, a symbolic constant for the value returned by
+   mmap() when it doesn't work.  Usually, this constant should be -1.
+   This only makes sense for files that use mmap() and include
+   sys/mman.h *before* sysdep.h, but doesn't hurt others.  */
+#ifdef HAVE_MMAP
+# include <sys/mman.h>
+# ifndef MAP_FAILED
+#  define MAP_FAILED ((void *) -1)
+# endif
+#endif
+
 #include "utils.h"
 #include "hash.h"
 
@@ -103,7 +109,7 @@ as that of the covered work.  */
 #include "exits.h"
 #include "c-strcase.h"
 
-static void _Noreturn
+_Noreturn static void
 memfatal (const char *context, long attempted_size)
 {
   /* Make sure we don't try to store part of the log line, and thus
@@ -225,14 +231,20 @@ xstrdup_lower (const char *s)
 
 /* Copy the string formed by two pointers (one on the beginning, other
    on the char after the last char) to a new, malloc-ed location.
-   0-terminate it.  */
+   0-terminate it.
+   If both pointers are NULL, the function returns an empty string.  */
 char *
 strdupdelim (const char *beg, const char *end)
 {
-  char *res = xmalloc (end - beg + 1);
-  memcpy (res, beg, end - beg);
-  res[end - beg] = '\0';
-  return res;
+  if (beg && beg <= end)
+    {
+      char *res = xmalloc (end - beg + 1);
+      memcpy (res, beg, end - beg);
+      res[end - beg] = '\0';
+      return res;
+    }
+
+  return xstrdup("");
 }
 
 /* Parse a string containing comma-separated elements, and return a
@@ -282,13 +294,6 @@ sepstring (const char *s)
    vsnprintf until the correct size is found.  Since Wget also ships a
    fallback implementation of vsnprintf, this should be portable.  */
 
-/* Constant is using for limits memory allocation for text buffer.
-   Applicable in situation when: vasprintf is not available in the system
-   and vsnprintf return -1 when long line is truncated (in old versions of
-   glibc and in other system where C99 doesn`t support) */
-
-#define FMT_MAX_LENGTH 1048576
-
 char *
 aprintf (const char *fmt, ...)
 {
@@ -308,6 +313,13 @@ aprintf (const char *fmt, ...)
   return str;
 #else  /* not HAVE_VASPRINTF */
 
+/* Constant is using for limits memory allocation for text buffer.
+   Applicable in situation when: vasprintf is not available in the system
+   and vsnprintf return -1 when long line is truncated (in old versions of
+   glibc and in other system where C99 doesn`t support) */
+
+#define FMT_MAX_LENGTH 1048576
+
   /* vasprintf is unavailable.  snprintf into a small buffer and
      resize it as necessary. */
   int size = 32;
@@ -557,7 +569,7 @@ int
 remove_link (const char *file)
 {
   int err = 0;
-  struct_stat st;
+  struct stat st;
 
   if (lstat (file, &st) == 0 && S_ISLNK (st.st_mode))
     {
@@ -570,21 +582,42 @@ remove_link (const char *file)
   return err;
 }
 
-/* Does FILENAME exist?  This is quite a lousy implementation, since
-   it supplies no error codes -- only a yes-or-no answer.  Thus it
-   will return that a file does not exist if, e.g., the directory is
-   unreadable.  I don't mind it too much currently, though.  The
-   proper way should, of course, be to have a third, error state,
-   other than true/false, but that would introduce uncalled-for
-   additional complexity to the callers.  */
+/* Does FILENAME exist? */
 bool
-file_exists_p (const char *filename)
+file_exists_p (const char *filename, file_stats_t *fstats)
 {
-#ifdef HAVE_ACCESS
-  return access (filename, F_OK) >= 0;
+  struct stat buf;
+
+#if defined(WINDOWS) || defined(__VMS)
+    int ret = stat (filename, &buf);
+    if (ret >= 0)
+    {
+      if (fstats != NULL)
+        fstats->access_err = errno;
+    }
+    return ret >= 0;
 #else
-  struct_stat buf;
-  return stat (filename, &buf) >= 0;
+  errno = 0;
+  if (stat (filename, &buf) == 0 && S_ISREG(buf.st_mode) &&
+              (((S_IRUSR & buf.st_mode) && (getuid() == buf.st_uid))  ||
+               ((S_IRGRP & buf.st_mode) && group_member(buf.st_gid))  ||
+                (S_IROTH & buf.st_mode))) {
+    if (fstats != NULL)
+    {
+      fstats->access_err = 0;
+      fstats->st_ino = buf.st_ino;
+      fstats->st_dev = buf.st_dev;
+    }
+    return true;
+  }
+  else
+  {
+    if (fstats != NULL)
+      fstats->access_err = (errno == 0 ? EACCES : errno);
+    errno = 0;
+    return false;
+  }
+  /* NOTREACHED */
 #endif
 }
 
@@ -593,7 +626,7 @@ file_exists_p (const char *filename)
 bool
 file_non_directory_p (const char *path)
 {
-  struct_stat buf;
+  struct stat buf;
   /* Use lstat() rather than stat() so that symbolic links pointing to
      directories can be identified correctly.  */
   if (lstat (path, &buf) != 0)
@@ -620,7 +653,7 @@ file_size (const char *filename)
   fclose (fp);
   return size;
 #else
-  struct_stat st;
+  struct stat st;
   if (stat (filename, &st) < 0)
     return -1;
   return st.st_size;
@@ -652,7 +685,7 @@ unique_name_1 (const char *prefix)
 
   do
     number_to_string (template_tail, count++);
-  while (file_exists_p (template));
+  while (file_exists_p (template, NULL));
 
   return xstrdup (template);
 }
@@ -680,7 +713,7 @@ unique_name (const char *file, bool allow_passthrough)
 {
   /* If the FILE itself doesn't exist, return it without
      modification. */
-  if (!file_exists_p (file))
+  if (!file_exists_p (file, NULL))
     return allow_passthrough ? (char *)file : xstrdup (file);
 
   /* Otherwise, find a numeric suffix that results in unused file name
@@ -809,7 +842,7 @@ fopen_excl (const char *fname, int binary)
   /* Manually check whether the file exists.  This is prone to race
      conditions, but systems without O_EXCL haven't deserved
      better.  */
-  if (file_exists_p (fname))
+  if (file_exists_p (fname, NULL))
     {
       errno = EEXIST;
       return NULL;
@@ -818,6 +851,113 @@ fopen_excl (const char *fname, int binary)
 #endif /* not O_EXCL */
 }
 
+/* fopen_stat() assumes that file_exists_p() was called earlier.
+   file_stats_t passed to this function was returned from file_exists_p()
+   This is to prevent TOCTTOU race condition.
+   Details : FIO45-C from https://www.securecoding.cert.org/
+   Note that for creating a new file, this check is not useful
+
+   Input:
+     fname  => Name of file to open
+     mode   => File open mode
+     fstats => Saved file_stats_t about file that was checked for existence
+
+   Returns:
+     NULL if there was an error
+     FILE * of opened file stream
+*/
+FILE *
+fopen_stat(const char *fname, const char *mode, file_stats_t *fstats)
+{
+  int fd;
+  FILE *fp;
+  struct stat fdstats;
+
+  fp = fopen (fname, mode);
+  if (fp == NULL)
+  {
+    logprintf (LOG_NOTQUIET, _("Failed to Fopen file %s\n"), fname);
+    return NULL;
+  }
+  fd = fileno (fp);
+  if (fd < 0)
+  {
+    logprintf (LOG_NOTQUIET, _("Failed to get FD for file %s\n"), fname);
+    fclose (fp);
+    return NULL;
+  }
+  memset(&fdstats, 0, sizeof(fdstats));
+  if (fstat (fd, &fdstats) == -1)
+  {
+    logprintf (LOG_NOTQUIET, _("Failed to stat file %s, (check permissions)\n"), fname);
+    fclose (fp);
+    return NULL;
+  }
+#if !(defined(WINDOWS) || defined(__VMS))
+  if (fstats != NULL &&
+      (fdstats.st_dev != fstats->st_dev ||
+       fdstats.st_ino != fstats->st_ino))
+  {
+    /* File changed since file_exists_p() : NOT SAFE */
+    logprintf (LOG_NOTQUIET, _("File %s changed since the last check. Security check failed."), fname);
+    fclose (fp);
+    return NULL;
+  }
+#endif
+
+  return fp;
+}
+
+/* open_stat assumes that file_exists_p() was called earlier to save file_stats
+   file_stats_t passed to this function was returned from file_exists_p()
+   This is to prevent TOCTTOU race condition.
+   Details : FIO45-C from https://www.securecoding.cert.org/
+   Note that for creating a new file, this check is not useful
+
+
+   Input:
+     fname  => Name of file to open
+     flags  => File open flags
+     mode   => File open mode
+     fstats => Saved file_stats_t about file that was checked for existence
+
+   Returns:
+     -1 if there was an error
+     file descriptor of opened file stream
+*/
+int
+open_stat(const char *fname, int flags, mode_t mode, file_stats_t *fstats)
+{
+  int fd;
+  struct stat fdstats;
+
+  fd = open (fname, flags, mode);
+  if (fd < 0)
+  {
+    logprintf (LOG_NOTQUIET, _("Failed to open file %s, reason :%s\n"), fname, strerror(errno));
+    return -1;
+  }
+  memset(&fdstats, 0, sizeof(fdstats));
+  if (fstat (fd, &fdstats) == -1)
+  {
+    logprintf (LOG_NOTQUIET, _("Failed to stat file %s, error: %s\n"), fname, strerror(errno));
+    return -1;
+  }
+#if !(defined(WINDOWS) || defined(__VMS))
+  if (fstats != NULL &&
+      (fdstats.st_dev != fstats->st_dev ||
+       fdstats.st_ino != fstats->st_ino))
+  {
+    /* File changed since file_exists_p() : NOT SAFE */
+    logprintf (LOG_NOTQUIET, _("Trying to open file %s but it changed since last check. Security check failed."), fname);
+    close (fd);
+    return -1;
+  }
+#endif
+
+  return fd;
+}
+
 /* Create DIRECTORY.  If some of the pathname components of DIRECTORY
    are missing, create them first.  In case any mkdir() call fails,
    return its error status.  Returns 0 on successful completion.
@@ -846,7 +986,7 @@ make_directory (const char *directory)
       /* Check whether the directory already exists.  Allow creation of
          of intermediate directories to fail, as the initial path components
          are not necessarily directories!  */
-      if (!file_exists_p (dir))
+      if (!file_exists_p (dir, NULL))
         ret = mkdir (dir, 0777);
       else
         ret = 0;
@@ -1174,7 +1314,7 @@ wget_read_file (const char *file)
 
 #ifdef HAVE_MMAP
   {
-    struct_fstat buf;
+    struct stat buf;
     if (fstat (fd, &buf) < 0)
       goto mmap_lose;
     fm->length = buf.st_size;
@@ -1317,7 +1457,7 @@ merge_vecs (char **v1, char **v2)
   for (j = 0; v2[j]; j++)
     ;
   /* Reallocate v1.  */
-  v1 = xrealloc (v1, (i + j + 1) * sizeof (char **));
+  v1 = xrealloc (v1, (i + j + 1) * sizeof (char *));
   memcpy (v1 + i, v2, (j + 1) * sizeof (char *));
   xfree (v2);
   return v1;
@@ -1590,7 +1730,7 @@ numdigit (wgint number)
 {
   int cnt = 1;
   if (number < 0)
-    ++cnt;                      /* accomodate '-' */
+    ++cnt;                      /* accommodate '-' */
   while ((number /= 10) != 0)
     ++cnt;
   return cnt;
@@ -1920,7 +2060,7 @@ random_float (void)
 
 static sigjmp_buf run_with_timeout_env;
 
-static void _Noreturn
+_Noreturn static void
 abort_run_with_timeout (int sig _GL_UNUSED)
 {
   assert (sig == SIGALRM);
@@ -2035,13 +2175,16 @@ run_with_timeout (double timeout, void (*fun) (void *), void *arg)
       return false;
     }
 
-  signal (SIGALRM, abort_run_with_timeout);
   if (SETJMP (run_with_timeout_env) != 0)
     {
       /* Longjumped out of FUN with a timeout. */
       signal (SIGALRM, SIG_DFL);
       return true;
     }
+  else
+    {
+      signal (SIGALRM, abort_run_with_timeout);
+    }
   alarm_set (timeout);
   fun (arg);
 
@@ -2130,7 +2273,7 @@ xsleep (double seconds)
    base64 data.  */
 
 size_t
-base64_encode (const void *data, size_t length, char *dest)
+wget_base64_encode (const void *data, size_t length, char *dest)
 {
   /* Conversion table.  */
   static const char tbl[64] = {
@@ -2187,7 +2330,7 @@ base64_encode (const void *data, size_t length, char *dest)
 
 /* Decode data from BASE64 (a null-terminated string) into memory
    pointed to by DEST.  DEST is assumed to be large enough to
-   accomodate the decoded data, which is guaranteed to be no more than
+   accommodate the decoded data, which is guaranteed to be no more than
    3/4*strlen(base64).
 
    Since DEST is assumed to contain binary data, it is not
@@ -2198,7 +2341,7 @@ base64_encode (const void *data, size_t length, char *dest)
    This function originates from Free Recode.  */
 
 ssize_t
-base64_decode (const char *base64, void *dest)
+wget_base64_decode (const char *base64, void *dest, size_t size)
 {
   /* Table of base64 values for first 128 characters.  Note that this
      assumes ASCII (but so does Wget in other places).  */
@@ -2222,7 +2365,8 @@ base64_decode (const char *base64, void *dest)
 #define IS_BASE64(c) ((IS_ASCII (c) && BASE64_CHAR_TO_VALUE (c) >= 0) || c == '=')
 
   const char *p = base64;
-  char *q = dest;
+  unsigned char *q = dest;
+  ssize_t n = 0;
 
   while (1)
     {
@@ -2244,7 +2388,12 @@ base64_decode (const char *base64, void *dest)
       if (c == '=' || !IS_BASE64 (c))
         return -1;              /* illegal char while decoding base64 */
       value |= BASE64_CHAR_TO_VALUE (c) << 12;
-      *q++ = value >> 16;
+      if (size)
+        {
+          *q++ = value >> 16;
+          size--;
+        }
+      n++;
 
       /* Process third byte of a quadruplet.  */
       NEXT_CHAR (c, p);
@@ -2264,7 +2413,12 @@ base64_decode (const char *base64, void *dest)
         }
 
       value |= BASE64_CHAR_TO_VALUE (c) << 6;
-      *q++ = 0xff & value >> 8;
+      if (size)
+        {
+          *q++ = 0xff & value >> 8;
+          size--;
+        }
+      n++;
 
       /* Process fourth byte of a quadruplet.  */
       NEXT_CHAR (c, p);
@@ -2276,12 +2430,17 @@ base64_decode (const char *base64, void *dest)
         return -1;              /* illegal char while decoding base64 */
 
       value |= BASE64_CHAR_TO_VALUE (c);
-      *q++ = 0xff & value;
+      if (size)
+        {
+          *q++ = 0xff & value;
+          size--;
+        }
+      n++;
     }
 #undef IS_BASE64
 #undef BASE64_CHAR_TO_VALUE
 
-  return q - (char *) dest;
+  return n;
 }
 
 #ifdef HAVE_LIBPCRE
@@ -2403,17 +2562,17 @@ mergesort_internal (void *base, void *temp, size_t size, size_t from, size_t to,
 }
 
 /* Stable sort with interface exactly like standard library's qsort.
-   Uses mergesort internally, allocating temporary storage with
-   alloca.  */
+   Uses mergesort internally. */
 
 void
 stable_sort (void *base, size_t nmemb, size_t size,
              int (*cmpfun) (const void *, const void *))
 {
-  if (size > 1)
+  if (nmemb > 1 && size > 1)
     {
-      void *temp = alloca (nmemb * size * sizeof (void *));
+      void *temp = xmalloc (nmemb * size);
       mergesort_internal (base, temp, size, 0, nmemb - 1, cmpfun);
+      xfree(temp);
     }
 }
 
@@ -2514,13 +2673,224 @@ wg_hex_to_string (char *str_buffer, const char *hex_buffer, size_t hex_len)
   for (i = 0; i < hex_len; i++)
     {
       /* Each byte takes 2 characters.  */
-      sprintf (str_buffer + 2 * i, "%02x", hex_buffer[i] & 0xFF);
+      sprintf (str_buffer + 2 * i, "%02x", (unsigned) (hex_buffer[i] & 0xFF));
     }
 
   /* Null-terminate result.  */
   str_buffer[2 * i] = '\0';
 }
 
+#ifdef HAVE_SSL
+
+/*
+ * Public key pem to der conversion
+ */
+
+static bool
+wg_pubkey_pem_to_der (const char *pem, unsigned char **der, size_t *der_len)
+{
+  char *stripped_pem, *begin_pos, *end_pos;
+  size_t pem_count, stripped_pem_count = 0, pem_len;
+  ssize_t size;
+  unsigned char *base64data;
+
+  *der = NULL;
+  *der_len = 0;
+
+  /* if no pem, exit. */
+  if (!pem)
+    return false;
+
+  begin_pos = strstr (pem, "-----BEGIN PUBLIC KEY-----");
+  if (!begin_pos)
+    return false;
+
+  pem_count = begin_pos - pem;
+  /* Invalid if not at beginning AND not directly following \n */
+  if (0 != pem_count && '\n' != pem[pem_count - 1])
+    return false;
+
+  /* 26 is length of "-----BEGIN PUBLIC KEY-----" */
+  pem_count += 26;
+
+  /* Invalid if not directly following \n */
+  end_pos = strstr (pem + pem_count, "\n-----END PUBLIC KEY-----");
+  if (!end_pos)
+    return false;
+
+  pem_len = end_pos - pem;
+
+  stripped_pem = xmalloc (pem_len - pem_count + 1);
+
+  /*
+   * Here we loop through the pem array one character at a time between the
+   * correct indices, and place each character that is not '\n' or '\r'
+   * into the stripped_pem array, which should represent the raw base64 string
+   */
+  while (pem_count < pem_len) {
+    if ('\n' != pem[pem_count] && '\r' != pem[pem_count])
+      stripped_pem[stripped_pem_count++] = pem[pem_count];
+    ++pem_count;
+  }
+  /* Place the null terminator in the correct place */
+  stripped_pem[stripped_pem_count] = '\0';
+
+  base64data = xmalloc (BASE64_LENGTH(stripped_pem_count));
+
+  size = wget_base64_decode (stripped_pem, base64data, BASE64_LENGTH(stripped_pem_count));
+
+  if (size < 0) {
+    xfree (base64data);           /* malformed base64 from server */
+  } else {
+    *der = base64data;
+    *der_len = (size_t) size;
+  }
+
+  xfree (stripped_pem);
+
+  return *der_len > 0;
+}
+
+/*
+ * Generic pinned public key check.
+ */
+
+bool
+wg_pin_peer_pubkey (const char *pinnedpubkey, const char *pubkey, size_t pubkeylen)
+{
+  struct file_memory *fm;
+  unsigned char *buf = NULL, *pem_ptr = NULL;
+  size_t size, pem_len;
+  bool pem_read;
+  bool result = false;
+
+  size_t pinkeylen;
+  ssize_t decoded_hash_length;
+  char *pinkeycopy, *begin_pos, *end_pos;
+  unsigned char *sha256sumdigest = NULL, *expectedsha256sumdigest = NULL;
+
+  /* if a path wasn't specified, don't pin */
+  if (!pinnedpubkey)
+    return true;
+  if (!pubkey || !pubkeylen)
+    return result;
+
+  /* only do this if pinnedpubkey starts with "sha256//", length 8 */
+  if (strncmp (pinnedpubkey, "sha256//", 8) == 0)
+    {
+      /* compute sha256sum of public key */
+      sha256sumdigest = xmalloc (SHA256_DIGEST_SIZE);
+      sha256_buffer (pubkey, pubkeylen, sha256sumdigest);
+      expectedsha256sumdigest = xmalloc (SHA256_DIGEST_SIZE);
+
+      /* it starts with sha256//, copy so we can modify it */
+      pinkeylen = strlen (pinnedpubkey) + 1;
+      pinkeycopy = xmalloc (pinkeylen);
+      memcpy (pinkeycopy, pinnedpubkey, pinkeylen);
+
+      /* point begin_pos to the copy, and start extracting keys */
+      begin_pos = pinkeycopy;
+      do
+        {
+          end_pos = strstr (begin_pos, ";sha256//");
+          /*
+           * if there is an end_pos, null terminate,
+           * otherwise it'll go to the end of the original string
+           */
+          if (end_pos)
+            end_pos[0] = '\0';
+
+          /* decode base64 pinnedpubkey, 8 is length of "sha256//" */
+          decoded_hash_length = wget_base64_decode (begin_pos + 8, expectedsha256sumdigest, SHA256_DIGEST_SIZE);
+
+          /* if valid base64, compare sha256 digests directly */
+          if (SHA256_DIGEST_SIZE == decoded_hash_length)
+            {
+              if (!memcmp (sha256sumdigest, expectedsha256sumdigest, SHA256_DIGEST_SIZE))
+                {
+                  result = true;
+                  break;
+                }
+            }
+          else
+            logprintf (LOG_VERBOSE, _ ("Skipping key with wrong size (%d/%d): %s\n"),
+                       (int) (strlen (begin_pos + 8) * 3) / 4, SHA256_DIGEST_SIZE,
+                       quote (begin_pos + 8));
+
+          /*
+           * change back the null-terminator we changed earlier,
+           * and look for next begin
+           */
+          if (end_pos)
+            {
+              end_pos[0] = ';';
+              begin_pos = strstr (end_pos, "sha256//");
+            }
+        }
+      while (end_pos && begin_pos);
+
+      xfree (sha256sumdigest);
+      xfree (expectedsha256sumdigest);
+      xfree (pinkeycopy);
+
+      return result;
+    }
+
+  /* fall back to assuming this is a file path */
+  fm = wget_read_file (pinnedpubkey);
+  if (!fm)
+    return result;
+
+  /* Check the file's size */
+  if (fm->length < 0 || fm->length > MAX_PINNED_PUBKEY_SIZE)
+    goto cleanup;
+
+  /*
+   * if the size of our certificate is bigger than the file
+   * size then it can't match
+   */
+  size = (size_t) fm->length;
+  if (pubkeylen > size)
+    goto cleanup;
+
+  /* If the sizes are the same, it can't be base64 encoded, must be der */
+  if (pubkeylen == size)
+    {
+      if (!memcmp (pubkey, fm->content, pubkeylen))
+        result = true;
+      goto cleanup;
+    }
+
+  /*
+   * Otherwise we will assume it's PEM and try to decode it
+   * after placing null terminator
+   */
+  buf = xmalloc (size + 1);
+  memcpy (buf, fm->content, size);
+  buf[size] = '\0';
+
+  pem_read = wg_pubkey_pem_to_der ((const char *) buf, &pem_ptr, &pem_len);
+  /* if it wasn't read successfully, exit */
+  if (!pem_read)
+    goto cleanup;
+
+  /*
+   * if the size of our certificate doesn't match the size of
+   * the decoded file, they can't be the same, otherwise compare
+   */
+  if (pubkeylen == pem_len && !memcmp (pubkey, pem_ptr, pubkeylen))
+    result = true;
+
+cleanup:
+  xfree (buf);
+  xfree (pem_ptr);
+  wget_read_file_free (fm);
+
+  return result;
+}
+
+#endif /* HAVE_SSL */
+
 #ifdef TESTING
 
 const char *

src/utils.h

@@ -1,7 +1,5 @@
 /* Declarations for utils.c.
-   Copyright (C) 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004,
-   2005, 2006, 2007, 2008, 2009, 2010, 2011, 2015 Free Software
-   Foundation, Inc.
+   Copyright (C) 1996-2011, 2015, 2018 Free Software Foundation, Inc.
 
 This file is part of GNU Wget.
 
@@ -37,6 +35,10 @@ as that of the covered work.  */
 /* Constant is using when we don`t know attempted size exactly */
 #define UNKNOWN_ATTEMPTED_SIZE -3
 
+#ifndef MAX_PINNED_PUBKEY_SIZE
+#define MAX_PINNED_PUBKEY_SIZE 1048576 /* 1MB */
+#endif
+
 /* Macros that interface to malloc, but know about type sizes, and
    cast the result to the appropriate type.  The casts are not
    necessary in standard C, but Wget performs them anyway for the sake
@@ -74,15 +76,23 @@ void fork_to_background (void);
 char *aprintf (const char *, ...) GCC_FORMAT_ATTR (1, 2);
 char *concat_strings (const char *, ...);
 
+typedef struct file_stat_s {
+  int access_err;               /* Error in accecssing file : Not present vs permission */
+  ino_t st_ino;                 /* st_ino from stats() on the file before open() */
+  dev_t st_dev;                 /* st_dev from stats() on the file before open() */
+} file_stats_t;
+
 void touch (const char *, time_t);
 int remove_link (const char *);
-bool file_exists_p (const char *);
+bool file_exists_p (const char *, file_stats_t *);
 bool file_non_directory_p (const char *);
 wgint file_size (const char *);
 int make_directory (const char *);
 char *unique_name (const char *, bool);
 FILE *unique_create (const char *, bool, char **);
 FILE *fopen_excl (const char *, int);
+FILE *fopen_stat (const char *, const char *, file_stats_t *);
+int   open_stat  (const char *, int, mode_t, file_stats_t *);
 char *file_merge (const char *, const char *);
 
 int fnmatch_nocase (const char *, const char *, int);
@@ -136,8 +146,8 @@ void xsleep (double);
 /* How many bytes it will take to store LEN bytes in base64.  */
 #define BASE64_LENGTH(len) (4 * (((len) + 2) / 3))
 
-size_t base64_encode (const void *, size_t, char *);
-ssize_t base64_decode (const char *, void *);
+size_t wget_base64_encode (const void *, size_t, char *);
+ssize_t wget_base64_decode (const char *, void *, size_t);
 
 #ifdef HAVE_LIBPCRE
 void *compile_pcre_regex (const char *);
@@ -161,4 +171,9 @@ void wg_hex_to_string (char *str_buffer, const char *hex_buffer, size_t hex_len)
 
 extern unsigned char char_prop[];
 
+#ifdef HAVE_SSL
+/* Check pinned public key. */
+bool wg_pin_peer_pubkey (const char *pinnedpubkey, const char *pubkey, size_t pubkeylen);
+#endif
+
 #endif /* UTILS_H */