* src/host.c (sufmatch): Fix to domain matching

* src/netrc.c: Fix Standalone compilation of netrc file
* NEWS: Update NEWS file for new release
2026-08-21 02:13:27 +08:00 · 2018-01-19 19:32:01 +01:00 · 2018-01-17 14:44:52 +01:00 · 2018-01-14 12:17:31 +01:00 · 2018-01-14 11:33:52 +01:00 · 2018-01-14 11:28:51 +01:00
376 changed files with 46995 additions and 32267 deletions
--- a/.gitignore
+++ b/.gitignore
@@ -2,6 +2,7 @@
 /.sc-start-sc_bindtextdomain
 /.sc-start-sc_prohibit_HAVE_MBRTOWC
 /.version
+/ABOUT-NLS
 /GNUmakefile
 /INSTALL
 /Makefile
@@ -19,10 +20,13 @@
 po/*.gmo*
 po/*.po*
 src/*.o
+tests/*.log
+tests/*.trs
 testenv/*.log
 testenv/*.trs
 # build-aux/
 build-aux/.gitignore
+build-aux/ar-lib
 build-aux/compile
 build-aux/config.guess
 build-aux/config.sub
@@ -54,6 +58,7 @@ po/Makefile.in
 po/Makevars
 po/POTFILES
 po/wget.pot
+po/stamp-po
 # src/
 src/.deps/
 src/Makefile
@@ -81,7 +86,6 @@ testenv/Makefile.in
 # tests/
 tests/Makefile
 tests/Makefile.in
-tests/WgetTest.pm
 tests/unit-tests
 # util/
 util/Makefile
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -0,0 +1,124 @@
+variables:
+  BUILD_IMAGES_PROJECT: gnuwget/build-images
+  DEBIAN_BUILD: buildenv-debian-stretch
+  FEDORA_BUILD: buildenv-f25
+  CENTOS7_BUILD: buildenv-centos7
+  GET_SOURCES_ATTEMPTS: "3"
+
+# In this build we combine
+#  * syntax-check
+#  * abi-check
+#  * build/valgrind-check
+#  * build/asan-check
+#  * build w/Werror and ubsan-check
+Debian GNU/Linux build:
+  image: $CI_REGISTRY/$BUILD_IMAGES_PROJECT:$DEBIAN_BUILD
+  script:
+  - echo "127.0.0.1 wgettestingserver" >>/etc/hosts
+  - cat /etc/hosts
+  - touch .manywarnings
+  - alias make="make -j$(nproc)"
+  - ./bootstrap
+  - ./configure -C
+  - make syntax-check
+  - make check
+#  - make abi-check
+  - make distcheck
+  - make check-valgrind
+  - ./configure -C --with-ssl=openssl
+  - make clean
+  - make check
+#  - ./configure -C --enable-fsanitize-asan
+#  - make clean
+#  - make check
+#  - ./configure -C --enable-fsanitize-ubsan
+#  - make clean
+#  - make check
+#  - ./configure -C --enable-fsanitize-msan
+#  - make clean
+#  - make check
+  tags:
+  - shared
+  artifacts:
+    expire_in: 2 weeks
+    when: on_failure
+    paths:
+      - tests/*.log
+#      - compat_reports/
+
+#Centos7 build:
+#  image: $CI_REGISTRY/$BUILD_IMAGES_PROJECT:$CENTOS7_BUILD
+#  script:
+#  - sed -i 's/AM_GNU_GETTEXT_VERSION(0.19.3)/AM_GNU_GETTEXT_VERSION(0.18.2)/g' configure.ac
+#  - ./bootstrap && ./configure --enable-gcc-warnings --disable-doc && make -j$(nproc) check
+#  tags:
+#  - shared
+#  artifacts:
+#    expire_in: 2 weeks
+#    when: on_failure
+#    paths:
+#      - tests/*.log
+#      - compat_reports/
+
+#clang-analyzer/Fedora:
+#  image: $CI_REGISTRY/$BUILD_IMAGES_PROJECT:$FEDORA_BUILD
+#  script:
+#  - ./bootstrap
+#  - scan-build ./configure --disable-doc --enable-gcc-warnings
+#  - make -j$(nproc) -C gl
+#  - scan-build --status-bugs -o scan-build-lib make -j$(nproc) -C lib
+#  tags:
+#  - shared
+#  except:
+#  - tags
+#  artifacts:
+#    expire_in: 2 weeks
+#    when: on_failure
+#    paths:
+#      - scan-build-lib/*
+
+#pages:
+#  image: $CI_REGISTRY/$BUILD_IMAGES_PROJECT:$DEBIAN_BUILD
+#  script:
+#  - ./bootstrap
+#  - CFLAGS="-g -O0" ./configure --enable-gtk-doc --enable-code-coverage --disable-gcc-warnings
+#  - make -j$(nproc) check
+#  - make local-code-coverage-output
+#  - mkdir -p public
+#  - rm -rf public/coverage
+#  - mv libidn2-*-coverage public/coverage
+#  - rm -rf public/reference
+#  - mv doc/reference/html public/reference
+#  - make -C doc manual
+#  - rm -rf public/manual
+#  - mv doc/manual public/manual
+#  tags:
+#  - shared
+#  artifacts:
+#    when: on_success
+#    paths:
+#      - public
+#  only:
+#    - master
+
+#MinGW64:
+#  image: $CI_REGISTRY/$BUILD_IMAGES_PROJECT:$FEDORA_BUILD
+#  before_script:
+#  - mount -t binfmt_misc binfmt_misc /proc/sys/fs/binfmt_misc &&
+#    echo ':DOSWin:M::MZ::/usr/bin/wine64:' > /proc/sys/fs/binfmt_misc/register
+#  script:
+#  - ./bootstrap
+#  - mingw64-configure --disable-valgrind-tests --disable-doc
+#  - mingw64-make -j$(nproc)
+#  - mingw64-make -j$(nproc) check
+#  tags:
+#  - shared
+#  - docker
+#  except:
+#  - tags
+#  artifacts:
+#    expire_in: 2 weeks
+#    when: on_failure
+#    paths:
+#      - ./*.log
+#      - tests/*.log
--- a/.travis.yml
+++ b/.travis.yml
@@ -0,0 +1,51 @@
+sudo: no
+dist: trusty
+
+language: c
+
+compiler:
+    - gcc
+
+notifications:
+    email:
+        recipients:
+            - darnir@gmail.com
+            - bug-wget@gnu.org
+        on_success: change
+        on_failure: always
+
+env:
+    global:
+        - PERLLIB=$PERLLIB:../../tests
+        - PERL5LIB=$PERL5LIB:../../tests
+    matrix:
+        - SSL=""
+        - SSL="openssl"
+        - SSL="gnutls"
+
+addons:
+    apt:
+        packages:
+            - automake
+            - autoconf
+            - autopoint
+            - flex
+            - texinfo
+            - pkg-config
+            - libgnutls-dev
+            - libssl-dev
+            - make
+            - libhttp-daemon-perl
+            - libio-socket-ssl-perl
+            - libidn2-dev
+            - gettext
+            - texlive
+            - python3
+            - valgrind
+            - language-pack-tr
+            - language-pack-ru
+
+script:
+    - make distclean || true
+    - ./bootstrap
+    - ./contrib/travis-ci $SSL
--- a/1067
+++ b/1067
--- a/8
+++ b/8
@@ -1,7 +1,7 @@
                    GNU GENERAL PUBLIC LICENSE
                       Version 3, 29 June 2007

- Copyright (C) 2007 Free Software Foundation, Inc. <http://fsf.org/>
+ Copyright (C) 2007 Free Software Foundation, Inc. <https://fsf.org/>
 Everyone is permitted to copy and distribute verbatim copies
 of this license document, but changing it is not allowed.

@@ -645,7 +645,7 @@ the "copyright" line and a pointer to where the full notice is found.
    GNU General Public License for more details.

    You should have received a copy of the GNU General Public License
-    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.

 Also add information on how to contact you by electronic and paper mail.

@@ -664,11 +664,11 @@ might be different; for a GUI interface, you would use an "about box".
  You should also get your employer (if you work as a programmer) or school,
 if any, to sign a "copyright disclaimer" for the program, if necessary.
 For more information on this, and how to apply and follow the GNU GPL, see
-<http://www.gnu.org/licenses/>.
+<https://www.gnu.org/licenses/>.

  The GNU General Public License does not permit incorporating your program
 into proprietary programs.  If your program is a subroutine library, you
 may consider it more useful to permit linking proprietary applications with
 the library.  If this is what you want to do, use the GNU Lesser General
 Public License instead of this License.  But first, please read
-<http://www.gnu.org/philosophy/why-not-lgpl.html>.
+<https://www.gnu.org/philosophy/why-not-lgpl.html>.
--- a/3222
+++ b/3222
--- a/21536
+++ b/21536
--- a/ChangeLog.README
+++ b/ChangeLog.README
@@ -1,16 +0,0 @@
-Please note that Wget has more than one ChangeLog file:
-
-    ./ChangeLog: documents changes to files in the top-level directory
-                 and to files in subdirectories like po/ that don't have
-                 their own ChangeLogs
-
-    src/ChangeLog: documents only changes to files in the src directory
-
-    doc/ChangeLog: documents only changes to files in the doc directory
-
-    windows/ChangeLog: documents only changes to files in the windows directory
-
-    msdos/ChangeLog: documents only changes to files in the msdos directory
-
-When checking to see if a patch you sent in has been applied, please
-look in the appropriate ChangeLog(s).
--- a/27
+++ b/27
@@ -7,30 +7,14 @@ Primary List
 The primary mailinglist for discussion, bug-reports, or questions about
 GNU Wget is at <bug-wget@gnu.org>. To subscribe, send an email to
 <bug-wget-join@gnu.org>, or visit
-`http://lists.gnu.org/mailman/listinfo/bug-wget'.
+`https://lists.gnu.org/mailman/listinfo/bug-wget'.

   You do not need to subscribe to send a message to the list; however,
 please note that unsubscribed messages are moderated, and may take a
 while before they hit the list--*usually around a day*.  If you want
 your message to show up immediately, please subscribe to the list
 before posting. Archives for the list may be found at
-`http://lists.gnu.org/pipermail/bug-wget/'.
-
-   An NNTP/Usenettish gateway is also available via Gmane
-(http://gmane.org/about.php). You can see the Gmane archives at
-`http://news.gmane.org/gmane.comp.web.wget.general'. Note that the
-Gmane archives conveniently include messages from both the current
-list, and the previous one. Messages also show up in the Gmane archives
-sooner than they do at `lists.gnu.org'.
-
-Bug Notices List
----------------
-
-Additionally, there is the <wget-notify@addictivecode.org> mailing
-list. This is a non-discussion list that receives bug report
-notifications from the bug-tracker. To subscribe to this list, send an
-email to <wget-notify-join@addictivecode.org>, or visit
-`http://addictivecode.org/mailman/listinfo/wget-notify'.
+`https://lists.gnu.org/pipermail/bug-wget/'.

 Obsolete Lists
 --------------
@@ -40,10 +24,7 @@ discussion list, and another list, <wget-patches@sunsite.dk> was used
 for submitting and discussing patches to GNU Wget.

   Messages from <wget@sunsite.dk> are archived at
-     `http://www.mail-archive.com/wget%40sunsite.dk/' and at
-
-     `http://news.gmane.org/gmane.comp.web.wget.general' (which also
-     continues to archive the current list, <bug-wget@gnu.org>).
+     `https://www.mail-archive.com/wget%40sunsite.dk/' and at

   Messages from <wget-patches@sunsite.dk> are archived at
-     `http://news.gmane.org/gmane.comp.web.wget.patches'.
+     `https://news.gmane.org/gmane.comp.web.wget.patches'.
--- a/Makefile.am
+++ b/Makefile.am
@@ -1,6 +1,6 @@
 # Makefile for `Wget' utility
-# Copyright (C) 1995, 1996, 1997, 2006, 2007, 2008, 2009, 2010, 2011
-# Free Software Foundation, Inc.
+# Copyright (C) 1995-1997, 2006-2018 Free Software Foundation,
+# Inc.

 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -13,7 +13,7 @@
 # GNU General Public License for more details.

 # You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.

 # Additional permission under GNU GPL version 3 section 7

@@ -27,7 +27,7 @@
 # as that of the covered work.

 #
-# Version: $(VERSION)
+# Version: @VERSION@
 #


@@ -43,8 +43,8 @@ ACLOCAL_AMFLAGS = -I m4
 # subdirectories in the distribution
 SUBDIRS = lib src doc po tests util testenv

-EXTRA_DIST = ChangeLog.README MAILING-LIST \
-             msdos/ChangeLog msdos/config.h msdos/Makefile.DJ \
+EXTRA_DIST = MAILING-LIST \
+             msdos/config.h msdos/Makefile.DJ \
             msdos/Makefile.WC ABOUT-NLS \
             build-aux/build_info.pl build-aux/git-version-gen .version

@@ -60,5 +60,23 @@ clean-generic:

 # Arrange so that .tarball-version appears only in the distribution
 # tarball, and never in a checked-out repository.
-dist-hook:
+dist-hook: gen-ChangeLog
 	$(AM_V_GEN)echo $(VERSION) > $(distdir)/.tarball-version
+
+gen_start_date = 2014-12-10
+.PHONY: gen-ChangeLog
+gen-ChangeLog:
+	$(AM_V_GEN)if test -d .git; then				\
+	  log_fix="$(srcdir)/build-aux/git-log-fix";			\
+	  test -e "$$log_fix"						\
+	    || amend_git_log=;						\
+	  $(top_srcdir)/build-aux/gitlog-to-changelog			\
+	    $$amend_git_log --since=$(gen_start_date) > $(distdir)/cl-t && \
+            echo >>  $(distdir)/cl-t &&                                 \
+            cat ChangeLog-2014-12-10 >>  $(distdir)/cl-t &&             \
+	    { rm -f $(distdir)/ChangeLog &&				\
+	      mv $(distdir)/cl-t $(distdir)/ChangeLog; }		\
+	fi
+
+check-valgrind:
+	TESTS_ENVIRONMENT="VALGRIND_TESTS=1" $(MAKE) check
--- a/280
+++ b/280
@@ -1,25 +1,219 @@
 GNU Wget NEWS -- history of user-visible changes.

-Copyright (C) 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004, 2005,
-2006, 2007, 2008, 2009, 2010, 2011, 2012 Free Software Foundation, Inc.
+Copyright (C) 1997-2018 Free Software Foundation, Inc.
 See the end for copying conditions.

 Please send GNU Wget bug reports to <bug-wget@gnu.org>.
-
-* Changes in Wget X.Y.Z

-** Use libpsl for verifying cookie domains
+* Changes in Wget 1.19.3

-** Default progress bar output changed
+* Prevent erroneous decompression of .gz and .tgz files with broken servers

-** Introduce --show-progress to force display the progress bar
+* Added support for HTTP 308 Permanent Redirect response

-** Introduce --no-config.
+* Fix a segfault in some cases where the Content-Type header is not sent
+
+* Support OpenSSL 1.1 builds without using deprecated features
+
+* Fix netrc file detection on Windows
+
+* Several minor bug fixes
+
+
+* Changes in Wget 1.19.2
+
+* Fix CVE-2017-13089 (Stack overflow in HTTP protocol handling)
+
+* Fix CVE-2017-13090 (Heap overflow in HTTP protocol handling)
+
+* New option --compression for gzip Content-Encoding
+
+* New option --[no]-netrc to control .netrc parsing
+
+* Added GNU extensions to .netrc parsing
+
+* Improved IDNA 2003 compatibility
+
+* Fix VPATH issues
+
+* Improved and extended the test suite
+
+* Support Wayback Machine's X-Archive-Orig-last-modified
+
+* Several bug fixes
+
+
+* Changes in Wget 1.19.1
+
+* Fix bugs, a regression, portability/build issues
+
+* Add new option --retry-on-http-error
+
+
+* Changes in Wget 1.19
+
+* New option --use-askpass=COMMAND. Fetch user/password by calling
+  an external program.
+
+* Use IDNA2008 (+ TR46 if available) through libidn2
+
+* When processing a Metalink header, --metalink-index=<number> allows
+  to process the header's application/metalink4+xml files.
+
+* When processing a Metalink file, --trust-server-names enables the
+  use of the destination file names specified in the Metalink file,
+  otherwise a safe destination file name is computed.
+
+* When processing a Metalink file, enforce a safe destination path.
+  Remove any drive letter prefix under w32, i.e. 'C:D:file'.  Call
+  libmetalink's metalink_check_safe_path() to prevent absolute,
+  relative, or home paths:
+  https://tools.ietf.org/html/rfc5854#section-4.1.2.1
+  https://tools.ietf.org/html/rfc5854#section-4.2.8.3
+
+* When processing a Metalink file, --directory-prefix=<prefix> sets
+  the top of the retrieval tree to prefix for Metalink downloads.
+
+* When processing a Metalink file, reject downloaded files which don't
+  agree with their own metalink:size value:
+  https://tools.ietf.org/html/rfc5854#section-4.2.16
+
+* When processing a Metalink file, with --continue resume partially
+  downloaded files and keep fully downloaded files even if they fail
+  the verification.
+
+* When processing a Metalink file, create the parent directories of a
+  "path/file" destination file name:
+  https://tools.ietf.org/html/rfc5854#section-4.1.2.1
+  https://tools.ietf.org/html/rfc5854#section-4.2.8.3
+
+* On a recursive download, append a .tmp suffix to temporary files
+  that will be deleted after being parsed, and create them
+  readable/writable only by the owner.
+
+* New make target 'check-valgrind'
+
+* Fix several bugs
+
+* Fix compatibility issues
+
+* Changes in Wget 1.18
+
+* By default, on server redirects to a FTP resource, use the original
+  URL to get the local file name. Close CVE-2016-4971.  This
+  introduces a backward-incompatibility for HTTP->FTP redirects and
+  any script that relies on the old  behaviour must use
+  --trust-server-names.
+
+* Check the HSTS file is not world-writable before using it.
+
+* Parse <img srcset> attributes on a recursive download.
+
+* Fix problem with SNI server names having trailing dot(s)
+
+* New options --bind-dns-address and --dns-servers.
+
+* When Wget is built with libiconv, it now converts non-ASCII URIs to
+  the locale's codeset when it creates files.  The encoding of the
+  remote files and URIs is taken from --remote-encoding, defaulting to
+  UTF-8.  The result is that non-ASCII URIs and files downloaded via
+  HTTP/HTTPS and FTP will have names on the local filesystem that
+  correspond to their remote names.
+
+* Changes in Wget 1.17.1
+
+* Fix compile error when IPv6 is disabled or SSL is not present.
+
+* Fix HSTS memory leak.
+
+* Fix progress output in non-C locales.
+
+* Fix SIGSEGV when -N and --content-disposition are used together.
+
+* Add --check-certificate=quiet to tell wget to not print any warning about
+  invalid certificates.
+
+* Changes in Wget 1.17
+
+** Remove FTP passive to active fallback due to privacy concerns.
+
+** Add support for --if-modified-since.
+
+** Add support for metalink through --input-metalink and --metalink-over-http.
+
+** Add support for HSTS through --hsts and --hsts-file.
+
+** Add option to restrict filenames under VMS.
+
+** Add support for --rejected-log which logs to a separate file the reasons why
+   URLs are being rejected and some context around it.
+
+** Add support for FTPS.
+
+** Do not download/save file on error when --spider enabled
+
+** Add --convert-file-only option. This option converts only the
+   filename part of the URLs, leaving the rest of the URLs untouched.
+
+* Changes in Wget 1.16.3
+
+** Fix a regression introduced by wget 1.16.2 that --quiet is not
+   really quiet anymore.
+
+* Changes in Wget 1.16.2
+
+** Native uuid generation on Windows
+
+** Fix build on Solaris
+
+** Allow progress bar on stderr when -o is used
+
+** Accept 5-digit port numbers in FTP EPSV responses.
+
+** Support older versions of flex.
+
+** Updated translations.
+
+* Changes in Wget 1.16.1
+
+** Add --enable-assert configure option.
+
+** Use pkg-config to check for libraries presence.
+
+** Do not limit --secure-protocol=auto|pfs to TLSv1.0.
+
+** Add --secure-protocol=TLSv1_1|TLSv1_2 .
+
+** Full C89 source code compliance.
+
+** Select and use the most secure authentication scheme with HTTP connections.
+
+** Fix issues with turkish locales.
+
+** Handle 504 Gateway Timeout.
+
+** New option --crl-file to load Certificate Revocation Lists.
+
+** Add valgrind support to tests suite.
+
+** Fix an off-by-one problem in the progress bar (introduced in 1.16).
+
+* Changes in Wget 1.16
+
+** No longer create local symbolic links by default.  Closes CVE-2014-4877.
+
+** Use libpsl for verifying cookie domains.
+
+** Default progress bar output changed.
+
+** Introduce --show-progress to force display the progress bar.
+
+** Introduce --no-config.  The wgetrc files will not be read.

 ** Introduce --start-pos to allow starting downloads from a specified position.

 ** Fix a problem with ISA Server Proxy and keep-alive connections.
-
+
 * Changes in Wget 1.15

 ** Add support for --method.
@@ -51,7 +245,7 @@ Please send GNU Wget bug reports to <bug-wget@gnu.org>.
 ** Specify Host with the HTTP CONNECT method.

 ** Use the correct HTTP method on a redirection.
-
+
 * Changes in Wget 1.14

 ** Add support for content-on-error.  It allows to store the HTTP
@@ -73,12 +267,12 @@ Please send GNU Wget bug reports to <bug-wget@gnu.org>.

 ** Add support for TLS Server Name Indication.

-** Accept the arguments --accept-reject and --reject-regex.
+** Accept the arguments --accept-regex and --reject-regex.

 ** The GNU TLS backend honors correctly the timeout value.

 ** Add support for RFC 2617 Digest Access Authentication.
-
+
 * Changes in Wget 1.13.4

 ** Now --version and --help work again.
@@ -91,7 +285,7 @@ Please send GNU Wget bug reports to <bug-wget@gnu.org>.
   is specified.

 ** Fix a segfault on an incomplete STYLE tag.
-
+
 * Changes in Wget 1.13.3

 ** Support HTTP/1.1
@@ -157,7 +351,7 @@ Please send GNU Wget bug reports to <bug-wget@gnu.org>.

 ** Introduce `show_all_dns_entries' to print all IP addresses corresponding to
   a DNS name when it is resolved.
-
+
 * Changes in Wget 1.12

 ** Mailing list MOVED to bug-wget@gnu.org
@@ -211,7 +405,7 @@ forces the percent-encoding of all non-ASCII bytes
 now documented: save_headers, spider, and user_agent,
 auth_no_challenge, and keep_session_cookies. Also added documentation
 for the "lowercase" and "uppercase" values for --restrict-file-names, which had been present since Wget 1.11.
-
+
 * Changes in Wget 1.11.4

 ** Fixed an issue (apparently a regression) where -O would refuse to
@@ -225,13 +419,13 @@ locally with content.
 to a pointer-to-time_t.

 ** Translation updates for Catalan.
-
+
 * Changes in Wget 1.11.3

 ** Downgraded -N with -O to a warning, rather than an error.

 ** Translation updates
-
+
 * Changes in Wget 1.11.2

 ** Fixed a problem in authenticating over HTTPS through a proxy.
@@ -253,7 +447,7 @@ multiple times. This has been fixed in 1.11.2.
 for improved conformance with RFC 3986. However, this behavior presents
 problems for some FTP setups, and so they are now preserved again, for
 FTP URLs only.
-
+
 * Changes in Wget 1.11.1.

 ** Interrupted downloads no longer result in renaming the file
@@ -275,7 +469,7 @@ be surprising, and notes that they may change in the future.

 ** Documentation of --no-parents now explains how a trailing slash, or
 lack thereof, in the specified URL, will affect behavior.
-
+
 * Changes in Wget 1.11.

 ** Timestamping now uses the value from the most recent HTTP response,
@@ -321,7 +515,7 @@ available at http://hg.addictivecode.org/. Prior to this, the source
 code was hosted on Subversion (migrated from the original CVS); you can
 still get access to older tags and branches for Wget in the Subversion
 repository at http://addictivecode.org/svn/wget/.
-
+
 * Changes in Wget 1.10.

 ** Downloading files larger than 2GB, sometimes referred to as "large
@@ -432,9 +626,9 @@ http_password and proxy_password respectively.  The login and passwd
 .wgetrc commands have been deprecated.

 * `wget -b' now works correctly under Windows.
-
+
 * Wget 1.9.1 is a bugfix release with no user-visible changes.
-
+
 * Changes in Wget 1.9.

 ** It is now possible to specify that POST method be used for HTTP
@@ -486,11 +680,11 @@ values "yes" and "no" along with the traditional "on" and "off".
 ** It is now possible to specify decimal values for timeouts, waiting
 periods, and download rate.  For instance, `--wait=0.5' now works as
 expected, as does `--dns-timeout=0.5' and even `--limit-rate=2.5k'.
-
+
 * Wget 1.8.2 is a bugfix release with no user-visible changes.
-
+
 * Wget 1.8.1 is a bugfix release with no user-visible changes.
-
+
 * Changes in Wget 1.8.

 ** A new progress indicator is now available and used by default.
@@ -542,7 +736,7 @@ addresses when accessing the first one fails.
 non-standard port.

 ** Wget now supports the robots.txt directives specified in
-<http://www.robotstxt.org/wc/norobots-rfc.txt>.
+<http://www.robotstxt.org/norobots-rfc.txt>.

 ** URL parser has been fixed, especially the infamous overzealous
 quoting.  Wget no longer dequotes reserved characters, e.g. `%3F' is
@@ -550,9 +744,9 @@ no longer translated to `?', nor `%2B' to `+'.  Unsafe characters
 which are not reserved are still escaped, of course.

 ** No more than 20 successive redirections are allowed.
-
+
 * Wget 1.7.1 is a bugfix release with no user-visible changes.
-
+
 * Changes in Wget 1.7.

 ** SSL (`https') pages now work if you compile Wget with SSL support;
@@ -594,7 +788,7 @@ large sites (thousands of documents).
 documentation.  (The last version that shipped with a man page was
 1.4.5).  To get this, you need to have pod2man from the Perl
 distribution installed on your system.
-
+
 * Changes in Wget 1.6

 ** Administrative changes.
@@ -645,13 +839,13 @@ passive_ftp is the only .wgetrc command which takes a lockable Boolean.
 bound to multiple IP addresses.

 ** wget now accepts (illegal per HTTP spec) relative URLs in HTTP redirects.
-
+
 * Wget 1.5.3 is a bugfix release with no user-visible changes.
-
+
 * Wget 1.5.2 is a bugfix release with no user-visible changes.
-
+
 * Wget 1.5.1 is a bugfix release with no user-visible changes.
-
+
 * Changes in Wget 1.5.0

 ** Wget speaks many languages!
@@ -697,11 +891,11 @@ download.

 ** The manual page, now hopelessly out of date, is no longer
 distributed with Wget.
-
+
 * Wget 1.4.5 is a bugfix release with no user-visible changes.
-
+
 * Wget 1.4.4 is a bugfix release with no user-visible changes.
-
+
 * Changes in Wget 1.4.3

 ** Wget is now a GNU utility.
@@ -717,7 +911,7 @@ distributed with Wget.
 ** Global wgetrc now goes to /usr/local/etc (i.e. $sysconfdir).

 ** Lots of bugfixes.
-
+
 * Changes in Wget 1.4.2

 ** New mirror site at ftp://sunsite.auc.dk/pub/infosystems/wget/,
@@ -735,7 +929,7 @@ files.
 ** --convert-links should work now.

 ** Minor bugfixes.
-
+
 * Changes in Wget 1.4.1

 ** Minor bugfixes.
@@ -743,7 +937,7 @@ files.
 ** Added -I (the opposite of -X).

 ** Dot tracing is now customizable; try wget --dot-style=binary
-
+
 * Changes in Wget 1.4.0

 ** Wget 1.4.0 [formerly known as Geturl] is an extensive rewrite of
@@ -824,7 +1018,7 @@ turn on mirroring options).
 ** Lots of small features I can't remember. :-)

 ** A host of bugfixes.
-
+
 * Changes in Geturl 1.3

 ** Added FTP globbing support (ftp://fly.cc.fer.hr/*)
@@ -842,9 +1036,9 @@ turn on mirroring options).
 ** Added support for robots.txt

 ** Fixed some minor bugs
-
+
 * Geturl 1.2 is a bugfix release with no user-visible changes.
-
+
 * Changes in Geturl 1.1

 ** REST supported in FTP
@@ -862,7 +1056,7 @@ geturl -vo log http://fly.cc.fer.hr/
 ** <base href="xxx"> supported

 ** autoconf supported
-
+
 ----------------------------------------------------------------------
 Copyright information:

--- a/6
+++ b/6
@@ -1,7 +1,7 @@
                                                          -*- text -*-
 GNU Wget
 ========
-                  Current Web home: http://www.gnu.org/software/wget/
+                  Current Web home: https://www.gnu.org/software/wget/

 GNU Wget is a free utility for non-interactive download of files from
 the Web.  It supports HTTP, HTTPS, and FTP protocols, as well as
@@ -50,10 +50,10 @@ resides at <ftp://ftp.gnu.org/pub/gnu/wget/>.
 Please report bugs in Wget to <bug-wget@gnu.org>.

 See the file `MAILING-LIST' for information about Wget mailing lists.
-Wget's home page is at <http://www.gnu.org/software/wget/>.
+Wget's home page is at <https://www.gnu.org/software/wget/>.

 If you would like to contribute code for Wget, please read
-http://wget.addictivecode.org/PatchGuidelines.
+https://wget.addictivecode.org/PatchGuidelines.

 Wget was originally written and mainained by Hrvoje Niksic.  Please see
 the file AUTHORS for a list of major contributors, and the ChangeLogs
--- a/README.checkout
+++ b/README.checkout
@@ -4,8 +4,9 @@ Compiling From Repository Sources
   does not contain automatically-generated files, even when these are
   normally present in the distribution tarballs. Therefore, to build GNU
   Wget from the sources in the repository, you'll need to have one or
-   more of the following (note that gettext, OpenSSL, libidn, libiconv,
-   libpsl and libmetalink are not absolutely required):
+   more of the following (note that gettext, OpenSSL, GnuTLS, libidn2,
+   libiconv, libpsl, libpcre, pkg-config, libmetalink and GnuPG are not
+   absolutely required):

     * [20]autoconf (currently, GNU Wget requires version 2.61). This is
       needed to generate the configure script from configure.in. This is
@@ -23,12 +24,18 @@ Compiling From Repository Sources
     * [23]Perl, if you wish to generate the wget(1) manpage, or run the
       tests in the tests/ sub directory. Tarball distributions include an
       already-generated wget.1 manual. The command "make check" runs the
-       test suite written in perl. To execute all the tests you need
-       [24]libwww-perl perl library. If "perl -MCPAN -e 'install
-       Bundle::LWP'" fails then you most likely don't have cpan module
-       installed. First download [25]CPAN and [26]install it. Then execute
-       "perl -MCPAN -e 'install Bundle::LWP'". Now "make check" should
-       pass most of the tests in the test suite.
+       test suite written in perl and python (see below). To execute all the tests
+       you need libwww-perl and libio-socket-ssl-perl perl library.
+       GNU/Linux distros likely have these two libraries packaged.
+       If not, install them directly with "perl -MCPAN -e 'install Bundle::LWP'"[24].
+       If that fails then you most likely don't have the CPAN module installed.
+       First download [25]CPAN and [26]install it, then try again the above command.
+       Now "make check" should pass most of the tests in the test suite.
+
+     * [45]Python3, if you want to run the tests in the testenv/ subdirectory.
+       Keep in mind that "make check" will try to run all the Perl and Python3
+       tests. More information about the test suite below in the section
+       "Testing and development".

     * [27]texinfo in order to generate Info, PostScript and/or HTML
       documentation. You don't need texinfo in order to generate the
@@ -46,37 +53,65 @@ Compiling From Repository Sources
       to export M4=<new m4 path>, to be sure that autoconf/automake use
       it instead of the old one.

-     * [29]OpenSSL to handle encrypted web sessions. You almost certainly
-       want this. It is not enough to have the library installed; you need
-       to have the header files available to compile against as well. For
-       GNU/Linux systems, this usually means installing a "developer"
-       package for openssl (for example, [30]Debian systems require
-       installation of the libssl-dev package). Alternatively, you can
-       compile GNU Wget against the [31]GNUTLS library instead, by
-       specifying --with-ssl=gnutls to the ./configure script; however,
-       building against GNUTLS is currently experimental, and not
-       officially supported. If you have OpenSSL and its development
-       headers installed, but don't want to build Wget with it, specify
-       --without-ssl to the ./configure script.
+     * [29]GnuTLS to allow encrypted data transfer (HTTPS).
+       You need the header files and the library installed.
+       As an alternative, you can use [30]OpenSSL by specifying
+       --with-ssl=openssl to the ./configure script.
+       If you do not want HTTPS support, specify --without-ssl to the
+       ./configure script.
+       If you want to compile+link a non-system library version use
+       --with-libgnutls-prefix (or if having pkg-config: see description
+       below).

-     * [32]libidn is required for IDN/IRI support (non-ASCII characters
+     * [30]OpenSSL to allow encrypted data transfer (HTTPS) an
+       alternative to [31]GnuTLS.
+       You need the header files and the library installed.
+       If you want to compile+link a non-system library version use
+       --with-libssl-prefix (or if having pkg-config: see description below).
+
+     * [31]libidn2 is required for IDN/IRI support (non-ASCII characters
       within what would otherwise be URLs).

-     * [33]libiconv is required on non-GNU systems, for IDN/IRI support.
+     * [32]libiconv is required on non-GNU systems, for IDN/IRI support.
       On GNU systems, the functionality provided by libiconv is already
       present in the system libraries.

-     * [34]git is used to fetch gnulib files trough the bootstrap.sh script.
+     * [33]git is used to fetch gnulib files trough the bootstrap.sh script.

-     * [45]libmetalink is required for compiling with Metalink support.
+     * [34]libpsl is required for using a public suffix list to check for valid
+       cookie domains. You need the header files and the library installed.
+
+     * [35]libpcre is required for using Perl-compatible regular expressions
+       with --accept-regex and --reject-regex. You need the header files and
+       the library installed to compile and link Wget with PCRE support.
+
+     * [36]pkg-config helps the ./configure script to find installed libraries.
+       Most libraries provide a pkg-config file (.pc extension) with
+       information about dependencies, header file and library locations.
+       Distributions deliver their specific .pc file to each library.
+       If you want to compile+link against your own library version, make a
+       copy of the appropriate .pc file and amend it to your needs (e.g. edit
+       the line starting with prefix=). Before you execute the ./configure
+       script, set (and export) PKG_CONFIG_PATH to the directory where you
+       saved the .pc file. Example:
+       $ PKG_CONFIG_PATH="." ./configure
+
+     * [46]libmetalink is needed to enable Metalink files support.
+
+     * [47]GnuPG with GPGME is used to verify GPG-signed Metalink resources.
+
+     * [48]libcares is needed to bind DNS resolving to a given IP address.
+       The command line options --dns-servers and --bind-dns-address are
+       only available when configured with --with-cares.
+
+     * [49]Gpef is required by some gnulib modules, most notably "unicase". You
+	   need the application installed on the machine to compile Wget from git.

-     * [35]libpsl is required for using a public suffix list to check for valid
-       cookie domains.

   For those who might be confused as to what to do once they check out
   the source code, considering configure and Makefile do not yet exist at
-   that point, a shell script called bootstrap.sh has been provided. After
-   calling ./bootstrap.sh you're ready to build GNU Wget in the normal
+   that point, a shell script called bootstrap has been provided. After
+   calling ./bootstrap you're ready to build GNU Wget in the normal
   fashion, with ./configure and make.

   So, to sum up, after checking out the source code as described above,
@@ -85,17 +120,51 @@ Compiling From Repository Sources
 $  cd wget        # assumes you've cloned a repository to "./wget"
    2. Generate all the automatically-generated files required prior to
       configuring the package:
- $  ./bootstrap.sh
+ $  ./bootstrap (or: bash ./bootstrap)
    3. Configure the package and compile it:
- $  ./configure [some_parameters]
+ $  ./configure --enable-assert [some_parameters]
 $  make
    4. Hack, compile, test, hack, compile, test...
 $  src/wget --version
 GNU Wget 1.12-devel (9cb2563197bc)

- Copyright © 2008,2010 Free Software Foundation, Inc.
+Testing and development
+
+ All developers are requested to enable the assertions on their development
+ builds to ensure a stable codebase. Assertions are added to state certain
+ assumptions about the code and its data which all developers should be mindful
+ of. To enable assertions, run the configure command with the --enable-assert
+ option, like this:
+
+ $ ./configure --enable-assert [other configure options]
+
+ Both the Perl and Python test suites (test/ and testenv/) include support for GDB and Valgrind.
+ The environment variables GDB_TESTS and VALGRIND_TESTS are available to
+ enable such wrappers. If specified, Wget would be run through either of them
+ during the test. For example:
+
+ $ cd testenv
+ $ VALGRIND_TESTS=1 ./Test-O.py
+
+ That would execute Test-O.py test case, but running Wget through Valgrind.
+
+ GDB has preference over Valgrind. If both variables have been asserted,
+ Wget would be run through GDB.
+
+ If you run a test case through GDB, please bear in mind that it could give
+ a false negative. This is because some tests that expect Wget to fail
+ rely on Wget's return code. However, when run through GDB, its return code
+ will always be zero, causing the test to claim failure. This wrapper for
+ GDB is, however, very useful to tackle bugs, allowing one to write a test case
+ for some specific bug and then using GDB to fix it more easily. Otherwise,
+ a dedicated server would have to be set up and write a custom CGI just to reproduce
+ that bug, which might be tedious. Tests should only be run through GDB for that purpose.
+
+
+ Copyright (C) 2008, 2010, 2014-2015, 2018 Free Software Foundation,
+ Inc.
 License GPLv3+: GNU GPL version 3 or later
- <http://www.gnu.org/licenses/gpl.html>.
+ <https://www.gnu.org/licenses/gpl.html>.
 This is free software: you are free to change and redistribute it.
 There is NO WARRANTY, to the extent permitted by law.

@@ -114,34 +183,38 @@ Compiling From Repository Sources
     * [43]Valid HTML 4.01
     __________________________________________________________________

-   All content © 2007 Free Software Foundation. For terms of use,
-   redistribution, and modification, please see the [44]WikiLicense page.
+   All content (C) 2007 Free Software Foundation. For terms of use,
+   redistribution, and modification, please see the [45]WikiLicense page.

 References

-  20. http://www.gnu.org/software/autoconf/
-  21. http://www.gnu.org/software/automake/
-  22. http://flex.sourceforge.net/
-  23. http://www.perl.org/
+  20. https://www.gnu.org/software/autoconf/
+  21. https://www.gnu.org/software/automake/
+  22. https://github.com/westes/flex
+  23. https://www.perl.org/
  24. http://search.cpan.org/dist/libwww-perl/lib/Bundle/LWP.pm
  25. http://search.cpan.org/CPAN/authors/id/A/AN/ANDK/CPAN-1.9402.tar.gz
  26. http://apthorpe.cynistar.net/code/configuring_cpan.html
-  27. http://www.gnu.org/software/texinfo/
-  28. http://www.gnu.org/software/gettext/
-  29. http://www.openssl.org/
-  30. http://www.debian.org/
-  31. http://www.gnu.org/software/gnutls/
-  32. http://www.gnu.org/software/libidn/
-  33. http://www.gnu.org/software/libiconv/
-  34. http://git-scm.com/
-  35. https://github.com/rockdaboot/libpsl
+  27. https://www.gnu.org/software/texinfo/
+  28. https://www.gnu.org/software/gettext/
+  29. https://www.gnu.org/software/gnutls/
+  30. https://www.openssl.org/
+  31. https://www.gnu.org/software/libidn/#libidn2
+  32. https://www.gnu.org/software/libiconv/
+  33. https://git-scm.com/
+  34. https://github.com/rockdaboot/libpsl
+  35. https://www.pcre.org
  36. http://wget.addictivecode.org/CompilingRepoSources?action=edit&editor=text
  37. http://wget.addictivecode.org/CompilingRepoSources
  38. http://wget.addictivecode.org/CompilingRepoSources?action=info
  39. http://wget.addictivecode.org/CompilingRepoSources?action=AttachFile
-  40. http://moinmo.in/
-  41. http://moinmo.in/Python
-  42. http://moinmo.in/GPL
-  43. http://validator.w3.org/check?uri=referer
+  40. https://moinmo.in/
+  41. https://moinmo.in/Python
+  42. https://moinmo.in/GPL
+  43. https://validator.w3.org/check?uri=referer
  44. http://wget.addictivecode.org/WikiLicense
-  45. https://launchpad.net/libmetalink
+  45. https://www.python.org/
+  46. https://launchpad.net/libmetalink
+  47. https://www.gnupg.org
+  48. https://c-ares.haxx.se/
+  49. https://www.gnu.org/software/gperf/
--- a/263
+++ b/263
@@ -1,10 +1,10 @@
 #! /bin/sh
 # Print a version string.
-scriptversion=2012-07-19.14; # UTC
+scriptversion=2017-09-19.08; # UTC

 # Bootstrap this package from checked-out sources.

-# Copyright (C) 2003-2012 Free Software Foundation, Inc.
+# Copyright (C) 2003-2018 Free Software Foundation, Inc.

 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -17,7 +17,7 @@ scriptversion=2012-07-19.14; # UTC
 # GNU General Public License for more details.

 # You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.

 # Originally written by Paul Eggert.  The canonical version of this
 # script is maintained as build-aux/bootstrap in gnulib, however, to
@@ -42,6 +42,9 @@ export LC_ALL

 local_gl_dir=gl

+# Honor $PERL, but work even if there is none.
+PERL="${PERL-perl}"
+
 me=$0

 usage() {
@@ -138,22 +141,23 @@ po_download_command_format=\
 # Fallback for downloading .po files (if rsync fails).
 po_download_command_format2=\
 "wget --mirror -nd -q -np -A.po -P '%s' \
- http://translationproject.org/latest/%s/"
+ https://translationproject.org/latest/%s/"

+# Prefer a non-empty tarname (4th argument of AC_INIT if given), else
+# fall back to the package name (1st argument with munging)
 extract_package_name='
-  /^AC_INIT(/{
-     /.*,.*,.*, */{
-       s///
-       s/[][]//g
-       s/)$//
+  /^AC_INIT(\[*/{
+     s///
+     /^[^,]*,[^,]*,[^,]*,[ []*\([^][ ,)]\)/{
+       s//\1/
+       s/[],)].*//
       p
       q
     }
-     s/AC_INIT(\[*//
-     s/]*,.*//
+     s/[],)].*//
     s/^GNU //
     y/ABCDEFGHIJKLMNOPQRSTUVWXYZ/abcdefghijklmnopqrstuvwxyz/
-     s/[^A-Za-z0-9_]/-/g
+     s/[^abcdefghijklmnopqrstuvwxyz0123456789_]/-/g
     p
  }
 '
@@ -208,12 +212,26 @@ bootstrap_sync=false
 # Use git to update gnulib sources
 use_git=true

+check_exists() {
+  if test "$1" = "--verbose"; then
+    ($2 --version </dev/null) >/dev/null 2>&1
+    if test $? -ge 126; then
+      # If not found, run with diagnostics as one may be
+      # presented with env variables to set to find the right version
+      ($2 --version </dev/null)
+    fi
+  else
+    ($1 --version </dev/null) >/dev/null 2>&1
+  fi
+
+  test $? -lt 126
+}
+
 # find_tool ENVVAR NAMES...
 # -------------------------
 # Search for a required program.  Use the value of ENVVAR, if set,
-# otherwise find the first of the NAMES that can be run (i.e.,
-# supports --version).  If found, set ENVVAR to the program name,
-# die otherwise.
+# otherwise find the first of the NAMES that can be run.
+# If found, set ENVVAR to the program name, die otherwise.
 #
 # FIXME: code duplication, see also gnu-web-doc-update.
 find_tool ()
@@ -223,27 +241,21 @@ find_tool ()
  find_tool_names=$@
  eval "find_tool_res=\$$find_tool_envvar"
  if test x"$find_tool_res" = x; then
-    for i
-    do
-      if ($i --version </dev/null) >/dev/null 2>&1; then
-       find_tool_res=$i
-       break
+    for i; do
+      if check_exists $i; then
+        find_tool_res=$i
+        break
      fi
    done
-  else
-    find_tool_error_prefix="\$$find_tool_envvar: "
  fi
-  test x"$find_tool_res" != x \
-    || die "one of these is required: $find_tool_names"
-  ($find_tool_res --version </dev/null) >/dev/null 2>&1 \
-    || die "${find_tool_error_prefix}cannot run $find_tool_res --version"
+  if test x"$find_tool_res" = x; then
+    warn_ "one of these is required: $find_tool_names;"
+    die   "alternatively set $find_tool_envvar to a compatible tool"
+  fi
  eval "$find_tool_envvar=\$find_tool_res"
  eval "export $find_tool_envvar"
 }

-# Find sha1sum, named gsha1sum on MacPorts, and shasum on Mac OS X 10.6.
-find_tool SHA1SUM sha1sum gsha1sum shasum
-
 # Override the default configuration, if necessary.
 # Make sure that bootstrap.conf is sourced from the current directory
 # if we were invoked as "sh bootstrap".
@@ -255,12 +267,12 @@ esac
 # Extra files from gnulib, which override files from other sources.
 test -z "${gnulib_extra_files}" && \
  gnulib_extra_files="
-        $build_aux/install-sh
-        $build_aux/mdate-sh
-        $build_aux/texinfo.tex
-        $build_aux/depcomp
-        $build_aux/config.guess
-        $build_aux/config.sub
+        build-aux/install-sh
+        build-aux/mdate-sh
+        build-aux/texinfo.tex
+        build-aux/depcomp
+        build-aux/config.guess
+        build-aux/config.sub
        doc/INSTALL
 "

@@ -306,34 +318,34 @@ if test -n "$checkout_only_file" && test ! -r "$checkout_only_file"; then
  die "Bootstrapping from a non-checked-out distribution is risky."
 fi

-# Ensure that lines starting with ! sort last, per gitignore conventions
-# for whitelisting exceptions after a more generic blacklist pattern.
-sort_patterns() {
-  sort -u "$@" | sed '/^!/ {
-    H
-    d
-  }
-  $ {
-    P
-    x
-    s/^\n//
-  }' | sed '/^$/d'
+# Strip blank and comment lines to leave significant entries.
+gitignore_entries() {
+  sed '/^#/d; /^$/d' "$@"
 }

-# If $STR is not already on a line by itself in $FILE, insert it,
-# sorting the new contents of the file and replacing $FILE with the result.
-insert_sorted_if_absent() {
+# If $STR is not already on a line by itself in $FILE, insert it at the start.
+# Entries are inserted at the start of the ignore list to ensure existing
+# entries starting with ! are not overridden.  Such entries support
+# whitelisting exceptions after a more generic blacklist pattern.
+insert_if_absent() {
  file=$1
  str=$2
  test -f $file || touch $file
-  echo "$str" | sort_patterns - $file | cmp -s - $file > /dev/null \
-    || { echo "$str" | sort_patterns - $file > $file.bak \
-      && mv $file.bak $file; } \
-    || die "insert_sorted_if_absent $file $str: failed"
+  test -r $file || die "Error: failed to read ignore file: $file"
+  duplicate_entries=$(gitignore_entries $file | sort | uniq -d)
+  if [ "$duplicate_entries" ] ; then
+    die "Error: Duplicate entries in $file: " $duplicate_entries
+  fi
+  linesold=$(gitignore_entries $file | wc -l)
+  linesnew=$( { echo "$str"; cat $file; } | gitignore_entries | sort -u | wc -l)
+  if [ $linesold != $linesnew ] ; then
+    { echo "$str" | cat - $file > $file.bak && mv $file.bak $file; } \
+      || die "insert_if_absent $file $str: failed"
+  fi
 }

 # Adjust $PATTERN for $VC_IGNORE_FILE and insert it with
-# insert_sorted_if_absent.
+# insert_if_absent.
 insert_vc_ignore() {
  vc_ignore_file="$1"
  pattern="$2"
@@ -344,7 +356,7 @@ insert_vc_ignore() {
    # .gitignore entry.
    pattern=$(echo "$pattern" | sed s,^,/,);;
  esac
-  insert_sorted_if_absent "$vc_ignore_file" "$pattern"
+  insert_if_absent "$vc_ignore_file" "$pattern"
 }

 # Die if there is no AC_CONFIG_AUX_DIR($build_aux) line in configure.ac.
@@ -406,28 +418,30 @@ sort_ver() { # sort -V is not generally available
  done
 }

+get_version_sed='
+# Move version to start of line.
+s/.*[v ]\([0-9]\)/\1/
+
+# Skip lines that do not start with version.
+/^[0-9]/!d
+
+# Remove characters after the version.
+s/[^.a-z0-9-].*//
+
+# The first component must be digits only.
+s/^\([0-9]*\)[a-z-].*/\1/
+
+#the following essentially does s/5.005/5.5/
+s/\.0*\([1-9]\)/.\1/g
+p
+q'
+
 get_version() {
  app=$1

-  $app --version >/dev/null 2>&1 || return 1
+  $app --version >/dev/null 2>&1 || { $app --version; return 1; }

-  $app --version 2>&1 |
-  sed -n '# Move version to start of line.
-          s/.*[v ]\([0-9]\)/\1/
-
-          # Skip lines that do not start with version.
-          /^[0-9]/!d
-
-          # Remove characters after the version.
-          s/[^.a-z0-9-].*//
-
-          # The first component must be digits only.
-          s/^\([0-9]*\)[a-z-].*/\1/
-
-          #the following essentially does s/5.005/5.5/
-          s/\.0*\([1-9]\)/.\1/g
-          p
-          q'
+  $app --version 2>&1 | sed -n "$get_version_sed"
 }

 check_versions() {
@@ -447,6 +461,7 @@ check_versions() {
    test "$appvar" = TAR && appvar=AMTAR
    case $appvar in
        GZIP) ;; # Do not use $GZIP:  it contains gzip options.
+        PERL::*) ;; # Keep perl modules as-is
        *) eval "app=\${$appvar-$app}" ;;
    esac

@@ -464,12 +479,22 @@ check_versions() {
          ret=1
          continue
        } ;;
+      # Another check is for perl modules.  These can be written as
+      # e.g. perl::XML::XPath in case of XML::XPath module, etc.
+      perl::*)
+        # Extract module name
+        app="${app#perl::}"
+        if ! $PERL -m"$app" -e 'exit 0' >/dev/null 2>&1; then
+          warn_ "Error: perl module '$app' not found"
+          ret=1
+        fi
+        continue
+        ;;
    esac
    if [ "$req_ver" = "-" ]; then
      # Merely require app to exist; not all prereq apps are well-behaved
      # so we have to rely on $? rather than get_version.
-      $app --version >/dev/null 2>&1
-      if [ 126 -le $? ]; then
+      if ! check_exists --verbose $app; then
        warn_ "Error: '$app' not found"
        ret=1
      fi
@@ -502,6 +527,12 @@ print_versions() {
  # can't depend on column -t
 }

+# Find sha1sum, named gsha1sum on MacPorts, shasum on Mac OS X 10.6.
+# Also find the compatible sha1 utility on the BSDs
+if test x"$SKIP_PO" = x; then
+  find_tool SHA1SUM sha1sum gsha1sum shasum sha1
+fi
+
 use_libtool=0
 # We'd like to use grep -E, to see if any of LT_INIT,
 # AC_PROG_LIBTOOL, AM_PROG_LIBTOOL is used in configure.ac,
@@ -547,13 +578,21 @@ if ! printf "$buildreq" | check_versions; then
  fi
 fi

+# Warn the user if autom4te appears to be broken; this causes known
+# issues with at least gettext 0.18.3.
+probe=$(echo 'm4_quote([hi])' | autom4te -l M4sugar -t 'm4_quote:$%' -)
+if test "x$probe" != xhi; then
+  warn_ "WARNING: your autom4te wrapper eats stdin;"
+  warn_ "if bootstrap fails, consider upgrading your autotools"
+fi
+
 echo "$0: Bootstrapping from checked-out $package sources..."

 # See if we can use gnulib's git-merge-changelog merge driver.
-if test -d .git && (git --version) >/dev/null 2>/dev/null ; then
+if $use_git && test -d .git && check_exists git; then
  if git config merge.merge-changelog.driver >/dev/null ; then
    :
-  elif (git-merge-changelog --version) >/dev/null 2>/dev/null ; then
+  elif check_exists git-merge-changelog; then
    echo "$0: initializing git-merge-changelog driver"
    git config merge.merge-changelog.name 'GNU-style ChangeLog merge driver'
    git config merge.merge-changelog.driver 'git-merge-changelog %O %A %B'
@@ -573,17 +612,21 @@ git_modules_config () {
  test -f .gitmodules && git config --file .gitmodules "$@"
 }

-gnulib_path=$(git_modules_config submodule.gnulib.path)
-test -z "$gnulib_path" && gnulib_path=gnulib
+if $use_git; then
+  gnulib_path=$(git_modules_config submodule.gnulib.path)
+  test -z "$gnulib_path" && gnulib_path=gnulib
+fi

-# Get gnulib files.
+# Get gnulib files.  Populate $GNULIB_SRCDIR, possibly updating a
+# submodule, for use in the rest of the script.

 case ${GNULIB_SRCDIR--} in
 -)
+  # Note that $use_git is necessarily true in this case.
  if git_modules_config submodule.gnulib.url >/dev/null; then
    echo "$0: getting gnulib files..."
-    git submodule init || exit $?
-    git submodule update || exit $?
+    git submodule init -- "$gnulib_path" || exit $?
+    git submodule update -- "$gnulib_path" || exit $?

  elif [ ! -d "$gnulib_path" ]; then
    echo "$0: getting gnulib files..."
@@ -600,8 +643,8 @@ case ${GNULIB_SRCDIR--} in
  GNULIB_SRCDIR=$gnulib_path
  ;;
 *)
-  # Use GNULIB_SRCDIR as a reference.
-  if test -d "$GNULIB_SRCDIR"/.git && \
+  # Use GNULIB_SRCDIR directly or as a reference.
+  if $use_git && test -d "$GNULIB_SRCDIR"/.git && \
        git_modules_config submodule.gnulib.url >/dev/null; then
    echo "$0: getting gnulib files..."
    if git submodule -h|grep -- --reference > /dev/null; then
@@ -612,13 +655,14 @@ case ${GNULIB_SRCDIR--} in
      # This fallback allows at least git 1.5.5.
      if test -f "$gnulib_path"/gnulib-tool; then
        # Since file already exists, assume submodule init already complete.
-        git submodule update || exit $?
+        git submodule update -- "$gnulib_path" || exit $?
      else
        # Older git can't clone into an empty directory.
        rmdir "$gnulib_path" 2>/dev/null
        git clone --reference "$GNULIB_SRCDIR" \
          "$(git_modules_config submodule.gnulib.url)" "$gnulib_path" \
-          && git submodule init && git submodule update \
+          && git submodule init -- "$gnulib_path" \
+          && git submodule update -- "$gnulib_path" \
          || exit $?
      fi
    fi
@@ -627,18 +671,28 @@ case ${GNULIB_SRCDIR--} in
  ;;
 esac

+# $GNULIB_SRCDIR now points to the version of gnulib to use, and
+# we no longer need to use git or $gnulib_path below here.
+
 if $bootstrap_sync; then
  cmp -s "$0" "$GNULIB_SRCDIR/build-aux/bootstrap" || {
    echo "$0: updating bootstrap and restarting..."
+    case $(sh -c 'echo "$1"' -- a) in
+      a) ignored=--;;
+      *) ignored=ignored;;
+    esac
    exec sh -c \
      'cp "$1" "$2" && shift && exec "${CONFIG_SHELL-/bin/sh}" "$@"' \
-      -- "$GNULIB_SRCDIR/build-aux/bootstrap" \
+      $ignored "$GNULIB_SRCDIR/build-aux/bootstrap" \
      "$0" "$@" --no-bootstrap-sync
  }
 fi

-gnulib_tool=$GNULIB_SRCDIR/gnulib-tool
-<$gnulib_tool || exit $?
+gnulib_tool=$GNULIB_SRCDIR/gnulib-tool.py
+if ! <$gnulib_tool; then
+  gnulib_tool=$GNULIB_SRCDIR/gnulib-tool
+  <$gnulib_tool || exit $?
+fi

 # Get translations.

@@ -648,7 +702,7 @@ download_po_files() {
  echo "$me: getting translations into $subdir for $domain..."
  cmd=$(printf "$po_download_command_format" "$domain" "$subdir")
  eval "$cmd" && return
-  # Fallback to HTTP.
+  # Fallback to HTTPS.
  cmd=$(printf "$po_download_command_format2" "$subdir" "$domain")
  eval "$cmd"
 }
@@ -680,11 +734,10 @@ update_po_files() {
    cksum_file="$ref_po_dir/$po.s1"
    if ! test -f "$cksum_file" ||
        ! test -f "$po_dir/$po.po" ||
-        ! $SHA1SUM -c --status "$cksum_file" \
-            < "$new_po" > /dev/null; then
+        ! $SHA1SUM -c "$cksum_file" < "$new_po" > /dev/null 2>&1; then
      echo "$me: updated $po_dir/$po.po..."
      cp "$new_po" "$po_dir/$po.po" \
-          && $SHA1SUM < "$new_po" > "$cksum_file"
+          && $SHA1SUM < "$new_po" > "$cksum_file" || return
    fi
  done
 }
@@ -740,9 +793,9 @@ symlink_to_dir()
      # Leave any existing symlink alone, if it already points to the source,
      # so that broken build tools that care about symlink times
      # aren't confused into doing unnecessary builds.  Conversely, if the
-      # existing symlink's time stamp is older than the source, make it afresh,
+      # existing symlink's timestamp is older than the source, make it afresh,
      # so that broken tools aren't confused into skipping needed builds.  See
-      # <http://lists.gnu.org/archive/html/bug-gnulib/2011-05/msg00326.html>.
+      # <https://lists.gnu.org/r/bug-gnulib/2011-05/msg00326.html>.
      test -h "$dst" &&
      src_ls=$(ls -diL "$src" 2>/dev/null) && set $src_ls && src_i=$1 &&
      dst_ls=$(ls -diL "$dst" 2>/dev/null) && set $dst_ls && dst_i=$1 &&
@@ -867,7 +920,8 @@ if test $use_libtool = 1; then
  esac
 fi
 echo "$0: $gnulib_tool $gnulib_tool_options --import ..."
-$gnulib_tool $gnulib_tool_options --import $gnulib_modules &&
+$gnulib_tool $gnulib_tool_options --import $gnulib_modules \
+  || die "gnulib-tool failed"

 for file in $gnulib_files; do
  symlink_to_dir "$GNULIB_SRCDIR" $file \
@@ -889,20 +943,21 @@ find "$m4_base" "$source_base" \
  -depth \( -name '*.m4' -o -name '*.[ch]' \) \
  -type l -xtype l -delete > /dev/null 2>&1

+# Invoke autoreconf with --force --install to ensure upgrades of tools
+# such as ylwrap.
+AUTORECONFFLAGS="--verbose --install --force -I $m4_base $ACLOCAL_FLAGS"
+
 # Some systems (RHEL 5) are using ancient autotools, for which the
 # --no-recursive option had not been invented.  Detect that lack and
 # omit the option when it's not supported.  FIXME in 2017: remove this
 # hack when RHEL 5 autotools are updated, or when they become irrelevant.
-no_recursive=
 case $($AUTORECONF --help) in
-  *--no-recursive*) no_recursive=--no-recursive;;
+  *--no-recursive*) AUTORECONFFLAGS="$AUTORECONFFLAGS --no-recursive";;
 esac

 # Tell autoreconf not to invoke autopoint or libtoolize; they were run above.
-echo "running: AUTOPOINT=true LIBTOOLIZE=true " \
-    "$AUTORECONF --verbose --install $no_recursive -I $m4_base $ACLOCAL_FLAGS"
-AUTOPOINT=true LIBTOOLIZE=true \
-    $AUTORECONF --verbose --install $no_recursive -I $m4_base $ACLOCAL_FLAGS \
+echo "running: AUTOPOINT=true LIBTOOLIZE=true $AUTORECONF $AUTORECONFFLAGS"
+AUTOPOINT=true LIBTOOLIZE=true $AUTORECONF $AUTORECONFFLAGS \
  || die "autoreconf failed"

 # Get some extra files from gnulib, overriding existing files.
@@ -971,6 +1026,6 @@ echo "$0: done.  Now you can run './configure'."
 # eval: (add-hook 'write-file-hooks 'time-stamp)
 # time-stamp-start: "scriptversion="
 # time-stamp-format: "%:y-%02m-%02d.%02H"
-# time-stamp-time-zone: "UTC"
+# time-stamp-time-zone: "UTC0"
 # time-stamp-end: "; # UTC"
 # End:
--- a/bootstrap.conf
+++ b/bootstrap.conf
@@ -1,6 +1,5 @@
 # bootstrap.conf - Bootstrap configuration.
-# Copyright (C) 2007, 2008, 2009, 2010, 2011, 2012 Free Software Foundation,
-# Inc.
+# Copyright (C) 2007-2012, 2018 Free Software Foundation, Inc.
 #
 # This file is part of GNU Wget.
 #
@@ -9,13 +8,13 @@
 # the Free Software Foundation, either version 3 of the License, or (at
 # your option) any later version.
 #
-# GNU Wgt is distributed in the hope that it will be useful, but
+# GNU Wget is distributed in the hope that it will be useful, but
 # WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 # General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
-# along with this program.  If not, see `http://www.gnu.org/licenses/'.
+# along with this program.  If not, see `https://www.gnu.org/licenses/'.

 # gnulib library name
 gnulib_name=libgnu
@@ -31,32 +30,57 @@ announce-gen
 base32
 bind
 c-ctype
+c-strcase
+c-strcasestr
 clock-time
 close
 connect
+dirname
 fcntl
+flock
+fnmatch
+fopen
 futimens
 ftello
 getaddrinfo
+getline
 getopt-gnu
 getpass-gnu
 getpeername
 getsockname
+gettext
+gitlog-to-changelog
 git-version-gen
 gnupload
+group-member
+hostent
 ioctl
 iconv
 iconv-h
+inet_ntop
+intprops
+inttypes
+langinfo
+limits-h
+link
 listen
 maintainer-makefile
+mbiter
 mbtowc
+memrchr
 mkdir
 mkstemp
 mkostemp
+nanosleep
+crypto/md2
+crypto/md4
 crypto/md5
 crypto/sha1
 crypto/sha256
-pipe
+crypto/sha512
+open
+pipe-posix
+posix_spawn
 quote
 quotearg
 recv
@@ -64,20 +88,38 @@ regex
 select
 send
 setsockopt
+signal-h
 sigprocmask
 sigpipe
 snprintf
 socket
+spawn-pipe
+stat
 stdbool
-strcasestr
+stdint
+strcase
 strerror_r-posix
+strpbrk
+strptime
 strtok_r
+strtoll
+symlink
+sys_types
+timegm
 tmpdir
+unlink
 unlocked-io
 update-copyright
+libunistring-optional
+unistr/u8-strlen
+unicase/u8-tolower
+utimens
 vasprintf
 vsnprintf
+wcwidth
 write
+xmemdup0
+xstrndup
 "

 gnulib_extra_files="
@@ -103,4 +145,22 @@ perl       5.5
 rsync      -
 tar        -
 xz         -
+flex       -
+gperf      -
 "
+
+bootstrap_post_import_hook ()
+{
+  # We need to patch unicase/special-casing-table.h for gperf < 3.1
+  GPERF_VER=$(gperf --version | {
+    IFS=' .' read x y major minor bugfix _
+    printf "%d%02d%02d" $major $minor $bugfix
+  })
+
+  if [ $GPERF_VER -lt 30100 ]; then
+    echo "Creating lib/unicase/special-casing-table.h for gperf < 3.1"
+     gperf -m 10 lib/unicase/special-casing-table.gperf | \
+       sed -e "s/gl_unicase_special_lookup.*/gl_unicase_special_lookup\ \(const char \*str, size_t len\)/g" \
+         > lib/unicase/special-casing-table.h
+  fi
+}
--- a/build-aux/build_info.pl
+++ b/build-aux/build_info.pl
@@ -2,7 +2,7 @@

 # Generate build_info.c.

-# Copyright (C) 2009, 2010, 2011 Free Software Foundation, Inc.
+# Copyright (C) 2009-2011, 2018 Free Software Foundation, Inc.

 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -137,3 +137,4 @@ __DATA__

 #include "wget.h"
 #include <stdio.h>
+#include "version.h"
--- a/cfg.mk
+++ b/cfg.mk
@@ -1,4 +1,7 @@
-export VC_LIST_EXCEPT_DEFAULT=^(lib/.*|m4/.*|md5/.*|build-aux/.*|src/gettext\.h|.*ChangeLog)$$
+# Explicit syntax-check exceptions.
+exclude_file_name_regexp--sc_trailing_blank = ^doc/annou.*$$
+
+export VC_LIST_EXCEPT_DEFAULT=^(lib/.*|m4/.*|md5/.*|build-aux/.*|src/gettext\.h|.*ChangeLog|tests/certs/.*)$$

 local-checks-to-skip = \
    sc_const_long_option \
@@ -19,10 +22,12 @@ local-checks-to-skip = \
    sc_prohibit_S_IS_definition \
    \
    sc_program_name \
-
+    sc_makefile_at_at_check \
+    sc_cast_of_argument_to_free \
+    sc_prohibit_undesirable_word_seq

 #SHELL=bash -x
 show-vc-list-except:
 	@$(VC_LIST_EXCEPT)

-VC_LIST_ALWAYS_EXCLUDE_REGEX = ^ABOUT-NLS|((msdos|vms)/.*)$$
+VC_LIST_ALWAYS_EXCLUDE_REGEX = ^ABOUT-NLS|((msdos|vms)/.*)|ChangeLog-2014-12-10|/.*.der|^tests/certs/.*$$
--- a/configure.ac
+++ b/configure.ac
@@ -1,6 +1,6 @@
 dnl Template file for GNU Autoconf
-dnl Copyright (C) 1995, 1996, 1997, 2001, 2007, 2008, 2009, 2010, 2011, 2012,
-dnl 2013, 2014 Free Software Foundation, Inc.
+dnl Copyright (C) 1995-1997, 2001, 2007-2014, 2018 Free Software
+dnl Foundation, Inc.

 dnl This program is free software; you can redistribute it and/or modify
 dnl it under the terms of the GNU General Public License as published by
@@ -57,60 +57,108 @@ AC_CANONICAL_HOST
 AC_DEFINE_UNQUOTED([OS_TYPE], "$host_os",
                   [Define to be the name of the operating system.])

+dnl Non-verbose make
+m4_ifdef([AM_SILENT_RULES], [AM_SILENT_RULES([yes])])
+
 dnl
-dnl Process features.
+dnl Process External Libraries
 dnl

-ENABLE_PSL=no
-AC_ARG_WITH(libpsl,
-    AS_HELP_STRING([--without-libpsl],
-                   [disable support for libpsl cookie checking.]),
-    [],
-    [AC_SEARCH_LIBS(psl_builtin, psl,
-                    [ENABLE_PSL=yes; AC_DEFINE([HAVE_LIBPSL], [1], [PSL Support Enabled])],
-                    [AC_MSG_WARN(*** libpsl not found. Falling back to Wget builtin cookie checking.)])
-    ])
+dnl Libpsl: Public Suffix List checking
+AC_ARG_WITH([libpsl],
+  [AS_HELP_STRING([--without-libpsl], [disable support for libpsl cookie checking.])])

-AC_ARG_WITH(ssl,
-[[  --without-ssl           disable SSL autodetection
-  --with-ssl={gnutls,openssl} specify the SSL backend.  GNU TLS is the default.]])
+dnl SSL: Configure SSL backend to use
+AC_ARG_WITH([ssl],
+  [AS_HELP_STRING([--with-ssl={gnutls,openssl}], [specify SSL backend. GNU TLS is the default.])])

-AC_ARG_WITH(zlib,
-[[  --without-zlib          disable zlib ]])
+dnl Zlib: Configure use of zlib for compression
+AC_ARG_WITH([zlib],
+  [AS_HELP_STRING([--without-zlib], [disable zlib.])])

-AC_ARG_ENABLE(opie,
-[  --disable-opie          disable support for opie or s/key FTP login],
-ENABLE_OPIE=$enableval, ENABLE_OPIE=yes)
-test x"${ENABLE_OPIE}" = xyes && AC_DEFINE([ENABLE_OPIE], 1,
-   [Define if you want the Opie support for FTP compiled in.])
+dnl Metalink: Configure use of the Metalink library
+AC_ARG_WITH([metalink],
+  [AS_HELP_STRING([--with-metalink], [enable support for metalinks.])])

-AC_ARG_ENABLE(digest,
-[  --disable-digest        disable support for HTTP digest authorization],
-ENABLE_DIGEST=$enableval, ENABLE_DIGEST=yes)
-test x"${ENABLE_DIGEST}" = xyes && AC_DEFINE([ENABLE_DIGEST], 1,
-   [Define if you want the HTTP Digest Authorization compiled in.])
+dnl C-Ares: Configure use of the c-ares library for DNS lookup
+AC_ARG_WITH(cares, AS_HELP_STRING([--with-cares], [enable support for C-Ares DNS lookup.]), with_cares=$withval, with_cares=no)

-AC_ARG_ENABLE(ntlm,
-[  --disable-ntlm          disable support for NTLM authorization],
-[ENABLE_NTLM=$enableval], [ENABLE_NTLM=auto])
+dnl
+dnl Process features
+dnl

-AC_ARG_ENABLE(debug,
-[  --disable-debug         disable support for debugging output],
-ENABLE_DEBUG=$enableval, ENABLE_DEBUG=yes)
-test x"${ENABLE_DEBUG}" = xyes && AC_DEFINE([ENABLE_DEBUG], 1,
-   [Define if you want the debug output support compiled in.])
+dnl Opie: Support for opie s/key FTP logins
+AC_ARG_ENABLE([opie],
+  [AS_HELP_STRING([--disable-opie], [disable support for opie or s/key FTP login])],
+  [ENABLE_OPIE=$enableval],
+  [ENABLE_OPIE=yes])

-AC_ARG_ENABLE(metalink,
-[  --disable-metalink      disable support for metalink file format],
-ENABLE_METALINK=$enableval, ENABLE_METALINK=no)
-test x"${ENABLE_METALINK}" = xyes && AC_DEFINE([ENABLE_METALINK], 1,
-   [Define if you want the metalink file format support compiled in.])
+AS_IF([test "x$ENABLE_OPIE" = xyes],
+  [AC_DEFINE([ENABLE_OPIE], [1], [Define if you want Opie support for FTP compiled in.])],
+  []
+)

-AC_ARG_ENABLE(race-detection,
-[  --enable-race-detection enable support for detecting race conditions],
-ENABLE_RACES=$enableval, ENABLE_RACES=no)
-test x"${ENABLE_RACES}" = xyes && AC_DEFINE([ENABLE_RACES], 1,
-   [Define if you want runtime Race Detection enabled.])
+
+dnl Digest: Support for HTTP Digest Authentication
+AC_ARG_ENABLE([digest],
+  [AS_HELP_STRING([--disable-digest], [disable support for HTTP digest authorization])],
+  [ENABLE_DIGEST=$enableval],
+  [ENABLE_DIGEST=yes])
+
+AS_IF([test "x$ENABLE_DIGEST" = xyes],
+  [AC_DEFINE([ENABLE_DIGEST], [1], [Define if you want the HTTP Digest Authorization compiled in.])],
+  []
+)
+
+
+dnl NTLM: Support for HTTP NTLM Authentication
+AC_ARG_ENABLE([ntlm],
+  [AS_HELP_STRING([--disable-ntlm], [disable support for NTLM authorization])],
+  [ENABLE_NTLM=$enableval],
+  [ENABLE_NTLM=auto]
+)
+
+
+dnl Debug: Support for printing debugging output
+AC_ARG_ENABLE([debug],
+  [AS_HELP_STRING([--disable-debug], [disable support for debugging output])],
+  [ENABLE_DEBUG=$enableval],
+  [ENABLE_DEBUG=yes])
+
+AS_IF([test "x$ENABLE_DEBUG" = xyes],
+  [AC_DEFINE([ENABLE_DEBUG], [1], [Define if you want the debug output support compiled in.])],
+  []
+)
+
+dnl Valgrind-tests: Should test suite be run under valgrind?
+AC_ARG_ENABLE(valgrind-tests,
+  [AS_HELP_STRING([--enable-valgrind-tests], [enable using Valgrind for tests])],
+  [ENABLE_VALGRIND=$enableval],
+  [ENABLE_VALGRIND=no])
+
+AS_IF([test "x$ENABLE_VALGRIND" != xno], [
+  AC_CHECK_PROG(HAVE_VALGRIND, valgrind, yes, no)
+  AS_IF([test "x$HAVE_VALGRIND" = xyes], [
+    VALGRIND_TESTS="1"
+    AC_SUBST(VALGRIND_TESTS)
+    VALGRIND_INFO="Test suite will be run under Valgrind"
+  ], [
+    VALGRIND_INFO="Valgrind not found"
+  ])
+], [
+  VALGRIND_INFO="Valgrind testing not enabled"
+])
+
+AC_ARG_ENABLE(assert,
+  [AS_HELP_STRING([--enable-assert], [enable assertions in code base])],
+  [ENABLE_ASSERTION=$enableval],
+  [ENABLE_ASSERTION=no]
+)
+
+AS_IF([test "x$ENABLE_ASSERTION" != xyes], [
+  CFLAGS="-DNDEBUG $CFLAGS"
+  ],
+  [])

 dnl
 dnl Find the compiler
@@ -126,11 +174,17 @@ AC_AIX

 gl_EARLY

+dnl
+dnl Find python3
+dnl
+AM_PATH_PYTHON([3.0],,[:])
+AM_CONDITIONAL([HAVE_PYTHON3], [test "$PYTHON" != :])
+
 dnl
 dnl Gettext
 dnl
 AM_GNU_GETTEXT([external],[need-ngettext])
-AM_GNU_GETTEXT_VERSION([0.17])
+AM_GNU_GETTEXT_VERSION([0.18.1])

 AC_PROG_RANLIB

@@ -143,9 +197,11 @@ dnl   if compiler is gcc, then
 dnl     use -O2 and some warning flags
 dnl   else
 dnl     use os-specific flags or -O
+dnl
+dnl For these tests, the system considers clang to be gcc
 if test -n "$auto_cflags"; then
  if test -n "$GCC"; then
-    CFLAGS="$CFLAGS -O2 -Wall"
+    CFLAGS="$CFLAGS -O2 -Wall -Wextra"
  else
    case "$host_os" in
      *hpux*)  CFLAGS="$CFLAGS +O3"                      ;;
@@ -219,7 +275,7 @@ AC_FUNC_MMAP
 AC_FUNC_FSEEKO
 AC_CHECK_FUNCS(strptime timegm vsnprintf vasprintf drand48 pathconf)
 AC_CHECK_FUNCS(strtoll usleep ftello sigblock sigsetjmp memrchr wcwidth mbtowc)
-AC_CHECK_FUNCS(sleep symlink utime)
+AC_CHECK_FUNCS(sleep symlink utime strlcpy random)

 if test x"$ENABLE_OPIE" = xyes; then
  AC_LIBOBJ([ftp-opie])
@@ -234,15 +290,6 @@ AC_DEFINE([HAVE_STRNCASECMP], 1, [Define to 1 if you have the `strncasecmp' func
 AC_DEFINE([HAVE_STRDUP], 1, [Define to 1 if you have the `strdup' function.])
 AC_DEFINE([HAVE_ISATTY], 1, [Define to 1 if you have the `isatty' function.])

-dnl
-dnl Call Wget-specific macros defined in aclocal.
-dnl
-WGET_STRUCT_UTIMBUF
-WGET_FNMATCH
-WGET_NANOSLEEP
-WGET_POSIX_CLOCK
-WGET_NSL_SOCKET
-
 dnl Deal with specific hosts
 case $host_os in
  *mingw32* )
@@ -256,48 +303,88 @@ dnl
 dnl Checks for libraries.
 dnl

+PKG_PROG_PKG_CONFIG
+
+AS_IF([test "x$with_libpsl" != xno], [
+  PKG_CHECK_MODULES([LIBPSL], libpsl, [
+    with_libpsl=yes
+    # correct $LIBPSL_LIBS (in libpsl <= 0.6.0)
+    AS_IF([test "x$LIBPSL_LIBS" = "x-llibpsl "], [LIBPSL_LIBS="-lpsl"])
+    LIBS="$LIBPSL_LIBS $LIBS"
+    CFLAGS="$LIBPSL_CFLAGS $CFLAGS"
+    AC_DEFINE([HAVE_LIBPSL], [1], [PSL support enabled])
+  ], [
+    AC_SEARCH_LIBS(psl_builtin, psl,
+      [with_libpsl=yes; AC_DEFINE([HAVE_LIBPSL], [1], [PSL support enabled])],
+      [with_libpsl=no;  AC_MSG_WARN(*** libpsl was not found. Fallback to builtin cookie checking.)])
+  ])
+])
+
+AS_IF([test x"$with_libpsl" = xyes], [
+  AC_CHECK_FUNCS(psl_latest)
+])
+
 AS_IF([test x"$with_zlib" != xno], [
  with_zlib=yes
-  AC_CHECK_LIB(z, compress)
+  PKG_CHECK_MODULES([ZLIB], zlib, [
+    LIBS="$ZLIB_LIBS $LIBS"
+    CFLAGS="$ZLIB_CFLAGS $CFLAGS"
+    AC_DEFINE([HAVE_LIBZ], [1], [Define if using zlib.])
+  ], [
+    AC_CHECK_LIB(z, compress)
+  ])
 ])

 AS_IF([test x"$with_ssl" = xopenssl], [
-  dnl As of this writing (OpenSSL 0.9.6), the libcrypto shared library
-  dnl doesn't record its dependency on libdl, so we need to make sure
-  dnl -ldl ends up in LIBS on systems that have it.  Most OSes use
-  dnl dlopen(), but HP-UX uses shl_load().
-  AC_CHECK_LIB(dl, dlopen, [], [
-    AC_CHECK_LIB(dl, shl_load)
-  ])
+  if [test x"$with_libssl_prefix" = x]; then
+    PKG_CHECK_MODULES([OPENSSL], [openssl], [
+      AC_MSG_NOTICE([compiling in support for SSL via OpenSSL])
+      AC_LIBOBJ([openssl])
+      LIBS="$OPENSSL_LIBS $LIBS"
+      CFLAGS="$OPENSSL_CFLAGS -DHAVE_LIBSSL $CFLAGS"
+      LIBSSL=" " # ntlm check below wants this
+      AC_CHECK_FUNCS([RAND_egd])
+      AC_DEFINE([HAVE_LIBSSL], [1], [Define if using openssl.])
+      ssl_found=yes
+    ])
+  fi
+  if [test x"$ssl_found" != xyes]; then
+    dnl As of this writing (OpenSSL 0.9.6), the libcrypto shared library
+    dnl doesn't record its dependency on libdl, so we need to make sure
+    dnl -ldl ends up in LIBS on systems that have it.  Most OSes use
+    dnl dlopen(), but HP-UX uses shl_load().
+    AC_CHECK_LIB(dl, dlopen, [], [
+      AC_CHECK_LIB(dl, shl_load)
+    ])

-  ssl_found=no
-  case $host_os in
-    *mingw32* )
-      dnl prefer link to openssl dlls if possible. if not then fallback on static libs. if not then error
+    ssl_found=no
+    case $host_os in
+      *mingw32* )
+        dnl prefer link to openssl dlls if possible. if not then fallback on static libs. if not then error

-      AC_CHECK_LIB(eay32, EVP_MD_CTX_init)
-      if test x"$ac_cv_lib_eay32_EVP_MD_CTX_init" != xno
-      then
-        AC_CHECK_LIB(ssl32, SSL_connect, [
-          ssl_found=yes
-          AC_MSG_NOTICE([Enabling support for SSL via OpenSSL (shared)])
-          AC_LIBOBJ([openssl])
-          LIBS="${LIBS} -lssl32"
-          AC_DEFINE([HAVE_LIBSSL32], [1], [Define to 1 if you have the `ssl32' library (-lssl32).])
-        ],
-        AC_MSG_ERROR([openssl not found: shared lib eay32 found but ssl32 not found]))
+        AC_CHECK_LIB(eay32, EVP_MD_CTX_init)
+        if test x"$ac_cv_lib_eay32_EVP_MD_CTX_init" != xno
+        then
+          AC_CHECK_LIB(ssl32, SSL_connect, [
+            ssl_found=yes
+            AC_MSG_NOTICE([Enabling support for SSL via OpenSSL (shared)])
+            AC_LIBOBJ([openssl])
+            LIBS="${LIBS} -lssl32"
+            AC_DEFINE([HAVE_LIBSSL32], [1], [Define to 1 if you have the `ssl32' library (-lssl32).])
+          ],
+          AC_MSG_ERROR([openssl not found: shared lib eay32 found but ssl32 not found]))

-      else
-        LIBS+=' -lgdi32'
-        dnl fallback and test static libs
-      fi
-      dnl add zdll lib as dep for above tests?
-    ;;
-  esac
+        else
+          LIBS+=' -lgdi32'
+          dnl fallback and test static libs
+        fi
+        dnl add zdll lib as dep for above tests?
+      ;;
+    esac

-  AS_IF([test x$ssl_found != xyes], [
-    dnl Now actually check for -lssl if it wasn't already found
-    AC_LIB_HAVE_LINKFLAGS([ssl], [crypto], [
+    AS_IF([test x$ssl_found != xyes], [
+      dnl Now actually check for -lssl if it wasn't already found
+      AC_LIB_HAVE_LINKFLAGS([ssl], [crypto], [
 #include <openssl/ssl.h>
 #include <openssl/x509.h>
 #include <openssl/err.h>
@@ -305,19 +392,20 @@ AS_IF([test x"$with_ssl" = xopenssl], [
 #include <openssl/des.h>
 #include <openssl/md4.h>
 #include <openssl/md5.h>
-    ], [SSL_library_init ()])
-    if test x"$LIBSSL" != x
-    then
-      ssl_found=yes
-      AC_MSG_NOTICE([compiling in support for SSL via OpenSSL])
-      AC_LIBOBJ([openssl])
-      LIBS="$LIBSSL $LIBS"
-    elif test x"$with_ssl" != x
-    then
-      AC_MSG_ERROR([--with-ssl=openssl was given, but SSL is not available.])
-    fi
-  ])
-
+      ], [SSL_library_init ()])
+      if test x"$LIBSSL" != x
+      then
+        ssl_found=yes
+        AC_MSG_NOTICE([compiling in support for SSL via OpenSSL])
+        AC_LIBOBJ([openssl])
+        LIBS="$LIBSSL $LIBS"
+        AC_CHECK_FUNCS([RAND_egd])
+      elif test x"$with_ssl" != x
+      then
+        AC_MSG_ERROR([--with-ssl=openssl was given, but SSL is not available.])
+      fi
+    ])
+  fi
 ], [
  # --with-ssl is not openssl: check if it's no
  AS_IF([test x"$with_ssl" != xno], [
@@ -325,17 +413,29 @@ AS_IF([test x"$with_ssl" = xopenssl], [
    with_ssl=gnutls

    dnl Now actually check for -lgnutls
-    AC_LIB_HAVE_LINKFLAGS([gnutls], [], [
+    if [test x"$with_libgnutls_prefix" = x]; then
+      PKG_CHECK_MODULES([GNUTLS], [gnutls], [
+        AC_MSG_NOTICE([compiling in support for SSL via GnuTLS])
+        AC_LIBOBJ([gnutls])
+        LIBS="$GNUTLS_LIBS $LIBS"
+        CFLAGS="$GNUTLS_CFLAGS -DHAVE_LIBGNUTLS $CFLAGS"
+        AC_DEFINE([HAVE_LIBGNUTLS], [1], [Define if using gnutls.])
+        ssl_found=yes
+      ])
+    fi
+    if [test x"$ssl_found" != xyes]; then
+      AC_LIB_HAVE_LINKFLAGS([gnutls], [], [
 #include <gnutls/gnutls.h>
-    ], [gnutls_global_init()])
-    if test x"$LIBGNUTLS" != x
-    then
-      ssl_found=yes
-      AC_MSG_NOTICE([compiling in support for SSL via GnuTLS])
-      AC_LIBOBJ([gnutls])
-      LIBS="$LIBGNUTLS $LIBS"
-    else
-      AC_MSG_ERROR([--with-ssl=gnutls was given, but GNUTLS is not available.])
+      ], [gnutls_global_init()])
+      if test x"$LIBGNUTLS" != x
+      then
+        ssl_found=yes
+        AC_MSG_NOTICE([compiling in support for SSL via GnuTLS])
+        AC_LIBOBJ([gnutls])
+        LIBS="$LIBGNUTLS $LIBS"
+      else
+        AC_MSG_ERROR([GnuTLS has not been found. Use --with-ssl=openssl if you explicitly want OpenSSL.])
+      fi
    fi

    AC_CHECK_FUNCS(gnutls_priority_set_direct)
@@ -377,74 +477,34 @@ else
  fi
 fi

-AS_IF([test x"$ENABLE_THREADS" != xno],[
-  AC_CHECK_LIB(pthread, pthread_create, [
-dnl    AC_CHECK_LIB(semaphore, sem_init, [
-      threads=yes
-      LIBS="${LIBS} -lpthread"
-      AC_DEFINE([ENABLE_THREADS], 1,
-        [Define if you want the thread support compiled in.])
-      AC_MSG_NOTICE([compiling with support for threads])
-dnl    ],[
-dnl      threads=no
-dnl      if test x"$ENABLE_THREADS" == xyes
-dnl      then
-dnl        AC_MSG_ERROR([Could not find the sem_init function in semaphore.])
-dnl      fi
-dnl    ])
-  ],[
-      threads=no
-      if test x"$ENABLE_THREADS" == xyes
-      then
-        AC_MSG_ERROR([Could not find the pthread_create function in pthread.])
-      fi
-  ])
-])
-
-AS_IF([test x"$ENABLE_RACES" != xno],[
-   CFLAGS="$CFLAGS -fsanitize=thread -fPIE";
-   LDFLAGS="$LDFLAGS -fsanitize=thread -pie";
-])
-
-AS_IF([test x"$ENABLE_METALINK" != xno],[
-  dnl If metalink support is requested
-  PKG_CHECK_MODULES(LIBMETALINK, libmetalink, [
-    dnl If metalink module is found
-    LIBS="$LIBMETALINK_LIBS $LIBS"
-    CFLAGS="$LIBMETALINK_CFLAGS $CFLAGS"
-    AC_DEFINE([ENABLE_METALINK], 1,
-     [Define if you want the metalink support compiled in.])
-
-    AC_CHECK_LIB(metalink, metalink_parse_file, [
-      if test x"$DISABLE_THREADS" == xyes
-      then
-        AC_MSG_ERROR([Metalink support depends on threads, which are requested to be disabled.])
-      elif test "X$threads" != "Xyes"
-      then
-        AC_MSG_ERROR([Metalink support depends on threads, support for which could not be enabled.])
-      else
-        metalink=yes
-        AC_MSG_NOTICE([compiling in support for metalink])
-      fi
+dnl
+dnl Check for libmetalink
+dnl
+AS_IF([test x"$with_metalink" != xno], [
+  PKG_CHECK_MODULES([METALINK], libmetalink, [
+      LIBS="$METALINK_LIBS $LIBS"
+      CFLAGS="$METALINK_CFLAGS $CFLAGS"
+      AC_DEFINE([HAVE_METALINK], [1], [Define if using metalink.])
+      with_metalink=yes
    ], [
-      if test x"$ENABLE_METALINK" == xyes
-      then
-        AC_MSG_ERROR([Could not find the metalink_parse_file function in libmetalink.])
-      fi
+      with_metalink=no
    ])
-  ], [
-    dnl If metalink module is not found
-    if test x"$ENABLE_METALINK" == xyes
-    then
-      AC_MSG_ERROR([libmetalink not found])
-    fi
-  ])
-],[
-  dnl If metalink support is not requested
-  metalink=no
-  AC_MSG_NOTICE([compiling without support for metalink])
-])

+  have_gpg=no
+  AS_IF([test x"$with_metalink" = xyes], [
+    dnl
+    dnl Check for GPGME
+    dnl
+    m4_ifdef([AM_PATH_GPGME], [
+      AM_PATH_GPGME([], [
+        LIBS="$GPGME_LIBS $LIBS"
+        CFLAGS="$GPGME_CFLAGS $CFLAGS"
+        AC_DEFINE([HAVE_GPGME], [1], [Define if GPGME is available.])
+        have_gpg=yes
+      ])
+    ])
+  ])
+])

 dnl **********************************************************************
 dnl Checks for IPv6
@@ -570,9 +630,6 @@ AC_ARG_ENABLE(iri,
  ]
 )

-AC_ARG_WITH(libidn, AC_HELP_STRING([--with-libidn=[DIR]],
-                                   [Support IDN/IRIs (needs GNU Libidn)]),
-                                   libidn=$withval, libidn="")
 AS_IF([test "X$iri" != "Xno"],[
  AM_ICONV

@@ -584,84 +641,151 @@ AS_IF([test "X$iri" != "Xno"],[
      AC_MSG_NOTICE([disabling IRIs because libiconv wasn't found])
    fi
  fi
-],[  # else
-  # For some reason, this seems to be set even when we don't check.
-  # Explicitly unset.
-  LIBICONV=
 ])

+AC_ARG_WITH(libidn, AC_HELP_STRING([--with-libidn=[DIR]],
+                                   [Support IDN2008/IRIs (needs GNU libidn2 + libunicode)]),
+                                   libidn=$withval, libidn="")
 if test "X$iri" != "Xno"; then
-  if test "$libidn" != ""; then
-    LDFLAGS="${LDFLAGS} -L$libidn/lib"
-    CPPFLAGS="${CPPFLAGS} -I$libidn/include"
-  fi
+  AS_IF([test "x$with_libidn2" != xno], [
+    AC_SEARCH_LIBS(idn2_lookup_u8, idn2,
+      [with_libidn2=yes; AC_DEFINE([ENABLE_IRI], 1, [Define if IRI support is enabled.])],
+      [with_libidn2=no; iri=no; AC_MSG_WARN(*** LIBIDN2 was not found. You will not be able to use IDN2008 support)])

-  # If idna.h can't be found, check to see if it was installed under
-  # /usr/include/idn (OpenSolaris, at least, places it there).
-  # Check for idn-int.h in that case, because idna.h won't find
-  # idn-int.h until we've decided to add -I/usr/include/idn.
-  AC_CHECK_HEADER(idna.h, ,
-    [AC_CHECK_HEADER(idn/idn-int.h,
-                    [CPPFLAGS="${CPPFLAGS} -I/usr/include/idn"],
-                    [iri=no])]
-  )
-
-  if test "X$iri" != "Xno"; then
-    AC_CHECK_LIB(idn, stringprep_check_version,
-      [iri=yes LIBS="${LIBS} -lidn"], iri=no)
-  fi
-
-  if test "X$iri" != "Xno" ; then
-    AC_DEFINE([ENABLE_IRI], 1, [Define if IRI support is enabled.])
-    AC_MSG_NOTICE([Enabling support for IRI.])
-  else
-    AC_MSG_WARN([Libidn not found])
-  fi
+#    AS_IF([test "x$with_libidn2" = xyes], [
+#      AC_SEARCH_LIBS(u8_tolower, unistring,
+#        [AC_DEFINE([ENABLE_IRI], 1, [Define if IRI support is enabled.])],
+#        [iri=no; AC_MSG_WARN(*** LIBUNISTRING was not found. You will not be able to use IDN2008 support)])
+#    ])
+  ])
 fi

 dnl
 dnl Check for UUID
 dnl
-AC_CHECK_HEADER(uuid.h,
-                AC_CHECK_FUNC(uuid, uuid_create,
-                  [AC_DEFINE([HAVE_UUID_CREATE], 1,
-                             [Define if uuid_create is available.])
-                  ])
-)

-AC_CHECK_HEADER(uuid/uuid.h,
-                AC_CHECK_LIB(uuid, uuid_generate,
-                  [LIBS="${LIBS} -luuid"
-                   AC_DEFINE([HAVE_LIBUUID], 1,
-                             [Define if libuuid is available.])
-                  ])
-)
+AC_ARG_WITH(libuuid, AC_HELP_STRING([--without-libuuid],
+                                    [Generate UUIDs for WARC files via libuuid]))
+
+AS_IF([test "x$with_libuuid" = xyes], [
+  # libuuid was explicitly requested
+  PKG_CHECK_MODULES([UUID], uuid, [
+    LIBS="$UUID_LIBS $LIBS"
+    CFLAGS="$UUID_CFLAGS $CFLAGS"
+    uuid_mode=1
+  ], [
+    AC_SEARCH_LIBS(uuid_generate, uuid,
+      [uuid_mode=1],
+      [AC_MSG_ERROR(*** libuuid was explicitly requested but wasn't found.)])
+  ])
+], [test "x$with_libuuid" = xno], [
+  # libuuid was explicitly *not* requested
+  AC_CHECK_HEADER(uuid.h,
+    AC_CHECK_FUNC(uuid_create, [uuid_mode=2]))
+], [
+  # default:
+  AC_CHECK_HEADER(uuid.h, [
+    AC_CHECK_FUNC(uuid_create, [uuid_mode=2])
+  ], [
+    AC_CHECK_HEADER(uuid/uuid.h,
+      AC_SEARCH_LIBS(uuid_generate, uuid, [uuid_mode=1]))
+  ])
+])
+
+AS_IF([test "x$uuid_mode" = x1], [
+  AC_DEFINE([HAVE_LIBUUID], [1], [Define if using libuuid.])
+], [test "x$uuid_mode" = x2], [
+  AC_DEFINE([HAVE_UUID_CREATE], 1, [Define if uuid_create is available.])
+])

 dnl
 dnl Check for PCRE
 dnl

-AC_CHECK_HEADER(pcre.h,
-                AC_CHECK_LIB(pcre, pcre_compile,
-                  [LIBS="${LIBS} -lpcre"
-                   AC_DEFINE([HAVE_LIBPCRE], 1,
-                             [Define if libpcre is available.])
-                  ])
-)
+AC_ARG_ENABLE(pcre, AC_HELP_STRING([--disable-pcre],
+                                   [Disable PCRE style regular expressions]))

+AS_IF([test "X$enable_pcre" != "Xno"],[
+  PKG_CHECK_MODULES([PCRE], libpcre, [
+    CFLAGS="$PCRE_CFLAGS $CFLAGS"
+    AC_CHECK_HEADER(pcre.h, [
+      LIBS="$PCRE_LIBS $LIBS"
+      AC_DEFINE([HAVE_LIBPCRE], [1], [Define if libpcre is available.])
+    ])
+  ], [
+    AC_CHECK_HEADER(pcre.h, [
+      AC_CHECK_LIB(pcre, pcre_compile, [
+        LIBS="${LIBS} -lpcre"
+        AC_DEFINE([HAVE_LIBPCRE], 1, [Define if libpcre is available.])
+      ])
+    ])
+  ])
+])
+
+dnl
+dnl Check for libcares (resolver library)
+dnl
+
+AS_IF([test "X$with_cares" = "Xyes"],[
+  PKG_CHECK_MODULES([CARES], libcares, [
+    CFLAGS="$CARES_CFLAGS $CFLAGS"
+    AC_CHECK_HEADER(ares.h, [
+      LIBS="$CARES_LIBS $LIBS"
+      AC_DEFINE([HAVE_LIBCARES], [1], [Define if libcares is available.])
+      RESOLVER_INFO="libcares, --bind-dns-address and --dns-servers available"
+    ])
+  ], [
+    AC_CHECK_HEADER(ares.h, [
+      AC_CHECK_LIB(cares, ares_set_local_ip4, [
+        LIBS="-lcares ${LIBS}"
+        AC_DEFINE([HAVE_LIBCARES], 1, [Define if libcares is available.])
+        RESOLVER_INFO="libcares, --bind-dns-address and --dns-servers available"
+      ])
+    ])
+  ])
+], [
+  RESOLVER_INFO="libc, --bind-dns-address and --dns-servers not available"
+])
+
+dnl
+dnl Extended Attribute support
+dnl
+
+AC_ARG_ENABLE([xattr],
+  [AS_HELP_STRING([--disable-xattr], [disable support for POSIX Extended Attributes])],
+  [ENABLE_XATTR=$enableval],
+  [ENABLE_XATTR=yes])
+
+case "$host_os" in
+  *linux* | *darwin*) xattr_syscalls="fsetxattr" ;;
+  freebsd*)           xattr_syscalls="extattr_set_fd" ;;
+  *)  AC_MSG_NOTICE([Disabling Extended Attribute support: your system is not known to support extended attributes.])
+      ENABLE_XATTR=no
+esac
+
+if test "X${ENABLE_XATTR}" = "Xyes"; then
+  AC_CHECK_FUNCS([$xattr_syscalls], [], [
+    AC_MSG_NOTICE([Disabling Extended Attribute support: your system does not support $xattr_syscalls])
+    ENABLE_XATTR=no
+  ])
+fi
+
+test "X${ENABLE_XATTR}" = "Xyes" && AC_DEFINE([ENABLE_XATTR], 1,
+    [Define if you want file meta-data storing into POSIX Extended Attributes compiled in.])

 dnl Needed by src/Makefile.am
-AM_CONDITIONAL([IRI_IS_ENABLED], [test "X$iri" == "Xyes"])
-AM_CONDITIONAL([METALINK_IS_ENABLED], [test "X$metalink" == "Xyes"])
-AM_CONDITIONAL([THREADS_ARE_ENABLED], [test "X$threads" == "Xyes"])
-AM_CONDITIONAL([RACE_CHECKING_IS_ENABLED], [test "X$race-detection" == "Xyes"])
+AM_CONDITIONAL([IRI_IS_ENABLED], [test "X$iri" != "Xno"])
+AM_CONDITIONAL([WITH_SSL], [test "X$with_ssl" != "Xno"])
+AM_CONDITIONAL([METALINK_IS_ENABLED], [test "X$with_metalink" != "Xno"])
+AM_CONDITIONAL([WITH_XATTR], [test "X$ENABLE_XATTR" != "Xno"])

 dnl
 dnl Create output
 dnl
 AC_CONFIG_FILES([Makefile src/Makefile doc/Makefile util/Makefile
-                 po/Makefile.in tests/Makefile tests/WgetTest.pm
-                 lib/Makefile testenv/Makefile])
+                 po/Makefile.in tests/Makefile
+                 lib/Makefile testenv/Makefile
+                 tests/certs/interca.conf tests/certs/rootca.conf])
 AC_CONFIG_HEADERS([src/config.h])
 AC_OUTPUT

@@ -676,11 +800,16 @@ AC_MSG_NOTICE([Summary of build options:
  Libs:              $LIBS
  SSL:               $with_ssl
  Zlib:              $with_zlib
-  PSL:               $ENABLE_PSL
+  PSL:               $with_libpsl
  Digest:            $ENABLE_DIGEST
  NTLM:              $ENABLE_NTLM
  OPIE:              $ENABLE_OPIE
-  Metalink:          $ENABLE_METALINK
+  POSIX xattr:       $ENABLE_XATTR
  Debugging:         $ENABLE_DEBUG
-  Race Detection:    $ENABLE_RACES
+  Assertions:        $ENABLE_ASSERTION
+  Valgrind:          $VALGRIND_INFO
+  Metalink:          $with_metalink
+  Resolver:          $RESOLVER_INFO
+  GPGME:             $have_gpg
+  IRI:               $iri
 ])
--- a/contrib/check-hard
+++ b/contrib/check-hard
@@ -0,0 +1,55 @@
+#!/bin/sh
+#
+# Do some checking before 'git push'.
+
+# Set a stricter bash mode
+set -e
+set -u
+
+CORES=$(grep -c ^processor /proc/cpuinfo 2> /dev/null || echo 0)
+if [ $CORES -lt 1 ]; then CORES=1; fi
+
+echo "Running: make distclean"
+make distclean > /dev/null || true
+
+#CFLAGS="-std=c89 -pedantic -O0 -g -Wall -Wextra -Wstrict-prototypes -Wold-style-definition -Wwrite-strings -Wshadow -Wformat -Wformat-security -Wunreachable-code -Wstrict-prototypes -Wmissing-prototypes -Wold-style-definition"
+CFLAGS="-Wall -Wextra -O2 -g"
+
+CACHEFILE=$PWD/config_check.cache
+
+# measure time consumed and print it at the end of the script
+START=$(date +%s.%N)
+
+echo "Running: ./bootstrap"
+./bootstrap > /dev/null
+echo
+
+for CC in gcc clang; do
+  export CC
+  echo "*** Testing with CC=$CC"
+
+  # the compiler changed, so we have to remove the cache file here
+  rm -f $CACHEFILE
+
+  for options in "" "--with-ssl=openssl" "--disable-iri"; do
+    export DISTCHECK_CONFIGURE_FLAGS="-C --cache-file=$CACHEFILE $options"
+    echo "  ./configure $DISTCHECK_CONFIGURE_FLAGS"
+    ./configure $DISTCHECK_CONFIGURE_FLAGS CFLAGS="$CFLAGS" > /dev/null
+
+    for xVALGRIND in 0 1; do
+      for xLCALL in C tr_TR.utf8; do
+        export TESTS_ENVIRONMENT="LC_ALL=$xLCALL VALGRIND_TESTS=$xVALGRIND"
+        echo "    TESTS_ENVIRONMENT=\"$TESTS_ENVIRONMENT\"" make check -j$CORES
+        make check -j$CORES > /dev/null
+      done
+    done
+
+    unset TESTS_ENVIRONMENT
+    export TESTS_ENVIRONMENT
+    echo "  make distcheck CFLAGS=$CFLAGS -j$CORES"
+    make distcheck CFLAGS="$CFLAGS" -j$CORES > /dev/null
+  done
+done
+
+END=$(date +%s.%N)
+echo "Duration: "$(echo "$END - $START" | bc)
--- a/contrib/release
+++ b/contrib/release
@@ -0,0 +1,123 @@
+#!/bin/bash -e
+#
+# Written 2016 by Tim Ruehsen
+#
+# To get the version number right: add a signed tag and
+#   make sure the working dir is clean (git diff).
+# If that fails, put the version into '.tarball-version'.
+
+export CFLAGS="-g -O2 -Wall"
+
+echo
+echo "* Bootstrap"
+#./bootstrap >/dev/null
+
+echo
+echo "* Configure"
+#./configure -q
+
+extensions="gz lz"
+app_name=$(grep "^PACKAGE_NAME='" configure|cut -d "'" -f2)
+app_ver=$(grep "^PACKAGE_VERSION='" configure|cut -d "'" -f2)
+
+echo "Current version of $app_name is $app_ver"
+echo
+echo -n "Is that OK ? [y/N] "
+read x
+[ "$x" != "y" -a "$x" != "Y" ] && exit 0
+
+echo
+grep -i "Changes in $app_name" NEWS|head -1
+echo -n "Did you edit the NEWS file ? [y/N] "
+read x
+[ "$x" != "y" -a "$x" != "Y" ] && exit 0
+
+echo
+echo "* Make"
+#make -s clean
+#make -s
+
+echo
+echo "* Make distcheck"
+#make -s distcheck >/dev/null
+
+echo
+echo "* Make dist"
+for ext in $extensions; do
+  [[ $ext = 'gz' ]] && ext="gzip"
+  [[ $ext = 'lz' ]] && ext="lzip"
+  make -s dist-$ext >/dev/null
+done
+
+while true; do
+  echo "Alpha or Regular release ? [R/A] "
+  read x
+  if test "$x" = "r" || test "$x" = "R"; then
+    directory="/incoming/ftp"
+    break
+  elif test "$x" = "a" || test "$x" = "A"; then
+    directory="/incoming/alpha"
+    break
+  fi
+done
+
+x=$(gpg --list-secret-keys 2>/dev/null|grep ^sec|awk '{ print $2 }')
+echo $x
+if [ "$x" = "" ]; then
+  echo No signing key found - please check with 'gpg --list-secret-keys'
+  exit 1
+fi
+
+# arrays just work in Bash
+keys=($x)
+if [ ${#keys[@]} -eq 1 ]; then
+  IFS='/' arr=($x)
+  signing_key=${arr[1]}
+else
+  for ((i=0; i < ${#keys[@]}; i++)); do
+    echo "$((i+1))) ${keys[i]}"
+  done
+  while true; do
+    echo -n "Enter 1-${#keys[@]}: "
+    read x
+
+    # check for digits only
+    if [ -n "${i//[0-9]/}" ]; then continue; fi
+
+    if test $x -ge 1 || test $x -le ${#keys[@]}; then
+      IFS='/' arr=(${keys[$((x-1))]})
+      signing_key=${arr[1]}
+      break
+    fi
+  done
+fi
+unset IFS
+
+echo
+echo -n "Really upload to ftp-upload.gnu.org/$directory ? "
+read x
+[ "$x" != "y" -a "$x" != "Y" ] && exit 0
+
+for ext in $extensions; do
+  app_tarball="${app_name}-${app_ver}.tar.$ext"
+  gpg --default-key $signing_key --yes -b $app_tarball
+
+  echo -e "version: 1.2\nfilename: ${app_tarball}\ndirectory: ${app_name}\n" >${app_tarball}.directive
+  gpg --default-key $signing_key --yes --clearsign ${app_tarball}.directive
+
+ftp -v -n -i -p ftp-upload.gnu.org <<EOF
+user anonymous
+cd $directory
+put ${app_tarball}
+put ${app_tarball}.sig
+put ${app_tarball}.directive.asc
+EOF
+
+done
+
+prev_tag=`git tag|tail -2|head -1`
+echo "Don't forget to mention the following contributors in the announcement:"
+(
+  git log ${prev_tag}.. --format="%aN" --reverse
+  git log ${prev_tag}.. --reverse|grep -i reported|cut -d':' -f2|cut -d'<' -f1|cut -d' ' -f2-|tr -d '"'
+) | sort -u
--- a/contrib/travis-ci
+++ b/contrib/travis-ci
@@ -0,0 +1,60 @@
+#!/bin/bash
+
+# Set a stricter bash mode
+set -e
+set -u
+
+# Set up the subset of tests this Matrix instance will run.
+
+# One instance is forked for each of the SSL library possibilities
+SSL=${1:-""}
+case $SSL in
+    "") SSL_LIB="--without-ssl";;
+    "openssl") SSL_LIB="--with-ssl=openssl";;
+    "gnutls") SSL_LIB="--with-ssl=gnutls";;
+esac
+
+# For some reason. /proc/cpuinfo reports 16 cores on Travis, while the docs
+# claim that each instance has only 2 cores. We believe the docs and force a
+# value of n+1 here.
+CORES=3
+
+# Define a large number of Warning flags for the compiler. Hopefully, someone
+# will sit and analyze the output to clean the warnings from the codebase.
+CFLAGS="-std=c89 -pedantic -O0 -Wall -Wextra -Wstrict-prototypes -Wold-style-definition -Wwrite-strings -Wshadow -Wformat -Wformat-security -Wunreachable-code -Wstrict-prototypes -Wmissing-prototypes -Wold-style-definition"
+
+# A cachefile for ./configure. Saves significant time when reconfiguring
+CACHEFILE=$PWD/config_check.cache
+
+# measure time consumed and print it at the end of the script
+START=$(date +%s)
+
+# Test different Permutations of Wget. Always under Valgrind
+#
+# Options:
+#   - All Features Enabled
+#     - C       [Default Locale]
+#     - Russian [Multibyte / Multicolumn Locale]
+#     - Turkish [Special Test due to issues with language]
+#   - IPv6 Disabled
+#     - C
+for options in "" "--disable-ipv6"; do
+  export DISTCHECK_CONFIGURE_FLAGS="-C --cache-file=$CACHEFILE --enable-assert --enable-valgrind-tests $SSL_LIB $options"
+  echo "  ./configure $DISTCHECK_CONFIGURE_FLAGS CFLAGS=\"$CFLAGS\""
+  ./configure $DISTCHECK_CONFIGURE_FLAGS CFLAGS="$CFLAGS"
+
+  if [[ -z $options ]]; then
+      LANG_ARR=("ru_RU.UTF-8" "tr_TR.UTF-8" "C")
+  else
+      LANG_ARR=("C")
+  fi
+
+  for xLCALL in "${LANG_ARR[@]}"; do
+    export TESTS_ENVIRONMENT="LC_ALL=$xLCALL VALGRIND_TESTS=1"
+    echo "    TESTS_ENVIRONMENT=\"$TESTS_ENVIRONMENT\" make distcheck CFLAGS=$CFLAGS -j$CORES"
+    make distcheck CFLAGS="$CFLAGS" -j$CORES
+  done
+done
+
+END=$(date +%s)
+echo "Duration: $((END-START))"
--- a/contrib/tsocked-wget
+++ b/contrib/tsocked-wget
@@ -0,0 +1,90 @@
+#!/bin/bash
+
+# Script that executes wget using a socks proxy if the environment variable
+# socks_proxy is set.
+#
+# The socks_proxy variable shall have one of the forms:
+#   socks://username:password@host:port
+#   socks4://username:password@host:port
+#   socks5://username:password@host:port
+# with username, password and port fields being optional
+#
+# As socksification applies to the whole process, domains defined in the
+# no_proxy setting are *not* excluded.
+#
+
+# Requisites: bash (version 3.2.3 or later), tsocks and wget
+
+####################################
+#  Copyright (C) 2014 Ángel González
+#  Permission is hereby granted, free of charge, to any person obtaining a copy
+#  of this software and associated documentation files (the "Software"), to deal
+#  in the Software without restriction, including without limitation the rights
+#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+#  copies of the Software, and to permit persons to whom the Software is
+#  furnished to do so, subject to the following conditions:
+#
+#  The above copyright notice and this permission notice shall be included in
+#  all copies or substantial portions of the Software.
+#
+#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+#  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+#  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+#  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+#  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+#  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+#  THE SOFTWARE.
+#
+
+set -eu
+
+# Binary to be executed. Use an absolute path if installing this script as 'wget'
+WGET=wget
+
+if [ -z "${socks_proxy:-}" ]; then
+exec "$WGET" "$@"
+fi
+
+CONFIG=""
+
+if [[ "${socks_proxy}" =~ ^socks[45]?:// ]]; then
+ if [[ "${socks_proxy:5:1}" != ":" ]]; then
+  CONFIG+="server_type = ${socks_proxy:5:1}"
+  socks_proxy="${socks_proxy:9}"
+ else
+  socks_proxy="${socks_proxy:8}"
+ fi
+elif [[ "${socks_proxy}" =~ ^[[:alnum:]]*:// ]]; then
+ echo "Bad value specified for socks_proxy: $socks_proxy" >&2
+ exit 2
+fi
+
+if [[ "${socks_proxy}" =~ ^([^@:]*)(:([^@]*))?@ ]]; then
+ unset TSOCKS_USERNAME
+ CONFIG+="
+   default_user = ${BASH_REMATCH[1]}"
+
+ if [ ! -z "${BASH_REMATCH[3]}" ]; then
+  unset TSOCKS_PASSWORD
+  CONFIG+="
+   default_pass = ${BASH_REMATCH[3]}"
+ fi
+ socks_proxy="${socks_proxy:${#BASH_REMATCH[0]}}"
+fi
+
+
+# Get rid of trailing slashes
+if [[ "${socks_proxy}" =~ ^([^/]*)/ ]]; then
+ socks_proxy="${socks_proxy:0:${#BASH_REMATCH[1]}}"
+fi
+
+if [[ "${socks_proxy}" =~ :([0-9]+)$ ]]; then
+  CONFIG+="
+   server_port = ${BASH_REMATCH[1]}"
+  socks_proxy=${socks_proxy:0:${#socks_proxy} - ${#BASH_REMATCH[0]}}
+fi
+
+CONFIG+="
+   server = ${socks_proxy}"
+
+TSOCKS_CONF_FILE=<(echo "$CONFIG") exec tsocks "$WGET" --no-proxy "$@"
--- a/doc/ChangeLog
+++ b/doc/ChangeLog
--- a/doc/Makefile.am
+++ b/doc/Makefile.am
@@ -1,6 +1,6 @@
 # Makefile for `wget' utility
-# Copyright (C) 1995, 1996, 1997, 2007, 2008, 2009, 2010, 2011 Free
-# Software Foundation, Inc.
+# Copyright (C) 1995-1997, 2007-2011, 2015, 2018 Free Software
+# Foundation, Inc.

 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -16,7 +16,7 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.

 #
-# Version: $(VERSION)
+# Version: @VERSION@
 #

 # Program to convert DVI files to PostScript
@@ -28,6 +28,7 @@ manext     = 1
 RM = rm -f

 TEXI2POD     = $(srcdir)/texi2pod.pl
+POD2MAN      = @POD2MAN@
 MAN          = wget.$(manext)
 WGETRC       = $(sysconfdir)/wgetrc
 SAMPLERCTEXI = sample.wgetrc.munged_for_texi_inclusion
@@ -38,7 +39,7 @@ SAMPLERCTEXI = sample.wgetrc.munged_for_texi_inclusion

 man_MANS = $(MAN)

-all: wget.info $(COMMENT_IF_NO_POD2MAN)$(MAN)
+all: wget.info @COMMENT_IF_NO_POD2MAN@$(MAN)

 everything: all wget_us.ps wget_a4.ps wget_toc.html

@@ -56,7 +57,8 @@ wget.pod: $(srcdir)/wget.texi version.texi
 	$(TEXI2POD) -D VERSION="$(VERSION)" $(srcdir)/wget.texi $@

 $(MAN): wget.pod
-	$(POD2MAN) --center="GNU Wget" --release="GNU Wget $(VERSION)" $? > $@
+	$(POD2MAN) --center="GNU Wget" --release="GNU Wget @VERSION@" --utf8 $? > $@ || \
+	$(POD2MAN) --center="GNU Wget" --release="GNU Wget @VERSION@" $? > $@

 #wget.cat: $(MAN)
 #	nroff -man $? > $@
@@ -75,10 +77,10 @@ wget_toc.html: $(srcdir)/wget.texi
 #

 # install all the documentation
-install-data-local: install.wgetrc $(COMMENT_IF_NO_POD2MAN)install.man
+install-data-local: install.wgetrc @COMMENT_IF_NO_POD2MAN@install.man

 # uninstall all the documentation
-uninstall-local: $(COMMENT_IF_NO_POD2MAN)uninstall.man
+uninstall-local: @COMMENT_IF_NO_POD2MAN@uninstall.man


 # install man page, creating install directory if necessary
--- a/doc/announcement_1.19.txt
+++ b/doc/announcement_1.19.txt
@@ -0,0 +1,104 @@
+Hello,
+
+we are pleased to announce the new version of GNU wget 1.19.
+
+It comes with major improvements for Metalink, IDNA2008 for
+international domain names, an option to call external tools
+for fetching user/password, several bugfixes and improvements.
+
+Many thanks go to all the contributors and list activists !
+
+Contributors (from the git log):
+Ander Juaristi
+Dale R. Worley 
+Eli Zaretskii 
+Gisle Vanem 
+Giuseppe Scrivano
+Göran Uddeborg 
+Jann Horn 
+Jeffery To
+Jérémie Courrèges-Anglas 
+Liam R. Howlett
+losgrandes
+Matthew White
+Misra, Deapesh 
+Nikos Mavrogiannopoulos
+Noël Köthe
+Piotr Wajda
+Rahul Bedarkar
+Sean Burford
+Tim Rühsen
+Tobias Stoeckmann
+vijeth-aradhya
+
+
+The new version is available for download here:
+
+https://ftp.gnu.org/gnu/wget/wget-1.19.tar.gz
+https://ftp.gnu.org/gnu/wget/wget-1.19.tar.xz
+
+and the GPG detached signatures using the key 0x08302DB6A2670428:
+
+https://ftp.gnu.org/gnu/wget/wget-1.19.tar.gz.sig
+https://ftp.gnu.org/gnu/wget/wget-1.19.tar.xz.sig
+
+To reduce load on the main server, you can use this redirector service
+which automatically redirects you to a mirror:
+
+http://ftpmirror.gnu.org/wget/wget-1.19.tar.gz
+http://ftpmirror.gnu.org/wget/wget-1.19.tar.xz
+
+
+Noteworthy changes:
+
+* New option --use-askpass=COMMAND. Fetch user/password by calling
+  an external program.
+
+* Use IDNA2008 (+ TR46 if available) through libidn2
+
+* When processing a Metalink header, --metalink-index=<number> allows
+  to process the header's application/metalink4+xml files.
+
+* When processing a Metalink file, --trust-server-names enables the
+  use of the destination file names specified in the Metalink file,
+  otherwise a safe destination file name is computed.
+
+* When processing a Metalink file, enforce a safe destination path.
+  Remove any drive letter prefix under w32, i.e. 'C:D:file'.  Call
+  libmetalink's metalink_check_safe_path() to prevent absolute,
+  relative, or home paths:
+  https://tools.ietf.org/html/rfc5854#section-4.1.2.1
+  https://tools.ietf.org/html/rfc5854#section-4.2.8.3
+
+* When processing a Metalink file, --directory-prefix=<prefix> sets
+  the top of the retrieval tree to prefix for Metalink downloads.
+
+* When processing a Metalink file, reject downloaded files which don't
+  agree with their own metalink:size value:
+  https://tools.ietf.org/html/rfc5854#section-4.2.16
+
+* When processing a Metalink file, with --continue resume partially
+  downloaded files and keep fully downloaded files even if they fail
+  the verification.
+
+* When processing a Metalink file, create the parent directories of a
+  "path/file" destination file name:
+  https://tools.ietf.org/html/rfc5854#section-4.1.2.1
+  https://tools.ietf.org/html/rfc5854#section-4.2.8.3
+
+* On a recursive download, append a .tmp suffix to temporary files
+  that will be deleted after being parsed, and create them
+  readable/writable only by the owner.
+
+* New make target 'check-valgrind'
+
+* Fix several bugs
+
+* Fix compatibility issues
+
+
+Please report any problem you may experience to the bug-wget@gnu.org
+mailing list.
+
+For the maintainers of Wget,
+Tim
--- a/doc/fdl.texi
+++ b/doc/fdl.texi
@@ -5,7 +5,8 @@
@c hence no sectioning command or @node.

@display
-Copyright @copyright{} 2000, 2001, 2002, 2007, 2008 Free Software Foundation, Inc.
+Copyright @copyright{} 2000-2002, 2007-2008, 2015, 2018 Free Software
+Foundation, Inc.
@uref{http://fsf.org/}

 Everyone is permitted to copy and distribute verbatim copies
--- a/doc/metalink-standard.txt
+++ b/doc/metalink-standard.txt
@@ -0,0 +1,233 @@
+GNU Wget Metalink recommended behaviour
+
+  Metalink/XML and Metalink/HTTP standard reference
+
+
+1. Security features
+********************
+
+Only metalink:file elements with safe "name" fields shall be accepted
+[1 #section-4.1.2.1]. If unsafe metalink:file elements are saved, any
+related test shall fail (see '2. Tests').
+
+By design, libmetalink rejects unsafe metalink:file elements [3]:
+* lib/metalink_helper.c (metalink_check_safe_path): Verify path
+
+1.1 Exceptions
+==============
+
+The option --directory-prefix could allow to use an absolute, relative
+or home path.
+
+2. Tests
+********
+
+Saving a file to an unexpected path poses a security problem. We must
+ensure that Wget's automated tests never modify the root and the home
+paths or descend/escalate to a relative path unexpectedly.
+
+2.1 Metalink/XML implemented tests
+==================================
+
+See testenv/Makefile.am (METALINK_TESTS).
+
+2.2 Metalink/HTTP implemented tests
+===================================
+
+See testenv/Makefile.am (METALINK_TESTS).
+
+3. Download file name
+*********************
+
+The download file name shall be decided by precise rules which prevent
+any naming uncertainty and security issues.
+
+3.1 Naming rules
+================
+
+The final name of downloaded files is computed starting from a trusted
+name, which is then combined with the "Directory Options".  The result
+is verified and eventually made safer following security rules. If the
+final name isn't found safe enough, then the file isn't downloaded.
+
+Depending on the options used, a suffix could be appended to the final
+name to not overwrite existing files.
+
+3.1.1 The trusted name
+======================
+
+The option --trust-server-names decides what is the trusted name.
+
+Any Metalink/XML element with an unsafe metalink:file "name" field is
+ignored, see '1. Security features'.
+
+3.1.1.1 Without --trust-server-names
+====================================
+
+When --trust-server-names is off, the basename of the --input-metalink
+file, if available, or of the mother URL is trusted. This trusted name
+is the radix of any subsequent file name.
+
+When a Metalink/HTTP in encountered, any fetched Metalink/XML file has
+its own ordinal number appended as suffix to the trusted name. In this
+case scenario, an unique Metalink/XML file is saved each time applying
+an additional suffix to the currently computed name when necessary.
+
+The files described by a Metalink/XML file will be named sequentially
+applying an additional suffix to the currently trusted/computed name.
+
+3.1.1.2 With --trust-server-names
+=================================
+
+When --trust-server-names is on, the metalink:file "name" field parsed
+from Metalink/XML files is trusted. When no Metalink/XML is available,
+the mother URL is trusted.
+
+Any Metalink/HTTP application/metalink4+xml file is saved using the
+basename of its own Link header "name" field, if available.
+
+In conjunction with the option --content-disposition, a 'Content-Type:
+application/metalink4+xml' file is saved using the basename of its own
+Content-Disposition header "filename" field, if available.
+
+3.1.2 The final name
+====================
+
+The "Directory Options" are combined with the trusted name. The result
+is evaluated again by the '1. Security features'. If the path is found
+unsafe, only the basename of the final name is considered.  If this is
+found unsafe too, the file is not downloaded.
+
+4. Metalink/XML
+***************
+
+4.1 Example files
+=================
+
+See [1 #section-1.1].
+
+cat > bugus.meta4 << EOF
+<?xml version="1.0" encoding="UTF-8"?>
+<metalink xmlns="urn:ietf:params:xml:ns:metalink">
+  <file name="/dir/A/File1">
+    <size>1617</size>
+    <hash type="sha256">ecb3dff2648667513e31554b3ad054ccd89fce38e33367c9459ac3a285153742</hash>
+    <url>http://another.url/common_name</url>
+    <url>http://ftpmirror.gnu.org/bash/bash-4.3-patches/bash43-001</url>
+  </file>
+  <file name="dir/B/File2">
+    <size>1594</size>
+    <hash type="sha256">eee7cd7062ab29a9e4f02924d9c367264dcb8b162703f74ff6eb8f175a91502b</hash>
+    <url>http://another.url/again/common_name</url>
+    <url>http://ftpmirror.gnu.org/bash/bash-4.3-patches/bash43-002</url>
+  </file>
+</metalink>
+EOF
+
+4.2 Command line example
+========================
+
+$ wget --input-metalink=bogus.meta4
+
+4.3 Metalink/XML file parsing
+=============================
+
+The metalink xml file is parsed by one of the following libmetalink's
+functions [3], depending upon the library configured to use:
+* lib/libexpat_metalink_parser.c (metalink_parse_file): Expat [4]
+* lib/libxml2_metalink_parser.c (metalink_parse_file): Libxml2 [5]
+
+The result returned doesn't include unsafe metalink:file elements, as
+stated at point '1. Security features'.
+
+An empty result shall not be considered an error.  Parsing errors will
+be informed to the caller of libmetalink's metalink_parse_file().
+
+4.4 Saving files
+================
+
+Fetched metalink:file elements shall be wrote using the unique "name"
+field as file name [1 #section-4.1.2.1].
+
+A metalink:file url's file name shall not substitute the "name" field.
+
+Security exceptions are explained in '3. Download file name'.
+
+4.5 Multi-Source download
+=========================
+
+Parallel range requests are allowed [1 #section-1].
+
+5. Metalink/HTTP
+****************
+
+5.1 HTTP server
+===============
+
+The local server http://127.0.0.1 is used as reference in the course
+of this chapter. Any server service capable of sending Metalink/HTTP
+header answers may be used.
+
+5.2 Command line example
+========================
+
+$ wget --metalink-over-http http://127.0.0.1/dir/file.ext
+
+5.3 Metalink/HTTP header answer
+===============================
+
+See [2 #section-1.1].
+
+Etag: "thvDyvhfIqlvFe+A9MYgxAfm1q5="
+Link: <http://www2.example.com/example.ext>; rel=duplicate
+Link: <ftp://ftp.example.com/example.ext>; rel=duplicate
+Link: <http://example.com/example.ext.torrent>; rel=describedby;
+type="application/x-bittorrent"
+Link: <http://example.com/example.ext.meta4>; rel=describedby;
+type="application/metalink4+xml"
+Link: <http://example.com/example.ext.asc>; rel=describedby;
+type="application/pgp-signature"
+Digest: SHA-256=MWVkMWQxYTRiMzk5MDQ0MzI3NGU5NDEyZTk5OWY1ZGFmNzgyZTJlO
+DYzYjRjYzFhOTlmNTQwYzI2M2QwM2U2MQ==
+
+See [2 #section-4].
+
+Link: <http://example.com/example.ext.torrent>; rel=describedby;
+type="application/x-bittorrent"; name="differentname.ext"
+Link: <http://example.com/example.ext.meta4>; rel=describedby;
+type="application/metalink4+xml"
+
+5.4 Saving files
+================
+
+When none of --output-document and/or --content-disposition is used,
+the file name to wrote is computed from the cli's url hierarchy. The
+purpose of the "Directory Options" is as usual, and the file name is
+the cli's url file name, see wget(1).
+
+The url followed to download the file shall not substitute the cli's
+url to compute the file name to wrote, except when it redirects to a
+Metalink/XML file, following the rules in '3. Download file name'.
+
+5.5 Multi-Source download
+=========================
+
+Parallel range requests are allowed [2 #section-7].
+
+4. References
+*************
+
+[1] The Metalink Download Description Format
+https://tools.ietf.org/html/rfc5854
+
+[2] Metalink/HTTP: Mirrors and Hashes
+https://tools.ietf.org/html/rfc6249
+
+[3] Libmetalink
+https://github.com/metalink-dev/libmetalink
+
+[4] Expat
+http://www.libexpat.org
+
+[5] Libxml2
+http://xmlsoft.org
--- a/doc/metalink.txt
+++ b/doc/metalink.txt
@@ -0,0 +1,165 @@
+GNU Wget Metalink module
+
+  Evaluation of the Metalink/XML and Metalink/HTTP implementations
+
+
+1. Introduction
+***************
+
+This document, and the results contained in it, is focused over the
+evaluation of the Metalink/XML and Metalink/HTTP implementations.
+
+The "Directory Options" mentioned here are used on the command line in
+conjunction with the option '--input-metalink=file' for Metalink/XML,
+and '--metalink-over-http' for Metalink/HTTP.
+
+$ wget --input-metalink=<file> [directory options]
+$ wget --metalink-over-http [directory options] <url>
+
+2. Notes
+********
+
+Tests for metalink:file names beginning with '/', '~/', './', or '../'
+(e.g. "/path/file") shall be run manually due to security concerns.
+
+3. Metalink files used as reference
+***********************************
+
+3.1 Test: metalink:file with "path/file" name format
+====================================================
+
+cat > test.meta4 << EOF
+<?xml version="1.0" encoding="UTF-8"?>
+<metalink xmlns="urn:ietf:params:xml:ns:metalink">
+  <file name="path/file">
+    <size>543</size>
+    <hash type="sha256">d37d3965f8e1a7b16504b4273b09c392776b7e4dd17e601256c7b2fd9ce5f56e</hash>
+    <hash type="md5">0f6ff5cdc15603f1b81227b5a296f001</hash>
+    <url>http://wrongurl.really/gnu/wget/wget-1.18.tar.xz.sig</url>
+    <url>http://ftpmirror.gnu.org/wget/wget-1.18.tar.xz.sig</url>
+    <url>http://ftp.gnu.org/gnu/wget/wget-1.18.tar.xz.sig</url>
+    <url>http://nl.mirror.babylon.network/gnu/wget/wget-1.18.tar.xz.sig</url>
+  </file>
+</metalink>
+EOF
+
+4. `wget --input-metalink=test.meta4`
+*************************************
+
+4.1 Implemented safety features
+===============================
+
+Any metalink:file name containing an absolute, relative, or home path
+(see '2. Notes') parsed from Metalink/XML files is rejected.
+
+This is a libmetalink's design decision implemented in the function
+metalink_check_safe_path().  This feature shall not be modified.
+
+All the above conform to the RFC5854 standard.
+
+References:
+ https://tools.ietf.org/html/rfc5854#section-4.1.2.1
+ https://tools.ietf.org/html/rfc5854#section-4.2.8.3
+
+4.2 File download behaviour
+===========================
+
+When a Metalink/XML file is parsed:
+1. create the metalink:file "path/file" tree;
+2. download the metalink:url file as "path/file";
+3. verify the "path/file" size, if declared;
+4. verify the "path/file" checksum.
+
+All the above conform to the RFC5854 standard.
+
+References:
+ https://tools.ietf.org/html/rfc5854
+
+4.3 Questionable behaviours
+===========================
+
+If more metalink:file elements are the same, wget downloads them all.
+
+5. `wget --metalink-over-http`
+******************************
+
+5.1 Implemented safety features
+===============================
+
+The function url_file_name() is responsible of parsing the url's file
+name and mixing in the "Directory Options" wrote on the command line.
+
+The use of libmetalink's metalink_check_safe_path() shouldn't be
+necessary (see '4.1 Implemented safety features').
+
+All the above comform to the usual Wget's download behaviour.
+
+References:
+ wget(1)
+
+5.2 File download behaviour
+===========================
+
+When a Metalink/HTTP header is parsed:
+1. extract metalink metadata from the header;
+2. download the file from the mirror with the highest priority;
+3. verify the file's size, if declared;
+4. verify the file's checksum.
+
+All the above comform to the usual Wget's download behaviour and to
+the RFC6249 standard.
+
+References:
+ wget(1)
+ https://tools.ietf.org/html/rfc6249
+
+6. Directory Options
+********************
+
+'-nd'
+'--no-directories'
+
+    Do not apply to Metalink/XML files (aka --input-metalink=<file>).
+
+    Apply to Metalink/HTTP urls as described in the Wget's manual, see
+    wget(1).  The target url is the url wrote on the command line.
+
+'-x'
+'--force-directories'
+
+    Do not apply to Metalink/XML files (aka --input-metalink=<file>).
+
+    Apply to Metalink/HTTP urls as described in the Wget's manual, see
+    wget(1).  The target url is the url wrote on the command line.
+
+'-nH'
+'--no-host-directories'
+
+    Do not apply to Metalink/XML files (aka --input-metalink=<file>).
+
+    Apply to Metalink/HTTP urls as described in the Wget's manual, see
+    wget(1).  The target url is the url wrote on the command line.
+
+'--protocol-directories'
+
+    Do not apply to Metalink/XML files (aka --input-metalink=<file>).
+
+    Apply to Metalink/HTTP urls as described in the Wget's manual, see
+    wget(1).  The target url is the url wrote on the command line.
+
+'--cut-dirs=number'
+
+    Do not apply to Metalink/XML files (aka --input-metalink=<file>).
+
+    Apply to Metalink/HTTP urls as described in the Wget's manual, see
+    wget(1).  The target url is the url wrote on the command line.
+
+'-P prefix'
+'--directory-prefix=prefix'
+
+    Set the top of the retrieval tree to prefix for both Metalink/XML
+    and Metalink/HTTP downloads, see wget(1).
+
+    If combining the prefix with the file name results in an absolute,
+    relative, or home path, the directory components are stripped and
+    only the basename is used. See '4.1 Implemented safety features'.
--- a/doc/sample.wgetrc
+++ b/doc/sample.wgetrc
@@ -16,6 +16,9 @@
 ## To use the settings in this file, you will have to uncomment them,
 ## as well as change them, in most cases, as the values on the
 ## commented-out lines are the default values (e.g. "off").
+##
+## Command are case-, underscore- and minus-insensitive.
+## For example ftp_proxy, ftp-proxy and ftpproxy are the same.


 ##
@@ -122,7 +125,7 @@
 #iri = off

 # Force the default system encoding
-#locale = UTF-8
+#localencoding = UTF-8

 # Force the default remote server encoding
 #remoteencoding = UTF-8
--- a/doc/texi2pod.pl
+++ b/doc/texi2pod.pl
@@ -1,6 +1,6 @@
 #! /usr/bin/env perl

-#   Copyright (C) 1999, 2000, 2001, 2003, 2007, 2009, 2010, 2011 Free
+#   Copyright (C) 1999-2001, 2003, 2007, 2009-2011, 2015, 2018 Free
 #   Software Foundation, Inc.

 # This file is part of GCC.
@@ -315,6 +315,8 @@ die "No filename or title\n" unless defined $fn && defined $tl;
 $sects{NAME} = "$fn \- $tl\n";
 $sects{FOOTNOTES} .= "=back\n" if exists $sects{FOOTNOTES};

+print "=encoding utf-8\n\n";
+
 for $sect (qw(NAME SYNOPSIS DESCRIPTION OPTIONS ENVIRONMENT EXITSTATUS
           FILES BUGS NOTES FOOTNOTES SEEALSO AUTHOR COPYRIGHT)) {
    if(exists $sects{$sect}) {
--- a/doc/wget.texi
+++ b/doc/wget.texi
--- a/2
+++ b/2
--- a/m4/exitfail.m4
+++ b/m4/exitfail.m4
@@ -1,5 +1,5 @@
 # exitfail.m4 serial 6
-dnl Copyright (C) 2002, 2003, 2005, 2006, 2010, 2011 Free Software
+dnl Copyright (C) 2002-2003, 2005-2006, 2010-2011, 2018 Free Software
 dnl Foundation, Inc.
 dnl This file is free software; the Free Software Foundation
 dnl gives unlimited permission to copy and/or distribute it,
--- a/m4/getpagesize.m4
+++ b/m4/getpagesize.m4
@@ -1,5 +1,5 @@
 # getpagesize.m4 serial 7
-dnl Copyright (C) 2002, 2004, 2005, 2007, 2010, 2011 Free Software
+dnl Copyright (C) 2002, 2004-2005, 2007, 2010-2011, 2018 Free Software
 dnl Foundation, Inc.
 dnl This file is free software; the Free Software Foundation
 dnl gives unlimited permission to copy and/or distribute it,
--- a/m4/wchar.m4
+++ b/m4/wchar.m4
@@ -1,7 +1,6 @@
 dnl A placeholder for ISO C99 <wchar.h>, for platforms that have issues.

-dnl Copyright (C) 2007, 2008, 2009, 2010, 2011 Free Software Foundation,
-dnl Inc.
+dnl Copyright (C) 2007-2011, 2018 Free Software Foundation, Inc.
 dnl This file is free software; the Free Software Foundation
 dnl gives unlimited permission to copy and/or distribute it,
 dnl with or without modifications, as long as this notice is preserved.
--- a/m4/wctype.m4
+++ b/m4/wctype.m4
@@ -2,7 +2,7 @@

 dnl A placeholder for ISO C99 <wctype.h>, for platforms that lack it.

-dnl Copyright (C) 2006, 2007, 2008, 2010, 2011 Free Software Foundation,
+dnl Copyright (C) 2006-2008, 2010-2011, 2018 Free Software Foundation,
 dnl Inc.
 dnl This file is free software; the Free Software Foundation
 dnl gives unlimited permission to copy and/or distribute it,
--- a/m4/wget.m4
+++ b/m4/wget.m4
@@ -1,7 +1,5 @@
 dnl Wget-specific Autoconf macros.
-dnl Copyright (C) 1995, 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003,
-dnl 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011 Free Software
-dnl Foundation, Inc.
+dnl Copyright (C) 1995-2011, 2018 Free Software Foundation, Inc.

 dnl This program is free software; you can redistribute it and/or modify
 dnl it under the terms of the GNU General Public License as published by
@@ -27,89 +25,6 @@ dnl Corresponding Source for a non-source form of such a combination
 dnl shall include the source code for the parts of OpenSSL used as well
 dnl as that of the covered work.

-dnl
-dnl Check for `struct utimbuf'.
-dnl
-
-AC_DEFUN([WGET_STRUCT_UTIMBUF], [
-  AC_CHECK_TYPES([struct utimbuf], [], [], [
-#include <stdio.h>
-#if HAVE_SYS_TYPES_H
-# include <sys/types.h>
-#endif
-#if HAVE_UTIME_H
-# include <utime.h>
-#endif
-  ])
-])
-
-dnl Check whether fnmatch.h can be included.  This doesn't use
-dnl AC_FUNC_FNMATCH because Wget is already careful to only use
-dnl fnmatch on certain OS'es.  However, fnmatch.h is sometimes broken
-dnl even on those because Apache installs its own fnmatch.h to
-dnl /usr/local/include (!), which GCC uses before /usr/include.
-
-AC_DEFUN([WGET_FNMATCH], [
-  AC_MSG_CHECKING([for working fnmatch.h])
-  AC_COMPILE_IFELSE([AC_LANG_SOURCE([#include <fnmatch.h>
-                    ])], [
-    AC_MSG_RESULT(yes)
-    AC_DEFINE([HAVE_WORKING_FNMATCH_H], 1,
-              [Define if fnmatch.h can be included.])
-  ], [
-    AC_MSG_RESULT(no)
-  ])
-])
-
-dnl Check for nanosleep.  For nanosleep to work on Solaris, we must
-dnl link with -lrt (recently) or with -lposix4 (older releases).
-
-AC_DEFUN([WGET_NANOSLEEP], [
-  AC_CHECK_FUNCS(nanosleep, [], [
-    AC_CHECK_LIB(rt, nanosleep, [
-      AC_DEFINE([HAVE_NANOSLEEP], 1,
-                [Define if you have the nanosleep function.])
-      LIBS="-lrt $LIBS"
-    ], [
-      AC_CHECK_LIB(posix4, nanosleep, [
-	AC_DEFINE([HAVE_NANOSLEEP], 1,
-		  [Define if you have the nanosleep function.])
-	LIBS="-lposix4 $LIBS"
-      ])
-    ])
-  ])
-])
-
-AC_DEFUN([WGET_POSIX_CLOCK], [
-  AC_CHECK_FUNCS(clock_gettime, [], [
-    AC_CHECK_LIB(rt, clock_gettime)
-  ])
-])
-
-dnl Check whether we need to link with -lnsl and -lsocket, as is the
-dnl case on e.g. Solaris.
-
-AC_DEFUN([WGET_NSL_SOCKET], [
-  dnl On Solaris, -lnsl is needed to use gethostbyname.  But checking
-  dnl for gethostbyname is not enough because on "NCR MP-RAS 3.0"
-  dnl gethostbyname is in libc, but -lnsl is still needed to use
-  dnl -lsocket, as well as for functions such as inet_ntoa.  We look
-  dnl for such known offenders and if one of them is not found, we
-  dnl check if -lnsl is needed.
-  wget_check_in_nsl=NONE
-  AC_CHECK_FUNCS(gethostbyname, [], [
-    wget_check_in_nsl=gethostbyname
-  ])
-  AC_CHECK_FUNCS(inet_ntoa, [], [
-    wget_check_in_nsl=inet_ntoa
-  ])
-  if test $wget_check_in_nsl != NONE; then
-    AC_CHECK_LIB(nsl, $wget_check_in_nsl)
-  fi
-  AC_CHECK_LIB(socket, socket)
-])
-
-
 dnl ************************************************************
 dnl START OF IPv6 AUTOCONFIGURATION SUPPORT MACROS
 dnl ************************************************************
--- a/msdos/ChangeLog
+++ b/msdos/ChangeLog
@@ -1,61 +0,0 @@
-2014-06-11  Giuseppe Scrivano  <gscrivan@redhat.com>
-
-	* Makefile.DJ: Remove terminating empty lines.
-	* Makefile.WC: Likewise.
-	* msdos.c: Likewise.
-
-2009-09-06  Gisle Vanem  <gvanem@broadpark.no>
-
-	* Makefile.WC: Added compilation of new file msdos.c.
-	Simplified; no need to have explicit rules for files in ./lib and
-	./md5. Just use the suffix search path mechanism in wmake.
-
-2009-09-06  Gisle Vanem  <gvanem@broadpark.no>
-
-	* Makefile.DJ: Added compilation of new file msdos.c.
-
-2009-09-05  Gisle Vanem  <gvanem@broadpark.no>
-
-	* config.h: Added 'HAVE_ALLOCA_H' for Watcom 1.5+.  Added
-	'USE_WATT32' since all DOS-targets use the Watt-32 tcp/ip stack.
-	Added meaningless dummy LOCALEDIR.
-
-	* Makefile.WC: A much needed update. Added rules for many files
-	in ./lib, css.c and version.c.
-
-	* Makefile.DJ: A much needed update. Added rules for css.c and
-	version.c.
-
-2008-01-25  Micah Cowan  <micah@cowan.name>
-
-	* config.h: Updated copyright year.
-
-2007-11-28  Micah Cowan <micah@cowan.name>
-
-	* config.h: Updated license exception for OpenSSL, per the SFLC.
-
-2007-10-15  Gisle Vanem  <gvanem@broadpark.no>
-
-	* config.h: Added some HAVE_* for djgpp 2.04 and Watcom 1.7+.
-
-	* Makefile.DJ: rewritten for including some files from ../lib.
-
-	* Makefile.WC: Ditto. Handling source-files out-of current
-	directory makes compliation a bit more painfull. AFAICS, one
-	must use explicit rules.
-
-2007-10-02  Gisle Vanem  <gvanem@broadpark.no>
-
-	* config.h: Removed unused defines, added needed 'HAVE_*' defines.
-
-	* Makefile.DJ: rewritten to be used from './src' directory.
-	Added '-DOPENSSL_NO_KRB5' for OpenSSL build. Target is
-	now wget.exe.
-
-	* Makefile.WC: Added for building with OpenWatcom targeting
-	32-bit DOS (DOS32A extender). 
-
-2007-09-24  Gisle Vanem  <giva@bgnett.no>
-
-	* Makefile.DJ, config.h: Added to support building on MS-DOS via
-	DJGPP.
--- a/msdos/config.h
+++ b/msdos/config.h
@@ -1,6 +1,6 @@
 /* Configuration header file for MS-DOS/Watt-32
-   Copyright (C) 2007, 2008, 2009, 2010, 2011 Free Software Foundation,
-   Inc.
+   Copyright (C) 2007, 2008, 2009, 2010, 2011, 2015 Free Software
+   Foundation, Inc.

   This file is part of GNU Wget.

--- a/msdos/msdos.c
+++ b/msdos/msdos.c
@@ -1,5 +1,5 @@
 /* MSDOS utility functions.
-   Copyright (C) 2009, 2010, 2011 Free Software Foundation, Inc.
+   Copyright (C) 2009, 2010, 2011, 2015 Free Software Foundation, Inc.

 This file is part of GNU Wget.

--- a/po/POTFILES.in
+++ b/po/POTFILES.in
@@ -1,7 +1,5 @@
 # List of files which containing translatable strings.
-# Copyright (C) 1995, 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003,
-# 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011 Free Software
-# Foundation, Inc.
+# Copyright (C) 1995-2011, 2018 Free Software Foundation, Inc.

 # Package source files
 lib/error.c
@@ -26,6 +24,7 @@ src/init.c
 src/iri.c
 src/log.c
 src/main.c
+src/metalink.c
 src/mswindows.c
 src/netrc.c
 src/openssl.c
@@ -34,7 +33,6 @@ src/ptimer.c
 src/recur.c
 src/res.c
 src/retr.c
-src/spider.c
 src/url.c
 src/utils.c
 src/warc.c
--- a/po/stamp-po
+++ b/po/stamp-po
@@ -1 +0,0 @@
-timestamp
--- a/src/ChangeLog
+++ b/src/ChangeLog
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -1,7 +1,5 @@
 # Makefile for `wget' utility
-# Copyright (C) 1995, 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003,
-# 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2014 Free Software
-# Foundation, Inc.
+# Copyright (C) 1995-2011, 2015, 2018 Free Software Foundation, Inc.

 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -28,50 +26,45 @@
 # as that of the covered work.

 #
-# Version: $(VERSION)
+# Version: @VERSION@
 #

 if IRI_IS_ENABLED
 IRI_OBJ = iri.c
 endif
+
 if METALINK_IS_ENABLED
 METALINK_OBJ = metalink.c
-METALINK_HEADER = metalink.h
 endif
-if THREADS_ARE_ENABLED
-THREAD_OBJ = multi.c
-MULTI_HEADER = multi.h
+
+if WITH_XATTR
+XATTR_OBJ = xattr.c
 endif

 # The following line is losing on some versions of make!
-DEFS     += -DSYSTEM_WGETRC=\"$(sysconfdir)/wgetrc\" -DLOCALEDIR=\"$(localedir)\"
-LIBS     += $(LIBICONV) $(LIBINTL) $(LIB_CLOCK_GETTIME)
+DEFS     = @DEFS@ -DSYSTEM_WGETRC=\"$(sysconfdir)/wgetrc\" -DLOCALEDIR=\"$(localedir)\"

-EXTRA_DIST = css.l css.c css_.c build_info.c.in iri.c multi.c multi.h metalink.c metalink.h
+EXTRA_DIST = css.l css.c css_.c build_info.c.in

 bin_PROGRAMS = wget
-
-wget_SOURCES = cmpt.c connect.c convert.c cookies.c ftp.c			\
-	       css_.c css-url.c							\
-	       ftp-basic.c ftp-ls.c hash.c host.c html-parse.c html-url.c	\
-	       http.c init.c log.c main.c netrc.c progress.c ptimer.c		\
-	       recur.c res.c retr.c spider.c url.c warc.c			\
-	       utils.c exits.c build_info.c $(IRI_OBJ)				\
-	       $(THREAD_OBJ) $(METALINK_OBJ)					\
-	       css-url.h css-tokens.h connect.h convert.h cookies.h		\
-	       ftp.h hash.h host.h html-parse.h html-url.h			\
-	       http.h http-ntlm.h init.h log.h mswindows.h netrc.h		\
-	       options.h progress.h ptimer.h recur.h res.h retr.h		\
-	       spider.h ssl.h sysdep.h url.h warc.h utils.h wget.h iri.h	\
-	       exits.h gettext.h $(THREAD_HEADER) $(METALINK_HEADER)
-
-am__wget_SOURCES_DIST = $(wget_SOURCES)
-am__libunittest_a_SOURCES_DIST = $(wget_SOURCES)
-
-
+wget_SOURCES = connect.c convert.c cookies.c ftp.c	\
+		css_.c css-url.c	\
+		ftp-basic.c ftp-ls.c hash.c host.c hsts.c html-parse.c html-url.c	\
+		http.c init.c log.c main.c netrc.c progress.c ptimer.c	\
+		recur.c res.c retr.c spider.c url.c warc.c $(XATTR_OBJ) \
+		utils.c exits.c build_info.c $(IRI_OBJ) $(METALINK_OBJ)	\
+		css-url.h css-tokens.h connect.h convert.h cookies.h	\
+		ftp.h hash.h host.h hsts.h  html-parse.h html-url.h	\
+		http.h http-ntlm.h init.h log.h mswindows.h netrc.h	\
+		options.h progress.h ptimer.h recur.h res.h retr.h	\
+		spider.h ssl.h sysdep.h url.h warc.h utils.h wget.h iri.h	\
+		exits.h version.h metalink.h xattr.h
 nodist_wget_SOURCES = version.c
-LDADD = $(LIBOBJS) ../lib/libgnu.a
-AM_LDFLAGS = $(THREAD_LIB)
+EXTRA_wget_SOURCES = iri.c
+LDADD = $(LIBOBJS) ../lib/libgnu.a $(GETADDRINFO_LIB) $(HOSTENT_LIB)\
+ $(INET_NTOP_LIB) $(LIBSOCKET) $(LIB_CLOCK_GETTIME) $(LIB_CRYPTO)\
+ $(LIB_NANOSLEEP) $(LIB_POSIX_SPAWN) $(LIB_SELECT) $(LIBICONV) $(LIBINTL)\
+ $(LIBTHREAD) $(LIBUNISTRING) $(SERVENT_LIB)
 AM_CPPFLAGS = -I$(top_builddir)/lib -I$(top_srcdir)/lib


@@ -89,7 +82,8 @@ version.c:  $(wget_SOURCES) ../lib/libgnu.a
 	echo '/* version.c */' > $@
 	echo '/* Autogenerated by Makefile - DO NOT EDIT */' >> $@
 	echo '' >> $@
-	echo 'const char *version_string = "$(VERSION)";' >> $@
+	echo '#include "version.h"' >> $@
+	echo 'const char *version_string = "@VERSION@";' >> $@
 	echo 'const char *compilation_string = "'$(COMPILE)'";' \
 	    | $(ESCAPEQUOTE) >> $@
 	echo 'const char *link_string = "'$(CCLD) $(AM_CFLAGS) $(CFLAGS) \
@@ -97,7 +91,7 @@ version.c:  $(wget_SOURCES) ../lib/libgnu.a
 	    | $(ESCAPEQUOTE) >> $@

 css.c: $(srcdir)/css.l
-	$(LEX) $(LFLAGS) -o $@ $^
+	$(LEX) $(LFLAGS) -o$@ $^

 css_.c: css.c
 	echo '#include "wget.h"' > $@
--- a/src/build_info.c.in
+++ b/src/build_info.c.in
@@ -2,14 +2,16 @@ digest          defined ENABLE_DIGEST
 https           defined HAVE_SSL
 ipv6            defined ENABLE_IPV6
 iri             defined ENABLE_IRI
-large-file      SIZEOF_OFF_T >= 8
+large-file      SIZEOF_OFF_T >= 8 || defined WINDOWS

 nls             defined ENABLE_NLS
 ntlm            defined ENABLE_NTLM
 opie            defined ENABLE_OPIE
-threads         defined ENABLE_THREADS
-metalink        defined ENABLE_METALINK
 psl             defined HAVE_LIBPSL
+cares            defined HAVE_LIBCARES
+
+metalink        defined HAVE_METALINK
+gpgme           defined HAVE_GPGME

 ssl choice:
    openssl     defined HAVE_LIBSSL || defined HAVE_LIBSSL32
--- a/src/cmpt.c
+++ b/src/cmpt.c
--- a/src/connect.c
+++ b/src/connect.c
@@ -1,7 +1,5 @@
 /* Establishing and handling network connections.
-   Copyright (C) 1995, 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003,
-   2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011 Free Software
-   Foundation, Inc.
+   Copyright (C) 1995-2011, 2015, 2018 Free Software Foundation, Inc.

 This file is part of GNU Wget.

@@ -31,6 +29,7 @@ as that of the covered work.  */

 #include "wget.h"

+#include "exits.h"
 #include <stdio.h>
 #include <stdlib.h>
 #include <unistd.h>
@@ -54,6 +53,7 @@ as that of the covered work.  */
 #include <errno.h>
 #include <string.h>
 #include <sys/time.h>
+
 #include "utils.h"
 #include "host.h"
 #include "connect.h"
@@ -168,7 +168,7 @@ sockaddr_size (const struct sockaddr *sa)
      abort ();
    }
 }
-
+
 /* Resolve the bind address specified via --bind-address and store it
   to SA.  The resolved value is stored in a static variable and
   reused after the first invocation of this function.
@@ -213,7 +213,7 @@ resolve_bind_address (struct sockaddr *sa)
  should_bind = true;
  return true;
 }
-
+
 struct cwt_context {
  int fd;
  const struct sockaddr *addr;
@@ -250,7 +250,7 @@ connect_with_timeout (int fd, const struct sockaddr *addr, socklen_t addrlen,
    errno = ETIMEDOUT;
  return ctx.result;
 }
-
+
 /* Connect via TCP to the specified address and port.

   If PRINT is non-NULL, it is the host name to print that we're
@@ -274,18 +274,14 @@ connect_to_ip (const ip_address *ip, int port, const char *print)

          if (opt.enable_iri && (name = idn_decode ((char *) print)) != NULL)
            {
-              int len = strlen (print) + strlen (name) + 4;
-              str = xmalloc (len);
-              snprintf (str, len, "%s (%s)", name, print);
-              str[len-1] = '\0';
+              str = aprintf ("%s (%s)", name, print);
              xfree (name);
            }

          logprintf (LOG_VERBOSE, _("Connecting to %s|%s|:%d... "),
                     str ? str : escnonprint_uri (print), txt_addr, port);

-          if (str)
-              xfree (str);
+          xfree (str);
        }
      else
        {
@@ -327,8 +323,10 @@ connect_to_ip (const ip_address *ip, int port, const char *print)
      if (bufsize < 512)
        bufsize = 512;          /* avoid pathologically small values */
 #ifdef SO_RCVBUF
-      setsockopt (sock, SOL_SOCKET, SO_RCVBUF,
-                  (void *)&bufsize, (socklen_t)sizeof (bufsize));
+      if (setsockopt (sock, SOL_SOCKET, SO_RCVBUF,
+                  (void *) &bufsize, (socklen_t) sizeof (bufsize)))
+        logprintf (LOG_NOTQUIET, _("setsockopt SO_RCVBUF failed: %s\n"),
+                   strerror (errno));
 #endif
      /* When we add limit_rate support for writing, which is useful
         for POST, we should also set SO_SNDBUF here.  */
@@ -365,9 +363,16 @@ connect_to_ip (const ip_address *ip, int port, const char *print)
       logprintf.  */
    int save_errno = errno;
    if (sock >= 0)
-      fd_close (sock);
+      {
+#ifdef WIN32
+	/* If the connection timed out, fd_close will hang in Gnulib's
+	   close_fd_maybe_socket, inside the call to WSAEnumNetworkEvents.  */
+	if (errno != ETIMEDOUT)
+#endif
+	  fd_close (sock);
+      }
    if (print)
-      logprintf (LOG_VERBOSE, _("failed: %s.\n"), strerror (errno));
+      logprintf (LOG_NOTQUIET, _("failed: %s.\n"), strerror (errno));
    errno = save_errno;
    return -1;
  }
@@ -429,7 +434,7 @@ connect_to_host (const char *host, int port)

  return -1;
 }
-
+
 /* Create a socket, bind it to local interface BIND_ADDRESS on port
   *PORT, set up a listen backlog, and return the resulting socket, or
   -1 in case of error.
@@ -460,7 +465,9 @@ bind_local (const ip_address *bind_address, int *port)
    return -1;

 #ifdef SO_REUSEADDR
-  setsockopt (sock, SOL_SOCKET, SO_REUSEADDR, setopt_ptr, setopt_size);
+  if (setsockopt (sock, SOL_SOCKET, SO_REUSEADDR, setopt_ptr, setopt_size))
+    logprintf (LOG_NOTQUIET, _("setsockopt SO_REUSEADDR failed: %s\n"),
+               strerror (errno));
 #endif

  xzero (ss);
@@ -678,6 +685,11 @@ select_fd (int fd, double maxtime, int wait_for)
  struct timeval tmout;
  int result;

+  if (fd >= FD_SETSIZE)
+    {
+      logprintf (LOG_NOTQUIET, _("Too many fds open.  Cannot use select on a fd >= %d\n"), FD_SETSIZE);
+      exit (WGET_EXIT_GENERIC_ERROR);
+    }
  FD_ZERO (&fdset);
  FD_SET (fd, &fdset);
  if (wait_for & WAIT_FOR_READ)
@@ -720,6 +732,11 @@ test_socket_open (int sock)
  struct timeval to;
  int ret = 0;

+  if (sock >= FD_SETSIZE)
+    {
+      logprintf (LOG_NOTQUIET, _("Too many fds open.  Cannot use select on a fd >= %d\n"), FD_SETSIZE);
+      exit (WGET_EXIT_GENERIC_ERROR);
+    }
  /* Check if we still have a valid (non-EOF) connection.  From Andrew
   * Maholski's code in the Unix Socket FAQ.  */

@@ -746,7 +763,7 @@ wget uses blocking sockets so we must convert them back to blocking
       or EOF/error. */
    return false;
 }
-
+
 /* Basic socket operations, mostly EINTR wrappers.  */

 static int
@@ -794,7 +811,7 @@ sock_close (int fd)
 #undef read
 #undef write
 #undef close
-
+
 /* Reading and writing from the network.  We build around the socket
   (file descriptor) API, but support "extended" operations for things
   that are not mere file descriptors under the hood, such as SSL
@@ -845,7 +862,7 @@ void *
 fd_transport_context (int fd)
 {
  struct transport_info *info = hash_table_get (transport_map, (void *)(intptr_t) fd);
-  return info->ctx;
+  return info ? info->ctx : NULL;
 }

 /* When fd_read/fd_write are called multiple times in a loop, they should
--- a/src/connect.h
+++ b/src/connect.h
@@ -1,7 +1,5 @@
 /* Declarations for connect.
-   Copyright (C) 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004,
-   2005, 2006, 2007, 2008, 2009, 2010, 2011 Free Software Foundation,
-   Inc.
+   Copyright (C) 1996-2011, 2015, 2018 Free Software Foundation, Inc.

 This file is part of GNU Wget.

@@ -60,11 +58,7 @@ enum {
  WAIT_FOR_READ = 1,
  WAIT_FOR_WRITE = 2
 };
-#ifdef ENABLE_THREADS
-int select_fds (int *, int *, int, double, int);
-#else
 int select_fd (int, double, int);
-#endif
 bool test_socket_open (int);

 struct transport_implementation {
--- a/src/convert.c
+++ b/src/convert.c
@@ -1,6 +1,6 @@
 /* Conversion of links to local files.
-   Copyright (C) 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2014
-   Free Software Foundation, Inc.
+   Copyright (C) 2003-2011, 2014-2015, 2018 Free Software Foundation,
+   Inc.

 This file is part of GNU Wget.

@@ -36,9 +36,6 @@ as that of the covered work.  */
 #include <unistd.h>
 #include <errno.h>
 #include <assert.h>
-#ifdef ENABLE_THREADS
-#include <pthread.h>
-#endif
 #include "convert.h"
 #include "url.h"
 #include "recur.h"
@@ -49,6 +46,7 @@ as that of the covered work.  */
 #include "html-url.h"
 #include "css-url.h"
 #include "iri.h"
+#include "xstrndup.h"

 static struct hash_table *dl_file_url_map;
 struct hash_table *dl_url_file_map;
@@ -58,49 +56,6 @@ struct hash_table *dl_url_file_map;
 struct hash_table *downloaded_html_set;
 struct hash_table *downloaded_css_set;

-#ifdef ENABLE_THREADS
-static pthread_mutex_t convert_mutex = PTHREAD_MUTEX_INITIALIZER;
-
-static void
-lock_convert_mutex()
-{
-  static int convert_mutex_init_p = 0;
-  if (! convert_mutex_init_p)
-    {
-      pthread_mutexattr_t mta;
-      pthread_mutexattr_init (&mta);
-      pthread_mutexattr_settype (&mta, PTHREAD_MUTEX_RECURSIVE);
-
-      pthread_mutex_init (&convert_mutex, &mta);
-      convert_mutex_init_p = 1;
-    }
-  pthread_mutex_lock (&convert_mutex);
-}
-
-
-#define THREAD_SAFE(ret, fn, args, argv)          \
-  ret fn args                                     \
-  {                                               \
-    ret r;                                        \
-    lock_convert_mutex();                         \
-    r = fn##_1 argv;                              \
-    pthread_mutex_unlock (&convert_mutex);        \
-    return r;                                     \
-  }
-
-#define THREAD_SAFE_VOID(fn, args, argv)            \
-  void fn args                                      \
-  {                                                 \
-    lock_convert_mutex();                           \
-    fn##_1 argv;                                    \
-    pthread_mutex_unlock (&convert_mutex);          \
-  }
-
-#define FNNAME_WTHREADS(fn) fn##_1
-#else
-#define FNNAME_WTHREADS(fn) fn
-#endif
-
 static void convert_links (const char *, struct urlpos *);


@@ -182,8 +137,9 @@ convert_links_in_hashtable (struct hash_table *downloaded_set,
                 form.  We do this even if the URL already is in
                 relative form, because our directory structure may
                 not be identical to that on the server (think `-nd',
-                 `--cut-dirs', etc.)  */
-              cur_url->convert = CO_CONVERT_TO_RELATIVE;
+                 `--cut-dirs', etc.). If --convert-file-only was passed,
+                 we only convert the basename portion of the URL.  */
+              cur_url->convert = (opt.convert_file_only ? CO_CONVERT_BASENAME_ONLY : CO_CONVERT_TO_RELATIVE);
              cur_url->local_name = xstrdup (local_name);
              DEBUGP (("will convert url %s to local %s\n", u->url, local_name));
            }
@@ -228,7 +184,7 @@ convert_links_in_hashtable (struct hash_table *downloaded_set,
   extracted from these two lists.  */

 void
-FNNAME_WTHREADS(convert_all_links) (void)
+convert_all_links (void)
 {
  double secs;
  int file_count = 0;
@@ -239,7 +195,7 @@ FNNAME_WTHREADS(convert_all_links) (void)
  convert_links_in_hashtable (downloaded_css_set, 1, &file_count);

  secs = ptimer_measure (timer);
-  logprintf (LOG_VERBOSE, _("Converted %d files in %s seconds.\n"),
+  logprintf (LOG_VERBOSE, _("Converted links in %d files in %s seconds.\n"),
             file_count, print_decimal (secs));

  ptimer_destroy (timer);
@@ -252,6 +208,7 @@ static const char *replace_attr_refresh_hack (const char *, int, FILE *,
                                              const char *, int);
 static char *local_quote_string (const char *, bool);
 static char *construct_relative (const char *, const char *);
+static char *convert_basename (const char *, const struct urlpos *);

 /* Change the links in one file.  LINKS is a list of links in the
   document, along with their positions and the desired direction of
@@ -267,7 +224,7 @@ convert_links (const char *file, struct urlpos *links)
  struct urlpos *link;
  int to_url_count = 0, to_file_count = 0;

-  logprintf (LOG_VERBOSE, _("Converting %s... "), file);
+  logprintf (LOG_VERBOSE, _("Converting links in %s... "), file);

  {
    /* First we do a "dry run": go through the list L and see whether
@@ -351,7 +308,7 @@ convert_links (const char *file, struct urlpos *links)
            char *quoted_newname = local_quote_string (newname,
                                                       link->link_css_p);

-            if (link->link_css_p)
+            if (link->link_css_p || link->link_noquote_html_p)
              p = replace_plain (p, link->size, fp, quoted_newname);
            else if (!link->link_refresh_p)
              p = replace_attr (p, link->size, fp, quoted_newname);
@@ -361,9 +318,32 @@ convert_links (const char *file, struct urlpos *links)

            DEBUGP (("TO_RELATIVE: %s to %s at position %d in %s.\n",
                     link->url->url, newname, link->pos, file));
+
            xfree (newname);
            xfree (quoted_newname);
            ++to_file_count;
+            break;
+          }
+        case CO_CONVERT_BASENAME_ONLY:
+          {
+            char *newname = convert_basename (p, link);
+            char *quoted_newname = local_quote_string (newname, link->link_css_p);
+
+            if (link->link_css_p || link->link_noquote_html_p)
+              p = replace_plain (p, link->size, fp, quoted_newname);
+            else if (!link->link_refresh_p)
+              p = replace_attr (p, link->size, fp, quoted_newname);
+            else
+              p = replace_attr_refresh_hack (p, link->size, fp, quoted_newname,
+                                             link->refresh_timeout);
+
+            DEBUGP (("Converted file part only: %s to %s at position %d in %s.\n",
+                     link->url->url, newname, link->pos, file));
+
+            xfree (newname);
+            xfree (quoted_newname);
+            ++to_file_count;
+
            break;
          }
        case CO_CONVERT_TO_COMPLETE:
@@ -372,7 +352,7 @@ convert_links (const char *file, struct urlpos *links)
            char *newlink = link->url->url;
            char *quoted_newlink = html_quote_string (newlink);

-            if (link->link_css_p)
+            if (link->link_css_p || link->link_noquote_html_p)
              p = replace_plain (p, link->size, fp, newlink);
            else if (!link->link_refresh_p)
              p = replace_attr (p, link->size, fp, quoted_newlink);
@@ -382,6 +362,7 @@ convert_links (const char *file, struct urlpos *links)

            DEBUGP (("TO_COMPLETE: <something> to %s at position %d in %s.\n",
                     newlink, link->pos, file));
+
            xfree (quoted_newlink);
            ++to_url_count;
            break;
@@ -460,14 +441,89 @@ construct_relative (const char *basefile, const char *linkfile)
        ++basedirs;
    }

-  /* Construct LINK as explained above. */
-  link = xmalloc (3 * basedirs + strlen (linkfile) + 1);
-  for (i = 0; i < basedirs; i++)
-    memcpy (link + 3 * i, "../", 3);
-  strcpy (link + 3 * i, linkfile);
+  if (!basedirs && (b = strpbrk (linkfile, "/:")) && *b == ':')
+    {
+      link = xmalloc (2 + strlen (linkfile) + 1);
+      memcpy (link, "./", 2);
+      strcpy (link + 2, linkfile);
+    }
+  else
+    {
+      /* Construct LINK as explained above. */
+      link = xmalloc (3 * basedirs + strlen (linkfile) + 1);
+      for (i = 0; i < basedirs; i++)
+        memcpy (link + 3 * i, "../", 3);
+      strcpy (link + 3 * i, linkfile);
+    }
+
  return link;
 }

+/* Construct and return a "transparent proxy" URL
+   reflecting changes made by --adjust-extension to the file component
+   (i.e., "basename") of the original URL, but leaving the "dirname"
+   of the URL (protocol://hostname... portion) untouched.
+
+   Think: populating a squid cache via a recursive wget scrape, where
+   changing URLs to work locally with "file://..." is NOT desirable.
+
+   Example:
+
+   if
+                     p = "//foo.com/bar.cgi?xyz"
+   and
+      link->local_name = "docroot/foo.com/bar.cgi?xyz.css"
+   then
+
+      new_construct_func(p, link);
+   will return
+      "//foo.com/bar.cgi?xyz.css"
+
+   Essentially, we do s/$(basename orig_url)/$(basename link->local_name)/
+*/
+static char *
+convert_basename (const char *p, const struct urlpos *link)
+{
+  int len = link->size;
+  char *url = NULL;
+  char *org_basename = NULL, *local_basename = NULL;
+  char *result = NULL;
+
+  if (*p == '"' || *p == '\'')
+    {
+      len -= 2;
+      p++;
+    }
+
+  url = xstrndup (p, len);
+
+  org_basename = strrchr (url, '/');
+  if (org_basename)
+    org_basename++;
+  else
+    org_basename = url;
+
+  local_basename = strrchr (link->local_name, '/');
+  if (local_basename)
+    local_basename++;
+  else
+    local_basename = url;
+
+  /*
+   * If the basenames differ, graft the adjusted basename (local_basename)
+   * onto the original URL.
+   */
+  if (strcmp (org_basename, local_basename) == 0)
+    result = url;
+  else
+    {
+      result = uri_merge (url, local_basename);
+      xfree (url);
+    }
+
+  return result;
+}
+
 /* Used by write_backup_file to remember which files have been
   written. */
 static struct hash_table *converted_files;
@@ -707,7 +763,7 @@ local_quote_string (const char *file, bool no_html_quote)

  return no_html_quote ? strdup (newname) : html_quote_string (newname);
 }
-
+
 /* Book-keeping code for dl_file_url_map, dl_url_file_map,
   downloaded_html_list, and downloaded_html_set.  Other code calls
   these functions to let us know that a file has been downloaded.  */
@@ -807,7 +863,7 @@ dissociate_urls_from_file (const char *file)
   URL has already been downloaded.  */

 void
-FNNAME_WTHREADS(register_download) (const char *url, const char *file)
+register_download (const char *url, const char *file)
 {
  char *old_file, *old_url;

@@ -887,7 +943,7 @@ FNNAME_WTHREADS(register_download) (const char *url, const char *file)
   register_download() above.  */

 void
-FNNAME_WTHREADS(register_redirection) (const char *from, const char *to)
+register_redirection (const char *from, const char *to)
 {
  char *file;

@@ -902,7 +958,7 @@ FNNAME_WTHREADS(register_redirection) (const char *from, const char *to)
 /* Register that the file has been deleted. */

 void
-FNNAME_WTHREADS(register_delete_file) (const char *file)
+register_delete_file (const char *file)
 {
  char *old_url, *old_file;

@@ -920,7 +976,7 @@ FNNAME_WTHREADS(register_delete_file) (const char *file)
 /* Register that FILE is an HTML file that has been downloaded. */

 void
-FNNAME_WTHREADS(register_html) (const char *url, const char *file)
+register_html (const char *file)
 {
  if (!downloaded_html_set)
    downloaded_html_set = make_string_hash_table (0);
@@ -930,7 +986,7 @@ FNNAME_WTHREADS(register_html) (const char *url, const char *file)
 /* Register that FILE is a CSS file that has been downloaded. */

 void
-FNNAME_WTHREADS(register_css) (const char *url, const char *file)
+register_css (const char *file)
 {
  if (!downloaded_css_set)
    downloaded_css_set = make_string_hash_table (0);
@@ -941,8 +997,8 @@ static void downloaded_files_free (void);

 /* Cleanup the data structures associated with this file.  */

-static void
-FNNAME_WTHREADS(convert_cleanup) (void)
+void
+convert_cleanup (void)
 {
  if (dl_file_url_map)
    {
@@ -962,13 +1018,13 @@ FNNAME_WTHREADS(convert_cleanup) (void)
  if (converted_files)
    string_set_free (converted_files);
 }
-
+
 /* Book-keeping code for downloaded files that enables extension
   hacks.  */

 /* This table should really be merged with dl_file_url_map and
   downloaded_html_files.  This was originally a list, but I changed
-   it to a hash table beause it was actually taking a lot of time to
+   it to a hash table because it was actually taking a lot of time to
   find things in it.  */

 static struct hash_table *downloaded_files_hash;
@@ -1017,7 +1073,7 @@ downloaded_mode_to_ptr (downloaded_file_t mode)
   URLs.  */

 downloaded_file_t
-FNNAME_WTHREADS(downloaded_file) (downloaded_file_t mode, const char *file)
+downloaded_file (downloaded_file_t mode, const char *file)
 {
  downloaded_file_t *ptr;

@@ -1058,7 +1114,7 @@ downloaded_files_free (void)
      downloaded_files_hash = NULL;
    }
 }
-
+
 /* The function returns the pointer to the malloc-ed quoted version of
   string s.  It will recognize and quote numeric and special graphic
   entities, as per RFC1866:
@@ -1131,17 +1187,6 @@ html_quote_string (const char *s)
  return res;
 }

-#ifdef ENABLE_THREADS
-THREAD_SAFE (downloaded_file_t, downloaded_file, (downloaded_file_t a, const char *b), (a, b));
-THREAD_SAFE_VOID (register_download, (const char *a, const char *b), (a, b));
-THREAD_SAFE_VOID (register_redirection, (const char *a, const char *b), (a, b));
-THREAD_SAFE_VOID (register_html, (const char *a, const char *b), (a, b));
-THREAD_SAFE_VOID (register_css, (const char *a, const char *b), (a, b));
-THREAD_SAFE_VOID (register_delete_file, (const char *a), (a));
-THREAD_SAFE_VOID (convert_cleanup, (void), ());
-THREAD_SAFE_VOID (convert_all_links, (void), ());
-#endif
-
 /*
 * vim: et ts=2 sw=2
 */
--- a/src/convert.h
+++ b/src/convert.h
@@ -1,5 +1,5 @@
 /* Declarations for convert.c
-   Copyright (C) 2003, 2004, 2005, 2006, 2009, 2010, 2011 Free Software
+   Copyright (C) 2003-2006, 2009-2011, 2015, 2018 Free Software
   Foundation, Inc.

 This file is part of GNU Wget.
@@ -40,6 +40,8 @@ enum convert_options {
  CO_NOCONVERT = 0,             /* don't convert this URL */
  CO_CONVERT_TO_RELATIVE,       /* convert to relative, e.g. to
                                   "../../otherdir/foo.gif" */
+  CO_CONVERT_BASENAME_ONLY,     /* convert the file portion only (basename)
+                                   leaving the rest of the URL unchanged */
  CO_CONVERT_TO_COMPLETE,       /* convert to absolute, e.g. to
                                   "http://orighost/somedir/bar.jpg". */
  CO_NULLIFY_BASE               /* change to empty string. */
@@ -67,6 +69,7 @@ struct urlpos {
  unsigned int link_base_p  :1;     /* the url came from <base href=...> */
  unsigned int link_inline_p    :1; /* needed to render the page */
  unsigned int link_css_p   :1;     /* the url came from CSS */
+  unsigned int link_noquote_html_p :1; /* from HTML, but doesn't need " */
  unsigned int link_expect_html :1; /* expected to contain HTML */
  unsigned int link_expect_css  :1; /* expected to contain CSS */

@@ -98,12 +101,14 @@ typedef enum
 } downloaded_file_t;

 downloaded_file_t downloaded_file (downloaded_file_t, const char *);
-void register_download (const char *a, const char *b);
+
+void register_download (const char *, const char *);
+void register_redirection (const char *, const char *);
+void register_html (const char *);
+void register_css (const char *);
+void register_delete_file (const char *);
 void convert_all_links (void);
-void register_delete_file (const char *a);
-void register_redirection (const char *a, const char *b);
-void register_css (const char *a, const char *b);
-void register_html (const char *a, const char *b);
+void convert_cleanup (void);

 char *html_quote_string (const char *);

--- a/src/cookies.c
+++ b/src/cookies.c
@@ -1,6 +1,5 @@
 /* Support for cookies.
-   Copyright (C) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009,
-   2010, 2011 Free Software Foundation, Inc.
+   Copyright (C) 2001-2011, 2015, 2018 Free Software Foundation, Inc.

 This file is part of GNU Wget.

@@ -45,6 +44,7 @@ as that of the covered work.  */

 #include "wget.h"

+#include <stdint.h>
 #include <stdio.h>
 #include <string.h>
 #include <stdlib.h>
@@ -58,7 +58,9 @@ as that of the covered work.  */
 #include "hash.h"
 #include "cookies.h"
 #include "http.h"               /* for http_atotm */
-
+#include "c-strcase.h"
+
+
 /* Declarations of `struct cookie' and the most basic functions. */

 /* Cookie jar serves as cookie storage and a means of retrieving
@@ -151,13 +153,13 @@ cookie_expired_p (const struct cookie *c)
 static void
 delete_cookie (struct cookie *cookie)
 {
-  xfree_null (cookie->domain);
-  xfree_null (cookie->path);
-  xfree_null (cookie->attr);
-  xfree_null (cookie->value);
+  xfree (cookie->domain);
+  xfree (cookie->path);
+  xfree (cookie->attr);
+  xfree (cookie->value);
  xfree (cookie);
 }
-
+
 /* Functions for storing cookies.

   All cookies can be reached beginning with jar->chains.  The key in
@@ -302,7 +304,12 @@ discard_matching_cookie (struct cookie_jar *jar, struct cookie *cookie)

          res = hash_table_get_pair (jar->chains, victim->domain,
                                     &chain_key, NULL);
-          assert (res != 0);
+
+          if (res == 0)
+            {
+              logprintf (LOG_VERBOSE, _("Unable to get cookie for %s\n"),
+                         victim->domain);
+            }
          if (!victim->next)
            {
              /* VICTIM was the only cookie in the chain.  Destroy the
@@ -317,7 +324,7 @@ discard_matching_cookie (struct cookie_jar *jar, struct cookie *cookie)
      DEBUGP (("Discarded old cookie.\n"));
    }
 }
-
+
 /* Functions for parsing the `Set-Cookie' header, and creating new
   cookies from the wire.  */

@@ -369,7 +376,7 @@ parse_set_cookie (const char *set_cookie, bool silent)
        {
          if (!TOKEN_NON_EMPTY (value))
            goto error;
-          xfree_null (cookie->domain);
+          xfree (cookie->domain);
          /* Strictly speaking, we should set cookie->domain_exact if the
             domain doesn't begin with a dot.  But many sites set the
             domain to "foo.com" and expect "subhost.foo.com" to get the
@@ -382,7 +389,7 @@ parse_set_cookie (const char *set_cookie, bool silent)
        {
          if (!TOKEN_NON_EMPTY (value))
            goto error;
-          xfree_null (cookie->path);
+          xfree (cookie->path);
          cookie->path = strdupdelim (value.b, value.e);
        }
      else if (TOKEN_IS (name, "expires"))
@@ -456,7 +463,7 @@ parse_set_cookie (const char *set_cookie, bool silent)

 #undef TOKEN_IS
 #undef TOKEN_NON_EMPTY
-
+
 /* Sanity checks.  These are important, otherwise it is possible for
   mailcious attackers to destroy important cookie information and/or
   violate your privacy.  */
@@ -477,7 +484,7 @@ parse_set_cookie (const char *set_cookie, bool silent)
 /* Check whether ADDR matches <digits>.<digits>.<digits>.<digits>.

   We don't want to call network functions like inet_addr() because
-   all we need is a check, preferrably one that is small, fast, and
+   all we need is a check, preferably one that is small, fast, and
   well-defined.  */

 static bool
@@ -508,7 +515,7 @@ numeric_address_p (const char *addr)
   psl on our own, if libpsl is compiled without a public suffix list,
   fall back to using the original "tail matching" heuristic. Also if
   libpsl is unable to convert the domain to lowercase, which means that
-   it doesnt have any runtime conversion support, we again fall back to
+   it doesn't have any runtime conversion support, we again fall back to
   "tail matching" since libpsl states the results are unpredictable with
   upper case strings.
   */
@@ -518,19 +525,51 @@ check_domain_match (const char *cookie_domain, const char *host)
 {

 #ifdef HAVE_LIBPSL
-  DEBUGP (("cdm: 1"));
+  static int init_psl;
+  static const psl_ctx_t *psl;
+
  char *cookie_domain_lower = NULL;
  char *host_lower = NULL;
-  const psl_ctx_t *psl;
  int is_acceptable;

-  if (!(psl = psl_builtin()))
+  DEBUGP (("cdm: 1\n"));
+  if (!init_psl)
    {
-      DEBUGP (("\nlibpsl not built with a public suffix list. "
-               "Falling back to simple heuristics.\n"));
-      goto no_psl;
-    }
+      init_psl = 1;

+#ifdef HAVE_PSL_LATEST
+      if ((psl = psl_latest (NULL)))
+        goto have_psl;
+
+      DEBUGP (("\nPSL: Failed to load any PSL data. "
+               "Falling back to insecure heuristics.\n"));
+#else
+      if ((psl = psl_builtin ()) && !psl_builtin_outdated ())
+        goto have_psl;
+
+      DEBUGP (("\nPSL: built-in data outdated. "
+               "Trying to load data from %s.\n",
+              quote (psl_builtin_filename ())));
+
+      if ((psl = psl_load_file (psl_builtin_filename ())))
+        goto have_psl;
+
+      DEBUGP (("\nPSL: %s not found or not readable. "
+               "Falling back to built-in data.\n",
+              quote (psl_builtin_filename ())));
+
+      if (!(psl = psl_builtin ()))
+        {
+          DEBUGP (("\nPSL: libpsl not built with a public suffix list. "
+                   "Falling back to insecure heuristics.\n"));
+          goto no_psl;
+        }
+#endif
+    }
+  else if (!psl)
+    goto no_psl;
+
+have_psl:
  if (psl_str_to_utf8lower (cookie_domain, NULL, NULL, &cookie_domain_lower) == PSL_SUCCESS &&
      psl_str_to_utf8lower (host, NULL, NULL, &host_lower) == PSL_SUCCESS)
    {
@@ -555,13 +594,13 @@ no_psl:
 #endif

  /* For efficiency make some elementary checks first */
-  DEBUGP (("cdm: 2"));
+  DEBUGP (("cdm: 2\n"));

  /* For the sake of efficiency, check for exact match first. */
  if (0 == strcasecmp (cookie_domain, host))
    return true;

-  DEBUGP ((" 3"));
+  DEBUGP (("cdm: 3\n"));

  /* HOST must match the tail of cookie_domain. */
  if (!match_tail (host, cookie_domain, true))
@@ -601,7 +640,7 @@ no_psl:
    if (*p == '.')
      /* Ignore leading period in this calculation. */
      ++p;
-    DEBUGP ((" 4"));
+    DEBUGP (("cdm: 4\n"));
    for (out = 0; !out; p++)
      switch (*p)
        {
@@ -627,12 +666,12 @@ no_psl:
          ++ldcl;
        }

-    DEBUGP ((" 5"));
+    DEBUGP (("cdm: 5\n"));

    if (dccount < 2)
      return false;

-    DEBUGP ((" 6"));
+    DEBUGP (("cdm: 6\n"));

    if (dccount == 2)
      {
@@ -652,7 +691,7 @@ no_psl:
      }
  }

-  DEBUGP ((" 7"));
+  DEBUGP (("cdm: 7\n"));

  /* Don't allow the host "foobar.com" to set a cookie for domain
     "bar.com".  */
@@ -667,7 +706,7 @@ no_psl:
        return false;
    }

-  DEBUGP ((" 8"));
+  DEBUGP (("cdm: 8\n"));

  return true;
 }
@@ -692,7 +731,7 @@ check_path_match (const char *cookie_path, const char *path)
  s = PS_newstr;                                                \
 } while (0)

-
+
 /* Process the HTTP `Set-Cookie' header.  This results in storing the
   cookie or discarding a matching one, or ignoring it completely, all
   depending on the contents.  */
@@ -776,7 +815,7 @@ cookie_handle_set_cookie (struct cookie_jar *jar,
  if (cookie)
    delete_cookie (cookie);
 }
-
+
 /* Support for sending out cookies in HTTP requests, based on
   previously stored cookies.  Entry point is
   `build_cookies_request'.  */
@@ -1011,7 +1050,7 @@ cookie_header (struct cookie_jar *jar, const char *host,

  struct cookie *cookie;
  struct weighed_cookie *outgoing;
-  int count, i, ocnt;
+  size_t count, i, ocnt;
  char *result;
  int result_size, pos;
  PREPEND_SLASH (path);         /* see cookie_handle_set_cookie */
@@ -1025,7 +1064,7 @@ cookie_header (struct cookie_jar *jar, const char *host,
  chain_count = find_chains_of_host (jar, host, chains);

  /* No cookies for this host. */
-  if (!chain_count)
+  if (chain_count <= 0)
    return NULL;

  cookies_now = time (NULL);
@@ -1036,7 +1075,7 @@ cookie_header (struct cookie_jar *jar, const char *host,

  /* Count the number of matching cookies. */
  count = 0;
-  for (i = 0; i < chain_count; i++)
+  for (i = 0; i < (unsigned) chain_count; i++)
    for (cookie = chains[i]; cookie; cookie = cookie->next)
      if (cookie_matches_url (cookie, host, port, path, secflag, NULL))
        ++count;
@@ -1044,12 +1083,14 @@ cookie_header (struct cookie_jar *jar, const char *host,
    return NULL;                /* no cookies matched */

  /* Allocate the array. */
-  outgoing = alloca_array (struct weighed_cookie, count);
+  if (count > SIZE_MAX / sizeof (struct weighed_cookie))
+    return NULL;                /* unable to process so many cookies */
+  outgoing = xmalloc (count * sizeof (struct weighed_cookie));

  /* Fill the array with all the matching cookies from the chains that
     match HOST. */
  ocnt = 0;
-  for (i = 0; i < chain_count; i++)
+  for (i = 0; i < (unsigned) chain_count; i++)
    for (cookie = chains[i]; cookie; cookie = cookie->next)
      {
        int pg;
@@ -1104,10 +1145,11 @@ cookie_header (struct cookie_jar *jar, const char *host,
        }
    }
  result[pos++] = '\0';
+  xfree (outgoing);
  assert (pos == result_size);
  return result;
 }
-
+
 /* Support for loading and saving cookies.  The format used for
   loading and saving should be the format of the `cookies.txt' file
   used by Netscape and Mozilla, at least the Unix versions.
@@ -1346,7 +1388,7 @@ cookie_jar_save (struct cookie_jar *jar, const char *file)

  DEBUGP (("Done saving cookies.\n"));
 }
-
+
 /* Clean up cookie-related data. */

 void
@@ -1369,7 +1411,7 @@ cookie_jar_delete (struct cookie_jar *jar)
  hash_table_destroy (jar->chains);
  xfree (jar);
 }
-
+
 /* Test cases.  Currently this is only tests parse_set_cookies.  To
   use, recompile Wget with -DTEST_COOKIES and call test_cookies()
   from main.  */
@@ -1438,8 +1480,8 @@ test_cookies (void)
              printf ("Invalid value %d for '%s' (expected '%s', got '%s')\n",
                      j / 2 + 1, data, expected[j + 1], v);
            j += 2;
-            free (n);
-            free (v);
+            xfree (n);
+            xfree (v);
          }
        if (expected[j])
          printf ("Too few parameters for '%s'\n", data);
--- a/src/cookies.h
+++ b/src/cookies.h
@@ -1,6 +1,5 @@
 /* Support for cookies.
-   Copyright (C) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009,
-   2010, 2011 Free Software Foundation, Inc.
+   Copyright (C) 2001-2011, 2015, 2018 Free Software Foundation, Inc.

 This file is part of GNU Wget.

--- a/src/css-tokens.h
+++ b/src/css-tokens.h
@@ -1,5 +1,6 @@
 /* Declarations for css.lex
-   Copyright (C) 2006, 2009, 2010, 2011 Free Software Foundation, Inc.
+   Copyright (C) 2006, 2009-2011, 2015, 2018 Free Software Foundation,
+   Inc.

 This file is part of GNU Wget.

--- a/src/css-url.c
+++ b/src/css-url.c
@@ -1,5 +1,5 @@
 /* Collect URLs from CSS source.
-   Copyright (C) 1998, 2000, 2001, 2002, 2003, 2009, 2010, 2011, 2014 Free
+   Copyright (C) 1998, 2000-2003, 2009-2011, 2014-2015, 2018 Free
   Software Foundation, Inc.

 This file is part of GNU Wget.
@@ -52,12 +52,14 @@ as that of the covered work.  */
 #include "html-url.h"
 #include "css-tokens.h"
 #include "css-url.h"
+#include "xstrndup.h"

 /* from lex.yy.c */
 extern char *yytext;
 extern int yyleng;
 typedef struct yy_buffer_state *YY_BUFFER_STATE;
 extern YY_BUFFER_STATE yy_scan_bytes (const char *bytes,int len  );
+extern void yy_delete_buffer (YY_BUFFER_STATE  b);
 extern int yylex (void);

 /*
@@ -97,7 +99,7 @@ get_uri_string (const char *at, int *pos, int *length)
      *length -= 2;
    }

-  return xstrdup (at + *pos);
+  return xstrndup (at + *pos, *length);
 }

 void
@@ -108,9 +110,10 @@ get_urls_css (struct map_context *ctx, int offset, int buf_length)
  int buffer_pos = 0;
  int pos, length;
  char *uri;
+  YY_BUFFER_STATE b;

  /* tell flex to scan from this buffer */
-  yy_scan_bytes (ctx->text + offset, buf_length);
+  b = yy_scan_bytes (ctx->text + offset, buf_length);

  while((token = yylex()) != CSSEOF)
    {
@@ -187,6 +190,9 @@ get_urls_css (struct map_context *ctx, int offset, int buf_length)
        }
      buffer_pos += yyleng;
    }
+
+  yy_delete_buffer(b);
+
  DEBUGP (("\n"));
 }

--- a/src/css-url.h
+++ b/src/css-url.h
@@ -1,5 +1,6 @@
 /* Declarations for css-url.c.
-   Copyright (C) 2006, 2009, 2010, 2011 Free Software Foundation, Inc.
+   Copyright (C) 2006, 2009-2011, 2015, 2018 Free Software Foundation,
+   Inc.

 This file is part of GNU Wget.

@@ -30,7 +31,6 @@ as that of the covered work.  */
 #ifndef CSS_URL_H
 #define CSS_URL_H

-void get_urls_css (struct map_context *, int, int);
 void get_urls_css (struct map_context *, int, int);
 struct urlpos *get_urls_css_file (const char *, const char *);

--- a/src/css.l
+++ b/src/css.l
@@ -6,7 +6,8 @@
 %{
 /* Lex source for CSS tokenizing.
   Taken from http://www.w3.org/TR/CSS21/grammar.html#q2
-   Copyright (C) 2006, 2009, 2010, 2011 Free Software Foundation, Inc.
+   Copyright (C) 2006, 2009-2011, 2015, 2018 Free Software Foundation,
+   Inc.

 This file is part of GNU Wget.

--- a/src/exits.c
+++ b/src/exits.c
@@ -1,5 +1,5 @@
 /* Exit status handling.
-   Copyright (C) 2009, 2010, 2011, 2012 Free Software Foundation, Inc.
+   Copyright (C) 2009-2012, 2015, 2018 Free Software Foundation, Inc.

   This file is part of GNU Wget.

@@ -56,7 +56,7 @@ get_status_for_err (uerr_t err)
    case FTPNSFOD: case FTPUNKNOWNTYPE: case FTPSRVERR:
    case FTPRETRINT: case FTPRESTFAIL: case FTPNOPASV:
    case CONTNOTSUPPORTED: case RANGEERR: case RETRBADPATTERN:
-    case PROXERR:
+    case PROXERR: case GATEWAYTIMEOUT:
      return WGET_EXIT_SERVER_ERROR;
    case URLERROR: case QUOTEXC: case SSLINITFAILED: case UNKNOWNATTR:
    default:
--- a/src/exits.h
+++ b/src/exits.h
@@ -1,5 +1,5 @@
 /* Exit status related declarations.
-   Copyright (C) 2009, 2010, 2011, 2012 Free Software Foundation, Inc.
+   Copyright (C) 2009-2012, 2015, 2018 Free Software Foundation, Inc.

 This file is part of GNU Wget.

--- a/src/ftp-basic.c
+++ b/src/ftp-basic.c
@@ -1,6 +1,5 @@
 /* Basic FTP routines.
-   Copyright (C) 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004,
-   2005, 2006, 2007, 2008, 2009, 2010, 2011, 2014 Free Software Foundation,
+   Copyright (C) 1996-2011, 2014-2015, 2018 Free Software Foundation,
   Inc.

 This file is part of GNU Wget.
@@ -43,8 +42,9 @@ as that of the covered work.  */
 #include "host.h"
 #include "ftp.h"
 #include "retr.h"
+#include "c-strcase.h"
+

-
 /* Get the response of FTP server and allocate enough room to handle
   it.  <CR> and <LF> characters are stripped from the line, and the
   line is 0-terminated.  All the response lines but the last one are
@@ -134,6 +134,23 @@ ftp_request (const char *command, const char *value)
  return res;
 }

+uerr_t
+ftp_greeting (int csock)
+{
+  uerr_t err = FTPOK;
+  char *response = NULL;
+
+  err = ftp_response (csock, &response);
+  if (err != FTPOK)
+    goto bail;
+  if (*response != '2')
+    err = FTPSRVERR;
+
+bail:
+  if (response)
+    xfree (response);
+  return err;
+}
 /* Sends the USER and PASS commands to the server, to control
   connection socket csock.  */
 uerr_t
@@ -143,16 +160,6 @@ ftp_login (int csock, const char *acc, const char *pass)
  char *request, *respline;
  int nwritten;

-  /* Get greeting.  */
-  err = ftp_response (csock, &respline);
-  if (err != FTPOK)
-    return err;
-  if (*respline != '2')
-    {
-      xfree (respline);
-      return FTPSRVERR;
-    }
-  xfree (respline);
  /* Send USER username.  */
  request = ftp_request ("USER", acc);
  nwritten = fd_write (csock, request, strlen (request), -1);
@@ -190,7 +197,7 @@ ftp_login (int csock, const char *acc, const char *pass)
    for (i = 0; i < countof (skey_head); i++)
      {
        int l = strlen (skey_head[i]);
-        if (0 == strncasecmp (skey_head[i], respline, l))
+        if (0 == c_strncasecmp (skey_head[i], respline, l))
          {
            seed = respline + l;
            break;
@@ -480,6 +487,119 @@ ftp_eprt (int csock, int *local_sock)
 }
 #endif

+#ifdef HAVE_SSL
+/*
+ * The following three functions defined into this #ifdef block
+ * wrap the extended FTP commands defined in RFC 2228 (FTP Security Extensions).
+ * Currently, only FTPS is supported, so these functions are only compiled when SSL
+ * support is available, because there's no point in using FTPS when there's no SSL.
+ * Shall someone add new secure FTP protocols in the future, feel free to remove this
+ * #ifdef, or add new constants to it.
+ */
+
+/*
+ * Sends an AUTH command as defined by RFC 2228,
+ * deriving its argument from the scheme. For example, if the provided scheme
+ * is SCHEME_FTPS, the command sent will be "AUTH TLS". Currently, this is the only
+ * scheme supported, so this function will return FTPNOAUTH when supplied a different
+ * one. It will also return FTPNOAUTH if the target server does not support FTPS.
+ */
+uerr_t
+ftp_auth (int csock, enum url_scheme scheme)
+{
+  uerr_t err = 0;
+  int written = 0;
+  char *request = NULL, *response = NULL;
+
+  if (scheme == SCHEME_FTPS)
+    {
+      request = ftp_request ("AUTH", "TLS");
+      written = fd_write (csock, request, strlen (request), -1);
+      if (written < 0)
+        {
+          err = WRITEFAILED;
+          goto bail;
+        }
+      err = ftp_response (csock, &response);
+      if (err != FTPOK)
+        goto bail;
+      if (*response != '2')
+        err = FTPNOAUTH;
+    }
+  else
+    err = FTPNOAUTH;
+
+bail:
+  xfree (request);
+  xfree (response);
+
+  return err;
+}
+
+uerr_t
+ftp_pbsz (int csock, int pbsz)
+{
+  uerr_t err = 0;
+  int written = 0;
+  char spbsz[5];
+  char *request = NULL, *response = NULL;
+
+  snprintf (spbsz, 5, "%d", pbsz);
+  request = ftp_request ("PBSZ", spbsz);
+  written = fd_write (csock, request, strlen (request), -1);
+  if (written < 0)
+    {
+      err = WRITEFAILED;
+      goto bail;
+    }
+
+  err = ftp_response (csock, &response);
+  if (err != FTPOK)
+    goto bail;
+  if (*response != '2')
+    err = FTPNOPBSZ;
+
+bail:
+  xfree (request);
+  xfree (response);
+
+  return err;
+}
+
+uerr_t
+ftp_prot (int csock, enum prot_level prot)
+{
+  uerr_t err = 0;
+  int written = 0;
+  char *request = NULL, *response = NULL;
+  /* value must be a single character value */
+  char value[2];
+
+  value[0] = prot;
+  value[1] = '\0';
+
+  request = ftp_request ("PROT", value);
+  written = fd_write (csock, request, strlen (request), -1);
+  if (written < 0)
+    {
+      err = WRITEFAILED;
+      goto bail;
+    }
+
+  err = ftp_response (csock, &response);
+  if (err != FTPOK)
+    goto bail;
+  if (*response != '2')
+    err = FTPNOPROT;
+
+bail:
+  xfree (request);
+  xfree (response);
+
+  return err;
+}
+#endif /* HAVE_SSL */
+
 /* Similar to ftp_port, but uses `PASV' to initiate the passive FTP
   transfer.  Reads the response from server and parses it.  Reads the
   host and port addresses and returns them.  */
@@ -784,16 +904,8 @@ ftp_epsv (int csock, ip_address *ip, int *port)
    }

  /* Finally, get the port number */
-  tport = 0;
-  for (i = 1; c_isdigit (*s); s++)
-    {
-      if (i > 5)
-        {
-          xfree (respline);
-          return FTPINVPASV;
-        }
+  for (tport = 0, i = 0; i < 5 && c_isdigit (*s); i++, s++)
      tport = (*s - '0') + 10 * tport;
-    }

  /* Make sure that the response terminates correcty */
  if (*s++ != delim)
@@ -965,16 +1077,18 @@ ftp_list (int csock, const char *file, bool avoid_list_a, bool avoid_list,
  bool ok = false;
  size_t i = 0;

-  *list_a_used = false;
-
  /* 2013-10-12 Andrea Urbani (matfanjol)
     For more information about LIST and "LIST -a" please look at ftp.c,
     function getftp, text "__LIST_A_EXPLANATION__".

     If somebody changes the following commands, please, checks also the
     later "i" variable.  */
-  const char *list_commands[] = { "LIST -a",
-                                  "LIST" };
+  static const char *list_commands[] = {
+    "LIST -a",
+    "LIST"
+  };
+
+  *list_a_used = false;

  if (avoid_list_a)
    {
@@ -1068,25 +1182,25 @@ ftp_syst (int csock, enum stype *server_type, enum ustype *unix_type)

  if (request == NULL)
    *server_type = ST_OTHER;
-  else if (!strcasecmp (request, "VMS"))
+  else if (!c_strcasecmp (request, "VMS"))
    *server_type = ST_VMS;
-  else if (!strcasecmp (request, "UNIX"))
+  else if (!c_strcasecmp (request, "UNIX"))
    {
      *server_type = ST_UNIX;
      /* 2013-10-17 Andrea Urbani (matfanjol)
         I check more in depth the system type */
-      if (!strncasecmp (ftp_last_respline, "215 UNIX Type: L8", 17))
+      if (!c_strncasecmp (ftp_last_respline, "215 UNIX Type: L8", 17))
        *unix_type = UST_TYPE_L8;
-      else if (!strncasecmp (ftp_last_respline,
+      else if (!c_strncasecmp (ftp_last_respline,
                             "215 UNIX MultiNet Unix Emulation V5.3(93)", 41))
        *unix_type = UST_MULTINET;
    }
-  else if (!strcasecmp (request, "WINDOWS_NT")
-           || !strcasecmp (request, "WINDOWS2000"))
+  else if (!c_strcasecmp (request, "WINDOWS_NT")
+           || !c_strcasecmp (request, "WINDOWS2000"))
    *server_type = ST_WINNT;
-  else if (!strcasecmp (request, "MACOS"))
+  else if (!c_strcasecmp (request, "MACOS"))
    *server_type = ST_MACOS;
-  else if (!strcasecmp (request, "OS/400"))
+  else if (!c_strcasecmp (request, "OS/400"))
    *server_type = ST_OS400;
  else
    *server_type = ST_OTHER;
@@ -1135,7 +1249,7 @@ ftp_pwd (int csock, char **pwd)
    goto err;

  /* Has the `pwd' been already allocated?  Free! */
-  xfree_null (*pwd);
+  xfree (*pwd);

  *pwd = xstrdup (request);

--- a/src/ftp-ls.c
+++ b/src/ftp-ls.c
@@ -1,7 +1,5 @@
 /* Parsing FTP `ls' output.
-   Copyright (C) 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004,
-   2005, 2006, 2007, 2008, 2009, 2010, 2011 Free Software Foundation,
-   Inc.
+   Copyright (C) 1996-2011, 2015, 2018 Free Software Foundation, Inc.

 This file is part of GNU Wget.

@@ -42,6 +40,7 @@ as that of the covered work.  */
 #include "url.h"
 #include "convert.h"            /* for html_quote_string prototype */
 #include "retr.h"               /* for output_stream */
+#include "c-strcase.h"

 /* Converts symbolic permissions to number-style ones, e.g. string
   rwxr-xr-x to 755.  For now, it knows nothing of
@@ -121,7 +120,7 @@ ftp_parse_unix_ls (const char *file, int ignore_perms)
    {
      len = clean_line (line, len);
      /* Skip if total...  */
-      if (!strncasecmp (line, "total", 5))
+      if (!c_strncasecmp (line, "total", 5))
        continue;
      /* Get the first token (permissions).  */
      tok = strtok (line, " ");
@@ -166,12 +165,12 @@ ftp_parse_unix_ls (const char *file, int ignore_perms)
              /*cur.perms = 1023;*/     /* #### What is this?  --hniksic */
              cur.perms = 0644;
            }
-          DEBUGP (("implicit perms %0o; ", cur.perms));
+          DEBUGP (("implicit perms %0o; ", (unsigned) cur.perms));
        }
       else
         {
           cur.perms = symperms (tok + 1);
-           DEBUGP (("perms %0o; ", cur.perms));
+           DEBUGP (("perms %0o; ", (unsigned) cur.perms));
         }

      error = ignore = 0;       /* Erroneous and ignoring entries are
@@ -199,7 +198,7 @@ ftp_parse_unix_ls (const char *file, int ignore_perms)
          if (next < 0)         /* a month name was not encountered */
            {
              for (i = 0; i < 12; i++)
-                if (!strcasecmp (tok, months[i]))
+                if (!c_strcasecmp (tok, months[i]))
                  break;
              /* If we got a month, it means the token before it is the
                 size, and the filename is three tokens away.  */
@@ -362,8 +361,8 @@ ftp_parse_unix_ls (const char *file, int ignore_perms)
      if (error || ignore)
        {
          DEBUGP (("Skipping.\n"));
-          xfree_null (cur.name);
-          xfree_null (cur.linkto);
+          xfree (cur.name);
+          xfree (cur.linkto);
          continue;
        }

@@ -477,6 +476,7 @@ ftp_parse_winnt_ls (const char *file)
        }
      /* Now it is possible to determine the position of the first symbol in
         filename. */
+      memset(&cur, 0, sizeof (cur));
      cur.name = xstrdup(filename);
      DEBUGP (("Name: '%s'\n", cur.name));

@@ -775,14 +775,14 @@ ftp_parse_vms_ls (const char *file)
         what will work in a CWD command.
      */
      len = strlen (tok);
-      if (!strncasecmp((tok + (len - 4)), ".DIR", 4))
+      if (!c_strncasecmp((tok + (len - 4)), ".DIR", 4))
        {
          *(tok+ (len - 4)) = '\0'; /* Discard ".DIR". */
          cur.type  = FT_DIRECTORY;
          cur.perms = VMS_DEFAULT_PROT_DIR;
          DEBUGP (("Directory (nv)\n"));
        }
-      else if (!strncasecmp ((tok + (len - 6)), ".DIR;1", 6))
+      else if (!c_strncasecmp ((tok + (len - 6)), ".DIR;1", 6))
        {
          *(tok+ (len - 6)) = '\0'; /* Discard ".DIR;1". */
          cur.type  = FT_DIRECTORY;
@@ -930,7 +930,7 @@ ftp_parse_vms_ls (const char *file)
                    }
                }
              cur.perms = perms;
-              DEBUGP (("Prot.  perms = %0o.\n", cur.perms));
+              DEBUGP (("Prot.  perms = %0o.\n", (unsigned) cur.perms));
            }
          else
            {
@@ -1051,7 +1051,7 @@ Unsupported listing type, trying Unix listing parser.\n"));
      return ftp_parse_unix_ls (file, 0);
    }
 }
-
+
 /* Stuff for creating FTP index. */

 /* The function creates an HTML index containing references to given
@@ -1088,7 +1088,7 @@ ftp_index (const char *file, struct url *u, struct fileinfo *f)
      else
        upwd = concat_strings (tmpu, "@", (char *) 0);
      xfree (tmpu);
-      xfree_null (tmpp);
+      xfree (tmpp);
    }
  else
    upwd = xstrdup ("");
--- a/src/ftp-opie.c
+++ b/src/ftp-opie.c
@@ -1,6 +1,5 @@
 /* Opie (s/key) support for FTP.
-   Copyright (C) 1998, 1999, 2000, 2001, 2002, 2003, 2004, 2005, 2006,
-   2007, 2008, 2009, 2010, 2011 Free Software Foundation, Inc.
+   Copyright (C) 1998-2011, 2015, 2018 Free Software Foundation, Inc.

 This file is part of GNU Wget.

--- a/src/ftp.c
+++ b/src/ftp.c
--- a/src/ftp.h
+++ b/src/ftp.h
@@ -1,7 +1,5 @@
 /* Declarations for FTP support.
-   Copyright (C) 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004,
-   2005, 2006, 2007, 2008, 2009, 2010, 2011 Free Software Foundation,
-   Inc.
+   Copyright (C) 1996-2011, 2015, 2018 Free Software Foundation, Inc.

 This file is part of GNU Wget.

@@ -32,9 +30,8 @@ as that of the covered work.  */
 #ifndef FTP_H
 #define FTP_H

-#include "wget.h"
-
 #include "host.h"
+#include "url.h"

 /* System types. */
 enum stype
@@ -55,12 +52,27 @@ enum ustype
  UST_OTHER
 };

-extern char ftp_last_respline[];
+#ifdef HAVE_SSL
+/* Data channel protection levels (to be used with PBSZ) */
+enum prot_level
+{
+  PROT_CLEAR = 'C',
+  PROT_SAFE = 'S',
+  PROT_CONFIDENTIAL = 'E',
+  PROT_PRIVATE = 'P'
+};
+#endif

 uerr_t ftp_response (int, char **);
+uerr_t ftp_greeting (int);
 uerr_t ftp_login (int, const char *, const char *);
 uerr_t ftp_port (int, int *);
 uerr_t ftp_pasv (int, ip_address *, int *);
+#ifdef HAVE_SSL
+uerr_t ftp_auth (int, enum url_scheme);
+uerr_t ftp_pbsz (int, int);
+uerr_t ftp_prot (int, enum prot_level);
+#endif
 #ifdef ENABLE_IPV6
 uerr_t ftp_lprt (int, int *);
 uerr_t ftp_lpsv (int, ip_address *, int *);
@@ -146,15 +158,17 @@ enum wget_ftp_fstatus
  AVOID_LIST    = 0x0008,   /* It tells us if during this
                               session we have to avoid to use
                               "LIST". */
-  LIST_AFTER_LIST_A_CHECK_DONE  = 0x0010
+  LIST_AFTER_LIST_A_CHECK_DONE  = 0x0010,
                            /* It tells us if we have already
                               checked "LIST" after the first
                               "LIST -a" to handle the case of
                               file/folders named "-a". */
+  DATA_CHANNEL_SECURITY = 0x0020 /* Establish a secure data channel */
 };

 struct fileinfo *ftp_parse_ls (const char *, const enum stype);
-uerr_t ftp_loop (struct url *, char **, int *, struct url *, bool, bool, struct range *);
+uerr_t ftp_loop (struct url *, struct url *, char **, int *, struct url *,
+                 bool, bool);

 uerr_t ftp_index (const char *, struct url *, struct fileinfo *);

--- a/src/gettext.h
+++ b/src/gettext.h
@@ -1,272 +0,0 @@
-/* Convenience header for conditional use of GNU <libintl.h>.
-   Copyright (C) 1995, 1996, 1997, 1998, 2000, 2001, 2002, 2004, 2005,
-   2006, 2009, 2010, 2011 Free Software Foundation, Inc.
-
-   This program is free software; you can redistribute it and/or modify it
-   under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 2, or (at your option)
-   any later version.
-
-   This program is distributed in the hope that it will be useful,
-   but WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Library General Public License for more details.
-
-   You should have received a copy of the GNU General Public
-   License along with this program; if not, write to the Free Software
-   Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301,
-   USA.  */
-
-#ifndef _LIBGETTEXT_H
-#define _LIBGETTEXT_H 1
-
-/* NLS can be disabled through the configure --disable-nls option.  */
-#if ENABLE_NLS
-
-/* Get declarations of GNU message catalog functions.  */
-# include <libintl.h>
-
-/* You can set the DEFAULT_TEXT_DOMAIN macro to specify the domain used by
-   the gettext() and ngettext() macros.  This is an alternative to calling
-   textdomain(), and is useful for libraries.  */
-# ifdef DEFAULT_TEXT_DOMAIN
-#  undef gettext
-#  define gettext(Msgid) \
-     dgettext (DEFAULT_TEXT_DOMAIN, Msgid)
-#  undef ngettext
-#  define ngettext(Msgid1, Msgid2, N) \
-     dngettext (DEFAULT_TEXT_DOMAIN, Msgid1, Msgid2, N)
-# endif
-
-#else
-
-/* Solaris /usr/include/locale.h includes /usr/include/libintl.h, which
-   chokes if dcgettext is defined as a macro.  So include it now, to make
-   later inclusions of <locale.h> a NOP.  We don't include <libintl.h>
-   as well because people using "gettext.h" will not include <libintl.h>,
-   and also including <libintl.h> would fail on SunOS 4, whereas <locale.h>
-   is OK.  */
-#if defined(__sun)
-# include <locale.h>
-#endif
-
-/* Many header files from the libstdc++ coming with g++ 3.3 or newer include
-   <libintl.h>, which chokes if dcgettext is defined as a macro.  So include
-   it now, to make later inclusions of <libintl.h> a NOP.  */
-#if defined(__cplusplus) && defined(__GNUG__) && (__GNUC__ >= 3)
-# include <cstdlib>
-# if (__GLIBC__ >= 2) || _GLIBCXX_HAVE_LIBINTL_H
-#  include <libintl.h>
-# endif
-#endif
-
-/* Disabled NLS.
-   The casts to 'const char *' serve the purpose of producing warnings
-   for invalid uses of the value returned from these functions.
-   On pre-ANSI systems without 'const', the config.h file is supposed to
-   contain "#define const".  */
-# define gettext(Msgid) ((const char *) (Msgid))
-# define dgettext(Domainname, Msgid) ((void) (Domainname), gettext (Msgid))
-# define dcgettext(Domainname, Msgid, Category) \
-    ((void) (Category), dgettext (Domainname, Msgid))
-# define ngettext(Msgid1, Msgid2, N) \
-    ((N) == 1 \
-     ? ((void) (Msgid2), (const char *) (Msgid1)) \
-     : ((void) (Msgid1), (const char *) (Msgid2)))
-# define dngettext(Domainname, Msgid1, Msgid2, N) \
-    ((void) (Domainname), ngettext (Msgid1, Msgid2, N))
-# define dcngettext(Domainname, Msgid1, Msgid2, N, Category) \
-    ((void) (Category), dngettext(Domainname, Msgid1, Msgid2, N))
-# define textdomain(Domainname) ((const char *) (Domainname))
-# define bindtextdomain(Domainname, Dirname) \
-    ((void) (Domainname), (const char *) (Dirname))
-# define bind_textdomain_codeset(Domainname, Codeset) \
-    ((void) (Domainname), (const char *) (Codeset))
-
-#endif
-
-/* A pseudo function call that serves as a marker for the automated
-   extraction of messages, but does not call gettext().  The run-time
-   translation is done at a different place in the code.
-   The argument, String, should be a literal string.  Concatenated strings
-   and other string expressions won't work.
-   The macro's expansion is not parenthesized, so that it is suitable as
-   initializer for static 'char[]' or 'const char[]' variables.  */
-#define gettext_noop(String) String
-
-/* The separator between msgctxt and msgid in a .mo file.  */
-#define GETTEXT_CONTEXT_GLUE "\004"
-
-/* Pseudo function calls, taking a MSGCTXT and a MSGID instead of just a
-   MSGID.  MSGCTXT and MSGID must be string literals.  MSGCTXT should be
-   short and rarely need to change.
-   The letter 'p' stands for 'particular' or 'special'.  */
-#ifdef DEFAULT_TEXT_DOMAIN
-# define pgettext(Msgctxt, Msgid) \
-   pgettext_aux (DEFAULT_TEXT_DOMAIN, Msgctxt GETTEXT_CONTEXT_GLUE Msgid, Msgid, LC_MESSAGES)
-#else
-# define pgettext(Msgctxt, Msgid) \
-   pgettext_aux (NULL, Msgctxt GETTEXT_CONTEXT_GLUE Msgid, Msgid, LC_MESSAGES)
-#endif
-#define dpgettext(Domainname, Msgctxt, Msgid) \
-  pgettext_aux (Domainname, Msgctxt GETTEXT_CONTEXT_GLUE Msgid, Msgid, LC_MESSAGES)
-#define dcpgettext(Domainname, Msgctxt, Msgid, Category) \
-  pgettext_aux (Domainname, Msgctxt GETTEXT_CONTEXT_GLUE Msgid, Msgid, Category)
-#ifdef DEFAULT_TEXT_DOMAIN
-# define npgettext(Msgctxt, Msgid, MsgidPlural, N) \
-   npgettext_aux (DEFAULT_TEXT_DOMAIN, Msgctxt GETTEXT_CONTEXT_GLUE Msgid, Msgid, MsgidPlural, N, LC_MESSAGES)
-#else
-# define npgettext(Msgctxt, Msgid, MsgidPlural, N) \
-   npgettext_aux (NULL, Msgctxt GETTEXT_CONTEXT_GLUE Msgid, Msgid, MsgidPlural, N, LC_MESSAGES)
-#endif
-#define dnpgettext(Domainname, Msgctxt, Msgid, MsgidPlural, N) \
-  npgettext_aux (Domainname, Msgctxt GETTEXT_CONTEXT_GLUE Msgid, Msgid, MsgidPlural, N, LC_MESSAGES)
-#define dcnpgettext(Domainname, Msgctxt, Msgid, MsgidPlural, N, Category) \
-  npgettext_aux (Domainname, Msgctxt GETTEXT_CONTEXT_GLUE Msgid, Msgid, MsgidPlural, N, Category)
-
-#ifdef __GNUC__
-__inline
-#else
-#ifdef __cplusplus
-inline
-#endif
-#endif
-static const char *
-pgettext_aux (const char *domain,
-              const char *msg_ctxt_id, const char *msgid,
-              int category)
-{
-  const char *translation = dcgettext (domain, msg_ctxt_id, category);
-  if (translation == msg_ctxt_id)
-    return msgid;
-  else
-    return translation;
-}
-
-#ifdef __GNUC__
-__inline
-#else
-#ifdef __cplusplus
-inline
-#endif
-#endif
-static const char *
-npgettext_aux (const char *domain,
-               const char *msg_ctxt_id, const char *msgid,
-               const char *msgid_plural, unsigned long int n,
-               int category)
-{
-  const char *translation =
-    dcngettext (domain, msg_ctxt_id, msgid_plural, n, category);
-  if (translation == msg_ctxt_id || translation == msgid_plural)
-    return (n == 1 ? msgid : msgid_plural);
-  else
-    return translation;
-}
-
-/* The same thing extended for non-constant arguments.  Here MSGCTXT and MSGID
-   can be arbitrary expressions.  But for string literals these macros are
-   less efficient than those above.  */
-
-#include <string.h>
-
-#define _LIBGETTEXT_HAVE_VARIABLE_SIZE_ARRAYS \
-  (((__GNUC__ >= 3 || __GNUG__ >= 2) && !__STRICT_ANSI__) \
-   /* || __STDC_VERSION__ >= 199901L */ )
-
-#if !_LIBGETTEXT_HAVE_VARIABLE_SIZE_ARRAYS
-#include <stdlib.h>
-#endif
-
-#define pgettext_expr(Msgctxt, Msgid) \
-  dcpgettext_expr (NULL, Msgctxt, Msgid, LC_MESSAGES)
-#define dpgettext_expr(Domainname, Msgctxt, Msgid) \
-  dcpgettext_expr (Domainname, Msgctxt, Msgid, LC_MESSAGES)
-
-#ifdef __GNUC__
-__inline
-#else
-#ifdef __cplusplus
-inline
-#endif
-#endif
-static const char *
-dcpgettext_expr (const char *domain,
-                 const char *msgctxt, const char *msgid,
-                 int category)
-{
-  size_t msgctxt_len = strlen (msgctxt) + 1;
-  size_t msgid_len = strlen (msgid) + 1;
-  const char *translation;
-#if _LIBGETTEXT_HAVE_VARIABLE_SIZE_ARRAYS
-  char msg_ctxt_id[msgctxt_len + msgid_len];
-#else
-  char buf[1024];
-  char *msg_ctxt_id =
-    (msgctxt_len + msgid_len <= sizeof (buf)
-     ? buf
-     : (char *) malloc (msgctxt_len + msgid_len));
-  if (msg_ctxt_id != NULL)
-#endif
-    {
-      memcpy (msg_ctxt_id, msgctxt, msgctxt_len - 1);
-      msg_ctxt_id[msgctxt_len - 1] = '\004';
-      memcpy (msg_ctxt_id + msgctxt_len, msgid, msgid_len);
-      translation = dcgettext (domain, msg_ctxt_id, category);
-#if !_LIBGETTEXT_HAVE_VARIABLE_SIZE_ARRAYS
-      if (msg_ctxt_id != buf)
-        free (msg_ctxt_id);
-#endif
-      if (translation != msg_ctxt_id)
-        return translation;
-    }
-  return msgid;
-}
-
-#define npgettext_expr(Msgctxt, Msgid, MsgidPlural, N) \
-  dcnpgettext_expr (NULL, Msgctxt, Msgid, MsgidPlural, N, LC_MESSAGES)
-#define dnpgettext_expr(Domainname, Msgctxt, Msgid, MsgidPlural, N) \
-  dcnpgettext_expr (Domainname, Msgctxt, Msgid, MsgidPlural, N, LC_MESSAGES)
-
-#ifdef __GNUC__
-__inline
-#else
-#ifdef __cplusplus
-inline
-#endif
-#endif
-static const char *
-dcnpgettext_expr (const char *domain,
-                  const char *msgctxt, const char *msgid,
-                  const char *msgid_plural, unsigned long int n,
-                  int category)
-{
-  size_t msgctxt_len = strlen (msgctxt) + 1;
-  size_t msgid_len = strlen (msgid) + 1;
-  const char *translation;
-#if _LIBGETTEXT_HAVE_VARIABLE_SIZE_ARRAYS
-  char msg_ctxt_id[msgctxt_len + msgid_len];
-#else
-  char buf[1024];
-  char *msg_ctxt_id =
-    (msgctxt_len + msgid_len <= sizeof (buf)
-     ? buf
-     : (char *) malloc (msgctxt_len + msgid_len));
-  if (msg_ctxt_id != NULL)
-#endif
-    {
-      memcpy (msg_ctxt_id, msgctxt, msgctxt_len - 1);
-      msg_ctxt_id[msgctxt_len - 1] = '\004';
-      memcpy (msg_ctxt_id + msgctxt_len, msgid, msgid_len);
-      translation = dcngettext (domain, msg_ctxt_id, msgid_plural, n, category);
-#if !_LIBGETTEXT_HAVE_VARIABLE_SIZE_ARRAYS
-      if (msg_ctxt_id != buf)
-        free (msg_ctxt_id);
-#endif
-      if (!(translation == msg_ctxt_id || translation == msgid_plural))
-        return translation;
-    }
-  return (n == 1 ? msgid : msgid_plural);
-}
-
-#endif /* _LIBGETTEXT_H */
--- a/src/gnutls.c
+++ b/src/gnutls.c
@@ -1,6 +1,5 @@
 /* SSL support via GnuTLS library.
-   Copyright (C) 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012 Free Software
-   Foundation, Inc.
+   Copyright (C) 2005-2012, 2015, 2018 Free Software Foundation, Inc.

 This file is part of GNU Wget.

@@ -36,7 +35,9 @@ as that of the covered work.  */
 #include <stdio.h>
 #include <dirent.h>
 #include <stdlib.h>
+#include <xalloc.h>

+#include <gnutls/abstract.h>
 #include <gnutls/gnutls.h>
 #include <gnutls/x509.h>
 #include <sys/ioctl.h>
@@ -48,7 +49,7 @@ as that of the covered work.  */
 #include "hash.h"
 #include "ssl.h"

-#include <sys/fcntl.h>
+#include <fcntl.h>

 #ifdef WIN32
 # include "w32sock.h"
@@ -56,6 +57,9 @@ as that of the covered work.  */

 #include "host.h"

+static int
+_do_handshake (gnutls_session_t session, int fd, double timeout);
+
 static int
 key_type_to_gnutls_type (enum keyfile_type type)
 {
@@ -122,9 +126,10 @@ ssl_init (void)
          while ((dent = readdir (dir)) != NULL)
            {
              struct stat st;
-              char ca_file[dirlen + strlen(dent->d_name) + 2];
+              size_t ca_file_length = dirlen + strlen(dent->d_name) + 2;
+              char *ca_file = alloca(ca_file_length);

-              snprintf (ca_file, sizeof(ca_file), "%s/%s", ca_directory, dent->d_name);
+              snprintf (ca_file, ca_file_length, "%s/%s", ca_directory, dent->d_name);
              if (stat (ca_file, &st) != 0)
                continue;

@@ -138,8 +143,7 @@ ssl_init (void)
              hash_table_put (inode_map, (void *)(intptr_t) st.st_ino, NULL);
              if ((rc = gnutls_certificate_set_x509_trust_file (credentials, ca_file,
                                                                GNUTLS_X509_FMT_PEM)) <= 0)
-                logprintf (LOG_NOTQUIET, _("ERROR: Failed to open cert %s: (%d).\n"),
-                           ca_file, rc);
+                DEBUGP (("WARNING: Failed to open cert %s: (%d).\n", ca_file, rc));
              else
                ncerts += rc;
            }
@@ -149,18 +153,48 @@ ssl_init (void)
        }
    }

+  if (opt.ca_cert)
+    {
+      int rc;
+
+      ncerts = 0;
+
+      if ((rc = gnutls_certificate_set_x509_trust_file (credentials, opt.ca_cert,
+                                                        GNUTLS_X509_FMT_PEM)) <= 0)
+        logprintf (LOG_NOTQUIET, _ ("ERROR: Failed to open cert %s: (%d).\n"),
+                   opt.ca_cert, rc);
+      else
+        {
+          ncerts += rc;
+          logprintf (LOG_NOTQUIET, _ ("Loaded CA certificate '%s'\n"), opt.ca_cert);
+        }
+    }
+
+  if (opt.crl_file)
+    {
+      int rc;
+
+      if ((rc = gnutls_certificate_set_x509_crl_file (credentials, opt.crl_file, GNUTLS_X509_FMT_PEM)) <= 0)
+        {
+          logprintf (LOG_NOTQUIET, _("ERROR: Failed to load CRL file '%s': (%d)\n"), opt.crl_file, rc);
+          return false;
+        }
+
+      logprintf (LOG_NOTQUIET, _ ("Loaded CRL file '%s'\n"), opt.crl_file);
+    }
+
  DEBUGP (("Certificates loaded: %d\n", ncerts));

  /* Use the private key from the cert file unless otherwise specified. */
  if (opt.cert_file && !opt.private_key)
    {
-      opt.private_key = opt.cert_file;
+      opt.private_key = xstrdup (opt.cert_file);
      opt.private_key_type = opt.cert_type;
    }
  /* Use the cert from the private key file unless otherwise specified. */
  if (!opt.cert_file && opt.private_key)
    {
-      opt.cert_file = opt.private_key;
+      opt.cert_file = xstrdup (opt.private_key);
      opt.cert_type = opt.private_key_type;
    }

@@ -181,10 +215,6 @@ cert to be of the same type.\n"));
                                            type);
    }

-  if (opt.ca_cert)
-    gnutls_certificate_set_x509_trust_file (credentials, opt.ca_cert,
-                                            GNUTLS_X509_FMT_PEM);
-
  ssl_initialized = true;

  return true;
@@ -193,6 +223,7 @@ cert to be of the same type.\n"));
 struct wgnutls_transport_context
 {
  gnutls_session_t session;       /* GnuTLS session handle */
+  gnutls_datum_t *session_data;
  int last_error;               /* last error returned by read/write/... */

  /* Since GnuTLS doesn't support the equivalent to recv(...,
@@ -203,11 +234,6 @@ struct wgnutls_transport_context
  int peeklen;
 };

-#ifndef MIN
-# define MIN(i, j) ((i) <= (j) ? (i) : (j))
-#endif
-
-
 static int
 wgnutls_read_timeout (int fd, char *buf, int bufsize, void *arg, double timeout)
 {
@@ -255,6 +281,12 @@ wgnutls_read_timeout (int fd, char *buf, int bufsize, void *arg, double timeout)
        {
          ret = gnutls_record_recv (ctx->session, buf, bufsize);
          timed_out = timeout && ptimer_measure (timer) >= timeout;
+          if (!timed_out && ret == GNUTLS_E_REHANDSHAKE)
+            {
+              DEBUGP (("GnuTLS: *** REHANDSHAKE while reading\n"));
+              if ((ret = _do_handshake (ctx->session, fd, timeout)) == 0)
+                ret = GNUTLS_E_AGAIN; /* restart reading */
+            }
        }
    }
  while (ret == GNUTLS_E_INTERRUPTED || (ret == GNUTLS_E_AGAIN && !timed_out));
@@ -384,6 +416,11 @@ wgnutls_close (int fd, void *arg)
 {
  struct wgnutls_transport_context *ctx = arg;
  /*gnutls_bye (ctx->session, GNUTLS_SHUT_RDWR);*/
+  if (ctx->session_data)
+    {
+      gnutls_free (ctx->session_data->data);
+      gnutls_free (ctx->session_data);
+    }
  gnutls_deinit (ctx->session);
  xfree (ctx);
  close (fd);
@@ -398,90 +435,15 @@ static struct transport_implementation wgnutls_transport =
  wgnutls_peek, wgnutls_errstr, wgnutls_close
 };

-bool
-ssl_connect_wget (int fd, const char *hostname)
+static int
+_do_handshake (gnutls_session_t session, int fd, double timeout)
 {
 #ifdef F_GETFL
  int flags = 0;
 #endif
-  struct wgnutls_transport_context *ctx;
-  gnutls_session_t session;
-  int err,alert;
-  gnutls_init (&session, GNUTLS_CLIENT);
-  const char *str;
+  int err;

-  /* We set the server name but only if it's not an IP address. */
-  if (! is_valid_ip_address (hostname))
-    {
-      gnutls_server_name_set (session, GNUTLS_NAME_DNS, hostname,
-                              strlen (hostname));
-    }
-
-  gnutls_set_default_priority (session);
-  gnutls_credentials_set (session, GNUTLS_CRD_CERTIFICATE, credentials);
-#ifndef FD_TO_SOCKET
-# define FD_TO_SOCKET(X) (X)
-#endif
-#ifdef HAVE_INTPTR_T
-  gnutls_transport_set_ptr (session, (gnutls_transport_ptr_t) (intptr_t) FD_TO_SOCKET (fd));
-#else
-  gnutls_transport_set_ptr (session, (gnutls_transport_ptr_t) FD_TO_SOCKET (fd));
-#endif
-
-  err = 0;
-#if HAVE_GNUTLS_PRIORITY_SET_DIRECT
-  switch (opt.secure_protocol)
-    {
-    case secure_protocol_auto:
-      break;
-    case secure_protocol_sslv2:
-    case secure_protocol_sslv3:
-      err = gnutls_priority_set_direct (session, "NORMAL:-VERS-TLS-ALL:+VERS-SSL3.0", NULL);
-      break;
-    case secure_protocol_tlsv1:
-      err = gnutls_priority_set_direct (session, "NORMAL:-VERS-SSL3.0", NULL);
-      break;
-    case secure_protocol_pfs:
-      err = gnutls_priority_set_direct (session, "PFS", NULL);
-      if (err != GNUTLS_E_SUCCESS)
-        /* fallback if PFS is not available */
-        err = gnutls_priority_set_direct (session, "NORMAL:-RSA", NULL);
-      break;
-    default:
-      abort ();
-    }
-#else
-  int allowed_protocols[4] = {0, 0, 0, 0};
-  switch (opt.secure_protocol)
-    {
-    case secure_protocol_auto:
-      break;
-    case secure_protocol_sslv2:
-    case secure_protocol_sslv3:
-      allowed_protocols[0] = GNUTLS_SSL3;
-      err = gnutls_protocol_set_priority (session, allowed_protocols);
-      break;
-
-    case secure_protocol_tlsv1:
-      allowed_protocols[0] = GNUTLS_TLS1_0;
-      allowed_protocols[1] = GNUTLS_TLS1_1;
-      allowed_protocols[2] = GNUTLS_TLS1_2;
-      err = gnutls_protocol_set_priority (session, allowed_protocols);
-      break;
-
-    default:
-      abort ();
-    }
-#endif
-
-  if (err < 0)
-    {
-      logprintf (LOG_NOTQUIET, "GnuTLS: %s\n", gnutls_strerror (err));
-      gnutls_deinit (session);
-      return false;
-    }
-
-  if (opt.connect_timeout)
+  if (timeout)
    {
 #ifdef F_GETFL
      flags = fcntl (fd, F_GETFL, 0);
@@ -502,17 +464,17 @@ ssl_connect_wget (int fd, const char *hostname)
    {
      err = gnutls_handshake (session);

-      if (opt.connect_timeout && err == GNUTLS_E_AGAIN)
+      if (timeout && err == GNUTLS_E_AGAIN)
        {
          if (gnutls_record_get_direction (session))
            {
              /* wait for writeability */
-              err = select_fd (fd, opt.connect_timeout, WAIT_FOR_WRITE);
+              err = select_fd (fd, timeout, WAIT_FOR_WRITE);
            }
          else
            {
              /* wait for readability */
-              err = select_fd (fd, opt.connect_timeout, WAIT_FOR_READ);
+              err = select_fd (fd, timeout, WAIT_FOR_READ);
            }

          if (err <= 0)
@@ -533,17 +495,16 @@ ssl_connect_wget (int fd, const char *hostname)
          if (err == GNUTLS_E_WARNING_ALERT_RECEIVED ||
              err == GNUTLS_E_FATAL_ALERT_RECEIVED)
            {
-              alert = gnutls_alert_get (session);
-              str = gnutls_alert_get_name (alert);
-              if (str == NULL)
-                str = "(unknown)";
-              logprintf (LOG_NOTQUIET, "GnuTLS: received alert [%d]: %s\n", alert, str);
+              gnutls_alert_description_t alert = gnutls_alert_get (session);
+              const char *str = gnutls_alert_get_name (alert);
+              logprintf (LOG_NOTQUIET, "GnuTLS: received alert [%u]: %s\n",
+                         alert, str ? str : "(unknown)");
            }
        }
    }
  while (err && gnutls_error_is_fatal (err) == 0);

-  if (opt.connect_timeout)
+  if (timeout)
    {
 #ifdef F_GETFL
      if (fcntl (fd, F_SETFL, flags) < 0)
@@ -555,6 +516,166 @@ ssl_connect_wget (int fd, const char *hostname)
 #endif
    }

+  return err;
+}
+
+static const char *
+_sni_hostname(const char *hostname)
+{
+  size_t len = strlen(hostname);
+
+  char *sni_hostname = xmemdup(hostname, len + 1);
+
+  /* Remove trailing dot(s) to fix #47408.
+   * Regarding RFC 6066 (SNI): The hostname is represented as a byte
+   * string using ASCII encoding without a trailing dot. */
+  while (len && sni_hostname[--len] == '.')
+    sni_hostname[len] = 0;
+
+  return sni_hostname;
+}
+
+bool
+ssl_connect_wget (int fd, const char *hostname, int *continue_session)
+{
+  struct wgnutls_transport_context *ctx;
+  gnutls_session_t session;
+  int err;
+
+  gnutls_init (&session, GNUTLS_CLIENT);
+
+  /* We set the server name but only if it's not an IP address. */
+  if (! is_valid_ip_address (hostname))
+    {
+      /* GnuTLS 3.4.x (x<=10) disrespects the length parameter, we have to construct a new string */
+      /* see https://gitlab.com/gnutls/gnutls/issues/78 */
+      const char *sni_hostname = _sni_hostname(hostname);
+
+      gnutls_server_name_set (session, GNUTLS_NAME_DNS, sni_hostname, strlen(sni_hostname));
+      xfree(sni_hostname);
+    }
+
+  gnutls_credentials_set (session, GNUTLS_CRD_CERTIFICATE, credentials);
+#ifndef FD_TO_SOCKET
+# define FD_TO_SOCKET(X) (X)
+#endif
+#ifdef HAVE_INTPTR_T
+  gnutls_transport_set_ptr (session, (gnutls_transport_ptr_t) (intptr_t) FD_TO_SOCKET (fd));
+#else
+  gnutls_transport_set_ptr (session, (gnutls_transport_ptr_t) FD_TO_SOCKET (fd));
+#endif
+
+#if HAVE_GNUTLS_PRIORITY_SET_DIRECT
+  switch (opt.secure_protocol)
+    {
+    case secure_protocol_auto:
+      err = gnutls_set_default_priority (session);
+      gnutls_session_enable_compatibility_mode(session);
+      break;
+
+    case secure_protocol_sslv2:
+    case secure_protocol_sslv3:
+      err = gnutls_priority_set_direct (session, "NORMAL:-VERS-TLS-ALL:+VERS-SSL3.0", NULL);
+      break;
+
+    case secure_protocol_tlsv1:
+      err = gnutls_priority_set_direct (session, "NORMAL:-VERS-SSL3.0", NULL);
+      break;
+
+    case secure_protocol_tlsv1_1:
+      err = gnutls_priority_set_direct (session, "NORMAL:-VERS-SSL3.0:-VERS-TLS1.0", NULL);
+      break;
+
+    case secure_protocol_tlsv1_2:
+      err = gnutls_priority_set_direct (session, "NORMAL:-VERS-SSL3.0:-VERS-TLS1.0:-VERS-TLS1.1", NULL);
+      break;
+
+    case secure_protocol_pfs:
+      err = gnutls_priority_set_direct (session, "PFS:-VERS-SSL3.0", NULL);
+      if (err != GNUTLS_E_SUCCESS)
+        /* fallback if PFS is not available */
+        err = gnutls_priority_set_direct (session, "NORMAL:-RSA:-VERS-SSL3.0", NULL);
+      break;
+
+    default:
+      logprintf (LOG_NOTQUIET, _("GnuTLS: unimplemented 'secure-protocol' option value %u\n"),
+                 (unsigned) opt.secure_protocol);
+      logprintf (LOG_NOTQUIET, _("Please report this issue to bug-wget@gnu.org\n"));
+      abort ();
+    }
+#else
+  int allowed_protocols[4] = {0, 0, 0, 0};
+  switch (opt.secure_protocol)
+    {
+    case secure_protocol_auto:
+      err = gnutls_set_default_priority (session);
+      break;
+
+    case secure_protocol_sslv2:
+    case secure_protocol_sslv3:
+      allowed_protocols[0] = GNUTLS_SSL3;
+      err = gnutls_protocol_set_priority (session, allowed_protocols);
+      break;
+
+    case secure_protocol_tlsv1:
+      allowed_protocols[0] = GNUTLS_TLS1_0;
+      allowed_protocols[1] = GNUTLS_TLS1_1;
+      allowed_protocols[2] = GNUTLS_TLS1_2;
+      err = gnutls_protocol_set_priority (session, allowed_protocols);
+      break;
+
+    case secure_protocol_tlsv1_1:
+      allowed_protocols[0] = GNUTLS_TLS1_1;
+      allowed_protocols[1] = GNUTLS_TLS1_2;
+      err = gnutls_protocol_set_priority (session, allowed_protocols);
+      break;
+
+    case secure_protocol_tlsv1_2:
+      allowed_protocols[0] = GNUTLS_TLS1_2;
+      err = gnutls_protocol_set_priority (session, allowed_protocols);
+      break;
+
+    default:
+      logprintf (LOG_NOTQUIET, _("GnuTLS: unimplemented 'secure-protocol' option value %d\n"), opt.secure_protocol);
+      logprintf (LOG_NOTQUIET, _("Please report this issue to bug-wget@gnu.org\n"));
+      abort ();
+    }
+#endif
+
+  if (err < 0)
+    {
+      logprintf (LOG_NOTQUIET, "GnuTLS: %s\n", gnutls_strerror (err));
+      gnutls_deinit (session);
+      return false;
+    }
+
+  if (continue_session)
+    {
+      ctx = (struct wgnutls_transport_context *) fd_transport_context (*continue_session);
+      if (!gnutls_session_is_resumed (session))
+        {
+          if (!ctx || !ctx->session_data || gnutls_session_set_data (session, ctx->session_data->data, ctx->session_data->size))
+            {
+              if (ctx && ctx->session_data)
+                {
+                  /* server does not want to continue the session */
+                  if (ctx->session_data->data)
+                    gnutls_free (ctx->session_data->data);
+                  gnutls_free (ctx->session_data);
+                }
+              gnutls_deinit (session);
+              return false;
+            }
+        }
+      else
+        {
+          logputs (LOG_ALWAYS, "SSL session has already been resumed. Continuing.\n");
+          continue_session = NULL;
+        }
+    }
+
+  err = _do_handshake (session, fd, opt.connect_timeout);
+
  if (err < 0)
    {
      gnutls_deinit (session);
@@ -562,11 +683,70 @@ ssl_connect_wget (int fd, const char *hostname)
    }

  ctx = xnew0 (struct wgnutls_transport_context);
+  ctx->session_data = xnew0 (gnutls_datum_t);
  ctx->session = session;
+  if (gnutls_session_get_data2 (session, ctx->session_data))
+    {
+      xfree (ctx->session_data);
+      logprintf (LOG_NOTQUIET, "WARNING: Could not save SSL session data for socket %d\n", fd);
+    }
  fd_register_transport (fd, &wgnutls_transport, ctx);
  return true;
 }

+static bool
+pkp_pin_peer_pubkey (gnutls_x509_crt_t cert, const char *pinnedpubkey)
+{
+  /* Scratch */
+  size_t len1 = 0, len2 = 0;
+  char *buff1 = NULL;
+
+  gnutls_pubkey_t key = NULL;
+
+  /* Result is returned to caller */
+  int ret = 0;
+  bool result = false;
+
+  /* if a path wasn't specified, don't pin */
+  if (NULL == pinnedpubkey)
+    return true;
+
+  if (NULL == cert)
+    return result;
+
+  /* Begin Gyrations to get the public key     */
+  gnutls_pubkey_init (&key);
+
+  ret = gnutls_pubkey_import_x509 (key, cert, 0);
+  if (ret < 0)
+    goto cleanup; /* failed */
+
+  ret = gnutls_pubkey_export (key, GNUTLS_X509_FMT_DER, NULL, &len1);
+  if (ret != GNUTLS_E_SHORT_MEMORY_BUFFER || len1 == 0)
+    goto cleanup; /* failed */
+
+  buff1 = xmalloc (len1);
+
+  len2 = len1;
+
+  ret = gnutls_pubkey_export (key, GNUTLS_X509_FMT_DER, buff1, &len2);
+  if (ret < 0 || len1 != len2)
+    goto cleanup; /* failed */
+
+  /* End Gyrations */
+
+  /* The one good exit point */
+  result = wg_pin_peer_pubkey (pinnedpubkey, buff1, len1);
+
+ cleanup:
+  if (NULL != key)
+    gnutls_pubkey_deinit (key);
+
+  xfree (buff1);
+
+  return result;
+}
+
 #define _CHECK_CERT(flag,msg) \
  if (status & (flag))\
    {\
@@ -587,6 +767,11 @@ ssl_check_certificate (int fd, const char *host)
     him about problems with the server's certificate.  */
  const char *severity = opt.check_cert ? _("ERROR") : _("WARNING");
  bool success = true;
+  bool pinsuccess = opt.pinnedpubkey == NULL;
+
+  /* The user explicitly said to not check for the certificate.  */
+  if (opt.check_cert == CHECK_CERT_QUIET && pinsuccess)
+    return success;

  err = gnutls_certificate_verify_peers2 (ctx->session, &status);
  if (err < 0)
@@ -611,6 +796,7 @@ ssl_check_certificate (int fd, const char *host)
      gnutls_x509_crt_t cert;
      const gnutls_datum_t *cert_list;
      unsigned int cert_list_size;
+      const char *sni_hostname;

      if ((err = gnutls_x509_crt_init (&cert)) < 0)
        {
@@ -645,13 +831,23 @@ ssl_check_certificate (int fd, const char *host)
          logprintf (LOG_NOTQUIET, _("The certificate has expired\n"));
          success = false;
        }
-      if (!gnutls_x509_crt_check_hostname (cert, host))
+      sni_hostname = _sni_hostname(host);
+      if (!gnutls_x509_crt_check_hostname (cert, sni_hostname))
        {
          logprintf (LOG_NOTQUIET,
                     _("The certificate's owner does not match hostname %s\n"),
-                     quote (host));
+                     quote (sni_hostname));
          success = false;
        }
+      xfree(sni_hostname);
+
+      pinsuccess = pkp_pin_peer_pubkey (cert, opt.pinnedpubkey);
+      if (!pinsuccess)
+        {
+          logprintf (LOG_ALWAYS, _("The public key does not match pinned public key!\n"));
+          success = false;
+        }
+
 crt_deinit:
      gnutls_x509_crt_deinit (cert);
    }
@@ -662,5 +858,6 @@ ssl_check_certificate (int fd, const char *host)
    }

 out:
-  return opt.check_cert ? success : true;
+  /* never return true if pinsuccess fails */
+  return !pinsuccess ? false : (opt.check_cert == CHECK_CERT_ON ? success : true);
 }
--- a/src/hash.c
+++ b/src/hash.c
@@ -1,6 +1,5 @@
 /* Hash tables.
-   Copyright (C) 2000, 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008,
-   2009, 2010, 2011 Free Software Foundation, Inc.
+   Copyright (C) 2000-2011, 2015, 2018 Free Software Foundation, Inc.

 This file is part of GNU Wget.

@@ -46,10 +45,11 @@ as that of the covered work.  */
 # include "utils.h"
 #else
 /* Make do without them. */
-# define xnew(x) xmalloc (sizeof (x))
-# define xnew_array(type, x) xmalloc (sizeof (type) * (x))
-# define xmalloc malloc
-# define xfree free
+# define xnew(type) (xmalloc (sizeof (type)))
+# define xnew0(type) (xcalloc (1, sizeof (type)))
+# define xnew_array(type, len) (xmalloc ((len) * sizeof (type)))
+# define xfree(p) do { free ((void *) (p)); p = NULL; } while (0)
+
 # ifndef countof
 #  define countof(x) (sizeof (x) / sizeof ((x)[0]))
 # endif
@@ -282,10 +282,10 @@ hash_table_new (int items,

  /* Calculate the size that ensures that the table will store at
     least ITEMS keys without the need to resize.  */
-  size = 1 + items / HASH_MAX_FULLNESS;
+  size = (int) (1 + items / HASH_MAX_FULLNESS);
  size = prime_size (size, &ht->prime_offset);
  ht->size = size;
-  ht->resize_threshold = size * HASH_MAX_FULLNESS;
+  ht->resize_threshold = (int) (size * HASH_MAX_FULLNESS);
  /*assert (ht->resize_threshold >= items);*/

  ht->cells = xnew_array (struct cell, ht->size);
@@ -393,7 +393,7 @@ grow_hash_table (struct hash_table *ht)
 #endif

  ht->size = newsize;
-  ht->resize_threshold = newsize * HASH_MAX_FULLNESS;
+  ht->resize_threshold = (int) (newsize * HASH_MAX_FULLNESS);

  cells = xnew_array (struct cell, newsize);
  memset (cells, INVALID_PTR_CHAR, newsize * sizeof (struct cell));
@@ -585,7 +585,7 @@ hash_table_count (const struct hash_table *ht)
 {
  return ht->count;
 }
-
+
 /* Functions from this point onward are meant for convenience and
   don't strictly belong to this file.  However, this is as good a
   place for them as any.  */
@@ -740,7 +740,7 @@ cmp_pointer (const void *ptr1, const void *ptr2)
 {
  return ptr1 == ptr2;
 }
-
+
 #ifdef TEST

 #include <stdio.h>
--- a/src/hash.h
+++ b/src/hash.h
@@ -1,6 +1,6 @@
 /* Hash table declarations.
-   Copyright (C) 2000, 2007, 2008, 2009, 2010, 2011 Free Software
-   Foundation, Inc.
+   Copyright (C) 2000, 2007-2011, 2015, 2018 Free Software Foundation,
+   Inc.

 This file is part of GNU Wget.

--- a/src/host.c
+++ b/src/host.c
@@ -1,7 +1,5 @@
 /* Host name resolution and matching.
-   Copyright (C) 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004,
-   2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012 Free Software Foundation,
-   Inc.
+   Copyright (C) 1996-2012, 2015, 2018 Free Software Foundation, Inc.

 This file is part of GNU Wget.

@@ -61,6 +59,7 @@ as that of the covered work.  */
 #include "host.h"
 #include "url.h"
 #include "hash.h"
+#include "ptimer.h"

 #ifndef NO_ADDRESS
 # define NO_ADDRESS NO_DATA
@@ -151,6 +150,13 @@ address_list_set_faulty (struct address_list *al, int index)
     "faulty" attempt is always preceded with all-faulty addresses,
     and this is how Wget uses it.  */
  assert (index == al->faulty);
+  if (index != al->faulty)
+    {
+      logprintf (LOG_ALWAYS, "index: %d\nal->faulty: %d\n", index, al->faulty);
+      logprintf (LOG_ALWAYS, _("Error in handling the address list.\n"));
+      logprintf (LOG_ALWAYS, _("Please report this issue to bug-wget@gnu.org\n"));
+      abort();
+    }

  ++al->faulty;
  if (al->faulty >= al->count)
@@ -309,7 +315,7 @@ address_list_release (struct address_list *al)
      address_list_delete (al);
    }
 }
-
+
 /* Versions of gethostbyname and getaddrinfo that support timeout. */

 #ifndef ENABLE_IPV6
@@ -408,7 +414,7 @@ getaddrinfo_with_timeout (const char *node, const char *service,
 }

 #endif /* ENABLE_IPV6 */
-
+
 /* Return a textual representation of ADDR, i.e. the dotted quad for
   IPv4 addresses, and the colon-separated list of hex words (with all
   zeros omitted, etc.) for IPv6 addresses.  */
@@ -416,14 +422,12 @@ getaddrinfo_with_timeout (const char *node, const char *service,
 const char *
 print_address (const ip_address *addr)
 {
-#ifdef ENABLE_IPV6
  static char buf[64];
+
  if (!inet_ntop (addr->family, IP_INADDR_DATA (addr), buf, sizeof buf))
    snprintf (buf, sizeof buf, "<error: %s>", strerror (errno));
+
  return buf;
-#else
-  return inet_ntoa (addr->data.d4);
-#endif
 }

 /* The following two functions were adapted from glibc's
@@ -570,7 +574,7 @@ is_valid_ipv6_address (const char *str, const char *end)

  return true;
 }
-
+
 /* Simple host cache, used by lookup_host to speed up resolving.  The
   cache doesn't handle TTL because Wget is a fairly short-lived
   application.  Refreshing is attempted when connect fails, though --
@@ -592,7 +596,7 @@ cache_query (const char *host)
  al = hash_table_get (host_name_addresses_map, host);
  if (al)
    {
-      DEBUGP (("Found %s in host_name_addresses_map (%p)\n", host, al));
+      DEBUGP (("Found %s in host_name_addresses_map (%p)\n", host, (void *) al));
      ++al->refcount;
      return al;
    }
@@ -637,7 +641,113 @@ cache_remove (const char *host)
      hash_table_remove (host_name_addresses_map, host);
    }
 }
-
+
+#ifdef HAVE_LIBCARES
+#include <sys/select.h>
+#include <ares.h>
+extern ares_channel ares;
+
+static struct address_list *
+merge_address_lists (struct address_list *al1, struct address_list *al2)
+{
+  int count = al1->count + al2->count;
+
+  /* merge al2 into al1 */
+  al1->addresses = xrealloc (al1->addresses, sizeof (ip_address) * count);
+  memcpy (al1->addresses + al1->count, al2->addresses, sizeof (ip_address) * al2->count);
+  al1->count = count;
+
+  address_list_delete (al2);
+
+  return al1;
+}
+
+static struct address_list *
+address_list_from_hostent (struct hostent *host)
+{
+  int count, i;
+  struct address_list *al = xnew0 (struct address_list);
+
+  for (count = 0; host->h_addr_list[count]; count++)
+    ;
+
+  assert (count > 0);
+
+  al->addresses = xnew_array (ip_address, count);
+  al->count     = count;
+  al->refcount  = 1;
+
+  for (i = 0; i < count; i++)
+    {
+      ip_address *ip = &al->addresses[i];
+      ip->family = host->h_addrtype;
+      memcpy (IP_INADDR_DATA (ip), host->h_addr_list[i], ip->family == AF_INET ? 4 : 16);
+    }
+
+  return al;
+}
+
+/* Since GnuLib's select() (i.e. rpl_select()) cannot handle socket-numbers
+ * returned from C-ares, we must use the original select() from Winsock.
+ */
+#ifdef WINDOWS
+#undef select
+#endif
+
+static void
+wait_ares (ares_channel channel)
+{
+  struct ptimer *timer = NULL;
+
+  if (opt.dns_timeout)
+    timer = ptimer_new ();
+
+  for (;;)
+    {
+      struct timeval *tvp, tv;
+      fd_set read_fds, write_fds;
+      int nfds, rc;
+
+      FD_ZERO (&read_fds);
+      FD_ZERO (&write_fds);
+      nfds = ares_fds (channel, &read_fds, &write_fds);
+      if (nfds == 0)
+        break;
+
+      if (timer)
+        {
+          double max = opt.dns_timeout - ptimer_measure (timer);
+
+          tv.tv_sec = (long) max;
+          tv.tv_usec = 1000000 * (max - (long) max);
+          tvp = ares_timeout (channel, &tv, &tv);
+        }
+      else
+        tvp = ares_timeout (channel, NULL, &tv);
+
+      rc = select (nfds, &read_fds, &write_fds, NULL, tvp);
+      if (rc == 0 && timer && ptimer_measure (timer) >= opt.dns_timeout)
+        ares_cancel (channel);
+      else
+        ares_process (channel, &read_fds, &write_fds);
+    }
+}
+
+static void
+callback (void *arg, int status, int timeouts _GL_UNUSED, struct hostent *host)
+{
+  struct address_list **al = (struct address_list **) arg;
+
+  if (!host || status != ARES_SUCCESS)
+    {
+      *al = NULL;
+      return;
+    }
+
+  *al = address_list_from_hostent (host);
+}
+#endif
+
 /* Look up HOST in DNS and return a list of IP addresses.

   This function caches its result so that, if the same host is passed
@@ -730,95 +840,123 @@ lookup_host (const char *host, int flags)

      if (opt.enable_iri && (name = idn_decode ((char *) host)) != NULL)
        {
-          int len = strlen (host) + strlen (name) + 4;
-          str = xmalloc (len);
-          snprintf (str, len, "%s (%s)", name, host);
-          str[len-1] = '\0';
+          str = aprintf ("%s (%s)", name, host);
          xfree (name);
        }

      logprintf (LOG_VERBOSE, _("Resolving %s... "),
                 quotearg_style (escape_quoting_style, str ? str : host));

-      if (str)
-        xfree (str);
+      xfree (str);
    }

 #ifdef ENABLE_IPV6
-  {
-    int err;
-    struct addrinfo hints, *res;
+#ifdef HAVE_LIBCARES
+  if (ares)
+    {
+      struct address_list *al4;
+      struct address_list *al6;

-    xzero (hints);
-    hints.ai_socktype = SOCK_STREAM;
-    if (opt.ipv4_only)
-      hints.ai_family = AF_INET;
-    else if (opt.ipv6_only)
-      hints.ai_family = AF_INET6;
-    else
-      /* We tried using AI_ADDRCONFIG, but removed it because: it
-         misinterprets IPv6 loopbacks, it is broken on AIX 5.1, and
-         it's unneeded since we sort the addresses anyway.  */
+      if (opt.ipv4_only || !opt.ipv6_only)
+        ares_gethostbyname (ares, host, AF_INET, callback, &al4);
+      if (opt.ipv6_only || !opt.ipv4_only)
+        ares_gethostbyname (ares, host, AF_INET6, callback, &al6);
+
+      wait_ares (ares);
+
+      if (al4 && al6)
+        al = merge_address_lists (al4, al6);
+      else if (al4)
+        al = al4;
+      else
+        al = al6;
+    }
+  else
+#endif
+    {
+      int err;
+      struct addrinfo hints, *res;
+
+      xzero (hints);
+      hints.ai_socktype = SOCK_STREAM;
+      if (opt.ipv4_only)
+        hints.ai_family = AF_INET;
+      else if (opt.ipv6_only)
+        hints.ai_family = AF_INET6;
+      else
+        /* We tried using AI_ADDRCONFIG, but removed it because: it
+           misinterprets IPv6 loopbacks, it is broken on AIX 5.1, and
+           it's unneeded since we sort the addresses anyway.  */
        hints.ai_family = AF_UNSPEC;

-    if (flags & LH_BIND)
-      hints.ai_flags |= AI_PASSIVE;
+      if (flags & LH_BIND)
+        hints.ai_flags |= AI_PASSIVE;

 #ifdef AI_NUMERICHOST
-    if (numeric_address)
-      {
-        /* Where available, the AI_NUMERICHOST hint can prevent costly
-           access to DNS servers.  */
-        hints.ai_flags |= AI_NUMERICHOST;
-        timeout = 0;            /* no timeout needed when "resolving"
+      if (numeric_address)
+        {
+          /* Where available, the AI_NUMERICHOST hint can prevent costly
+             access to DNS servers.  */
+          hints.ai_flags |= AI_NUMERICHOST;
+          timeout = 0; /* no timeout needed when "resolving"
                                   numeric hosts -- avoid setting up
                                   signal handlers and such. */
-      }
+        }
 #endif

-    err = getaddrinfo_with_timeout (host, NULL, &hints, &res, timeout);
-    if (err != 0 || res == NULL)
-      {
-        if (!silent)
-          logprintf (LOG_VERBOSE, _("failed: %s.\n"),
-                     err != EAI_SYSTEM ? gai_strerror (err) : strerror (errno));
-        return NULL;
-      }
-    al = address_list_from_addrinfo (res);
-    freeaddrinfo (res);
-    if (!al)
-      {
-        logprintf (LOG_VERBOSE,
-                   _("failed: No IPv4/IPv6 addresses for host.\n"));
-        return NULL;
-      }
+      err = getaddrinfo_with_timeout (host, NULL, &hints, &res, timeout);

-    /* Reorder addresses so that IPv4 ones (or IPv6 ones, as per
-       --prefer-family) come first.  Sorting is stable so the order of
-       the addresses with the same family is undisturbed.  */
-    if (al->count > 1 && opt.prefer_family != prefer_none)
-      stable_sort (al->addresses, al->count, sizeof (ip_address),
-                   opt.prefer_family == prefer_ipv4
-                   ? cmp_prefer_ipv4 : cmp_prefer_ipv6);
-  }
+      if (err != 0 || res == NULL)
+        {
+          if (!silent)
+            logprintf (LOG_VERBOSE, _ ("failed: %s.\n"),
+                       err != EAI_SYSTEM ? gai_strerror (err) : strerror (errno));
+          return NULL;
+        }
+      al = address_list_from_addrinfo (res);
+      freeaddrinfo (res);
+    }
+
+  if (!al)
+    {
+      logprintf (LOG_VERBOSE,
+                 _ ("failed: No IPv4/IPv6 addresses for host.\n"));
+      return NULL;
+    }
+
+  /* Reorder addresses so that IPv4 ones (or IPv6 ones, as per
+     --prefer-family) come first.  Sorting is stable so the order of
+     the addresses with the same family is undisturbed.  */
+  if (al->count > 1 && opt.prefer_family != prefer_none)
+    stable_sort (al->addresses, al->count, sizeof (ip_address),
+                 opt.prefer_family == prefer_ipv4
+                 ? cmp_prefer_ipv4 : cmp_prefer_ipv6);
 #else  /* not ENABLE_IPV6 */
-  {
-    struct hostent *hptr = gethostbyname_with_timeout (host, timeout);
-    if (!hptr)
-      {
-        if (!silent)
-          {
-            if (errno != ETIMEDOUT)
-              logprintf (LOG_VERBOSE, _("failed: %s.\n"),
-                         host_errstr (h_errno));
-            else
-              logputs (LOG_VERBOSE, _("failed: timed out.\n"));
-          }
-        return NULL;
-      }
-    /* Do older systems have h_addr_list?  */
-    al = address_list_from_ipv4_addresses (hptr->h_addr_list);
-  }
+#ifdef HAVE_LIBCARES
+  if (ares)
+    {
+      ares_gethostbyname (ares, host, AF_INET, callback, &al);
+      wait_ares (ares);
+    }
+  else
+#endif
+    {
+      struct hostent *hptr = gethostbyname_with_timeout (host, timeout);
+      if (!hptr)
+        {
+          if (!silent)
+            {
+              if (errno != ETIMEDOUT)
+                logprintf (LOG_VERBOSE, _ ("failed: %s.\n"),
+                           host_errstr (h_errno));
+              else
+                logputs (LOG_VERBOSE, _ ("failed: timed out.\n"));
+            }
+          return NULL;
+        }
+      /* Do older systems have h_addr_list?  */
+      al = address_list_from_ipv4_addresses (hptr->h_addr_list);
+    }
 #endif /* not ENABLE_IPV6 */

  /* Print the addresses determined by DNS lookup, but no more than
@@ -848,7 +986,7 @@ lookup_host (const char *host, int flags)

  return al;
 }
-
+
 /* Determine whether a URL is acceptable to be followed, according to
   a list of domains to accept.  */
 bool
@@ -879,18 +1017,25 @@ sufmatch (const char **list, const char *what)
  int i, j, k, lw;

  lw = strlen (what);
+
  for (i = 0; list[i]; i++)
    {
-      if (list[i][0] == '\0')
-        continue;
+      j = strlen (list[i]);
+      if (lw < j)
+        continue; /* what is no (sub)domain of list[i] */

-      for (j = strlen (list[i]), k = lw; j >= 0 && k >= 0; j--, k--)
+      for (k = lw; j >= 0 && k >= 0; j--, k--)
        if (c_tolower (list[i][j]) != c_tolower (what[k]))
          break;
-      /* The domain must be first to reach to beginning.  */
-      if (j == -1)
+
+      /* Domain or subdomain match
+       * k == -1: exact match
+       * k >= 0 && what[k] == '.': subdomain match
+       */
+      if (j == -1 && (k == -1 || what[k] == '.'))
        return true;
    }
+
  return false;
 }

--- a/src/host.h
+++ b/src/host.h
@@ -1,7 +1,5 @@
 /* Declarations for host.c
-   Copyright (C) 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004,
-   2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012 Free Software Foundation,
-   Inc.
+   Copyright (C) 1996-2012, 2015, 2018 Free Software Foundation, Inc.

 This file is part of GNU Wget.

--- a/src/hsts.c
+++ b/src/hsts.c
@@ -0,0 +1,827 @@
+/* HTTP Strict Transport Security (HSTS) support.
+   Copyright (C) 1996-2012, 2015, 2018 Free Software Foundation, Inc.
+
+This file is part of GNU Wget.
+
+GNU Wget is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 3 of the License, or
+ (at your option) any later version.
+
+GNU Wget is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with Wget.  If not, see <http://www.gnu.org/licenses/>.
+
+Additional permission under GNU GPL version 3 section 7
+
+If you modify this program, or any covered work, by linking or
+combining it with the OpenSSL project's OpenSSL library (or a
+modified version of that library), containing parts covered by the
+terms of the OpenSSL or SSLeay licenses, the Free Software Foundation
+grants you additional permission to convey the resulting work.
+Corresponding Source for a non-source form of such a combination
+shall include the source code for the parts of OpenSSL used as well
+as that of the covered work.  */
+#include "wget.h"
+
+#ifdef HAVE_HSTS
+#include "hsts.h"
+#include "utils.h"
+#include "host.h" /* for is_valid_ip_address() */
+#include "init.h" /* for home_dir() */
+#include "hash.h"
+#include "c-ctype.h"
+#ifdef TESTING
+#include "test.h"
+#endif
+
+#include <unistd.h>
+#include <sys/types.h>
+#include <stdlib.h>
+#include <time.h>
+#include <sys/stat.h>
+#include <string.h>
+#include <stdio.h>
+#include <sys/file.h>
+
+struct hsts_store {
+  struct hash_table *table;
+  time_t last_mtime;
+  bool changed;
+};
+
+struct hsts_kh {
+  char *host;
+  int explicit_port;
+};
+
+struct hsts_kh_info {
+  time_t created;
+  time_t max_age;
+  bool include_subdomains;
+};
+
+enum hsts_kh_match {
+  NO_MATCH,
+  SUPERDOMAIN_MATCH,
+  CONGRUENT_MATCH
+};
+
+#define hsts_is_host_name_valid(host) (!is_valid_ip_address (host))
+#define hsts_is_scheme_valid(scheme) (scheme == SCHEME_HTTPS)
+#define hsts_is_host_eligible(scheme, host) \
+    (hsts_is_scheme_valid (scheme) && hsts_is_host_name_valid (host))
+
+#define DEFAULT_HTTP_PORT 80
+#define DEFAULT_SSL_PORT  443
+#define MAKE_EXPLICIT_PORT(s, p) (s == SCHEME_HTTPS ? (p == DEFAULT_SSL_PORT ? 0 : p) \
+    : (p == DEFAULT_HTTP_PORT ? 0 : p))
+
+/* Hashing and comparison functions for the hash table */
+
+static unsigned long
+hsts_hash_func (const void *key)
+{
+  struct hsts_kh *k = (struct hsts_kh *) key;
+  const char *h = NULL;
+  unsigned int hash = k->explicit_port;
+
+  for (h = k->host; *h; h++)
+    hash = hash * 31 + *h;
+
+  return hash;
+}
+
+static int
+hsts_cmp_func (const void *h1, const void *h2)
+{
+  struct hsts_kh *kh1 = (struct hsts_kh *) h1,
+      *kh2 = (struct hsts_kh *) h2;
+
+  return (!strcmp (kh1->host, kh2->host)) && (kh1->explicit_port == kh2->explicit_port);
+}
+
+/* Private functions. Feel free to make some of these public when needed. */
+
+static struct hsts_kh_info *
+hsts_find_entry (hsts_store_t store,
+                 const char *host, int explicit_port,
+                 enum hsts_kh_match *match_type,
+                 struct hsts_kh *kh)
+{
+  struct hsts_kh *k = NULL;
+  struct hsts_kh_info *khi = NULL;
+  enum hsts_kh_match match = NO_MATCH;
+  char *pos = NULL;
+  char *org_ptr = NULL;
+
+  k = (struct hsts_kh *) xnew (struct hsts_kh);
+  k->host = xstrdup_lower (host);
+  k->explicit_port = explicit_port;
+
+  /* save pointer so that we don't get into trouble later when freeing */
+  org_ptr = k->host;
+
+  khi = (struct hsts_kh_info *) hash_table_get (store->table, k);
+  if (khi)
+    {
+      match = CONGRUENT_MATCH;
+      goto end;
+    }
+
+  while (match == NO_MATCH &&
+      (pos = strchr (k->host, '.')) && pos - k->host > 0 &&
+      strchr (pos + 1, '.'))
+    {
+      k->host += (pos - k->host + 1);
+      khi = (struct hsts_kh_info *) hash_table_get (store->table, k);
+      if (khi)
+        match = SUPERDOMAIN_MATCH;
+    }
+
+end:
+  /* restore pointer or we'll get a SEGV */
+  k->host = org_ptr;
+
+  /* copy parameters to previous frame */
+  if (match_type)
+    *match_type = match;
+  if (kh)
+    memcpy (kh, k, sizeof (struct hsts_kh));
+  else
+    xfree (k->host);
+
+  xfree (k);
+  return khi;
+}
+
+static bool
+hsts_new_entry_internal (hsts_store_t store,
+                         const char *host, int port,
+                         time_t created, time_t max_age,
+                         bool include_subdomains,
+                         bool check_validity,
+                         bool check_expired,
+                         bool check_duplicates)
+{
+  struct hsts_kh *kh = xnew (struct hsts_kh);
+  struct hsts_kh_info *khi = xnew0 (struct hsts_kh_info);
+  bool success = false;
+
+  kh->host = xstrdup_lower (host);
+  kh->explicit_port = MAKE_EXPLICIT_PORT (SCHEME_HTTPS, port);
+
+  khi->created = created;
+  khi->max_age = max_age;
+  khi->include_subdomains = include_subdomains;
+
+  /* Check validity */
+  if (check_validity && !hsts_is_host_name_valid (host))
+    goto bail;
+
+  if (check_expired && ((khi->created + khi->max_age) < khi->created))
+    goto bail;
+
+  if (check_duplicates && hash_table_contains (store->table, kh))
+    goto bail;
+
+  /* Now store the new entry */
+  hash_table_put (store->table, kh, khi);
+  success = true;
+
+bail:
+  if (!success)
+    {
+      /* abort! */
+      xfree (kh->host);
+      xfree (kh);
+      xfree (khi);
+    }
+
+  return success;
+}
+
+/*
+   Creates a new entry, but does not check whether that entry already exists.
+   This function assumes that check has already been done by the caller.
+ */
+static bool
+hsts_add_entry (hsts_store_t store,
+                const char *host, int port,
+                time_t max_age, bool include_subdomains)
+{
+  time_t t = time (NULL);
+
+  /* It might happen time() returned -1 */
+  return (t < 0 ?
+      false :
+      hsts_new_entry_internal (store, host, port, t, max_age, include_subdomains, false, true, false));
+}
+
+/* Creates a new entry, unless an identical one already exists. */
+static bool
+hsts_new_entry (hsts_store_t store,
+                const char *host, int port,
+                time_t created, time_t max_age,
+                bool include_subdomains)
+{
+  return hsts_new_entry_internal (store, host, port, created, max_age, include_subdomains, true, true, true);
+}
+
+static void
+hsts_remove_entry (hsts_store_t store, struct hsts_kh *kh)
+{
+  hash_table_remove (store->table, kh);
+}
+
+static bool
+hsts_store_merge (hsts_store_t store,
+                  const char *host, int port,
+                  time_t created, time_t max_age,
+                  bool include_subdomains)
+{
+  enum hsts_kh_match match_type = NO_MATCH;
+  struct hsts_kh_info *khi = NULL;
+  bool success = false;
+
+  port = MAKE_EXPLICIT_PORT (SCHEME_HTTPS, port);
+  khi = hsts_find_entry (store, host, port, &match_type, NULL);
+  if (khi && match_type == CONGRUENT_MATCH && created > khi->created)
+    {
+      /* update the entry with the new info */
+      khi->created = created;
+      khi->max_age = max_age;
+      khi->include_subdomains = include_subdomains;
+
+      success = true;
+    }
+  else if (!khi)
+    success = hsts_new_entry (store, host, port, created, max_age, include_subdomains);
+
+  return success;
+}
+
+static bool
+hsts_read_database (hsts_store_t store, FILE *fp, bool merge_with_existing_entries)
+{
+  char *line = NULL, *p;
+  size_t len = 0;
+  int items_read;
+  bool result = false;
+  bool (*func)(hsts_store_t, const char *, int, time_t, time_t, bool);
+
+  char host[256];
+  int port;
+  time_t created, max_age;
+  int include_subdomains;
+
+  func = (merge_with_existing_entries ? hsts_store_merge : hsts_new_entry);
+
+  while (getline (&line, &len, fp) > 0)
+    {
+      for (p = line; c_isspace (*p); p++)
+        ;
+
+      if (*p == '#')
+        continue;
+
+      items_read = sscanf (p, "%255s %d %d %lu %lu",
+                           host,
+                           &port,
+                           &include_subdomains,
+                           (unsigned long *) &created,
+                           (unsigned long *) &max_age);
+
+      if (items_read == 5)
+        func (store, host, port, created, max_age, !!include_subdomains);
+    }
+
+  xfree (line);
+  result = true;
+
+  return result;
+}
+
+static void
+hsts_store_dump (hsts_store_t store, FILE *fp)
+{
+  hash_table_iterator it;
+
+  /* Print preliminary comments. We don't care if any of these fail. */
+  fputs ("# HSTS 1.0 Known Hosts database for GNU Wget.\n", fp);
+  fputs ("# Edit at your own risk.\n", fp);
+  fputs ("# <hostname>\t<port>\t<incl. subdomains>\t<created>\t<max-age>\n", fp);
+
+  /* Now cycle through the HSTS store in memory and dump the entries */
+  for (hash_table_iterate (store->table, &it); hash_table_iter_next (&it);)
+    {
+      struct hsts_kh *kh = (struct hsts_kh *) it.key;
+      struct hsts_kh_info *khi = (struct hsts_kh_info *) it.value;
+
+      if (fprintf (fp, "%s\t%d\t%d\t%lu\t%lu\n",
+                   kh->host, kh->explicit_port, khi->include_subdomains,
+                   (unsigned long) khi->created,
+                   (unsigned long) khi->max_age) < 0)
+        {
+          logprintf (LOG_ALWAYS, "Could not write the HSTS database correctly.\n");
+          break;
+        }
+    }
+}
+
+/*
+ * Test:
+ *  - The file is a regular file (ie. not a symlink), and
+ *  - The file is not world-writable.
+ */
+static bool
+hsts_file_access_valid (const char *filename)
+{
+  struct stat st;
+
+  if (stat (filename, &st) == -1)
+    return false;
+
+  return
+#ifndef WINDOWS
+      /*
+       * The world-writable concept is a Unix-centric notion.
+       * We bypass this test on Windows.
+       */
+      !(st.st_mode & S_IWOTH) &&
+#endif
+      S_ISREG (st.st_mode);
+}
+
+/* HSTS API */
+
+/*
+   Changes the given URLs according to the HSTS policy.
+
+   If there's no host in the store that either congruently
+   or not, matches the given URL, no changes are made.
+   Returns true if the URL was changed, or false
+   if it was left intact.
+ */
+bool
+hsts_match (hsts_store_t store, struct url *u)
+{
+  bool url_changed = false;
+  struct hsts_kh_info *entry = NULL;
+  struct hsts_kh *kh = xnew(struct hsts_kh);
+  enum hsts_kh_match match = NO_MATCH;
+  int port = MAKE_EXPLICIT_PORT (u->scheme, u->port);
+
+  /* avoid doing any computation if we're already in HTTPS */
+  if (!hsts_is_scheme_valid (u->scheme))
+    {
+      entry = hsts_find_entry (store, u->host, port, &match, kh);
+      if (entry)
+        {
+          if ((entry->created + entry->max_age) >= time(NULL))
+            {
+              if ((match == CONGRUENT_MATCH) ||
+                  (match == SUPERDOMAIN_MATCH && entry->include_subdomains))
+                {
+                  /* we found a matching Known HSTS Host
+                     rewrite the URL */
+                  u->scheme = SCHEME_HTTPS;
+                  if (u->port == 80)
+                    u->port = 443;
+                  url_changed = true;
+                  store->changed = true;
+                }
+            }
+          else
+            {
+              hsts_remove_entry (store, kh);
+              store->changed = true;
+            }
+        }
+      xfree (kh->host);
+    }
+
+  xfree (kh);
+
+  return url_changed;
+}
+
+/*
+   Add a new HSTS Known Host to the HSTS store.
+
+   If the host already exists, its information is updated,
+   or it'll be removed from the store if max_age is zero.
+
+   Bear in mind that the store is kept in memory, and will not
+   be written to disk until hsts_store_save is called.
+   This function regrows the in-memory HSTS store if necessary.
+
+   Currently, for a host to be taken into consideration,
+   two conditions have to be met:
+     - Connection must be through a secure channel (HTTPS).
+     - The host must not be an IPv4 or IPv6 address.
+
+   The RFC 6797 states that hosts that match IPv4 or IPv6 format
+   should be discarded at URI rewrite time. But we short-circuit
+   that check here, since there's no point in storing a host that
+   will never be matched.
+
+   Returns true if a new entry was actually created, or false
+   if an existing entry was updated/deleted. */
+bool
+hsts_store_entry (hsts_store_t store,
+                  enum url_scheme scheme, const char *host, int port,
+                  time_t max_age, bool include_subdomains)
+{
+  bool result = false;
+  enum hsts_kh_match match = NO_MATCH;
+  struct hsts_kh *kh = xnew(struct hsts_kh);
+  struct hsts_kh_info *entry = NULL;
+
+  if (hsts_is_host_eligible (scheme, host))
+    {
+      port = MAKE_EXPLICIT_PORT (scheme, port);
+      entry = hsts_find_entry (store, host, port, &match, kh);
+      if (entry && match == CONGRUENT_MATCH)
+        {
+          if (max_age == 0)
+            {
+              hsts_remove_entry (store, kh);
+              store->changed = true;
+            }
+          else if (max_age > 0)
+            {
+              /* RFC 6797 states that 'max_age' is a TTL relative to the
+               * reception of the STS header so we have to update the
+               * 'created' field too. The RFC also states that we have to
+               * update the entry each time we see HSTS header.
+               * See also Section 11.2. */
+              time_t t = time (NULL);
+
+              if (t != -1 && t != entry->created)
+                {
+                  entry->created = t;
+                  entry->max_age = max_age;
+                  entry->include_subdomains = include_subdomains;
+                  store->changed = true;
+                }
+            }
+          /* we ignore negative max_ages */
+        }
+      else if (entry == NULL || match == SUPERDOMAIN_MATCH)
+        {
+          /* Either we didn't find a matching host,
+             or we got a superdomain match.
+             In either case, we create a new entry.
+
+             We have to perform an explicit check because it might
+             happen we got a non-existent entry with max_age == 0.
+          */
+          result = hsts_add_entry (store, host, port, max_age, include_subdomains);
+          if (result)
+            store->changed = true;
+        }
+      /* we ignore new entries with max_age == 0 */
+      xfree (kh->host);
+    }
+
+  xfree (kh);
+
+  return result;
+}
+
+hsts_store_t
+hsts_store_open (const char *filename)
+{
+  hsts_store_t store = NULL;
+  file_stats_t fstats;
+
+  store = xnew0 (struct hsts_store);
+  store->table = hash_table_new (0, hsts_hash_func, hsts_cmp_func);
+  store->last_mtime = 0;
+  store->changed = false;
+
+  if (file_exists_p (filename, &fstats))
+    {
+      if (hsts_file_access_valid (filename))
+        {
+          struct stat st;
+          FILE *fp = fopen_stat (filename, "r", &fstats);
+
+          if (!fp || !hsts_read_database (store, fp, false))
+            {
+              /* abort! */
+              hsts_store_close (store);
+              xfree (store);
+              if (fp)
+                fclose (fp);
+              goto out;
+            }
+
+          if (fstat (fileno (fp), &st) == 0)
+            store->last_mtime = st.st_mtime;
+
+          fclose (fp);
+        }
+      else
+        {
+          /*
+           * If we're not reading the HSTS database,
+           * then by all means act as if HSTS was disabled.
+           */
+          hsts_store_close (store);
+          xfree (store);
+
+          logprintf (LOG_NOTQUIET, "Will not apply HSTS. "
+                     "The HSTS database must be a regular and non-world-writable file.\n");
+        }
+    }
+
+out:
+  return store;
+}
+
+void
+hsts_store_save (hsts_store_t store, const char *filename)
+{
+  struct stat st;
+  FILE *fp = NULL;
+  int fd = 0;
+
+  if (filename && hash_table_count (store->table) > 0)
+    {
+      fp = fopen (filename, "a+");
+      if (fp)
+        {
+          /* Lock the file to avoid potential race conditions */
+          fd = fileno (fp);
+          flock (fd, LOCK_EX);
+
+          /* If the file has changed, merge the changes with our in-memory data
+             before dumping them to the file.
+             Otherwise we could potentially overwrite the data stored by other Wget processes.
+           */
+          if (store->last_mtime && stat (filename, &st) == 0 && st.st_mtime > store->last_mtime)
+            hsts_read_database (store, fp, true);
+
+          /* We've merged the latest changes so we can now truncate the file
+             and dump everything. */
+          fseek (fp, 0, SEEK_SET);
+          ftruncate (fd, 0);
+
+          /* now dump to the file */
+          hsts_store_dump (store, fp);
+
+          /* fclose is expected to unlock the file for us */
+          fclose (fp);
+        }
+    }
+}
+
+bool
+hsts_store_has_changed (hsts_store_t store)
+{
+  return (store ? store->changed : false);
+}
+
+void
+hsts_store_close (hsts_store_t store)
+{
+  hash_table_iterator it;
+
+  /* free all the host fields */
+  for (hash_table_iterate (store->table, &it); hash_table_iter_next (&it);)
+    {
+      xfree (((struct hsts_kh *) it.key)->host);
+      xfree (it.key);
+      xfree (it.value);
+    }
+
+  hash_table_destroy (store->table);
+}
+
+#ifdef TESTING
+/* I know I'm really evil because I'm writing macros
+   that change control flow. But we're testing, who will tell? :D
+ */
+#define TEST_URL_RW(s, u, p) do { \
+    if (test_url_rewrite (s, u, p, true)) \
+      return test_url_rewrite (s, u, p, true); \
+  } while (0)
+
+#define TEST_URL_NORW(s, u, p) do { \
+    if (test_url_rewrite (s, u, p, false)) \
+      return test_url_rewrite (s, u, p, false); \
+  } while (0)
+
+static char *
+get_hsts_store_filename (void)
+{
+  char *home = NULL, *filename = NULL;
+  FILE *fp = NULL;
+
+  home = home_dir ();
+  if (home)
+    {
+      filename = aprintf ("%s/.wget-hsts-test", home);
+      fp = fopen (filename, "w");
+      if (fp)
+        fclose (fp);
+    }
+
+  xfree (home);
+  return filename;
+}
+
+static hsts_store_t
+open_hsts_test_store (void)
+{
+  char *filename = NULL;
+  hsts_store_t table = NULL;
+
+  filename = get_hsts_store_filename ();
+  table = hsts_store_open (filename);
+  xfree (filename);
+
+  return table;
+}
+
+static void
+close_hsts_test_store (hsts_store_t store)
+{
+  char *filename = NULL;
+
+  filename = get_hsts_store_filename ();
+  unlink (filename);
+  xfree (filename);
+  xfree (store);
+}
+
+static const char*
+test_url_rewrite (hsts_store_t s, const char *url, int port, bool rewrite)
+{
+  bool result;
+  struct url u;
+
+  u.host = xstrdup (url);
+  u.port = port;
+  u.scheme = SCHEME_HTTP;
+
+  result = hsts_match (s, &u);
+
+  if (rewrite)
+    {
+      if (port == 80)
+        mu_assert("URL: port should've been rewritten to 443", u.port == 443);
+      else
+        mu_assert("URL: port should've been left intact", u.port == port);
+      mu_assert("URL: scheme should've been rewritten to HTTPS", u.scheme == SCHEME_HTTPS);
+      mu_assert("result should've been true", result == true);
+    }
+  else
+    {
+      mu_assert("URL: port should've been left intact", u.port == port);
+      mu_assert("URL: scheme should've been left intact", u.scheme == SCHEME_HTTP);
+      mu_assert("result should've been false", result == false);
+    }
+
+  xfree (u.host);
+  return NULL;
+}
+
+const char *
+test_hsts_new_entry (void)
+{
+  enum hsts_kh_match match = NO_MATCH;
+  struct hsts_kh_info *khi;
+  hsts_store_t s;
+  bool created;
+
+  s = open_hsts_test_store ();
+  mu_assert("Could not open the HSTS store. This could be due to lack of memory.", s != NULL);
+
+  created = hsts_store_entry (s, SCHEME_HTTP, "www.foo.com", 80, 1234, true);
+  mu_assert("No entry should have been created.", created == false);
+
+  created = hsts_store_entry (s, SCHEME_HTTPS, "www.foo.com", 443, 1234, true);
+  mu_assert("A new entry should have been created", created == true);
+
+  khi = hsts_find_entry (s, "www.foo.com", MAKE_EXPLICIT_PORT (SCHEME_HTTPS, 443), &match, NULL);
+  mu_assert("Should've been a congruent match", match == CONGRUENT_MATCH);
+  mu_assert("No valid HSTS info was returned", khi != NULL);
+  mu_assert("Variable 'max_age' should be 1234", khi->max_age == 1234);
+  mu_assert("Variable 'include_subdomains' should be asserted", khi->include_subdomains == true);
+
+  khi = hsts_find_entry (s, "b.www.foo.com", MAKE_EXPLICIT_PORT (SCHEME_HTTPS, 443), &match, NULL);
+  mu_assert("Should've been a superdomain match", match == SUPERDOMAIN_MATCH);
+  mu_assert("No valid HSTS info was returned", khi != NULL);
+  mu_assert("Variable 'max_age' should be 1234", khi->max_age == 1234);
+  mu_assert("Variable 'include_subdomains' should be asserted", khi->include_subdomains == true);
+
+  khi = hsts_find_entry (s, "ww.foo.com", MAKE_EXPLICIT_PORT (SCHEME_HTTPS, 443), &match, NULL);
+  mu_assert("Should've been no match", match == NO_MATCH);
+
+  khi = hsts_find_entry (s, "foo.com", MAKE_EXPLICIT_PORT (SCHEME_HTTPS, 443), &match, NULL);
+  mu_assert("Should've been no match", match == NO_MATCH);
+
+  khi = hsts_find_entry (s, ".foo.com", MAKE_EXPLICIT_PORT (SCHEME_HTTPS, 443), &match, NULL);
+  mu_assert("Should've been no match", match == NO_MATCH);
+
+  khi = hsts_find_entry (s, ".www.foo.com", MAKE_EXPLICIT_PORT (SCHEME_HTTPS, 443), &match, NULL);
+  mu_assert("Should've been no match", match == NO_MATCH);
+
+  hsts_store_close (s);
+  close_hsts_test_store (s);
+
+  return NULL;
+}
+
+const char*
+test_hsts_url_rewrite_superdomain (void)
+{
+  hsts_store_t s;
+  bool created;
+
+  s = open_hsts_test_store ();
+  mu_assert("Could not open the HSTS store", s != NULL);
+
+  created = hsts_store_entry (s, SCHEME_HTTPS, "www.foo.com", 443, 1234, true);
+  mu_assert("A new entry should've been created", created == true);
+
+  TEST_URL_RW (s, "www.foo.com", 80);
+  TEST_URL_RW (s, "bar.www.foo.com", 80);
+
+  hsts_store_close (s);
+  close_hsts_test_store (s);
+
+  return NULL;
+}
+
+const char*
+test_hsts_url_rewrite_congruent (void)
+{
+  hsts_store_t s;
+  bool created;
+
+  s = open_hsts_test_store ();
+  mu_assert("Could not open the HSTS store", s != NULL);
+
+  created = hsts_store_entry (s, SCHEME_HTTPS, "foo.com", 443, 1234, false);
+  mu_assert("A new entry should've been created", created == true);
+
+  TEST_URL_RW (s, "foo.com", 80);
+  TEST_URL_NORW (s, "www.foo.com", 80);
+
+  hsts_store_close (s);
+  close_hsts_test_store (s);
+
+  return NULL;
+}
+
+const char*
+test_hsts_read_database (void)
+{
+  hsts_store_t table;
+  char *home = home_dir();
+  char *file = NULL;
+  FILE *fp = NULL;
+  time_t created = time(NULL) - 10;
+
+  if (home)
+    {
+      file = aprintf ("%s/.wget-hsts-testing", home);
+      fp = fopen (file, "w");
+      if (fp)
+        {
+          fputs ("# dummy comment\n", fp);
+          fprintf (fp, "foo.example.com\t0\t1\t%lu\t123\n",(unsigned long) created);
+          fprintf (fp, "bar.example.com\t0\t0\t%lu\t456\n", (unsigned long) created);
+          fprintf (fp, "test.example.com\t8080\t0\t%lu\t789\n", (unsigned long) created);
+          fclose (fp);
+
+          table = hsts_store_open (file);
+
+          TEST_URL_RW (table, "foo.example.com", 80);
+          TEST_URL_RW (table, "www.foo.example.com", 80);
+          TEST_URL_RW (table, "bar.example.com", 80);
+
+          TEST_URL_NORW(table, "www.bar.example.com", 80);
+
+          TEST_URL_RW (table, "test.example.com", 8080);
+
+          hsts_store_close (table);
+          close_hsts_test_store (table);
+          unlink (file);
+        }
+      xfree (file);
+      xfree (home);
+    }
+
+  return NULL;
+}
+#endif /* TESTING */
+#endif /* HAVE_HSTS */
--- a/src/multi.h
+++ b/src/multi.h
@@ -1,13 +1,12 @@
-/* Declarations for HTTP.
-   Copyright (C) 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012 Free Software
-   Foundation, Inc.
+/* Declarations for hsts.c
+   Copyright (C) 1996-2012, 2015, 2018 Free Software Foundation, Inc.

 This file is part of GNU Wget.

 GNU Wget is free software; you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
 the Free Software Foundation; either version 3 of the License, or
-(at your option) any later version.
+ (at your option) any later version.

 GNU Wget is distributed in the hope that it will be useful,
 but WITHOUT ANY WARRANTY; without even the implied warranty of
@@ -27,63 +26,27 @@ grants you additional permission to convey the resulting work.
 Corresponding Source for a non-source form of such a combination
 shall include the source code for the parts of OpenSSL used as well
 as that of the covered work.  */
-
-#ifndef MULTI_H
-#define MULTI_H
-
-#include <semaphore.h>
-
 #include "wget.h"

-#include "iri.h"
+#ifdef HAVE_HSTS
+
+#ifndef HSTS_H
+#define HSTS_H
+
 #include "url.h"

-#define MIN_CHUNK_SIZE 2048
+typedef struct hsts_store *hsts_store_t;

-struct s_thread_ctx
-{
-  pthread_t thread;
-  int used;
-  int terminated;
-  int dt, url_err;
-  char *redirected;
-  char *referer;
-  struct url *url_parsed;
-  struct iri *i;
-  struct range *range;
-  char *file;
-  char *url;
-#ifdef ENABLE_THREADS
-  sem_t *retr_sem;
-#else
-  /* Not used.  */
-  void *retr_sem;
-#endif
-  uerr_t status;
-};
+hsts_store_t hsts_store_open (const char *);

-void init_temp_files();
+void hsts_store_save (hsts_store_t, const char *);
+void hsts_store_close (hsts_store_t);
+bool hsts_store_has_changed (hsts_store_t);

-void name_temp_files();
+bool hsts_store_entry (hsts_store_t,
+                       enum url_scheme, const char *, int,
+                       time_t, bool);
+bool hsts_match (hsts_store_t, struct url *);

-void merge_temp_files(char *);
-
-void delete_temp_files();
-
-void clean_temp_files();
-
-void init_ranges();
-
-int fill_ranges_data(int, long long int, long int);
-
-void clean_range_res_data();
-
-void clean_ranges();
-
-int spawn_thread (struct s_thread_ctx*, int, int);
-
-int collect_thread (sem_t *, struct s_thread_ctx *);
-
-static void * segmented_retrieve_url (void *);
-
-#endif /* MULTI_H */
+#endif /* HSTS_H */
+#endif /* HAVE_HSTS */
--- a/src/html-parse.c
+++ b/src/html-parse.c
@@ -1,6 +1,5 @@
 /* HTML parser for Wget.
-   Copyright (C) 1998, 1999, 2000, 2001, 2002, 2003, 2004, 2005, 2006,
-   2007, 2008, 2009, 2010, 2011 Free Software Foundation, Inc.
+   Copyright (C) 1998-2011, 2015, 2018 Free Software Foundation, Inc.

 This file is part of GNU Wget.

@@ -176,8 +175,8 @@ struct pool {
  P->orig_size = P->size;                                       \
 } while (0)

-/* Grow the pool to accomodate at least SIZE new bytes.  If the pool
-   already has room to accomodate SIZE bytes of data, this is a no-op.  */
+/* Grow the pool to accommodate at least SIZE new bytes.  If the pool
+   already has room to accommodate SIZE bytes of data, this is a no-op.  */

 #define POOL_GROW(p, increase)                                  \
  GROW_ARRAY ((p)->contents, (p)->size, (p)->tail + (increase), \
@@ -253,7 +252,7 @@ struct pool {
      (sizevar) = ga_newsize;                                                   \
    }                                                                           \
 } while (0)
-
+
 /* Test whether n+1-sized entity name fits in P.  We don't support
   IE-style non-terminated entities, e.g. "&ltfoo" -> "<foo".
   However, "&lt;foo" will work, as will "&lt!foo", "&lt", etc.  In
@@ -517,7 +516,7 @@ convert_and_copy (struct pool *pool, const char *beg, const char *end, int flags
        *p = c_tolower (*p);
    }
 }
-
+
 /* Originally we used to adhere to rfc 1866 here, and allowed only
   letters, digits, periods, and hyphens as names (of tags or
   attributes).  However, this broke too many pages which used
@@ -770,7 +769,7 @@ find_comment_end (const char *beg, const char *end)
      }
  return NULL;
 }
-
+
 /* Return true if the string containing of characters inside [b, e) is
   present in hash table HT.  */

@@ -801,14 +800,6 @@ name_allowed (const struct hash_table *ht, const char *b, const char *e)
  }                                             \
 } while (0)

-/* Skip non-whitespace, if any. */
-
-#define SKIP_NON_WS(p) do {                     \
-  while (!c_isspace (*p)) {                       \
-    ADVANCE (p);                                \
-  }                                             \
-} while (0)
-
 #ifdef STANDALONE
 static int tag_backout_count;
 #endif
@@ -881,7 +872,7 @@ map_html_tags (const char *text, int size,
    if (*p == '!')
      {
        if (!(flags & MHT_STRICT_COMMENTS)
-            && p < end + 3 && p[1] == '-' && p[2] == '-')
+            && p + 3 < end && p[1] == '-' && p[2] == '-')
          {
            /* If strict comments are not enforced and if we know
               we're looking at a comment, simply look for the
@@ -1167,7 +1158,7 @@ map_html_tags (const char *text, int size,
 #undef ADVANCE
 #undef SKIP_WS
 #undef SKIP_NON_WS
-
+
 #ifdef STANDALONE
 static void
 test_mapper (struct taginfo *taginfo, void *arg)
--- a/src/html-parse.h
+++ b/src/html-parse.h
@@ -1,6 +1,5 @@
 /* Declarations for html-parse.c.
-   Copyright (C) 1998, 1999, 2000, 2001, 2002, 2003, 2004, 2005, 2006,
-   2007, 2008, 2009, 2010, 2011 Free Software Foundation, Inc.
+   Copyright (C) 1998-2011, 2015, 2018 Free Software Foundation, Inc.

 This file is part of GNU Wget.

--- a/src/html-url.c
+++ b/src/html-url.c
@@ -1,6 +1,5 @@
 /* Collect URLs from HTML source.
-   Copyright (C) 1998, 1999, 2000, 2001, 2002, 2003, 2004, 2005, 2006,
-   2007, 2008, 2009, 2010, 2011, 2012 Free Software Foundation, Inc.
+   Copyright (C) 1998-2012, 2015, 2018 Free Software Foundation, Inc.

 This file is part of GNU Wget.

@@ -45,6 +44,7 @@ as that of the covered work.  */
 #include "recur.h"
 #include "html-url.h"
 #include "css-url.h"
+#include "c-strcase.h"

 typedef void (*tag_handler_t) (int, struct taginfo *, struct map_context *);

@@ -54,6 +54,7 @@ typedef void (*tag_handler_t) (int, struct taginfo *, struct map_context *);
 DECLARE_TAG_HANDLER (tag_find_urls);
 DECLARE_TAG_HANDLER (tag_handle_base);
 DECLARE_TAG_HANDLER (tag_handle_form);
+DECLARE_TAG_HANDLER (tag_handle_img);
 DECLARE_TAG_HANDLER (tag_handle_link);
 DECLARE_TAG_HANDLER (tag_handle_meta);

@@ -103,7 +104,7 @@ static struct known_tag {
  { TAG_FORM,    "form",        tag_handle_form },
  { TAG_FRAME,   "frame",       tag_find_urls },
  { TAG_IFRAME,  "iframe",      tag_find_urls },
-  { TAG_IMG,     "img",         tag_find_urls },
+  { TAG_IMG,     "img",         tag_handle_img },
  { TAG_INPUT,   "input",       tag_find_urls },
  { TAG_LAYER,   "layer",       tag_find_urls },
  { TAG_LINK,    "link",        tag_handle_link },
@@ -181,7 +182,8 @@ static const char *additional_attributes[] = {
  "name",                       /* used by tag_handle_meta  */
  "content",                    /* used by tag_handle_meta  */
  "action",                     /* used by tag_handle_form  */
-  "style"                       /* used by check_style_attr */
+  "style",                      /* used by check_style_attr */
+  "srcset",                     /* used by tag_handle_img */
 };

 static struct hash_table *interesting_tags;
@@ -255,7 +257,7 @@ find_attr (struct taginfo *tag, const char *name, int *attrind)
 {
  int i;
  for (i = 0; i < tag->nattrs; i++)
-    if (!strcasecmp (tag->attrs[i].name, name))
+    if (!c_strcasecmp (tag->attrs[i].name, name))
      {
        if (attrind)
          *attrind = i;
@@ -302,6 +304,7 @@ append_url (const char *link_uri, int position, int size,
          logprintf (LOG_NOTQUIET,
                     _("%s: Cannot resolve incomplete link %s.\n"),
                     ctx->document_file, link_uri);
+          iri_free (iri);
          return NULL;
        }

@@ -310,6 +313,7 @@ append_url (const char *link_uri, int position, int size,
        {
          DEBUGP (("%s: link \"%s\" doesn't parse.\n",
                   ctx->document_file, link_uri));
+          iri_free (iri);
          return NULL;
        }
    }
@@ -333,6 +337,7 @@ append_url (const char *link_uri, int position, int size,
          DEBUGP (("%s: merged link \"%s\" doesn't parse.\n",
                   ctx->document_file, complete_uri));
          xfree (complete_uri);
+          iri_free (iri);
          return NULL;
        }
      xfree (complete_uri);
@@ -378,7 +383,7 @@ append_url (const char *link_uri, int position, int size,

  return newel;
 }
-
+
 static void
 check_style_attr (struct taginfo *tag, struct map_context *ctx)
 {
@@ -487,8 +492,7 @@ tag_handle_base (int tagid _GL_UNUSED, struct taginfo *tag, struct map_context *
  base_urlpos->ignore_when_downloading = 1;
  base_urlpos->link_base_p = 1;

-  if (ctx->base)
-    xfree (ctx->base);
+  xfree (ctx->base);
  if (ctx->parent_base)
    ctx->base = uri_merge (ctx->parent_base, newbase);
  else
@@ -536,12 +540,12 @@ tag_handle_link (int tagid _GL_UNUSED, struct taginfo *tag, struct map_context *
          char *rel = find_attr (tag, "rel", NULL);
          if (rel)
            {
-              if (0 == strcasecmp (rel, "stylesheet"))
+              if (0 == c_strcasecmp (rel, "stylesheet"))
                {
                  up->link_inline_p = 1;
                  up->link_expect_css = 1;
                }
-              else if (0 == strcasecmp (rel, "shortcut icon"))
+              else if (0 == c_strcasecmp (rel, "shortcut icon"))
                {
                  up->link_inline_p = 1;
                }
@@ -553,7 +557,7 @@ tag_handle_link (int tagid _GL_UNUSED, struct taginfo *tag, struct map_context *
                     <link rel="alternate" type="application/rss+xml" href=".../?feed=rss2" />
                  */
                  char *type = find_attr (tag, "type", NULL);
-                  if (!type || strcasecmp (type, "text/html") == 0)
+                  if (!type || c_strcasecmp (type, "text/html") == 0)
                    up->link_expect_html = 1;
                }
            }
@@ -570,7 +574,7 @@ tag_handle_meta (int tagid _GL_UNUSED, struct taginfo *tag, struct map_context *
  char *name = find_attr (tag, "name", NULL);
  char *http_equiv = find_attr (tag, "http-equiv", NULL);

-  if (http_equiv && 0 == strcasecmp (http_equiv, "refresh"))
+  if (http_equiv && 0 == c_strcasecmp (http_equiv, "refresh"))
    {
      /* Some pages use a META tag to specify that the page be
         refreshed by a new page after a given number of seconds.  The
@@ -615,7 +619,7 @@ tag_handle_meta (int tagid _GL_UNUSED, struct taginfo *tag, struct map_context *
          entry->link_expect_html = 1;
        }
    }
-  else if (http_equiv && 0 == strcasecmp (http_equiv, "content-type"))
+  else if (http_equiv && 0 == c_strcasecmp (http_equiv, "content-type"))
    {
      /* Handle stuff like:
         <meta http-equiv="Content-Type" content="text/html; charset=CHARSET"> */
@@ -629,17 +633,17 @@ tag_handle_meta (int tagid _GL_UNUSED, struct taginfo *tag, struct map_context *
      if (!mcharset)
        return;

-      xfree_null (meta_charset);
+      xfree (meta_charset);
      meta_charset = mcharset;
    }
-  else if (name && 0 == strcasecmp (name, "robots"))
+  else if (name && 0 == c_strcasecmp (name, "robots"))
    {
      /* Handle stuff like:
         <meta name="robots" content="index,nofollow"> */
      char *content = find_attr (tag, "content", NULL);
      if (!content)
        return;
-      if (!strcasecmp (content, "none"))
+      if (!c_strcasecmp (content, "none"))
        ctx->nofollow = true;
      else
        {
@@ -651,7 +655,7 @@ tag_handle_meta (int tagid _GL_UNUSED, struct taginfo *tag, struct map_context *
              /* Find the next occurrence of ',' or whitespace,
               * or the end of the string.  */
              end = content + strcspn (content, ", \f\n\r\t\v");
-              if (!strncasecmp (content, "nofollow", end - content))
+              if (!c_strncasecmp (content, "nofollow", end - content))
                ctx->nofollow = true;
              /* Skip past the next comma, if any. */
              if (*end == ',')
@@ -670,6 +674,91 @@ tag_handle_meta (int tagid _GL_UNUSED, struct taginfo *tag, struct map_context *
    }
 }

+/* Handle the IMG tag.  This requires special handling for the srcset attr,
+   while the traditional src/lowsrc/href attributes can be handled generically.
+*/
+
+static void
+tag_handle_img (int tagid, struct taginfo *tag, struct map_context *ctx) {
+  int attrind;
+  char *srcset;
+
+  /* Use the generic approach for the attributes without special syntax. */
+  tag_find_urls(tagid, tag, ctx);
+
+  srcset = find_attr (tag, "srcset", &attrind);
+  if (srcset)
+    {
+      /* These are relative to the input text. */
+      int base_ind = ATTR_POS (tag,attrind,ctx);
+      int size = strlen (srcset);
+
+      /* These are relative to srcset. */
+      int offset, url_start, url_end;
+
+      /* Make sure to line up base_ind with srcset[0], not outside quotes. */
+      if (ctx->text[base_ind] == '"' || ctx->text[base_ind] == '\'')
+        ++base_ind;
+
+      offset = 0;
+      while (offset < size)
+        {
+          bool has_descriptor = true;
+
+          /* Skip over initial whitespace and commas. Note there is no \v
+            in HTML5 whitespace. */
+          url_start = offset + strspn (srcset + offset, " \f\n\r\t,");
+
+          if (url_start == size)
+            return;
+
+          /* URL is any non-whitespace chars (including commas) - but with
+             trailing commas removed. */
+          url_end = url_start + strcspn (srcset + url_start, " \f\n\r\t");
+          while ((url_end - 1) > url_start && srcset[url_end - 1] == ',')
+            {
+              has_descriptor = false;
+              --url_end;
+            }
+
+          if (url_end > url_start)
+            {
+              char *url_text = strdupdelim (srcset + url_start,
+                                            srcset + url_end);
+              struct urlpos *up = append_url (url_text, base_ind + url_start,
+                                              url_end - url_start, ctx);
+              if (up)
+                {
+                  up->link_inline_p = 1;
+                  up->link_noquote_html_p = 1;
+                }
+              xfree (url_text);
+            }
+
+          /* If the URL wasn't terminated by a , there may also be a descriptor
+             which we just skip. */
+          if (has_descriptor)
+            {
+              /* This is comma-terminated, except there may be one level of
+                 parentheses escaping that. */
+              bool in_paren = false;
+              for (offset = url_end; offset < size; ++offset)
+                {
+                  char c = srcset[offset];
+                  if (c == '(')
+                    in_paren = true;
+                  else if (c == ')' && in_paren)
+                    in_paren = false;
+                  else if (c == ',' && !in_paren)
+                    break;
+                }
+            }
+          else
+            offset = url_end;
+        }
+    }
+}
+
 /* Dispatch the tag handler appropriate for the tag we're mapping
   over.  See known_tags[] for definition of tag handlers.  */

@@ -692,7 +781,7 @@ collect_tags_mapper (struct taginfo *tag, void *arg)

  check_style_attr (tag, ctx);

-  if (tag->end_tag_p && (0 == strcasecmp (tag->name, "style"))
+  if (tag->end_tag_p && (0 == c_strcasecmp (tag->name, "style"))
      && tag->contents_begin && tag->contents_end
      && tag->contents_begin <= tag->contents_end)
  {
@@ -701,7 +790,7 @@ collect_tags_mapper (struct taginfo *tag, void *arg)
                  tag->contents_end - tag->contents_begin);
  }
 }
-
+
 /* Analyze HTML tags FILE and construct a list of URLs referenced from
   it.  It merges relative links in FILE with URL.  It is aware of
   <base href=...> and does the right thing.  */
@@ -748,15 +837,19 @@ get_urls_html (const char *file, const char *url, bool *meta_disallow_follow,
  map_html_tags (fm->content, fm->length, collect_tags_mapper, &ctx, flags,
                 NULL, interesting_attributes);

-  /* If meta charset isn't null, override content encoding */
-  if (iri && meta_charset)
+#ifdef ENABLE_IRI
+  /* Meta charset is only valid if there was no HTTP header Content-Type charset. */
+  /* This is true for HTTP 1.0 and 1.1. */
+  if (iri && !iri->content_encoding && meta_charset)
    set_content_encoding (iri, meta_charset);
+#endif
+  xfree (meta_charset);

  DEBUGP (("no-follow in %s: %d\n", file, ctx.nofollow));
  if (meta_disallow_follow)
    *meta_disallow_follow = ctx.nofollow;

-  xfree_null (ctx.base);
+  xfree (ctx.base);
  wget_read_file_free (fm);
  return ctx.head;
 }
@@ -787,6 +880,7 @@ get_urls_file (const char *file)
    {
      int up_error_code;
      char *url_text;
+      char *new_url;
      struct urlpos *entry;
      struct url *url;

@@ -821,7 +915,7 @@ get_urls_file (const char *file)
          url_text = merged;
        }

-      char *new_url = rewrite_shorthand_url (url_text);
+      new_url = rewrite_shorthand_url (url_text);
      if (new_url)
        {
          xfree (url_text);
--- a/src/html-url.h
+++ b/src/html-url.h
@@ -1,5 +1,5 @@
 /* Declarations for html-url.c.
-   Copyright (C) 1995, 1996, 1997, 2009, 2010, 2011 Free Software
+   Copyright (C) 1995-1997, 2009-2011, 2015, 2018 Free Software
   Foundation, Inc.

 This file is part of GNU Wget.
--- a/src/http-ntlm.c
+++ b/src/http-ntlm.c
@@ -1,6 +1,5 @@
 /* NTLM code.
-   Copyright (C) 2005, 2006, 2007, 2008, 2009, 2010, 2011 Free Software
-   Foundation, Inc.
+   Copyright (C) 2005-2011, 2015, 2018 Free Software Foundation, Inc.
   Contributed by Daniel Stenberg.

 This file is part of GNU Wget.
@@ -74,13 +73,13 @@ as that of the covered work.  */
 /* Define this to make the type-3 message include the NT response message */
 #define USE_NTRESPONSES 1

-
+
 /* Flag bits definitions available at on
   http://davenport.sourceforge.net/ntlm.html */

 #define NTLMFLAG_NEGOTIATE_OEM                   (1<<1)
 #define NTLMFLAG_NEGOTIATE_NTLM_KEY              (1<<9)
-
+
 /*
  (*) = A "security buffer" is a triplet consisting of two shorts and one
  long:
@@ -122,7 +121,7 @@ ntlm_input (struct ntlmdata *ntlm, const char *header)

      DEBUGP (("Received a type-2 NTLM message.\n"));

-      size = base64_decode (header, buffer);
+      size = wget_base64_decode (header, buffer, strlen (header));
      if (size < 0)
        return false;           /* malformed base64 from server */

@@ -411,7 +410,7 @@ ntlm_output (struct ntlmdata *ntlm, const char *user, const char *passwd,
    size = 32 + hostlen + domlen;

    base64 = (char *) alloca (BASE64_LENGTH (size) + 1);
-    base64_encode (ntlmbuf, size, base64);
+    wget_base64_encode (ntlmbuf, size, base64);

    output = concat_strings ("NTLM ", base64, (char *) 0);
    break;
@@ -474,82 +473,82 @@ ntlm_output (struct ntlmdata *ntlm, const char *user, const char *passwd,

    /* Create the big type-3 message binary blob */

-    size = (size_t) snprintf (ntlmbuf, sizeof(ntlmbuf),
-                     "NTLMSSP%c"
-                     "\x03%c%c%c" /* type-3, 32 bits */
+    snprintf (ntlmbuf, sizeof (ntlmbuf),
+              "NTLMSSP%c"
+              "\x03%c%c%c" /* type-3, 32 bits */

-                     "%c%c%c%c" /* LanManager length + allocated space */
-                     "%c%c" /* LanManager offset */
-                     "%c%c" /* 2 zeroes */
+              "%c%c%c%c" /* LanManager length + allocated space */
+              "%c%c" /* LanManager offset */
+              "%c%c" /* 2 zeroes */

-                     "%c%c" /* NT-response length */
-                     "%c%c" /* NT-response allocated space */
-                     "%c%c" /* NT-response offset */
-                     "%c%c" /* 2 zeroes */
+              "%c%c" /* NT-response length */
+              "%c%c" /* NT-response allocated space */
+              "%c%c" /* NT-response offset */
+              "%c%c" /* 2 zeroes */

-                     "%c%c"  /* domain length */
-                     "%c%c"  /* domain allocated space */
-                     "%c%c"  /* domain name offset */
-                     "%c%c"  /* 2 zeroes */
+              "%c%c" /* domain length */
+              "%c%c" /* domain allocated space */
+              "%c%c" /* domain name offset */
+              "%c%c" /* 2 zeroes */

-                     "%c%c"  /* user length */
-                     "%c%c"  /* user allocated space */
-                     "%c%c"  /* user offset */
-                     "%c%c"  /* 2 zeroes */
+              "%c%c" /* user length */
+              "%c%c" /* user allocated space */
+              "%c%c" /* user offset */
+              "%c%c" /* 2 zeroes */

-                     "%c%c"  /* host length */
-                     "%c%c"  /* host allocated space */
-                     "%c%c"  /* host offset */
-                     "%c%c%c%c%c%c"  /* 6 zeroes */
+              "%c%c" /* host length */
+              "%c%c" /* host allocated space */
+              "%c%c" /* host offset */
+              "%c%c%c%c%c%c" /* 6 zeroes */

-                     "\xff\xff"  /* message length */
-                     "%c%c"  /* 2 zeroes */
+              "\xff\xff" /* message length */
+              "%c%c" /* 2 zeroes */

-                     "\x01\x82" /* flags */
-                     "%c%c"  /* 2 zeroes */
+              "\x01\x82" /* flags */
+              "%c%c" /* 2 zeroes */

-                     /* domain string */
-                     /* user string */
-                     /* host string */
-                     /* LanManager response */
-                     /* NT response */
-                     ,
-                     0, /* zero termination */
-                     0,0,0, /* type-3 long, the 24 upper bits */
+              /* domain string */
+              /* user string */
+              /* host string */
+              /* LanManager response */
+              /* NT response */
+              ,
+              0, /* zero termination */
+              0, 0, 0, /* type-3 long, the 24 upper bits */

-                     SHORTPAIR(0x18),  /* LanManager response length, twice */
-                     SHORTPAIR(0x18),
-                     SHORTPAIR(lmrespoff),
-                     0x0, 0x0,
+              SHORTPAIR (0x18), /* LanManager response length, twice */
+              SHORTPAIR (0x18),
+              SHORTPAIR (lmrespoff),
+              0x0, 0x0,

 #ifdef USE_NTRESPONSES
-                     SHORTPAIR(0x18),  /* NT-response length, twice */
-                     SHORTPAIR(0x18),
+              SHORTPAIR (0x18), /* NT-response length, twice */
+              SHORTPAIR (0x18),
 #else
-                     0x0, 0x0,
-                     0x0, 0x0,
+              0x0, 0x0,
+              0x0, 0x0,
 #endif
-                     SHORTPAIR(ntrespoff),
-                     0x0, 0x0,
+              SHORTPAIR (ntrespoff),
+              0x0, 0x0,

-                     SHORTPAIR(domlen),
-                     SHORTPAIR(domlen),
-                     SHORTPAIR(domoff),
-                     0x0, 0x0,
+              SHORTPAIR (domlen),
+              SHORTPAIR (domlen),
+              SHORTPAIR (domoff),
+              0x0, 0x0,

-                     SHORTPAIR(userlen),
-                     SHORTPAIR(userlen),
-                     SHORTPAIR(useroff),
-                     0x0, 0x0,
+              SHORTPAIR (userlen),
+              SHORTPAIR (userlen),
+              SHORTPAIR (useroff),
+              0x0, 0x0,

-                     SHORTPAIR(hostlen),
-                     SHORTPAIR(hostlen),
-                     SHORTPAIR(hostoff),
-                     0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
+              SHORTPAIR (hostlen),
+              SHORTPAIR (hostlen),
+              SHORTPAIR (hostoff),
+              0x0, 0x0, 0x0, 0x0, 0x0, 0x0,

-                     0x0, 0x0,
+              0x0, 0x0,

-                     0x0, 0x0);
+              0x0, 0x0);

    /* size is now 64 */
    size=64;
@@ -584,7 +583,7 @@ ntlm_output (struct ntlmdata *ntlm, const char *user, const char *passwd,

    /* convert the binary blob into base64 */
    base64 = (char *) alloca (BASE64_LENGTH (size) + 1);
-    base64_encode (ntlmbuf, size, base64);
+    wget_base64_encode (ntlmbuf, size, base64);

    output = concat_strings ("NTLM ", base64, (char *) 0);

--- a/src/http-ntlm.h
+++ b/src/http-ntlm.h
@@ -1,8 +1,8 @@
 #ifndef __HTTP_NTLM_H
 #define __HTTP_NTLM_H
 /* Declarations for http_ntlm.c
-   Copyright (C) 1995, 1996, 1997, 2000, 2007, 2008, 2009, 2010, 2011
-   Free Software Foundation, Inc.
+   Copyright (C) 1995-1997, 2000, 2007-2011, 2015, 2018 Free Software
+   Foundation, Inc.
   Contributed by Daniel Stenberg.

 This file is part of GNU Wget.
--- a/src/http.c
+++ b/src/http.c
--- a/src/http.h
+++ b/src/http.h
@@ -1,6 +1,5 @@
 /* Declarations for HTTP.
-   Copyright (C) 2005, 2006, 2007, 2008, 2009, 2010, 2011 Free Software
-   Foundation, Inc.
+   Copyright (C) 2005-2011, 2015, 2018 Free Software Foundation, Inc.

 This file is part of GNU Wget.

@@ -31,12 +30,12 @@ as that of the covered work.  */
 #ifndef HTTP_H
 #define HTTP_H

-#include "wget.h"
+#include "hsts.h"

 struct url;

-uerr_t http_loop (struct url *, struct url *, char **, char **, const char *,
-                  int *, struct url *, struct iri *, struct range *);
+uerr_t http_loop (const struct url *, struct url *, char **, char **, const char *,
+                  int *, struct url *, struct iri *);
 void save_cookies (void);
 void http_cleanup (void);
 time_t http_atotm (const char *);
--- a/src/init.c
+++ b/src/init.c
@@ -1,6 +1,5 @@
 /* Reading/parsing the initialization file.
-   Copyright (C) 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004,
-   2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012, 2014 Free Software Foundation,
+   Copyright (C) 1996-2012, 2014-2015, 2018 Free Software Foundation,
   Inc.

 This file is part of GNU Wget.
@@ -69,6 +68,8 @@ as that of the covered work.  */
 #include "retr.h"               /* for output_stream */
 #include "warc.h"               /* for warc_close */
 #include "spider.h"             /* for spider_cleanup */
+#include "html-url.h"           /* for cleanup_html_url */
+#include "c-strcase.h"

 #ifdef TESTING
 #include "test.h"
@@ -90,10 +91,16 @@ CMD_DECLARE (cmd_number_inf);
 CMD_DECLARE (cmd_string);
 CMD_DECLARE (cmd_string_uppercase);
 CMD_DECLARE (cmd_file);
+CMD_DECLARE (cmd_file_once);
 CMD_DECLARE (cmd_directory);
 CMD_DECLARE (cmd_time);
 CMD_DECLARE (cmd_vector);

+CMD_DECLARE (cmd_use_askpass);
+
+#ifdef HAVE_LIBZ
+CMD_DECLARE (cmd_spec_compression);
+#endif
 CMD_DECLARE (cmd_spec_dirstruct);
 CMD_DECLARE (cmd_spec_header);
 CMD_DECLARE (cmd_spec_warc_header);
@@ -101,6 +108,7 @@ CMD_DECLARE (cmd_spec_htmlify);
 CMD_DECLARE (cmd_spec_mirror);
 CMD_DECLARE (cmd_spec_prefer_family);
 CMD_DECLARE (cmd_spec_progress);
+CMD_DECLARE (cmd_spec_progressdisp);
 CMD_DECLARE (cmd_spec_recursive);
 CMD_DECLARE (cmd_spec_regex_type);
 CMD_DECLARE (cmd_spec_restrict_file_names);
@@ -111,6 +119,7 @@ CMD_DECLARE (cmd_spec_secure_protocol);
 CMD_DECLARE (cmd_spec_timeout);
 CMD_DECLARE (cmd_spec_useragent);
 CMD_DECLARE (cmd_spec_verbose);
+CMD_DECLARE (cmd_check_cert);

 /* List of recognized commands, each consisting of name, place and
   function.  When adding a new command, simply add it to the list,
@@ -138,6 +147,9 @@ static const struct {
  { "backups",          &opt.backups,           cmd_number },
  { "base",             &opt.base_href,         cmd_string },
  { "bindaddress",      &opt.bind_address,      cmd_string },
+#ifdef HAVE_LIBCARES
+  { "binddnsaddress",   &opt.bind_dns_address,  cmd_string },
+#endif
  { "bodydata",         &opt.body_data,         cmd_string },
  { "bodyfile",         &opt.body_file,         cmd_string },
 #ifdef HAVE_SSL
@@ -148,15 +160,22 @@ static const struct {
  { "cadirectory",      &opt.ca_directory,      cmd_directory },
  { "certificate",      &opt.cert_file,         cmd_file },
  { "certificatetype",  &opt.cert_type,         cmd_cert_type },
-  { "checkcertificate", &opt.check_cert,        cmd_boolean },
+  { "checkcertificate", &opt.check_cert,        cmd_check_cert },
 #endif
  { "chooseconfig",     &opt.choose_config,     cmd_file },
+#ifdef HAVE_LIBZ
+  { "compression",      &opt.compression,       cmd_spec_compression },
+#endif
  { "connecttimeout",   &opt.connect_timeout,   cmd_time },
  { "contentdisposition", &opt.content_disposition, cmd_boolean },
  { "contentonerror",   &opt.content_on_error,  cmd_boolean },
  { "continue",         &opt.always_rest,       cmd_boolean },
+  { "convertfileonly",  &opt.convert_file_only, cmd_boolean },
  { "convertlinks",     &opt.convert_links,     cmd_boolean },
  { "cookies",          &opt.cookies,           cmd_boolean },
+#ifdef HAVE_SSL
+  { "crlfile",          &opt.crl_file,          cmd_file_once },
+#endif
  { "cutdirs",          &opt.cut_dirs,          cmd_number },
  { "debug",            &opt.debug,             cmd_boolean },
  { "defaultpage",      &opt.default_page,      cmd_string },
@@ -164,6 +183,9 @@ static const struct {
  { "dirprefix",        &opt.dir_prefix,        cmd_directory },
  { "dirstruct",        NULL,                   cmd_spec_dirstruct },
  { "dnscache",         &opt.dns_cache,         cmd_boolean },
+#ifdef HAVE_LIBCARES
+  { "dnsservers",       &opt.dns_servers,       cmd_string },
+#endif
  { "dnstimeout",       &opt.dns_timeout,       cmd_time },
  { "domains",          &opt.domains,           cmd_vector },
  { "dotbytes",         &opt.dot_bytes,         cmd_bytes },
@@ -181,12 +203,22 @@ static const struct {
  { "ftppasswd",        &opt.ftp_passwd,        cmd_string }, /* deprecated */
  { "ftppassword",      &opt.ftp_passwd,        cmd_string },
  { "ftpproxy",         &opt.ftp_proxy,         cmd_string },
+#ifdef HAVE_SSL
+  { "ftpscleardataconnection", &opt.ftps_clear_data_connection, cmd_boolean },
+  { "ftpsfallbacktoftp", &opt.ftps_fallback_to_ftp, cmd_boolean },
+  { "ftpsimplicit",     &opt.ftps_implicit,     cmd_boolean },
+  { "ftpsresumessl",    &opt.ftps_resume_ssl,   cmd_boolean },
+#endif
 #ifdef __VMS
  { "ftpstmlf",         &opt.ftp_stmlf,         cmd_boolean },
 #endif /* def __VMS */
  { "ftpuser",          &opt.ftp_user,          cmd_string },
  { "glob",             &opt.ftp_glob,          cmd_boolean },
  { "header",           NULL,                   cmd_spec_header },
+#ifdef HAVE_HSTS
+  { "hsts",             &opt.hsts,              cmd_boolean },
+  { "hstsfile",         &opt.hsts_file,         cmd_file },
+#endif
  { "htmlextension",    &opt.adjust_extension,  cmd_boolean }, /* deprecated */
  { "htmlify",          NULL,                   cmd_spec_htmlify },
  { "httpkeepalive",    &opt.http_keep_alive,   cmd_boolean },
@@ -198,6 +230,7 @@ static const struct {
 #endif
  { "httpsproxy",       &opt.https_proxy,       cmd_string },
  { "httpuser",         &opt.http_user,         cmd_string },
+  { "ifmodifiedsince",  &opt.if_modified_since, cmd_boolean },
  { "ignorecase",       &opt.ignore_case,       cmd_boolean },
  { "ignorelength",     &opt.ignore_length,     cmd_boolean },
  { "ignoretags",       &opt.ignore_tags,       cmd_vector },
@@ -207,10 +240,11 @@ static const struct {
  { "inet6only",        &opt.ipv6_only,         cmd_boolean },
 #endif
  { "input",            &opt.input_filename,    cmd_file },
-  { "iri",              &opt.enable_iri,        cmd_boolean },
-#ifdef ENABLE_THREADS
-  { "jobs",             &opt.jobs,              cmd_number },
+#ifdef HAVE_METALINK
+  { "inputmetalink",    &opt.input_metalink,    cmd_file },
 #endif
+  { "iri",              &opt.enable_iri,        cmd_boolean },
+  { "keepbadhash",      &opt.keep_badhash,      cmd_boolean },
  { "keepsessioncookies", &opt.keep_session_cookies, cmd_boolean },
  { "limitrate",        &opt.limit_rate,        cmd_bytes },
  { "loadcookies",      &opt.cookies_input,     cmd_file },
@@ -218,8 +252,9 @@ static const struct {
  { "logfile",          &opt.lfilename,         cmd_file },
  { "login",            &opt.ftp_user,          cmd_string },/* deprecated*/
  { "maxredirect",      &opt.max_redirect,      cmd_number },
-#ifdef ENABLE_METALINK
-  { "metalink",         &opt.metalink_file,     cmd_file },
+#ifdef HAVE_METALINK
+  { "metalinkindex",    &opt.metalink_index,     cmd_number_inf },
+  { "metalinkoverhttp", &opt.metalink_over_http, cmd_boolean },
 #endif
  { "method",           &opt.method,            cmd_string_uppercase },
  { "mirror",           NULL,                   cmd_spec_mirror },
@@ -234,9 +269,15 @@ static const struct {
  { "passiveftp",       &opt.ftp_pasv,          cmd_boolean },
  { "passwd",           &opt.ftp_passwd,        cmd_string },/* deprecated*/
  { "password",         &opt.passwd,            cmd_string },
+#ifdef HAVE_SSL
+  { "pinnedpubkey",     &opt.pinnedpubkey,      cmd_string },
+#endif
  { "postdata",         &opt.post_data,         cmd_string },
  { "postfile",         &opt.post_file_name,    cmd_file },
  { "preferfamily",     NULL,                   cmd_spec_prefer_family },
+#ifdef HAVE_METALINK
+  { "preferredlocation", &opt.preferred_location, cmd_string },
+#endif
  { "preservepermissions", &opt.preserve_perm,  cmd_boolean },
 #ifdef HAVE_SSL
  { "privatekey",       &opt.private_key,       cmd_file },
@@ -259,17 +300,16 @@ static const struct {
  { "referer",          &opt.referer,           cmd_string },
  { "regextype",        &opt.regex_type,        cmd_spec_regex_type },
  { "reject",           &opt.rejects,           cmd_vector },
+  { "rejectedlog",      &opt.rejected_log,      cmd_file },
  { "rejectregex",      &opt.rejectregex_s,     cmd_string },
  { "relativeonly",     &opt.relative_only,     cmd_boolean },
  { "remoteencoding",   &opt.encoding_remote,   cmd_string },
  { "removelisting",    &opt.remove_listing,    cmd_boolean },
  { "reportspeed",             &opt.report_bps, cmd_spec_report_speed},
  { "restrictfilenames", NULL,                  cmd_spec_restrict_file_names },
-#ifdef ENABLE_METALINK
-  { "retries",          &opt.n_retries,         cmd_number_inf },
-#endif
  { "retrsymlinks",     &opt.retr_symlinks,     cmd_boolean },
  { "retryconnrefused", &opt.retry_connrefused, cmd_boolean },
+  { "retryonhttperror", &opt.retry_on_http_error, cmd_string },
  { "robots",           &opt.use_robots,        cmd_boolean },
  { "savecookies",      &opt.cookies_output,    cmd_file },
  { "saveheaders",      &opt.save_headers,      cmd_boolean },
@@ -278,7 +318,7 @@ static const struct {
 #endif
  { "serverresponse",   &opt.server_response,   cmd_boolean },
  { "showalldnsentries", &opt.show_all_dns_entries, cmd_boolean },
-  { "showprogress",     &opt.show_progress,      cmd_boolean },
+  { "showprogress",     &opt.show_progress,     cmd_spec_progressdisp },
  { "spanhosts",        &opt.spanhost,          cmd_boolean },
  { "spider",           &opt.spider,            cmd_boolean },
  { "startpos",         &opt.start_pos,         cmd_bytes },
@@ -287,7 +327,8 @@ static const struct {
  { "timestamping",     &opt.timestamping,      cmd_boolean },
  { "tries",            &opt.ntry,              cmd_number_inf },
  { "trustservernames", &opt.trustservernames,  cmd_boolean },
-  { "unlink",           &opt.unlink,            cmd_boolean },
+  { "unlink",           &opt.unlink_requested,  cmd_boolean },
+  { "useaskpass" ,      &opt.use_askpass,       cmd_use_askpass },
  { "useproxy",         &opt.use_proxy,         cmd_boolean },
  { "user",             &opt.user,              cmd_string },
  { "useragent",        NULL,                   cmd_spec_useragent },
@@ -309,6 +350,9 @@ static const struct {
 #ifdef USE_WATT32
  { "wdebug",           &opt.wdebug,            cmd_boolean },
 #endif
+#ifdef ENABLE_XATTR
+  { "xattr",            &opt.enable_xattr,      cmd_boolean },
+#endif
 };

 /* Look up CMDNAME in the commands[] and return its position in the
@@ -324,7 +368,7 @@ command_by_name (const char *cmdname)
  while (lo <= hi)
    {
      int mid = (lo + hi) >> 1;
-      int cmp = strcasecmp (cmdname, commands[mid].name);
+      int cmp = c_strcasecmp (cmdname, commands[mid].name);
      if (cmp < 0)
        hi = mid - 1;
      else if (cmp > 0)
@@ -334,7 +378,7 @@ command_by_name (const char *cmdname)
    }
  return -1;
 }
-
+
 /* Reset the variables to default values.  */
 void
 defaults (void)
@@ -348,15 +392,13 @@ defaults (void)
     bit pattern will be the least of the implementors' worries.  */
  xzero (opt);

+#ifdef HAVE_METALINK
+  opt.metalink_index = -1;
+#endif
+
  opt.cookies = true;
  opt.verbose = -1;
  opt.ntry = 20;
-#ifdef ENABLE_THREADS
-  opt.jobs = 1;
-#endif
-#ifdef ENABLE_METALINK
-  opt.n_retries = 1;
-#endif
  opt.reclevel = 5;
  opt.add_hostdir = true;
  opt.netrc = true;
@@ -364,11 +406,13 @@ defaults (void)
  opt.htmlify = true;
  opt.http_keep_alive = true;
  opt.use_proxy = true;
+  opt.convert_file_only = false;
  tmp = getenv ("no_proxy");
  if (tmp)
    opt.no_proxy = sepstring (tmp);
  opt.prefer_family = prefer_none;
  opt.allow_cache = true;
+  opt.if_modified_since = true;

  opt.read_timeout = 900;
  opt.use_robots = true;
@@ -381,14 +425,40 @@ defaults (void)

  opt.dns_cache = true;
  opt.ftp_pasv = true;
+  /* 2014-09-07  Darshit Shah  <darnir@gmail.com>
+   * opt.retr_symlinks is set to true by default. Creating symbolic links on the
+   * local filesystem pose a security threat by malicious FTP Servers that
+   * server a specially crafted .listing file akin to this:
+   *
+   * lrwxrwxrwx   1 root     root           33 Dec 25  2012 JoCxl6d8rFU -> /
+   * drwxrwxr-x  15 1024     106          4096 Aug 28 02:02 JoCxl6d8rFU
+   *
+   * A .listing file in this fashion makes Wget susceptiple to a symlink attack
+   * wherein the attacker is able to create arbitrary files, directories and
+   * symbolic links on the target system and even set permissions.
+   *
+   * Hence, by default Wget attempts to retrieve the pointed-to files and does
+   * not create the symbolic links locally.
+   */
+  opt.retr_symlinks = true;

 #ifdef HAVE_SSL
-  opt.check_cert = true;
+  opt.check_cert = CHECK_CERT_ON;
+  opt.ftps_resume_ssl = true;
+  opt.ftps_fallback_to_ftp = false;
+  opt.ftps_implicit = false;
+  opt.ftps_clear_data_connection = false;
+#endif
+
+#ifdef HAVE_LIBZ
+  opt.compression = compression_auto;
 #endif

  /* The default for file name restriction defaults to the OS type. */
 #if defined(WINDOWS) || defined(MSDOS) || defined(__CYGWIN__)
  opt.restrict_files_os = restrict_windows;
+#elif defined(__VMS)
+  opt.restrict_files_os = restrict_vms;
 #else
  opt.restrict_files_os = restrict_unix;
 #endif
@@ -427,10 +497,21 @@ defaults (void)

  /* Use a negative value to mark the absence of --start-pos option */
  opt.start_pos = -1;
-  opt.show_progress = false;
+  opt.show_progress = -1;
  opt.noscroll = false;
+
+#ifdef HAVE_HSTS
+  /* HSTS is enabled by default */
+  opt.hsts = true;
+#endif
+
+#ifdef ENABLE_XATTR
+  opt.enable_xattr = true;
+#else
+  opt.enable_xattr = false;
+#endif
 }
-
+
 /* Return the user's home directory (strdup-ed), or NULL if none is
   found.  */
 char *
@@ -479,7 +560,7 @@ home_dir (void)
    }

  ret = home ? xstrdup (home) : NULL;
-  free (buf);
+  xfree (buf);

  return ret;
 }
@@ -494,10 +575,11 @@ wgetrc_env_file_name (void)
  char *env = getenv ("WGETRC");
  if (env && *env)
    {
-      if (!file_exists_p (env))
+      file_stats_t flstat;
+      if (!file_exists_p (env, &flstat))
        {
-          fprintf (stderr, _("%s: WGETRC points to %s, which doesn't exist.\n"),
-                   exec_name, env);
+          fprintf (stderr, _("%s: WGETRC points to %s, which couldn't be accessed because of error: %s.\n"),
+                   exec_name, env, strerror(flstat.access_err));
          exit (WGET_EXIT_GENERIC_ERROR);
        }
      return xstrdup (env);
@@ -505,7 +587,7 @@ wgetrc_env_file_name (void)
  return NULL;
 }

-/* Check for the existance of '$HOME/.wgetrc' and return its path
+/* Check for the existence of '$HOME/.wgetrc' and return its path
   if it exists and is set.  */
 char *
 wgetrc_user_file_name (void)
@@ -520,12 +602,12 @@ wgetrc_user_file_name (void)
  home = home_dir ();
  if (home)
    file = aprintf ("%s/.wgetrc", home);
-  xfree_null (home);
+  xfree (home);
 #endif /* def __VMS [else] */

  if (!file)
    return NULL;
-  if (!file_exists_p (file))
+  if (!file_exists_p (file, NULL))
    {
      xfree (file);
      return NULL;
@@ -554,17 +636,13 @@ wgetrc_file_name (void)
     SYSTEM_WGETRC should not be defined under WINDOWS.  */
  if (!file)
    {
-      char *home = home_dir ();
-      xfree_null (file);
-      file = NULL;
-      home = ws_mypath ();
+      char *home = ws_mypath ();
      if (home)
        {
          file = aprintf ("%s/wget.ini", home);
-          if (!file_exists_p (file))
+          if (!file_exists_p (file, NULL))
            {
              xfree (file);
-              file = NULL;
            }
          xfree (home);
        }
@@ -590,7 +668,7 @@ static bool setval_internal_tilde (int, const char *, const char *);
   there were errors in the file.  */

 bool
-run_wgetrc (const char *file)
+run_wgetrc (const char *file, file_stats_t *flstats)
 {
  FILE *fp;
  char *line = NULL;
@@ -598,7 +676,7 @@ run_wgetrc (const char *file)
  int ln;
  int errcnt = 0;

-  fp = fopen (file, "r");
+  fp = fopen_stat (file, "r", flstats);
  if (!fp)
    {
      fprintf (stderr, _("%s: Cannot read %s (%s).\n"), exec_name,
@@ -638,8 +716,8 @@ run_wgetrc (const char *file)
        default:
          abort ();
        }
-      xfree_null (com);
-      xfree_null (val);
+      xfree (com);
+      xfree (val);
      ++ln;
    }
  xfree (line);
@@ -654,14 +732,16 @@ void
 initialize (void)
 {
  char *file, *env_sysrc;
+  file_stats_t flstats;
  bool ok = true;

+  memset(&flstats, 0, sizeof(flstats));
  /* Run a non-standard system rc file when the according environment
     variable has been set. For internal testing purposes only!  */
  env_sysrc = getenv ("SYSTEM_WGETRC");
-  if (env_sysrc && file_exists_p (env_sysrc))
+  if (env_sysrc && file_exists_p (env_sysrc, &flstats))
    {
-      ok &= run_wgetrc (env_sysrc);
+      ok &= run_wgetrc (env_sysrc, &flstats);
      /* If there are any problems parsing the system wgetrc file, tell
         the user and exit */
      if (! ok)
@@ -675,8 +755,8 @@ or specify a different file using --config.\n"), env_sysrc);
    }
  /* Otherwise, if SYSTEM_WGETRC is defined, use it.  */
 #ifdef SYSTEM_WGETRC
-  else if (file_exists_p (SYSTEM_WGETRC))
-    ok &= run_wgetrc (SYSTEM_WGETRC);
+  else if (file_exists_p (SYSTEM_WGETRC, &flstats))
+    ok &= run_wgetrc (SYSTEM_WGETRC, &flstats);
  /* If there are any problems parsing the system wgetrc file, tell
     the user and exit */
  if (! ok)
@@ -703,7 +783,8 @@ or specify a different file using --config.\n"), SYSTEM_WGETRC);
    }
  else
 #endif
-    ok &= run_wgetrc (file);
+	if (file_exists_p (file, &flstats))
+        ok &= run_wgetrc (file, &flstats);

  /* If there were errors processing either `.wgetrc', abort. */
  if (!ok)
@@ -810,6 +891,10 @@ static bool
 setval_internal (int comind, const char *com, const char *val)
 {
  assert (0 <= comind && ((size_t) comind) < countof (commands));
+
+  if ((unsigned) comind >= countof (commands))
+    return NULL;
+
  DEBUGP (("Setting %s (%s) to %s\n", com, commands[comind].name, val));
  return commands[comind].action (com, val, commands[comind].place);
 }
@@ -892,7 +977,7 @@ run_command (const char *cmdopt)
      exit (WGET_EXIT_PARSE_ERROR);
    }
 }
-
+
 /* Generic helper functions, for use with `commands'. */

 /* Forward declarations: */
@@ -916,6 +1001,18 @@ static bool simple_atof (const char *, const char *, double *);
                     && (p)[3] == '\0')


+static int
+cmd_boolean_internal (const char *com _GL_UNUSED, const char *val, void *place _GL_UNUSED)
+{
+  if (CMP2 (val, 'o', 'n') || CMP3 (val, 'y', 'e', 's') || CMP1 (val, '1'))
+    /* "on", "yes" and "1" mean true. */
+    return 1;
+  else if (CMP3 (val, 'o', 'f', 'f') || CMP2 (val, 'n', 'o') || CMP1 (val, '0'))
+    /* "off", "no" and "0" mean false. */
+    return 0;
+  return -1;
+}
+
 /* Store the boolean value from VAL to PLACE.  COM is ignored,
   except for error messages.  */
 static bool
@@ -923,24 +1020,62 @@ cmd_boolean (const char *com, const char *val, void *place)
 {
  bool value;

-  if (CMP2 (val, 'o', 'n') || CMP3 (val, 'y', 'e', 's') || CMP1 (val, '1'))
-    /* "on", "yes" and "1" mean true. */
-    value = true;
-  else if (CMP3 (val, 'o', 'f', 'f') || CMP2 (val, 'n', 'o') || CMP1 (val, '0'))
-    /* "off", "no" and "0" mean false. */
-    value = false;
-  else
+  switch (cmd_boolean_internal (com, val, place))
    {
-      fprintf (stderr,
-               _("%s: %s: Invalid boolean %s; use `on' or `off'.\n"),
-               exec_name, com, quote (val));
-      return false;
-    }
+    case 0:
+      value = false;
+      break;

+    case 1:
+      value = true;
+      break;
+
+    default:
+      {
+        fprintf (stderr,
+                 _("%s: %s: Invalid boolean %s; use `on' or `off'.\n"),
+                 exec_name, com, quote (val));
+        return false;
+      }
+    }
  *(bool *) place = value;
  return true;
 }

+/* Store the check_cert value from VAL to PLACE.  COM is ignored,
+   except for error messages.  */
+static bool
+cmd_check_cert (const char *com, const char *val, void *place)
+{
+  int value;
+
+  switch (cmd_boolean_internal (com, val, place))
+    {
+    case 0:
+      value = CHECK_CERT_OFF;
+      break;
+
+    case 1:
+      value = CHECK_CERT_ON;
+      break;
+
+    default:
+      {
+        if (!c_strcasecmp (val, "quiet"))
+          value = CHECK_CERT_QUIET;
+        else
+          {
+            fprintf (stderr,
+                     _("%s: %s: Invalid %s; use `on', `off' or `quiet'.\n"),
+                     exec_name, com, quote (val));
+            return false;
+          }
+      }
+    }
+  *(int *) place = value;
+  return true;
+}
+
 /* Set the non-negative integer value from VAL to PLACE.  With
   incorrect specification, the number remains unchanged.  */
 static bool
@@ -960,7 +1095,7 @@ cmd_number (const char *com, const char *val, void *place)
 static bool
 cmd_number_inf (const char *com, const char *val, void *place)
 {
-  if (!strcasecmp (val, "inf"))
+  if (!c_strcasecmp (val, "inf"))
    {
      *(int *) place = 0;
      return true;
@@ -975,7 +1110,7 @@ cmd_string (const char *com _GL_UNUSED, const char *val, void *place)
 {
  char **pstring = (char **)place;

-  xfree_null (*pstring);
+  xfree (*pstring);
  *pstring = xstrdup (val);
  return true;
 }
@@ -986,7 +1121,7 @@ cmd_string_uppercase (const char *com _GL_UNUSED, const char *val, void *place)
 {
  char *q, **pstring;
  pstring = (char **)place;
-  xfree_null (*pstring);
+  xfree (*pstring);

  *pstring = xmalloc (strlen (val) + 1);

@@ -1006,7 +1141,7 @@ cmd_file (const char *com _GL_UNUSED, const char *val, void *place)
 {
  char **pstring = (char **)place;

-  xfree_null (*pstring);
+  xfree (*pstring);

  /* #### If VAL is empty, perhaps should set *PLACE to NULL.  */

@@ -1024,6 +1159,20 @@ cmd_file (const char *com _GL_UNUSED, const char *val, void *place)
  return true;
 }

+/* like cmd_file, but insist on just a single option usage */
+static bool
+cmd_file_once (const char *com _GL_UNUSED, const char *val, void *place)
+{
+  if (*(char **)place)
+    {
+      fprintf (stderr, _("%s: %s must only be used once\n"),
+               exec_name, com);
+      return false;
+    }
+
+  return cmd_file(com, val, place);
+}
+
 /* Like cmd_file, but strips trailing '/' characters.  */
 static bool
 cmd_directory (const char *com, const char *val, void *place)
@@ -1258,6 +1407,32 @@ cmd_time (const char *com, const char *val, void *place)
  return true;
 }

+
+static bool
+cmd_use_askpass (const char *com _GL_UNUSED, const char *val, void *place)
+{
+  const char *env_name = "WGET_ASKPASS";
+  const char *env;
+
+  if (val && *val)
+    return cmd_string (com, val, place);
+
+  env = getenv (env_name);
+  if (!(env && *env))
+    {
+      env_name = "SSH_ASKPASS";
+      env = getenv (env_name);
+    }
+
+  if (!(env && *env))
+    {
+      fprintf (stderr, _("use-askpass requires a string or either environment variable WGET_ASKPASS or SSH_ASKPASS to be set.\n"));
+      exit (WGET_EXIT_GENERIC_ERROR);
+    }
+
+  return cmd_string (com, env, place);
+}
+
 #ifdef HAVE_SSL
 static bool
 cmd_cert_type (const char *com, const char *val, void *place)
@@ -1273,12 +1448,31 @@ cmd_cert_type (const char *com, const char *val, void *place)
  return ok;
 }
 #endif
-
+
 /* Specialized helper functions, used by `commands' to handle some
   options specially.  */

 static bool check_user_specified_header (const char *);

+#ifdef HAVE_LIBZ
+static bool
+cmd_spec_compression (const char *com, const char *val, void *place)
+{
+  static const struct decode_item choices[] = {
+    { "auto", compression_auto },
+    { "gzip", compression_gzip },
+    { "none", compression_none },
+  };
+  int ok = decode_string (val, choices, countof (choices), place);
+  if (!ok)
+    {
+      fprintf (stderr, _("%s: %s: Invalid value %s.\n"), exec_name, com,
+               quote (val));
+    }
+  return ok;
+}
+#endif
+
 static bool
 cmd_spec_dirstruct (const char *com, const char *val, void *place_ignored _GL_UNUSED)
 {
@@ -1397,7 +1591,7 @@ cmd_spec_progress (const char *com, const char *val, void *place_ignored _GL_UNU
               exec_name, com, quote (val));
      return false;
    }
-  xfree_null (opt.progress_type);
+  xfree (opt.progress_type);

  /* Don't call set_progress_implementation here.  It will be called
     in main when it becomes clear what the log output is.  */
@@ -1461,6 +1655,8 @@ cmd_spec_restrict_file_names (const char *com, const char *val, void *place_igno

      if (VAL_IS ("unix"))
        restrict_os = restrict_unix;
+      else if (VAL_IS ("vms"))
+        restrict_os = restrict_vms;
      else if (VAL_IS ("windows"))
        restrict_os = restrict_windows;
      else if (VAL_IS ("lowercase"))
@@ -1475,7 +1671,7 @@ cmd_spec_restrict_file_names (const char *com, const char *val, void *place_igno
        {
          fprintf (stderr, _("\
 %s: %s: Invalid restriction %s,\n\
-    use [unix|windows],[lowercase|uppercase],[nocontrol],[ascii].\n"),
+    use [unix|vms|windows],[lowercase|uppercase],[nocontrol],[ascii].\n"),
                   exec_name, com, quote (val));
          return false;
        }
@@ -1498,7 +1694,7 @@ cmd_spec_restrict_file_names (const char *com, const char *val, void *place_igno
 static bool
 cmd_spec_report_speed (const char *com, const char *val, void *place_ignored _GL_UNUSED)
 {
-  opt.report_bps = strcasecmp (val, "bits") == 0;
+  opt.report_bps = c_strcasecmp (val, "bits") == 0;
  if (!opt.report_bps)
    fprintf (stderr, _("%s: %s: Invalid value %s.\n"), exec_name, com, quote (val));
  return opt.report_bps;
@@ -1513,6 +1709,8 @@ cmd_spec_secure_protocol (const char *com, const char *val, void *place)
    { "sslv2", secure_protocol_sslv2 },
    { "sslv3", secure_protocol_sslv3 },
    { "tlsv1", secure_protocol_tlsv1 },
+    { "tlsv1_1", secure_protocol_tlsv1_1 },
+    { "tlsv1_2", secure_protocol_tlsv1_2 },
    { "pfs", secure_protocol_pfs },
  };
  int ok = decode_string (val, choices, countof (choices), place);
@@ -1546,11 +1744,27 @@ cmd_spec_useragent (const char *com, const char *val, void *place_ignored _GL_UN
               exec_name, com, quote (val));
      return false;
    }
-  xfree_null (opt.useragent);
+  xfree (opt.useragent);
  opt.useragent = xstrdup (val);
  return true;
 }

+/* The --show-progress option is not a cmd_boolean since we need to keep track
+ * of whether the user explicitly requested the option or not. -1 means
+ * uninitialized. */
+static bool
+cmd_spec_progressdisp (const char *com, const char *val, void *place _GL_UNUSED)
+{
+  bool flag;
+  if (cmd_boolean (com, val, &flag))
+    {
+      opt.show_progress = flag;
+      return true;
+    }
+  return false;
+}
+
+
 /* The "verbose" option cannot be cmd_boolean because the variable is
   not bool -- it's of type int (-1 means uninitialized because of
   some random hackery for disallowing -q -v).  */
@@ -1562,11 +1776,12 @@ cmd_spec_verbose (const char *com, const char *val, void *place_ignored _GL_UNUS
  if (cmd_boolean (com, val, &flag))
    {
      opt.verbose = flag;
+      opt.show_progress = -1;
      return true;
    }
  return false;
 }
-
+
 /* Miscellaneous useful routines.  */

 /* A very simple atoi clone, more useful than atoi because it works on
@@ -1701,7 +1916,7 @@ decode_string (const char *val, const struct decode_item *items, int itemcount,
 {
  int i;
  for (i = 0; i < itemcount; i++)
-    if (0 == strcasecmp (val, items[i].name))
+    if (0 == c_strcasecmp (val, items[i].name))
      {
        *place = items[i].code;
        return true;
@@ -1744,55 +1959,76 @@ cleanup (void)
  spider_cleanup ();
  host_cleanup ();
  log_cleanup ();
-  netrc_cleanup (netrc_list);
+  netrc_cleanup ();

-  for (i = 0; i < nurl; i++)
-    xfree (url[i]);
-
-  xfree_null (opt.choose_config);
-  xfree_null (opt.lfilename);
-  xfree_null (opt.dir_prefix);
-  xfree_null (opt.input_filename);
-  xfree_null (opt.output_document);
+  xfree (opt.choose_config);
+  xfree (opt.lfilename);
+  xfree (opt.dir_prefix);
+  xfree (opt.input_filename);
+#ifdef HAVE_METALINK
+  xfree (opt.input_metalink);
+  xfree (opt.preferred_location);
+#endif
+  xfree (opt.output_document);
  free_vec (opt.accepts);
  free_vec (opt.rejects);
-  free_vec (opt.excludes);
-  free_vec (opt.includes);
+  free_vec ((char **)opt.excludes);
+  free_vec ((char **)opt.includes);
  free_vec (opt.domains);
  free_vec (opt.follow_tags);
  free_vec (opt.ignore_tags);
-  xfree_null (opt.progress_type);
-  xfree_null (opt.ftp_user);
-  xfree_null (opt.ftp_passwd);
-  xfree_null (opt.ftp_proxy);
-  xfree_null (opt.https_proxy);
-  xfree_null (opt.http_proxy);
+  xfree (opt.progress_type);
+  xfree (opt.ftp_user);
+  xfree (opt.ftp_passwd);
+  xfree (opt.ftp_proxy);
+  xfree (opt.https_proxy);
+  xfree (opt.http_proxy);
  free_vec (opt.no_proxy);
-  xfree_null (opt.useragent);
-  xfree_null (opt.referer);
-  xfree_null (opt.http_user);
-  xfree_null (opt.http_passwd);
+  xfree (opt.useragent);
+  xfree (opt.referer);
+  xfree (opt.http_user);
+  xfree (opt.http_passwd);
  free_vec (opt.user_headers);
  free_vec (opt.warc_user_headers);
 # ifdef HAVE_SSL
-  xfree_null (opt.cert_file);
-  xfree_null (opt.private_key);
-  xfree_null (opt.ca_directory);
-  xfree_null (opt.ca_cert);
-  xfree_null (opt.random_file);
-  xfree_null (opt.egd_file);
+  xfree (opt.cert_file);
+  xfree (opt.private_key);
+  xfree (opt.ca_directory);
+  xfree (opt.ca_cert);
+  xfree (opt.crl_file);
+  xfree (opt.random_file);
+  xfree (opt.egd_file);
 # endif
-  xfree_null (opt.bind_address);
-  xfree_null (opt.cookies_input);
-  xfree_null (opt.cookies_output);
-  xfree_null (opt.user);
-  xfree_null (opt.passwd);
-  xfree_null (opt.base_href);
-  xfree_null (opt.method);
+  xfree (opt.bind_address);
+  xfree (opt.cookies_input);
+  xfree (opt.cookies_output);
+  xfree (opt.user);
+  xfree (opt.passwd);
+  xfree (opt.base_href);
+  xfree (opt.method);
+  xfree (opt.post_file_name);
+  xfree (opt.post_data);
+  xfree (opt.body_data);
+  xfree (opt.body_file);
+  xfree (opt.rejected_log);
+  xfree (opt.use_askpass);
+  xfree (opt.retry_on_http_error);
+
+#ifdef HAVE_LIBCARES
+#include <ares.h>
+  {
+    extern ares_channel ares;
+
+    xfree (opt.bind_dns_address);
+    xfree (opt.dns_servers);
+    ares_destroy (ares);
+    ares_library_cleanup ();
+  }
+#endif

 #endif /* DEBUG_MALLOC */
 }
-
+
 /* Unit testing routines.  */

 #ifdef TESTING
@@ -1804,7 +2040,7 @@ test_commands_sorted(void)

  for (i = 1; i < countof(commands); ++i)
    {
-      if (strcasecmp (commands[i - 1].name, commands[i].name) > 0)
+      if (c_strcasecmp (commands[i - 1].name, commands[i].name) > 0)
        {
          mu_assert ("FAILED", false);
          break;
@@ -1820,7 +2056,7 @@ test_cmd_spec_restrict_file_names(void)
  static const struct {
    const char *val;
    int expected_restrict_files_os;
-    int expected_restrict_files_ctrl;
+    bool expected_restrict_files_ctrl;
    int expected_restrict_files_case;
    bool result;
  } test_array[] = {
@@ -1845,9 +2081,9 @@ test_cmd_spec_restrict_file_names(void)
      */
      mu_assert ("test_cmd_spec_restrict_file_names: wrong result",
                 res == test_array[i].result
-                 && opt.restrict_files_os   == test_array[i].expected_restrict_files_os
+                 && (int) opt.restrict_files_os   == test_array[i].expected_restrict_files_os
                 && opt.restrict_files_ctrl == test_array[i].expected_restrict_files_ctrl
-                 && opt.restrict_files_case == test_array[i].expected_restrict_files_case);
+                 && (int) opt.restrict_files_case == test_array[i].expected_restrict_files_case);
    }

  return NULL;
--- a/src/init.h
+++ b/src/init.h
@@ -1,7 +1,5 @@
 /* Declarations for init.c.
-   Copyright (C) 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004,
-   2005, 2006, 2007, 2008, 2009, 2010, 2011 Free Software Foundation,
-   Inc.
+   Copyright (C) 1996-2011, 2015, 2018 Free Software Foundation, Inc.

 This file is part of GNU Wget.

@@ -41,6 +39,6 @@ void setoptval (const char *, const char *, const char *);
 char *home_dir (void);
 void cleanup (void);
 void defaults (void);
-bool run_wgetrc (const char *file);
+bool run_wgetrc (const char *file, file_stats_t *);

 #endif /* INIT_H */
--- a/src/iri.c
+++ b/src/iri.c
@@ -1,5 +1,5 @@
 /* IRI related functions.
-   Copyright (C) 2008, 2009, 2010, 2011 Free Software Foundation, Inc.
+   Copyright (C) 2008-2011, 2015, 2018 Free Software Foundation, Inc.

 This file is part of GNU Wget.

@@ -32,44 +32,49 @@ as that of the covered work.  */
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
-#include <iconv.h>
-#include <stringprep.h>
-#include <idna.h>
+#include <langinfo.h>
 #include <errno.h>
+#ifdef HAVE_ICONV
+# include <iconv.h>
+#endif
+#include <idn2.h>
+#if IDN2_VERSION_NUMBER < 0x00140000
+# include <unicase.h>
+# include <unistr.h>
+#endif

 #include "utils.h"
-
-/* RFC3987 section 3.1 mandates STD3 ASCII RULES */
-#define IDNA_FLAGS  IDNA_USE_STD3_ASCII_RULES
+#include "url.h"
+#include "c-strcase.h"
+#include "c-strcasestr.h"
+#include "xstrndup.h"

 /* Note: locale encoding is kept in options struct (opt.locale) */

-static bool do_conversion (iconv_t cd, char *in, size_t inlen, char **out);
-
-
 /* Given a string containing "charset=XXX", return the encoding if found,
   or NULL otherwise */
 char *
-parse_charset (char *str)
+parse_charset (const char *str)
 {
+  const char *end;
  char *charset;

  if (!str || !*str)
    return NULL;

-  str = strcasestr (str, "charset=");
+  str = c_strcasestr (str, "charset=");
  if (!str)
    return NULL;

  str += 8;
-  charset = str;
+  end = str;

  /* sXXXav: which chars should be banned ??? */
-  while (*charset && !c_isspace (*charset))
-    charset++;
+  while (*end && !c_isspace (*end))
+    end++;

  /* sXXXav: could strdupdelim return NULL ? */
-  charset = strdupdelim (str, charset);
+  charset = strdupdelim (str, end);

  /* Do a minimum check on the charset value */
  if (!check_encoding_name (charset))
@@ -84,17 +89,22 @@ parse_charset (char *str)
 }

 /* Find the locale used, or fall back on a default value */
-char *
+const char *
 find_locale (void)
 {
-  return (char *) stringprep_locale_charset ();
+	const char *encoding = nl_langinfo(CODESET);
+
+	if (!encoding || !*encoding)
+		return "ASCII";
+
+   return encoding;
 }

 /* Basic check of an encoding name. */
 bool
-check_encoding_name (char *encoding)
+check_encoding_name (const char *encoding)
 {
-  char *s = encoding;
+  const char *s = encoding;

  while (*s)
    {
@@ -110,60 +120,54 @@ check_encoding_name (char *encoding)
  return true;
 }

-/* Try converting string str from locale to UTF-8. Return a new string
-   on success, or str on error or if conversion isn't needed. */
-const char *
-locale_to_utf8 (const char *str)
-{
-  iconv_t l2u;
-  char *new;
-
-  /* That shouldn't happen, just in case */
-  if (!opt.locale)
-    {
-      logprintf (LOG_VERBOSE, _("locale_to_utf8: locale is unset\n"));
-      opt.locale = find_locale ();
-    }
-
-  if (!opt.locale || !strcasecmp (opt.locale, "utf-8"))
-    return str;
-
-  l2u = iconv_open ("UTF-8", opt.locale);
-  if (l2u == (iconv_t)(-1))
-    {
-      logprintf (LOG_VERBOSE, _("Conversion from %s to %s isn't supported\n"),
-                 quote (opt.locale), quote ("UTF-8"));
-      return str;
-    }
-
-  if (do_conversion (l2u, (char *) str, strlen ((char *) str), &new))
-    return (const char *) new;
-
-  return str;
-}
-
+#ifdef HAVE_ICONV
 /* Do the conversion according to the passed conversion descriptor cd. *out
   will contain the transcoded string on success. *out content is
   unspecified otherwise. */
 static bool
-do_conversion (iconv_t cd, char *in, size_t inlen, char **out)
+do_conversion (const char *tocode, const char *fromcode, char const *in_org, size_t inlen, char **out)
 {
+  iconv_t cd;
  /* sXXXav : hummm hard to guess... */
-  size_t len, done, outlen = inlen * 2;
+  size_t len, done, outlen;
  int invalid = 0, tooshort = 0;
-  char *s;
+  char *s, *in, *in_save;

-  s = xmalloc (outlen + 1);
-  *out = s;
-  len = outlen;
+  cd = iconv_open (tocode, fromcode);
+  if (cd == (iconv_t)(-1))
+    {
+      logprintf (LOG_VERBOSE, _("Conversion from %s to %s isn't supported\n"),
+                 quote (fromcode), quote (tocode));
+      *out = NULL;
+      return false;
+    }
+
+  /* iconv() has to work on an unescaped string */
+  in_save = in = xstrndup (in_org, inlen);
+  url_unescape_except_reserved (in);
+  inlen = strlen(in);
+
+  len = outlen = inlen * 2;
+  *out = s = xmalloc (outlen + 1);
  done = 0;

  for (;;)
    {
-      if (iconv (cd, &in, &inlen, out, &outlen) != (size_t)(-1))
+      if (iconv (cd, (ICONV_CONST char **) &in, &inlen, out, &outlen) != (size_t)(-1) &&
+          iconv (cd, NULL, NULL, out, &outlen) != (size_t)(-1))
        {
          *out = s;
          *(s + len - outlen - done) = '\0';
+          xfree(in_save);
+          iconv_close(cd);
+          IF_DEBUG
+          {
+            /* not not print out embedded passwords, in_org might be an URL */
+            if (!strchr(in_org, '@') && !strchr(*out, '@'))
+              debug_logprintf ("converted '%s' (%s) -> '%s' (%s)\n", in_org, fromcode, *out, tocode);
+            else
+              debug_logprintf ("logging suppressed, strings may contain password\n");
+          }
          return true;
        }

@@ -183,16 +187,10 @@ do_conversion (iconv_t cd, char *in, size_t inlen, char **out)
        }
      else if (errno == E2BIG) /* Output buffer full */
        {
-          char *new;
-
          tooshort++;
          done = len;
-          outlen = done + inlen * 2;
-          new = xmalloc (outlen + 1);
-          memcpy (new, s, done);
-          xfree (s);
-          s = new;
-          len = outlen;
+          len = outlen = done + inlen * 2;
+          s = xrealloc (s, outlen + 1);
          *out = s + done;
        }
      else /* Weird, we got an unspecified error */
@@ -202,63 +200,148 @@ do_conversion (iconv_t cd, char *in, size_t inlen, char **out)
        }
    }

+    xfree(in_save);
+    iconv_close(cd);
+    IF_DEBUG
+    {
+      /* not not print out embedded passwords, in_org might be an URL */
+      if (!strchr(in_org, '@') && !strchr(*out, '@'))
+        debug_logprintf ("converted '%s' (%s) -> '%s' (%s)\n", in_org, fromcode, *out, tocode);
+      else
+        debug_logprintf ("logging suppressed, strings may contain password\n");
+    }
    return false;
 }
+#else
+static bool
+do_conversion (const char *tocode _GL_UNUSED, const char *fromcode _GL_UNUSED,
+               char const *in_org _GL_UNUSED, size_t inlen _GL_UNUSED, char **out)
+{
+  *out = NULL;
+  return false;
+}
+#endif
+
+/* Try converting string str from locale to UTF-8. Return a new string
+   on success, or str on error or if conversion isn't needed. */
+const char *
+locale_to_utf8 (const char *str)
+{
+  char *new;
+
+  /* That shouldn't happen, just in case */
+  if (!opt.locale)
+    {
+      logprintf (LOG_VERBOSE, _("locale_to_utf8: locale is unset\n"));
+      opt.locale = find_locale ();
+    }
+
+  if (!opt.locale || !c_strcasecmp (opt.locale, "utf-8"))
+    return str;
+
+  if (do_conversion ("UTF-8", opt.locale, (char *) str, strlen ((char *) str), &new))
+    return (const char *) new;
+
+  xfree (new);
+  return str;
+}

 /* Try to "ASCII encode" UTF-8 host. Return the new domain on success or NULL
   on error. */
 char *
-idn_encode (struct iri *i, char *host)
+idn_encode (const struct iri *i, const char *host)
 {
-  char *new;
  int ret;
+  char *ascii_encoded;
+  char *utf8_encoded = NULL;
+  const char *src;
+#if IDN2_VERSION_NUMBER < 0x00140000
+  uint8_t *lower;
+  size_t len = 0;
+#endif

  /* Encode to UTF-8 if not done */
  if (!i->utf8_encode)
    {
-      if (!remote_to_utf8 (i, (const char *) host, (const char **) &new))
-          return NULL;  /* Nothing to encode or an error occured */
-      host = new;
+      if (!remote_to_utf8 (i, host, &utf8_encoded))
+          return NULL;  /* Nothing to encode or an error occurred */
+      src = utf8_encoded;
    }
+  else
+    src = host;

-  /* toASCII UTF-8 NULL terminated string */
-  ret = idna_to_ascii_8z (host, &new, IDNA_FLAGS);
-  if (ret != IDNA_SUCCESS)
+#if IDN2_VERSION_NUMBER >= 0x00140000
+  /* IDN2_TRANSITIONAL implies input NFC encoding */
+  ret = idn2_lookup_u8 ((uint8_t *) src, (uint8_t **) &ascii_encoded, IDN2_NONTRANSITIONAL);
+  if (ret != IDN2_OK)
+    /* fall back to TR46 Transitional mode, max IDNA2003 compatibility */
+    ret = idn2_lookup_u8 ((uint8_t *) src, (uint8_t **) &ascii_encoded, IDN2_TRANSITIONAL);
+
+  if (ret != IDN2_OK)
+    logprintf (LOG_VERBOSE, _("idn_encode failed (%d): %s\n"), ret,
+               quote (idn2_strerror (ret)));
+#else
+  /* we need a conversion to lowercase */
+  lower = u8_tolower ((uint8_t *) src, u8_strlen ((uint8_t *) src) + 1, 0, UNINORM_NFKC, NULL, &len);
+  if (!lower)
    {
-      /* sXXXav : free new when needed ! */
-      logprintf (LOG_VERBOSE, _("idn_encode failed (%d): %s\n"), ret,
-                 quote (idna_strerror (ret)));
+      logprintf (LOG_VERBOSE, _("Failed to convert to lower: %d: %s\n"),
+                 errno, quote (src));
+      xfree (utf8_encoded);
      return NULL;
    }

-  return new;
+  if ((ret = idn2_lookup_u8 (lower, (uint8_t **) &ascii_encoded, IDN2_NFC_INPUT)) != IDN2_OK)
+    {
+      logprintf (LOG_VERBOSE, _("idn_encode failed (%d): %s\n"), ret,
+                 quote (idn2_strerror (ret)));
+    }
+
+  xfree (lower);
+#endif
+
+  xfree (utf8_encoded);
+
+  if (ret == IDN2_OK && ascii_encoded)
+    {
+      char *tmp = xstrdup (ascii_encoded);
+      idn2_free (ascii_encoded);
+      ascii_encoded = tmp;
+    }
+
+  return ret == IDN2_OK ? ascii_encoded : NULL;
 }

 /* Try to decode an "ASCII encoded" host. Return the new domain in the locale
   on success or NULL on error. */
 char *
-idn_decode (char *host)
+idn_decode (const char *host)
 {
+/*
  char *new;
  int ret;

-  ret = idna_to_unicode_8zlz (host, &new, IDNA_FLAGS);
-  if (ret != IDNA_SUCCESS)
+  ret = idn2_register_u8 (NULL, host, (uint8_t **) &new, 0);
+  if (ret != IDN2_OK)
    {
-      logprintf (LOG_VERBOSE, _("idn_decode failed (%d): %s\n"), ret,
-                 quote (idna_strerror (ret)));
+      logprintf (LOG_VERBOSE, _("idn2_register_u8 failed (%d): %s: %s\n"), ret,
+                 quote (idn2_strerror (ret)), host);
      return NULL;
    }

  return new;
+*/
+  /* idn2_register_u8() just works label by label.
+   * That is pretty much overhead for just displaying the original ulabels.
+   * To keep at least the debug output format, return a cloned host. */
+  return xstrdup(host);
 }

 /* Try to transcode string str from remote encoding to UTF-8. On success, *new
   contains the transcoded string. *new content is unspecified otherwise. */
 bool
-remote_to_utf8 (struct iri *iri, const char *str, const char **new)
+remote_to_utf8 (const struct iri *iri, const char *str, char **new)
 {
-  iconv_t cd;
  bool ret = false;

  if (!iri->uri_encoding)
@@ -267,11 +350,11 @@ remote_to_utf8 (struct iri *iri, const char *str, const char **new)
  /* When `i->uri_encoding' == "UTF-8" there is nothing to convert.  But we must
     test for non-ASCII symbols for correct hostname processing in `idn_encode'
     function. */
-  if (!strcmp (iri->uri_encoding, "UTF-8"))
+  if (!c_strcasecmp (iri->uri_encoding, "UTF-8"))
    {
-      const char *p = str;
-      for (p = str; *p; p++)
-        if (*p < 0)
+      const unsigned char *p;
+      for (p = (unsigned char *) str; *p; p++)
+        if (*p > 127)
          {
            *new = strdup (str);
            return true;
@@ -279,19 +362,13 @@ remote_to_utf8 (struct iri *iri, const char *str, const char **new)
      return false;
    }

-  cd = iconv_open ("UTF-8", iri->uri_encoding);
-  if (cd == (iconv_t)(-1))
-    return false;
-
-  if (do_conversion (cd, (char *) str, strlen ((char *) str), (char **) new))
+  if (do_conversion ("UTF-8", iri->uri_encoding, str, strlen (str), new))
    ret = true;

-  iconv_close (cd);
-
  /* Test if something was converted */
-  if (!strcmp (str, *new))
+  if (*new && !strcmp (str, *new))
    {
-      xfree ((char *) *new);
+      xfree (*new);
      return false;
    }

@@ -325,23 +402,26 @@ struct iri *iri_dup (const struct iri *src)
 void
 iri_free (struct iri *i)
 {
-  xfree_null (i->uri_encoding);
-  xfree_null (i->content_encoding);
-  xfree_null (i->orig_url);
-  xfree (i);
+  if (i)
+    {
+      xfree (i->uri_encoding);
+      xfree (i->content_encoding);
+      xfree (i->orig_url);
+      xfree (i);
+    }
 }

 /* Set uri_encoding of struct iri i. If a remote encoding was specified, use
   it unless force is true. */
 void
-set_uri_encoding (struct iri *i, char *charset, bool force)
+set_uri_encoding (struct iri *i, const char *charset, bool force)
 {
  DEBUGP (("URI encoding = %s\n", charset ? quote (charset) : "None"));
  if (!force && opt.encoding_remote)
    return;
  if (i->uri_encoding)
    {
-      if (charset && !strcasecmp (i->uri_encoding, charset))
+      if (charset && !c_strcasecmp (i->uri_encoding, charset))
        return;
      xfree (i->uri_encoding);
    }
@@ -351,14 +431,14 @@ set_uri_encoding (struct iri *i, char *charset, bool force)

 /* Set content_encoding of struct iri i. */
 void
-set_content_encoding (struct iri *i, char *charset)
+set_content_encoding (struct iri *i, const char *charset)
 {
  DEBUGP (("URI content encoding = %s\n", charset ? quote (charset) : "None"));
  if (opt.encoding_remote)
    return;
  if (i->content_encoding)
    {
-      if (charset && !strcasecmp (i->content_encoding, charset))
+      if (charset && !c_strcasecmp (i->content_encoding, charset))
        return;
      xfree (i->content_encoding);
    }
--- a/src/iri.h
+++ b/src/iri.h
@@ -1,5 +1,5 @@
 /* Internationalization related declarations.
-   Copyright (C) 2008, 2009, 2010, 2011 Free Software Foundation, Inc.
+   Copyright (C) 2008-2011, 2015, 2018 Free Software Foundation, Inc.

 This file is part of GNU Wget.

@@ -39,29 +39,30 @@ struct iri {

 #ifdef ENABLE_IRI

-char *parse_charset (char *str);
-char *find_locale (void);
-bool check_encoding_name (char *encoding);
+char *parse_charset (const char *str);
+const char *find_locale (void);
+bool check_encoding_name (const char *encoding);
 const char *locale_to_utf8 (const char *str);
-char *idn_encode (struct iri *i, char *host);
-char *idn_decode (char *host);
-bool remote_to_utf8 (struct iri *i, const char *str, const char **new);
+char *idn_encode (const struct iri *i, const char *host);
+char *idn_decode (const char *host);
+bool remote_to_utf8 (const struct iri *i, const char *str, char **new);
 struct iri *iri_new (void);
 struct iri *iri_dup (const struct iri *);
 void iri_free (struct iri *i);
-void set_uri_encoding (struct iri *i, char *charset, bool force);
-void set_content_encoding (struct iri *i, char *charset);
+void set_uri_encoding (struct iri *i, const char *charset, bool force);
+void set_content_encoding (struct iri *i, const char *charset);

 #else /* ENABLE_IRI */

 extern struct iri dummy_iri;

-#define parse_charset(str)          (str, NULL)
+#define parse_charset(str)          NULL
 #define find_locale()               NULL
 #define check_encoding_name(str)    false
 #define locale_to_utf8(str)         (str)
 #define idn_encode(a,b)             NULL
 #define idn_decode(str)             NULL
+#define idn2_free(str)              ((void)0)
 #define remote_to_utf8(a,b,c)       false
 #define iri_new()                   (&dummy_iri)
 #define iri_dup(a)                  (&dummy_iri)
--- a/src/log.c
+++ b/src/log.c
@@ -1,6 +1,5 @@
 /* Messages logging.
-   Copyright (C) 1998, 1999, 2000, 2001, 2002, 2003, 2004, 2005, 2006,
-   2007, 2008, 2009, 2010, 2011 Free Software Foundation, Inc.
+   Copyright (C) 1998-2011, 2015, 2018 Free Software Foundation, Inc.

 This file is part of GNU Wget.

@@ -73,13 +72,25 @@ as that of the covered work.  */
   - Inhibiting output.  When Wget receives SIGHUP, but redirecting
   the output fails, logging is inhibited.  */

-
+
 /* The file descriptor used for logging.  This is NULL before log_init
   is called; logging functions log to stderr then.  log_init sets it
   either to stderr or to a file pointer obtained from fopen().  If
   logging is inhibited, logfp is set back to NULL. */
 static FILE *logfp;

+/* Descriptor of the stdout|stderr */
+static FILE *stdlogfp;
+
+/* Descriptor of the wget.log* file (if created) */
+static FILE *filelogfp;
+
+/* Name of log file */
+static char *logfile;
+
+/* Is interactive shell ? */
+static int shell_is_interactive;
+
 /* A second file descriptor pointing to the temporary log file for the
   WARC writer.  If WARC writing is disabled, this is NULL.  */
 static FILE *warclogfp;
@@ -142,7 +153,7 @@ static int log_line_current = -1;
 static bool trailing_line;

 static void check_redirect_output (void);
-
+
 #define ROT_ADVANCE(num) do {                   \
  if (++num >= SAVED_LOG_LINES)                 \
    num = 0;                                    \
@@ -156,11 +167,7 @@ static void
 free_log_line (int num)
 {
  struct log_ln *ln = log_lines + num;
-  if (ln->malloced_line)
-    {
-      xfree (ln->malloced_line);
-      ln->malloced_line = NULL;
-    }
+  xfree (ln->malloced_line);
  ln->content = NULL;
 }

@@ -265,7 +272,7 @@ saved_append (const char *s)
      s = end;
    }
 }
-
+
 /* Check X against opt.verbose and opt.quiet.  The semantics is as
   follows:

@@ -314,6 +321,14 @@ get_log_fp (void)
  return stderr;
 }

+static FILE *
+get_progress_fp (void)
+{
+  if (opt.show_progress == true)
+      return stderr;
+  return get_log_fp();
+}
+
 /* Returns the file descriptor for the secondary log file. This is
   WARCLOGFP, except if called before log_init, in which case it
   returns stderr.  This is useful in case someone calls a logging
@@ -328,7 +343,9 @@ get_warc_log_fp (void)
    return NULL;
  if (warclogfp)
    return warclogfp;
-  return NULL;
+  if (logfp)
+    return NULL;
+  return stderr;
 }

 /* Sets the file descriptor for the secondary log file.  */
@@ -338,7 +355,7 @@ log_set_warc_log_fp (FILE * fp)
 {
  warclogfp = fp;
 }
-
+
 /* Log a literal string S.  The string is logged as-is, without a
   newline appended.  */

@@ -347,11 +364,22 @@ logputs (enum log_options o, const char *s)
 {
  FILE *fp;
  FILE *warcfp;
+  int errno_save = errno;

  check_redirect_output ();
-  if ((fp = get_log_fp ()) == NULL)
+  if (o == LOG_PROGRESS)
+    fp = get_progress_fp ();
+  else
+    fp = get_log_fp ();
+
+  errno = errno_save;
+
+  if (fp == NULL)
    return;
+
  warcfp = get_warc_log_fp ();
+  errno = errno_save;
+
  CHECK_VERBOSE (o);

  FPUTS (s, fp);
@@ -363,6 +391,8 @@ logputs (enum log_options o, const char *s)
    logflush ();
  else
    needs_flushing = true;
+
+  errno = errno_save;
 }

 struct logvprintf_state {
@@ -385,7 +415,7 @@ struct logvprintf_state {
   (An alternative approach would be to use va_copy, but that's not
   portable.)  */

-static bool
+static bool GCC_FORMAT_ATTR (2, 0)
 log_vprintf_internal (struct logvprintf_state *state, const char *fmt,
                      va_list args)
 {
@@ -451,8 +481,7 @@ log_vprintf_internal (struct logvprintf_state *state, const char *fmt,
  FPUTS (write_ptr, fp);
  if (warcfp != NULL)
    FPUTS (write_ptr, warcfp);
-  if (state->bigmsg)
-    xfree (state->bigmsg);
+  xfree (state->bigmsg);

 flush:
  if (flush_log_p)
@@ -504,7 +533,7 @@ log_set_flush (bool flush)
    }
  else
    {
-      /* Reenable flushing.  If anything was printed in no-flush mode,
+      /* Re-enable flushing.  If anything was printed in no-flush mode,
         flush the log now.  */
      if (needs_flushing)
        logflush ();
@@ -534,13 +563,16 @@ logprintf (enum log_options o, const char *fmt, ...)
  va_list args;
  struct logvprintf_state lpstate;
  bool done;
+  int errno_saved = errno;

  check_redirect_output ();
+  errno = errno_saved;
  if (inhibit_logging)
    return;
  CHECK_VERBOSE (o);

  xzero (lpstate);
+  errno = 0;
  do
    {
      va_start (args, fmt);
@@ -551,6 +583,8 @@ logprintf (enum log_options o, const char *fmt, ...)
        exit (WGET_EXIT_GENERIC_ERROR);
    }
  while (!done);
+
+  errno = errno_saved;
 }

 #ifdef ENABLE_DEBUG
@@ -580,7 +614,7 @@ debug_logprintf (const char *fmt, ...)
    }
 }
 #endif /* ENABLE_DEBUG */
-
+
 /* Open FILE and set up a logging stream.  If FILE cannot be opened,
   exit with status of 1.  */
 void
@@ -588,11 +622,20 @@ log_init (const char *file, bool appendp)
 {
  if (file)
    {
-      logfp = fopen (file, appendp ? "a" : "w");
-      if (!logfp)
+      if (HYPHENP (file))
        {
-          fprintf (stderr, "%s: %s: %s\n", exec_name, file, strerror (errno));
-          exit (WGET_EXIT_GENERIC_ERROR);
+          stdlogfp = stdout;
+          logfp = stdlogfp;
+        }
+      else
+        {
+          filelogfp = fopen (file, appendp ? "a" : "w");
+          if (!filelogfp)
+            {
+              fprintf (stderr, "%s: %s: %s\n", exec_name, file, strerror (errno));
+              exit (WGET_EXIT_GENERIC_ERROR);
+            }
+          logfp = filelogfp;
        }
    }
  else
@@ -603,7 +646,8 @@ log_init (const char *file, bool appendp)
         stderr only if the user actually specifies `-O -'.  He says
         this inconsistency is harder to document, but is overall
         easier on the user.  */
-      logfp = stderr;
+      stdlogfp = stderr;
+      logfp = stdlogfp;

      if (1
 #ifdef HAVE_ISATTY
@@ -618,6 +662,11 @@ log_init (const char *file, bool appendp)
          save_context_p = true;
        }
    }
+
+#ifndef WINDOWS
+  /* Initialize this values so we don't have to ask every time we print line */
+  shell_is_interactive = isatty (STDIN_FILENO);
+#endif
 }

 /* Close LOGFP (only if we opened it, not if it's stderr), inhibit
@@ -675,7 +724,7 @@ log_dump_context (void)
  fflush (fp);
  fflush (warcfp);
 }
-
+
 /* String escape functions. */

 /* Return the number of non-printable characters in SOURCE.
@@ -847,64 +896,85 @@ log_cleanup (void)
 {
  size_t i;
  for (i = 0; i < countof (ring); i++)
-    xfree_null (ring[i].buffer);
+    xfree (ring[i].buffer);
 }
-
+
 /* When SIGHUP or SIGUSR1 are received, the output is redirected
   elsewhere.  Such redirection is only allowed once. */
-static enum { RR_NONE, RR_REQUESTED, RR_DONE } redirect_request = RR_NONE;
 static const char *redirect_request_signal_name;

-/* Redirect output to `wget-log'.  */
+/* Redirect output to `wget-log' or back to stdout/stderr.  */

-static void
-redirect_output (void)
+void
+redirect_output (bool to_file, const char *signal_name)
 {
-  char *logfile;
-  logfp = unique_create (DEFAULT_LOGFILE, false, &logfile);
-  if (logfp)
+  if (to_file && logfp != filelogfp)
    {
-      fprintf (stderr, _("\n%s received, redirecting output to %s.\n"),
-               redirect_request_signal_name, quote (logfile));
-      xfree (logfile);
-      /* Dump the context output to the newly opened log.  */
+      if (signal_name)
+        {
+          fprintf (stderr, "\n%s received.", signal_name);
+        }
+      if (!filelogfp)
+        {
+          filelogfp = unique_create (DEFAULT_LOGFILE, false, &logfile);
+          if (filelogfp)
+            {
+              fprintf (stderr, _("\nRedirecting output to %s.\n"),
+                  quote (logfile));
+              /* Store signal name to tell wget it's permanent redirect to log file */
+              redirect_request_signal_name = signal_name;
+              logfp = filelogfp;
+              /* Dump the context output to the newly opened log.  */
+              log_dump_context ();
+            }
+          else
+            {
+              /* Eek!  Opening the alternate log file has failed.  Nothing we
+                can do but disable printing completely. */
+              fprintf (stderr, _("%s: %s; disabling logging.\n"),
+                      (logfile) ? logfile : DEFAULT_LOGFILE, strerror (errno));
+              inhibit_logging = true;
+            }
+        }
+      else
+        {
+          fprintf (stderr, _("\nRedirecting output to %s.\n"),
+              quote (logfile));
+          logfp = filelogfp;
+          log_dump_context ();
+        }
+    }
+  else if (!to_file && logfp != stdlogfp)
+    {
+      logfp = stdlogfp;
      log_dump_context ();
    }
-  else
-    {
-      /* Eek!  Opening the alternate log file has failed.  Nothing we
-         can do but disable printing completely. */
-      fprintf (stderr, _("\n%s received.\n"), redirect_request_signal_name);
-      fprintf (stderr, _("%s: %s; disabling logging.\n"),
-               (logfile) ? logfile : DEFAULT_LOGFILE, strerror (errno));
-      inhibit_logging = true;
-    }
-  save_context_p = false;
 }

-/* Check whether a signal handler requested the output to be
-   redirected. */
+/* Check whether there's a need to redirect output. */

 static void
 check_redirect_output (void)
 {
-  if (redirect_request == RR_REQUESTED)
+#ifndef WINDOWS
+  /* If it was redirected already to log file by SIGHUP, SIGUSR1 or -o parameter,
+   * it was permanent.
+   * If there was no SIGHUP or SIGUSR1 and shell is interactive
+   * we check if process is fg or bg before every line is printed.*/
+  if (!redirect_request_signal_name && shell_is_interactive && !opt.lfilename)
    {
-      redirect_request = RR_DONE;
-      redirect_output ();
+      pid_t foreground_pgrp = tcgetpgrp (STDIN_FILENO);
+
+      if (foreground_pgrp != -1 && foreground_pgrp != getpgrp ())
+        {
+          /* Process backgrounded */
+          redirect_output (true,NULL);
+        }
+      else
+        {
+          /* Process foregrounded */
+          redirect_output (false,NULL);
+        }
    }
-}
-
-/* Request redirection at a convenient time.  This may be called from
-   a signal handler. */
-
-void
-log_request_redirect_output (const char *signal_name)
-{
-  if (redirect_request == RR_NONE && save_context_p)
-    /* Request output redirection.  The request will be processed by
-       check_redirect_output(), which is called from entry point log
-       functions. */
-    redirect_request = RR_REQUESTED;
-  redirect_request_signal_name = signal_name;
+#endif /* WINDOWS */
 }
--- a/src/log.h
+++ b/src/log.h
@@ -1,6 +1,5 @@
 /* Declarations for log.c.
-   Copyright (C) 1998, 1999, 2000, 2001, 2002, 2003, 2004, 2005, 2006,
-   2007, 2008, 2009, 2010, 2011 Free Software Foundation, Inc.
+   Copyright (C) 1998-2011, 2015, 2018 Free Software Foundation, Inc.

 This file is part of GNU Wget.

@@ -52,6 +51,7 @@ void log_init (const char *, bool);
 void log_close (void);
 void log_cleanup (void);
 void log_request_redirect_output (const char *);
+void redirect_output (bool, const char *);

 const char *escnonprint (const char *);
 const char *escnonprint_uri (const char *);
--- a/src/main.c
+++ b/src/main.c
--- a/src/metalink.c
+++ b/src/metalink.c
--- a/src/metalink.h
+++ b/src/metalink.h
@@ -1,7 +1,5 @@
 /* Declarations for metalink.c.
-   Copyright (C) 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004,
-   2005, 2006, 2007, 2008, 2009, 2010, 2011 Free Software Foundation,
-   Inc.
+   Copyright (C) 2015, 2018 Free Software Foundation, Inc.

 This file is part of GNU Wget.

@@ -28,79 +26,50 @@ grants you additional permission to convey the resulting work.
 Corresponding Source for a non-source form of such a combination
 shall include the source code for the parts of OpenSSL used as well
 as that of the covered work.  */
+#if ! defined METALINK_H && defined HAVE_METALINK
+#define	METALINK_H

-#ifndef MLINK_H
-#define MLINK_H
+#include <metalink/metalink_types.h>
+#include "wget.h"

-typedef struct metalink_piece_hash
-{
-  struct metalink_piece_hash *next;
+#ifdef HAVE_SSL
+# define RES_TYPE_SUPPORTED(x)\
+    ((!x) || !strcmp (x, "http") || !strcmp (x, "https") || !strcmp (x, "ftp") || !strcmp (x, "ftps"))
+#else
+# define RES_TYPE_SUPPORTED(x)\
+    ((!x) || !strcmp (x, "ftp") || !strcmp (x, "http"))
+#endif

-  int piece;
-  char *hash;
-} mlink_piece_hash;
+#define DEFAULT_PRI 999999
+#define VALID_PRI_RANGE(x) ((x) > 0 && (x) < 1000000)

-typedef struct metalink_checksum
-{
-  struct metalink_checksum *next;
+uerr_t retrieve_from_metalink (const metalink_t *metalink);

-  char *type;
-  char *hash;
-} mlink_checksum;
+int metalink_res_cmp (const void *res1, const void *res2);
+int metalink_meta_cmp (const void* meta1, const void* meta2);

-typedef struct metalink_resource
-{
-  struct metalink_resource *next;
+int metalink_check_safe_path (const char *path);

-  char *url;
-  char *type;
-  char *location;
-  int preference;
-  int maxconnections;
-} mlink_resource;
+char *last_component (char const *name);
+void replace_metalink_basename (char **name, char *ref);
+char *get_metalink_basename (char *name);
+void append_suffix_number (char **str, const char *sep, wgint num);
+void clean_metalink_string (char **str);
+void dequote_metalink_string (char **str);
+void badhash_suffix (char *name);
+void badhash_or_remove (char *name);
+uerr_t fetch_metalink_file (const char *url_str,
+                            bool resume, bool metalink_http,
+                            const char *filename, char **destname);

-typedef struct
-{
-  char *type;
-  int length;
-  mlink_piece_hash *piece_hashes;
-} mlink_chunk_checksum;
+bool find_key_value (const char *start,
+                     const char *end,
+                     const char *key,
+                     char **value);
+bool has_key (const char *start, const char *end, const char *key);
+const char *find_key_values (const char *start,
+                             const char *end,
+                             char **key,
+                             char **value);

-typedef struct metalink_file
-{
-  struct metalink_file *next;
-
-  char *name;
-  long long int size;
-  char *version;
-  char *language;
-  char *os;
-  int maxconnections;
-  mlink_resource *resources;
-  int num_of_res;
-  mlink_checksum *checksums;
-  int num_of_checksums;
-  mlink_chunk_checksum *chunk_checksum;
-} mlink_file;
-
-typedef struct
-{
-  char *identity;
-  char *tags;
-  mlink_file *files;
-  int num_of_files;
-} mlink;
-
-mlink *parse_metalink (char *);
-
-void elect_resources (mlink *);
-
-void elect_checksums (mlink *);
-
-void delete_mlink (mlink *);
-
-metalink_t *metalink_context (const char *);
-
-int verify_file_hash (const char *, mlink_checksum *);
-
-#endif /* MLINK_H */
+#endif	/* METALINK_H */
--- a/src/mswindows.c
+++ b/src/mswindows.c
@@ -1,6 +1,5 @@
 /* mswindows.c -- Windows-specific support
-   Copyright (C) 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004,
-   2005, 2006, 2007, 2008, 2009, 2010, 2011, 2014 Free Software Foundation,
+   Copyright (C) 1996-2011, 2014-2015, 2018 Free Software Foundation,
   Inc.

 This file is part of GNU Wget.
@@ -53,16 +52,13 @@ as that of the covered work.  */
 #endif


-/* Defined in log.c.  */
-void log_request_redirect_output (const char *);
-
 /* Windows version of xsleep in utils.c.  */

 void
 xsleep (double seconds)
 {
 #if defined(HAVE_USLEEP) && defined(HAVE_SLEEP)
-  if (seconds > 1000)
+  if (seconds >= 1)
    {
      /* Explained in utils.c. */
      sleep (seconds);
@@ -85,11 +81,11 @@ windows_main (char **exec_name)
  if (p)
    *p = '\0';
 }
-
+
 static void
 ws_cleanup (void)
 {
-  xfree ((char*)exec_name);
+  xfree (exec_name);
  WSACleanup ();
 }

@@ -98,7 +94,7 @@ static void
 ws_hangup (const char *reason)
 {
  fprintf (stderr, _("Continuing in background.\n"));
-  log_request_redirect_output (reason);
+  redirect_output (true, reason);

  /* Detach process from the current console.  Under Windows 9x, if we
     were launched from a 16-bit process (which is usually the case;
@@ -367,8 +363,8 @@ static int old_percentage = -1;
 void
 ws_changetitle (const char *url)
 {
-  xfree_null (title_buf);
-  xfree_null (curr_url);
+  xfree (title_buf);
+  xfree (curr_url);
  title_buf = xmalloc (strlen (url) + 20);
  curr_url = xstrdup (url);
  old_percentage = -1;
@@ -477,7 +473,7 @@ ws_startup (void)
  set_sleep_mode ();
  SetConsoleCtrlHandler (ws_handler, TRUE);
 }
-
+
 /* run_with_timeout Windows implementation.  */

 /* Stack size 0 uses default thread stack-size (reserve+commit).
@@ -583,7 +579,7 @@ run_with_timeout (double seconds, void (*fun) (void *), void *arg)
 const char *
 inet_ntop (int af, const void *src, char *dst, socklen_t cnt)
 {
-  /* struct sockaddr can't accomodate struct sockaddr_in6. */
+  /* struct sockaddr can't accommodate struct sockaddr_in6. */
  union {
    struct sockaddr_in6 sin6;
    struct sockaddr_in sin;
--- a/src/mswindows.h
+++ b/src/mswindows.h
@@ -1,7 +1,5 @@
 /* Declarations for windows
-   Copyright (C) 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004,
-   2005, 2006, 2007, 2008, 2009, 2010, 2011 Free Software Foundation,
-   Inc.
+   Copyright (C) 1996-2011, 2015, 2018 Free Software Foundation, Inc.

 This file is part of GNU Wget.

@@ -57,6 +55,9 @@ as that of the covered work.  */
 /* Declares getpid(). */
 #include <process.h>

+/* Declares inet_ntop() and inet_pton(). */
+#include <arpa/inet.h>
+
 /* We have strcasecmp and strncasecmp, just under different names.  */
 #ifndef HAVE_STRCASECMP
 # define strcasecmp stricmp
@@ -85,11 +86,6 @@ typedef __int64 wgint;

 #define PATH_SEPARATOR '\\'

-/* Additional declarations needed for IPv6: */
-#ifdef ENABLE_IPV6
-const char *inet_ntop (int, const void *, char *, socklen_t);
-#endif
-
 /* ioctl needed by set_windows_fd_as_blocking_socket() */
 #include <sys/ioctl.h>

--- a/src/multi.c
+++ b/src/multi.c
@@ -1,239 +0,0 @@
-/* Declarations for HTTP.
-   Copyright (C) 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012, 2014 Free Software
-   Foundation, Inc.
-
-This file is part of GNU Wget.
-
-GNU Wget is free software; you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation; either version 3 of the License, or
-(at your option) any later version.
-
-GNU Wget is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License
-along with Wget.  If not, see <http://www.gnu.org/licenses/>.
-
-Additional permission under GNU GPL version 3 section 7
-
-If you modify this program, or any covered work, by linking or
-combining it with the OpenSSL project's OpenSSL library (or a
-modified version of that library), containing parts covered by the
-terms of the OpenSSL or SSLeay licenses, the Free Software Foundation
-grants you additional permission to convey the resulting work.
-Corresponding Source for a non-source form of such a combination
-shall include the source code for the parts of OpenSSL used as well
-as that of the covered work.  */
-
-#include "wget.h"
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <errno.h>
-#include <pthread.h>
-#include <semaphore.h>
-#include <unistd.h>
-
-#include "multi.h"
-#include "url.h"
-#include "exits.h"
-
-static struct range *ranges;
-char **files;
-
-/*  Allocate space for temporary file names. */
-void
-init_temp_files()
-{
-  int i;
-
-  if(!(files = malloc (opt.jobs * (sizeof *files))))
-    {
-      logprintf (LOG_VERBOSE, "Space for temporary file data could not be allocated.\n");
-      exit (WGET_EXIT_GENERIC_ERROR);
-    }
-  for (i = 0; i < opt.jobs; ++i)
-    if(!(files[i] = malloc (L_tmpnam * sizeof(char))))
-      {
-        logprintf (LOG_VERBOSE, "Space for temporary file names could not be allocated.\n");
-        exit (WGET_EXIT_GENERIC_ERROR);
-      }
-}
-
-/*  Assign names to temporary files to be used. */
-void
-name_temp_files()
-{
-  int i;
-
-  for (i = 0; i < opt.jobs; ++i)
-    if(!tmpnam(files[i]))
-      {
-        logprintf (LOG_VERBOSE, "Temporary file name could not be assigned.\n");
-        exit (WGET_EXIT_GENERIC_ERROR);
-      }
-}
-
-/*  Merge the temporary files in which the chunks are stored to form the
-    resulting file(output). */
-void
-merge_temp_files(char *output)
-{
-  FILE *out, *in;
-  int j, ret;
-  void *buf = malloc (MIN_CHUNK_SIZE);
-
-  out = fopen (output, "wb");
-  for(j = 0; j < opt.jobs; ++j)
-    {
-      in = fopen(files[j],"rb");
-      ret = MIN_CHUNK_SIZE;
-      while(ret == MIN_CHUNK_SIZE)
-        {
-          ret = fread(buf, 1, MIN_CHUNK_SIZE, in);
-          fwrite(buf, 1, ret, out);
-        }
-      fclose(in);
-    }
-  fclose(out);
-  free(buf);
-}
-
-/* Delete the temporary files used. */
-void
-delete_temp_files()
-{
-  int j = 0;
-
-  while(j < opt.jobs)
-    unlink(files[j++]);
-}
-
-/* Clean the space allocated for temporary files data. */
-void
-clean_temp_files()
-{
-  int i;
-
-  for (i = 0; i < opt.jobs; ++i)
-    free (files[i]);
-  free(files);
-}
-
-/* Allocate ranges array to store the ranges data. */
-void
-init_ranges()
-{
-  if(!(ranges = malloc (opt.jobs * (sizeof *ranges))))
-    {
-      logprintf (LOG_VERBOSE, "Space for ranges data could not be allocated.\n");
-      exit (WGET_EXIT_GENERIC_ERROR);
-    }
-}
-
-/* Assign values to the ranges.
-   Also allocates the resources array each struct range must have.
-
-   Returns the number of ranges to which values are assigned. */
-int
-fill_ranges_data(int num_of_resources, long long int file_size,
-                 long int chunk_size)
-{
-  int i, r;
-  i = 0;
-  do
-  {
-      ranges[i].first_byte = i * chunk_size;
-      ranges[i].last_byte = (i+1) * chunk_size - 1;
-      ranges[i].bytes_covered = ranges[i].is_assigned = 0;
-      ranges[i].resources = malloc(num_of_resources * sizeof(bool));
-      ranges[i].status_least_severe = RETROK;
-      for (r = 0; r < num_of_resources; ++r)
-        ranges[i].resources[r] = false;
-      ++i;
-    } while (ranges[i-1].last_byte < (file_size - 1));
-  ranges[i-1].last_byte = file_size -1;
-
-  return i;
-}
-
-/* Free the resources array of each range allocated by fill_ranges_data(). */
-void
-clean_range_res_data()
-{
-  int i;
-  for (i = 0; i < opt.jobs; ++i)
-    free (ranges[i].resources);
-}
-
-/* Free the ranges array that is used for storing ranges' data. */
-void
-clean_ranges()
-{
-  free (ranges);
-  ranges = NULL;
-}
-
-/* Assign 'last minute' data to struct s_thread_ctx instances regarding their
-   usage and range information. Then create a thread using that instance. */
-int
-spawn_thread (struct s_thread_ctx *thread_ctx, int index, int resource)
-{
-  static pthread_t thread;
-
-  thread_ctx[index].url_parsed = url_parse (thread_ctx[index].url,
-                       &(thread_ctx[index].url_err), thread_ctx[index].i, true);
-  if(!thread_ctx[index].url_parsed)
-    return 1;
-
-  thread_ctx[index].file = files[index];
-  thread_ctx[index].range = ranges + index;
-  (thread_ctx[index].range)->is_assigned = 1;
-  (thread_ctx[index].range)->resources[resource] = true;
-
-  thread_ctx[index].used = 1;
-  thread_ctx[index].terminated = 0;
-
-  return pthread_create (&thread, NULL, segmented_retrieve_url, &thread_ctx[index]);
-}
-
-/* Collects the first thread to terminate and updates struct s_thread_ctx
-   instance's data regarding its 'business' (i.e. being used by a thread).
-
-   Returns the index of the struct s_thread_ctx instance that was used in the
-   terminating thread. */
-int
-collect_thread (sem_t *retr_sem, struct s_thread_ctx *thread_ctx)
-{
-  int k, ret;
-  do
-    ret = sem_wait (retr_sem);
-  while (ret < 0 && errno == EINTR);
-
-  for (k = 0; k < opt.jobs; k++)
-    if (thread_ctx[k].used && thread_ctx[k].terminated)
-      {
-        url_free (thread_ctx[k].url_parsed);
-        thread_ctx[k].used = 0;
-        (thread_ctx[k].range)->is_assigned = 0;
-        return k;
-      }
-}
-
-/* The function which is being called by pthread_create in spawn_thread(). It
-   is used to call retrieve_url(), which requires many arguments. */
-static void *
-segmented_retrieve_url (void *arg)
-{
-  struct s_thread_ctx *ctx = (struct s_thread_ctx *) arg;
-
-  ctx->status = retrieve_url (ctx->url_parsed, ctx->url,
-                              &ctx->file, &ctx->redirected,
-                              ctx->referer, &ctx->dt,
-                              false, ctx->i, true, ctx->range);
-  ctx->terminated = 1;
-  sem_post (ctx->retr_sem);
-}
--- a/src/netrc.c
+++ b/src/netrc.c
@@ -1,6 +1,6 @@
 /* Read and parse the .netrc file to get hosts, accounts, and passwords.
-   Copyright (C) 1996, 2007, 2008, 2009, 2010, 2011 Free Software
-   Foundation, Inc.
+   Copyright (C) 1996, 2007-2011, 2015, 2018 Free Software Foundation,
+   Inc.

 This file is part of GNU Wget.

@@ -42,7 +42,11 @@ as that of the covered work.  */
 #include "netrc.h"
 #include "init.h"

-#define NETRC_FILE_NAME ".netrc"
+#ifdef WINDOWS
+#  define NETRC_FILE_NAME "_netrc"
+#else
+#  define NETRC_FILE_NAME ".netrc"
+#endif

 static acc_t *netrc_list;

@@ -75,7 +79,7 @@ search_netrc (const char *host, const char **acc, const char **passwd,
 #ifdef __VMS

      int err;
-      struct_stat buf;
+      struct stat buf;
      char *path = "SYS$LOGIN:.netrc";

      netrc_list = NULL;
@@ -94,7 +98,7 @@ search_netrc (const char *host, const char **acc, const char **passwd,
      if (home)
        {
          int err;
-          struct_stat buf;
+          struct stat buf;
          char *path = (char *)alloca (strlen (home) + 1
                                       + strlen (NETRC_FILE_NAME) + 1);
          sprintf (path, "%s/%s", home, NETRC_FILE_NAME);
@@ -163,7 +167,7 @@ search_netrc (const char *host, const char **acc, const char **passwd,

 /* Normally, these functions would be defined by your package.  */
 # define xmalloc malloc
-# define xfree free
+# define xfree(p) do { free ((void *) (p)); p = NULL; } while (0)
 # define xstrdup strdup

 # define xrealloc realloc
@@ -183,9 +187,9 @@ maybe_add_to_list (acc_t **newentry, acc_t **list)
  if (a && ! a->acc)
    {
      /* Free any allocated space.  */
-      xfree_null (a->host);
-      xfree_null (a->acc);
-      xfree_null (a->passwd);
+      xfree (a->host);
+      xfree (a->acc);
+      xfree (a->passwd);
    }
  else
    {
@@ -235,7 +239,7 @@ parse_netrc (const char *path)
  /* The latest token we've seen in the file.  */
  enum
  {
-    tok_nothing, tok_account, tok_login, tok_macdef, tok_machine, tok_password
+    tok_nothing, tok_account, tok_login, tok_macdef, tok_machine, tok_password, tok_port, tok_force
  } last_token = tok_nothing;

  current = retval = NULL;
@@ -344,6 +348,18 @@ parse_netrc (const char *path)
                premature_token = "account";
              break;

+              /* We don't handle the port keyword at all.  */
+            case tok_port:
+              if (!current)
+                premature_token = "port";
+              break;
+
+              /* We don't handle the force keyword at all.  */
+            case tok_force:
+              if (!current)
+                premature_token = "force";
+              break;
+
              /* We handle tok_nothing below this switch.  */
            case tok_nothing:
              break;
@@ -365,10 +381,10 @@ parse_netrc (const char *path)
              /* Fetch the next token.  */
              if (!strcmp (tok, "account"))
                last_token = tok_account;
+
              else if (!strcmp (tok, "default"))
-                {
                  maybe_add_to_list (&current, &retval);
-                }
+
              else if (!strcmp (tok, "login"))
                last_token = tok_login;

@@ -381,6 +397,16 @@ parse_netrc (const char *path)
              else if (!strcmp (tok, "password"))
                last_token = tok_password;

+				  /* GNU extensions 'port' and 'force', not operational
+					* see https://www.gnu.org/software/emacs/manual/html_node/gnus/NNTP.html#index-nntp_002dauthinfo_002dfunction-2003
+					* see https://savannah.gnu.org/bugs/index.php?52066
+					*/
+              else if (!strcmp (tok, "port"))
+                last_token = tok_port;
+
+              else if (!strcmp (tok, "force"))
+                last_token = tok_force;
+
              else
                fprintf (stderr, _("%s: %s:%d: unknown token \"%s\"\n"),
                         exec_name, path, ln, tok);
@@ -424,9 +450,9 @@ free_netrc(acc_t *l)
  while (l)
    {
      t = l->next;
-      xfree_null (l->acc);
-      xfree_null (l->passwd);
-      xfree_null (l->host);
+      xfree (l->acc);
+      xfree (l->passwd);
+      xfree (l->host);
      xfree (l);
      l = t;
    }
@@ -435,11 +461,14 @@ free_netrc(acc_t *l)
 #ifdef STANDALONE
 #include <sys/types.h>
 #include <sys/stat.h>
+#include "exits.h"
+
+const char *program_argstring = NULL; /* Needed by warc.c */

 int
 main (int argc, char **argv)
 {
-  struct_stat sb;
+  struct stat sb;
  char *program_name, *file, *target;
  acc_t *head, *a;

--- a/src/netrc.h
+++ b/src/netrc.h
@@ -1,6 +1,6 @@
 /* Declarations for netrc.c
-   Copyright (C) 1996, 1996, 1997, 2007, 2008, 2009, 2010, 2011 Free
-   Software Foundation, Inc.
+   Copyright (C) 1996, 1996-1997, 2007-2011, 2015, 2018 Free Software
+   Foundation, Inc.

 This file is part of GNU Wget.

--- a/src/openssl.c
+++ b/src/openssl.c
@@ -1,6 +1,5 @@
 /* SSL support via OpenSSL library.
-   Copyright (C) 2000, 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008,
-   2009, 2010, 2011, 2012 Free Software Foundation, Inc.
+   Copyright (C) 2000-2012, 2015, 2018 Free Software Foundation, Inc.
   Originally contributed by Christian Fraenkel.

 This file is part of GNU Wget.
@@ -35,11 +34,17 @@ as that of the covered work.  */
 #include <errno.h>
 #include <unistd.h>
 #include <string.h>
+#include <xalloc.h>

 #include <openssl/ssl.h>
 #include <openssl/x509v3.h>
 #include <openssl/err.h>
 #include <openssl/rand.h>
+#include <openssl/bio.h>
+#if OPENSSL_VERSION_NUMBER >= 0x00907000
+#include <openssl/conf.h>
+#include <openssl/engine.h>
+#endif

 #include "utils.h"
 #include "connect.h"
@@ -86,9 +91,11 @@ init_prng (void)
  if (RAND_status ())
    return;

+#ifdef HAVE_RAND_EGD
  /* Get random data from EGD if opt.egd_file was used.  */
  if (opt.egd_file && *opt.egd_file)
    RAND_egd (opt.egd_file);
+#endif

  if (RAND_status ())
    return;
@@ -164,16 +171,22 @@ static int ssl_true_initialized = 0;
 bool
 ssl_init (void)
 {
+  SSL_METHOD const *meth;
+  long ssl_options = 0;
+#if !defined(LIBRESSL_VERSION_NUMBER) && (OPENSSL_VERSION_NUMBER >= 0x10100000L)
+  int ssl_proto_version = 0;
+#endif
+
 #if OPENSSL_VERSION_NUMBER >= 0x00907000
  if (ssl_true_initialized == 0)
    {
+#if OPENSSL_API_COMPAT < 0x10100000L
      OPENSSL_config (NULL);
+#endif
      ssl_true_initialized = 1;
    }
 #endif

-  SSL_METHOD const *meth;
-
  if (ssl_ctx)
    /* The SSL has already been initialized. */
    return true;
@@ -187,29 +200,82 @@ ssl_init (void)
      goto error;
    }

+#if OPENSSL_VERSION_NUMBER >= 0x00907000
+  OPENSSL_load_builtin_modules();
+  ENGINE_load_builtin_engines();
+  CONF_modules_load_file(NULL, NULL,
+      CONF_MFLAGS_DEFAULT_SECTION|CONF_MFLAGS_IGNORE_MISSING_FILE);
+#endif
+#if OPENSSL_API_COMPAT >= 0x10100000L
+  OPENSSL_init_ssl(0, NULL);
+#else
  SSL_library_init ();
  SSL_load_error_strings ();
+#endif
+#if OPENSSL_VERSION_NUMBER < 0x10100000L
  SSLeay_add_all_algorithms ();
  SSLeay_add_ssl_algorithms ();
+#endif

  switch (opt.secure_protocol)
    {
-    case secure_protocol_auto:
-      meth = SSLv23_client_method ();
-      break;
-#ifndef OPENSSL_NO_SSL2
+#if !defined OPENSSL_NO_SSL2 && OPENSSL_VERSION_NUMBER < 0x10100000L
    case secure_protocol_sslv2:
      meth = SSLv2_client_method ();
      break;
 #endif
+
+#ifndef OPENSSL_NO_SSL3_METHOD
    case secure_protocol_sslv3:
      meth = SSLv3_client_method ();
      break;
+#endif
+
+    case secure_protocol_auto:
    case secure_protocol_pfs:
-    case secure_protocol_tlsv1:
-      meth = TLSv1_client_method ();
+      meth = SSLv23_client_method ();
+      ssl_options |= SSL_OP_NO_SSLv2 | SSL_OP_NO_SSLv3;
      break;
+    case secure_protocol_tlsv1:
+#if !defined(LIBRESSL_VERSION_NUMBER) && (OPENSSL_VERSION_NUMBER >= 0x10100000L)
+      meth = TLS_client_method();
+      ssl_proto_version = TLS1_VERSION;
+#else
+      meth = TLSv1_client_method ();
+#endif
+      break;
+
+#if OPENSSL_VERSION_NUMBER >= 0x10001000
+    case secure_protocol_tlsv1_1:
+#if !defined(LIBRESSL_VERSION_NUMBER) && (OPENSSL_VERSION_NUMBER >= 0x10100000L)
+      meth = TLS_client_method();
+      ssl_proto_version = TLS1_1_VERSION;
+#else
+      meth = TLSv1_1_client_method ();
+#endif
+      break;
+
+    case secure_protocol_tlsv1_2:
+#if !defined(LIBRESSL_VERSION_NUMBER) && (OPENSSL_VERSION_NUMBER >= 0x10100000L)
+      meth = TLS_client_method();
+      ssl_proto_version = TLS1_2_VERSION;
+#else
+      meth = TLSv1_2_client_method ();
+#endif
+      break;
+#else
+    case secure_protocol_tlsv1_1:
+      logprintf (LOG_NOTQUIET, _("Your OpenSSL version is too old to support TLSv1.1\n"));
+      goto error;
+
+    case secure_protocol_tlsv1_2:
+      logprintf (LOG_NOTQUIET, _("Your OpenSSL version is too old to support TLSv1.2\n"));
+      goto error;
+#endif
+
    default:
+      logprintf (LOG_NOTQUIET, _("OpenSSL: unimplemented 'secure-protocol' option value %d\n"), opt.secure_protocol);
+      logprintf (LOG_NOTQUIET, _("Please report this issue to bug-wget@gnu.org\n"));
      abort ();
    }

@@ -219,6 +285,14 @@ ssl_init (void)
  if (!ssl_ctx)
    goto error;

+  if (ssl_options)
+    SSL_CTX_set_options (ssl_ctx, ssl_options);
+
+#if !defined(LIBRESSL_VERSION_NUMBER) && (OPENSSL_VERSION_NUMBER >= 0x10100000L)
+  if (ssl_proto_version)
+    SSL_CTX_set_min_proto_version(ssl_ctx, ssl_proto_version);
+#endif
+
  /* OpenSSL ciphers: https://www.openssl.org/docs/apps/ciphers.html
   * Since we want a good protection, we also use HIGH (that excludes MD4 ciphers and some more)
   */
@@ -228,6 +302,18 @@ ssl_init (void)
  SSL_CTX_set_default_verify_paths (ssl_ctx);
  SSL_CTX_load_verify_locations (ssl_ctx, opt.ca_cert, opt.ca_directory);

+  if (opt.crl_file)
+    {
+      X509_STORE *store = SSL_CTX_get_cert_store (ssl_ctx);
+      X509_LOOKUP *lookup;
+
+      if (!(lookup = X509_STORE_add_lookup (store, X509_LOOKUP_file ()))
+          || (!X509_load_crl_file (lookup, opt.crl_file, X509_FILETYPE_PEM)))
+        goto error;
+
+      X509_STORE_set_flags (store, X509_V_FLAG_CRL_CHECK | X509_V_FLAG_CRL_CHECK_ALL);
+    }
+
  /* SSL_VERIFY_NONE instructs OpenSSL not to abort SSL_connect if the
     certificate is invalid.  We verify the certificate separately in
     ssl_check_certificate, which provides much better diagnostics
@@ -237,10 +323,17 @@ ssl_init (void)
  /* Use the private key from the cert file unless otherwise specified. */
  if (opt.cert_file && !opt.private_key)
    {
-      opt.private_key = opt.cert_file;
+      opt.private_key = xstrdup (opt.cert_file);
      opt.private_key_type = opt.cert_type;
    }

+  /* Use cert from private key file unless otherwise specified. */
+  if (opt.private_key && !opt.cert_file)
+    {
+      opt.cert_file = xstrdup (opt.private_key);
+      opt.cert_type = opt.private_key_type;
+    }
+
  if (opt.cert_file)
    if (SSL_CTX_use_certificate_file (ssl_ctx, opt.cert_file,
                                      key_type_to_ssl_type (opt.cert_type))
@@ -272,6 +365,7 @@ ssl_init (void)
 struct openssl_transport_context
 {
  SSL *conn;                    /* SSL connection handle */
+  SSL_SESSION *sess;            /* SSL session info */
  char *last_error;             /* last error printed with openssl_errstr */
 };

@@ -370,7 +464,7 @@ openssl_errstr (int fd _GL_UNUSED, void *arg)
    return NULL;

  /* Get rid of previous contents of ctx->last_error, if any.  */
-  xfree_null (ctx->last_error);
+  xfree (ctx->last_error);

  /* Iterate over OpenSSL's error stack and accumulate errors in the
     last_error buffer, separated by "; ".  This is better than using
@@ -414,7 +508,7 @@ openssl_close (int fd, void *arg)

  SSL_shutdown (conn);
  SSL_free (conn);
-  xfree_null (ctx->last_error);
+  xfree (ctx->last_error);
  xfree (ctx);

  close (fd);
@@ -443,6 +537,22 @@ ssl_connect_with_timeout_callback(void *arg)
  ctx->result = SSL_connect(ctx->ssl);
 }

+static const char *
+_sni_hostname(const char *hostname)
+{
+  size_t len = strlen(hostname);
+
+  char *sni_hostname = xmemdup(hostname, len + 1);
+
+  /* Remove trailing dot(s) to fix #47408.
+   * Regarding RFC 6066 (SNI): The hostname is represented as a byte
+   * string using ASCII encoding without a trailing dot. */
+  while (len && sni_hostname[--len] == '.')
+    sni_hostname[len] = 0;
+
+  return sni_hostname;
+}
+
 /* Perform the SSL handshake on file descriptor FD, which is assumed
   to be connected to an SSL server.  The SSL handle provided by
   OpenSSL is registered with the file descriptor FD using
@@ -452,7 +562,7 @@ ssl_connect_with_timeout_callback(void *arg)
   Returns true on success, false on failure.  */

 bool
-ssl_connect_wget (int fd, const char *hostname)
+ssl_connect_wget (int fd, const char *hostname, int *continue_session)
 {
  SSL *conn;
  struct scwt_context scwt_ctx;
@@ -465,11 +575,16 @@ ssl_connect_wget (int fd, const char *hostname)
  if (!conn)
    goto error;
 #if OPENSSL_VERSION_NUMBER >= 0x0090806fL && !defined(OPENSSL_NO_TLSEXT)
-  /* If the SSL library was build with support for ServerNameIndication
+  /* If the SSL library was built with support for ServerNameIndication
     then use it whenever we have a hostname.  If not, don't, ever. */
  if (! is_valid_ip_address (hostname))
    {
-      if (! SSL_set_tlsext_host_name (conn, hostname))
+      const char *sni_hostname = _sni_hostname(hostname);
+
+      long rc = SSL_set_tlsext_host_name (conn, sni_hostname);
+      xfree(sni_hostname);
+
+      if (rc == 0)
        {
          DEBUGP (("Failed to set TLS server-name indication."));
          goto error;
@@ -477,6 +592,14 @@ ssl_connect_wget (int fd, const char *hostname)
    }
 #endif

+  if (continue_session)
+    {
+      /* attempt to resume a previous SSL session */
+      ctx = (struct openssl_transport_context *) fd_transport_context (*continue_session);
+      if (!ctx || !ctx->sess || !SSL_set_session (conn, ctx->sess))
+        goto error;
+    }
+
 #ifndef FD_TO_SOCKET
 # define FD_TO_SOCKET(X) (X)
 #endif
@@ -490,11 +613,14 @@ ssl_connect_wget (int fd, const char *hostname)
    DEBUGP (("SSL handshake timed out.\n"));
    goto timeout;
  }
-  if (scwt_ctx.result <= 0 || conn->state != SSL_ST_OK)
+  if (scwt_ctx.result <= 0 || !SSL_is_init_finished(conn))
    goto error;

  ctx = xnew0 (struct openssl_transport_context);
  ctx->conn = conn;
+  ctx->sess = SSL_get0_session (conn);
+  if (!ctx->sess)
+    logprintf (LOG_NOTQUIET, "WARNING: Could not save SSL session data for socket %d\n", fd);

  /* Register FD with Wget's transport layer, i.e. arrange that our
     functions are used for reading, writing, and polling.  */
@@ -555,6 +681,86 @@ pattern_match (const char *pattern, const char *string)
  return *n == '\0';
 }

+static char *_get_rfc2253_formatted (X509_NAME *name)
+{
+  int len;
+  char *out = NULL;
+  BIO* b;
+
+  if ((b = BIO_new (BIO_s_mem ())))
+    {
+      if (X509_NAME_print_ex (b, name, 0, XN_FLAG_RFC2253) >= 0
+          && (len = BIO_number_written (b)) > 0)
+        {
+          out = xmalloc (len + 1);
+          BIO_read (b, out, len);
+          out[len] = 0;
+        }
+      BIO_free (b);
+    }
+
+  return out ? out : xstrdup("");
+}
+
+/*
+ * Heavily modified from:
+ * https://www.owasp.org/index.php/Certificate_and_Public_Key_Pinning#OpenSSL
+ */
+static bool
+pkp_pin_peer_pubkey (X509* cert, const char *pinnedpubkey)
+{
+  /* Scratch */
+  int len1 = 0, len2 = 0;
+  char *buff1 = NULL, *temp = NULL;
+
+  /* Result is returned to caller */
+  bool result = false;
+
+  /* if a path wasn't specified, don't pin */
+  if (!pinnedpubkey)
+    return true;
+
+  if (!cert)
+    return result;
+
+  /* Begin Gyrations to get the subjectPublicKeyInfo     */
+  /* Thanks to Viktor Dukhovni on the OpenSSL mailing list */
+
+  /* https://groups.google.com/group/mailing.openssl.users/browse_thread
+   /thread/d61858dae102c6c7 */
+  len1 = i2d_X509_PUBKEY (X509_get_X509_PUBKEY (cert), NULL);
+  if (len1 < 1)
+    goto cleanup; /* failed */
+
+  /* https://www.openssl.org/docs/crypto/buffer.html */
+  buff1 = temp = OPENSSL_malloc (len1);
+  if (!buff1)
+    goto cleanup; /* failed */
+
+  /* https://www.openssl.org/docs/crypto/d2i_X509.html */
+  len2 = i2d_X509_PUBKEY (X509_get_X509_PUBKEY (cert), (unsigned char **) &temp);
+
+  /*
+   * These checks are verifying we got back the same values as when we
+   * sized the buffer. It's pretty weak since they should always be the
+   * same. But it gives us something to test.
+   */
+  if ((len1 != len2) || !temp || ((temp - buff1) != len1))
+    goto cleanup; /* failed */
+
+  /* End Gyrations */
+
+  /* The one good exit point */
+  result = wg_pin_peer_pubkey (pinnedpubkey, buff1, len1);
+
+ cleanup:
+  /* https://www.openssl.org/docs/crypto/buffer.html */
+  if (NULL != buff1)
+    OPENSSL_free (buff1);
+
+  return result;
+}
+
 /* Verify the validity of the certificate presented by the server.
   Also check that the "common name" of the server, as presented by
   its certificate, corresponds to HOST.  (HOST typically comes from
@@ -578,6 +784,7 @@ ssl_check_certificate (int fd, const char *host)
  long vresult;
  bool success = true;
  bool alt_name_checked = false;
+  bool pinsuccess = opt.pinnedpubkey == NULL;

  /* If the user has specified --no-check-cert, we still want to warn
     him about problems with the server's certificate.  */
@@ -587,6 +794,10 @@ ssl_check_certificate (int fd, const char *host)
  SSL *conn = ctx->conn;
  assert (conn != NULL);

+  /* The user explicitly said to not check for the certificate.  */
+  if (opt.check_cert == CHECK_CERT_QUIET && pinsuccess)
+    return success;
+
  cert = SSL_get_peer_certificate (conn);
  if (!cert)
    {
@@ -598,23 +809,25 @@ ssl_check_certificate (int fd, const char *host)

  IF_DEBUG
    {
-      char *subject = X509_NAME_oneline (X509_get_subject_name (cert), 0, 0);
-      char *issuer = X509_NAME_oneline (X509_get_issuer_name (cert), 0, 0);
+      char *subject = _get_rfc2253_formatted (X509_get_subject_name (cert));
+      char *issuer = _get_rfc2253_formatted (X509_get_issuer_name (cert));
      DEBUGP (("certificate:\n  subject: %s\n  issuer:  %s\n",
               quotearg_n_style (0, escape_quoting_style, subject),
               quotearg_n_style (1, escape_quoting_style, issuer)));
-      OPENSSL_free (subject);
-      OPENSSL_free (issuer);
+      xfree (subject);
+      xfree (issuer);
    }

  vresult = SSL_get_verify_result (conn);
  if (vresult != X509_V_OK)
    {
-      char *issuer = X509_NAME_oneline (X509_get_issuer_name (cert), 0, 0);
+      char *issuer = _get_rfc2253_formatted (X509_get_issuer_name (cert));
      logprintf (LOG_NOTQUIET,
                 _("%s: cannot verify %s's certificate, issued by %s:\n"),
                 severity, quotearg_n_style (0, escape_quoting_style, host),
                 quote_n (1, issuer));
+      xfree(issuer);
+
      /* Try to print more user-friendly (and translated) messages for
         the frequent verification errors.  */
      switch (vresult)
@@ -661,9 +874,12 @@ ssl_check_certificate (int fd, const char *host)
    {
      /* Test subject alternative names */

+      /* SNI hostname must not have a trailing dot */
+      const char *sni_hostname = _sni_hostname(host);
+
      /* Do we want to check for dNSNAmes or ipAddresses (see RFC 2818)?
       * Signal it by host_in_octet_string. */
-      ASN1_OCTET_STRING *host_in_octet_string = a2i_IPADDRESS (host);
+      ASN1_OCTET_STRING *host_in_octet_string = a2i_IPADDRESS (sni_hostname);

      int numaltnames = sk_GENERAL_NAME_num (subjectAltNames);
      int i;
@@ -698,7 +914,7 @@ ssl_check_certificate (int fd, const char *host)
                  if (0 <= ASN1_STRING_to_UTF8 (&name_in_utf8, name->d.dNSName))
                    {
                      /* Compare and check for NULL attack in ASN1_STRING */
-                      if (pattern_match ((char *)name_in_utf8, host) &&
+                      if (pattern_match ((char *)name_in_utf8, sni_hostname) &&
                            (strlen ((char *)name_in_utf8) ==
                                (size_t) ASN1_STRING_length (name->d.dNSName)))
                        {
@@ -710,7 +926,7 @@ ssl_check_certificate (int fd, const char *host)
                }
            }
        }
-      sk_GENERAL_NAME_free (subjectAltNames);
+      sk_GENERAL_NAME_pop_free(subjectAltNames, GENERAL_NAME_free);
      if (host_in_octet_string)
        ASN1_OCTET_STRING_free(host_in_octet_string);

@@ -719,9 +935,11 @@ ssl_check_certificate (int fd, const char *host)
          logprintf (LOG_NOTQUIET,
              _("%s: no certificate subject alternative name matches\n"
                "\trequested host name %s.\n"),
-                     severity, quote_n (1, host));
+                     severity, quote_n (1, sni_hostname));
          success = false;
        }
+
+      xfree(sni_hostname);
    }

  if (alt_name_checked == false)
@@ -776,6 +994,13 @@ ssl_check_certificate (int fd, const char *host)
        }
    }

+    pinsuccess = pkp_pin_peer_pubkey (cert, opt.pinnedpubkey);
+    if (!pinsuccess)
+      {
+        logprintf (LOG_ALWAYS, _("The public key does not match pinned public key!\n"));
+        success = false;
+      }
+

  if (success)
    DEBUGP (("X509 certificate successfully verified and matches host %s\n",
@@ -783,13 +1008,13 @@ ssl_check_certificate (int fd, const char *host)
  X509_free (cert);

 no_cert:
-  if (opt.check_cert && !success)
+  if (opt.check_cert == CHECK_CERT_ON && !success)
    logprintf (LOG_NOTQUIET, _("\
 To connect to %s insecurely, use `--no-check-certificate'.\n"),
               quotearg_style (escape_quoting_style, host));

-  /* Allow --no-check-cert to disable certificate checking. */
-  return opt.check_cert ? success : true;
+  /* never return true if pinsuccess fails */
+  return !pinsuccess ? false : (opt.check_cert == CHECK_CERT_ON ? success : true);
 }

 /*
--- a/src/options.h
+++ b/src/options.h
@@ -1,7 +1,5 @@
 /* struct options.
-   Copyright (C) 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004,
-   2005, 2006, 2007, 2008, 2009, 2010, 2011 Free Software Foundation,
-   Inc.
+   Copyright (C) 1996-2011, 2015, 2018 Free Software Foundation, Inc.

 This file is part of GNU Wget.

@@ -29,18 +27,21 @@ Corresponding Source for a non-source form of such a combination
 shall include the source code for the parts of OpenSSL used as well
 as that of the covered work.  */

+enum CHECK_CERT_MODES
+{
+  CHECK_CERT_OFF,
+  CHECK_CERT_ON,
+  CHECK_CERT_QUIET
+};
+
 struct options
 {
  int verbose;                  /* Are we verbose?  (First set to -1,
                                   hence not boolean.) */
  bool quiet;                   /* Are we quiet? */
  int ntry;                     /* Number of tries per URL */
-#ifdef ENABLE_METALINK
-  int n_retries;                /* Number of tries per file */
-#endif
-  char *metalink_file;          /* Metalink filename */
-
  bool retry_connrefused;       /* Treat CONNREFUSED as non-fatal. */
+  char *retry_on_http_error;    /* Treat given HTTP errors as non-fatal. */
  bool background;              /* Whether we should work in background. */
  bool ignore_length;           /* Do we heed content-length at all?  */
  bool recursive;               /* Are we recursive? */
@@ -59,10 +60,16 @@ struct options
  bool add_hostdir;             /* Do we add hostname directory? */
  bool protocol_directories;    /* Whether to prepend "http"/"ftp" to dirs. */
  bool noclobber;               /* Disables clobbering of existing data. */
-  bool unlink;                  /* remove file before clobbering */
+  bool unlink_requested;        /* remove file before clobbering */
  char *dir_prefix;             /* The top of directory tree */
  char *lfilename;              /* Log filename */
  char *input_filename;         /* Input filename */
+#ifdef HAVE_METALINK
+  char *input_metalink;         /* Input metalink file */
+  int metalink_index;           /* Metalink application/metalink4+xml metaurl ordinal number. */
+  bool metalink_over_http;      /* Use Metalink if present in HTTP response */
+  char *preferred_location;     /* Preferred location for Metalink resources */
+#endif
  char *choose_config;          /* Specified config file */
  bool noconfig;                /* Ignore all config files? */
  bool force_html;              /* Is the input file an HTML file? */
@@ -92,6 +99,11 @@ struct options
  void *(*regex_compile_fun)(const char *);             /* Function to compile a regex. */
  bool (*regex_match_fun)(const void *, const char *);  /* Function to match a string to a regex. */

+#ifdef HAVE_LIBCARES
+  char *bind_dns_address;
+  char *dns_servers;
+#endif
+
  char **domains;               /* See host.c */
  char **exclude_domains;
  bool dns_cache;               /* whether we cache DNS lookups. */
@@ -115,9 +127,12 @@ struct options
  bool warc_keep_log;           /* Store the log file in a WARC record. */
  char **warc_user_headers;     /* User-defined WARC header(s). */

+  bool enable_xattr;            /* Store metadata in POSIX extended attributes. */
+
  char *user;                   /* Generic username */
  char *passwd;                 /* Generic password */
  bool ask_passwd;              /* Ask for password? */
+  char *use_askpass;           /* value to use for use-askpass if WGET_ASKPASS is not set */

  bool always_rest;             /* Always use REST. */
  wgint start_pos;              /* Start position of a download. */
@@ -138,7 +153,7 @@ struct options
  char **no_proxy;
  char *base_href;
  char *progress_type;          /* progress indicator type. */
-  bool show_progress;           /* Show only the progress bar */
+  int  show_progress;           /* Show only the progress bar */
  bool noscroll;                /* Don't scroll the filename in the progressbar */
  char *proxy_user; /*oli*/
  char *proxy_passwd;
@@ -170,6 +185,7 @@ struct options
 #endif

  bool timestamping;            /* Whether to use time-stamping. */
+  bool if_modified_since;       /* Whether to use conditional get requests.  */

  bool backup_converted;        /* Do we save pre-converted files as *.orig? */
  int backups;                  /* Are numeric backups made? */
@@ -181,6 +197,9 @@ struct options
                                   NULL. */
  bool convert_links;           /* Will the links be converted
                                   locally? */
+  bool convert_file_only;       /* Convert only the file portion of the URI (i.e. basename).
+                                   Leave everything else untouched. */
+
  bool remove_listing;          /* Do we remove .listing files
                                   generated by FTP? */
  bool htmlify;                 /* Do we HTML-ify the OS-dependent
@@ -207,9 +226,11 @@ struct options
    secure_protocol_sslv2,
    secure_protocol_sslv3,
    secure_protocol_tlsv1,
+    secure_protocol_tlsv1_1,
+    secure_protocol_tlsv1_2,
    secure_protocol_pfs
  } secure_protocol;            /* type of secure protocol to use. */
-  bool check_cert;              /* whether to validate the server's cert */
+  int check_cert;               /* whether to validate the server's cert */
  char *cert_file;              /* external client certificate to use. */
  char *private_key;            /* private key file (if not internal). */
  enum keyfile_type {
@@ -221,15 +242,26 @@ struct options

  char *ca_directory;           /* CA directory (hash files) */
  char *ca_cert;                /* CA certificate file to use */
+  char *crl_file;               /* file with CRLs */
+
+  char *pinnedpubkey;           /* Public key (PEM/DER) file, or any number
+                                   of base64 encoded sha256 hashes preceded by
+                                   \'sha256//\' and separated by \';\', to verify
+                                   peer against */

  char *random_file;            /* file with random data to seed the PRNG */
  char *egd_file;               /* file name of the egd daemon socket */
  bool https_only;              /* whether to follow HTTPS only */
+  bool ftps_resume_ssl;
+  bool ftps_fallback_to_ftp;
+  bool ftps_implicit;
+  bool ftps_clear_data_connection;
 #endif /* HAVE_SSL */

  bool cookies;                 /* whether cookies are used. */
  char *cookies_input;          /* file we're loading the cookies from. */
  char *cookies_output;         /* file we're saving the cookies to. */
+  bool keep_badhash;            /* Keep files with checksum mismatch. */
  bool keep_session_cookies;    /* whether session cookies should be
                                   saved and loaded. */

@@ -241,6 +273,7 @@ struct options

  enum {
    restrict_unix,
+    restrict_vms,
    restrict_windows
  } restrict_files_os;          /* file name restriction ruleset. */
  bool restrict_files_ctrl;     /* non-zero if control chars in URLs
@@ -277,7 +310,7 @@ struct options

  bool enable_iri;
  char *encoding_remote;
-  char *locale;
+  const char *locale;

  bool trustservernames;
 #ifdef __VMS
@@ -291,7 +324,20 @@ struct options
                                   name. */
  bool report_bps;              /*Output bandwidth in bits format*/

-  int jobs;                 /* How many threads use at the same time.  */
+#ifdef HAVE_LIBZ
+  enum compression_options {
+    compression_auto,
+    compression_gzip,
+    compression_none
+  } compression;                /* type of HTTP compression to use */
+#endif
+
+  char *rejected_log;           /* The file to log rejected URLS to. */
+
+#ifdef HAVE_HSTS
+  bool hsts;
+  char *hsts_file;
+#endif
 };

 extern struct options opt;
--- a/src/progress.c
+++ b/src/progress.c
@@ -1,6 +1,5 @@
 /* Download progress.
-   Copyright (C) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009,
-   2010, 2011 Free Software Foundation, Inc.
+   Copyright (C) 2001-2011, 2015, 2018 Free Software Foundation, Inc.

 This file is part of GNU Wget.

@@ -36,24 +35,20 @@ as that of the covered work.  */
 #include <assert.h>
 #include <unistd.h>
 #include <signal.h>
-#include <time.h>
 #include <wchar.h>
-
-#ifdef ENABLE_THREADS
-#include <pthread.h>
-#endif
+#include <mbiter.h>

 #include "progress.h"
 #include "utils.h"
 #include "retr.h"
-#include "ptimer.h"
+#include "c-strcase.h"

 struct progress_implementation {
  const char *name;
  bool interactive;
  void *(*create) (const char *, wgint, wgint);
  void (*update) (void *, wgint, double);
-  void (*draw) (void *, bool);
+  void (*draw) (void *);
  void (*finish) (void *, double);
  void (*set_params) (char *);
 };
@@ -63,33 +58,15 @@ struct progress_implementation {
 static void *dot_create (const char *, wgint, wgint);
 static void dot_update (void *, wgint, double);
 static void dot_finish (void *, double);
-static void dot_draw (void *, bool);
+static void dot_draw (void *);
 static void dot_set_params (char *);

 static void *bar_create (const char *, wgint, wgint);
 static void bar_update (void *, wgint, double);
-static void bar_draw (void *, bool);
+static void bar_draw (void *);
 static void bar_finish (void *, double);
 static void bar_set_params (char *);

-struct progress_header
-{
-  struct progress_header *next;
-};
-
-static struct progress_header *progress_list = NULL;
-static struct progress_header *current_progress = NULL;
-
-#ifdef ENABLE_THREADS
-static pthread_mutex_t progress_mutex = PTHREAD_MUTEX_INITIALIZER;
-# define LOCK_PROGRESS() pthread_mutex_lock (&progress_mutex);
-# define UNLOCK_PROGRESS() pthread_mutex_unlock (&progress_mutex);
-#else
-# define LOCK_PROGRESS()
-# define UNLOCK_PROGRESS()
-#endif
-
-
 static struct progress_implementation implementations[] = {
  { "dot", 0, dot_create, dot_update, dot_draw, dot_finish, dot_set_params },
  { "bar", 1, bar_create, bar_update, bar_draw, bar_finish, bar_set_params }
@@ -97,7 +74,7 @@ static struct progress_implementation implementations[] = {
 static struct progress_implementation *current_impl;
 static int current_impl_locked;

-/* Progress implementation used by default.  Can be overriden in
+/* Progress implementation used by default.  Can be overridden in
   wgetrc or by the fallback one.  */

 #define DEFAULT_PROGRESS_IMPLEMENTATION "bar"
@@ -110,37 +87,6 @@ static int current_impl_locked;

 #define FALLBACK_PROGRESS_IMPLEMENTATION "dot"

-
-/* Time between screen refreshes will not be shorter than this, so
-   that Wget doesn't swamp the TTY with output.  */
-#define REFRESH_INTERVAL 0.2
-
-/* Time between screen refreshes the file to show when there are multiple
-   downloads at the same time.  */
-#define REFRESH_SWITCH_FILE 1.5
-
-/* Assumed screen width if we can't find the real value.  */
-#define DEFAULT_SCREEN_WIDTH 80
-
-/* Minimum screen width we'll try to work with.  If this is too small,
-   create_image will overflow the buffer.  */
-#define MINIMUM_SCREEN_WIDTH 45
-
-/* The last known screen width.  This can be updated by the code that
-   detects that SIGWINCH was received (but it's never updated from the
-   signal handler).  */
-static int screen_width;
-
-/* A flag that, when set, means SIGWINCH was received.  */
-static volatile sig_atomic_t received_sigwinch;
-
-static int bp_width;            /* screen width we're using at the
-                                   time the progress gauge was
-                                   created.  this is different from
-                                   the screen_width global variable in
-                                   that the latter can be changed by a
-                                   signal. */
-
 /* Return true if NAME names a valid progress bar implementation.  The
   characters after the first : will be ignored.  */

@@ -210,7 +156,6 @@ void *
 progress_create (const char *f_download, wgint initial, wgint total)
 {
  /* Check if the log status has changed under our feet. */
-  struct progress_header *ret;
  if (output_redirected)
    {
      if (!current_impl_locked)
@@ -218,17 +163,7 @@ progress_create (const char *f_download, wgint initial, wgint total)
      output_redirected = 0;
    }

-  ret = current_impl->create (f_download, initial, total);
-  if (ret)
-    {
-      LOCK_PROGRESS ();
-
-      ret->next = progress_list;
-      progress_list = ret;
-
-      UNLOCK_PROGRESS ();
-    }
-  return ret;
+  return current_impl->create (f_download, initial, total);
 }

 /* Return true if the progress gauge is "interactive", i.e. if it can
@@ -248,50 +183,8 @@ progress_interactive_p (void *progress _GL_UNUSED)
 void
 progress_update (void *progress, wgint howmuch, double dltime)
 {
-  static struct ptimer *last_draw = NULL;
-  static struct ptimer *last_switch = NULL;
-  bool force_screen_update = false;
-
  current_impl->update (progress, howmuch, dltime);
-  LOCK_PROGRESS ();
-
-  if (last_draw == NULL)
-    last_draw = ptimer_new ();
-  if (last_switch == NULL)
-    last_switch = ptimer_new ();
-
-  /* If SIGWINCH (the window size change signal) been received,
-     determine the new screen size and update the screen.  */
-  if (received_sigwinch)
-    {
-      int old_width = screen_width;
-      screen_width = determine_screen_width ();
-      if (!screen_width)
-        screen_width = DEFAULT_SCREEN_WIDTH;
-      else if (screen_width < MINIMUM_SCREEN_WIDTH)
-        screen_width = MINIMUM_SCREEN_WIDTH;
-      if (screen_width != old_width)
-        force_screen_update = true;
-      received_sigwinch = 0;
-    }
-
-  if (ptimer_measure (last_draw) >= REFRESH_INTERVAL || force_screen_update)
-    {
-      if (current_progress == NULL)
-        current_progress = progress_list;
-
-      bp_width = screen_width - 1;
-      current_impl->draw (current_progress, force_screen_update);
-      ptimer_reset (last_draw);
-
-      if (ptimer_measure (last_switch) >= REFRESH_SWITCH_FILE)
-        {
-          ptimer_reset (last_switch);
-          current_progress = current_progress->next;
-        }
-    }
-
-  UNLOCK_PROGRESS ();
+  current_impl->draw (progress);
 }

 /* Tell the progress gauge to clean up.  Calling this will free the
@@ -300,38 +193,12 @@ progress_update (void *progress, wgint howmuch, double dltime)
 void
 progress_finish (void *progress, double dltime)
 {
-  struct progress_header *header = progress;
-    {
-      struct progress_header *it, *prev = NULL;
-
-      LOCK_PROGRESS ();
-
-      for (it = progress_list; it; it = it->next)
-        {
-          if (it == progress)
-            {
-              if (current_progress == progress)
-                current_progress = it->next;
-
-              if (it == progress_list)
-                progress_list = progress_list->next;
-              else
-                prev->next = it->next;
-              break;
-            }
-          prev = it;
-        }
-
-      UNLOCK_PROGRESS ();
-    }
  current_impl->finish (progress, dltime);
 }
-
+
 /* Dot-printing. */

 struct dot_progress {
-  struct progress_header header;
-
  wgint initial_length;         /* how many bytes have been downloaded
                                   previously. */
  wgint total_length;           /* expected total byte count when the
@@ -486,16 +353,12 @@ dot_update (void *progress, wgint howmuch, double dltime)
 }

 static void
-dot_draw (void *progress, bool force)
+dot_draw (void *progress)
 {
  struct dot_progress *dp = progress;
  int dot_bytes = opt.dot_bytes;
  wgint ROW_BYTES = opt.dot_bytes * opt.dots_in_line;

-  /* FIXME: support threads.  */
-  if (opt.jobs > 1)
-    return;
-
  log_set_flush (false);

  for (; dp->accumulated >= dot_bytes; dp->accumulated -= dot_bytes)
@@ -564,7 +427,7 @@ dot_set_params (char *params)
    return;

  /* We use this to set the retrieval style.  */
-  if (!strcasecmp (params, "default"))
+  if (!c_strcasecmp (params, "default"))
    {
      /* Default style: 1K dots, 10 dots in a cluster, 50 dots in a
         line.  */
@@ -572,7 +435,7 @@ dot_set_params (char *params)
      opt.dot_spacing = 10;
      opt.dots_in_line = 50;
    }
-  else if (!strcasecmp (params, "binary"))
+  else if (!c_strcasecmp (params, "binary"))
    {
      /* "Binary" retrieval: 8K dots, 16 dots in a cluster, 48 dots
         (384K) in a line.  */
@@ -580,7 +443,7 @@ dot_set_params (char *params)
      opt.dot_spacing = 16;
      opt.dots_in_line = 48;
    }
-  else if (!strcasecmp (params, "mega"))
+  else if (!c_strcasecmp (params, "mega"))
    {
      /* "Mega" retrieval, for retrieving very long files; each dot is
         64K, 8 dots in a cluster, 6 clusters (3M) in a line.  */
@@ -588,7 +451,7 @@ dot_set_params (char *params)
      opt.dot_spacing = 8;
      opt.dots_in_line = 48;
    }
-  else if (!strcasecmp (params, "giga"))
+  else if (!c_strcasecmp (params, "giga"))
    {
      /* "Giga" retrieval, for retrieving very very *very* long files;
         each dot is 1M, 8 dots in a cluster, 4 clusters (32M) in a
@@ -602,9 +465,24 @@ dot_set_params (char *params)
             _("Invalid dot style specification %s; leaving unchanged.\n"),
             quote (params));
 }
-
+
 /* "Thermometer" (bar) progress. */

+/* Assumed screen width if we can't find the real value.  */
+#define DEFAULT_SCREEN_WIDTH 80
+
+/* Minimum screen width we'll try to work with.  If this is too small,
+   create_image will overflow the buffer.  */
+#define MINIMUM_SCREEN_WIDTH 45
+
+/* The last known screen width.  This can be updated by the code that
+   detects that SIGWINCH was received (but it's never updated from the
+   signal handler).  */
+static int screen_width;
+
+/* A flag that, when set, means SIGWINCH was received.  */
+static volatile sig_atomic_t received_sigwinch;
+
 /* Size of the download speed history ring. */
 #define DLSPEED_HISTORY_SIZE 20

@@ -619,12 +497,15 @@ dot_set_params (char *params)
   download speeds are scratched.  */
 #define STALL_START_TIME 5

+/* Time between screen refreshes will not be shorter than this, so
+   that Wget doesn't swamp the TTY with output.  */
+#define REFRESH_INTERVAL 0.2
+
 /* Don't refresh the ETA too often to avoid jerkiness in predictions.
   This allows ETA to change approximately once per second.  */
 #define ETA_REFRESH_INTERVAL 0.99

 struct bar_progress {
-  struct progress_header header;
  const char *f_download;       /* Filename of the downloaded file */
  wgint initial_length;         /* how many bytes have been downloaded
                                   previously. */
@@ -632,8 +513,19 @@ struct bar_progress {
                                   download finishes */
  wgint count;                  /* bytes downloaded so far */

-  double dltime;                /* download time so far */
+  double last_screen_update;    /* time of the last screen update,
+                                   measured since the beginning of
+                                   download. */

+  double dltime;                /* download time so far */
+  int width;                    /* screen width we're using at the
+                                   time the progress gauge was
+                                   created.  this is different from
+                                   the screen_width global variable in
+                                   that the latter can be changed by a
+                                   signal. */
+  char *buffer;                 /* buffer where the bar "image" is
+                                   stored. */
  int tick;                     /* counter used for drawing the
                                   progress bar where the total size
                                   is not known. */
@@ -668,7 +560,7 @@ struct bar_progress {
  int last_eta_value;
 };

-static void create_image (struct bar_progress *, char *buffer, double, bool);
+static void create_image (struct bar_progress *, double, bool);
 static void display_image (char *);

 static void *
@@ -698,11 +590,16 @@ bar_create (const char *f_download, wgint initial, wgint total)
    }

  /* - 1 because we don't want to use the last screen column. */
-  bp_width = screen_width - 1;
+  bp->width = screen_width - 1;
+  /* + enough space for the terminating zero, and hopefully enough room
+   * for multibyte characters. */
+#define BUF_LEN (bp->width + 100)
+  bp->buffer = xmalloc (BUF_LEN);

  logputs (LOG_VERBOSE, "\n");

-  bar_draw (bp, false);
+  create_image (bp, 0, false);
+  display_image (bp->buffer);

  return bp;
 }
@@ -729,20 +626,37 @@ bar_update (void *progress, wgint howmuch, double dltime)
 }

 static void
-bar_draw (void *progress, bool force)
+bar_draw (void *progress)
 {
+  bool force_screen_update = false;
  struct bar_progress *bp = progress;
-  static char *buffer = NULL;

-  if (buffer == NULL)
-    buffer = xmalloc (bp_width + 100);
-
-  if (force)
+  /* If SIGWINCH (the window size change signal) been received,
+     determine the new screen size and update the screen.  */
+  if (received_sigwinch)
    {
-      buffer = xrealloc (buffer, bp_width + 100);
+      int old_width = screen_width;
+      screen_width = determine_screen_width ();
+      if (!screen_width)
+        screen_width = DEFAULT_SCREEN_WIDTH;
+      else if (screen_width < MINIMUM_SCREEN_WIDTH)
+        screen_width = MINIMUM_SCREEN_WIDTH;
+      if (screen_width != old_width)
+        {
+          bp->width = screen_width - 1;
+          bp->buffer = xrealloc (bp->buffer, bp->width + 100);
+          force_screen_update = true;
+        }
+      received_sigwinch = 0;
    }
-  create_image (bp, buffer, bp->dltime, false);
-  display_image (buffer);
+
+  if (bp->dltime - bp->last_screen_update < REFRESH_INTERVAL && !force_screen_update)
+    /* Don't update more often than five times per second. */
+    return;
+
+  create_image (bp, bp->dltime, false);
+  display_image (bp->buffer);
+  bp->last_screen_update = bp->dltime;
 }

 static void
@@ -755,11 +669,13 @@ bar_finish (void *progress, double dltime)
    /* See bar_update() for explanation. */
    bp->total_length = bp->initial_length + bp->count;

-  bar_draw (bp, false);
+  create_image (bp, dltime, true);
+  display_image (bp->buffer);

  logputs (LOG_VERBOSE, "\n");
  logputs (LOG_PROGRESS, "\n");

+  xfree (bp->buffer);
  xfree (bp);
 }

@@ -777,7 +693,7 @@ bar_finish (void *progress, double dltime)
   The idea is that for fast downloads, we get the speed over exactly
   the last three seconds.  For slow downloads (where a network read
   takes more than 150ms to complete), we get the speed over a larger
-   time period, as large as it takes to complete thirty reads.  This
+   time period, as large as it takes to complete twenty reads.  This
   is good because slow downloads tend to fluctuate more and a
   3-second average would be too erratic.  */

@@ -898,8 +814,39 @@ count_cols (const char *mbs)
    }
  return cols;
 }
+
+static int
+cols_to_bytes (const char *mbs, const int cols, int *ncols)
+{
+  int p_cols = 0, bytes = 0;
+  mbchar_t mbc;
+  mbi_iterator_t iter;
+  mbi_init (iter, mbs, strlen(mbs));
+  while (p_cols < cols && mbi_avail (iter))
+    {
+      mbc = mbi_cur (iter);
+      p_cols += mb_width (mbc);
+      /* The multibyte character has exceeded the total number of columns we
+       * have available. The remaining bytes will be padded with a space. */
+      if (p_cols > cols)
+        {
+          p_cols -= mb_width (mbc);
+          break;
+        }
+      bytes += mb_len (mbc);
+      mbi_advance (iter);
+    }
+  *ncols = p_cols;
+  return bytes;
+}
 #else
-# define count_cols(mbs) ((int)(strlen(mbs)))
+static int count_cols (const char *mbs) { return (int) strlen(mbs); }
+static int
+cols_to_bytes (const char *mbs _GL_UNUSED, const int cols, int *ncols)
+{
+  *ncols = cols;
+  return cols;
+}
 #endif

 static const char *
@@ -907,7 +854,7 @@ get_eta (int *bcd)
 {
  /* TRANSLATORS: "ETA" is English-centric, but this must
     be short, ideally 3 chars.  Abbreviate if necessary.  */
-  static const char eta_str[] = N_("   eta %s");
+  static const char eta_str[] = N_("    eta %s");
  static const char *eta_trans;
  static int bytes_cols_diff;
  if (eta_trans == NULL)
@@ -944,33 +891,17 @@ get_eta (int *bcd)
  p += sizeof (s) - 1;                          \
 } while (0)

-/* Use move_to_end (s) to get S to point the end of the string (the
-   terminating \0).  This is faster than s+=strlen(s), but some people
-   are confused when they see strchr (s, '\0') in the code.  */
-#define move_to_end(s) s = strchr (s, '\0');
-
-#ifndef MAX
-# define MAX(a, b) ((a) >= (b) ? (a) : (b))
-#endif
-#ifndef MIN
-# define MIN(a, b) ((a) <= (b) ? (a) : (b))
-#endif
-
 static void
-create_image (struct bar_progress *bp, char *buffer, double dl_total_time, bool done)
+create_image (struct bar_progress *bp, double dl_total_time, bool done)
 {
-  const int MAX_FILENAME_LEN = bp_width / 4;
-  char *p = buffer;
+  const int MAX_FILENAME_COLS = bp->width / 4;
+  char *p = bp->buffer;
  wgint size = bp->initial_length + bp->count;

-  const char *size_grouped = with_thousand_seps (size);
-  int size_grouped_len = count_cols (size_grouped);
-  /* Difference between num cols and num bytes: */
-  int size_grouped_diff = strlen (size_grouped) - size_grouped_len;
-  int size_grouped_pad; /* Used to pad the field width for size_grouped. */
-
  struct bar_progress_hist *hist = &bp->hist;
-  int orig_filename_len = strlen (bp->f_download);
+  int orig_filename_cols = count_cols (bp->f_download);
+
+  int padding;

  /* The progress bar should look like this:
     file xx% [=======>             ] nnn.nnK 12.34KB/s  eta 36m 51s
@@ -982,9 +913,8 @@ create_image (struct bar_progress *bp, char *buffer, double dl_total_time, bool
     It would be especially bad for the progress bar to be resized
     randomly.

-     "file "           - Downloaded filename      - MAX_FILENAME_LEN chars + 1
+     "file "           - Downloaded filename      - MAX_FILENAME_COLS chars + 1
     "xx% " or "100%"  - percentage               - 4 chars
-     " url "           - string                   - MAX MAX_URL_LEN chars + 2
     "[]"              - progress bar decorations - 2 chars
     " nnn.nnK"        - downloaded bytes         - 7 chars + 1
     " 12.5KB/s"       - download rate            - 8 chars + 1
@@ -993,43 +923,76 @@ create_image (struct bar_progress *bp, char *buffer, double dl_total_time, bool
     "=====>..."       - progress bar             - the rest
  */

-#define PROGRESS_FILENAME_LEN  MAX_FILENAME_LEN + 1
+  /* TODO: Ask the Turkish Translators to fix their translation for the "done"
+   * mode of progress bar. Use one less character. Once that is done, redice
+   * PROGRESS_ETA_LEN by 1.
+   */
+#define PROGRESS_FILENAME_LEN  MAX_FILENAME_COLS + 1
 #define PROGRESS_PERCENT_LEN   4
 #define PROGRESS_DECORAT_LEN   2
 #define PROGRESS_FILESIZE_LEN  7 + 1
-#define PROGRESS_DWNLOAD_RATE  8 + 1
-#define PROGRESS_ETA_LEN       14
+#define PROGRESS_DWNLOAD_RATE  8 + 2
+#define PROGRESS_ETA_LEN       15

-  int progress_size = bp_width - (PROGRESS_FILENAME_LEN + PROGRESS_PERCENT_LEN +
+  int progress_size = bp->width - (PROGRESS_FILENAME_LEN + PROGRESS_PERCENT_LEN +
                                   PROGRESS_DECORAT_LEN + PROGRESS_FILESIZE_LEN +
                                   PROGRESS_DWNLOAD_RATE + PROGRESS_ETA_LEN);

  /* The difference between the number of bytes used,
     and the number of columns used. */
  int bytes_cols_diff = 0;
+  int cols_diff;
+  const char *down_size;
+
+  memset (bp->buffer, '\0', BUF_LEN);

  if (progress_size < 5)
    progress_size = 0;

-  if (orig_filename_len <= MAX_FILENAME_LEN)
+  if (orig_filename_cols <= MAX_FILENAME_COLS)
    {
-      int padding = MAX_FILENAME_LEN - orig_filename_len;
-      sprintf (p, "%s ", bp->f_download);
-      p += orig_filename_len + 1;
-      for (;padding;padding--)
-        *p++ = ' ';
+      padding = MAX_FILENAME_COLS - orig_filename_cols;
+      p += sprintf (p, "%s ", bp->f_download);
+      memset (p, ' ', padding);
+      p += padding;
    }
  else
    {
-      int offset;
+      int offset_cols;
+      int bytes_in_filename, offset_bytes, col;
+      int *cols_ret = &col;

-      if (((orig_filename_len > MAX_FILENAME_LEN) && !opt.noscroll) && !done)
-        offset = ((int) bp->tick) % (orig_filename_len - MAX_FILENAME_LEN);
+#define MIN_SCROLL_TEXT 5
+      if ((orig_filename_cols > MAX_FILENAME_COLS + MIN_SCROLL_TEXT) &&
+          !opt.noscroll &&
+          !done)
+        {
+          offset_cols = ((int) bp->tick + orig_filename_cols + MAX_FILENAME_COLS / 2)
+                        % (orig_filename_cols + MAX_FILENAME_COLS);
+          if (offset_cols > orig_filename_cols)
+            {
+              padding = MAX_FILENAME_COLS - (offset_cols - orig_filename_cols);
+              memset(p, ' ', padding);
+              p += padding;
+              offset_cols = 0;
+            }
+          else
+            padding = 0;
+        }
      else
-        offset = 0;
-      memcpy (p, bp->f_download + offset, MAX_FILENAME_LEN);
-      p += MAX_FILENAME_LEN;
-      *p++ = ' ';
+        {
+          padding = 0;
+          offset_cols = 0;
+        }
+      offset_bytes = cols_to_bytes (bp->f_download, offset_cols, cols_ret);
+      bytes_in_filename = cols_to_bytes (bp->f_download + offset_bytes,
+                                         MAX_FILENAME_COLS - padding,
+                                         cols_ret);
+      memcpy (p, bp->f_download + offset_bytes, bytes_in_filename);
+      p += bytes_in_filename;
+      padding = MAX_FILENAME_COLS - (padding + *cols_ret);
+      memset (p, ' ', padding + 1);
+      p += padding + 1;
    }

  /* "xx% " */
@@ -1037,15 +1000,13 @@ create_image (struct bar_progress *bp, char *buffer, double dl_total_time, bool
    {
      int percentage = 100.0 * size / bp->total_length;
      assert (percentage <= 100);
-
-      if (percentage < 100)
-        sprintf (p, "%3d%%", percentage);
-      else
-        strcpy (p, "100%");
-      p += 4;
+      p += sprintf (p, "%3d%%", percentage);
    }
  else
-    APPEND_LITERAL ("    ");
+    {
+      memset (p, ' ', PROGRESS_PERCENT_LEN);
+      p += PROGRESS_PERCENT_LEN;
+    }

  /* The progress bar: "[====>      ]" or "[++==>      ]". */
  if (progress_size && bp->total_length > 0)
@@ -1057,7 +1018,6 @@ create_image (struct bar_progress *bp, char *buffer, double dl_total_time, bool
      int dlsz = (double)size / bp->total_length * progress_size;

      char *begin;
-      int i;

      assert (dlsz <= progress_size);
      assert (insz <= dlsz);
@@ -1067,18 +1027,19 @@ create_image (struct bar_progress *bp, char *buffer, double dl_total_time, bool

      /* Print the initial portion of the download with '+' chars, the
         rest with '=' and one '>'.  */
-      for (i = 0; i < insz; i++)
-        *p++ = '+';
+      memset (p, '+', insz);
+      p += insz;
+
      dlsz -= insz;
      if (dlsz > 0)
        {
-          for (i = 0; i < dlsz - 1; i++)
-            *p++ = '=';
+          memset (p, '=', dlsz-1);
+          p += dlsz - 1;
          *p++ = '>';
        }

-      while (p - begin < progress_size)
-        *p++ = ' ';
+      memset (p, ' ', (progress_size - (p - begin)));
+      p += (progress_size - (p - begin));
      *p++ = ']';
    }
  else if (progress_size)
@@ -1105,28 +1066,15 @@ create_image (struct bar_progress *bp, char *buffer, double dl_total_time, bool
        }
      *p++ = ']';

-      ++bp->tick;
    }
+  ++bp->tick;

  /* " 234.56M" */
-  const char * down_size = human_readable (size, 1000, 2);
-  int cols_diff = 7 - count_cols (down_size);
-  while (cols_diff > 0)
-  {
-    *p++=' ';
-    cols_diff--;
-  }
-  sprintf (p, " %s", down_size);
-  move_to_end (p);
-  /* Pad with spaces to 7 chars for the size_grouped field;
-   * couldn't use the field width specifier in sprintf, because
-   * it counts in bytes, not characters. */
-  for (size_grouped_pad = PROGRESS_FILESIZE_LEN - 7;
-       size_grouped_pad > 0;
-       --size_grouped_pad)
-    {
-      *p++ = ' ';
-    }
+  down_size = human_readable (size, 1000, 2);
+  cols_diff = PROGRESS_FILESIZE_LEN - count_cols (down_size);
+  memset (p, ' ', cols_diff);
+  p += cols_diff;
+  p += sprintf (p, "%s", down_size);

  /* " 12.52Kb/s or 12.52KB/s" */
  if (hist->total_time > 0 && hist->total_bytes)
@@ -1139,12 +1087,11 @@ create_image (struct bar_progress *bp, char *buffer, double dl_total_time, bool
      wgint dlquant = hist->total_bytes + bp->recent_bytes;
      double dltime = hist->total_time + (dl_total_time - bp->recent_start);
      double dlspeed = calc_rate (dlquant, dltime, &units);
-      sprintf (p, " %4.*f%s", dlspeed >= 99.95 ? 0 : dlspeed >= 9.995 ? 1 : 2,
+      p += sprintf (p, "  %4.*f%s", dlspeed >= 99.95 ? 0 : dlspeed >= 9.995 ? 1 : 2,
               dlspeed,  !opt.report_bps ? short_units[units] : short_units_bits[units]);
-      move_to_end (p);
    }
  else
-    APPEND_LITERAL (" --.-KB/s");
+    APPEND_LITERAL ("  --.-KB/s");

  if (!done)
    {
@@ -1179,14 +1126,14 @@ create_image (struct bar_progress *bp, char *buffer, double dl_total_time, bool
              bp->last_eta_time = dl_total_time;
            }

-          sprintf (p, get_eta(&bytes_cols_diff),
+          p += sprintf (p, get_eta(&bytes_cols_diff),
                   eta_to_human_short (eta, false));
-          move_to_end (p);
        }
      else if (bp->total_length > 0)
        {
        skip_eta:
-          APPEND_LITERAL ("             ");
+          memset (p, ' ', PROGRESS_ETA_LEN);
+          p += PROGRESS_ETA_LEN;
        }
    }
  else
@@ -1196,22 +1143,38 @@ create_image (struct bar_progress *bp, char *buffer, double dl_total_time, bool
      int ncols;

      /* Note to translators: this should not take up more room than
-         available here.  Abbreviate if necessary.  */
-      strcpy (p, _("   in "));
+         available here (6 columns).  Abbreviate if necessary.  */
+      strcpy (p, _("    in "));
      nbytes = strlen (p);
      ncols  = count_cols (p);
      bytes_cols_diff = nbytes - ncols;
-      p += nbytes;
      if (dl_total_time >= 10)
-        strcpy (p, eta_to_human_short ((int) (dl_total_time + 0.5), false));
+        ncols += sprintf (p + nbytes, "%s",  eta_to_human_short ((int) (dl_total_time + 0.5), false));
      else
-        sprintf (p, "%ss", print_decimal (dl_total_time));
-      move_to_end (p);
+        ncols += sprintf (p + nbytes, "%ss", print_decimal (dl_total_time));
+      p += ncols + bytes_cols_diff;
+      memset (p, ' ', PROGRESS_ETA_LEN - ncols);
+      p += PROGRESS_ETA_LEN - ncols;
    }

-  while (p - buffer - bytes_cols_diff - size_grouped_diff < bp_width)
-    *p++ = ' ';
+  padding = bp->width - count_cols (bp->buffer);
+  assert (padding >= 0 && "Padding length became non-positive!");
+  padding = padding > 0 ? padding : 0;
+  memset (p, ' ', padding);
+  p += padding;
  *p = '\0';
+
+  /* 2014-11-14  Darshit Shah  <darnir@gmail.com>
+   * Assert that the length of the progress bar is lesser than the size of the
+   * screen with which we are dealing. This assertion *MUST* always be removed
+   * from the release code since we do not want Wget to crash and burn when the
+   * assertion fails. Instead Wget should continue downloading and display a
+   * horrible and irritating progress bar that spams the screen with newlines.
+   *
+   * By default, all assertions are disabled in a Wget build and are enabled
+   * only with the --enable-assert configure option.
+   */
+  assert (count_cols (bp->buffer) == bp->width);
 }

 /* Print the contents of the buffer as a one-line ASCII "image" so
@@ -1229,8 +1192,6 @@ display_image (char *buf)
 static void
 bar_set_params (char *params)
 {
-  char *term = getenv ("TERM");
-
  if (params)
    {
      char *param = strtok (params, ":");
@@ -1243,19 +1204,13 @@ bar_set_params (char *params)
        } while ((param = strtok (NULL, ":")) != NULL);
    }

-  if ((opt.lfilename
+  if (((opt.lfilename && opt.show_progress != 1)
 #ifdef HAVE_ISATTY
       /* The progress bar doesn't make sense if the output is not a
          TTY -- when logging to file, it is better to review the
          dots.  */
       || !isatty (fileno (stderr))
 #endif
-       /* Normally we don't depend on terminal type because the
-          progress bar only uses ^M to move the cursor to the
-          beginning of line, which works even on dumb terminals.  But
-          Jamie Zawinski reports that ^M and ^H tricks don't work in
-          Emacs shell buffers, and only make a mess.  */
-       || (term && 0 == strcmp (term, "emacs"))
       )
      && !current_impl_locked)
    {
--- a/src/progress.h
+++ b/src/progress.h
@@ -1,6 +1,5 @@
 /* Download progress.
-   Copyright (C) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009,
-   2010, 2011 Free Software Foundation, Inc.
+   Copyright (C) 2001-2011, 2015, 2018 Free Software Foundation, Inc.

 This file is part of GNU Wget.

--- a/src/ptimer.c
+++ b/src/ptimer.c
@@ -1,6 +1,5 @@
 /* Portable timers.
-   Copyright (C) 2005, 2006, 2007, 2008, 2009, 2010, 2011 Free Software
-   Foundation, Inc.
+   Copyright (C) 2005-2011, 2015, 2018 Free Software Foundation, Inc.

 This file is part of GNU Wget.

@@ -291,7 +290,7 @@ windows_resolution (void)
    return 10;                  /* according to MSDN */
 }
 #endif  /* PTIMER_WINDOWS */
-
+
 /* The code below this point is independent of timer implementation. */

 struct ptimer {
--- a/src/ptimer.h
+++ b/src/ptimer.h
@@ -1,6 +1,5 @@
 /* Declarations for ptimer.c.
-   Copyright (C) 2005, 2006, 2007, 2008, 2009, 2010, 2011 Free Software
-   Foundation, Inc.
+   Copyright (C) 2005-2011, 2015, 2018 Free Software Foundation, Inc.

 This file is part of GNU Wget.

--- a/src/recur.c
+++ b/src/recur.c
@@ -1,7 +1,5 @@
 /* Handling of recursive HTTP retrieving.
-   Copyright (C) 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004,
-   2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012 Free Software Foundation,
-   Inc.
+   Copyright (C) 1996-2012, 2015, 2018 Free Software Foundation, Inc.

 This file is part of GNU Wget.

@@ -37,12 +35,6 @@ as that of the covered work.  */
 #include <unistd.h>
 #include <errno.h>
 #include <assert.h>
-#ifdef ENABLE_THREADS
-#include <pthread.h>
-#include <semaphore.h>
-#endif
-
-#include "multi.h"

 #include "url.h"
 #include "recur.h"
@@ -56,7 +48,8 @@ as that of the covered work.  */
 #include "html-url.h"
 #include "css-url.h"
 #include "spider.h"
-
+#include "exits.h"
+
 /* Functions for maintaining the URL queue.  */

 struct queue_element {
@@ -165,33 +158,42 @@ url_dequeue (struct url_queue *queue, struct iri **i,
  xfree (qel);
  return true;
 }
-
-static bool download_child_p (const struct urlpos *, struct url *, int,
-                              struct url *, struct hash_table *, struct iri *);
-static bool descend_redirect_p (const char *, struct url *, int,
-                                struct url *, struct hash_table *, struct iri *);

-#if !ENABLE_THREADS
-# define THREAD_JOIN(...)  (0)
-# define SEM_INIT(...)     (0)
-# define SEM_WAIT(...)     (0)
-#else
-# define THREAD_JOIN pthread_join
-# define SEM_INIT sem_init
-# define SEM_WAIT sem_wait
-static void *
-start_retrieve_url (void *arg)
+static void blacklist_add (struct hash_table *blacklist, const char *url)
 {
-  struct s_thread_ctx *ctx = (struct s_thread_ctx *) arg;
+  char *url_unescaped = xstrdup (url);

-  ctx->status = retrieve_url (ctx->url_parsed, ctx->url,
-                              &ctx->file, &ctx->redirected,
-                              ctx->referer, &ctx->dt,
-                              false, ctx->i, true, NULL);
-  ctx->terminated = 1;
-  sem_post (ctx->retr_sem);
+  url_unescape (url_unescaped);
+  string_set_add (blacklist, url_unescaped);
+  xfree (url_unescaped);
 }
-#endif
+
+static int blacklist_contains (struct hash_table *blacklist, const char *url)
+{
+  char *url_unescaped = xstrdup(url);
+  int ret;
+
+  url_unescape (url_unescaped);
+  ret = string_set_contains (blacklist, url_unescaped);
+  xfree (url_unescaped);
+
+  return ret;
+}
+
+typedef enum
+{
+  WG_RR_SUCCESS, WG_RR_BLACKLIST, WG_RR_NOTHTTPS, WG_RR_NONHTTP, WG_RR_ABSOLUTE,
+  WG_RR_DOMAIN, WG_RR_PARENT, WG_RR_LIST, WG_RR_REGEX, WG_RR_RULES,
+  WG_RR_SPANNEDHOST, WG_RR_ROBOTS
+} reject_reason;
+
+static reject_reason download_child (const struct urlpos *, struct url *, int,
+                              struct url *, struct hash_table *, struct iri *);
+static reject_reason descend_redirect (const char *, struct url *, int,
+                              struct url *, struct hash_table *, struct iri *);
+static void write_reject_log_header (FILE *);
+static void write_reject_log_reason (FILE *, reject_reason,
+                              const struct url *, const struct url *);

 /* Retrieve a part of the web beginning with START_URL.  This used to
   be called "recursive retrieval", because the old function was
@@ -218,21 +220,6 @@ uerr_t
 retrieve_tree (struct url *start_url_parsed, struct iri *pi)
 {
  uerr_t status = RETROK;
-  struct s_thread_ctx *thread_ctx;
-  char *next_url = NULL, *next_referer;
-  int next_depth;
-  bool next_html_allowed, next_css_allowed;
-  struct iri *next_i = NULL;
-
-#ifdef ENABLE_THREADS
-  const int N_THREADS = opt.jobs > 0 ? opt.jobs : 1;
-  sem_t retr_sem;
-#else
-  const int N_THREADS = 1;
-  void *retr_sem;
-#endif
-
-  int free_threads = N_THREADS;

  /* The queue of URLs we need to load. */
  struct url_queue *queue;
@@ -243,22 +230,21 @@ retrieve_tree (struct url *start_url_parsed, struct iri *pi)

  struct iri *i = iri_new ();

-#define COPYSTR(x)  (x) ? xstrdup(x) : NULL;
+  FILE *rejectedlog = NULL; /* Don't write a rejected log. */
+
  /* Duplicate pi struct if not NULL */
  if (pi)
    {
+#define COPYSTR(x)  (x) ? xstrdup(x) : NULL;
      i->uri_encoding = COPYSTR (pi->uri_encoding);
      i->content_encoding = COPYSTR (pi->content_encoding);
      i->utf8_encode = pi->utf8_encode;
+#undef COPYSTR
    }
+#ifdef ENABLE_IRI
  else
    set_uri_encoding (i, opt.locale, true);
-#undef COPYSTR
-
-  thread_ctx = calloc (N_THREADS, sizeof *thread_ctx);
-
-  /* FIXME: CHECK FOR ERRORS.  */
-  SEM_INIT (&retr_sem, 0, 0);
+#endif

  queue = url_queue_new ();
  blacklist = make_string_hash_table (0);
@@ -267,19 +253,24 @@ retrieve_tree (struct url *start_url_parsed, struct iri *pi)
     just URL so we enqueue the canonical form of the URL.  */
  url_enqueue (queue, i, xstrdup (start_url_parsed->url), NULL, 0, true,
               false);
-  string_set_add (blacklist, start_url_parsed->url);
+  blacklist_add (blacklist, start_url_parsed->url);
+
+  if (opt.rejected_log)
+    {
+      rejectedlog = fopen (opt.rejected_log, "w");
+      write_reject_log_header (rejectedlog);
+      if (!rejectedlog)
+        logprintf (LOG_NOTQUIET, "%s: %s\n", opt.rejected_log, strerror (errno));
+    }

  while (1)
    {
      bool descend = false;
-      char *file = NULL;
-      bool is_css = false;
-      bool dash_p_leaf_HTML = false;
-      char *url = NULL, *referer;
+      char *url, *referer, *file = NULL;
      int depth;
      bool html_allowed, css_allowed;
-      bool dequed = false;
-      int index = 0;
+      bool is_css = false;
+      bool dash_p_leaf_HTML = false;

      if (opt.quota && total_downloaded_bytes > opt.quota)
        break;
@@ -288,30 +279,20 @@ retrieve_tree (struct url *start_url_parsed, struct iri *pi)

      /* Get the next URL from the queue... */

-      if (next_url == NULL)
-        {
-          if (url_dequeue (queue, (struct iri **) &next_i,
-                           (const char **)&next_url, (const char **)&next_referer,
-                           &next_depth, &next_html_allowed, &next_css_allowed))
-            dequed = true;
-        }
-
-      i = next_i;
-      url = next_url;
-      referer = next_referer;
-      depth = next_depth;
-      html_allowed = next_html_allowed;
-      css_allowed = next_css_allowed;
+      if (!url_dequeue (queue, (struct iri **) &i,
+                        (const char **)&url, (const char **)&referer,
+                        &depth, &html_allowed, &css_allowed))
+        break;

      /* ...and download it.  Note that this download is in most cases
-         unconditional, as download_child_p already makes sure a file
+         unconditional, as download_child already makes sure a file
         doesn't get enqueued twice -- and yet this check is here, and
-         not in download_child_p.  This is so that if you run `wget -r
+         not in download_child.  This is so that if you run `wget -r
         URL1 URL2', and a random URL is encountered once under URL1
         and again under URL2, but at a different (possibly smaller)
         depth, we want the URL's children to be taken into account
         the second time.  */
-      if (dequed && url && dl_url_file_map && hash_table_contains (dl_url_file_map, url))
+      if (dl_url_file_map && hash_table_contains (dl_url_file_map, url))
        {
          bool is_css_bool;

@@ -333,153 +314,73 @@ retrieve_tree (struct url *start_url_parsed, struct iri *pi)
        }
      else
        {
-          int j;
-retry:
-          if (! url)
+          int dt = 0, url_err;
+          char *redirected = NULL;
+          struct url *url_parsed = url_parse (url, &url_err, i, true);
+
+          if (!url_parsed)
            {
-              int used = 0;
-              for (j = 0; j < N_THREADS; j++)
-                {
-                  if (thread_ctx[j].used)
-                    {
-                      used = 1;
-                      break;
-                    }
-                }
-
-              if (! used)
-                break;
-            }
-
-          if (url && free_threads)
-            {
-              int err;
-              for (j = 0; j < N_THREADS; j++)
-                if (! thread_ctx[j].used)
-                  {
-                    index = j;
-                    free_threads--;
-                    thread_ctx[j].used = 1;
-                    thread_ctx[j].terminated = 0;
-                    break;
-                  }
-              assert (j < N_THREADS);
-
-              thread_ctx[index].file = file;
-              thread_ctx[index].referer = referer;
-              thread_ctx[index].dt = 0;
-              thread_ctx[index].i = i;
-              thread_ctx[index].redirected = NULL;
-              thread_ctx[index].range = NULL;
-              thread_ctx[index].url = url;
-              thread_ctx[index].retr_sem = &retr_sem;
-              thread_ctx[index].url_parsed = url_parse (thread_ctx[index].url,
-                                                        &thread_ctx[index].url_err,
-                                                        i, true);
-
-#ifdef ENABLE_THREADS
-              err = pthread_create (&thread_ctx[index].thread, NULL,
-                                    start_retrieve_url, &thread_ctx[index]);
-#else
-              thread_ctx[index].status = retrieve_url (thread_ctx[index].url_parsed,
-                                                       thread_ctx[index].url,
-                                                       &thread_ctx[index].file,
-                                                       &thread_ctx[index].redirected,
-                                                       thread_ctx[index].referer,
-                                                       &thread_ctx[index].dt,
-                                                       false, i, true,
-                                                       NULL);
-              thread_ctx[index].used = 1;
-              thread_ctx[index].terminated = 1;
-              err = 0;
-#endif
-
-              if (err == 0)
-                next_url = NULL;
-              else
-                {
-                  logprintf (LOG_NOTQUIET, "pthread_create: %s\n", strerror (errno));
-                  url_free (thread_ctx[index].url_parsed);
-                  thread_ctx[index].used = 0;
-                  free_threads++;
-                  return THREADS_ERR;
-                }
-              continue;
-            }
-
-          index = -1;
-          for (j = 0; j < N_THREADS; j++)
-            if (thread_ctx[j].used && thread_ctx[j].terminated)
-              {
-                index = j;
-                thread_ctx[j].used = 0;
-                THREAD_JOIN (thread_ctx[j].thread, NULL);
-                free_threads++;
-                break;
-              }
-
-          if (index < 0)
-            {
-              int ret;
-              do
-                ret = SEM_WAIT (&retr_sem);
-              while (ret < 0 && errno == EINTR);
-              if (ret < 0)
-                return SEM_ERR;
-
-              goto retry;
-            }
-
-          file = thread_ctx[index].file;
-          referer = thread_ctx[index].referer;
-          i = thread_ctx[index].i;
-          url = thread_ctx[index].url;
-
-          if (html_allowed && file && status == RETROK
-              && (thread_ctx[index].dt & RETROKF) && (thread_ctx[index].dt & TEXTHTML))
-            {
-              descend = true;
-              is_css = false;
-            }
-
-          /* a little different, css_allowed can override content type
-             lots of web servers serve css with an incorrect content type
-          */
-          if (file && status == RETROK
-              && (thread_ctx[index].dt & RETROKF) &&
-              ((thread_ctx[index].dt & TEXTCSS) || css_allowed)
-             )
-            {
-              descend = true;
-              is_css = true;
-            }
-
-          if (thread_ctx[index].redirected)
-            {
-              /* We have been redirected, possibly to another host, or
-                 different path, or wherever.  Check whether we really
-                 want to follow it.  */
-              if (descend)
-                {
-                  if (!descend_redirect_p (thread_ctx[index].redirected,
-                                           thread_ctx[index].url_parsed, depth,
-                                           start_url_parsed, blacklist, i))
-                    descend = false;
-                  else
-                    /* Make sure that the old pre-redirect form gets
-                       blacklisted. */
-                    string_set_add (blacklist, url);
-                }
-
-              xfree (thread_ctx[index].url);
-              url = thread_ctx[index].redirected;
+              char *error = url_error (url, url_err);
+              logprintf (LOG_NOTQUIET, "%s: %s.\n",url, error);
+              xfree (error);
+              inform_exit_status (URLERROR);
            }
          else
            {
-              xfree (thread_ctx[index].url);
-              url = xstrdup (thread_ctx[index].url_parsed->url);
+
+              status = retrieve_url (url_parsed, url, &file, &redirected, referer,
+                                     &dt, false, i, true);
+
+              if (html_allowed && file && status == RETROK
+                  && (dt & RETROKF) && (dt & TEXTHTML))
+                {
+                  descend = true;
+                  is_css = false;
+                }
+
+              /* a little different, css_allowed can override content type
+                 lots of web servers serve css with an incorrect content type
+              */
+              if (file && status == RETROK
+                  && (dt & RETROKF) &&
+                  ((dt & TEXTCSS) || css_allowed))
+                {
+                  descend = true;
+                  is_css = true;
+                }
+
+              if (redirected)
+                {
+                  /* We have been redirected, possibly to another host, or
+                     different path, or wherever.  Check whether we really
+                     want to follow it.  */
+                  if (descend)
+                    {
+                      reject_reason r = descend_redirect (redirected, url_parsed,
+                                        depth, start_url_parsed, blacklist, i);
+                      if (r == WG_RR_SUCCESS)
+                        {
+                          /* Make sure that the old pre-redirect form gets
+                             blacklisted. */
+                          blacklist_add (blacklist, url);
+                        }
+                      else
+                        {
+                          write_reject_log_reason (rejectedlog, r, url_parsed, start_url_parsed);
+                          descend = false;
+                        }
+                    }
+
+                  xfree (url);
+                  url = redirected;
+                }
+              else
+                {
+                  xfree (url);
+                  url = xstrdup (url_parsed->url);
+                }
+              url_free (url_parsed);
            }
-          url_free(thread_ctx[index].url_parsed);
        }

      if (opt.spider)
@@ -535,22 +436,38 @@ retry:
              struct url *url_parsed = url_parse (url, NULL, i, true);
              struct iri *ci;
              char *referer_url = url;
-              bool strip_auth = (url_parsed != NULL
-                                 && url_parsed->user != NULL);
+              bool strip_auth;
+
              assert (url_parsed != NULL);

+              if (!url_parsed)
+                continue;
+
+              strip_auth = (url_parsed && url_parsed->user);
+
              /* Strip auth info if present */
              if (strip_auth)
                referer_url = url_string (url_parsed, URL_AUTH_HIDE);

              for (; child; child = child->next)
                {
+                  reject_reason r;
+
                  if (child->ignore_when_downloading)
-                    continue;
+                    {
+                      DEBUGP (("Not following due to 'ignore' flag: %s\n", child->url->url));
+                      continue;
+                    }
+
                  if (dash_p_leaf_HTML && !child->link_inline_p)
-                    continue;
-                  if (download_child_p (child, url_parsed, depth, start_url_parsed,
-                                        blacklist, i))
+                    {
+                      DEBUGP (("Not following due to 'link inline' flag: %s\n", child->url->url));
+                      continue;
+                    }
+
+                  r = download_child (child, url_parsed, depth,
+                                      start_url_parsed, blacklist, i);
+                  if (r == WG_RR_SUCCESS)
                    {
                      ci = iri_new ();
                      set_uri_encoding (ci, i->content_encoding, false);
@@ -561,7 +478,11 @@ retry:
                      /* We blacklist the URL we have enqueued, because we
                         don't want to enqueue (and hence download) the
                         same URL twice.  */
-                      string_set_add (blacklist, child->url->url);
+                      blacklist_add (blacklist, child->url->url);
+                    }
+                  else
+                    {
+                      write_reject_log_reason (rejectedlog, r, child->url, url_parsed);
                    }
                }

@@ -574,7 +495,7 @@ retry:

      if (file
          && (opt.delete_after
-              || opt.spider /* opt.recursive is implicitely true */
+              || opt.spider /* opt.recursive is implicitly true */
              || !acceptable (file)))
        {
          /* Either --delete-after was specified, or we loaded this
@@ -595,14 +516,16 @@ retry:
          logputs (LOG_VERBOSE, "\n");
          register_delete_file (file);
        }
-#ifndef ENABLE_THREADS
+
      xfree (url);
-      xfree_null (referer);
-      xfree_null (file);
+      xfree (referer);
+      xfree (file);
      iri_free (i);
-#endif
    }

+  if (rejectedlog)
+    fclose (rejectedlog);
+
  /* If anything is left of the queue due to a premature exit, free it
     now.  */
  {
@@ -615,7 +538,7 @@ retry:
      {
        iri_free (d6);
        xfree (d1);
-        xfree_null (d2);
+        xfree (d2);
      }
  }
  url_queue_delete (queue);
@@ -638,27 +561,29 @@ retry:
   by storing these URLs to BLACKLIST.  This may or may not help.  It
   will help if those URLs are encountered many times.  */

-static bool
-download_child_p (const struct urlpos *upos, struct url *parent, int depth,
+static reject_reason
+download_child (const struct urlpos *upos, struct url *parent, int depth,
                  struct url *start_url_parsed, struct hash_table *blacklist,
                  struct iri *iri)
 {
  struct url *u = upos->url;
  const char *url = u->url;
  bool u_scheme_like_http;
+  reject_reason reason = WG_RR_SUCCESS;

  DEBUGP (("Deciding whether to enqueue \"%s\".\n", url));

-  if (string_set_contains (blacklist, url))
+  if (blacklist_contains (blacklist, url))
    {
      if (opt.spider)
        {
          char *referrer = url_string (parent, URL_AUTH_HIDE_PASSWD);
-          DEBUGP (("download_child_p: parent->url is: %s\n", quote (parent->url)));
+          DEBUGP (("download_child: parent->url is: %s\n", quote (parent->url)));
          visited_url (url, referrer);
          xfree (referrer);
        }
      DEBUGP (("Already on the black list.\n"));
+      reason = WG_RR_BLACKLIST;
      goto out;
    }

@@ -688,6 +613,7 @@ download_child_p (const struct urlpos *upos, struct url *parent, int depth,
  if (opt.https_only && u->scheme != SCHEME_HTTPS)
    {
      DEBUGP (("Not following non-HTTPS links.\n"));
+      reason = WG_RR_NOTHTTPS;
      goto out;
    }
 #endif
@@ -696,9 +622,14 @@ download_child_p (const struct urlpos *upos, struct url *parent, int depth,
  u_scheme_like_http = schemes_are_similar_p (u->scheme, SCHEME_HTTP);

  /* 1. Schemes other than HTTP are normally not recursed into. */
-  if (!u_scheme_like_http && !(u->scheme == SCHEME_FTP && opt.follow_ftp))
+  if (!u_scheme_like_http && !((u->scheme == SCHEME_FTP
+#ifdef HAVE_SSL
+      || u->scheme == SCHEME_FTPS
+#endif
+      ) && opt.follow_ftp))
    {
      DEBUGP (("Not following non-HTTP schemes.\n"));
+      reason = WG_RR_NONHTTP;
      goto out;
    }

@@ -708,6 +639,7 @@ download_child_p (const struct urlpos *upos, struct url *parent, int depth,
    if (opt.relative_only && !upos->link_relative_p)
      {
        DEBUGP (("It doesn't really look like a relative link.\n"));
+        reason = WG_RR_ABSOLUTE;
        goto out;
      }

@@ -716,6 +648,7 @@ download_child_p (const struct urlpos *upos, struct url *parent, int depth,
  if (!accept_domain (u))
    {
      DEBUGP (("The domain was not accepted.\n"));
+      reason = WG_RR_DOMAIN;
      goto out;
    }

@@ -735,6 +668,7 @@ download_child_p (const struct urlpos *upos, struct url *parent, int depth,
        {
          DEBUGP (("Going to \"%s\" would escape \"%s\" with no_parent on.\n",
                   u->dir, start_url_parsed->dir));
+          reason = WG_RR_PARENT;
          goto out;
        }
    }
@@ -747,12 +681,14 @@ download_child_p (const struct urlpos *upos, struct url *parent, int depth,
      if (!accdir (u->dir))
        {
          DEBUGP (("%s (%s) is excluded/not-included.\n", url, u->dir));
+          reason = WG_RR_LIST;
          goto out;
        }
    }
  if (!accept_url (url))
    {
      DEBUGP (("%s is excluded/not-included through regex.\n", url));
+      reason = WG_RR_REGEX;
      goto out;
    }

@@ -760,7 +696,7 @@ download_child_p (const struct urlpos *upos, struct url *parent, int depth,
     for directories (no file name to match) and for non-leaf HTMLs,
     which can lead to other files that do need to be downloaded.  (-p
     automatically implies non-leaf because with -p we can, if
-     necesary, overstep the maximum depth to get the page requisites.)  */
+     necessary, overstep the maximum depth to get the page requisites.)  */
  if (u->file[0] != '\0'
      && !(has_html_suffix_p (u->file)
           /* The exception only applies to non-leaf HTMLs (but -p
@@ -777,6 +713,7 @@ download_child_p (const struct urlpos *upos, struct url *parent, int depth,
        {
          DEBUGP (("%s (%s) does not match acc/rej rules.\n",
                   url, u->file));
+          reason = WG_RR_RULES;
          goto out;
        }
    }
@@ -787,6 +724,7 @@ download_child_p (const struct urlpos *upos, struct url *parent, int depth,
      {
        DEBUGP (("This is not the same hostname as the parent's (%s and %s).\n",
                 u->host, parent->host));
+        reason = WG_RR_SPANNEDHOST;
        goto out;
      }

@@ -828,36 +766,37 @@ download_child_p (const struct urlpos *upos, struct url *parent, int depth,
      if (!res_match_path (specs, u->path))
        {
          DEBUGP (("Not following %s because robots.txt forbids it.\n", url));
-          string_set_add (blacklist, url);
+          blacklist_add (blacklist, url);
+          reason = WG_RR_ROBOTS;
          goto out;
        }
    }

-  /* The URL has passed all the tests.  It can be placed in the
-     download queue. */
-  DEBUGP (("Decided to load it.\n"));
+  out:

-  return true;
+  if (reason == WG_RR_SUCCESS)
+    /* The URL has passed all the tests.  It can be placed in the
+       download queue. */
+    DEBUGP (("Decided to load it.\n"));
+  else
+    DEBUGP (("Decided NOT to load it.\n"));

- out:
-  DEBUGP (("Decided NOT to load it.\n"));
-
-  return false;
+  return reason;
 }

 /* This function determines whether we will consider downloading the
   children of a URL whose download resulted in a redirection,
   possibly to another host, etc.  It is needed very rarely, and thus
-   it is merely a simple-minded wrapper around download_child_p.  */
+   it is merely a simple-minded wrapper around download_child.  */

-static bool
-descend_redirect_p (const char *redirected, struct url *orig_parsed, int depth,
+static reject_reason
+descend_redirect (const char *redirected, struct url *orig_parsed, int depth,
                    struct url *start_url_parsed, struct hash_table *blacklist,
                    struct iri *iri)
 {
  struct url *new_parsed;
  struct urlpos *upos;
-  bool success;
+  reject_reason reason;

  assert (orig_parsed != NULL);

@@ -867,16 +806,110 @@ descend_redirect_p (const char *redirected, struct url *orig_parsed, int depth,
  upos = xnew0 (struct urlpos);
  upos->url = new_parsed;

-  success = download_child_p (upos, orig_parsed, depth,
+  reason = download_child (upos, orig_parsed, depth,
                              start_url_parsed, blacklist, iri);

+  if (reason == WG_RR_SUCCESS)
+    blacklist_add (blacklist, upos->url->url);
+  else if (reason == WG_RR_LIST || reason == WG_RR_REGEX)
+    {
+      DEBUGP (("Ignoring decision for redirects, decided to load it.\n"));
+      blacklist_add (blacklist, upos->url->url);
+      reason = WG_RR_SUCCESS;
+    }
+  else
+    DEBUGP (("Redirection \"%s\" failed the test.\n", redirected));
+
  url_free (new_parsed);
  xfree (upos);

-  if (!success)
-    DEBUGP (("Redirection \"%s\" failed the test.\n", redirected));
+  return reason;
+}

-  return success;
+
+/* This function writes the rejected log header. */
+static void
+write_reject_log_header (FILE *f)
+{
+  if (!f)
+    return;
+
+  /* Note: Update this header when columns change in any way. */
+  fprintf (f, "REASON\t"
+    "U_URL\tU_SCHEME\tU_HOST\tU_PORT\tU_PATH\tU_PARAMS\tU_QUERY\tU_FRAGMENT\t"
+    "P_URL\tP_SCHEME\tP_HOST\tP_PORT\tP_PATH\tP_PARAMS\tP_QUERY\tP_FRAGMENT\n");
+}
+
+/* This function writes a URL to the reject log. Internal use only. */
+static void
+write_reject_log_url (FILE *fp, const struct url *url)
+{
+  const char *escaped_str;
+  const char *scheme_str;
+
+  if (!fp)
+    return;
+
+  escaped_str = url_escape (url->url);
+
+  switch (url->scheme)
+    {
+      case SCHEME_HTTP:  scheme_str = "SCHEME_HTTP";    break;
+#ifdef HAVE_SSL
+      case SCHEME_HTTPS: scheme_str = "SCHEME_HTTPS";   break;
+      case SCHEME_FTPS:  scheme_str = "SCHEME_FTPS";    break;
+#endif
+      case SCHEME_FTP:   scheme_str = "SCHEME_FTP";     break;
+      default:           scheme_str = "SCHEME_INVALID"; break;
+    }
+
+  fprintf (fp, "%s\t%s\t%s\t%i\t%s\t%s\t%s\t%s",
+    escaped_str,
+    scheme_str,
+    url->host,
+    url->port,
+    url->path,
+    url->params ? url->params : "",
+    url->query ? url->query : "",
+    url->fragment ? url->fragment : "");
+
+  xfree (escaped_str);
+}
+
+/* This function writes out information on why a URL was rejected and its
+   context from download_child such as the URL being rejected and it's
+   parent's URL. The format it uses is comma separated values but with tabs. */
+static void
+write_reject_log_reason (FILE *fp, reject_reason reason,
+                         const struct url *url, const struct url *parent)
+{
+  const char *reason_str;
+
+  if (!fp)
+    return;
+
+  switch (reason)
+    {
+      case WG_RR_SUCCESS:     reason_str = "SUCCESS";     break;
+      case WG_RR_BLACKLIST:   reason_str = "BLACKLIST";   break;
+      case WG_RR_NOTHTTPS:    reason_str = "NOTHTTPS";    break;
+      case WG_RR_NONHTTP:     reason_str = "NONHTTP";     break;
+      case WG_RR_ABSOLUTE:    reason_str = "ABSOLUTE";    break;
+      case WG_RR_DOMAIN:      reason_str = "DOMAIN";      break;
+      case WG_RR_PARENT:      reason_str = "PARENT";      break;
+      case WG_RR_LIST:        reason_str = "LIST";        break;
+      case WG_RR_REGEX:       reason_str = "REGEX";       break;
+      case WG_RR_RULES:       reason_str = "RULES";       break;
+      case WG_RR_SPANNEDHOST: reason_str = "SPANNEDHOST"; break;
+      case WG_RR_ROBOTS:      reason_str = "ROBOTS";      break;
+      default:                reason_str = "UNKNOWN";     break;
+    }
+
+  fprintf (fp, "%s\t", reason_str);
+  write_reject_log_url (fp, url);
+  fprintf (fp, "\t");
+  write_reject_log_url (fp, parent);
+  fprintf (fp, "\n");
 }

 /* vim:set sts=2 sw=2 cino+={s: */
--- a/Show More
+++ b/Show More