* src/utils.c: Move macro FMT_MAX_LENGTH into scope

* bootstrap.conf: Add intprops, inttypes, limits-h, signal-h,
  stat, sys_types
* src/ftp.c: Replace 'struct_stat' by 'struct stat'
* src/hsts.c: Likewise
* src/http.c: Likewise
* src/main.c: Likewise
* src/netrc.c: Likewise
* src/retr.c: Likewise
* src/url.c: Likewise
* src/utils.c: Likewise
* src/sysdep.h: Remove old portability code

Further portability issues should be addressed by gnulib.

.travis.yml

@@ -36,7 +36,7 @@ addons:
             - make
             - libhttp-daemon-perl
             - libio-socket-ssl-perl
-            - libidn11-dev
+            - libidn2-dev
             - gettext
             - texlive
             - python3

ChangeLog-2014-12-10

@@ -729,7 +729,7 @@
 	* testenv/Makefile.am: Run the tests in Python's Optimizedmode
 	* testenv/conf/__init__.py (gen_hook): Use try..except instead of if..else
 	* testenv/misc/color_terminal.py: System and check will not change while a test is
-	run. Do not test for them on every invokation of printer()
+	run. Do not test for them on every invocation of printer()
 	* testenv/server/http/http_server.py: The ssl and re modules are required by
 	specific functions. Load them lazily
 	(HTTPSServer.__init__): Lazy load ssl module here

Makefile.am

@@ -77,3 +77,6 @@ gen-ChangeLog:
 	    { rm -f $(distdir)/ChangeLog &&				\
 	      mv $(distdir)/cl-t $(distdir)/ChangeLog; }		\
 	fi
+
+check-valgrind:
+	TESTS_ENVIRONMENT="VALGRIND_TESTS=1" $(MAKE) check

NEWS

@@ -1,12 +1,57 @@
 GNU Wget NEWS -- history of user-visible changes.
 
-Copyright (C) 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004, 2005,
-2006, 2007, 2008, 2009, 2010, 2011, 2012, 2013, 2014, 2015 Free
-Software Foundation, Inc.
+Copyright (C) 1997-2017 Free Software Foundation, Inc.
 See the end for copying conditions.
 
 Please send GNU Wget bug reports to <bug-wget@gnu.org>.
 
+* Changes in Wget 1.19
+
+* New option --use-askpass=COMMAND. Fetch user/password by calling
+  an external program.
+
+* Use IDNA2008 (+ TR46 if available) through libidn2
+
+* When processing a Metalink header, --metalink-index=<number> allows
+  to process the header's application/metalink4+xml files.
+
+* When processing a Metalink file, --trust-server-names enables the
+  use of the destination file names specified in the Metalink file,
+  otherwise a safe destination file name is computed.
+
+* When processing a Metalink file, enforce a safe destination path.
+  Remove any drive letter prefix under w32, i.e. 'C:D:file'.  Call
+  libmetalink's metalink_check_safe_path() to prevent absolute,
+  relative, or home paths:
+  https://tools.ietf.org/html/rfc5854#section-4.1.2.1
+  https://tools.ietf.org/html/rfc5854#section-4.2.8.3
+
+* When processing a Metalink file, --directory-prefix=<prefix> sets
+  the top of the retrieval tree to prefix for Metalink downloads.
+
+* When processing a Metalink file, reject downloaded files which don't
+  agree with their own metalink:size value:
+  https://tools.ietf.org/html/rfc5854#section-4.2.16
+
+* When processing a Metalink file, with --continue resume partially
+  downloaded files and keep fully downloaded files even if they fail
+  the verification.
+
+* When processing a Metalink file, create the parent directories of a
+  "path/file" destination file name:
+  https://tools.ietf.org/html/rfc5854#section-4.1.2.1
+  https://tools.ietf.org/html/rfc5854#section-4.2.8.3
+
+* On a recursive download, append a .tmp suffix to temporary files
+  that will be deleted after being parsed, and create them
+  readable/writable only by the owner.
+
+* New make target 'check-valgrind'
+
+* Fix several bugs
+
+* Fix compatibility issues
+
 * Changes in Wget 1.18
 
 * By default, on server redirects to a FTP resource, use the original

README.checkout

@@ -4,7 +4,7 @@ Compiling From Repository Sources
    does not contain automatically-generated files, even when these are
    normally present in the distribution tarballs. Therefore, to build GNU
    Wget from the sources in the repository, you'll need to have one or
-   more of the following (note that gettext, OpenSSL, GnuTLS, libidn,
+   more of the following (note that gettext, OpenSSL, GnuTLS, libidn2,
    libiconv, libpsl, libpcre, pkg-config, libmetalink and GnuPG are not
    absolutely required):
 
@@ -32,8 +32,8 @@ Compiling From Repository Sources
        First download [25]CPAN and [26]install it, then try again the above command.
        Now "make check" should pass most of the tests in the test suite.
 
-     * [45]Python, if you want to run the tests in the testenv/ subdirectory.
-       Keep in mind that "make check" will try to run all the perl and python
+     * [45]Python3, if you want to run the tests in the testenv/ subdirectory.
+       Keep in mind that "make check" will try to run all the Perl and Python3
        tests. More information about the test suite below in the section
        "Testing and development".
 
@@ -69,7 +69,7 @@ Compiling From Repository Sources
        If you want to compile+link a non-system library version use
        --with-libssl-prefix (or if having pkg-config: see description below).
 
-     * [31]libidn is required for IDN/IRI support (non-ASCII characters
+     * [31]libidn2 is required for IDN/IRI support (non-ASCII characters
        within what would otherwise be URLs).
 
      * [32]libiconv is required on non-GNU systems, for IDN/IRI support.
@@ -159,7 +159,7 @@ Testing and development
 
  Copyright (C) 2008,2010,2014,2015 Free Software Foundation, Inc.
  License GPLv3+: GNU GPL version 3 or later
- <http://www.gnu.org/licenses/gpl.html>.
+ <https://www.gnu.org/licenses/gpl.html>.
  This is free software: you are free to change and redistribute it.
  There is NO WARRANTY, to the extent permitted by law.
 
@@ -183,32 +183,32 @@ Testing and development
 
 References
 
-  20. http://www.gnu.org/software/autoconf/
-  21. http://www.gnu.org/software/automake/
-  22. http://flex.sourceforge.net/
-  23. http://www.perl.org/
+  20. https://www.gnu.org/software/autoconf/
+  21. https://www.gnu.org/software/automake/
+  22. https://github.com/westes/flex
+  23. https://www.perl.org/
   24. http://search.cpan.org/dist/libwww-perl/lib/Bundle/LWP.pm
   25. http://search.cpan.org/CPAN/authors/id/A/AN/ANDK/CPAN-1.9402.tar.gz
   26. http://apthorpe.cynistar.net/code/configuring_cpan.html
-  27. http://www.gnu.org/software/texinfo/
-  28. http://www.gnu.org/software/gettext/
-  29. http://www.gnu.org/software/gnutls/
-  30. http://www.openssl.org/
-  31. http://www.gnu.org/software/libidn/
-  32. http://www.gnu.org/software/libiconv/
-  33. http://git-scm.com/
+  27. https://www.gnu.org/software/texinfo/
+  28. https://www.gnu.org/software/gettext/
+  29. https://www.gnu.org/software/gnutls/
+  30. https://www.openssl.org/
+  31. https://www.gnu.org/software/libidn/#libidn2
+  32. https://www.gnu.org/software/libiconv/
+  33. https://git-scm.com/
   34. https://github.com/rockdaboot/libpsl
   35. http://www.pcre.org
   36. http://wget.addictivecode.org/CompilingRepoSources?action=edit&editor=text
   37. http://wget.addictivecode.org/CompilingRepoSources
   38. http://wget.addictivecode.org/CompilingRepoSources?action=info
   39. http://wget.addictivecode.org/CompilingRepoSources?action=AttachFile
-  40. http://moinmo.in/
-  41. http://moinmo.in/Python
-  42. http://moinmo.in/GPL
-  43. http://validator.w3.org/check?uri=referer
+  40. https://moinmo.in/
+  41. https://moinmo.in/Python
+  42. https://moinmo.in/GPL
+  43. https://validator.w3.org/check?uri=referer
   44. http://wget.addictivecode.org/WikiLicense
   45. https://www.python.org/
   46. https://launchpad.net/libmetalink
   47. https://www.gnupg.org
-  48. http://c-ares.haxx.se/
+  48. https://c-ares.haxx.se/

bootstrap

@@ -1,10 +1,10 @@
 #! /bin/sh
 # Print a version string.
-scriptversion=2013-12-05.23; # UTC
+scriptversion=2016-01-24.06; # UTC
 
 # Bootstrap this package from checked-out sources.
 
-# Copyright (C) 2003-2014 Free Software Foundation, Inc.
+# Copyright (C) 2003-2016 Free Software Foundation, Inc.
 
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -42,6 +42,9 @@ export LC_ALL
 
 local_gl_dir=gl
 
+# Honor $PERL, but work even if there is none.
+PERL="${PERL-perl}"
+
 me=$0
 
 usage() {
@@ -456,6 +459,7 @@ check_versions() {
     test "$appvar" = TAR && appvar=AMTAR
     case $appvar in
         GZIP) ;; # Do not use $GZIP:  it contains gzip options.
+        PERL::*) ;; # Keep perl modules as-is
         *) eval "app=\${$appvar-$app}" ;;
     esac
 
@@ -473,6 +477,17 @@ check_versions() {
           ret=1
           continue
         } ;;
+      # Another check is for perl modules.  These can be written as
+      # e.g. perl::XML::XPath in case of XML::XPath module, etc.
+      perl::*)
+        # Extract module name
+        app="${app#perl::}"
+        if ! $PERL -m"$app" -e 'exit 0' >/dev/null 2>&1; then
+          warn_ "Error: perl module '$app' not found"
+          ret=1
+        fi
+        continue
+        ;;
     esac
     if [ "$req_ver" = "-" ]; then
       # Merely require app to exist; not all prereq apps are well-behaved
@@ -900,7 +915,8 @@ if test $use_libtool = 1; then
   esac
 fi
 echo "$0: $gnulib_tool $gnulib_tool_options --import ..."
-$gnulib_tool $gnulib_tool_options --import $gnulib_modules &&
+$gnulib_tool $gnulib_tool_options --import $gnulib_modules \
+  || die "gnulib-tool failed"
 
 for file in $gnulib_files; do
   symlink_to_dir "$GNULIB_SRCDIR" $file \
@@ -1005,6 +1021,6 @@ echo "$0: done.  Now you can run './configure'."
 # eval: (add-hook 'write-file-hooks 'time-stamp)
 # time-stamp-start: "scriptversion="
 # time-stamp-format: "%:y-%02m-%02d.%02H"
-# time-stamp-time-zone: "UTC"
+# time-stamp-time-zone: "UTC0"
 # time-stamp-end: "; # UTC"
 # End:

bootstrap.conf

@@ -40,6 +40,7 @@ dirname
 fcntl
 flock
 fnmatch
+fopen
 futimens
 ftello
 getaddrinfo
@@ -52,9 +53,16 @@ gettext
 gitlog-to-changelog
 git-version-gen
 gnupload
+hostent
 ioctl
 iconv
 iconv-h
+inet_ntop
+intprops
+inttypes
+langinfo
+limits-h
+link
 listen
 maintainer-makefile
 mbiter
@@ -63,9 +71,16 @@ memrchr
 mkdir
 mkstemp
 mkostemp
+nanosleep
+crypto/md2
+crypto/md4
 crypto/md5
 crypto/sha1
 crypto/sha256
+crypto/sha512
+open
+pipe-posix
+posix_spawn
 quote
 quotearg
 recv
@@ -73,24 +88,36 @@ regex
 select
 send
 setsockopt
+signal-h
 sigprocmask
 sigpipe
 snprintf
 socket
 spawn-pipe
+stat
 stdbool
+stdint
 strcase
 strerror_r-posix
+strpbrk
 strptime
 strtok_r
 strtoll
+symlink
+sys_types
 timegm
 tmpdir
+unlink
 unlocked-io
 update-copyright
+libunistring-optional
+unistr/base
+unicase/tolower
+utimens
 vasprintf
 vsnprintf
 write
+xmemdup0
 xstrndup
 "
 

cfg.mk

@@ -26,4 +26,4 @@ local-checks-to-skip = \
 show-vc-list-except:
 	@$(VC_LIST_EXCEPT)
 
-VC_LIST_ALWAYS_EXCLUDE_REGEX = ^ABOUT-NLS|((msdos|vms)/.*)|ChangeLog-2014-12-10$$
+VC_LIST_ALWAYS_EXCLUDE_REGEX = ^ABOUT-NLS|((msdos|vms)/.*)|ChangeLog-2014-12-10|/.*.der$$

configure.ac

@@ -290,15 +290,6 @@ AC_DEFINE([HAVE_STRNCASECMP], 1, [Define to 1 if you have the `strncasecmp' func
 AC_DEFINE([HAVE_STRDUP], 1, [Define to 1 if you have the `strdup' function.])
 AC_DEFINE([HAVE_ISATTY], 1, [Define to 1 if you have the `isatty' function.])
 
-dnl
-dnl Call Wget-specific macros defined in aclocal.
-dnl
-WGET_STRUCT_UTIMBUF
-WGET_FNMATCH
-WGET_NANOSLEEP
-WGET_POSIX_CLOCK
-WGET_NSL_SOCKET
-
 dnl Deal with specific hosts
 case $host_os in
   *mingw32* )
@@ -329,6 +320,10 @@ AS_IF([test "x$with_libpsl" != xno], [
   ])
 ])
 
+AS_IF([test x"$with_libpsl" = xyes], [
+  AC_CHECK_FUNCS(psl_latest)
+])
+
 AS_IF([test x"$with_zlib" != xno], [
   with_zlib=yes
   PKG_CHECK_MODULES([ZLIB], zlib, [
@@ -635,9 +630,6 @@ AC_ARG_ENABLE(iri,
   ]
 )
 
-AC_ARG_WITH(libidn, AC_HELP_STRING([--with-libidn=[DIR]],
-                                   [Support IDN/IRIs (needs GNU Libidn)]),
-                                   libidn=$withval, libidn="")
 AS_IF([test "X$iri" != "Xno"],[
   AM_ICONV
 
@@ -655,33 +647,26 @@ AS_IF([test "X$iri" != "Xno"],[
   LIBICONV=
 ])
 
+AC_ARG_WITH(libidn, AC_HELP_STRING([--with-libidn=[DIR]],
+                                   [Support IDN2008/IRIs (needs GNU libidn2 + libunicode)]),
+                                   libidn=$withval, libidn="")
 if test "X$iri" != "Xno"; then
-  if test "$libidn" != ""; then
-    LDFLAGS="${LDFLAGS} -L$libidn/lib"
-    CPPFLAGS="${CPPFLAGS} -I$libidn/include"
-  fi
+  AS_IF([test "x$with_libidn2" != xno], [
+    AC_SEARCH_LIBS(idn2_lookup_u8, idn2,
+      [with_libidn2=yes; AC_DEFINE([ENABLE_IRI], 1, [Define if IRI support is enabled.])],
+      [with_libidn2=no; iri=no; AC_MSG_WARN(*** LIBIDN2 was not found. You will not be able to use IDN2008 support)])
 
-  # If idna.h can't be found, check to see if it was installed under
-  # /usr/include/idn (OpenSolaris, at least, places it there).
-  # Check for idn-int.h in that case, because idna.h won't find
-  # idn-int.h until we've decided to add -I/usr/include/idn.
-  AC_CHECK_HEADER(idna.h, ,
-    [AC_CHECK_HEADER(idn/idn-int.h,
-                    [CPPFLAGS="${CPPFLAGS} -I/usr/include/idn"],
-                    [iri=no])]
-  )
-
-  if test "X$iri" != "Xno"; then
-    AC_CHECK_LIB(idn, stringprep_check_version,
-      [iri=yes LIBS="${LIBS} -lidn"], iri=no)
-  fi
-
-  if test "X$iri" != "Xno" ; then
-    AC_DEFINE([ENABLE_IRI], 1, [Define if IRI support is enabled.])
-    AC_MSG_NOTICE([Enabling support for IRI.])
-  else
-    AC_MSG_WARN([Libidn not found])
-  fi
+#    AS_IF([test "x$with_libidn2" = xyes], [
+#      AC_SEARCH_LIBS(u8_tolower, unistring,
+#        [AC_DEFINE([ENABLE_IRI], 1, [Define if IRI support is enabled.])],
+#        [iri=no; AC_MSG_WARN(*** LIBUNISTRING was not found. You will not be able to use IDN2008 support)])
+#    ])
+  ])
+fi
+if test "X$iri" = "Xno"; then
+  # we don't need libunistring - clear settings from gnulib module
+  LIBUNISTRING=""
+  LTLIBUNISTRING=""
 fi
 
 dnl
@@ -750,7 +735,7 @@ dnl
 dnl Check for libcares (resolver library)
 dnl
 
-AS_IF([test "X$with_cares" == "Xyes"],[
+AS_IF([test "X$with_cares" = "Xyes"],[
   PKG_CHECK_MODULES([CARES], libcares, [
     CFLAGS="$CARES_CFLAGS $CFLAGS"
     AC_CHECK_HEADER(ares.h, [
@@ -771,11 +756,37 @@ AS_IF([test "X$with_cares" == "Xyes"],[
   RESOLVER_INFO="libc, --bind-dns-address and --dns-servers not available"
 ])
 
+dnl
+dnl Extended Attribute support
+dnl
+
+AC_ARG_ENABLE([xattr],
+  [AS_HELP_STRING([--disable-xattr], [disable support for POSIX Extended Attributes])],
+  [ENABLE_XATTR=$enableval],
+  [ENABLE_XATTR=yes])
+
+case "$host_os" in
+  *linux* | *darwin*) xattr_syscalls="fsetxattr" ;;
+  freebsd*)           xattr_syscalls="extattr_set_fd" ;;
+  *)  AC_MSG_NOTICE([Disabling Extended Attribute support: your system is not known to support extended attributes.])
+      ENABLE_XATTR=no
+esac
+
+if test "X${ENABLE_XATTR}" = "Xyes"; then
+  AC_CHECK_FUNCS([$xattr_syscalls], [], [
+    AC_MSG_NOTICE([Disabling Extended Attribute support: your system does not support $xattr_syscalls])
+    ENABLE_XATTR=no
+  ])
+fi
+
+test "X${ENABLE_XATTR}" = "Xyes" && AC_DEFINE([ENABLE_XATTR], 1,
+    [Define if you want file meta-data storing into POSIX Extended Attributes compiled in.])
 
 dnl Needed by src/Makefile.am
 AM_CONDITIONAL([IRI_IS_ENABLED], [test "X$iri" != "Xno"])
 AM_CONDITIONAL([WITH_SSL], [test "X$with_ssl" != "Xno"])
 AM_CONDITIONAL([METALINK_IS_ENABLED], [test "X$with_metalink" != "Xno"])
+AM_CONDITIONAL([WITH_XATTR], [test "X$ENABLE_XATTR" != "Xno"])
 
 dnl
 dnl Create output
@@ -801,10 +812,12 @@ AC_MSG_NOTICE([Summary of build options:
   Digest:            $ENABLE_DIGEST
   NTLM:              $ENABLE_NTLM
   OPIE:              $ENABLE_OPIE
+  POSIX xattr:       $ENABLE_XATTR
   Debugging:         $ENABLE_DEBUG
   Assertions:        $ENABLE_ASSERTION
   Valgrind:          $VALGRIND_INFO
   Metalink:          $with_metalink
   Resolver:          $RESOLVER_INFO
   GPGME:             $have_gpg
+  IRI:               $iri
 ])

contrib/check-hard

@@ -12,7 +12,7 @@ if [ $CORES -lt 1 ]; then CORES=1; fi
 echo "Running: make distclean"
 make distclean > /dev/null || true
 
-CFLAGS="-std=c89 -pedantic -O2 -g -Wall -Wextra -Wstrict-prototypes -Wold-style-definition -Wwrite-strings -Wshadow -Wformat -Wformat-security -Wunreachable-code -Wstrict-prototypes -Wmissing-prototypes -Wold-style-definition"
+CFLAGS="-std=c89 -pedantic -O0 -g -Wall -Wextra -Wstrict-prototypes -Wold-style-definition -Wwrite-strings -Wshadow -Wformat -Wformat-security -Wunreachable-code -Wstrict-prototypes -Wmissing-prototypes -Wold-style-definition"
 
 CACHEFILE=$PWD/config_check.cache
 

contrib/travis-ci

@@ -21,7 +21,7 @@ CORES=3
 
 # Define a large number of Warning flags for the compiler. Hopefully, someone
 # will sit and analyze the output to clean the warnings from the codebase.
-CFLAGS="-std=c89 -pedantic -O2 -Wall -Wextra -Wstrict-prototypes -Wold-style-definition -Wwrite-strings -Wshadow -Wformat -Wformat-security -Wunreachable-code -Wstrict-prototypes -Wmissing-prototypes -Wold-style-definition"
+CFLAGS="-std=c89 -pedantic -O0 -Wall -Wextra -Wstrict-prototypes -Wold-style-definition -Wwrite-strings -Wshadow -Wformat -Wformat-security -Wunreachable-code -Wstrict-prototypes -Wmissing-prototypes -Wold-style-definition"
 
 # A cachefile for ./configure. Saves significant time when reconfiguring
 CACHEFILE=$PWD/config_check.cache

doc/Makefile.am

@@ -57,10 +57,8 @@ wget.pod: $(srcdir)/wget.texi version.texi
 	$(TEXI2POD) -D VERSION="$(VERSION)" $(srcdir)/wget.texi $@
 
 $(MAN): wget.pod
-	$(POD2MAN) --center="GNU Wget" --release="GNU Wget @VERSION@" --utf8 $? > $@; \
-	if [ $? != 0 ]; then \
-          $(POD2MAN) --center="GNU Wget" --release="GNU Wget @VERSION@" $? > $@; \
-	fi
+	$(POD2MAN) --center="GNU Wget" --release="GNU Wget @VERSION@" --utf8 $? > $@ || \
+	$(POD2MAN) --center="GNU Wget" --release="GNU Wget @VERSION@" $? > $@
 
 #wget.cat: $(MAN)
 #	nroff -man $? > $@

doc/metalink-standard.txt

@@ -0,0 +1,233 @@
+GNU Wget Metalink recommended behaviour
+
+  Metalink/XML and Metalink/HTTP standard reference
+
+
+1. Security features
+********************
+
+Only metalink:file elements with safe "name" fields shall be accepted
+[1 #section-4.1.2.1]. If unsafe metalink:file elements are saved, any
+related test shall fail (see '2. Tests').
+
+By design, libmetalink rejects unsafe metalink:file elements [3]:
+* lib/metalink_helper.c (metalink_check_safe_path): Verify path
+
+1.1 Exceptions
+==============
+
+The option --directory-prefix could allow to use an absolute, relative
+or home path.
+
+2. Tests
+********
+
+Saving a file to an unexpected path poses a security problem. We must
+ensure that Wget's automated tests never modify the root and the home
+paths or descend/escalate to a relative path unexpectedly.
+
+2.1 Metalink/XML implemented tests
+==================================
+
+See testenv/Makefile.am (METALINK_TESTS).
+
+2.2 Metalink/HTTP implemented tests
+===================================
+
+See testenv/Makefile.am (METALINK_TESTS).
+
+3. Download file name
+*********************
+
+The download file name shall be decided by precise rules which prevent
+any naming uncertainty and security issues.
+
+3.1 Naming rules
+================
+
+The final name of downloaded files is computed starting from a trusted
+name, which is then combined with the "Directory Options".  The result
+is verified and eventually made safer following security rules. If the
+final name isn't found safe enough, then the file isn't downloaded.
+
+Depending on the options used, a suffix could be appended to the final
+name to not overwrite existing files.
+
+3.1.1 The trusted name
+======================
+
+The option --trust-server-names decides what is the trusted name.
+
+Any Metalink/XML element with an unsafe metalink:file "name" field is
+ignored, see '1. Security features'.
+
+3.1.1.1 Without --trust-server-names
+====================================
+
+When --trust-server-names is off, the basename of the --input-metalink
+file, if available, or of the mother URL is trusted. This trusted name
+is the radix of any subsequent file name.
+
+When a Metalink/HTTP in encountered, any fetched Metalink/XML file has
+its own ordinal number appended as suffix to the trusted name. In this
+case scenario, an unique Metalink/XML file is saved each time applying
+an additional suffix to the currently computed name when necessary.
+
+The files described by a Metalink/XML file will be named sequentially
+applying an additional suffix to the currently trusted/computed name.
+
+3.1.1.2 With --trust-server-names
+=================================
+
+When --trust-server-names is on, the metalink:file "name" field parsed
+from Metalink/XML files is trusted. When no Metalink/XML is available,
+the mother URL is trusted.
+
+Any Metalink/HTTP application/metalink4+xml file is saved using the
+basename of its own Link header "name" field, if available.
+
+In conjunction with the option --content-disposition, a 'Content-Type:
+application/metalink4+xml' file is saved using the basename of its own
+Content-Disposition header "filename" field, if available.
+
+3.1.2 The final name
+====================
+
+The "Directory Options" are combined with the trusted name. The result
+is evaluated again by the '1. Security features'. If the path is found
+unsafe, only the basename of the final name is considered.  If this is
+found unsafe too, the file is not downloaded.
+
+4. Metalink/XML
+***************
+
+4.1 Example files
+=================
+
+See [1 #section-1.1].
+
+cat > bugus.meta4 << EOF
+<?xml version="1.0" encoding="UTF-8"?>
+<metalink xmlns="urn:ietf:params:xml:ns:metalink">
+  <file name="/dir/A/File1">
+    <size>1617</size>
+    <hash type="sha256">ecb3dff2648667513e31554b3ad054ccd89fce38e33367c9459ac3a285153742</hash>
+    <url>http://another.url/common_name</url>
+    <url>http://ftpmirror.gnu.org/bash/bash-4.3-patches/bash43-001</url>
+  </file>
+  <file name="dir/B/File2">
+    <size>1594</size>
+    <hash type="sha256">eee7cd7062ab29a9e4f02924d9c367264dcb8b162703f74ff6eb8f175a91502b</hash>
+    <url>http://another.url/again/common_name</url>
+    <url>http://ftpmirror.gnu.org/bash/bash-4.3-patches/bash43-002</url>
+  </file>
+</metalink>
+EOF
+
+4.2 Command line example
+========================
+
+$ wget --input-metalink=bogus.meta4
+
+4.3 Metalink/XML file parsing
+=============================
+
+The metalink xml file is parsed by one of the following libmetalink's
+functions [3], depending upon the library configured to use:
+* lib/libexpat_metalink_parser.c (metalink_parse_file): Expat [4]
+* lib/libxml2_metalink_parser.c (metalink_parse_file): Libxml2 [5]
+
+The result returned doesn't include unsafe metalink:file elements, as
+stated at point '1. Security features'.
+
+An empty result shall not be considered an error.  Parsing errors will
+be informed to the caller of libmetalink's metalink_parse_file().
+
+4.4 Saving files
+================
+
+Fetched metalink:file elements shall be wrote using the unique "name"
+field as file name [1 #section-4.1.2.1].
+
+A metalink:file url's file name shall not substitute the "name" field.
+
+Security exceptions are explained in '3. Download file name'.
+
+4.5 Multi-Source download
+=========================
+
+Parallel range requests are allowed [1 #section-1].
+
+5. Metalink/HTTP
+****************
+
+5.1 HTTP server
+===============
+
+The local server http://127.0.0.1 is used as reference in the course
+of this chapter. Any server service capable of sending Metalink/HTTP
+header answers may be used.
+
+5.2 Command line example
+========================
+
+$ wget --metalink-over-http http://127.0.0.1/dir/file.ext
+
+5.3 Metalink/HTTP header answer
+===============================
+
+See [2 #section-1.1].
+
+Etag: "thvDyvhfIqlvFe+A9MYgxAfm1q5="
+Link: <http://www2.example.com/example.ext>; rel=duplicate
+Link: <ftp://ftp.example.com/example.ext>; rel=duplicate
+Link: <http://example.com/example.ext.torrent>; rel=describedby;
+type="application/x-bittorrent"
+Link: <http://example.com/example.ext.meta4>; rel=describedby;
+type="application/metalink4+xml"
+Link: <http://example.com/example.ext.asc>; rel=describedby;
+type="application/pgp-signature"
+Digest: SHA-256=MWVkMWQxYTRiMzk5MDQ0MzI3NGU5NDEyZTk5OWY1ZGFmNzgyZTJlO
+DYzYjRjYzFhOTlmNTQwYzI2M2QwM2U2MQ==
+
+See [2 #section-4].
+
+Link: <http://example.com/example.ext.torrent>; rel=describedby;
+type="application/x-bittorrent"; name="differentname.ext"
+Link: <http://example.com/example.ext.meta4>; rel=describedby;
+type="application/metalink4+xml"
+
+5.4 Saving files
+================
+
+When none of --output-document and/or --content-disposition is used,
+the file name to wrote is computed from the cli's url hierarchy. The
+purpose of the "Directory Options" is as usual, and the file name is
+the cli's url file name, see wget(1).
+
+The url followed to download the file shall not substitute the cli's
+url to compute the file name to wrote, except when it redirects to a
+Metalink/XML file, following the rules in '3. Download file name'.
+
+5.5 Multi-Source download
+=========================
+
+Parallel range requests are allowed [2 #section-7].
+
+4. References
+*************
+
+[1] The Metalink Download Description Format
+https://tools.ietf.org/html/rfc5854
+
+[2] Metalink/HTTP: Mirrors and Hashes
+https://tools.ietf.org/html/rfc6249
+
+[3] Libmetalink
+https://github.com/metalink-dev/libmetalink
+
+[4] Expat
+http://www.libexpat.org
+
+[5] Libxml2
+http://xmlsoft.org

doc/metalink.txt

@@ -0,0 +1,165 @@
+GNU Wget Metalink module
+
+  Evaluation of the Metalink/XML and Metalink/HTTP implementations
+
+
+1. Introduction
+***************
+
+This document, and the results contained in it, is focused over the
+evaluation of the Metalink/XML and Metalink/HTTP implementations.
+
+The "Directory Options" mentioned here are used on the command line in
+conjunction with the option '--input-metalink=file' for Metalink/XML,
+and '--metalink-over-http' for Metalink/HTTP.
+
+$ wget --input-metalink=<file> [directory options]
+$ wget --metalink-over-http [directory options] <url>
+
+2. Notes
+********
+
+Tests for metalink:file names beginning with '/', '~/', './', or '../'
+(e.g. "/path/file") shall be run manually due to security concerns.
+
+3. Metalink files used as reference
+***********************************
+
+3.1 Test: metalink:file with "path/file" name format
+====================================================
+
+cat > test.meta4 << EOF
+<?xml version="1.0" encoding="UTF-8"?>
+<metalink xmlns="urn:ietf:params:xml:ns:metalink">
+  <file name="path/file">
+    <size>543</size>
+    <hash type="sha256">d37d3965f8e1a7b16504b4273b09c392776b7e4dd17e601256c7b2fd9ce5f56e</hash>
+    <hash type="md5">0f6ff5cdc15603f1b81227b5a296f001</hash>
+    <url>http://wrongurl.really/gnu/wget/wget-1.18.tar.xz.sig</url>
+    <url>http://ftpmirror.gnu.org/wget/wget-1.18.tar.xz.sig</url>
+    <url>http://ftp.gnu.org/gnu/wget/wget-1.18.tar.xz.sig</url>
+    <url>http://nl.mirror.babylon.network/gnu/wget/wget-1.18.tar.xz.sig</url>
+  </file>
+</metalink>
+EOF
+
+4. `wget --input-metalink=test.meta4`
+*************************************
+
+4.1 Implemented safety features
+===============================
+
+Any metalink:file name containing an absolute, relative, or home path
+(see '2. Notes') parsed from Metalink/XML files is rejected.
+
+This is a libmetalink's design decision implemented in the function
+metalink_check_safe_path().  This feature shall not be modified.
+
+All the above conform to the RFC5854 standard.
+
+References:
+ https://tools.ietf.org/html/rfc5854#section-4.1.2.1
+ https://tools.ietf.org/html/rfc5854#section-4.2.8.3
+
+4.2 File download behaviour
+===========================
+
+When a Metalink/XML file is parsed:
+1. create the metalink:file "path/file" tree;
+2. download the metalink:url file as "path/file";
+3. verify the "path/file" size, if declared;
+4. verify the "path/file" checksum.
+
+All the above conform to the RFC5854 standard.
+
+References:
+ https://tools.ietf.org/html/rfc5854
+
+4.3 Questionable behaviours
+===========================
+
+If more metalink:file elements are the same, wget downloads them all.
+
+5. `wget --metalink-over-http`
+******************************
+
+5.1 Implemented safety features
+===============================
+
+The function url_file_name() is responsible of parsing the url's file
+name and mixing in the "Directory Options" wrote on the command line.
+
+The use of libmetalink's metalink_check_safe_path() shouldn't be
+necessary (see '4.1 Implemented safety features').
+
+All the above comform to the usual Wget's download behaviour.
+
+References:
+ wget(1)
+
+5.2 File download behaviour
+===========================
+
+When a Metalink/HTTP header is parsed:
+1. extract metalink metadata from the header;
+2. download the file from the mirror with the highest priority;
+3. verify the file's size, if declared;
+4. verify the file's checksum.
+
+All the above comform to the usual Wget's download behaviour and to
+the RFC6249 standard.
+
+References:
+ wget(1)
+ https://tools.ietf.org/html/rfc6249
+
+6. Directory Options
+********************
+
+'-nd'
+'--no-directories'
+
+    Do not apply to Metalink/XML files (aka --input-metalink=<file>).
+
+    Apply to Metalink/HTTP urls as described in the Wget's manual, see
+    wget(1).  The target url is the url wrote on the command line.
+
+'-x'
+'--force-directories'
+
+    Do not apply to Metalink/XML files (aka --input-metalink=<file>).
+
+    Apply to Metalink/HTTP urls as described in the Wget's manual, see
+    wget(1).  The target url is the url wrote on the command line.
+
+'-nH'
+'--no-host-directories'
+
+    Do not apply to Metalink/XML files (aka --input-metalink=<file>).
+
+    Apply to Metalink/HTTP urls as described in the Wget's manual, see
+    wget(1).  The target url is the url wrote on the command line.
+
+'--protocol-directories'
+
+    Do not apply to Metalink/XML files (aka --input-metalink=<file>).
+
+    Apply to Metalink/HTTP urls as described in the Wget's manual, see
+    wget(1).  The target url is the url wrote on the command line.
+
+'--cut-dirs=number'
+
+    Do not apply to Metalink/XML files (aka --input-metalink=<file>).
+
+    Apply to Metalink/HTTP urls as described in the Wget's manual, see
+    wget(1).  The target url is the url wrote on the command line.
+
+'-P prefix'
+'--directory-prefix=prefix'
+
+    Set the top of the retrieval tree to prefix for both Metalink/XML
+    and Metalink/HTTP downloads, see wget(1).
+
+    If combining the prefix with the file name results in an absolute,
+    relative, or home path, the directory components are stripped and
+    only the basename is used. See '4.1 Implemented safety features'.

doc/wget.texi

@@ -512,11 +512,27 @@ href if none was specified.
 Downloads files covered in local Metalink @var{file}. Metalink version 3
 and 4 are supported.
 
+@cindex keep-badhash
+@item --keep-badhash
+Keeps downloaded Metalink's files with a bad hash. It appends .badhash
+to the name of Metalink's files which have a checksum mismatch, except
+without overwriting existing files.
+
 @cindex metalink-over-http
 @item --metalink-over-http
 Issues HTTP HEAD request instead of GET and extracts Metalink metadata
 from response headers. Then it switches to Metalink download.
 If no valid Metalink metadata is found, it falls back to ordinary HTTP download.
+Enables @samp{Content-Type: application/metalink4+xml} files download/processing.
+
+@cindex metalink-index
+@item --metalink-index=@var{number}
+Set the Metalink @samp{application/metalink4+xml} metaurl ordinal
+NUMBER. From 1 to the total number of ``application/metalink4+xml''
+available.  Specify 0 or @samp{inf} to choose the first good one.
+Metaurls, such as those from a @samp{--metalink-over-http}, may have
+been sorted by priority key's value; keep this in mind to choose the
+right NUMBER.
 
 @cindex preferred-location
 @item --preferred-location
@@ -802,7 +818,7 @@ With @samp{--progress=bar}, there are currently two possible parameters,
 @var{force} and @var{noscroll}.
 
 When the output is not a TTY, the progress bar always falls back to ``dot'',
-even if @samp{--progress=bar} was passed to Wget during invokation. This
+even if @samp{--progress=bar} was passed to Wget during invocation. This
 behaviour can be overridden and the ``bar'' output forced by using the ``force''
 parameter as @samp{--progress=bar:force}.
 
@@ -1154,6 +1170,15 @@ options for @sc{http} connections.
 Prompt for a password for each connection established. Cannot be specified
 when @samp{--password} is being used, because they are mutually exclusive.
 
+@item --use-askpass=@var{command}
+Prompt for a user and password using the specified command.  If no command is
+specified then the command in the environment variable WGET_ASKPASS is used.
+If WGET_ASKPASS is not set then the command in the environment variable
+SSH_ASKPASS is used.
+
+You can set the default command for use-askpass in the @file{.wgetrc}.  That
+setting may be overridden from the command line.
+
 @cindex iri support
 @cindex idn support
 @item --no-iri
@@ -1327,10 +1352,10 @@ the @code{digest}, or the Windows @code{NTLM} authentication scheme.
 Another way to specify username and password is in the @sc{url} itself
 (@pxref{URL Format}).  Either method reveals your password to anyone who
 bothers to run @code{ps}.  To prevent the passwords from being seen,
-store them in @file{.wgetrc} or @file{.netrc}, and make sure to protect
-those files from other users with @code{chmod}.  If the passwords are
-really important, do not leave them lying in those files either---edit
-the files and delete them after Wget has started the download.
+use the @samp{--use-askpass} or store them in @file{.wgetrc} or @file{.netrc},
+and make sure to protect those files from other users with @code{chmod}.  If
+the passwords are really important, do not leave them lying in those files
+either---edit the files and delete them after Wget has started the download.
 
 @iftex
 @xref{Security Considerations}, for more information about security
@@ -1662,6 +1687,10 @@ This option is useful for some file-downloading CGI programs that use
 @code{Content-Disposition} headers to describe what the name of a
 downloaded file should be.
 
+When combined with @samp{--metalink-over-http} and @samp{--trust-server-names},
+a @samp{Content-Type: application/metalink4+xml} file is named using the
+@code{Content-Disposition} filename field, if available.
+
 @cindex Content On Error
 @item --content-on-error
 
@@ -1671,9 +1700,11 @@ with a http status code that indicates error.
 @cindex Trust server names
 @item --trust-server-names
 
-If this is set to on, on a redirect the last component of the
-redirection URL will be used as the local file name.  By default it is
-used the last component in the original URL.
+If this is set, on a redirect, the local file name will be based
+on the redirection URL.  By default the local file name is based on
+the original URL.  When doing recursive retrieving this can be helpful
+because in many web sites redirected URLs correspond to an underlying
+file structure, while link URLs do not.
 
 @cindex authentication
 @item --auth-no-challenge
@@ -3199,8 +3230,8 @@ Turn on recognition of the (non-standard) @samp{Content-Disposition}
 HTTP header---if set to @samp{on}, the same as @samp{--content-disposition}.
 
 @item trust_server_names = on/off
-If set to on, use the last component of a redirection URL for the local
-file name.
+If set to on, construct the local file name from redirection URLs
+rather than original URLs.
 
 @item continue = on/off
 If set to on, force continuation of preexistent partially retrieved

m4/wget.m4

@@ -27,89 +27,6 @@ dnl Corresponding Source for a non-source form of such a combination
 dnl shall include the source code for the parts of OpenSSL used as well
 dnl as that of the covered work.
 
-dnl
-dnl Check for `struct utimbuf'.
-dnl
-
-AC_DEFUN([WGET_STRUCT_UTIMBUF], [
-  AC_CHECK_TYPES([struct utimbuf], [], [], [
-#include <stdio.h>
-#if HAVE_SYS_TYPES_H
-# include <sys/types.h>
-#endif
-#if HAVE_UTIME_H
-# include <utime.h>
-#endif
-  ])
-])
-
-dnl Check whether fnmatch.h can be included.  This doesn't use
-dnl AC_FUNC_FNMATCH because Wget is already careful to only use
-dnl fnmatch on certain OS'es.  However, fnmatch.h is sometimes broken
-dnl even on those because Apache installs its own fnmatch.h to
-dnl /usr/local/include (!), which GCC uses before /usr/include.
-
-AC_DEFUN([WGET_FNMATCH], [
-  AC_MSG_CHECKING([for working fnmatch.h])
-  AC_COMPILE_IFELSE([AC_LANG_SOURCE([#include <fnmatch.h>
-                    ])], [
-    AC_MSG_RESULT(yes)
-    AC_DEFINE([HAVE_WORKING_FNMATCH_H], 1,
-              [Define if fnmatch.h can be included.])
-  ], [
-    AC_MSG_RESULT(no)
-  ])
-])
-
-dnl Check for nanosleep.  For nanosleep to work on Solaris, we must
-dnl link with -lrt (recently) or with -lposix4 (older releases).
-
-AC_DEFUN([WGET_NANOSLEEP], [
-  AC_CHECK_FUNCS(nanosleep, [], [
-    AC_CHECK_LIB(rt, nanosleep, [
-      AC_DEFINE([HAVE_NANOSLEEP], 1,
-                [Define if you have the nanosleep function.])
-      LIBS="-lrt $LIBS"
-    ], [
-      AC_CHECK_LIB(posix4, nanosleep, [
-	AC_DEFINE([HAVE_NANOSLEEP], 1,
-		  [Define if you have the nanosleep function.])
-	LIBS="-lposix4 $LIBS"
-      ])
-    ])
-  ])
-])
-
-AC_DEFUN([WGET_POSIX_CLOCK], [
-  AC_CHECK_FUNCS(clock_gettime, [], [
-    AC_CHECK_LIB(rt, clock_gettime)
-  ])
-])
-
-dnl Check whether we need to link with -lnsl and -lsocket, as is the
-dnl case on e.g. Solaris.
-
-AC_DEFUN([WGET_NSL_SOCKET], [
-  dnl On Solaris, -lnsl is needed to use gethostbyname.  But checking
-  dnl for gethostbyname is not enough because on "NCR MP-RAS 3.0"
-  dnl gethostbyname is in libc, but -lnsl is still needed to use
-  dnl -lsocket, as well as for functions such as inet_ntoa.  We look
-  dnl for such known offenders and if one of them is not found, we
-  dnl check if -lnsl is needed.
-  wget_check_in_nsl=NONE
-  AC_CHECK_FUNCS(gethostbyname, [], [
-    wget_check_in_nsl=gethostbyname
-  ])
-  AC_CHECK_FUNCS(inet_ntoa, [], [
-    wget_check_in_nsl=inet_ntoa
-  ])
-  if test $wget_check_in_nsl != NONE; then
-    AC_CHECK_LIB(nsl, $wget_check_in_nsl)
-  fi
-  AC_CHECK_LIB(socket, socket)
-])
-
-
 dnl ************************************************************
 dnl START OF IPv6 AUTOCONFIGURATION SUPPORT MACROS
 dnl ************************************************************

src/Makefile.am

@@ -39,9 +39,12 @@ if METALINK_IS_ENABLED
 METALINK_OBJ = metalink.c
 endif
 
+if WITH_XATTR
+XATTR_OBJ = xattr.c
+endif
+
 # The following line is losing on some versions of make!
 DEFS     = @DEFS@ -DSYSTEM_WGETRC=\"$(sysconfdir)/wgetrc\" -DLOCALEDIR=\"$(localedir)\"
-LIBS     = @LIBICONV@ @LIBINTL@ @LIBS@ $(LIB_CLOCK_GETTIME)
 
 EXTRA_DIST = css.l css.c css_.c build_info.c.in
 
@@ -50,17 +53,20 @@ wget_SOURCES = connect.c convert.c cookies.c ftp.c	\
 		css_.c css-url.c	\
 		ftp-basic.c ftp-ls.c hash.c host.c hsts.c html-parse.c html-url.c	\
 		http.c init.c log.c main.c netrc.c progress.c ptimer.c	\
-		recur.c res.c retr.c spider.c url.c warc.c	\
+		recur.c res.c retr.c spider.c url.c warc.c $(XATTR_OBJ) \
 		utils.c exits.c build_info.c $(IRI_OBJ) $(METALINK_OBJ)	\
 		css-url.h css-tokens.h connect.h convert.h cookies.h	\
 		ftp.h hash.h host.h hsts.h  html-parse.h html-url.h	\
 		http.h http-ntlm.h init.h log.h mswindows.h netrc.h	\
 		options.h progress.h ptimer.h recur.h res.h retr.h	\
 		spider.h ssl.h sysdep.h url.h warc.h utils.h wget.h iri.h	\
-		exits.h version.h metalink.h
+		exits.h version.h metalink.h xattr.h
 nodist_wget_SOURCES = version.c
 EXTRA_wget_SOURCES = iri.c
-LDADD = $(LIBOBJS) ../lib/libgnu.a
+LDADD = $(LIBOBJS) ../lib/libgnu.a $(GETADDRINFO_LIB) $(HOSTENT_LIB)\
+ $(INET_NTOP_LIB) $(LIBSOCKET) $(LIB_CLOCK_GETTIME) $(LIB_CRYPTO)\
+ $(LIB_NANOSLEEP) $(LIB_POSIX_SPAWN) $(LIB_SELECT) $(LTLIBICONV) $(LTLIBINTL)\
+ $(LTLIBTHREAD) $(LTLIBUNISTRING) $(SERVENT_LIB)
 AM_CPPFLAGS = -I$(top_builddir)/lib -I$(top_srcdir)/lib
 
 

src/connect.c

@@ -31,6 +31,7 @@ as that of the covered work.  */
 
 #include "wget.h"
 
+#include "exits.h"
 #include <stdio.h>
 #include <stdlib.h>
 #include <unistd.h>
@@ -56,7 +57,7 @@ as that of the covered work.  */
 #include <sys/time.h>
 
 #ifdef ENABLE_IRI
-#include <idn-free.h>
+#include <idn2.h>
 #endif
 
 #include "utils.h"
@@ -283,7 +284,7 @@ connect_to_ip (const ip_address *ip, int port, const char *print)
               str = xmalloc (len);
               snprintf (str, len, "%s (%s)", name, print);
               str[len-1] = '\0';
-              idn_free (name);
+              idn2_free (name);
             }
 
           logprintf (LOG_VERBOSE, _("Connecting to %s|%s|:%d... "),
@@ -331,8 +332,10 @@ connect_to_ip (const ip_address *ip, int port, const char *print)
       if (bufsize < 512)
         bufsize = 512;          /* avoid pathologically small values */
 #ifdef SO_RCVBUF
-      setsockopt (sock, SOL_SOCKET, SO_RCVBUF,
-                  (void *)&bufsize, (socklen_t)sizeof (bufsize));
+      if (setsockopt (sock, SOL_SOCKET, SO_RCVBUF,
+                  (void *) &bufsize, (socklen_t) sizeof (bufsize)))
+        logprintf (LOG_NOTQUIET, _("setsockopt SO_RCVBUF failed: %s\n"),
+                   strerror (errno));
 #endif
       /* When we add limit_rate support for writing, which is useful
          for POST, we should also set SO_SNDBUF here.  */
@@ -471,7 +474,9 @@ bind_local (const ip_address *bind_address, int *port)
     return -1;
 
 #ifdef SO_REUSEADDR
-  setsockopt (sock, SOL_SOCKET, SO_REUSEADDR, setopt_ptr, setopt_size);
+  if (setsockopt (sock, SOL_SOCKET, SO_REUSEADDR, setopt_ptr, setopt_size))
+    logprintf (LOG_NOTQUIET, _("setsockopt SO_REUSEADDR failed: %s\n"),
+               strerror (errno));
 #endif
 
   xzero (ss);
@@ -689,6 +694,11 @@ select_fd (int fd, double maxtime, int wait_for)
   struct timeval tmout;
   int result;
 
+  if (fd >= FD_SETSIZE)
+    {
+      logprintf (LOG_NOTQUIET, _("Too many fds open.  Cannot use select on a fd >= %d\n"), FD_SETSIZE);
+      exit (WGET_EXIT_GENERIC_ERROR);
+    }
   FD_ZERO (&fdset);
   FD_SET (fd, &fdset);
   if (wait_for & WAIT_FOR_READ)
@@ -731,6 +741,11 @@ test_socket_open (int sock)
   struct timeval to;
   int ret = 0;
 
+  if (sock >= FD_SETSIZE)
+    {
+      logprintf (LOG_NOTQUIET, _("Too many fds open.  Cannot use select on a fd >= %d\n"), FD_SETSIZE);
+      exit (WGET_EXIT_GENERIC_ERROR);
+    }
   /* Check if we still have a valid (non-EOF) connection.  From Andrew
    * Maholski's code in the Unix Socket FAQ.  */
 

src/cookies.c

@@ -45,6 +45,7 @@ as that of the covered work.  */
 
 #include "wget.h"
 
+#include <stdint.h>
 #include <stdio.h>
 #include <string.h>
 #include <stdlib.h>
@@ -525,19 +526,51 @@ check_domain_match (const char *cookie_domain, const char *host)
 {
 
 #ifdef HAVE_LIBPSL
+  static int init_psl;
+  static const psl_ctx_t *psl;
+
   char *cookie_domain_lower = NULL;
   char *host_lower = NULL;
-  const psl_ctx_t *psl;
   int is_acceptable;
 
-  DEBUGP (("cdm: 1"));
-  if (!(psl = psl_builtin()))
+  DEBUGP (("cdm: 1\n"));
+  if (!init_psl)
     {
-      DEBUGP (("\nlibpsl not built with a public suffix list. "
-               "Falling back to simple heuristics.\n"));
-      goto no_psl;
-    }
+      init_psl = 1;
 
+#ifdef HAVE_PSL_LATEST
+      if ((psl = psl_latest (NULL)))
+        goto have_psl;
+
+      DEBUGP (("\nPSL: Failed to load any PSL data. "
+               "Falling back to insecure heuristics.\n"));
+#else
+      if ((psl = psl_builtin ()) && !psl_builtin_outdated ())
+        goto have_psl;
+
+      DEBUGP (("\nPSL: built-in data outdated. "
+               "Trying to load data from %s.\n",
+              quote (psl_builtin_filename ())));
+
+      if ((psl = psl_load_file (psl_builtin_filename ())))
+        goto have_psl;
+
+      DEBUGP (("\nPSL: %s not found or not readable. "
+               "Falling back to built-in data.\n",
+              quote (psl_builtin_filename ())));
+
+      if (!(psl = psl_builtin ()))
+        {
+          DEBUGP (("\nPSL: libpsl not built with a public suffix list. "
+                   "Falling back to insecure heuristics.\n"));
+          goto no_psl;
+        }
+#endif
+    }
+  else if (!psl)
+    goto no_psl;
+
+have_psl:
   if (psl_str_to_utf8lower (cookie_domain, NULL, NULL, &cookie_domain_lower) == PSL_SUCCESS &&
       psl_str_to_utf8lower (host, NULL, NULL, &host_lower) == PSL_SUCCESS)
     {
@@ -562,13 +595,13 @@ no_psl:
 #endif
 
   /* For efficiency make some elementary checks first */
-  DEBUGP (("cdm: 2"));
+  DEBUGP (("cdm: 2\n"));
 
   /* For the sake of efficiency, check for exact match first. */
   if (0 == strcasecmp (cookie_domain, host))
     return true;
 
-  DEBUGP ((" 3"));
+  DEBUGP (("cdm: 3\n"));
 
   /* HOST must match the tail of cookie_domain. */
   if (!match_tail (host, cookie_domain, true))
@@ -608,7 +641,7 @@ no_psl:
     if (*p == '.')
       /* Ignore leading period in this calculation. */
       ++p;
-    DEBUGP ((" 4"));
+    DEBUGP (("cdm: 4\n"));
     for (out = 0; !out; p++)
       switch (*p)
         {
@@ -634,12 +667,12 @@ no_psl:
           ++ldcl;
         }
 
-    DEBUGP ((" 5"));
+    DEBUGP (("cdm: 5\n"));
 
     if (dccount < 2)
       return false;
 
-    DEBUGP ((" 6"));
+    DEBUGP (("cdm: 6\n"));
 
     if (dccount == 2)
       {
@@ -659,7 +692,7 @@ no_psl:
       }
   }
 
-  DEBUGP ((" 7"));
+  DEBUGP (("cdm: 7\n"));
 
   /* Don't allow the host "foobar.com" to set a cookie for domain
      "bar.com".  */
@@ -674,7 +707,7 @@ no_psl:
         return false;
     }
 
-  DEBUGP ((" 8"));
+  DEBUGP (("cdm: 8\n"));
 
   return true;
 }
@@ -1018,7 +1051,7 @@ cookie_header (struct cookie_jar *jar, const char *host,
 
   struct cookie *cookie;
   struct weighed_cookie *outgoing;
-  int count, i, ocnt;
+  size_t count, i, ocnt;
   char *result;
   int result_size, pos;
   PREPEND_SLASH (path);         /* see cookie_handle_set_cookie */
@@ -1032,7 +1065,7 @@ cookie_header (struct cookie_jar *jar, const char *host,
   chain_count = find_chains_of_host (jar, host, chains);
 
   /* No cookies for this host. */
-  if (!chain_count)
+  if (chain_count <= 0)
     return NULL;
 
   cookies_now = time (NULL);
@@ -1043,7 +1076,7 @@ cookie_header (struct cookie_jar *jar, const char *host,
 
   /* Count the number of matching cookies. */
   count = 0;
-  for (i = 0; i < chain_count; i++)
+  for (i = 0; i < (unsigned) chain_count; i++)
     for (cookie = chains[i]; cookie; cookie = cookie->next)
       if (cookie_matches_url (cookie, host, port, path, secflag, NULL))
         ++count;
@@ -1051,12 +1084,14 @@ cookie_header (struct cookie_jar *jar, const char *host,
     return NULL;                /* no cookies matched */
 
   /* Allocate the array. */
-  outgoing = alloca_array (struct weighed_cookie, count);
+  if (count > SIZE_MAX / sizeof (struct weighed_cookie))
+    return NULL;                /* unable to process so many cookies */
+  outgoing = xmalloc (count * sizeof (struct weighed_cookie));
 
   /* Fill the array with all the matching cookies from the chains that
      match HOST. */
   ocnt = 0;
-  for (i = 0; i < chain_count; i++)
+  for (i = 0; i < (unsigned) chain_count; i++)
     for (cookie = chains[i]; cookie; cookie = cookie->next)
       {
         int pg;
@@ -1111,6 +1146,7 @@ cookie_header (struct cookie_jar *jar, const char *host,
         }
     }
   result[pos++] = '\0';
+  xfree (outgoing);
   assert (pos == result_size);
   return result;
 }

src/css-url.c

@@ -59,6 +59,7 @@ extern char *yytext;
 extern int yyleng;
 typedef struct yy_buffer_state *YY_BUFFER_STATE;
 extern YY_BUFFER_STATE yy_scan_bytes (const char *bytes,int len  );
+extern void yy_delete_buffer (YY_BUFFER_STATE  b);
 extern int yylex (void);
 
 /*
@@ -109,9 +110,10 @@ get_urls_css (struct map_context *ctx, int offset, int buf_length)
   int buffer_pos = 0;
   int pos, length;
   char *uri;
+  YY_BUFFER_STATE b;
 
   /* tell flex to scan from this buffer */
-  yy_scan_bytes (ctx->text + offset, buf_length);
+  b = yy_scan_bytes (ctx->text + offset, buf_length);
 
   while((token = yylex()) != CSSEOF)
     {
@@ -188,6 +190,9 @@ get_urls_css (struct map_context *ctx, int offset, int buf_length)
         }
       buffer_pos += yyleng;
     }
+
+  yy_delete_buffer(b);
+
   DEBUGP (("\n"));
 }
 

src/css-url.h

@@ -31,7 +31,6 @@ as that of the covered work.  */
 #ifndef CSS_URL_H
 #define CSS_URL_H
 
-void get_urls_css (struct map_context *, int, int);
 void get_urls_css (struct map_context *, int, int);
 struct urlpos *get_urls_css_file (const char *, const char *);
 

src/ftp-ls.c

@@ -167,12 +167,12 @@ ftp_parse_unix_ls (const char *file, int ignore_perms)
               /*cur.perms = 1023;*/     /* #### What is this?  --hniksic */
               cur.perms = 0644;
             }
-          DEBUGP (("implicit perms %0o; ", cur.perms));
+          DEBUGP (("implicit perms %0o; ", (unsigned) cur.perms));
         }
        else
          {
            cur.perms = symperms (tok + 1);
-           DEBUGP (("perms %0o; ", cur.perms));
+           DEBUGP (("perms %0o; ", (unsigned) cur.perms));
          }
 
       error = ignore = 0;       /* Erroneous and ignoring entries are
@@ -478,6 +478,7 @@ ftp_parse_winnt_ls (const char *file)
         }
       /* Now it is possible to determine the position of the first symbol in
          filename. */
+      memset(&cur, 0, sizeof (cur));
       cur.name = xstrdup(filename);
       DEBUGP (("Name: '%s'\n", cur.name));
 
@@ -931,7 +932,7 @@ ftp_parse_vms_ls (const char *file)
                     }
                 }
               cur.perms = perms;
-              DEBUGP (("Prot.  perms = %0o.\n", cur.perms));
+              DEBUGP (("Prot.  perms = %0o.\n", (unsigned) cur.perms));
             }
           else
             {

src/ftp.c

@@ -52,6 +52,9 @@ as that of the covered work.  */
 #include "recur.h"              /* for INFINITE_RECURSION */
 #include "warc.h"
 #include "c-strcase.h"
+#ifdef ENABLE_XATTR
+#include "xattr.h"
+#endif
 
 #ifdef __VMS
 # include "vms.h"
@@ -239,7 +242,7 @@ print_length (wgint size, wgint start, bool authoritative)
 static uerr_t ftp_get_listing (struct url *, struct url *, ccon *, struct fileinfo **);
 
 static uerr_t
-get_ftp_greeting(int csock, ccon *con)
+get_ftp_greeting (int csock, ccon *con)
 {
   uerr_t err = 0;
 
@@ -356,12 +359,35 @@ getftp (struct url *u, struct url *original_url,
 
   *qtyread = restval;
 
-  user = u->user;
-  passwd = u->passwd;
-  search_netrc (u->host, (const char **)&user, (const char **)&passwd, 1);
-  user = user ? user : (opt.ftp_user ? opt.ftp_user : opt.user);
+  /* Find the username with priority */
+  if (u->user)
+    user = u->user;
+  else if (opt.user && (opt.use_askpass || opt.ask_passwd))
+    user = opt.user;
+  else if (opt.ftp_user)
+    user = opt.ftp_user;
+  else if (opt.user)
+    user = opt.user;
+  else
+    user = NULL;
+
+  /* Find the password with priority */
+  if (u->passwd)
+    passwd = u->passwd;
+  else if (opt.passwd && (opt.use_askpass || opt.ask_passwd))
+    passwd = opt.passwd;
+  else if (opt.ftp_passwd)
+    passwd = opt.ftp_passwd;
+  else if (opt.passwd)
+    passwd = opt.passwd;
+  else
+    passwd = NULL;
+
+  /* Check for ~/.netrc if none of the above match */
+  if (opt.netrc && (!user || !passwd))
+    search_netrc (u->host, (const char **) &user, (const char **) &passwd, 1);
+
   if (!user) user = "anonymous";
-  passwd = passwd ? passwd : (opt.ftp_passwd ? opt.ftp_passwd : opt.passwd);
   if (!passwd) passwd = "-wget@";
 
   dtsock = -1;
@@ -535,10 +561,10 @@ Error in server response, closing control connection.\n"));
                 logputs (LOG_VERBOSE, "done.");
 
               if (!opt.server_response)
-                logprintf (LOG_VERBOSE, "  ==> PROT %c ... ", prot);
+                logprintf (LOG_VERBOSE, "  ==> PROT %c ... ", (int) prot);
               if ((err = ftp_prot (csock, prot)) == FTPNOPROT)
                 {
-                  logprintf (LOG_NOTQUIET, _("Server did not accept the 'PROT %c' command.\n"), prot);
+                  logprintf (LOG_NOTQUIET, _("Server did not accept the 'PROT %c' command.\n"), (int) prot);
                   return err;
                 }
               if (!opt.server_response)
@@ -872,7 +898,7 @@ Error in server response, closing control connection.\n"));
              Unlike the rest of this block, this particular behavior
              _is_ VMS-specific, so it gets its own VMS test.
           */
-          if ((con->rs == ST_VMS) && (strchr( target, '/') != NULL))
+          if ((con->rs == ST_VMS) && (strchr (target, '/') != NULL))
             {
               cwd_end = 3;
               DEBUGP (("Using extra \"CWD []\" step for VMS server.\n"));
@@ -1188,6 +1214,7 @@ Error in server response, closing control connection.\n"));
       if (opt.spider)
         {
           bool exists = false;
+          bool all_exist = true;
           struct fileinfo *f;
           uerr_t _res = ftp_get_listing (u, original_url, con, &f);
           /* Set the DO_RETR command flag again, because it gets unset when
@@ -1203,6 +1230,8 @@ Error in server response, closing control connection.\n"));
                     {
                       exists = true;
                       break;
+                    } else {
+                      all_exist = false;
                     }
                   f = f->next;
                 }
@@ -1223,7 +1252,11 @@ Error in server response, closing control connection.\n"));
           con->csock = -1;
           fd_close (dtsock);
           fd_close (local_sock);
-          return RETRFINISHED;
+          if (all_exist) {
+              return RETRFINISHED;
+          } else {
+              return FTPNSFOD;
+          }
         }
 
       if (opt.verbose)
@@ -1546,6 +1579,11 @@ Error in server response, closing control connection.\n"));
   tmrate = retr_rate (rd_size, con->dltime);
   total_download_time += con->dltime;
 
+#ifdef ENABLE_XATTR
+  if (opt.enable_xattr)
+    set_file_metadata (u->url, NULL, fp);
+#endif
+
   fd_close (local_sock);
   /* Close the local file.  */
   if (!output_stream || con->cmd & DO_LIST)
@@ -1641,10 +1679,10 @@ Error in server response, closing control connection.\n"));
 #ifdef __VMS
       char *targ;
 
-      targ = ods_conform( con->target);
+      targ = ods_conform (con->target);
       if (targ != con->target)
         {
-          xfree( con->target);
+          xfree (con->target);
           con->target = targ;
         }
 #endif /* def __VMS */
@@ -1788,7 +1826,7 @@ ftp_loop_internal (struct url *u, struct url *original_url, struct fileinfo *f,
   char *tms, *locf;
   const char *tmrate = NULL;
   uerr_t err;
-  struct_stat st;
+  struct stat st;
 
   /* Declare WARC variables. */
   bool warc_enabled = (opt.warc_filename != NULL);
@@ -2211,7 +2249,7 @@ ftp_retrieve_list (struct url *u, struct url *original_url,
       dlthis = true;
       if (opt.timestamping && f->type == FT_PLAINFILE)
         {
-          struct_stat st;
+          struct stat st;
           /* If conversion of HTML files retrieved via FTP is ever implemented,
              we'll need to stat() <file>.orig here when -K has been specified.
              I'm not implementing it now since files on an FTP server are much
@@ -2274,7 +2312,7 @@ The sizes do not match (local %s) -- retrieving.\n\n"),
                          _("Invalid name of the symlink, skipping.\n"));
               else
                 {
-                  struct_stat st;
+                  struct stat st;
                   /* Check whether we already have the correct
                      symbolic link.  */
                   int rc = lstat (con->target, &st);
@@ -2334,6 +2372,7 @@ Already have correct symlink %s -> %s\n\n"),
             }
           break;
         case FT_UNKNOWN:
+        default:
           logprintf (LOG_NOTQUIET, _("%s: unknown/unsupported file type.\n"),
                      quote (f->name));
           break;
@@ -2355,7 +2394,12 @@ Already have correct symlink %s -> %s\n\n"),
        (f->type == FT_PLAINFILE) && opt.preserve_perm)
         {
           if (f->perms)
-            chmod (actual_target, f->perms);
+            {
+              if (chmod (actual_target, f->perms))
+                logprintf (LOG_NOTQUIET,
+                           _("Failed to set permissions for %s.\n"),
+                           actual_target);
+            }
           else
             DEBUGP (("Unrecognized permissions for %s.\n", actual_target));
         }
@@ -2503,7 +2547,7 @@ is_invalid_entry (struct fileinfo *f)
   while (cur->next)
     {
       cur = cur->next;
-      if (strcmp(f_name, cur->name) == 0)
+      if (strcmp (f_name, cur->name) == 0)
           return true;
     }
   return false;
@@ -2697,7 +2741,7 @@ ftp_loop (struct url *u, struct url *original_url, char **local_file, int *dt,
                 {
                   if (!opt.output_document)
                     {
-                      struct_stat st;
+                      struct stat st;
                       wgint sz;
                       if (stat (filename, &st) == 0)
                         sz = st.st_size;

src/gnutls.c

@@ -50,7 +50,7 @@ as that of the covered work.  */
 #include "hash.h"
 #include "ssl.h"
 
-#include <sys/fcntl.h>
+#include <fcntl.h>
 
 #ifdef WIN32
 # include "w32sock.h"
@@ -498,7 +498,7 @@ _do_handshake (gnutls_session_t session, int fd, double timeout)
             {
               gnutls_alert_description_t alert = gnutls_alert_get (session);
               const char *str = gnutls_alert_get_name (alert);
-              logprintf (LOG_NOTQUIET, "GnuTLS: received alert [%d]: %s\n",
+              logprintf (LOG_NOTQUIET, "GnuTLS: received alert [%u]: %s\n",
                          alert, str ? str : "(unknown)");
             }
         }
@@ -556,7 +556,6 @@ ssl_connect_wget (int fd, const char *hostname, int *continue_session)
       xfree(sni_hostname);
     }
 
-  gnutls_set_default_priority (session);
   gnutls_credentials_set (session, GNUTLS_CRD_CERTIFICATE, credentials);
 #ifndef FD_TO_SOCKET
 # define FD_TO_SOCKET(X) (X)
@@ -571,7 +570,8 @@ ssl_connect_wget (int fd, const char *hostname, int *continue_session)
   switch (opt.secure_protocol)
     {
     case secure_protocol_auto:
-      err = gnutls_priority_set_direct (session, "NORMAL:%COMPAT:-VERS-SSL3.0", NULL);
+      err = gnutls_set_default_priority (session);
+      gnutls_session_enable_compatibility_mode(session);
       break;
 
     case secure_protocol_sslv2:
@@ -599,7 +599,8 @@ ssl_connect_wget (int fd, const char *hostname, int *continue_session)
       break;
 
     default:
-      logprintf (LOG_NOTQUIET, _("GnuTLS: unimplemented 'secure-protocol' option value %d\n"), opt.secure_protocol);
+      logprintf (LOG_NOTQUIET, _("GnuTLS: unimplemented 'secure-protocol' option value %u\n"),
+                 (unsigned) opt.secure_protocol);
       logprintf (LOG_NOTQUIET, _("Please report this issue to bug-wget@gnu.org\n"));
       abort ();
     }
@@ -608,6 +609,7 @@ ssl_connect_wget (int fd, const char *hostname, int *continue_session)
   switch (opt.secure_protocol)
     {
     case secure_protocol_auto:
+      err = gnutls_set_default_priority (session);
       break;
 
     case secure_protocol_sslv2:

src/hash.c

@@ -283,10 +283,10 @@ hash_table_new (int items,
 
   /* Calculate the size that ensures that the table will store at
      least ITEMS keys without the need to resize.  */
-  size = 1 + items / HASH_MAX_FULLNESS;
+  size = (int) (1 + items / HASH_MAX_FULLNESS);
   size = prime_size (size, &ht->prime_offset);
   ht->size = size;
-  ht->resize_threshold = size * HASH_MAX_FULLNESS;
+  ht->resize_threshold = (int) (size * HASH_MAX_FULLNESS);
   /*assert (ht->resize_threshold >= items);*/
 
   ht->cells = xnew_array (struct cell, ht->size);
@@ -394,7 +394,7 @@ grow_hash_table (struct hash_table *ht)
 #endif
 
   ht->size = newsize;
-  ht->resize_threshold = newsize * HASH_MAX_FULLNESS;
+  ht->resize_threshold = (int) (newsize * HASH_MAX_FULLNESS);
 
   cells = xnew_array (struct cell, newsize);
   memset (cells, INVALID_PTR_CHAR, newsize * sizeof (struct cell));

src/host.c

@@ -58,7 +58,7 @@ as that of the covered work.  */
 #include <errno.h>
 
 #ifdef ENABLE_IRI
-#include <idn-free.h>
+#include <idn2.h>
 #endif
 
 #include "utils.h"
@@ -428,14 +428,12 @@ getaddrinfo_with_timeout (const char *node, const char *service,
 const char *
 print_address (const ip_address *addr)
 {
-#ifdef ENABLE_IPV6
   static char buf[64];
+
   if (!inet_ntop (addr->family, IP_INADDR_DATA (addr), buf, sizeof buf))
     snprintf (buf, sizeof buf, "<error: %s>", strerror (errno));
+
   return buf;
-#else
-  return inet_ntoa (addr->data.d4);
-#endif
 }
 
 /* The following two functions were adapted from glibc's
@@ -852,7 +850,7 @@ lookup_host (const char *host, int flags)
           str = xmalloc (len);
           snprintf (str, len, "%s (%s)", name, host);
           str[len-1] = '\0';
-          idn_free (name);
+          idn2_free (name);
         }
 
       logprintf (LOG_VERBOSE, _("Resolving %s... "),

src/hsts.c

@@ -80,7 +80,6 @@ enum hsts_kh_match {
 
 #define DEFAULT_HTTP_PORT 80
 #define DEFAULT_SSL_PORT  443
-#define CHECK_EXPLICIT_PORT(p1, p2) (p1 == 0 || p1 == p2)
 #define MAKE_EXPLICIT_PORT(s, p) (s == SCHEME_HTTPS ? (p == DEFAULT_SSL_PORT ? 0 : p) \
     : (p == DEFAULT_HTTP_PORT ? 0 : p))
 
@@ -327,7 +326,8 @@ hsts_store_dump (hsts_store_t store, FILE *fp)
 
       if (fprintf (fp, "%s\t%d\t%d\t%lu\t%lu\n",
                    kh->host, kh->explicit_port, khi->include_subdomains,
-                   khi->created, khi->max_age) < 0)
+                   (unsigned long) khi->created,
+                   (unsigned long) khi->max_age) < 0)
         {
           logprintf (LOG_ALWAYS, "Could not write the HSTS database correctly.\n");
           break;
@@ -343,12 +343,20 @@ hsts_store_dump (hsts_store_t store, FILE *fp)
 static bool
 hsts_file_access_valid (const char *filename)
 {
-  struct_stat st;
+  struct stat st;
 
   if (stat (filename, &st) == -1)
     return false;
 
-  return !(st.st_mode & S_IWOTH) && S_ISREG (st.st_mode);
+  return
+#ifndef WINDOWS
+      /*
+       * The world-writable concept is a Unix-centric notion.
+       * We bypass this test on Windows.
+       */
+      !(st.st_mode & S_IWOTH) &&
+#endif
+      S_ISREG (st.st_mode);
 }
 
 /* HSTS API */
@@ -502,7 +510,7 @@ hsts_store_open (const char *filename)
     {
       if (hsts_file_access_valid (filename))
         {
-          struct_stat st;
+          struct stat st;
           FILE *fp = fopen (filename, "r");
 
           if (!fp || !hsts_read_database (store, fp, false))
@@ -510,7 +518,8 @@ hsts_store_open (const char *filename)
               /* abort! */
               hsts_store_close (store);
               xfree (store);
-              fclose (fp);
+              if (fp)
+                fclose (fp);
               goto out;
             }
 
@@ -540,7 +549,7 @@ out:
 void
 hsts_store_save (hsts_store_t store, const char *filename)
 {
-  struct_stat st;
+  struct stat st;
   FILE *fp = NULL;
   int fd = 0;
 
@@ -790,9 +799,9 @@ test_hsts_read_database (void)
       if (fp)
         {
           fputs ("# dummy comment\n", fp);
-          fprintf (fp, "foo.example.com\t0\t1\t%ld\t123\n",(long) created);
-          fprintf (fp, "bar.example.com\t0\t0\t%ld\t456\n", (long) created);
-          fprintf (fp, "test.example.com\t8080\t0\t%ld\t789\n", (long) created);
+          fprintf (fp, "foo.example.com\t0\t1\t%lu\t123\n",(unsigned long) created);
+          fprintf (fp, "bar.example.com\t0\t0\t%lu\t456\n", (unsigned long) created);
+          fprintf (fp, "test.example.com\t8080\t0\t%lu\t789\n", (unsigned long) created);
           fclose (fp);
 
           table = hsts_store_open (file);

src/html-parse.c

@@ -801,14 +801,6 @@ name_allowed (const struct hash_table *ht, const char *b, const char *e)
   }                                             \
 } while (0)
 
-/* Skip non-whitespace, if any. */
-
-#define SKIP_NON_WS(p) do {                     \
-  while (!c_isspace (*p)) {                       \
-    ADVANCE (p);                                \
-  }                                             \
-} while (0)
-
 #ifdef STANDALONE
 static int tag_backout_count;
 #endif

src/html-url.c

@@ -729,8 +729,11 @@ tag_handle_img (int tagid, struct taginfo *tag, struct map_context *ctx) {
                                             srcset + url_end);
               struct urlpos *up = append_url (url_text, base_ind + url_start,
                                               url_end - url_start, ctx);
-              up->link_inline_p = 1;
-              up->link_noquote_html_p = 1;
+              if (up)
+                {
+                  up->link_inline_p = 1;
+                  up->link_noquote_html_p = 1;
+                }
               xfree (url_text);
             }
 
@@ -842,6 +845,7 @@ get_urls_html (const char *file, const char *url, bool *meta_disallow_follow,
   if (iri && !iri->content_encoding && meta_charset)
     set_content_encoding (iri, meta_charset);
 #endif
+  xfree (meta_charset);
 
   DEBUGP (("no-follow in %s: %d\n", file, ctx.nofollow));
   if (meta_disallow_follow)

src/http-ntlm.c

@@ -122,7 +122,7 @@ ntlm_input (struct ntlmdata *ntlm, const char *header)
 
       DEBUGP (("Received a type-2 NTLM message.\n"));
 
-      size = base64_decode (header, buffer);
+      size = wget_base64_decode (header, buffer);
       if (size < 0)
         return false;           /* malformed base64 from server */
 
@@ -411,7 +411,7 @@ ntlm_output (struct ntlmdata *ntlm, const char *user, const char *passwd,
     size = 32 + hostlen + domlen;
 
     base64 = (char *) alloca (BASE64_LENGTH (size) + 1);
-    base64_encode (ntlmbuf, size, base64);
+    wget_base64_encode (ntlmbuf, size, base64);
 
     output = concat_strings ("NTLM ", base64, (char *) 0);
     break;
@@ -584,7 +584,7 @@ ntlm_output (struct ntlmdata *ntlm, const char *user, const char *passwd,
 
     /* convert the binary blob into base64 */
     base64 = (char *) alloca (BASE64_LENGTH (size) + 1);
-    base64_encode (ntlmbuf, size, base64);
+    wget_base64_encode (ntlmbuf, size, base64);
 
     output = concat_strings ("NTLM ", base64, (char *) 0);
 

src/http.c

@@ -39,6 +39,7 @@ as that of the covered work.  */
 #include <errno.h>
 #include <time.h>
 #include <locale.h>
+#include <fcntl.h>
 
 #include "hash.h"
 #include "http.h"
@@ -66,6 +67,9 @@ as that of the covered work.  */
 # include "metalink.h"
 # include "xstrndup.h"
 #endif
+#ifdef ENABLE_XATTR
+#include "xattr.h"
+#endif
 
 #ifdef TESTING
 #include "test.h"
@@ -341,7 +345,7 @@ request_send (const struct request *req, int fd, FILE *warc_tmp)
   /* "\r\n\0" */
   size += 3;
 
-  p = request_string = alloca_array (char, size);
+  p = request_string = xmalloc (size);
 
   /* Generate the request. */
 
@@ -376,8 +380,9 @@ request_send (const struct request *req, int fd, FILE *warc_tmp)
       /* Write a copy of the data to the WARC record. */
       int warc_tmp_written = fwrite (request_string, 1, size - 1, warc_tmp);
       if (warc_tmp_written != size - 1)
-        return -2;
+        write_error = -2;
     }
+  xfree (request_string);
   return write_error;
 }
 
@@ -420,7 +425,7 @@ maybe_send_basic_creds (const char *hostname, const char *user,
       do_challenge = true;
     }
   else if (basic_authed_hosts
-      && hash_table_contains(basic_authed_hosts, hostname))
+      && hash_table_contains (basic_authed_hosts, hostname))
     {
       DEBUGP (("Found %s in basic_authed_hosts.\n", quote (hostname)));
       do_challenge = true;
@@ -446,9 +451,9 @@ register_basic_auth_host (const char *hostname)
     {
       basic_authed_hosts = make_nocase_string_hash_table (1);
     }
-  if (!hash_table_contains(basic_authed_hosts, hostname))
+  if (!hash_table_contains (basic_authed_hosts, hostname))
     {
-      hash_table_put (basic_authed_hosts, xstrdup(hostname), NULL);
+      hash_table_put (basic_authed_hosts, xstrdup (hostname), NULL);
       DEBUGP (("Inserted %s into basic_authed_hosts\n", quote (hostname)));
     }
 }
@@ -846,7 +851,7 @@ resp_free (struct response **resp_ref)
    caused crashes in UTF-8 locales.  */
 
 static void
-print_response_line(const char *prefix, const char *b, const char *e)
+print_response_line (const char *prefix, const char *b, const char *e)
 {
   char *copy;
   BOUNDED_TO_ALLOCA(b, e, copy);
@@ -872,7 +877,7 @@ print_server_response (const struct response *resp, const char *prefix)
         --e;
       if (b < e && e[-1] == '\r')
         --e;
-      print_response_line(prefix, b, e);
+      print_response_line (prefix, b, e);
     }
 }
 
@@ -1026,18 +1031,18 @@ skip_short_body (int fd, wgint contlen, bool chunked)
    or a fragment of a long parameter value
 */
 static int
-modify_param_name(param_token *name)
+modify_param_name (param_token *name)
 {
   const char *delim1 = memchr (name->b, '*', name->e - name->b);
   const char *delim2 = memrchr (name->b, '*', name->e - name->b);
 
   int result;
 
-  if(delim1 == NULL)
+  if (delim1 == NULL)
     {
       result = NOT_RFC2231;
     }
-  else if(delim1 == delim2)
+  else if (delim1 == delim2)
     {
       if ((name->e - 1) == delim1)
         {
@@ -1155,12 +1160,12 @@ extract_param (const char **source, param_token *name, param_token *value,
     }
   *source = p;
 
-  param_type = modify_param_name(name);
+  param_type = modify_param_name (name);
   if (param_type != NOT_RFC2231)
     {
       if (param_type == RFC2231_ENCODING && is_url_encoded)
         *is_url_encoded = true;
-      modify_param_value(value, param_type);
+      modify_param_value (value, param_type);
     }
   return true;
 }
@@ -1175,8 +1180,8 @@ static void
 append_value_to_filename (char **filename, param_token const * const value,
                           bool is_url_encoded)
 {
-  int original_length = strlen(*filename);
-  int new_length = strlen(*filename) + (value->e - value->b);
+  int original_length = strlen (*filename);
+  int new_length = strlen (*filename) + (value->e - value->b);
   *filename = xrealloc (*filename, new_length+1);
   memcpy (*filename + original_length, value->b, (value->e - value->b));
   (*filename)[new_length] = '\0';
@@ -1281,7 +1286,7 @@ parse_strict_transport_security (const char *header, time_t *max_age, bool *incl
         {
           if (BOUNDED_EQUAL_NO_CASE (name.b, name.e, "max-age"))
             {
-              xfree(c_max_age);
+              xfree (c_max_age);
               c_max_age = strdupdelim (value.b, value.e);
             }
           else if (BOUNDED_EQUAL_NO_CASE (name.b, name.e, "includeSubDomains"))
@@ -1564,6 +1569,7 @@ struct http_stat
 #ifdef HAVE_METALINK
   metalink_t *metalink;
 #endif
+  bool temporary;               /* downloading a temporary file */
 };
 
 static void
@@ -1805,7 +1811,7 @@ time_to_rfc1123 (time_t time, char *buf, size_t bufsize)
 }
 
 static struct request *
-initialize_request (struct url *u, struct http_stat *hs, int *dt, struct url *proxy,
+initialize_request (const struct url *u, struct http_stat *hs, int *dt, struct url *proxy,
                     bool inhibit_keep_alive, bool *basic_auth_finished,
                     wgint *body_data_size, char **user, char **passwd, uerr_t *ret)
 {
@@ -1869,21 +1875,42 @@ initialize_request (struct url *u, struct http_stat *hs, int *dt, struct url *pr
   request_set_header (req, "Accept", "*/*", rel_none);
   request_set_header (req, "Accept-Encoding", "identity", rel_none);
 
-  /* Find the username and password for authentication. */
-  *user = u->user;
-  *passwd = u->passwd;
-  search_netrc (u->host, (const char **)user, (const char **)passwd, 0);
-  *user = *user ? *user : (opt.http_user ? opt.http_user : opt.user);
-  *passwd = *passwd ? *passwd : (opt.http_passwd ? opt.http_passwd : opt.passwd);
+  /* Find the username with priority */
+  if (u->user)
+    *user = u->user;
+  else if (opt.user && (opt.use_askpass || opt.ask_passwd))
+    *user = opt.user;
+  else if (opt.http_user)
+    *user = opt.http_user;
+  else if (opt.user)
+    *user = opt.user;
+  else
+    *user = NULL;
+
+  /* Find the password with priority */
+  if (u->passwd)
+    *passwd = u->passwd;
+  else if (opt.passwd && (opt.use_askpass || opt.ask_passwd))
+    *passwd = opt.passwd;
+  else if (opt.http_passwd)
+    *passwd = opt.http_passwd;
+  else if (opt.passwd)
+    *passwd = opt.passwd;
+  else
+    *passwd = NULL;
+
+  /* Check for ~/.netrc if none of the above match */
+  if (opt.netrc && (!user || !passwd))
+    search_netrc (u->host, (const char **) user, (const char **) passwd, 0);
 
   /* We only do "site-wide" authentication with "global" user/password
    * values unless --auth-no-challange has been requested; URL user/password
    * info overrides. */
-  if (user && *passwd && (!u->user || opt.auth_without_challenge))
+  if (*user && *passwd && (!u->user || opt.auth_without_challenge))
     {
       /* If this is a host for which we've already received a Basic
        * challenge, we'll go ahead and send Basic authentication creds. */
-      *basic_auth_finished = maybe_send_basic_creds(u->host, *user, *passwd, req);
+      *basic_auth_finished = maybe_send_basic_creds (u->host, *user, *passwd, req);
     }
 
   /* Generate the Host header, HOST:PORT.  Take into account that:
@@ -1951,7 +1978,7 @@ initialize_request (struct url *u, struct http_stat *hs, int *dt, struct url *pr
 }
 
 static void
-initialize_proxy_configuration (struct url *u, struct request *req,
+initialize_proxy_configuration (const struct url *u, struct request *req,
                                 struct url *proxy, char **proxyauth)
 {
   char *proxy_user, *proxy_passwd;
@@ -1983,7 +2010,7 @@ initialize_proxy_configuration (struct url *u, struct request *req,
 }
 
 static uerr_t
-establish_connection (struct url *u, struct url **conn_ref,
+establish_connection (const struct url *u, const struct url **conn_ref,
                       struct http_stat *hs, struct url *proxy,
                       char **proxyauth,
                       struct request **req_ref, bool *using_ssl,
@@ -1993,7 +2020,7 @@ establish_connection (struct url *u, struct url **conn_ref,
   bool host_lookup_failed = false;
   int sock = *sock_ref;
   struct request *req = *req_ref;
-  struct url *conn = *conn_ref;
+  const struct url *conn = *conn_ref;
   struct response *resp;
   int write_error;
   int statcode;
@@ -2004,7 +2031,7 @@ establish_connection (struct url *u, struct url **conn_ref,
          proxy is used.  The exception is when SSL is in use, in which
          case the proxy is nothing but a passthrough to the target
          host, registered as a connection to the latter.  */
-      struct url *relevant = conn;
+      const struct url *relevant = conn;
 #ifdef HAVE_SSL
       if (u->scheme == SCHEME_HTTPS)
         relevant = u;
@@ -2119,7 +2146,7 @@ establish_connection (struct url *u, struct url **conn_ref,
               xfree (head);
               return HERR;
             }
-          xfree(hs->message);
+          xfree (hs->message);
           hs->message = xstrdup (message);
           resp_free (&resp);
           xfree (head);
@@ -2168,7 +2195,7 @@ set_file_timestamp (struct http_stat *hs)
   char *filename_plus_orig_suffix = alloca (filename_len + sizeof (ORIG_SFX));
   bool local_dot_orig_file_exists = false;
   char *local_filename = NULL;
-  struct_stat st;
+  struct stat st;
 
   if (opt.backup_converted)
     /* If -K is specified, we'll act on the assumption that it was specified
@@ -2225,7 +2252,7 @@ set_file_timestamp (struct http_stat *hs)
 }
 
 static uerr_t
-check_file_output (struct url *u, struct http_stat *hs,
+check_file_output (const struct url *u, struct http_stat *hs,
                    struct response *resp, char *hdrval, size_t hdrsize)
 {
   /* Determine the local filename if needed. Notice that if -O is used
@@ -2254,6 +2281,14 @@ check_file_output (struct url *u, struct http_stat *hs,
       xfree (local_file);
     }
 
+  hs->temporary = opt.delete_after || opt.spider || !acceptable (hs->local_file);
+  if (hs->temporary)
+    {
+      char *tmp = aprintf ("%s.tmp", hs->local_file);
+      xfree (hs->local_file);
+      hs->local_file = tmp;
+    }
+
   /* TODO: perform this check only once. */
   if (!hs->existence_checked && file_exists_p (hs->local_file))
     {
@@ -2285,7 +2320,7 @@ check_file_output (struct url *u, struct http_stat *hs,
 }
 
 static uerr_t
-check_auth (struct url *u, char *user, char *passwd, struct response *resp,
+check_auth (const struct url *u, char *user, char *passwd, struct response *resp,
             struct request *req, bool *ntlm_seen_ref, bool *retry,
             bool *basic_auth_finished_ref, bool *auth_finished_ref)
 {
@@ -2467,7 +2502,15 @@ open_output_stream (struct http_stat *hs, int count, FILE **fp)
           open_id = 22;
           *fp = fopen (hs->local_file, "wb", FOPEN_OPT_ARGS);
 #else /* def __VMS */
-          *fp = fopen (hs->local_file, "wb");
+          if (hs->temporary)
+            {
+              *fp = fdopen (open (hs->local_file, O_BINARY | O_CREAT | O_TRUNC | O_WRONLY, S_IRUSR | S_IWUSR), "wb");
+            }
+          else
+            {
+              *fp = fopen (hs->local_file, "wb");
+            }
+
 #endif /* def __VMS [else] */
         }
       else
@@ -2533,7 +2576,7 @@ metalink_from_http (const struct response *resp, const struct http_stat *hs,
   metalink_t *metalink = NULL;
   metalink_file_t *mfile = xnew0 (metalink_file_t);
   const char *val_beg, *val_end;
-  int res_count = 0, hash_count = 0, sig_count = 0, i;
+  int res_count = 0, meta_count = 0, hash_count = 0, sig_count = 0, i;
 
   DEBUGP (("Checking for Metalink in HTTP response\n"));
 
@@ -2546,6 +2589,88 @@ metalink_from_http (const struct response *resp, const struct http_stat *hs,
   /* Begin with 1-element array (for 0-termination). */
   mfile->checksums = xnew0 (metalink_checksum_t *);
   mfile->resources = xnew0 (metalink_resource_t *);
+  mfile->metaurls = xnew0 (metalink_metaurl_t *);
+
+  /* Process the Content-Type header.  */
+  if (resp_header_locate (resp, "Content-Type", 0, &val_beg, &val_end) != -1)
+    {
+      metalink_metaurl_t murl = {0};
+
+      const char *type_beg, *type_end;
+      char *typestr = NULL;
+      char *namestr = NULL;
+      size_t type_len;
+
+      DEBUGP (("Processing Content-Type header...\n"));
+
+      /* Find beginning of type.  */
+      type_beg = val_beg;
+      while (type_beg < val_end && c_isspace (*type_beg))
+        type_beg++;
+
+      /* Find end of type.  */
+      type_end = type_beg + 1;
+      while (type_end < val_end &&
+             *type_end != ';' &&
+             *type_end != ' ' &&
+             *type_end != '\r' &&
+             *type_end != '\n')
+        type_end++;
+
+      if (type_beg >= val_end || type_end > val_end)
+        {
+          DEBUGP (("Invalid Content-Type header. Ignoring.\n"));
+          goto skip_content_type;
+        }
+
+      type_len = type_end - type_beg;
+      typestr = xstrndup (type_beg, type_len);
+
+      DEBUGP (("Content-Type: %s\n", typestr));
+
+      if (strcmp (typestr, "application/metalink4+xml"))
+        {
+          xfree (typestr);
+          goto skip_content_type;
+        }
+
+      /*
+        Valid ranges for the "pri" attribute are from
+        1 to 999999.  Mirror servers with a lower value of the "pri"
+        attribute have a higher priority, while mirrors with an undefined
+        "pri" attribute are considered to have a value of 999999, which is
+        the lowest priority.
+
+        rfc6249 section 3.1
+      */
+      murl.priority = DEFAULT_PRI;
+
+      murl.mediatype = typestr;
+      typestr = NULL;
+
+      if (opt.content_disposition
+          && resp_header_locate (resp, "Content-Disposition", 0, &val_beg, &val_end) != -1)
+        {
+          find_key_value (val_beg, val_end, "filename", &namestr);
+          murl.name = namestr;
+          namestr = NULL;
+        }
+
+      murl.url = xstrdup (u->url);
+
+      DEBUGP (("URL=%s\n", murl.url));
+      DEBUGP (("MEDIATYPE=%s\n", murl.mediatype));
+      DEBUGP (("NAME=%s\n", murl.name ? murl.name : ""));
+      DEBUGP (("PRIORITY=%d\n", murl.priority));
+
+      /* 1 slot from new resource, 1 slot for null-termination.  */
+      mfile->metaurls = xrealloc (mfile->metaurls,
+                                  sizeof (metalink_metaurl_t *) * (meta_count + 2));
+      mfile->metaurls[meta_count] = xnew0 (metalink_metaurl_t);
+      *mfile->metaurls[meta_count] = murl;
+      meta_count++;
+    }
+skip_content_type:
 
   /* Find all Link headers.  */
   for (i = 0;
@@ -2606,14 +2731,14 @@ metalink_from_http (const struct response *resp, const struct http_stat *hs,
       DEBUGP (("URL=%s\n", urlstr));
       DEBUGP (("rel=%s\n", rel));
 
+      if (!strcmp (rel, "describedby"))
+        find_key_value (attrs_beg, val_end, "type", &reltype);
+
       /* Handle signatures.
          Libmetalink only supports one signature per file. Therefore we stop
          as soon as we successfully get first supported signature.  */
       if (sig_count == 0 &&
-          !strcmp (rel, "describedby") &&
-          find_key_value (attrs_beg, val_end, "type", &reltype) &&
-          !strcmp (reltype, "application/pgp-signature")
-          )
+          reltype && !strcmp (reltype, "application/pgp-signature"))
         {
           /* Download the signature to a temporary file.  */
           FILE *_output_stream = output_stream;
@@ -2779,6 +2904,60 @@ metalink_from_http (const struct response *resp, const struct http_stat *hs,
               res_count++;
             }
         } /* Handle resource link (rel=duplicate).  */
+
+      /* Handle Metalink/XML resources.  */
+      else if (reltype && !strcmp (reltype, "application/metalink4+xml"))
+        {
+          metalink_metaurl_t murl = {0};
+          char *pristr;
+
+          /*
+             Valid ranges for the "pri" attribute are from
+             1 to 999999.  Mirror servers with a lower value of the "pri"
+             attribute have a higher priority, while mirrors with an undefined
+             "pri" attribute are considered to have a value of 999999, which is
+             the lowest priority.
+
+             rfc6249 section 3.1
+           */
+          murl.priority = DEFAULT_PRI;
+          if (find_key_value (url_end, val_end, "pri", &pristr))
+            {
+              long pri;
+              char *end_pristr;
+              /* Do not care for errno since 0 is error in this case.  */
+              pri = strtol (pristr, &end_pristr, 10);
+              if (end_pristr != pristr + strlen (pristr) ||
+                  !VALID_PRI_RANGE (pri))
+                {
+                  /* This is against the specification, so let's inform the user.  */
+                  logprintf (LOG_NOTQUIET,
+                             _("Invalid pri value. Assuming %d.\n"),
+                             DEFAULT_PRI);
+                }
+              else
+                murl.priority = pri;
+              xfree (pristr);
+            }
+
+          murl.mediatype = xstrdup (reltype);
+
+          DEBUGP (("MEDIATYPE=%s\n", murl.mediatype));
+
+          /* At this point we have validated the new resource.  */
+
+          find_key_value (url_end, val_end, "name", &murl.name);
+
+          murl.url = urlstr;
+          urlstr = NULL;
+
+          /* 1 slot from new resource, 1 slot for null-termination.  */
+          mfile->metaurls = xrealloc (mfile->metaurls,
+                                       sizeof (metalink_metaurl_t *) * (meta_count + 2));
+          mfile->metaurls[meta_count] = xnew0 (metalink_metaurl_t);
+          *mfile->metaurls[meta_count] = murl;
+          meta_count++;
+        } /* Handle resource link (rel=describedby).  */
       else
         DEBUGP (("This link header was not used for Metalink\n"));
 
@@ -2789,8 +2968,9 @@ metalink_from_http (const struct response *resp, const struct http_stat *hs,
 
   /* Null-terminate resources array.  */
   mfile->resources[res_count] = 0;
+  mfile->metaurls[meta_count] = 0;
 
-  if (res_count == 0)
+  if (res_count == 0 && meta_count == 0)
     {
       DEBUGP (("No valid metalink references found.\n"));
       goto fail;
@@ -2816,9 +2996,17 @@ metalink_from_http (const struct response *resp, const struct http_stat *hs,
              Therefore we convert: base64 -> binary -> hex.  */
           const size_t dig_hash_str_len = strlen (dig_hash);
           char *bin_hash = alloca (dig_hash_str_len * 3 / 4 + 1);
-          size_t hash_bin_len;
+          ssize_t hash_bin_len;
 
-          hash_bin_len = base64_decode (dig_hash, bin_hash);
+          hash_bin_len = wget_base64_decode (dig_hash, bin_hash);
+
+          /* Detect malformed base64 input.  */
+          if (hash_bin_len < 0)
+            {
+              xfree (dig_type);
+              xfree (dig_hash);
+              continue;
+            }
 
           /* One slot for me, one for zero-termination.  */
           mfile->checksums =
@@ -2827,8 +3015,8 @@ metalink_from_http (const struct response *resp, const struct http_stat *hs,
           mfile->checksums[hash_count] = xnew (metalink_checksum_t);
           mfile->checksums[hash_count]->type = dig_type;
 
-          mfile->checksums[hash_count]->hash = xmalloc (hash_bin_len * 2 + 1);
-          wg_hex_to_string (mfile->checksums[hash_count]->hash, bin_hash, hash_bin_len);
+          mfile->checksums[hash_count]->hash = xmalloc ((size_t)hash_bin_len * 2 + 1);
+          wg_hex_to_string (mfile->checksums[hash_count]->hash, bin_hash, (size_t)hash_bin_len);
 
           xfree (dig_hash);
 
@@ -2845,7 +3033,7 @@ metalink_from_http (const struct response *resp, const struct http_stat *hs,
 
     rfc6249 section 6
    */
-  if (hash_count == 0)
+  if (res_count && hash_count == 0)
     {
       logputs (LOG_VERBOSE,
                _("Could not find acceptable digest for Metalink resources.\n"
@@ -2856,6 +3044,7 @@ metalink_from_http (const struct response *resp, const struct http_stat *hs,
   /* Metalink data is OK. Now we just need to sort the resources based
      on their priorities, preference, and perhaps location.  */
   stable_sort (mfile->resources, res_count, sizeof (metalink_resource_t *), metalink_res_cmp);
+  stable_sort (mfile->metaurls, meta_count, sizeof (metalink_metaurl_t *), metalink_meta_cmp);
 
   /* Restore sensible preference values (in case someone cares to look).  */
   for (i = 0; i < res_count; ++i)
@@ -2892,8 +3081,8 @@ fail:
    If PROXY is non-NULL, the connection will be made to the proxy
    server, and u->url will be requested.  */
 static uerr_t
-gethttp (struct url *u, struct http_stat *hs, int *dt, struct url *proxy,
-         struct iri *iri, int count)
+gethttp (const struct url *u, struct url *original_url, struct http_stat *hs,
+         int *dt, struct url *proxy, struct iri *iri, int count)
 {
   struct request *req = NULL;
 
@@ -2903,7 +3092,7 @@ gethttp (struct url *u, struct http_stat *hs, int *dt, struct url *proxy,
   int statcode;
   int write_error;
   wgint contlen, contrange;
-  struct url *conn;
+  const struct url *conn;
   FILE *fp;
   int err;
   uerr_t retval;
@@ -2997,7 +3186,7 @@ gethttp (struct url *u, struct http_stat *hs, int *dt, struct url *proxy,
   hs->res = -1;
   hs->rderrmsg = NULL;
   hs->newloc = NULL;
-  xfree(hs->remote_time);
+  xfree (hs->remote_time);
   hs->error = NULL;
   hs->message = NULL;
 
@@ -3136,7 +3325,7 @@ gethttp (struct url *u, struct http_stat *hs, int *dt, struct url *proxy,
       bool warc_result;
 
       /* Generate a timestamp and uuid for this request. */
-      warc_timestamp (warc_timestamp_str, sizeof(warc_timestamp_str));
+      warc_timestamp (warc_timestamp_str, sizeof (warc_timestamp_str));
       warc_uuid_str (warc_request_uuid);
 
       /* Create a request record and store it in the WARC file. */
@@ -3182,7 +3371,7 @@ gethttp (struct url *u, struct http_stat *hs, int *dt, struct url *proxy,
         resp = resp_new (head);
 
         /* Check for status line.  */
-        xfree(message);
+        xfree (message);
         statcode = resp_status (resp, &message);
         if (statcode < 0)
           {
@@ -3211,7 +3400,7 @@ gethttp (struct url *u, struct http_stat *hs, int *dt, struct url *proxy,
     while (_repeat);
   }
 
-  xfree(hs->message);
+  xfree (hs->message);
   hs->message = xstrdup (message);
   if (!opt.server_response)
     logprintf (LOG_VERBOSE, "%2d %s\n", statcode,
@@ -3291,6 +3480,9 @@ gethttp (struct url *u, struct http_stat *hs, int *dt, struct url *proxy,
   if (metalink)
     {
       hs->metalink = metalink_from_http (resp, hs, u);
+      /* Bugfix: hs->local_file is NULL (opt.content_disposition).  */
+      if (!hs->local_file && hs->metalink && hs->metalink->origin)
+        hs->local_file = xstrdup (hs->metalink->origin);
       xfree (hs->message);
       retval = RETR_WITH_METALINK;
       CLOSE_FINISH (sock);
@@ -3403,16 +3595,16 @@ gethttp (struct url *u, struct http_stat *hs, int *dt, struct url *proxy,
         {
           /* process strict transport security */
           if (hsts_store_entry (hsts_store, u->scheme, u->host, u->port, max_age, include_subdomains))
-            DEBUGP(("Added new HSTS host: %s:%u (max-age: %u, includeSubdomains: %s)\n",
+            DEBUGP(("Added new HSTS host: %s:%u (max-age: %lu, includeSubdomains: %s)\n",
                    u->host,
-                   u->port,
-                   (unsigned int) max_age,
+                   (unsigned) u->port,
+                   (unsigned long) max_age,
                    (include_subdomains ? "true" : "false")));
           else
-            DEBUGP(("Updated HSTS host: %s:%u (max-age: %u, includeSubdomains: %s)\n",
+            DEBUGP(("Updated HSTS host: %s:%u (max-age: %lu, includeSubdomains: %s)\n",
                    u->host,
-                   u->port,
-                   (unsigned int) max_age,
+                   (unsigned) u->port,
+                   (unsigned long) max_age,
                    (include_subdomains ? "true" : "false")));
         }
     }
@@ -3440,7 +3632,7 @@ gethttp (struct url *u, struct http_stat *hs, int *dt, struct url *proxy,
               tmp = parse_charset (tmp2);
               if (tmp)
                 set_content_encoding (iri, tmp);
-              xfree(tmp);
+              xfree (tmp);
             }
 #endif
         }
@@ -3646,7 +3838,7 @@ gethttp (struct url *u, struct http_stat *hs, int *dt, struct url *proxy,
       goto cleanup;
     }
   if ((contrange != 0 && contrange != hs->restval)
-      || (H_PARTIAL (statcode) && !contrange))
+      || (H_PARTIAL (statcode) && !contrange && hs->restval))
     {
       /* The Range request was somehow misunderstood by the server.
          Bail out.  */
@@ -3754,6 +3946,16 @@ gethttp (struct url *u, struct http_stat *hs, int *dt, struct url *proxy,
       goto cleanup;
     }
 
+#ifdef ENABLE_XATTR
+  if (opt.enable_xattr)
+    {
+      if (original_url != u)
+        set_file_metadata (u->url, original_url->url, fp);
+      else
+        set_file_metadata (u->url, NULL, fp);
+    }
+#endif
+
   err = read_response_body (hs, sock, fp, contlen, contrange,
                             chunked_transfer_encoding,
                             u->url, warc_timestamp_str,
@@ -3783,7 +3985,7 @@ gethttp (struct url *u, struct http_stat *hs, int *dt, struct url *proxy,
 /* The genuine HTTP loop!  This is the part where the retrieval is
    retried, and retried, and retried, and...  */
 uerr_t
-http_loop (struct url *u, struct url *original_url, char **newloc,
+http_loop (const struct url *u, struct url *original_url, char **newloc,
            char **local_file, const char *referer, int *dt, struct url *proxy,
            struct iri *iri)
 {
@@ -3796,7 +3998,7 @@ http_loop (struct url *u, struct url *original_url, char **newloc,
   uerr_t err, ret = TRYLIMEXC;
   time_t tmr = -1;               /* remote time-stamp */
   struct http_stat hstat;        /* HTTP status */
-  struct_stat st;
+  struct stat st;
   bool send_head_first = true;
   bool force_full_retrieve = false;
 
@@ -3972,7 +4174,7 @@ http_loop (struct url *u, struct url *original_url, char **newloc,
         *dt &= ~SEND_NOCACHE;
 
       /* Try fetching the document, or at least its head.  */
-      err = gethttp (u, &hstat, dt, proxy, iri, count);
+      err = gethttp (u, original_url, &hstat, dt, proxy, iri, count);
 
       /* Time?  */
       tms = datetime_str (time (NULL));
@@ -4400,7 +4602,10 @@ exit:
   if ((ret == RETROK || opt.content_on_error) && local_file)
     {
       xfree (*local_file);
-      *local_file = xstrdup (hstat.local_file);
+      /* Bugfix: Prevent SIGSEGV when hstat.local_file was left NULL
+         (i.e. due to opt.content_disposition).  */
+      if (hstat.local_file)
+        *local_file = xstrdup (hstat.local_file);
     }
   free_hstat (&hstat);
 
@@ -4546,7 +4751,7 @@ basic_authentication_encode (const char *user, const char *passwd)
   sprintf (t1, "%s:%s", user, passwd);
 
   t2 = (char *)alloca (BASE64_LENGTH (len1) + 1);
-  base64_encode (t1, len1, t2);
+  wget_base64_encode (t1, len1, t2);
 
   return concat_strings ("Basic ", t2, (char *) 0);
 }
@@ -4593,7 +4798,7 @@ digest_authentication_encode (const char *au, const char *user,
     { "algorithm", &algorithm }
   };
   char cnonce[16] = "";
-  char *res;
+  char *res = NULL;
   int res_len;
   size_t res_size;
   param_token name, value;
@@ -4616,29 +4821,22 @@ digest_authentication_encode (const char *au, const char *user,
           }
     }
 
-  if (qop != NULL && strcmp(qop,"auth"))
+  if (qop && strcmp (qop, "auth"))
     {
       logprintf (LOG_NOTQUIET, _("Unsupported quality of protection '%s'.\n"), qop);
-      xfree (qop); /* force freeing mem and return */
+      xfree (qop); /* force freeing mem and continue */
     }
-  else if (algorithm != NULL && strcmp (algorithm,"MD5") && strcmp (algorithm,"MD5-sess"))
+  else if (algorithm && strcmp (algorithm,"MD5") && strcmp (algorithm,"MD5-sess"))
     {
       logprintf (LOG_NOTQUIET, _("Unsupported algorithm '%s'.\n"), algorithm);
-      xfree (qop); /* force freeing mem and return */
+      xfree (algorithm); /* force freeing mem and continue */
     }
 
   if (!realm || !nonce || !user || !passwd || !path || !method)
     {
       *auth_err = ATTRMISSING;
-
-      xfree (realm);
-      xfree (opaque);
-      xfree (nonce);
-      xfree (qop);
-      xfree (algorithm);
-
-      return NULL;
-    }
+      goto cleanup;
+   }
 
   /* Calculate the digest value.  */
   {
@@ -4661,7 +4859,8 @@ digest_authentication_encode (const char *au, const char *user,
     if (algorithm && !strcmp (algorithm, "MD5-sess"))
       {
         /* A1BUF = H( H(user ":" realm ":" password) ":" nonce ":" cnonce ) */
-        snprintf (cnonce, sizeof (cnonce), "%08x", random_number(INT_MAX));
+        snprintf (cnonce, sizeof (cnonce), "%08x",
+          (unsigned) random_number (INT_MAX));
 
         md5_init_ctx (&ctx);
         /* md5_process_bytes (hash, MD5_DIGEST_SIZE, &ctx); */
@@ -4683,12 +4882,13 @@ digest_authentication_encode (const char *au, const char *user,
     md5_finish_ctx (&ctx, hash);
     dump_hash (a2buf, hash);
 
-    if (qop && !strcmp(qop, "auth"))
+    if (qop && !strcmp (qop, "auth"))
       {
         /* RFC 2617 Digest Access Authentication */
         /* generate random hex string */
         if (!*cnonce)
-          snprintf(cnonce, sizeof(cnonce), "%08x", random_number(INT_MAX));
+          snprintf (cnonce, sizeof (cnonce), "%08x",
+            (unsigned) random_number (INT_MAX));
 
         /* RESPONSE_DIGEST = H(A1BUF ":" nonce ":" noncecount ":" clientnonce ":" qop ": " A2BUF) */
         md5_init_ctx (&ctx);
@@ -4698,9 +4898,9 @@ digest_authentication_encode (const char *au, const char *user,
         md5_process_bytes ((unsigned char *)":", 1, &ctx);
         md5_process_bytes ((unsigned char *)"00000001", 8, &ctx); /* TODO: keep track of server nonce values */
         md5_process_bytes ((unsigned char *)":", 1, &ctx);
-        md5_process_bytes ((unsigned char *)cnonce, strlen(cnonce), &ctx);
+        md5_process_bytes ((unsigned char *)cnonce, strlen (cnonce), &ctx);
         md5_process_bytes ((unsigned char *)":", 1, &ctx);
-        md5_process_bytes ((unsigned char *)qop, strlen(qop), &ctx);
+        md5_process_bytes ((unsigned char *)qop, strlen (qop), &ctx);
         md5_process_bytes ((unsigned char *)":", 1, &ctx);
         md5_process_bytes ((unsigned char *)a2buf, MD5_DIGEST_SIZE * 2, &ctx);
         md5_finish_ctx (&ctx, hash);
@@ -4750,15 +4950,16 @@ digest_authentication_encode (const char *au, const char *user,
 
     if (opaque)
       {
-        res_len += snprintf(res + res_len, res_size - res_len, ", opaque=\"%s\"", opaque);
+        res_len += snprintf (res + res_len, res_size - res_len, ", opaque=\"%s\"", opaque);
       }
 
     if (algorithm)
       {
-        snprintf(res + res_len, res_size - res_len, ", algorithm=\"%s\"", algorithm);
+        snprintf (res + res_len, res_size - res_len, ", algorithm=\"%s\"", algorithm);
       }
   }
 
+cleanup:
   xfree (realm);
   xfree (opaque);
   xfree (nonce);
@@ -4904,7 +5105,7 @@ ensure_extension (struct http_stat *hs, const char *ext, int *dt)
 #ifdef TESTING
 
 const char *
-test_parse_range_header(void)
+test_parse_range_header (void)
 {
   unsigned i;
   static const struct {
@@ -4922,8 +5123,10 @@ test_parse_range_header(void)
       { "bytes 1-999/1000", 1, 999, 1000, true },
       { "bytes 42-1233/1234", 42, 1233, 1234, true },
       { "bytes 42-1233/*", 42, 1233, -1, true },
-      { "bytes 0-2147483648/2147483649", 0, 2147483648, 2147483649, true },
-      { "bytes 2147483648-4294967296/4294967297", 2147483648, 4294967296, 4294967297, true }
+      { "bytes 0-2147483648/2147483649", 0, 2147483648U, 2147483649U, true },
+#if SIZEOF_WGINT >= 8
+      { "bytes 2147483648-4294967296/4294967297", 2147483648U, 4294967296ULL, 4294967297ULL, true },
+#endif
   };
 
   wgint firstbyteptr[sizeof(wgint)];
@@ -4949,7 +5152,7 @@ test_parse_range_header(void)
 }
 
 const char *
-test_parse_content_disposition(void)
+test_parse_content_disposition (void)
 {
   unsigned i;
   static const struct {
@@ -4971,7 +5174,7 @@ test_parse_content_disposition(void)
 filename*1=\"B\"", "AA.ext", true },
   };
 
-  for (i = 0; i < countof(test_array); ++i)
+  for (i = 0; i < countof (test_array); ++i)
     {
       char *filename;
       bool res;

src/http.h

@@ -35,7 +35,7 @@ as that of the covered work.  */
 
 struct url;
 
-uerr_t http_loop (struct url *, struct url *, char **, char **, const char *,
+uerr_t http_loop (const struct url *, struct url *, char **, char **, const char *,
                   int *, struct url *, struct iri *);
 void save_cookies (void);
 void http_cleanup (void);

src/init.c

@@ -97,6 +97,8 @@ CMD_DECLARE (cmd_directory);
 CMD_DECLARE (cmd_time);
 CMD_DECLARE (cmd_vector);
 
+CMD_DECLARE (cmd_use_askpass);
+
 CMD_DECLARE (cmd_spec_dirstruct);
 CMD_DECLARE (cmd_spec_header);
 CMD_DECLARE (cmd_spec_warc_header);
@@ -210,7 +212,7 @@ static const struct {
   { "header",           NULL,                   cmd_spec_header },
 #ifdef HAVE_HSTS
   { "hsts",             &opt.hsts,              cmd_boolean },
-  { "hsts-file",        &opt.hsts_file,         cmd_file },
+  { "hstsfile",         &opt.hsts_file,         cmd_file },
 #endif
   { "htmlextension",    &opt.adjust_extension,  cmd_boolean }, /* deprecated */
   { "htmlify",          NULL,                   cmd_spec_htmlify },
@@ -223,7 +225,7 @@ static const struct {
 #endif
   { "httpsproxy",       &opt.https_proxy,       cmd_string },
   { "httpuser",         &opt.http_user,         cmd_string },
-  { "if-modified-since", &opt.if_modified_since, cmd_boolean },
+  { "ifmodifiedsince",  &opt.if_modified_since, cmd_boolean },
   { "ignorecase",       &opt.ignore_case,       cmd_boolean },
   { "ignorelength",     &opt.ignore_length,     cmd_boolean },
   { "ignoretags",       &opt.ignore_tags,       cmd_vector },
@@ -234,9 +236,10 @@ static const struct {
 #endif
   { "input",            &opt.input_filename,    cmd_file },
 #ifdef HAVE_METALINK
-  { "input-metalink",   &opt.input_metalink,    cmd_file },
+  { "inputmetalink",    &opt.input_metalink,    cmd_file },
 #endif
   { "iri",              &opt.enable_iri,        cmd_boolean },
+  { "keepbadhash",      &opt.keep_badhash,      cmd_boolean },
   { "keepsessioncookies", &opt.keep_session_cookies, cmd_boolean },
   { "limitrate",        &opt.limit_rate,        cmd_bytes },
   { "loadcookies",      &opt.cookies_input,     cmd_file },
@@ -245,7 +248,8 @@ static const struct {
   { "login",            &opt.ftp_user,          cmd_string },/* deprecated*/
   { "maxredirect",      &opt.max_redirect,      cmd_number },
 #ifdef HAVE_METALINK
-  { "metalink-over-http", &opt.metalink_over_http, cmd_boolean },
+  { "metalinkindex",    &opt.metalink_index,     cmd_number_inf },
+  { "metalinkoverhttp", &opt.metalink_over_http, cmd_boolean },
 #endif
   { "method",           &opt.method,            cmd_string_uppercase },
   { "mirror",           NULL,                   cmd_spec_mirror },
@@ -267,7 +271,7 @@ static const struct {
   { "postfile",         &opt.post_file_name,    cmd_file },
   { "preferfamily",     NULL,                   cmd_spec_prefer_family },
 #ifdef HAVE_METALINK
-  { "preferred-location", &opt.preferred_location, cmd_string },
+  { "preferredlocation", &opt.preferred_location, cmd_string },
 #endif
   { "preservepermissions", &opt.preserve_perm,  cmd_boolean },
 #ifdef HAVE_SSL
@@ -318,6 +322,7 @@ static const struct {
   { "tries",            &opt.ntry,              cmd_number_inf },
   { "trustservernames", &opt.trustservernames,  cmd_boolean },
   { "unlink",           &opt.unlink,            cmd_boolean },
+  { "useaskpass" ,      &opt.use_askpass,       cmd_use_askpass },
   { "useproxy",         &opt.use_proxy,         cmd_boolean },
   { "user",             &opt.user,              cmd_string },
   { "useragent",        NULL,                   cmd_spec_useragent },
@@ -339,6 +344,9 @@ static const struct {
 #ifdef USE_WATT32
   { "wdebug",           &opt.wdebug,            cmd_boolean },
 #endif
+#ifdef ENABLE_XATTR
+  { "xattr",            &opt.enable_xattr,      cmd_boolean },
+#endif
 };
 
 /* Look up CMDNAME in the commands[] and return its position in the
@@ -378,6 +386,10 @@ defaults (void)
      bit pattern will be the least of the implementors' worries.  */
   xzero (opt);
 
+#ifdef HAVE_METALINK
+  opt.metalink_index = -1;
+#endif
+
   opt.cookies = true;
   opt.verbose = -1;
   opt.ntry = 20;
@@ -482,6 +494,12 @@ defaults (void)
   /* HSTS is enabled by default */
   opt.hsts = true;
 #endif
+
+#ifdef ENABLE_XATTR
+  opt.enable_xattr = true;
+#else
+  opt.enable_xattr = false;
+#endif
 }
 
 /* Return the user's home directory (strdup-ed), or NULL if none is
@@ -859,6 +877,10 @@ static bool
 setval_internal (int comind, const char *com, const char *val)
 {
   assert (0 <= comind && ((size_t) comind) < countof (commands));
+
+  if ((unsigned) comind >= countof (commands))
+    return NULL;
+
   DEBUGP (("Setting %s (%s) to %s\n", com, commands[comind].name, val));
   return commands[comind].action (com, val, commands[comind].place);
 }
@@ -1371,6 +1393,32 @@ cmd_time (const char *com, const char *val, void *place)
   return true;
 }
 
+
+static bool
+cmd_use_askpass (const char *com _GL_UNUSED, const char *val, void *place)
+{
+  const char *env_name = "WGET_ASKPASS";
+  const char *env;
+
+  if (val && *val)
+    return cmd_string (com, val, place);
+
+  env = getenv (env_name);
+  if (!(env && *env))
+    {
+      env_name = "SSH_ASKPASS";
+      env = getenv (env_name);
+    }
+
+  if (!(env && *env))
+    {
+      fprintf (stderr, _("use-askpass requires a string or either environment variable WGET_ASKPASS or SSH_ASKPASS to be set.\n"));
+      exit (WGET_EXIT_GENERIC_ERROR);
+    }
+
+  return cmd_string (com, env, place);
+}
+
 #ifdef HAVE_SSL
 static bool
 cmd_cert_type (const char *com, const char *val, void *place)
@@ -1930,6 +1978,7 @@ cleanup (void)
   xfree (opt.body_data);
   xfree (opt.body_file);
   xfree (opt.rejected_log);
+  xfree (opt.use_askpass);
 
 #ifdef HAVE_LIBCARES
 #include <ares.h>

src/iri.c

@@ -33,11 +33,14 @@ as that of the covered work.  */
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
-#include <iconv.h>
-#include <stringprep.h>
-#include <idna.h>
-#include <idn-free.h>
+#include <langinfo.h>
 #include <errno.h>
+#ifdef HAVE_ICONV
+# include <iconv.h>
+#endif
+#include <idn2.h>
+#include <unicase.h>
+#include <unistr.h>
 
 #include "utils.h"
 #include "url.h"
@@ -45,9 +48,6 @@ as that of the covered work.  */
 #include "c-strcasestr.h"
 #include "xstrndup.h"
 
-/* RFC3987 section 3.1 mandates STD3 ASCII RULES */
-#define IDNA_FLAGS  IDNA_USE_STD3_ASCII_RULES
-
 /* Note: locale encoding is kept in options struct (opt.locale) */
 
 /* Given a string containing "charset=XXX", return the encoding if found,
@@ -88,10 +88,15 @@ parse_charset (const char *str)
 }
 
 /* Find the locale used, or fall back on a default value */
-char *
+const char *
 find_locale (void)
 {
-  return (char *) stringprep_locale_charset ();
+	const char *encoding = nl_langinfo(CODESET);
+
+	if (!encoding || !*encoding)
+		return "ASCII";
+
+   return encoding;
 }
 
 /* Basic check of an encoding name. */
@@ -146,7 +151,7 @@ do_conversion (const char *tocode, const char *fromcode, char const *in_org, siz
 
   for (;;)
     {
-      if (iconv (cd, &in, &inlen, out, &outlen) != (size_t)(-1) &&
+      if (iconv (cd, (ICONV_CONST char **) &in, &inlen, out, &outlen) != (size_t)(-1) &&
           iconv (cd, NULL, NULL, out, &outlen) != (size_t)(-1))
         {
           *out = s;
@@ -159,7 +164,7 @@ do_conversion (const char *tocode, const char *fromcode, char const *in_org, siz
             if (!strchr(in_org, '@') && !strchr(*out, '@'))
               debug_logprintf ("converted '%s' (%s) -> '%s' (%s)\n", in_org, fromcode, *out, tocode);
             else
-              debug_logprintf ("%s: logging suppressed, strings may contain password\n", __func__);
+              debug_logprintf ("logging suppressed, strings may contain password\n");
           }
           return true;
         }
@@ -201,7 +206,7 @@ do_conversion (const char *tocode, const char *fromcode, char const *in_org, siz
       if (!strchr(in_org, '@') && !strchr(*out, '@'))
         debug_logprintf ("converted '%s' (%s) -> '%s' (%s)\n", in_org, fromcode, *out, tocode);
       else
-        debug_logprintf ("%s: logging suppressed, strings may contain password\n", __func__);
+        debug_logprintf ("logging suppressed, strings may contain password\n");
     }
     return false;
 }
@@ -230,50 +235,6 @@ locale_to_utf8 (const char *str)
   return str;
 }
 
-/*
- * Work around a libidn <= 1.30 vulnerability.
- *
- * The function checks for a valid UTF-8 character sequence before
- * passing it to idna_to_ascii_8z().
- *
- * [1] http://lists.gnu.org/archive/html/help-libidn/2015-05/msg00002.html
- * [2] https://lists.gnu.org/archive/html/bug-wget/2015-06/msg00002.html
- * [3] http://curl.haxx.se/mail/lib-2015-06/0143.html
- */
-static bool
-_utf8_is_valid(const char *utf8)
-{
-  const unsigned char *s = (const unsigned char *) utf8;
-
-  while (*s)
-    {
-      if ((*s & 0x80) == 0) /* 0xxxxxxx ASCII char */
-        s++;
-      else if ((*s & 0xE0) == 0xC0) /* 110xxxxx 10xxxxxx */
-        {
-          if ((s[1] & 0xC0) != 0x80)
-            return false;
-          s+=2;
-        }
-      else if ((*s & 0xF0) == 0xE0) /* 1110xxxx 10xxxxxx 10xxxxxx */
-        {
-          if ((s[1] & 0xC0) != 0x80 || (s[2] & 0xC0) != 0x80)
-            return false;
-          s+=3;
-        }
-      else if ((*s & 0xF8) == 0xF0) /* 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx */
-        {
-          if ((s[1] & 0xC0) != 0x80 || (s[2] & 0xC0) != 0x80 || (s[3] & 0xC0) != 0x80)
-            return false;
-          s+=4;
-        }
-      else
-        return false;
-    }
-
-  return true;
-}
-
 /* Try to "ASCII encode" UTF-8 host. Return the new domain on success or NULL
    on error. */
 char *
@@ -282,34 +243,50 @@ idn_encode (const struct iri *i, const char *host)
   int ret;
   char *ascii_encoded;
   char *utf8_encoded = NULL;
+  const char *src;
+#if IDN2_VERSION_NUMBER < 0x00140000
+  uint8_t *lower;
+  size_t len = 0;
+#endif
 
   /* Encode to UTF-8 if not done */
   if (!i->utf8_encode)
     {
       if (!remote_to_utf8 (i, host, &utf8_encoded))
           return NULL;  /* Nothing to encode or an error occured */
+      src = utf8_encoded;
     }
+  else
+    src = host;
 
-  if (!_utf8_is_valid(utf8_encoded ? utf8_encoded : host))
+#if IDN2_VERSION_NUMBER >= 0x00140000
+  /* IDN2_TRANSITIONAL implies input NFC encoding */
+  if ((ret = idn2_lookup_u8 ((uint8_t *) src, (uint8_t **) &ascii_encoded, IDN2_NONTRANSITIONAL)) != IDN2_OK)
     {
-      logprintf (LOG_VERBOSE, _("Invalid UTF-8 sequence: %s\n"),
-                 quote(utf8_encoded ? utf8_encoded : host));
+      logprintf (LOG_VERBOSE, _("idn_encode failed (%d): %s\n"), ret,
+                 quote (idn2_strerror (ret)));
+    }
+#else
+  /* we need a conversion to lowercase */
+  lower = u8_tolower ((uint8_t *) src, u8_strlen ((uint8_t *) src) + 1, 0, UNINORM_NFKC, NULL, &len);
+  if (!lower)
+    {
+      logprintf (LOG_VERBOSE, _("Failed to convert to lower: %d: %s\n"),
+                 errno, quote (src));
       xfree (utf8_encoded);
       return NULL;
     }
 
-  /* Store in ascii_encoded the ASCII UTF-8 NULL terminated string */
-  ret = idna_to_ascii_8z (utf8_encoded ? utf8_encoded : host, &ascii_encoded, IDNA_FLAGS);
-  xfree (utf8_encoded);
-
-  if (ret != IDNA_SUCCESS)
+  if ((ret = idn2_lookup_u8 (lower, (uint8_t **) &ascii_encoded, IDN2_NFC_INPUT)) != IDN2_OK)
     {
       logprintf (LOG_VERBOSE, _("idn_encode failed (%d): %s\n"), ret,
-                 quote (idna_strerror (ret)));
-      return NULL;
+                 quote (idn2_strerror (ret)));
     }
 
-  return ascii_encoded;
+  xfree (lower);
+#endif
+
+  return ret == IDN2_OK ? ascii_encoded : NULL;
 }
 
 /* Try to decode an "ASCII encoded" host. Return the new domain in the locale
@@ -317,18 +294,24 @@ idn_encode (const struct iri *i, const char *host)
 char *
 idn_decode (const char *host)
 {
+/*
   char *new;
   int ret;
 
-  ret = idna_to_unicode_8zlz (host, &new, IDNA_FLAGS);
-  if (ret != IDNA_SUCCESS)
+  ret = idn2_register_u8 (NULL, host, (uint8_t **) &new, 0);
+  if (ret != IDN2_OK)
     {
-      logprintf (LOG_VERBOSE, _("idn_decode failed (%d): %s\n"), ret,
-                 quote (idna_strerror (ret)));
+      logprintf (LOG_VERBOSE, _("idn2_register_u8 failed (%d): %s: %s\n"), ret,
+                 quote (idn2_strerror (ret)), host);
       return NULL;
     }
 
   return new;
+*/
+  /* idn2_register_u8() just works label by label.
+   * That is pretty much overhead for just displaying the original ulabels.
+   * To keep at least the debug output format, return a cloned host. */
+  return xstrdup(host);
 }
 
 /* Try to transcode string str from remote encoding to UTF-8. On success, *new

src/iri.h

@@ -40,11 +40,10 @@ struct iri {
 
 #ifdef ENABLE_IRI
 
-# include <idna.h>
-# include <idn-free.h>
+# include <idn2.h>
 
 char *parse_charset (const char *str);
-char *find_locale (void);
+const char *find_locale (void);
 bool check_encoding_name (const char *encoding);
 const char *locale_to_utf8 (const char *str);
 char *idn_encode (const struct iri *i, const char *host);
@@ -66,7 +65,7 @@ extern struct iri dummy_iri;
 #define locale_to_utf8(str)         (str)
 #define idn_encode(a,b)             NULL
 #define idn_decode(str)             NULL
-#define idn_free(str)               ((void)0)
+#define idn2_free(str)              ((void)0)
 #define remote_to_utf8(a,b,c)       false
 #define iri_new()                   (&dummy_iri)
 #define iri_dup(a)                  (&dummy_iri)

src/log.c

@@ -80,6 +80,18 @@ as that of the covered work.  */
    logging is inhibited, logfp is set back to NULL. */
 static FILE *logfp;
 
+/* Descriptor of the stdout|stderr */
+static FILE *stdlogfp;
+
+/* Descriptor of the wget.log* file (if created) */
+static FILE *filelogfp;
+
+/* Name of log file */
+static char *logfile;
+
+/* Is interactive shell ? */
+static int shell_is_interactive;
+
 /* A second file descriptor pointing to the temporary log file for the
    WARC writer.  If WARC writing is disabled, this is NULL.  */
 static FILE *warclogfp;
@@ -611,16 +623,18 @@ log_init (const char *file, bool appendp)
     {
       if (HYPHENP (file))
         {
-          logfp = stdout;
+          stdlogfp = stdout;
+          logfp = stdlogfp;
         }
       else
         {
-          logfp = fopen (file, appendp ? "a" : "w");
-          if (!logfp)
+          filelogfp = fopen (file, appendp ? "a" : "w");
+          if (!filelogfp)
             {
               fprintf (stderr, "%s: %s: %s\n", exec_name, file, strerror (errno));
               exit (WGET_EXIT_GENERIC_ERROR);
             }
+          logfp = filelogfp;
         }
     }
   else
@@ -631,7 +645,8 @@ log_init (const char *file, bool appendp)
          stderr only if the user actually specifies `-O -'.  He says
          this inconsistency is harder to document, but is overall
          easier on the user.  */
-      logfp = stderr;
+      stdlogfp = stderr;
+      logfp = stdlogfp;
 
       if (1
 #ifdef HAVE_ISATTY
@@ -646,6 +661,11 @@ log_init (const char *file, bool appendp)
           save_context_p = true;
         }
     }
+
+#ifndef WINDOWS
+  /* Initialize this values so we don't have to ask every time we print line */
+  shell_is_interactive = isatty (STDIN_FILENO);
+#endif
 }
 
 /* Close LOGFP (only if we opened it, not if it's stderr), inhibit
@@ -880,59 +900,78 @@ log_cleanup (void)
 
 /* When SIGHUP or SIGUSR1 are received, the output is redirected
    elsewhere.  Such redirection is only allowed once. */
-static enum { RR_NONE, RR_REQUESTED, RR_DONE } redirect_request = RR_NONE;
 static const char *redirect_request_signal_name;
 
-/* Redirect output to `wget-log'.  */
+/* Redirect output to `wget-log' or back to stdout/stderr.  */
 
-static void
-redirect_output (void)
+void
+redirect_output (bool to_file, const char *signal_name)
 {
-  char *logfile;
-  logfp = unique_create (DEFAULT_LOGFILE, false, &logfile);
-  if (logfp)
+  if (to_file && logfp != filelogfp)
     {
-      fprintf (stderr, _("\n%s received, redirecting output to %s.\n"),
-               redirect_request_signal_name, quote (logfile));
-      xfree (logfile);
-      /* Dump the context output to the newly opened log.  */
+      if (signal_name)
+        {
+          fprintf (stderr, "\n%s received.", signal_name);
+        }
+      if (!filelogfp)
+        {
+          filelogfp = unique_create (DEFAULT_LOGFILE, false, &logfile);
+          if (filelogfp)
+            {
+              fprintf (stderr, _("\nRedirecting output to %s.\n"),
+                  quote (logfile));
+              /* Store signal name to tell wget it's permanent redirect to log file */
+              redirect_request_signal_name = signal_name;
+              logfp = filelogfp;
+              /* Dump the context output to the newly opened log.  */
+              log_dump_context ();
+            }
+          else
+            {
+              /* Eek!  Opening the alternate log file has failed.  Nothing we
+                can do but disable printing completely. */
+              fprintf (stderr, _("%s: %s; disabling logging.\n"),
+                      (logfile) ? logfile : DEFAULT_LOGFILE, strerror (errno));
+              inhibit_logging = true;
+            }
+        }
+      else
+        {
+          fprintf (stderr, _("\nRedirecting output to %s.\n"),
+              quote (logfile));
+          logfp = filelogfp;
+          log_dump_context ();
+        }
+    }
+  else if (!to_file && logfp != stdlogfp)
+    {
+      logfp = stdlogfp;
       log_dump_context ();
     }
-  else
-    {
-      /* Eek!  Opening the alternate log file has failed.  Nothing we
-         can do but disable printing completely. */
-      fprintf (stderr, _("\n%s received.\n"), redirect_request_signal_name);
-      fprintf (stderr, _("%s: %s; disabling logging.\n"),
-               (logfile) ? logfile : DEFAULT_LOGFILE, strerror (errno));
-      inhibit_logging = true;
-    }
-  save_context_p = false;
 }
 
-/* Check whether a signal handler requested the output to be
-   redirected. */
+/* Check whether there's a need to redirect output. */
 
 static void
 check_redirect_output (void)
 {
-  if (redirect_request == RR_REQUESTED)
+#ifndef WINDOWS
+  /* If it was redirected already to log file by SIGHUP, SIGUSR1 or -o parameter,
+   * it was permanent.
+   * If there was no SIGHUP or SIGUSR1 and shell is interactive
+   * we check if process is fg or bg before every line is printed.*/
+  if (!redirect_request_signal_name && shell_is_interactive && !opt.lfilename)
     {
-      redirect_request = RR_DONE;
-      redirect_output ();
+      if (tcgetpgrp (STDIN_FILENO) != getpgrp ())
+        {
+          /* Process backgrounded */
+          redirect_output (true,NULL);
+        }
+      else
+        {
+          /* Process foregrounded */
+          redirect_output (false,NULL);
+        }
     }
-}
-
-/* Request redirection at a convenient time.  This may be called from
-   a signal handler. */
-
-void
-log_request_redirect_output (const char *signal_name)
-{
-  if (redirect_request == RR_NONE && save_context_p)
-    /* Request output redirection.  The request will be processed by
-       check_redirect_output(), which is called from entry point log
-       functions. */
-    redirect_request = RR_REQUESTED;
-  redirect_request_signal_name = signal_name;
+#endif /* WINDOWS */
 }

src/log.h

@@ -52,6 +52,7 @@ void log_init (const char *, bool);
 void log_close (void);
 void log_cleanup (void);
 void log_request_redirect_output (const char *);
+void redirect_output (bool, const char *);
 
 const char *escnonprint (const char *);
 const char *escnonprint_uri (const char *);

src/main.c

@@ -36,6 +36,7 @@ as that of the covered work.  */
 #include <unistd.h>
 #include <string.h>
 #include <signal.h>
+#include <spawn.h>
 #ifdef ENABLE_NLS
 # include <locale.h>
 #endif
@@ -60,6 +61,7 @@ as that of the covered work.  */
 #include "version.h"
 #include "c-strcase.h"
 #include "dirname.h"
+#include "xmemdup0.h"
 #include <getopt.h>
 #include <getpass.h>
 #include <quote.h>
@@ -130,7 +132,7 @@ redirect_output_signal (int sig)
     signal_name = "SIGUSR1";
 #endif
 
-  log_request_redirect_output (signal_name);
+  redirect_output (true,signal_name);
   progress_schedule_redirect ();
   signal (sig, redirect_output_signal);
 }
@@ -219,8 +221,8 @@ save_hsts (void)
 
 /* Definition of command-line options. */
 
-static void _Noreturn print_help (void);
-static void _Noreturn print_version (void);
+_Noreturn static void print_help (void);
+_Noreturn static void print_version (void);
 
 #ifdef HAVE_SSL
 # define IF_SSL(x) x
@@ -321,7 +323,7 @@ static struct cmdline_option option_data[] =
     { "host-directories", 0, OPT_BOOLEAN, "addhostdir", -1 },
 #ifdef HAVE_HSTS
     { "hsts", 0, OPT_BOOLEAN, "hsts", -1},
-    { "hsts-file", 0, OPT_VALUE, "hsts-file", -1 },
+    { "hsts-file", 0, OPT_VALUE, "hstsfile", -1 },
 #endif
     { "html-extension", 'E', OPT_BOOLEAN, "adjustextension", -1 }, /* deprecated */
     { "htmlify", 0, OPT_BOOLEAN, "htmlify", -1 },
@@ -340,9 +342,10 @@ static struct cmdline_option option_data[] =
 #endif
     { "input-file", 'i', OPT_VALUE, "input", -1 },
 #ifdef HAVE_METALINK
-    { "input-metalink", 0, OPT_VALUE, "input-metalink", -1 },
+    { "input-metalink", 0, OPT_VALUE, "inputmetalink", -1 },
 #endif
     { "iri", 0, OPT_BOOLEAN, "iri", -1 },
+    { "keep-badhash", 0, OPT_BOOLEAN, "keepbadhash", -1 },
     { "keep-session-cookies", 0, OPT_BOOLEAN, "keepsessioncookies", -1 },
     { "level", 'l', OPT_VALUE, "reclevel", -1 },
     { "limit-rate", 0, OPT_VALUE, "limitrate", -1 },
@@ -351,7 +354,8 @@ static struct cmdline_option option_data[] =
     { "rejected-log", 0, OPT_VALUE, "rejectedlog", -1 },
     { "max-redirect", 0, OPT_VALUE, "maxredirect", -1 },
 #ifdef HAVE_METALINK
-    { "metalink-over-http", 0, OPT_BOOLEAN, "metalink-over-http", -1 },
+    { "metalink-index", 0, OPT_VALUE, "metalinkindex", -1 },
+    { "metalink-over-http", 0, OPT_BOOLEAN, "metalinkoverhttp", -1 },
 #endif
     { "method", 0, OPT_VALUE, "method", -1 },
     { "mirror", 'm', OPT_BOOLEAN, "mirror", -1 },
@@ -370,7 +374,7 @@ static struct cmdline_option option_data[] =
     { "post-file", 0, OPT_VALUE, "postfile", -1 },
     { "prefer-family", 0, OPT_VALUE, "preferfamily", -1 },
 #ifdef HAVE_METALINK
-    { "preferred-location", 0, OPT_VALUE, "preferred-location", -1 },
+    { "preferred-location", 0, OPT_VALUE, "preferredlocation", -1 },
 #endif
     { "preserve-permissions", 0, OPT_BOOLEAN, "preservepermissions", -1 },
     { IF_SSL ("private-key"), 0, OPT_VALUE, "privatekey", -1 },
@@ -410,10 +414,11 @@ static struct cmdline_option option_data[] =
     { "strict-comments", 0, OPT_BOOLEAN, "strictcomments", -1 },
     { "timeout", 'T', OPT_VALUE, "timeout", -1 },
     { "timestamping", 'N', OPT_BOOLEAN, "timestamping", -1 },
-    { "if-modified-since", 0, OPT_BOOLEAN, "if-modified-since", -1 },
+    { "if-modified-since", 0, OPT_BOOLEAN, "ifmodifiedsince", -1 },
     { "tries", 't', OPT_VALUE, "tries", -1 },
     { "unlink", 0, OPT_BOOLEAN, "unlink", -1 },
     { "trust-server-names", 0, OPT_BOOLEAN, "trustservernames", -1 },
+    { "use-askpass", 0, OPT_VALUE, "useaskpass", -1},
     { "use-server-timestamps", 0, OPT_BOOLEAN, "useservertimestamps", -1 },
     { "user", 0, OPT_VALUE, "user", -1 },
     { "user-agent", 'U', OPT_VALUE, "useragent", -1 },
@@ -435,6 +440,9 @@ static struct cmdline_option option_data[] =
     { "warc-tempdir", 0, OPT_VALUE, "warctempdir", -1 },
 #ifdef USE_WATT32
     { "wdebug", 0, OPT_BOOLEAN, "wdebug", -1 },
+#endif
+#ifdef ENABLE_XATTR
+    { "xattr", 0, OPT_BOOLEAN, "xattr", -1 },
 #endif
   };
 
@@ -547,7 +555,7 @@ print_usage (int error)
 
 /* Print the help message, describing all the available options.  If
    you add an option, be sure to update this list.  */
-static void _Noreturn
+_Noreturn static void
 print_help (void)
 {
   /* We split the help text this way to ease translation of individual
@@ -690,6 +698,11 @@ Download:\n"),
        --password=PASS             set both ftp and http password to PASS\n"),
     N_("\
        --ask-password              prompt for passwords\n"),
+    N_("\
+       --use-askpass=COMMAND       specify credential handler for requesting \n\
+                                     username and password.  If no COMMAND is \n\
+                                     specified the WGET_ASKPASS or the SSH_ASKPASS \n\
+                                     environment variable is used.\n"),
     N_("\
        --no-iri                    turn off IRI support\n"),
     N_("\
@@ -699,10 +712,18 @@ Download:\n"),
     N_("\
        --unlink                    remove file before clobber\n"),
 #ifdef HAVE_METALINK
+    N_("\
+       --keep-badhash              keep files with checksum mismatch (append .badhash)\n"),
+    N_("\
+       --metalink-index=NUMBER     Metalink application/metalink4+xml metaurl ordinal NUMBER\n"),
     N_("\
        --metalink-over-http        use Metalink metadata from HTTP response headers\n"),
     N_("\
        --preferred-location        preferred location for Metalink resources\n"),
+#endif
+#ifdef ENABLE_XATTR
+    N_("\
+       --no-xattr                  turn off storage of metadata in extended file attributes\n"),
 #endif
     "\n",
 
@@ -809,7 +830,7 @@ HTTPS (SSL/TLS) options:\n"),
     N_("\
        --pinnedpubkey=FILE/HASHES  Public key (PEM/DER) file, or any number\n\
                                    of base64 encoded sha256 hashes preceded by\n\
-                                   \'sha256//\' and seperated by \';\', to verify\n\
+                                   \'sha256//\' and separated by \';\', to verify\n\
                                    peer against\n"),
 #if defined(HAVE_LIBSSL) || defined(HAVE_LIBSSL32)
     N_("\
@@ -1019,6 +1040,103 @@ prompt_for_password (void)
   return getpass("");
 }
 
+
+/* Execute external application opt.use_askpass */
+static void
+run_use_askpass (char *question, char **answer)
+{
+  char tmp[1024];
+  pid_t pid;
+  int status;
+  int com[2];
+  ssize_t bytes = 0;
+  char *argv[3], *p;
+  posix_spawn_file_actions_t fa;
+
+  if (pipe (com) == -1)
+    {
+      fprintf (stderr, _("Cannot create pipe\n"));
+      exit (WGET_EXIT_GENERIC_ERROR);
+    }
+
+  status = posix_spawn_file_actions_init (&fa);
+  if (status)
+    {
+      fprintf (stderr,
+              _("Error initializing spawn file actions for use-askpass: %d\n"),
+              status);
+      exit (WGET_EXIT_GENERIC_ERROR);
+    }
+
+  status = posix_spawn_file_actions_adddup2 (&fa, com[1], STDOUT_FILENO);
+  if (status)
+    {
+      fprintf (stderr,
+              _("Error setting spawn file actions for use-askpass: %d\n"),
+              status);
+      exit (WGET_EXIT_GENERIC_ERROR);
+    }
+
+  /* C89 initializer lists must be computable at load time,
+   * thus this explicit initialization. */
+  argv[0] = opt.use_askpass;
+  argv[1] = question;
+  argv[2] = NULL;
+
+  status = posix_spawnp (&pid, opt.use_askpass, &fa, NULL, argv, environ);
+  if (status)
+    {
+      fprintf (stderr, "Error spawning %s: %d\n", opt.use_askpass, status);
+      exit (WGET_EXIT_GENERIC_ERROR);
+    }
+
+  /* Parent process reads from child. */
+  close (com[1]);
+  bytes = read (com[0], tmp, sizeof (tmp) - 1);
+  if (bytes <= 0)
+    {
+      fprintf (stderr,
+              _("Error reading response from command \"%s %s\": %s\n"),
+              opt.use_askpass, question, strerror (errno));
+      exit (WGET_EXIT_GENERIC_ERROR);
+    }
+
+  /* Make sure there is a trailing 0 */
+  tmp[bytes] = '\0';
+
+  /* Remove a possible new line */
+  if ((p = strpbrk (tmp, "\r\n")))
+    bytes = p - tmp;
+
+  *answer = xmemdup0 (tmp, bytes);
+}
+
+/* set the user name and password*/
+static void
+use_askpass (struct url *u)
+{
+  static char question[1024];
+
+  if (u->user == NULL || u->user[0] == '\0')
+    {
+      snprintf (question, sizeof (question),  _("Username for '%s%s': "),
+                scheme_leading_string(u->scheme), u->host);
+      /* Prompt for username */
+      run_use_askpass (question, &u->user);
+      if (opt.recursive)
+        opt.user = xstrdup (u->user);
+    }
+
+  if (u->passwd == NULL || u->passwd[0] == '\0')
+    {
+      snprintf(question, sizeof (question), _("Password for '%s%s@%s': "),
+               scheme_leading_string (u->scheme), u->user, u->host);
+      /* Prompt for password */
+      run_use_askpass (question, &u->passwd);
+      if (opt.recursive)
+        opt.passwd = xstrdup (u->passwd);
+    }
+}
 /* Function that prints the line argument while limiting it
    to at most line_length. prefix is printed on the first line
    and an appropriate number of spaces are added on subsequent
@@ -1080,7 +1198,7 @@ format_and_print_line (const char *prefix, const char *line,
   return 0;
 }
 
-static void _Noreturn
+_Noreturn static void
 print_version (void)
 {
   const char *wgetrc_title  = _("Wgetrc: ");
@@ -1339,7 +1457,8 @@ main (int argc, char **argv)
           append_to_log = true;
           break;
         case OPT__EXECUTE:
-          run_command (optarg);
+          if (optarg) /* check silences static analyzer */
+            run_command (optarg);
           break;
         case OPT__NO:
           {
@@ -1623,7 +1742,7 @@ for details.\n\n"));
       else if (opt.method)
         {
           fprintf (stderr, _("You cannot use --post-data or --post-file along with --method. "
-                             "--method expects data through --body-data and --body-file options"));
+                             "--method expects data through --body-data and --body-file options\n"));
           exit (WGET_EXIT_GENERIC_ERROR);
         }
     }
@@ -1702,6 +1821,16 @@ for details.\n\n"));
         exit (WGET_EXIT_GENERIC_ERROR);
     }
 
+  if (opt.use_askpass)
+  {
+    if (opt.use_askpass[0] == '\0')
+      {
+        fprintf (stderr,
+                 _("use-askpass requires a string or either environment variable WGET_ASKPASS or SSH_ASKPASS to be set.\n"));
+        exit (WGET_EXIT_GENERIC_ERROR);
+      }
+  }
+
 #ifdef USE_WATT32
   if (opt.wdebug)
      dbug_init();
@@ -1764,7 +1893,7 @@ for details.\n\n"));
         }
       else
         {
-          struct_fstat st;
+          struct stat st;
 
 #ifdef __VMS
 /* Common fopen() optional arguments:
@@ -1920,6 +2049,10 @@ only if outputting to a regular file.\n"));
         }
       else
         {
+          /* Request credentials if use_askpass is set. */
+          if (opt.use_askpass)
+            use_askpass (url_parsed);
+
           if ((opt.recursive || opt.page_requisites)
               && ((url_scheme (*t) != SCHEME_FTP
 #ifdef HAVE_SSL

src/metalink.h

@@ -46,6 +46,21 @@ as that of the covered work.  */
 uerr_t retrieve_from_metalink (const metalink_t *metalink);
 
 int metalink_res_cmp (const void *res1, const void *res2);
+int metalink_meta_cmp (const void* meta1, const void* meta2);
+
+int metalink_check_safe_path (const char *path);
+
+char *last_component (char const *name);
+void replace_metalink_basename (char **name, char *ref);
+char *get_metalink_basename (char *name);
+void append_suffix_number (char **str, const char *sep, wgint num);
+void clean_metalink_string (char **str);
+void dequote_metalink_string (char **str);
+void badhash_suffix (char *name);
+void badhash_or_remove (char *name);
+uerr_t fetch_metalink_file (const char *url_str,
+                            bool resume, bool metalink_http,
+                            const char *filename, char **destname);
 
 bool find_key_value (const char *start,
                      const char *end,

src/mswindows.c

@@ -53,9 +53,6 @@ as that of the covered work.  */
 #endif
 
 
-/* Defined in log.c.  */
-void log_request_redirect_output (const char *);
-
 /* Windows version of xsleep in utils.c.  */
 
 void
@@ -98,7 +95,7 @@ static void
 ws_hangup (const char *reason)
 {
   fprintf (stderr, _("Continuing in background.\n"));
-  log_request_redirect_output (reason);
+  redirect_output (true, reason);
 
   /* Detach process from the current console.  Under Windows 9x, if we
      were launched from a 16-bit process (which is usually the case;

src/netrc.c

@@ -75,7 +75,7 @@ search_netrc (const char *host, const char **acc, const char **passwd,
 #ifdef __VMS
 
       int err;
-      struct_stat buf;
+      struct stat buf;
       char *path = "SYS$LOGIN:.netrc";
 
       netrc_list = NULL;
@@ -94,7 +94,7 @@ search_netrc (const char *host, const char **acc, const char **passwd,
       if (home)
         {
           int err;
-          struct_stat buf;
+          struct stat buf;
           char *path = (char *)alloca (strlen (home) + 1
                                        + strlen (NETRC_FILE_NAME) + 1);
           sprintf (path, "%s/%s", home, NETRC_FILE_NAME);
@@ -439,7 +439,7 @@ free_netrc(acc_t *l)
 int
 main (int argc, char **argv)
 {
-  struct_stat sb;
+  struct stat sb;
   char *program_name, *file, *target;
   acc_t *head, *a;
 

src/openssl.c

@@ -204,18 +204,20 @@ ssl_init (void)
 #endif
   SSL_library_init ();
   SSL_load_error_strings ();
+#if OPENSSL_VERSION_NUMBER < 0x10100000L
   SSLeay_add_all_algorithms ();
   SSLeay_add_ssl_algorithms ();
+#endif
 
   switch (opt.secure_protocol)
     {
-#ifndef OPENSSL_NO_SSL2
+#if !defined OPENSSL_NO_SSL2 && OPENSSL_VERSION_NUMBER < 0x10100000L
     case secure_protocol_sslv2:
       meth = SSLv2_client_method ();
       break;
 #endif
 
-#ifndef OPENSSL_NO_SSL3
+#ifndef OPENSSL_NO_SSL3_METHOD
     case secure_protocol_sslv3:
       meth = SSLv3_client_method ();
       break;
@@ -583,7 +585,7 @@ ssl_connect_wget (int fd, const char *hostname, int *continue_session)
     DEBUGP (("SSL handshake timed out.\n"));
     goto timeout;
   }
-  if (scwt_ctx.result <= 0 || SSL_state(conn) != SSL_ST_OK)
+  if (scwt_ctx.result <= 0 || !SSL_is_init_finished(conn))
     goto error;
 
   ctx = xnew0 (struct openssl_transport_context);

src/options.h

@@ -67,6 +67,7 @@ struct options
   char *input_filename;         /* Input filename */
 #ifdef HAVE_METALINK
   char *input_metalink;         /* Input metalink file */
+  int metalink_index;           /* Metalink application/metalink4+xml metaurl ordinal number. */
   bool metalink_over_http;      /* Use Metalink if present in HTTP response */
   char *preferred_location;     /* Preferred location for Metalink resources */
 #endif
@@ -127,9 +128,12 @@ struct options
   bool warc_keep_log;           /* Store the log file in a WARC record. */
   char **warc_user_headers;     /* User-defined WARC header(s). */
 
+  bool enable_xattr;            /* Store metadata in POSIX extended attributes. */
+
   char *user;                   /* Generic username */
   char *passwd;                 /* Generic password */
   bool ask_passwd;              /* Ask for password? */
+  char *use_askpass;           /* value to use for use-askpass if WGET_ASKPASS is not set */
 
   bool always_rest;             /* Always use REST. */
   wgint start_pos;              /* Start position of a download. */
@@ -243,7 +247,7 @@ struct options
 
   char *pinnedpubkey;           /* Public key (PEM/DER) file, or any number
                                    of base64 encoded sha256 hashes preceded by
-                                   \'sha256//\' and seperated by \';\', to verify
+                                   \'sha256//\' and separated by \';\', to verify
                                    peer against */
 
   char *random_file;            /* file with random data to seed the PRNG */
@@ -258,6 +262,7 @@ struct options
   bool cookies;                 /* whether cookies are used. */
   char *cookies_input;          /* file we're loading the cookies from. */
   char *cookies_output;         /* file we're saving the cookies to. */
+  bool keep_badhash;            /* Keep files with checksum mismatch. */
   bool keep_session_cookies;    /* whether session cookies should be
                                    saved and loaded. */
 
@@ -306,7 +311,7 @@ struct options
 
   bool enable_iri;
   char *encoding_remote;
-  char *locale;
+  const char *locale;
 
   bool trustservernames;
 #ifdef __VMS

src/progress.c

@@ -892,11 +892,6 @@ get_eta (int *bcd)
   p += sizeof (s) - 1;                          \
 } while (0)
 
-/* Use move_to_end (s) to get S to point the end of the string (the
-   terminating \0).  This is faster than s+=strlen(s), but some people
-   are confused when they see strchr (s, '\0') in the code.  */
-#define move_to_end(s) s = strchr (s, '\0');
-
 static void
 create_image (struct bar_progress *bp, double dl_total_time, bool done)
 {

src/recur.c

@@ -438,10 +438,15 @@ retrieve_tree (struct url *start_url_parsed, struct iri *pi)
               struct url *url_parsed = url_parse (url, NULL, i, true);
               struct iri *ci;
               char *referer_url = url;
-              bool strip_auth = (url_parsed != NULL
-                                 && url_parsed->user != NULL);
+              bool strip_auth;
+
               assert (url_parsed != NULL);
 
+              if (!url_parsed)
+                continue;
+
+              strip_auth = (url_parsed && url_parsed->user);
+
               /* Strip auth info if present */
               if (strip_auth)
                 referer_url = url_string (url_parsed, URL_AUTH_HIDE);
@@ -451,9 +456,16 @@ retrieve_tree (struct url *start_url_parsed, struct iri *pi)
                   reject_reason r;
 
                   if (child->ignore_when_downloading)
-                    continue;
+                    {
+                      DEBUGP (("Not following due to 'ignore' flag: %s\n", child->url->url));
+                      continue;
+                    }
+
                   if (dash_p_leaf_HTML && !child->link_inline_p)
-                    continue;
+                    {
+                      DEBUGP (("Not following due to 'link inline' flag: %s\n", child->url->url));
+                      continue;
+                    }
 
                   r = download_child (child, url_parsed, depth,
                                       start_url_parsed, blacklist, i);
@@ -801,6 +813,12 @@ descend_redirect (const char *redirected, struct url *orig_parsed, int depth,
 
   if (reason == WG_RR_SUCCESS)
     blacklist_add (blacklist, upos->url->url);
+  else if (reason == WG_RR_LIST || reason == WG_RR_REGEX)
+    {
+      DEBUGP (("Ignoring decision for redirects, decided to load it.\n"));
+      blacklist_add (blacklist, upos->url->url);
+      reason = WG_RR_SUCCESS;
+    }
   else
     DEBUGP (("Redirection \"%s\" failed the test.\n", redirected));
 

src/retr.c

@@ -634,7 +634,7 @@ retr_rate (wgint bytes, double secs)
   double dlrate = calc_rate (bytes, secs, &units);
   /* Use more digits for smaller numbers (regardless of unit used),
      e.g. "1022", "247", "12.5", "2.38".  */
-  sprintf (res, "%.*f %s",
+  snprintf (res, sizeof(res), "%.*f %s",
            dlrate >= 99.95 ? 0 : dlrate >= 9.995 ? 1 : 2,
            dlrate, !opt.report_bps ? rate_names[units]: rate_names_bits[units]);
 
@@ -869,7 +869,7 @@ retrieve_url (struct url * orig_parsed, const char *origurl, char **file,
          redirects, but a ton of boneheaded webservers and CGIs out
          there break the rules and use relative URLs, and popular
          browsers are lenient about this, so wget should be too. */
-      construced_newloc = uri_merge (url, mynewloc);
+      construced_newloc = uri_merge (url, mynewloc ? mynewloc : "");
       xfree (mynewloc);
       mynewloc = construced_newloc;
 
@@ -1147,7 +1147,7 @@ retrieve_from_file (const char *file, bool html, int *count)
 Removing file due to --delete-after in retrieve_from_file():\n"));
           logprintf (LOG_VERBOSE, _("Removing %s.\n"), filename);
           if (unlink (filename))
-            logprintf (LOG_NOTQUIET, "unlink: %s\n", strerror (errno));
+            logprintf (LOG_NOTQUIET, "Failed to unlink %s: (%d) %s\n", filename, errno, strerror (errno));
           dt &= ~RETROKF;
         }
 
@@ -1248,9 +1248,9 @@ rotate_backups(const char *fname)
 #endif
 
   int maxlen = strlen (fname) + sizeof (SEP) + numdigit (opt.backups) + AVSL;
-  char *from = (char *)alloca (maxlen);
-  char *to = (char *)alloca (maxlen);
-  struct_stat sb;
+  char *from = alloca (maxlen);
+  char *to = alloca (maxlen);
+  struct stat sb;
   int i;
 
   if (stat (fname, &sb) == 0)
@@ -1267,17 +1267,21 @@ rotate_backups(const char *fname)
        */
       if (i == opt.backups)
         {
-          sprintf (to, "%s%s%d%s", fname, SEP, i, AVS);
+          snprintf (to, sizeof(to), "%s%s%d%s", fname, SEP, i, AVS);
           delete (to);
         }
 #endif
-      sprintf (to, "%s%s%d", fname, SEP, i);
-      sprintf (from, "%s%s%d", fname, SEP, i - 1);
-      rename (from, to);
+      snprintf (to, maxlen, "%s%s%d", fname, SEP, i);
+      snprintf (from, maxlen, "%s%s%d", fname, SEP, i - 1);
+      if (rename (from, to))
+        logprintf (LOG_NOTQUIET, "Failed to rename %s to %s: (%d) %s\n",
+                   from, to, errno, strerror (errno));
     }
 
-  sprintf (to, "%s%s%d", fname, SEP, 1);
-  rename(fname, to);
+  snprintf (to, maxlen, "%s%s%d", fname, SEP, 1);
+  if (rename(fname, to))
+    logprintf (LOG_NOTQUIET, "Failed to rename %s to %s: (%d) %s\n",
+               fname, to, errno, strerror (errno));
 }
 
 static bool no_proxy_match (const char *, const char **);

src/sysdep.h

@@ -35,53 +35,11 @@ as that of the covered work.  */
 #ifndef SYSDEP_H
 #define SYSDEP_H
 
-/* Testing for __sun is not enough because it's also defined on SunOS.  */
-#ifdef __sun
-# ifdef __SVR4
-#  define solaris
-# endif
-#endif
-
-#if defined(__INTERIX) && !defined(_ALL_SOURCE)
-# define _ALL_SOURCE
-#endif
-
-/* The "namespace tweaks" below attempt to set a friendly "compilation
-   environment" under popular operating systems.  Default compilation
-   environment often means that some functions that are "extensions"
-   are not declared -- `strptime' is one example.
-
-   But non-default environments can expose bugs in the system header
-   files, crippling compilation in _very_ non-obvious ways.  Because
-   of that, we define them only on well-tested architectures where we
-   know they will work.  */
-
-#undef NAMESPACE_TWEAKS
-
-#ifdef solaris
-# define NAMESPACE_TWEAKS
-#endif
-
-#if defined(__linux__) || defined(__GLIBC__)
-# define NAMESPACE_TWEAKS
-#endif
-
-#ifdef NAMESPACE_TWEAKS
-
-#endif /* NAMESPACE_TWEAKS */
-
-
-/* Alloca declaration, based on recommendation in the Autoconf manual.
-   These have to be after the above namespace tweaks, but before any
-   non-preprocessor code.  */
+/* Provided by gnulib on systems that don't have it: */
 
 #include <alloca.h>
-
-/* Must include these, so we can test for the missing stat macros and
-   define them as necessary.  */
 #include <sys/types.h>
 #include <sys/stat.h>
-
 #include <stdint.h>
 #include <inttypes.h>
 
@@ -95,98 +53,9 @@ as that of the covered work.  */
 # include "mswindows.h"
 #endif
 
-/* Provided by gnulib on systems that don't have it: */
-# include <stdbool.h>
-
-#ifndef struct_stat
-# define struct_stat struct stat
-#endif
-#ifndef struct_fstat
-# define struct_fstat struct stat
-#endif
-
-#include <intprops.h>
-
-/* For CHAR_BIT, LONG_MAX, etc. */
+#include <stdbool.h>
 #include <limits.h>
-
-#ifndef CHAR_BIT
-# define CHAR_BIT 8
-#endif
-
-/* These are defined in cmpt.c if missing, so we must declare
-   them.  */
-#ifndef HAVE_STRCASECMP
-int strcasecmp ();
-#endif
-#ifndef HAVE_STRNCASECMP
-int strncasecmp ();
-#endif
-#ifndef HAVE_STRPTIME
-char *strptime ();
-#endif
-#ifndef HAVE_TIMEGM
-# include <time.h>
-time_t timegm (struct tm *);
-#endif
-#ifndef HAVE_MEMRCHR
-void *memrchr (const void *, int, size_t);
-#endif
-
-/* These are defined in snprintf.c.  It would be nice to have an
-   snprintf.h, though.  */
-#ifndef HAVE_SNPRINTF
-int snprintf (char *str, size_t count, const char *fmt, ...);
-#endif
-#ifndef HAVE_VSNPRINTF
-#include <stdarg.h>
-int vsnprintf (char *str, size_t count, const char *fmt, va_list arg);
-#endif
-
-/* Enable system fnmatch only on systems where fnmatch.h is usable.
-   If the fnmatch on your system is buggy, undef this symbol and a
-   replacement implementation will be used instead.  */
-#ifdef HAVE_WORKING_FNMATCH_H
-# define SYSTEM_FNMATCH
-#endif
-
 #include <fnmatch.h>
-
-/* Provide sig_atomic_t if the system doesn't.  */
-#ifndef HAVE_SIG_ATOMIC_T
-typedef int sig_atomic_t;
-#endif
-
-/* Provide uint32_t on the platforms that don't define it.  Although
-   most code should be agnostic about integer sizes, some code really
-   does need a 32-bit integral type.  Such code should use uint32_t.
-   (The exception is gnu-md5.[ch], which uses its own detection for
-   portability across platforms.)  */
-
-#ifndef HAVE_UINT32_T
-# if SIZEOF_INT == 4
-typedef unsigned int uint32_t;
-# else
-#  if SIZEOF_LONG == 4
-typedef unsigned long uint32_t;
-#  else
-#   if SIZEOF_SHORT == 4
-typedef unsigned short uint32_t;
-#   else
- #error "Cannot determine a 32-bit unsigned integer type"
-#   endif
-#  endif
-# endif
-#endif
-
-/* If uintptr_t isn't defined, simply typedef it to unsigned long. */
-#ifndef HAVE_UINTPTR_T
-typedef unsigned long uintptr_t;
-#endif
-
-/* If intptr_t isn't defined, simply typedef it to long. */
-#ifndef HAVE_INTPTR_T
-typedef long intptr_t;
-#endif
+#include "intprops.h"
 
 #endif /* SYSDEP_H */

src/url.c

@@ -43,10 +43,10 @@ as that of the covered work.  */
 #include "host.h"  /* for is_valid_ipv6_address */
 #include "c-strcase.h"
 
-#if HAVE_ICONV
-#include <iconv.h>
-#include <langinfo.h>
+#ifdef HAVE_ICONV
+# include <iconv.h>
 #endif
+#include <langinfo.h>
 
 #ifdef __VMS
 #include "vms.h"
@@ -512,6 +512,12 @@ scheme_disable (enum url_scheme scheme)
   supported_schemes[scheme].flags |= scm_disabled;
 }
 
+const char *
+scheme_leading_string (enum url_scheme scheme)
+{
+  return supported_schemes[scheme].leading_string;
+}
+
 /* Skip the username and password, if present in the URL.  The
    function should *not* be called with the complete URL, but with the
    portion after the scheme.
@@ -1207,7 +1213,7 @@ url_free (struct url *url)
   if (url)
     {
       if (url->idn_allocated) {
-        idn_free (url->host);      /* A dummy if !defined(ENABLE_IRI) */
+        idn2_free (url->host);      /* A dummy if !defined(ENABLE_IRI) */
         url->host = NULL;
       }
       else
@@ -1236,7 +1242,7 @@ mkalldirs (const char *path)
 {
   const char *p;
   char *t;
-  struct_stat st;
+  struct stat st;
   int res;
 
   p = path + strlen (path);
@@ -1271,12 +1277,14 @@ mkalldirs (const char *path)
              name exists, we just remove it and create the directory
              anyway.  */
           DEBUGP (("Removing %s because of directory danger!\n", t));
-          unlink (t);
+          if (unlink (t))
+            logprintf (LOG_NOTQUIET, "Failed to unlink %s (%d): %s\n",
+                       t, errno, strerror(errno));
         }
     }
   res = make_directory (t);
   if (res != 0)
-    logprintf (LOG_NOTQUIET, "%s: %s", t, strerror (errno));
+    logprintf (LOG_NOTQUIET, "%s: %s\n", t, strerror (errno));
   xfree (t);
   return res;
 }
@@ -1540,7 +1548,6 @@ static char *
 convert_fname (char *fname)
 {
   char *converted_fname = fname;
-#if HAVE_ICONV
   const char *from_encoding = opt.encoding_remote;
   const char *to_encoding = opt.locale;
   iconv_t cd;
@@ -1567,7 +1574,7 @@ convert_fname (char *fname)
 
       for (;;)
 	{
-	  if (iconv (cd, &fname, &inlen, &s, &outlen) != (size_t)(-1)
+	  if (iconv (cd, (ICONV_CONST char **) &fname, &inlen, &s, &outlen) != (size_t)(-1)
 	      && iconv (cd, NULL, NULL, &s, &outlen) != (size_t)(-1))
 	    {
 	      *(converted_fname + len - outlen - done) = '\0';
@@ -1607,7 +1614,6 @@ convert_fname (char *fname)
     }
 
     iconv_close(cd);
-#endif
 
   return converted_fname;
 }

src/url.h

@@ -100,8 +100,8 @@ struct url
   char *user;
   char *passwd;
 
-  /* 'host' is allocated by idna_to_ascii_8z() via idn_encode().
-   * Call 'idn_free()' to free this memory. */
+  /* 'host' is allocated by idn2_lookup_u8() via idn_encode().
+   * Call 'idn2_free()' to free this memory. */
   bool idn_allocated;
 };
 
@@ -124,6 +124,7 @@ bool url_has_scheme (const char *);
 bool url_valid_scheme (const char *);
 int scheme_default_port (enum url_scheme);
 void scheme_disable (enum url_scheme);
+const char *scheme_leading_string (enum url_scheme);
 
 char *url_string (const struct url *, enum url_auth_mode);
 char *url_file_name (const struct url *, char *);

src/utils.c

@@ -113,7 +113,7 @@ as that of the covered work.  */
 #include "exits.h"
 #include "c-strcase.h"
 
-static void _Noreturn
+_Noreturn static void
 memfatal (const char *context, long attempted_size)
 {
   /* Make sure we don't try to store part of the log line, and thus
@@ -235,14 +235,20 @@ xstrdup_lower (const char *s)
 
 /* Copy the string formed by two pointers (one on the beginning, other
    on the char after the last char) to a new, malloc-ed location.
-   0-terminate it.  */
+   0-terminate it.
+   If both pointers are NULL, the function returns an empty string.  */
 char *
 strdupdelim (const char *beg, const char *end)
 {
-  char *res = xmalloc (end - beg + 1);
-  memcpy (res, beg, end - beg);
-  res[end - beg] = '\0';
-  return res;
+  if (beg && beg <= end)
+    {
+      char *res = xmalloc (end - beg + 1);
+      memcpy (res, beg, end - beg);
+      res[end - beg] = '\0';
+      return res;
+    }
+
+  return xstrdup("");
 }
 
 /* Parse a string containing comma-separated elements, and return a
@@ -292,13 +298,6 @@ sepstring (const char *s)
    vsnprintf until the correct size is found.  Since Wget also ships a
    fallback implementation of vsnprintf, this should be portable.  */
 
-/* Constant is using for limits memory allocation for text buffer.
-   Applicable in situation when: vasprintf is not available in the system
-   and vsnprintf return -1 when long line is truncated (in old versions of
-   glibc and in other system where C99 doesn`t support) */
-
-#define FMT_MAX_LENGTH 1048576
-
 char *
 aprintf (const char *fmt, ...)
 {
@@ -318,6 +317,13 @@ aprintf (const char *fmt, ...)
   return str;
 #else  /* not HAVE_VASPRINTF */
 
+/* Constant is using for limits memory allocation for text buffer.
+   Applicable in situation when: vasprintf is not available in the system
+   and vsnprintf return -1 when long line is truncated (in old versions of
+   glibc and in other system where C99 doesn`t support) */
+
+#define FMT_MAX_LENGTH 1048576
+
   /* vasprintf is unavailable.  snprintf into a small buffer and
      resize it as necessary. */
   int size = 32;
@@ -567,7 +573,7 @@ int
 remove_link (const char *file)
 {
   int err = 0;
-  struct_stat st;
+  struct stat st;
 
   if (lstat (file, &st) == 0 && S_ISLNK (st.st_mode))
     {
@@ -593,7 +599,7 @@ file_exists_p (const char *filename)
 #ifdef HAVE_ACCESS
   return access (filename, F_OK) >= 0;
 #else
-  struct_stat buf;
+  struct stat buf;
   return stat (filename, &buf) >= 0;
 #endif
 }
@@ -603,7 +609,7 @@ file_exists_p (const char *filename)
 bool
 file_non_directory_p (const char *path)
 {
-  struct_stat buf;
+  struct stat buf;
   /* Use lstat() rather than stat() so that symbolic links pointing to
      directories can be identified correctly.  */
   if (lstat (path, &buf) != 0)
@@ -630,7 +636,7 @@ file_size (const char *filename)
   fclose (fp);
   return size;
 #else
-  struct_stat st;
+  struct stat st;
   if (stat (filename, &st) < 0)
     return -1;
   return st.st_size;
@@ -1184,7 +1190,7 @@ wget_read_file (const char *file)
 
 #ifdef HAVE_MMAP
   {
-    struct_fstat buf;
+    struct stat buf;
     if (fstat (fd, &buf) < 0)
       goto mmap_lose;
     fm->length = buf.st_size;
@@ -1327,7 +1333,7 @@ merge_vecs (char **v1, char **v2)
   for (j = 0; v2[j]; j++)
     ;
   /* Reallocate v1.  */
-  v1 = xrealloc (v1, (i + j + 1) * sizeof (char **));
+  v1 = xrealloc (v1, (i + j + 1) * sizeof (char *));
   memcpy (v1 + i, v2, (j + 1) * sizeof (char *));
   xfree (v2);
   return v1;
@@ -1930,7 +1936,7 @@ random_float (void)
 
 static sigjmp_buf run_with_timeout_env;
 
-static void _Noreturn
+_Noreturn static void
 abort_run_with_timeout (int sig _GL_UNUSED)
 {
   assert (sig == SIGALRM);
@@ -2045,13 +2051,16 @@ run_with_timeout (double timeout, void (*fun) (void *), void *arg)
       return false;
     }
 
-  signal (SIGALRM, abort_run_with_timeout);
   if (SETJMP (run_with_timeout_env) != 0)
     {
       /* Longjumped out of FUN with a timeout. */
       signal (SIGALRM, SIG_DFL);
       return true;
     }
+  else
+    {
+      signal (SIGALRM, abort_run_with_timeout);
+    }
   alarm_set (timeout);
   fun (arg);
 
@@ -2140,7 +2149,7 @@ xsleep (double seconds)
    base64 data.  */
 
 size_t
-base64_encode (const void *data, size_t length, char *dest)
+wget_base64_encode (const void *data, size_t length, char *dest)
 {
   /* Conversion table.  */
   static const char tbl[64] = {
@@ -2208,7 +2217,7 @@ base64_encode (const void *data, size_t length, char *dest)
    This function originates from Free Recode.  */
 
 ssize_t
-base64_decode (const char *base64, void *dest)
+wget_base64_decode (const char *base64, void *dest)
 {
   /* Table of base64 values for first 128 characters.  Note that this
      assumes ASCII (but so does Wget in other places).  */
@@ -2413,17 +2422,17 @@ mergesort_internal (void *base, void *temp, size_t size, size_t from, size_t to,
 }
 
 /* Stable sort with interface exactly like standard library's qsort.
-   Uses mergesort internally, allocating temporary storage with
-   alloca.  */
+   Uses mergesort internally. */
 
 void
 stable_sort (void *base, size_t nmemb, size_t size,
              int (*cmpfun) (const void *, const void *))
 {
-  if (size > 1)
+  if (nmemb > 1 && size > 1)
     {
-      void *temp = alloca (nmemb * size * sizeof (void *));
+      void *temp = xmalloc (nmemb * size);
       mergesort_internal (base, temp, size, 0, nmemb - 1, cmpfun);
+      xfree(temp);
     }
 }
 
@@ -2524,7 +2533,7 @@ wg_hex_to_string (char *str_buffer, const char *hex_buffer, size_t hex_len)
   for (i = 0; i < hex_len; i++)
     {
       /* Each byte takes 2 characters.  */
-      sprintf (str_buffer + 2 * i, "%02x", hex_buffer[i] & 0xFF);
+      sprintf (str_buffer + 2 * i, "%02x", (unsigned) (hex_buffer[i] & 0xFF));
     }
 
   /* Null-terminate result.  */
@@ -2588,7 +2597,7 @@ wg_pubkey_pem_to_der (const char *pem, unsigned char **der, size_t *der_len)
 
   base64data = xmalloc (BASE64_LENGTH(stripped_pem_count));
 
-  size = base64_decode (stripped_pem, base64data);
+  size = wget_base64_decode (stripped_pem, base64data);
 
   if (size < 0) {
     xfree (base64data);           /* malformed base64 from server */
@@ -2651,7 +2660,7 @@ wg_pin_peer_pubkey (const char *pinnedpubkey, const char *pubkey, size_t pubkeyl
           end_pos[0] = '\0';
 
         /* decode base64 pinnedpubkey, 8 is length of "sha256//" */
-        decoded_hash_length = base64_decode (begin_pos + 8, expectedsha256sumdigest);
+        decoded_hash_length = wget_base64_decode (begin_pos + 8, expectedsha256sumdigest);
         /* if valid base64, compare sha256 digests directly */
         if (SHA256_DIGEST_SIZE == decoded_hash_length &&
            !memcmp (sha256sumdigest, expectedsha256sumdigest, SHA256_DIGEST_SIZE)) {

src/utils.h

@@ -140,8 +140,8 @@ void xsleep (double);
 /* How many bytes it will take to store LEN bytes in base64.  */
 #define BASE64_LENGTH(len) (4 * (((len) + 2) / 3))
 
-size_t base64_encode (const void *, size_t, char *);
-ssize_t base64_decode (const char *, void *);
+size_t wget_base64_encode (const void *, size_t, char *);
+ssize_t wget_base64_decode (const char *, void *);
 
 #ifdef HAVE_LIBPCRE
 void *compile_pcre_regex (const char *);

src/wget.h

@@ -361,7 +361,8 @@ typedef enum
   TIMECONV_ERR,
   METALINK_PARSE_ERROR, METALINK_RETR_ERROR,
   METALINK_CHKSUM_ERROR, METALINK_SIG_ERROR, METALINK_MISSING_RESOURCE,
-  RETR_WITH_METALINK
+  RETR_WITH_METALINK,
+  METALINK_SIZE_ERROR
 } uerr_t;
 
 /* 2005-02-19 SMS.

src/xattr.c

@@ -0,0 +1,79 @@
+/* xattr.h -- POSIX Extended Attribute support.
+
+   Copyright (C) 2016 Free Software Foundation, Inc.
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3, or (at your option)
+   any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, see <http://www.gnu.org/licenses/>.  */
+
+#include "wget.h"
+
+#include <stdio.h>
+#include <string.h>
+
+#include "log.h"
+#include "xattr.h"
+
+#ifdef USE_XATTR
+
+static int
+write_xattr_metadata (const char *name, const char *value, FILE *fp)
+{
+  int retval = -1;
+
+  if (name && value && fp)
+    {
+      retval = fsetxattr (fileno (fp), name, value, strlen (value), 0);
+      /* FreeBSD's extattr_set_fd returns the length of the extended attribute. */
+      retval = (retval < 0) ? retval : 0;
+      if (retval)
+        DEBUGP (("Failed to set xattr %s.\n", quote(name)));
+    }
+
+  return retval;
+}
+
+#else /* USE_XATTR */
+
+static int
+write_xattr_metadata (const char *name, const char *value, FILE *fp)
+{
+  (void)name;
+  (void)value;
+  (void)fp;
+
+  return 0;
+}
+
+#endif /* USE_XATTR */
+
+int
+set_file_metadata (const char *origin_url, const char *referrer_url, FILE *fp)
+{
+  /* Save metadata about where the file came from (requested, final URLs) to
+   * user POSIX Extended Attributes of retrieved file.
+   *
+   * For more details about the user namespace see
+   * [http://freedesktop.org/wiki/CommonExtendedAttributes] and
+   * [http://0pointer.de/lennart/projects/mod_mime_xattr/].
+   */
+  int retval = -1;
+
+  if (!origin_url || !fp)
+    return retval;
+
+  retval = write_xattr_metadata ("user.xdg.origin.url", escnonprint_uri (origin_url), fp);
+  if ((!retval) && referrer_url)
+    retval = write_xattr_metadata ("user.xdg.referrer.url", escnonprint_uri (referrer_url), fp);
+
+  return retval;
+}

src/xattr.h

@@ -0,0 +1,45 @@
+/* xattr.h -- POSIX Extended Attribute function mappings.
+
+   Copyright (C) 2016 Free Software Foundation, Inc.
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3, or (at your option)
+   any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, see <http://www.gnu.org/licenses/>.  */
+
+#include <stdio.h>
+
+#ifndef _XATTR_H
+#define _XATTR_H
+
+/* Store metadata name/value attributes against fp. */
+int set_file_metadata (const char *origin_url, const char *referrer_url, FILE *fp);
+
+#if defined(__linux)
+/* libc on Linux has fsetxattr (5 arguments). */
+#  include <sys/xattr.h>
+#  define USE_XATTR
+#elif defined(__APPLE__)
+/* libc on OS/X has fsetxattr (6 arguments). */
+#  include <sys/xattr.h>
+#  define fsetxattr (file, name, buffer, size, flags) \
+          fsetxattr ((file), (name), (buffer), (size), 0, (flags))
+#  define USE_XATTR
+#elif defined(__FreeBSD_version) && (__FreeBSD_version > 500000)
+/* FreeBSD */
+#  include <sys/types.h>
+#  include <sys/extattr.h>
+#  define fsetxattr (file, name, buffer, size, flags) \
+          extattr_set_fd ((file), EXTATTR_NAMESPACE_USER, (name), (buffer), (size))
+#  define USE_XATTR
+#endif
+
+#endif /* _XATTR_H */

testenv/Makefile.am

@@ -27,8 +27,38 @@
 
 
 if METALINK_IS_ENABLED
-  METALINK_TESTS = Test-metalink-xml.py             \
-    Test-metalink-http.py
+  METALINK_TESTS = Test-metalink-http.py            \
+    Test-metalink-http-quoted.py                    \
+    Test-metalink-http-baddigest.py                 \
+    Test-metalink-http-xml.py                       \
+    Test-metalink-http-xml-trust.py                 \
+    Test-metalink-http-xml-trust-name.py            \
+    Test-metalink-http-xml-type.py                  \
+    Test-metalink-http-xml-type-trust.py            \
+    Test-metalink-http-xml-type-content.py          \
+    Test-metalink-http-xml-type-trust-content.py    \
+    Test-metalink-xml.py                            \
+    Test-metalink-xml-continue.py                   \
+    Test-metalink-xml-relpath.py                    \
+    Test-metalink-xml-abspath.py                    \
+    Test-metalink-xml-homepath.py                   \
+    Test-metalink-xml-trust.py                      \
+    Test-metalink-xml-relpath-trust.py              \
+    Test-metalink-xml-abspath-trust.py              \
+    Test-metalink-xml-homepath-trust.py             \
+    Test-metalink-xml-prefix.py                     \
+    Test-metalink-xml-relprefix.py                  \
+    Test-metalink-xml-absprefix.py                  \
+    Test-metalink-xml-homeprefix.py                 \
+    Test-metalink-xml-prefix-trust.py               \
+    Test-metalink-xml-relprefix-trust.py            \
+    Test-metalink-xml-absprefix-trust.py            \
+    Test-metalink-xml-homeprefix-trust.py           \
+    Test-metalink-xml-emptyprefix-trust.py          \
+    Test-metalink-xml-size.py                       \
+    Test-metalink-xml-nohash.py                     \
+    Test-metalink-xml-nourls.py                     \
+    Test-metalink-xml-urlbreak.py
 else
   METALINK_TESTS =
 endif
@@ -42,7 +72,8 @@ if WITH_SSL
 endif
 
 if HAVE_PYTHON3
-  TESTS = Test-auth-basic-fail.py                   \
+  TESTS = Test-504.py                               \
+    Test-auth-basic-fail.py                         \
     Test-auth-basic.py                              \
     Test-auth-both.py                               \
     Test-auth-digest.py                             \
@@ -51,6 +82,7 @@ if HAVE_PYTHON3
     Test-auth-retcode.py                            \
     Test-auth-with-content-disposition.py           \
     Test-c-full.py                                  \
+    Test-condget.py                                 \
     Test-Content-disposition-2.py                   \
     Test-Content-disposition.py                     \
     Test--convert-links--content-on-error.py        \
@@ -59,24 +91,26 @@ if HAVE_PYTHON3
     Test-cookie-expires.py                          \
     Test-cookie.py                                  \
     Test-Head.py                                    \
+    Test-hsts.py                                    \
     Test--https.py                                  \
     Test--https-crl.py                              \
     Test-missing-scheme-retval.py                   \
+    Test-O.py                                       \
     Test-pinnedpubkey-der-https.py                  \
     Test-pinnedpubkey-der-no-check-https.py         \
     Test-pinnedpubkey-hash-https.py                 \
     Test-pinnedpubkey-hash-no-check-fail-https.py   \
     Test-pinnedpubkey-pem-fail-https.py             \
     Test-pinnedpubkey-pem-https.py                  \
-    Test-hsts.py                                    \
-    Test-O.py                                       \
     Test-Post.py                                    \
-    Test-504.py                                     \
-    Test--spider-r.py                               \
-    Test--rejected-log.py                           \
+    Test-recursive-basic.py                         \
+    Test-recursive-include.py                       \
+    Test-recursive-redirect.py                      \
+    Test-redirect.py                                \
     Test-redirect-crash.py                          \
+    Test--rejected-log.py                           \
     Test-reserved-chars.py                          \
-    Test-condget.py                                 \
+    Test--spider-r.py                               \
     $(METALINK_TESTS)
 
 endif

testenv/README

@@ -43,7 +43,8 @@ Working:
 ================================================================================
 
 The Test Files are valid Python scripts and the default mask for them is 755.
-A singular Test must be invoked in the following manner:
+A singular Test must be invoked in the following manner, with the current
+directory being the testenv directory:
 $ ./python3 <Name of Test File>      OR
 $ ./<Name of Test File>
 The script will then initialize the various elements and pass them to an object
@@ -117,7 +118,6 @@ from misc.wget_file import WgetFile
 
 It is recommended that a small description of the Test Case is provided next.
 This would be very helpful to future contributors.
-Next, is the const variable, TEST_NAME that defines the name of the Test.
 
 Each File in the Test must be represented as a WgetFile object. The WgetFile
 Class has the following prototype:
@@ -130,14 +130,15 @@ The rules object is a dictionary element, with the key as the Rule Name and
 value as the Rule Data. In most cases, the Rule Data is another dictionary.
 
 Various variables used consistently across all tests are:
-    * WGET_OPTIONS: The command line string passed to Wget upon invokation. This
+    * WGET_OPTIONS: The command line string passed to Wget upon invocation. This
     string may contain URLs, like in the case where in-URL authentication is
     used. Variable names passed like {{var_name}} will be replaced by the
     contents of the variable self.var_name before being passed to Wget
     * WGET_URLS: This is a list of filenames which will be appended as the URLs
-    to Wget during invokation. This is a list of lists, where WGET_URLS[0]
+    to Wget during invocation. This is a list of lists, where WGET_URLS[0]
     represents the list of Filenames called from Server[0], WGET_URLS[1] is a
-    list of files downloaded from Server[2], etc.
+    list of files downloaded from Server[2], etc.  They must be relative URLs,
+    i.e., not start with "/".
     * Files: This variable defines the files that exist in the Server's
     filesystem. The Files variable is a list of lists of WgetFile objects.
     This means that File[0] is a list of WgetFile objects that lie on Server[0],
@@ -154,13 +155,11 @@ Various variables used consistently across all tests are:
 
 Both, the HTTPTest and FTPTest modules have the same prototype:
 {
-    name,
     pre_hook,
     test_options,
     post_hook,
     protocols
 }
-name should be a string, and is usually passed to the TEST_NAME variable,
 the three hooks should be Python dict objects and protocols should be a list of
 protocols, like [HTTP, HTTPS].
 
@@ -215,7 +214,7 @@ executed. The currently supported options are:
     download. The complete URL will be created and passed to Wget
     automatically. (alias URLs)
     * WgetCommands  : A string consisting of the various commandline switches
-    sent to Wget upon invokation. Any data placed between {{ }} in this string
+    sent to Wget upon invocation. Any data placed between {{ }} in this string
     will be replaced with the contents of self.<data> before being passed to
     Wget. This is particularly useful for getting the hostname and port for a
     file. While all Download URL's are passed to Urls, a notable exception is
@@ -281,13 +280,6 @@ red-flagged as a XPASS. Currently, tests expected to fail under valgrind are not
 explicitly marked as XFAIL. Tests failing under valgrind must always be
 considered a blocking error.
 
-Remember to always name the Test correctly using the TEST_NAME variable. This
-is essential since a directory with the Test Name is created and this can
-cause synchronization problems when the Parallel Test Harness is used.
-One can use the following command on Unix systems to check for TEST_NAME
-clashes:
-$ grep -r -h "TEST_NAME =" | cut -c13- | uniq -c -d
-
 Work Remaining:
 ================================================================================
 

testenv/Test-Proto.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python3
 from sys import exit
 from test.http_test import HTTPTest
-from misc.constants import HTTP, HTTPS
+from test.base_test import HTTP, HTTPS
 from misc.wget_file import WgetFile
 
 """

testenv/Test-metalink-http-baddigest.py

@@ -0,0 +1,93 @@
+#!/usr/bin/env python3
+from sys import exit
+from test.http_test import HTTPTest
+from misc.wget_file import WgetFile
+import hashlib
+from base64 import b64encode
+
+"""
+    This is to test Metalink/HTTP with a malformed base64 Digest header.
+
+    With --trust-server-names, trust the metalink:file names.
+
+    Without --trust-server-names, don't trust the metalink:file names:
+    use the basename of --input-metalink, and add a sequential number
+    (e.g. .#1, .#2, etc.).
+
+    Strip the directory from unsafe paths.
+"""
+
+############# File Definitions ###############################################
+bad = "Ouch!"
+bad_sha256 = b64encode (hashlib.sha256 (bad.encode ('UTF-8')).digest ()).decode ('ascii')
+
+LinkHeaders = ["<http://{{SRV_HOST}}:{{SRV_PORT}}/wrong_file>; rel=duplicate; pri=1"]
+DigestHeader = "SHA-256=bad_base64,SHA-256={{BAD_HASH}}"
+
+# This will be filled as soon as we know server hostname and port
+MetaHTTPRules = {'SendHeader' : {}}
+
+MetaHTTP = WgetFile ("main.metalink", rules=MetaHTTPRules)
+
+wrong_file = WgetFile ("wrong_file", bad)
+wrong_file_down = WgetFile ("main.metalink", bad)
+
+WGET_OPTIONS = "--metalink-over-http"
+WGET_URLS = [["main.metalink"]]
+
+RequestList = [[
+    "HEAD /main.metalink",
+    "GET /wrong_file"
+]]
+
+Files = [[
+    MetaHTTP,
+    wrong_file
+]]
+Existing_Files = []
+
+ExpectedReturnCode = 0
+ExpectedDownloadedFiles = [wrong_file_down]
+
+################ Pre and Post Test Hooks #####################################
+pre_test = {
+    "ServerFiles"       : Files,
+    "LocalFiles"        : Existing_Files
+}
+test_options = {
+    "WgetCommands"      : WGET_OPTIONS,
+    "Urls"              : WGET_URLS
+}
+post_test = {
+    "ExpectedFiles"     : ExpectedDownloadedFiles,
+    "ExpectedRetcode"   : ExpectedReturnCode,
+    "FilesCrawled"      : RequestList
+}
+
+http_test = HTTPTest (
+                pre_hook=pre_test,
+                test_params=test_options,
+                post_hook=post_test
+)
+
+http_test.server_setup()
+### Get and use dynamic server sockname
+srv_host, srv_port = http_test.servers[0].server_inst.socket.getsockname ()
+
+DigestHeader = DigestHeader.replace('{{BAD_HASH}}', bad_sha256)
+
+# Helper function for hostname, port and digest substitution
+def SubstituteServerInfo (text, host, port):
+    text = text.replace('{{SRV_HOST}}', host)
+    text = text.replace('{{SRV_PORT}}', str (port))
+    return text
+
+MetaHTTPRules["SendHeader"] = {
+        'Link': [ SubstituteServerInfo (LinkHeader, srv_host, srv_port)
+                    for LinkHeader in LinkHeaders ],
+        'Digest': DigestHeader
+}
+
+err = http_test.begin ()
+
+exit (err)

testenv/Test-metalink-http-quoted.py

@@ -0,0 +1,126 @@
+#!/usr/bin/env python3
+from sys import exit
+from test.http_test import HTTPTest
+from misc.wget_file import WgetFile
+import hashlib
+from base64 import b64encode
+
+"""
+    This is to test Metalink/HTTP quoted values support in Wget.
+"""
+
+# Helper function for hostname, port and digest substitution
+def SubstituteServerInfo (text, host, port, digest):
+    text = text.replace('{{FILE1_HASH}}', digest)
+    text = text.replace('{{SRV_HOST}}', host)
+    text = text.replace('{{SRV_PORT}}', str (port))
+    return text
+
+############# File Definitions ###############################################
+File1 = "Would you like some Tea?"
+File1_corrupted = "Would you like some Coffee?"
+File1_lowPref = "Do not take this"
+File1_sha256 = b64encode (hashlib.sha256 (File1.encode ('UTF-8')).digest ()).decode ('ascii')
+Signature = '''-----BEGIN PGP SIGNATURE-----
+Version: GnuPG v1.0.7 (GNU/Linux)
+
+This is no valid signature. But it should be downloaded.
+The attempt to verify should fail but should not prevent
+a successful metalink resource retrieval (the sig failure
+should not be fatal).
+-----END PGP SIGNATURE-----
+'''
+File2 = "No meta data for this file."
+
+LinkHeaders = [
+    # This file has low priority and should not be picked.
+    "<http://{{SRV_HOST}}:{{SRV_PORT}}/File1_lowPref>; rel=\"duplicate\"; pri=\"9\"; geo=\"pl\"",
+    # This file should be picked second, after hash failure.
+    "<http://0.0.0.0/File1_try2_badconnection>; rel =\"duplicate\";pref; pri=\"7\"",
+    # This signature download will fail.
+    "<http://{{SRV_HOST}}:{{SRV_PORT}}/Sig2.asc>; rel=\"describedby\"; type=\"application/pgp-signature\"",
+    # Two good signatures
+    "<http://{{SRV_HOST}}:{{SRV_PORT}}/Sig.asc>; rel=\"describedby\"; type=\"application/pgp-signature\"",
+    "<http://{{SRV_HOST}}:{{SRV_PORT}}/Sig.asc>; rel=\"describedby\"; type=\"application/pgp-signature\"",
+    # Bad URL scheme
+    "<invalid_url>; rel=\"duplicate\"; pri=\"4\"",
+    # rel missing
+    "<http://{{SRV_HOST}}:{{SRV_PORT}}/File1>; pri=\"1\"; pref",
+    # invalid rel
+    "<http://{{SRV_HOST}}:{{SRV_PORT}}/File1>; rel=\"strange\"; pri=\"4\"",
+    # This file should be picked first, because it has the lowest pri among preferred.
+    "<http://{{SRV_HOST}}:{{SRV_PORT}}/File1_try1_corrupted>; rel=\"duplicate\"; geo=\"su\"; pri=\"4\"; pref",
+    # This file should NOT be picked third due to preferred location set to 'uk'
+    "<http://{{SRV_HOST}}:{{SRV_PORT}}/File1_badgeo>; rel =\"duplicate\";pri=\"5\"",
+    # This file should be picked as third try, and it should succeed
+    "<http://{{SRV_HOST}}:{{SRV_PORT}}/File1_try3_ok>; rel=\'duplicate\'; pri=\"5\";geo=\"uk\""
+    ]
+DigestHeader = "SHA-256=\'{{FILE1_HASH}}\'"
+
+# This will be filled as soon as we know server hostname and port
+MetaFileRules = {'SendHeader' : {}}
+
+FileOkServer = WgetFile ("File1_try3_ok", File1)
+FileBadPref = WgetFile ("File1_lowPref", File1_lowPref)
+FileBadHash = WgetFile ("File1_try1_corrupted", File1_corrupted)
+MetaFile = WgetFile ("test.meta", rules=MetaFileRules)
+# In case of Metalink over HTTP, the local file name is
+# derived from the URL suffix.
+FileOkLocal = WgetFile ("test.meta", File1)
+SigFile = WgetFile ("Sig.asc", Signature)
+FileNoMeta = WgetFile ("File2", File2)
+
+WGET_OPTIONS = "--metalink-over-http --preferred-location=uk"
+WGET_URLS = [["test.meta", "File2"]]
+
+Files = [[FileOkServer, FileBadPref, FileBadHash, MetaFile, SigFile, FileNoMeta]]
+Existing_Files = []
+
+ExpectedReturnCode = 0
+ExpectedDownloadedFiles = [FileNoMeta, FileOkLocal]
+
+RequestList = [
+    [
+        "HEAD /test.meta",
+        "GET /Sig2.asc",
+        "GET /Sig.asc",
+        "GET /File1_try1_corrupted",
+        "GET /File1_try3_ok",
+        "HEAD /File2",
+        "GET /File2",
+    ]
+]
+
+################ Pre and Post Test Hooks #####################################
+pre_test = {
+    "ServerFiles"       : Files,
+    "LocalFiles"        : Existing_Files
+}
+test_options = {
+    "WgetCommands"      : WGET_OPTIONS,
+    "Urls"              : WGET_URLS
+}
+post_test = {
+    "ExpectedFiles"     : ExpectedDownloadedFiles,
+    "ExpectedRetcode"   : ExpectedReturnCode,
+    "FilesCrawled"      : RequestList,
+}
+
+http_test = HTTPTest (
+                pre_hook=pre_test,
+                test_params=test_options,
+                post_hook=post_test,
+)
+
+http_test.server_setup()
+srv_host, srv_port = http_test.servers[0].server_inst.socket.getsockname ()
+
+MetaFileRules["SendHeader"] = {
+        'Link': [ SubstituteServerInfo (LinkHeader, srv_host, srv_port, File1_sha256)
+                    for LinkHeader in LinkHeaders ],
+        'Digest': SubstituteServerInfo (DigestHeader, srv_host, srv_port, File1_sha256),
+}
+
+err = http_test.begin ()
+
+exit (err)

testenv/Test-metalink-http-xml-trust-name.py

@@ -0,0 +1,272 @@
+#!/usr/bin/env python3
+from sys import exit
+from test.http_test import HTTPTest
+from misc.wget_file import WgetFile
+import hashlib
+from base64 import b64encode
+
+"""
+    This is to test Metalink/HTTP with Metalink/XML Link headers.
+
+    With --trust-server-names, trust the metalink:file names.
+
+    Without --trust-server-names, don't trust the metalink:file names:
+    use the basename of --input-metalink, and add a sequential number
+    (e.g. .#1, .#2, etc.).
+
+    Strip the directory from unsafe paths.
+"""
+
+############# File Definitions ###############################################
+bad = "Ouch!"
+bad_sha256 = hashlib.sha256 (bad.encode ('UTF-8')).hexdigest ()
+
+File1 = "Would you like some Tea?"
+File1_lowPref = "Do not take this"
+File1_sha256 = hashlib.sha256 (File1.encode ('UTF-8')).hexdigest ()
+
+File2 = "This is gonna be good"
+File2_lowPref = "Not this one too"
+File2_sha256 = hashlib.sha256 (File2.encode ('UTF-8')).hexdigest ()
+
+File3 = "A little more, please"
+File3_lowPref = "That's just too much"
+File3_sha256 = hashlib.sha256 (File3.encode ('UTF-8')).hexdigest ()
+
+File4 = "Maybe a biscuit?"
+File4_lowPref = "No, thanks"
+File4_sha256 = hashlib.sha256 (File4.encode ('UTF-8')).hexdigest ()
+
+File5 = "More Tea...?"
+File5_lowPref = "I have to go..."
+File5_sha256 = hashlib.sha256 (File5.encode ('UTF-8')).hexdigest ()
+
+MetaXml1 = \
+"""<?xml version="1.0" encoding="utf-8"?>
+<metalink version="3.0" xmlns="http://www.metalinker.org/">
+  <publisher>
+    <name>GNU Wget</name>
+  </publisher>
+  <license>
+    <name>GNU GPL</name>
+    <url>http://www.gnu.org/licenses/gpl.html</url>
+  </license>
+  <identity>Wget Test Files</identity>
+  <version>1.2.3</version>
+  <description>Wget Test Files description</description>
+  <files>
+    <file name="dir/File1">
+      <verification>
+        <hash type="sha256">{{FILE1_HASH}}</hash>
+      </verification>
+      <resources>
+        <url type="http" preference="35">http://{{SRV_HOST}}:{{SRV_PORT}}/wrong_file</url>
+        <url type="http" preference="40">http://{{SRV_HOST}}:{{SRV_PORT}}/404</url>
+        <url type="http" preference="25">http://{{SRV_HOST}}:{{SRV_PORT}}/File1_lowPref</url>
+        <url type="http" preference="30">http://{{SRV_HOST}}:{{SRV_PORT}}/File1</url>
+      </resources>
+    </file>
+    <file name="dir/File2">
+      <verification>
+        <hash type="sha256">{{FILE2_HASH}}</hash>
+      </verification>
+      <resources>
+        <url type="http" preference="35">http://{{SRV_HOST}}:{{SRV_PORT}}/wrong_file</url>
+        <url type="http" preference="40">http://{{SRV_HOST}}:{{SRV_PORT}}/404</url>
+        <url type="http" preference="25">http://{{SRV_HOST}}:{{SRV_PORT}}/File2_lowPref</url>
+        <url type="http" preference="30">http://{{SRV_HOST}}:{{SRV_PORT}}/File2</url>
+      </resources>
+    </file>
+    <file name="/dir/File3"> <!-- rejected by libmetalink -->
+      <verification>
+        <hash type="sha256">{{FILE3_HASH}}</hash>
+      </verification>
+      <resources>
+        <url type="http" preference="35">http://{{SRV_HOST}}:{{SRV_PORT}}/wrong_file</url>
+        <url type="http" preference="40">http://{{SRV_HOST}}:{{SRV_PORT}}/404</url>
+        <url type="http" preference="25">http://{{SRV_HOST}}:{{SRV_PORT}}/File3_lowPref</url>
+        <url type="http" preference="30">http://{{SRV_HOST}}:{{SRV_PORT}}/File3</url>
+      </resources>
+    </file>
+    <file name="dir/File4">
+      <verification>
+        <hash type="sha256">{{FILE4_HASH}}</hash>
+      </verification>
+      <resources>
+        <url type="http" preference="35">http://{{SRV_HOST}}:{{SRV_PORT}}/wrong_file</url>
+        <url type="http" preference="40">http://{{SRV_HOST}}:{{SRV_PORT}}/404</url>
+        <url type="http" preference="25">http://{{SRV_HOST}}:{{SRV_PORT}}/File4_lowPref</url>
+        <url type="http" preference="30">http://{{SRV_HOST}}:{{SRV_PORT}}/File4</url>
+      </resources>
+    </file>
+    <file name="dir/File5">
+      <verification>
+        <hash type="sha256">{{FILE5_HASH}}</hash>
+      </verification>
+      <resources>
+        <url type="http" preference="35">http://{{SRV_HOST}}:{{SRV_PORT}}/wrong_file</url>
+        <url type="http" preference="40">http://{{SRV_HOST}}:{{SRV_PORT}}/404</url>
+        <url type="http" preference="25">http://{{SRV_HOST}}:{{SRV_PORT}}/File5_lowPref</url>
+        <url type="http" preference="30">http://{{SRV_HOST}}:{{SRV_PORT}}/File5</url>
+      </resources>
+    </file>
+  </files>
+</metalink>
+"""
+
+MetaXml2 = \
+"""<?xml version="1.0" encoding="utf-8"?>
+<metalink version="3.0" xmlns="http://www.metalinker.org/">
+  <publisher>
+    <name>GNU Wget</name>
+  </publisher>
+  <license>
+    <name>GNU GPL</name>
+    <url>http://www.gnu.org/licenses/gpl.html</url>
+  </license>
+  <identity>Wget Test Files</identity>
+  <version>1.2.3</version>
+  <description>Wget Test Files description</description>
+  <files>
+    <file name="bad">
+      <verification>
+        <hash type="sha256">{{BAD_HASH}}</hash>
+      </verification>
+      <resources>
+        <url type="http" preference="35">http://{{SRV_HOST}}:{{SRV_PORT}}/wrong_file</url>
+        <url type="http" preference="40">http://{{SRV_HOST}}:{{SRV_PORT}}/404</url>
+        <url type="http" preference="30">http://{{SRV_HOST}}:{{SRV_PORT}}/bad</url>
+      </resources>
+    </file>
+  </files>
+</metalink>
+"""
+
+LinkHeaders = [
+    # This file has the lowest priority, and should go last
+    "<http://{{SRV_HOST}}:{{SRV_PORT}}/test1.metalink>; rel=describedby; pri=2; type=\"application/metalink4+xml\"; name=\"newname.metalink\"",
+    # This file has the highest priority, and should go first
+    "<http://{{SRV_HOST}}:{{SRV_PORT}}/test2.metalink>; rel=describedby; pri=1; type=\"application/metalink4+xml\""
+]
+
+# This will be filled as soon as we know server hostname and port
+MetaHTTPRules = {'SendHeader' : {}}
+
+MetaHTTP = WgetFile ("main.metalink", rules=MetaHTTPRules)
+
+wrong_file = WgetFile ("wrong_file", bad)
+
+File1_orig = WgetFile ("File1", File1)
+File1_down = WgetFile ("dir/File1", File1)
+File1_nono = WgetFile ("File1_lowPref", File1_lowPref)
+
+File2_orig = WgetFile ("File2", File2)
+File2_down = WgetFile ("dir/File2", File2)
+File2_nono = WgetFile ("File2_lowPref", File2_lowPref)
+
+# rejected by libmetalink
+File3_orig = WgetFile ("File3", File3)
+File3_nono = WgetFile ("File3_lowPref", File3_lowPref)
+
+File4_orig = WgetFile ("File4", File4)
+File4_down = WgetFile ("dir/File4", File4)
+File4_nono = WgetFile ("File4_lowPref", File4_lowPref)
+
+File5_orig = WgetFile ("File5", File5)
+File5_down = WgetFile ("dir/File5", File5)
+File5_nono = WgetFile ("File5_lowPref", File5_lowPref)
+
+MetaFile1 = WgetFile ("test1.metalink", MetaXml1)
+MetaFile1_down = WgetFile ("newname.metalink", MetaXml1)
+
+MetaFile2 = WgetFile ("test2.metalink", MetaXml2)
+
+WGET_OPTIONS = "--trust-server-names --metalink-over-http --metalink-index=2"
+WGET_URLS = [["main.metalink"]]
+
+RequestList = [[
+    "HEAD /main.metalink",
+    "GET /404",
+    "GET /wrong_file",
+    "GET /test1.metalink",
+    "GET /File1",
+    "GET /File2",
+    "GET /File4",
+    "GET /File5"
+]]
+
+Files = [[
+    MetaHTTP,
+    wrong_file,
+    MetaFile1, MetaFile2,
+    File1_orig, File1_nono,
+    File2_orig, File2_nono,
+    File3_orig, File3_nono,
+    File4_orig, File4_nono,
+    File5_orig, File5_nono
+]]
+Existing_Files = []
+
+ExpectedReturnCode = 0
+ExpectedDownloadedFiles = [
+    MetaFile1_down,
+    File1_down,
+    File2_down,
+    File4_down,
+    File5_down
+]
+
+################ Pre and Post Test Hooks #####################################
+pre_test = {
+    "ServerFiles"       : Files,
+    "LocalFiles"        : Existing_Files
+}
+test_options = {
+    "WgetCommands"      : WGET_OPTIONS,
+    "Urls"              : WGET_URLS
+}
+post_test = {
+    "ExpectedFiles"     : ExpectedDownloadedFiles,
+    "ExpectedRetcode"   : ExpectedReturnCode,
+    "FilesCrawled"      : RequestList
+}
+
+http_test = HTTPTest (
+                pre_hook=pre_test,
+                test_params=test_options,
+                post_hook=post_test
+)
+
+http_test.server_setup()
+### Get and use dynamic server sockname
+srv_host, srv_port = http_test.servers[0].server_inst.socket.getsockname ()
+
+MetaXml1 = MetaXml1.replace('{{FILE1_HASH}}', File1_sha256)
+MetaXml1 = MetaXml1.replace('{{FILE2_HASH}}', File2_sha256)
+MetaXml1 = MetaXml1.replace('{{FILE3_HASH}}', File3_sha256)
+MetaXml1 = MetaXml1.replace('{{FILE4_HASH}}', File4_sha256)
+MetaXml1 = MetaXml1.replace('{{FILE5_HASH}}', File5_sha256)
+MetaXml1 = MetaXml1.replace('{{SRV_HOST}}', srv_host)
+MetaXml1 = MetaXml1.replace('{{SRV_PORT}}', str (srv_port))
+MetaFile1.content = MetaXml1
+MetaFile1_down.content = MetaXml1
+
+MetaXml2 = MetaXml2.replace('{{BAD_HASH}}', bad_sha256)
+MetaXml2 = MetaXml2.replace('{{SRV_HOST}}', srv_host)
+MetaXml2 = MetaXml2.replace('{{SRV_PORT}}', str (srv_port))
+MetaFile2.content = MetaXml2
+
+# Helper function for hostname, port and digest substitution
+def SubstituteServerInfo (text, host, port):
+    text = text.replace('{{SRV_HOST}}', host)
+    text = text.replace('{{SRV_PORT}}', str (port))
+    return text
+
+MetaHTTPRules["SendHeader"] = {
+        'Link': [ SubstituteServerInfo (LinkHeader, srv_host, srv_port)
+                    for LinkHeader in LinkHeaders ]
+}
+
+err = http_test.begin ()
+
+exit (err)

testenv/Test-metalink-http-xml-trust.py

@@ -0,0 +1,272 @@
+#!/usr/bin/env python3
+from sys import exit
+from test.http_test import HTTPTest
+from misc.wget_file import WgetFile
+import hashlib
+from base64 import b64encode
+
+"""
+    This is to test Metalink/HTTP with Metalink/XML Link headers.
+
+    With --trust-server-names, trust the metalink:file names.
+
+    Without --trust-server-names, don't trust the metalink:file names:
+    use the basename of --input-metalink, and add a sequential number
+    (e.g. .#1, .#2, etc.).
+
+    Strip the directory from unsafe paths.
+"""
+
+############# File Definitions ###############################################
+bad = "Ouch!"
+bad_sha256 = hashlib.sha256 (bad.encode ('UTF-8')).hexdigest ()
+
+File1 = "Would you like some Tea?"
+File1_lowPref = "Do not take this"
+File1_sha256 = hashlib.sha256 (File1.encode ('UTF-8')).hexdigest ()
+
+File2 = "This is gonna be good"
+File2_lowPref = "Not this one too"
+File2_sha256 = hashlib.sha256 (File2.encode ('UTF-8')).hexdigest ()
+
+File3 = "A little more, please"
+File3_lowPref = "That's just too much"
+File3_sha256 = hashlib.sha256 (File3.encode ('UTF-8')).hexdigest ()
+
+File4 = "Maybe a biscuit?"
+File4_lowPref = "No, thanks"
+File4_sha256 = hashlib.sha256 (File4.encode ('UTF-8')).hexdigest ()
+
+File5 = "More Tea...?"
+File5_lowPref = "I have to go..."
+File5_sha256 = hashlib.sha256 (File5.encode ('UTF-8')).hexdigest ()
+
+MetaXml1 = \
+"""<?xml version="1.0" encoding="utf-8"?>
+<metalink version="3.0" xmlns="http://www.metalinker.org/">
+  <publisher>
+    <name>GNU Wget</name>
+  </publisher>
+  <license>
+    <name>GNU GPL</name>
+    <url>http://www.gnu.org/licenses/gpl.html</url>
+  </license>
+  <identity>Wget Test Files</identity>
+  <version>1.2.3</version>
+  <description>Wget Test Files description</description>
+  <files>
+    <file name="dir/File1">
+      <verification>
+        <hash type="sha256">{{FILE1_HASH}}</hash>
+      </verification>
+      <resources>
+        <url type="http" preference="35">http://{{SRV_HOST}}:{{SRV_PORT}}/wrong_file</url>
+        <url type="http" preference="40">http://{{SRV_HOST}}:{{SRV_PORT}}/404</url>
+        <url type="http" preference="25">http://{{SRV_HOST}}:{{SRV_PORT}}/File1_lowPref</url>
+        <url type="http" preference="30">http://{{SRV_HOST}}:{{SRV_PORT}}/File1</url>
+      </resources>
+    </file>
+    <file name="dir/File2">
+      <verification>
+        <hash type="sha256">{{FILE2_HASH}}</hash>
+      </verification>
+      <resources>
+        <url type="http" preference="35">http://{{SRV_HOST}}:{{SRV_PORT}}/wrong_file</url>
+        <url type="http" preference="40">http://{{SRV_HOST}}:{{SRV_PORT}}/404</url>
+        <url type="http" preference="25">http://{{SRV_HOST}}:{{SRV_PORT}}/File2_lowPref</url>
+        <url type="http" preference="30">http://{{SRV_HOST}}:{{SRV_PORT}}/File2</url>
+      </resources>
+    </file>
+    <file name="/dir/File3"> <!-- rejected by libmetalink -->
+      <verification>
+        <hash type="sha256">{{FILE3_HASH}}</hash>
+      </verification>
+      <resources>
+        <url type="http" preference="35">http://{{SRV_HOST}}:{{SRV_PORT}}/wrong_file</url>
+        <url type="http" preference="40">http://{{SRV_HOST}}:{{SRV_PORT}}/404</url>
+        <url type="http" preference="25">http://{{SRV_HOST}}:{{SRV_PORT}}/File3_lowPref</url>
+        <url type="http" preference="30">http://{{SRV_HOST}}:{{SRV_PORT}}/File3</url>
+      </resources>
+    </file>
+    <file name="dir/File4">
+      <verification>
+        <hash type="sha256">{{FILE4_HASH}}</hash>
+      </verification>
+      <resources>
+        <url type="http" preference="35">http://{{SRV_HOST}}:{{SRV_PORT}}/wrong_file</url>
+        <url type="http" preference="40">http://{{SRV_HOST}}:{{SRV_PORT}}/404</url>
+        <url type="http" preference="25">http://{{SRV_HOST}}:{{SRV_PORT}}/File4_lowPref</url>
+        <url type="http" preference="30">http://{{SRV_HOST}}:{{SRV_PORT}}/File4</url>
+      </resources>
+    </file>
+    <file name="dir/File5">
+      <verification>
+        <hash type="sha256">{{FILE5_HASH}}</hash>
+      </verification>
+      <resources>
+        <url type="http" preference="35">http://{{SRV_HOST}}:{{SRV_PORT}}/wrong_file</url>
+        <url type="http" preference="40">http://{{SRV_HOST}}:{{SRV_PORT}}/404</url>
+        <url type="http" preference="25">http://{{SRV_HOST}}:{{SRV_PORT}}/File5_lowPref</url>
+        <url type="http" preference="30">http://{{SRV_HOST}}:{{SRV_PORT}}/File5</url>
+      </resources>
+    </file>
+  </files>
+</metalink>
+"""
+
+MetaXml2 = \
+"""<?xml version="1.0" encoding="utf-8"?>
+<metalink version="3.0" xmlns="http://www.metalinker.org/">
+  <publisher>
+    <name>GNU Wget</name>
+  </publisher>
+  <license>
+    <name>GNU GPL</name>
+    <url>http://www.gnu.org/licenses/gpl.html</url>
+  </license>
+  <identity>Wget Test Files</identity>
+  <version>1.2.3</version>
+  <description>Wget Test Files description</description>
+  <files>
+    <file name="bad">
+      <verification>
+        <hash type="sha256">{{BAD_HASH}}</hash>
+      </verification>
+      <resources>
+        <url type="http" preference="35">http://{{SRV_HOST}}:{{SRV_PORT}}/wrong_file</url>
+        <url type="http" preference="40">http://{{SRV_HOST}}:{{SRV_PORT}}/404</url>
+        <url type="http" preference="30">http://{{SRV_HOST}}:{{SRV_PORT}}/bad</url>
+      </resources>
+    </file>
+  </files>
+</metalink>
+"""
+
+LinkHeaders = [
+    # This file has the lowest priority, and should go last
+    "<http://{{SRV_HOST}}:{{SRV_PORT}}/test1.metalink>; rel=describedby; pri=2; type=\"application/metalink4+xml\"",
+    # This file has the highest priority, and should go first
+    "<http://{{SRV_HOST}}:{{SRV_PORT}}/test2.metalink>; rel=describedby; pri=1; type=\"application/metalink4+xml\""
+]
+
+# This will be filled as soon as we know server hostname and port
+MetaHTTPRules = {'SendHeader' : {}}
+
+MetaHTTP = WgetFile ("main.metalink", rules=MetaHTTPRules)
+
+wrong_file = WgetFile ("wrong_file", bad)
+
+File1_orig = WgetFile ("File1", File1)
+File1_down = WgetFile ("dir/File1", File1)
+File1_nono = WgetFile ("File1_lowPref", File1_lowPref)
+
+File2_orig = WgetFile ("File2", File2)
+File2_down = WgetFile ("dir/File2", File2)
+File2_nono = WgetFile ("File2_lowPref", File2_lowPref)
+
+# rejected by libmetalink
+File3_orig = WgetFile ("File3", File3)
+File3_nono = WgetFile ("File3_lowPref", File3_lowPref)
+
+File4_orig = WgetFile ("File4", File4)
+File4_down = WgetFile ("dir/File4", File4)
+File4_nono = WgetFile ("File4_lowPref", File4_lowPref)
+
+File5_orig = WgetFile ("File5", File5)
+File5_down = WgetFile ("dir/File5", File5)
+File5_nono = WgetFile ("File5_lowPref", File5_lowPref)
+
+MetaFile1 = WgetFile ("test1.metalink", MetaXml1)
+MetaFile1_down = WgetFile ("test1.metalink", MetaXml1)
+
+MetaFile2 = WgetFile ("test2.metalink", MetaXml2)
+
+WGET_OPTIONS = "--trust-server-names --metalink-over-http --metalink-index=2"
+WGET_URLS = [["main.metalink"]]
+
+RequestList = [[
+    "HEAD /main.metalink",
+    "GET /404",
+    "GET /wrong_file",
+    "GET /test1.metalink",
+    "GET /File1",
+    "GET /File2",
+    "GET /File4",
+    "GET /File5"
+]]
+
+Files = [[
+    MetaHTTP,
+    wrong_file,
+    MetaFile1, MetaFile2,
+    File1_orig, File1_nono,
+    File2_orig, File2_nono,
+    File3_orig, File3_nono,
+    File4_orig, File4_nono,
+    File5_orig, File5_nono
+]]
+Existing_Files = []
+
+ExpectedReturnCode = 0
+ExpectedDownloadedFiles = [
+    MetaFile1_down,
+    File1_down,
+    File2_down,
+    File4_down,
+    File5_down
+]
+
+################ Pre and Post Test Hooks #####################################
+pre_test = {
+    "ServerFiles"       : Files,
+    "LocalFiles"        : Existing_Files
+}
+test_options = {
+    "WgetCommands"      : WGET_OPTIONS,
+    "Urls"              : WGET_URLS
+}
+post_test = {
+    "ExpectedFiles"     : ExpectedDownloadedFiles,
+    "ExpectedRetcode"   : ExpectedReturnCode,
+    "FilesCrawled"      : RequestList
+}
+
+http_test = HTTPTest (
+                pre_hook=pre_test,
+                test_params=test_options,
+                post_hook=post_test
+)
+
+http_test.server_setup()
+### Get and use dynamic server sockname
+srv_host, srv_port = http_test.servers[0].server_inst.socket.getsockname ()
+
+MetaXml1 = MetaXml1.replace('{{FILE1_HASH}}', File1_sha256)
+MetaXml1 = MetaXml1.replace('{{FILE2_HASH}}', File2_sha256)
+MetaXml1 = MetaXml1.replace('{{FILE3_HASH}}', File3_sha256)
+MetaXml1 = MetaXml1.replace('{{FILE4_HASH}}', File4_sha256)
+MetaXml1 = MetaXml1.replace('{{FILE5_HASH}}', File5_sha256)
+MetaXml1 = MetaXml1.replace('{{SRV_HOST}}', srv_host)
+MetaXml1 = MetaXml1.replace('{{SRV_PORT}}', str (srv_port))
+MetaFile1.content = MetaXml1
+MetaFile1_down.content = MetaXml1
+
+MetaXml2 = MetaXml2.replace('{{BAD_HASH}}', bad_sha256)
+MetaXml2 = MetaXml2.replace('{{SRV_HOST}}', srv_host)
+MetaXml2 = MetaXml2.replace('{{SRV_PORT}}', str (srv_port))
+MetaFile2.content = MetaXml2
+
+# Helper function for hostname, port and digest substitution
+def SubstituteServerInfo (text, host, port):
+    text = text.replace('{{SRV_HOST}}', host)
+    text = text.replace('{{SRV_PORT}}', str (port))
+    return text
+
+MetaHTTPRules["SendHeader"] = {
+        'Link': [ SubstituteServerInfo (LinkHeader, srv_host, srv_port)
+                    for LinkHeader in LinkHeaders ]
+}
+
+err = http_test.begin ()
+
+exit (err)

testenv/Test-metalink-http-xml-type-content.py

@@ -0,0 +1,221 @@
+#!/usr/bin/env python3
+from sys import exit
+from test.http_test import HTTPTest
+from misc.wget_file import WgetFile
+import hashlib
+from base64 import b64encode
+
+"""
+    This is to test Metalink/HTTP with Content-Type Metalink/XML.
+
+    With --trust-server-names, trust the metalink:file names.
+
+    Without --trust-server-names, don't trust the metalink:file names:
+    use the basename of --input-metalink, and add a sequential number
+    (e.g. .#1, .#2, etc.).
+
+    Strip the directory from unsafe paths.
+"""
+
+############# File Definitions ###############################################
+bad = "Ouch!"
+
+File1 = "Would you like some Tea?"
+File1_lowPref = "Do not take this"
+File1_sha256 = hashlib.sha256 (File1.encode ('UTF-8')).hexdigest ()
+
+File2 = "This is gonna be good"
+File2_lowPref = "Not this one too"
+File2_sha256 = hashlib.sha256 (File2.encode ('UTF-8')).hexdigest ()
+
+File3 = "A little more, please"
+File3_lowPref = "That's just too much"
+File3_sha256 = hashlib.sha256 (File3.encode ('UTF-8')).hexdigest ()
+
+File4 = "Maybe a biscuit?"
+File4_lowPref = "No, thanks"
+File4_sha256 = hashlib.sha256 (File4.encode ('UTF-8')).hexdigest ()
+
+File5 = "More Tea...?"
+File5_lowPref = "I have to go..."
+File5_sha256 = hashlib.sha256 (File5.encode ('UTF-8')).hexdigest ()
+
+MetaXml = \
+"""<?xml version="1.0" encoding="utf-8"?>
+<metalink version="3.0" xmlns="http://www.metalinker.org/">
+  <publisher>
+    <name>GNU Wget</name>
+  </publisher>
+  <license>
+    <name>GNU GPL</name>
+    <url>http://www.gnu.org/licenses/gpl.html</url>
+  </license>
+  <identity>Wget Test Files</identity>
+  <version>1.2.3</version>
+  <description>Wget Test Files description</description>
+  <files>
+    <file name="dir/File1">
+      <verification>
+        <hash type="sha256">{{FILE1_HASH}}</hash>
+      </verification>
+      <resources>
+        <url type="http" preference="35">http://{{SRV_HOST}}:{{SRV_PORT}}/wrong_file</url>
+        <url type="http" preference="40">http://{{SRV_HOST}}:{{SRV_PORT}}/404</url>
+        <url type="http" preference="25">http://{{SRV_HOST}}:{{SRV_PORT}}/File1_lowPref</url>
+        <url type="http" preference="30">http://{{SRV_HOST}}:{{SRV_PORT}}/File1</url>
+      </resources>
+    </file>
+    <file name="dir/File2">
+      <verification>
+        <hash type="sha256">{{FILE2_HASH}}</hash>
+      </verification>
+      <resources>
+        <url type="http" preference="35">http://{{SRV_HOST}}:{{SRV_PORT}}/wrong_file</url>
+        <url type="http" preference="40">http://{{SRV_HOST}}:{{SRV_PORT}}/404</url>
+        <url type="http" preference="25">http://{{SRV_HOST}}:{{SRV_PORT}}/File2_lowPref</url>
+        <url type="http" preference="30">http://{{SRV_HOST}}:{{SRV_PORT}}/File2</url>
+      </resources>
+    </file>
+    <file name="/dir/File3"> <!-- rejected by libmetalink -->
+      <verification>
+        <hash type="sha256">{{FILE3_HASH}}</hash>
+      </verification>
+      <resources>
+        <url type="http" preference="35">http://{{SRV_HOST}}:{{SRV_PORT}}/wrong_file</url>
+        <url type="http" preference="40">http://{{SRV_HOST}}:{{SRV_PORT}}/404</url>
+        <url type="http" preference="25">http://{{SRV_HOST}}:{{SRV_PORT}}/File3_lowPref</url>
+        <url type="http" preference="30">http://{{SRV_HOST}}:{{SRV_PORT}}/File3</url>
+      </resources>
+    </file>
+    <file name="dir/File4">
+      <verification>
+        <hash type="sha256">{{FILE4_HASH}}</hash>
+      </verification>
+      <resources>
+        <url type="http" preference="35">http://{{SRV_HOST}}:{{SRV_PORT}}/wrong_file</url>
+        <url type="http" preference="40">http://{{SRV_HOST}}:{{SRV_PORT}}/404</url>
+        <url type="http" preference="25">http://{{SRV_HOST}}:{{SRV_PORT}}/File4_lowPref</url>
+        <url type="http" preference="30">http://{{SRV_HOST}}:{{SRV_PORT}}/File4</url>
+      </resources>
+    </file>
+    <file name="dir/File5">
+      <verification>
+        <hash type="sha256">{{FILE5_HASH}}</hash>
+      </verification>
+      <resources>
+        <url type="http" preference="35">http://{{SRV_HOST}}:{{SRV_PORT}}/wrong_file</url>
+        <url type="http" preference="40">http://{{SRV_HOST}}:{{SRV_PORT}}/404</url>
+        <url type="http" preference="25">http://{{SRV_HOST}}:{{SRV_PORT}}/File5_lowPref</url>
+        <url type="http" preference="30">http://{{SRV_HOST}}:{{SRV_PORT}}/File5</url>
+      </resources>
+    </file>
+  </files>
+</metalink>
+"""
+
+# This will be filled as soon as we know server hostname and port
+MetaHTTPRules = {'SendHeader' : {}}
+
+MetaHTTP = WgetFile ("main.metalink", rules=MetaHTTPRules)
+MetaHTTP_down = WgetFile ("main.metalink.meta#1", MetaXml)
+
+wrong_file = WgetFile ("wrong_file", bad)
+
+File1_orig = WgetFile ("File1", File1)
+File1_down = WgetFile ("main.metalink.meta#1.#1", File1)
+File1_nono = WgetFile ("File1_lowPref", File1_lowPref)
+
+File2_orig = WgetFile ("File2", File2)
+File2_down = WgetFile ("main.metalink.meta#1.#2", File2)
+File2_nono = WgetFile ("File2_lowPref", File2_lowPref)
+
+# rejected by libmetalink
+File3_orig = WgetFile ("File3", File3)
+File3_nono = WgetFile ("File3_lowPref", File3_lowPref)
+
+File4_orig = WgetFile ("File4", File4)
+File4_down = WgetFile ("main.metalink.meta#1.#3", File4)
+File4_nono = WgetFile ("File4_lowPref", File4_lowPref)
+
+File5_orig = WgetFile ("File5", File5)
+File5_down = WgetFile ("main.metalink.meta#1.#4", File5)
+File5_nono = WgetFile ("File5_lowPref", File5_lowPref)
+
+WGET_OPTIONS = "--metalink-over-http --content-disposition --metalink-index=0"
+WGET_URLS = [["main.metalink"]]
+
+RequestList = [[
+    "HEAD /main.metalink",
+    "GET /main.metalink",
+    "GET /404",
+    "GET /wrong_file",
+    "GET /File1",
+    "GET /File2",
+    "GET /File4",
+    "GET /File5"
+]]
+
+Files = [[
+    MetaHTTP,
+    wrong_file,
+    File1_orig, File1_nono,
+    File2_orig, File2_nono,
+    File3_orig, File3_nono,
+    File4_orig, File4_nono,
+    File5_orig, File5_nono
+]]
+Existing_Files = []
+
+ExpectedReturnCode = 0
+ExpectedDownloadedFiles = [
+    MetaHTTP_down,
+    File1_down,
+    File2_down,
+    File4_down,
+    File5_down
+]
+
+################ Pre and Post Test Hooks #####################################
+pre_test = {
+    "ServerFiles"       : Files,
+    "LocalFiles"        : Existing_Files
+}
+test_options = {
+    "WgetCommands"      : WGET_OPTIONS,
+    "Urls"              : WGET_URLS
+}
+post_test = {
+    "ExpectedFiles"     : ExpectedDownloadedFiles,
+    "ExpectedRetcode"   : ExpectedReturnCode,
+    "FilesCrawled"      : RequestList
+}
+
+http_test = HTTPTest (
+                pre_hook=pre_test,
+                test_params=test_options,
+                post_hook=post_test
+)
+
+http_test.server_setup()
+### Get and use dynamic server sockname
+srv_host, srv_port = http_test.servers[0].server_inst.socket.getsockname ()
+
+MetaXml = MetaXml.replace('{{FILE1_HASH}}', File1_sha256)
+MetaXml = MetaXml.replace('{{FILE2_HASH}}', File2_sha256)
+MetaXml = MetaXml.replace('{{FILE3_HASH}}', File3_sha256)
+MetaXml = MetaXml.replace('{{FILE4_HASH}}', File4_sha256)
+MetaXml = MetaXml.replace('{{FILE5_HASH}}', File5_sha256)
+MetaXml = MetaXml.replace('{{SRV_HOST}}', srv_host)
+MetaXml = MetaXml.replace('{{SRV_PORT}}', str (srv_port))
+MetaHTTP_down.content = MetaXml
+
+MetaHTTP.content = MetaXml
+
+MetaHTTPRules["SendHeader"] = {
+    'Content-Type': 'application/metalink4+xml',
+    'Content-Disposition': 'filename="newname.metalink"'
+}
+
+err = http_test.begin ()
+
+exit (err)

testenv/Test-metalink-http-xml-type-trust-content.py

@@ -0,0 +1,221 @@
+#!/usr/bin/env python3
+from sys import exit
+from test.http_test import HTTPTest
+from misc.wget_file import WgetFile
+import hashlib
+from base64 import b64encode
+
+"""
+    This is to test Metalink/HTTP with Content-Type Metalink/XML.
+
+    With --trust-server-names, trust the metalink:file names.
+
+    Without --trust-server-names, don't trust the metalink:file names:
+    use the basename of --input-metalink, and add a sequential number
+    (e.g. .#1, .#2, etc.).
+
+    Strip the directory from unsafe paths.
+"""
+
+############# File Definitions ###############################################
+bad = "Ouch!"
+
+File1 = "Would you like some Tea?"
+File1_lowPref = "Do not take this"
+File1_sha256 = hashlib.sha256 (File1.encode ('UTF-8')).hexdigest ()
+
+File2 = "This is gonna be good"
+File2_lowPref = "Not this one too"
+File2_sha256 = hashlib.sha256 (File2.encode ('UTF-8')).hexdigest ()
+
+File3 = "A little more, please"
+File3_lowPref = "That's just too much"
+File3_sha256 = hashlib.sha256 (File3.encode ('UTF-8')).hexdigest ()
+
+File4 = "Maybe a biscuit?"
+File4_lowPref = "No, thanks"
+File4_sha256 = hashlib.sha256 (File4.encode ('UTF-8')).hexdigest ()
+
+File5 = "More Tea...?"
+File5_lowPref = "I have to go..."
+File5_sha256 = hashlib.sha256 (File5.encode ('UTF-8')).hexdigest ()
+
+MetaXml = \
+"""<?xml version="1.0" encoding="utf-8"?>
+<metalink version="3.0" xmlns="http://www.metalinker.org/">
+  <publisher>
+    <name>GNU Wget</name>
+  </publisher>
+  <license>
+    <name>GNU GPL</name>
+    <url>http://www.gnu.org/licenses/gpl.html</url>
+  </license>
+  <identity>Wget Test Files</identity>
+  <version>1.2.3</version>
+  <description>Wget Test Files description</description>
+  <files>
+    <file name="dir/File1">
+      <verification>
+        <hash type="sha256">{{FILE1_HASH}}</hash>
+      </verification>
+      <resources>
+        <url type="http" preference="35">http://{{SRV_HOST}}:{{SRV_PORT}}/wrong_file</url>
+        <url type="http" preference="40">http://{{SRV_HOST}}:{{SRV_PORT}}/404</url>
+        <url type="http" preference="25">http://{{SRV_HOST}}:{{SRV_PORT}}/File1_lowPref</url>
+        <url type="http" preference="30">http://{{SRV_HOST}}:{{SRV_PORT}}/File1</url>
+      </resources>
+    </file>
+    <file name="dir/File2">
+      <verification>
+        <hash type="sha256">{{FILE2_HASH}}</hash>
+      </verification>
+      <resources>
+        <url type="http" preference="35">http://{{SRV_HOST}}:{{SRV_PORT}}/wrong_file</url>
+        <url type="http" preference="40">http://{{SRV_HOST}}:{{SRV_PORT}}/404</url>
+        <url type="http" preference="25">http://{{SRV_HOST}}:{{SRV_PORT}}/File2_lowPref</url>
+        <url type="http" preference="30">http://{{SRV_HOST}}:{{SRV_PORT}}/File2</url>
+      </resources>
+    </file>
+    <file name="/dir/File3"> <!-- rejected by libmetalink -->
+      <verification>
+        <hash type="sha256">{{FILE3_HASH}}</hash>
+      </verification>
+      <resources>
+        <url type="http" preference="35">http://{{SRV_HOST}}:{{SRV_PORT}}/wrong_file</url>
+        <url type="http" preference="40">http://{{SRV_HOST}}:{{SRV_PORT}}/404</url>
+        <url type="http" preference="25">http://{{SRV_HOST}}:{{SRV_PORT}}/File3_lowPref</url>
+        <url type="http" preference="30">http://{{SRV_HOST}}:{{SRV_PORT}}/File3</url>
+      </resources>
+    </file>
+    <file name="dir/File4">
+      <verification>
+        <hash type="sha256">{{FILE4_HASH}}</hash>
+      </verification>
+      <resources>
+        <url type="http" preference="35">http://{{SRV_HOST}}:{{SRV_PORT}}/wrong_file</url>
+        <url type="http" preference="40">http://{{SRV_HOST}}:{{SRV_PORT}}/404</url>
+        <url type="http" preference="25">http://{{SRV_HOST}}:{{SRV_PORT}}/File4_lowPref</url>
+        <url type="http" preference="30">http://{{SRV_HOST}}:{{SRV_PORT}}/File4</url>
+      </resources>
+    </file>
+    <file name="dir/File5">
+      <verification>
+        <hash type="sha256">{{FILE5_HASH}}</hash>
+      </verification>
+      <resources>
+        <url type="http" preference="35">http://{{SRV_HOST}}:{{SRV_PORT}}/wrong_file</url>
+        <url type="http" preference="40">http://{{SRV_HOST}}:{{SRV_PORT}}/404</url>
+        <url type="http" preference="25">http://{{SRV_HOST}}:{{SRV_PORT}}/File5_lowPref</url>
+        <url type="http" preference="30">http://{{SRV_HOST}}:{{SRV_PORT}}/File5</url>
+      </resources>
+    </file>
+  </files>
+</metalink>
+"""
+
+# This will be filled as soon as we know server hostname and port
+MetaHTTPRules = {'SendHeader' : {}}
+
+MetaHTTP = WgetFile ("main.metalink", rules=MetaHTTPRules)
+MetaHTTP_down = WgetFile ("newname.metalink", MetaXml)
+
+wrong_file = WgetFile ("wrong_file", bad)
+
+File1_orig = WgetFile ("File1", File1)
+File1_down = WgetFile ("dir/File1", File1)
+File1_nono = WgetFile ("File1_lowPref", File1_lowPref)
+
+File2_orig = WgetFile ("File2", File2)
+File2_down = WgetFile ("dir/File2", File2)
+File2_nono = WgetFile ("File2_lowPref", File2_lowPref)
+
+# rejected by libmetalink
+File3_orig = WgetFile ("File3", File3)
+File3_nono = WgetFile ("File3_lowPref", File3_lowPref)
+
+File4_orig = WgetFile ("File4", File4)
+File4_down = WgetFile ("dir/File4", File4)
+File4_nono = WgetFile ("File4_lowPref", File4_lowPref)
+
+File5_orig = WgetFile ("File5", File5)
+File5_down = WgetFile ("dir/File5", File5)
+File5_nono = WgetFile ("File5_lowPref", File5_lowPref)
+
+WGET_OPTIONS = "--trust-server-names --metalink-over-http --content-disposition --metalink-index=0"
+WGET_URLS = [["main.metalink"]]
+
+RequestList = [[
+    "HEAD /main.metalink",
+    "GET /main.metalink",
+    "GET /404",
+    "GET /wrong_file",
+    "GET /File1",
+    "GET /File2",
+    "GET /File4",
+    "GET /File5"
+]]
+
+Files = [[
+    MetaHTTP,
+    wrong_file,
+    File1_orig, File1_nono,
+    File2_orig, File2_nono,
+    File3_orig, File3_nono,
+    File4_orig, File4_nono,
+    File5_orig, File5_nono
+]]
+Existing_Files = []
+
+ExpectedReturnCode = 0
+ExpectedDownloadedFiles = [
+    MetaHTTP_down,
+    File1_down,
+    File2_down,
+    File4_down,
+    File5_down
+]
+
+################ Pre and Post Test Hooks #####################################
+pre_test = {
+    "ServerFiles"       : Files,
+    "LocalFiles"        : Existing_Files
+}
+test_options = {
+    "WgetCommands"      : WGET_OPTIONS,
+    "Urls"              : WGET_URLS
+}
+post_test = {
+    "ExpectedFiles"     : ExpectedDownloadedFiles,
+    "ExpectedRetcode"   : ExpectedReturnCode,
+    "FilesCrawled"      : RequestList
+}
+
+http_test = HTTPTest (
+                pre_hook=pre_test,
+                test_params=test_options,
+                post_hook=post_test
+)
+
+http_test.server_setup()
+### Get and use dynamic server sockname
+srv_host, srv_port = http_test.servers[0].server_inst.socket.getsockname ()
+
+MetaXml = MetaXml.replace('{{FILE1_HASH}}', File1_sha256)
+MetaXml = MetaXml.replace('{{FILE2_HASH}}', File2_sha256)
+MetaXml = MetaXml.replace('{{FILE3_HASH}}', File3_sha256)
+MetaXml = MetaXml.replace('{{FILE4_HASH}}', File4_sha256)
+MetaXml = MetaXml.replace('{{FILE5_HASH}}', File5_sha256)
+MetaXml = MetaXml.replace('{{SRV_HOST}}', srv_host)
+MetaXml = MetaXml.replace('{{SRV_PORT}}', str (srv_port))
+MetaHTTP_down.content = MetaXml
+
+MetaHTTP.content = MetaXml
+
+MetaHTTPRules["SendHeader"] = {
+    'Content-Type': 'application/metalink4+xml',
+    'Content-Disposition': 'filename="newname.metalink"'
+}
+
+err = http_test.begin ()
+
+exit (err)

testenv/Test-metalink-http-xml-type-trust.py

@@ -0,0 +1,221 @@
+#!/usr/bin/env python3
+from sys import exit
+from test.http_test import HTTPTest
+from misc.wget_file import WgetFile
+import hashlib
+from base64 import b64encode
+
+"""
+    This is to test Metalink/HTTP with Content-Type Metalink/XML.
+
+    With --trust-server-names, trust the metalink:file names.
+
+    Without --trust-server-names, don't trust the metalink:file names:
+    use the basename of --input-metalink, and add a sequential number
+    (e.g. .#1, .#2, etc.).
+
+    Strip the directory from unsafe paths.
+"""
+
+############# File Definitions ###############################################
+bad = "Ouch!"
+
+File1 = "Would you like some Tea?"
+File1_lowPref = "Do not take this"
+File1_sha256 = hashlib.sha256 (File1.encode ('UTF-8')).hexdigest ()
+
+File2 = "This is gonna be good"
+File2_lowPref = "Not this one too"
+File2_sha256 = hashlib.sha256 (File2.encode ('UTF-8')).hexdigest ()
+
+File3 = "A little more, please"
+File3_lowPref = "That's just too much"
+File3_sha256 = hashlib.sha256 (File3.encode ('UTF-8')).hexdigest ()
+
+File4 = "Maybe a biscuit?"
+File4_lowPref = "No, thanks"
+File4_sha256 = hashlib.sha256 (File4.encode ('UTF-8')).hexdigest ()
+
+File5 = "More Tea...?"
+File5_lowPref = "I have to go..."
+File5_sha256 = hashlib.sha256 (File5.encode ('UTF-8')).hexdigest ()
+
+MetaXml = \
+"""<?xml version="1.0" encoding="utf-8"?>
+<metalink version="3.0" xmlns="http://www.metalinker.org/">
+  <publisher>
+    <name>GNU Wget</name>
+  </publisher>
+  <license>
+    <name>GNU GPL</name>
+    <url>http://www.gnu.org/licenses/gpl.html</url>
+  </license>
+  <identity>Wget Test Files</identity>
+  <version>1.2.3</version>
+  <description>Wget Test Files description</description>
+  <files>
+    <file name="dir/File1">
+      <verification>
+        <hash type="sha256">{{FILE1_HASH}}</hash>
+      </verification>
+      <resources>
+        <url type="http" preference="35">http://{{SRV_HOST}}:{{SRV_PORT}}/wrong_file</url>
+        <url type="http" preference="40">http://{{SRV_HOST}}:{{SRV_PORT}}/404</url>
+        <url type="http" preference="25">http://{{SRV_HOST}}:{{SRV_PORT}}/File1_lowPref</url>
+        <url type="http" preference="30">http://{{SRV_HOST}}:{{SRV_PORT}}/File1</url>
+      </resources>
+    </file>
+    <file name="dir/File2">
+      <verification>
+        <hash type="sha256">{{FILE2_HASH}}</hash>
+      </verification>
+      <resources>
+        <url type="http" preference="35">http://{{SRV_HOST}}:{{SRV_PORT}}/wrong_file</url>
+        <url type="http" preference="40">http://{{SRV_HOST}}:{{SRV_PORT}}/404</url>
+        <url type="http" preference="25">http://{{SRV_HOST}}:{{SRV_PORT}}/File2_lowPref</url>
+        <url type="http" preference="30">http://{{SRV_HOST}}:{{SRV_PORT}}/File2</url>
+      </resources>
+    </file>
+    <file name="/dir/File3"> <!-- rejected by libmetalink -->
+      <verification>
+        <hash type="sha256">{{FILE3_HASH}}</hash>
+      </verification>
+      <resources>
+        <url type="http" preference="35">http://{{SRV_HOST}}:{{SRV_PORT}}/wrong_file</url>
+        <url type="http" preference="40">http://{{SRV_HOST}}:{{SRV_PORT}}/404</url>
+        <url type="http" preference="25">http://{{SRV_HOST}}:{{SRV_PORT}}/File3_lowPref</url>
+        <url type="http" preference="30">http://{{SRV_HOST}}:{{SRV_PORT}}/File3</url>
+      </resources>
+    </file>
+    <file name="dir/File4">
+      <verification>
+        <hash type="sha256">{{FILE4_HASH}}</hash>
+      </verification>
+      <resources>
+        <url type="http" preference="35">http://{{SRV_HOST}}:{{SRV_PORT}}/wrong_file</url>
+        <url type="http" preference="40">http://{{SRV_HOST}}:{{SRV_PORT}}/404</url>
+        <url type="http" preference="25">http://{{SRV_HOST}}:{{SRV_PORT}}/File4_lowPref</url>
+        <url type="http" preference="30">http://{{SRV_HOST}}:{{SRV_PORT}}/File4</url>
+      </resources>
+    </file>
+    <file name="dir/File5">
+      <verification>
+        <hash type="sha256">{{FILE5_HASH}}</hash>
+      </verification>
+      <resources>
+        <url type="http" preference="35">http://{{SRV_HOST}}:{{SRV_PORT}}/wrong_file</url>
+        <url type="http" preference="40">http://{{SRV_HOST}}:{{SRV_PORT}}/404</url>
+        <url type="http" preference="25">http://{{SRV_HOST}}:{{SRV_PORT}}/File5_lowPref</url>
+        <url type="http" preference="30">http://{{SRV_HOST}}:{{SRV_PORT}}/File5</url>
+      </resources>
+    </file>
+  </files>
+</metalink>
+"""
+
+# This will be filled as soon as we know server hostname and port
+MetaHTTPRules = {'SendHeader' : {}}
+
+MetaHTTP = WgetFile ("main.metalink", rules=MetaHTTPRules)
+MetaHTTP_down = WgetFile ("main.metalink", MetaXml)
+
+wrong_file = WgetFile ("wrong_file", bad)
+
+File1_orig = WgetFile ("File1", File1)
+File1_down = WgetFile ("dir/File1", File1)
+File1_nono = WgetFile ("File1_lowPref", File1_lowPref)
+
+File2_orig = WgetFile ("File2", File2)
+File2_down = WgetFile ("dir/File2", File2)
+File2_nono = WgetFile ("File2_lowPref", File2_lowPref)
+
+# rejected by libmetalink
+File3_orig = WgetFile ("File3", File3)
+File3_nono = WgetFile ("File3_lowPref", File3_lowPref)
+
+File4_orig = WgetFile ("File4", File4)
+File4_down = WgetFile ("dir/File4", File4)
+File4_nono = WgetFile ("File4_lowPref", File4_lowPref)
+
+File5_orig = WgetFile ("File5", File5)
+File5_down = WgetFile ("dir/File5", File5)
+File5_nono = WgetFile ("File5_lowPref", File5_lowPref)
+
+WGET_OPTIONS = "--trust-server-names --metalink-over-http --metalink-index=0"
+WGET_URLS = [["main.metalink"]]
+
+RequestList = [[
+    "HEAD /main.metalink",
+    "GET /main.metalink",
+    "GET /404",
+    "GET /wrong_file",
+    "GET /File1",
+    "GET /File2",
+    "GET /File4",
+    "GET /File5"
+]]
+
+Files = [[
+    MetaHTTP,
+    wrong_file,
+    File1_orig, File1_nono,
+    File2_orig, File2_nono,
+    File3_orig, File3_nono,
+    File4_orig, File4_nono,
+    File5_orig, File5_nono
+]]
+Existing_Files = []
+
+ExpectedReturnCode = 0
+ExpectedDownloadedFiles = [
+    MetaHTTP_down,
+    File1_down,
+    File2_down,
+    File4_down,
+    File5_down
+]
+
+################ Pre and Post Test Hooks #####################################
+pre_test = {
+    "ServerFiles"       : Files,
+    "LocalFiles"        : Existing_Files
+}
+test_options = {
+    "WgetCommands"      : WGET_OPTIONS,
+    "Urls"              : WGET_URLS
+}
+post_test = {
+    "ExpectedFiles"     : ExpectedDownloadedFiles,
+    "ExpectedRetcode"   : ExpectedReturnCode,
+    "FilesCrawled"      : RequestList
+}
+
+http_test = HTTPTest (
+                pre_hook=pre_test,
+                test_params=test_options,
+                post_hook=post_test
+)
+
+http_test.server_setup()
+### Get and use dynamic server sockname
+srv_host, srv_port = http_test.servers[0].server_inst.socket.getsockname ()
+
+MetaXml = MetaXml.replace('{{FILE1_HASH}}', File1_sha256)
+MetaXml = MetaXml.replace('{{FILE2_HASH}}', File2_sha256)
+MetaXml = MetaXml.replace('{{FILE3_HASH}}', File3_sha256)
+MetaXml = MetaXml.replace('{{FILE4_HASH}}', File4_sha256)
+MetaXml = MetaXml.replace('{{FILE5_HASH}}', File5_sha256)
+MetaXml = MetaXml.replace('{{SRV_HOST}}', srv_host)
+MetaXml = MetaXml.replace('{{SRV_PORT}}', str (srv_port))
+MetaHTTP_down.content = MetaXml
+
+MetaHTTP.content = MetaXml
+
+MetaHTTPRules["SendHeader"] = {
+    'Content-Type': 'application/metalink4+xml',
+    'Content-Disposition': 'filename="newname.metalink"'
+}
+
+err = http_test.begin ()
+
+exit (err)

testenv/Test-metalink-http-xml-type.py

@@ -0,0 +1,221 @@
+#!/usr/bin/env python3
+from sys import exit
+from test.http_test import HTTPTest
+from misc.wget_file import WgetFile
+import hashlib
+from base64 import b64encode
+
+"""
+    This is to test Metalink/HTTP with Content-Type Metalink/XML.
+
+    With --trust-server-names, trust the metalink:file names.
+
+    Without --trust-server-names, don't trust the metalink:file names:
+    use the basename of --input-metalink, and add a sequential number
+    (e.g. .#1, .#2, etc.).
+
+    Strip the directory from unsafe paths.
+"""
+
+############# File Definitions ###############################################
+bad = "Ouch!"
+
+File1 = "Would you like some Tea?"
+File1_lowPref = "Do not take this"
+File1_sha256 = hashlib.sha256 (File1.encode ('UTF-8')).hexdigest ()
+
+File2 = "This is gonna be good"
+File2_lowPref = "Not this one too"
+File2_sha256 = hashlib.sha256 (File2.encode ('UTF-8')).hexdigest ()
+
+File3 = "A little more, please"
+File3_lowPref = "That's just too much"
+File3_sha256 = hashlib.sha256 (File3.encode ('UTF-8')).hexdigest ()
+
+File4 = "Maybe a biscuit?"
+File4_lowPref = "No, thanks"
+File4_sha256 = hashlib.sha256 (File4.encode ('UTF-8')).hexdigest ()
+
+File5 = "More Tea...?"
+File5_lowPref = "I have to go..."
+File5_sha256 = hashlib.sha256 (File5.encode ('UTF-8')).hexdigest ()
+
+MetaXml = \
+"""<?xml version="1.0" encoding="utf-8"?>
+<metalink version="3.0" xmlns="http://www.metalinker.org/">
+  <publisher>
+    <name>GNU Wget</name>
+  </publisher>
+  <license>
+    <name>GNU GPL</name>
+    <url>http://www.gnu.org/licenses/gpl.html</url>
+  </license>
+  <identity>Wget Test Files</identity>
+  <version>1.2.3</version>
+  <description>Wget Test Files description</description>
+  <files>
+    <file name="dir/File1">
+      <verification>
+        <hash type="sha256">{{FILE1_HASH}}</hash>
+      </verification>
+      <resources>
+        <url type="http" preference="35">http://{{SRV_HOST}}:{{SRV_PORT}}/wrong_file</url>
+        <url type="http" preference="40">http://{{SRV_HOST}}:{{SRV_PORT}}/404</url>
+        <url type="http" preference="25">http://{{SRV_HOST}}:{{SRV_PORT}}/File1_lowPref</url>
+        <url type="http" preference="30">http://{{SRV_HOST}}:{{SRV_PORT}}/File1</url>
+      </resources>
+    </file>
+    <file name="dir/File2">
+      <verification>
+        <hash type="sha256">{{FILE2_HASH}}</hash>
+      </verification>
+      <resources>
+        <url type="http" preference="35">http://{{SRV_HOST}}:{{SRV_PORT}}/wrong_file</url>
+        <url type="http" preference="40">http://{{SRV_HOST}}:{{SRV_PORT}}/404</url>
+        <url type="http" preference="25">http://{{SRV_HOST}}:{{SRV_PORT}}/File2_lowPref</url>
+        <url type="http" preference="30">http://{{SRV_HOST}}:{{SRV_PORT}}/File2</url>
+      </resources>
+    </file>
+    <file name="/dir/File3"> <!-- rejected by libmetalink -->
+      <verification>
+        <hash type="sha256">{{FILE3_HASH}}</hash>
+      </verification>
+      <resources>
+        <url type="http" preference="35">http://{{SRV_HOST}}:{{SRV_PORT}}/wrong_file</url>
+        <url type="http" preference="40">http://{{SRV_HOST}}:{{SRV_PORT}}/404</url>
+        <url type="http" preference="25">http://{{SRV_HOST}}:{{SRV_PORT}}/File3_lowPref</url>
+        <url type="http" preference="30">http://{{SRV_HOST}}:{{SRV_PORT}}/File3</url>
+      </resources>
+    </file>
+    <file name="dir/File4">
+      <verification>
+        <hash type="sha256">{{FILE4_HASH}}</hash>
+      </verification>
+      <resources>
+        <url type="http" preference="35">http://{{SRV_HOST}}:{{SRV_PORT}}/wrong_file</url>
+        <url type="http" preference="40">http://{{SRV_HOST}}:{{SRV_PORT}}/404</url>
+        <url type="http" preference="25">http://{{SRV_HOST}}:{{SRV_PORT}}/File4_lowPref</url>
+        <url type="http" preference="30">http://{{SRV_HOST}}:{{SRV_PORT}}/File4</url>
+      </resources>
+    </file>
+    <file name="dir/File5">
+      <verification>
+        <hash type="sha256">{{FILE5_HASH}}</hash>
+      </verification>
+      <resources>
+        <url type="http" preference="35">http://{{SRV_HOST}}:{{SRV_PORT}}/wrong_file</url>
+        <url type="http" preference="40">http://{{SRV_HOST}}:{{SRV_PORT}}/404</url>
+        <url type="http" preference="25">http://{{SRV_HOST}}:{{SRV_PORT}}/File5_lowPref</url>
+        <url type="http" preference="30">http://{{SRV_HOST}}:{{SRV_PORT}}/File5</url>
+      </resources>
+    </file>
+  </files>
+</metalink>
+"""
+
+# This will be filled as soon as we know server hostname and port
+MetaHTTPRules = {'SendHeader' : {}}
+
+MetaHTTP = WgetFile ("main.metalink", rules=MetaHTTPRules)
+MetaHTTP_down = WgetFile ("main.metalink.meta#1", MetaXml)
+
+wrong_file = WgetFile ("wrong_file", bad)
+
+File1_orig = WgetFile ("File1", File1)
+File1_down = WgetFile ("main.metalink.meta#1.#1", File1)
+File1_nono = WgetFile ("File1_lowPref", File1_lowPref)
+
+File2_orig = WgetFile ("File2", File2)
+File2_down = WgetFile ("main.metalink.meta#1.#2", File2)
+File2_nono = WgetFile ("File2_lowPref", File2_lowPref)
+
+# rejected by libmetalink
+File3_orig = WgetFile ("File3", File3)
+File3_nono = WgetFile ("File3_lowPref", File3_lowPref)
+
+File4_orig = WgetFile ("File4", File4)
+File4_down = WgetFile ("main.metalink.meta#1.#3", File4)
+File4_nono = WgetFile ("File4_lowPref", File4_lowPref)
+
+File5_orig = WgetFile ("File5", File5)
+File5_down = WgetFile ("main.metalink.meta#1.#4", File5)
+File5_nono = WgetFile ("File5_lowPref", File5_lowPref)
+
+WGET_OPTIONS = "--metalink-over-http --metalink-index=0"
+WGET_URLS = [["main.metalink"]]
+
+RequestList = [[
+    "HEAD /main.metalink",
+    "GET /main.metalink",
+    "GET /404",
+    "GET /wrong_file",
+    "GET /File1",
+    "GET /File2",
+    "GET /File4",
+    "GET /File5"
+]]
+
+Files = [[
+    MetaHTTP,
+    wrong_file,
+    File1_orig, File1_nono,
+    File2_orig, File2_nono,
+    File3_orig, File3_nono,
+    File4_orig, File4_nono,
+    File5_orig, File5_nono
+]]
+Existing_Files = []
+
+ExpectedReturnCode = 0
+ExpectedDownloadedFiles = [
+    MetaHTTP_down,
+    File1_down,
+    File2_down,
+    File4_down,
+    File5_down
+]
+
+################ Pre and Post Test Hooks #####################################
+pre_test = {
+    "ServerFiles"       : Files,
+    "LocalFiles"        : Existing_Files
+}
+test_options = {
+    "WgetCommands"      : WGET_OPTIONS,
+    "Urls"              : WGET_URLS
+}
+post_test = {
+    "ExpectedFiles"     : ExpectedDownloadedFiles,
+    "ExpectedRetcode"   : ExpectedReturnCode,
+    "FilesCrawled"      : RequestList
+}
+
+http_test = HTTPTest (
+                pre_hook=pre_test,
+                test_params=test_options,
+                post_hook=post_test
+)
+
+http_test.server_setup()
+### Get and use dynamic server sockname
+srv_host, srv_port = http_test.servers[0].server_inst.socket.getsockname ()
+
+MetaXml = MetaXml.replace('{{FILE1_HASH}}', File1_sha256)
+MetaXml = MetaXml.replace('{{FILE2_HASH}}', File2_sha256)
+MetaXml = MetaXml.replace('{{FILE3_HASH}}', File3_sha256)
+MetaXml = MetaXml.replace('{{FILE4_HASH}}', File4_sha256)
+MetaXml = MetaXml.replace('{{FILE5_HASH}}', File5_sha256)
+MetaXml = MetaXml.replace('{{SRV_HOST}}', srv_host)
+MetaXml = MetaXml.replace('{{SRV_PORT}}', str (srv_port))
+MetaHTTP_down.content = MetaXml
+
+MetaHTTP.content = MetaXml
+
+MetaHTTPRules["SendHeader"] = {
+    'Content-Type': 'application/metalink4+xml',
+    'Content-Disposition': 'filename="newname.metalink"'
+}
+
+err = http_test.begin ()
+
+exit (err)

testenv/Test-metalink-http-xml.py

@@ -0,0 +1,272 @@
+#!/usr/bin/env python3
+from sys import exit
+from test.http_test import HTTPTest
+from misc.wget_file import WgetFile
+import hashlib
+from base64 import b64encode
+
+"""
+    This is to test Metalink/HTTP with Metalink/XML Link headers.
+
+    With --trust-server-names, trust the metalink:file names.
+
+    Without --trust-server-names, don't trust the metalink:file names:
+    use the basename of --input-metalink, and add a sequential number
+    (e.g. .#1, .#2, etc.).
+
+    Strip the directory from unsafe paths.
+"""
+
+############# File Definitions ###############################################
+bad = "Ouch!"
+bad_sha256 = hashlib.sha256 (bad.encode ('UTF-8')).hexdigest ()
+
+File1 = "Would you like some Tea?"
+File1_lowPref = "Do not take this"
+File1_sha256 = hashlib.sha256 (File1.encode ('UTF-8')).hexdigest ()
+
+File2 = "This is gonna be good"
+File2_lowPref = "Not this one too"
+File2_sha256 = hashlib.sha256 (File2.encode ('UTF-8')).hexdigest ()
+
+File3 = "A little more, please"
+File3_lowPref = "That's just too much"
+File3_sha256 = hashlib.sha256 (File3.encode ('UTF-8')).hexdigest ()
+
+File4 = "Maybe a biscuit?"
+File4_lowPref = "No, thanks"
+File4_sha256 = hashlib.sha256 (File4.encode ('UTF-8')).hexdigest ()
+
+File5 = "More Tea...?"
+File5_lowPref = "I have to go..."
+File5_sha256 = hashlib.sha256 (File5.encode ('UTF-8')).hexdigest ()
+
+MetaXml1 = \
+"""<?xml version="1.0" encoding="utf-8"?>
+<metalink version="3.0" xmlns="http://www.metalinker.org/">
+  <publisher>
+    <name>GNU Wget</name>
+  </publisher>
+  <license>
+    <name>GNU GPL</name>
+    <url>http://www.gnu.org/licenses/gpl.html</url>
+  </license>
+  <identity>Wget Test Files</identity>
+  <version>1.2.3</version>
+  <description>Wget Test Files description</description>
+  <files>
+    <file name="dir/File1">
+      <verification>
+        <hash type="sha256">{{FILE1_HASH}}</hash>
+      </verification>
+      <resources>
+        <url type="http" preference="35">http://{{SRV_HOST}}:{{SRV_PORT}}/wrong_file</url>
+        <url type="http" preference="40">http://{{SRV_HOST}}:{{SRV_PORT}}/404</url>
+        <url type="http" preference="25">http://{{SRV_HOST}}:{{SRV_PORT}}/File1_lowPref</url>
+        <url type="http" preference="30">http://{{SRV_HOST}}:{{SRV_PORT}}/File1</url>
+      </resources>
+    </file>
+    <file name="dir/File2">
+      <verification>
+        <hash type="sha256">{{FILE2_HASH}}</hash>
+      </verification>
+      <resources>
+        <url type="http" preference="35">http://{{SRV_HOST}}:{{SRV_PORT}}/wrong_file</url>
+        <url type="http" preference="40">http://{{SRV_HOST}}:{{SRV_PORT}}/404</url>
+        <url type="http" preference="25">http://{{SRV_HOST}}:{{SRV_PORT}}/File2_lowPref</url>
+        <url type="http" preference="30">http://{{SRV_HOST}}:{{SRV_PORT}}/File2</url>
+      </resources>
+    </file>
+    <file name="/dir/File3"> <!-- rejected by libmetalink -->
+      <verification>
+        <hash type="sha256">{{FILE3_HASH}}</hash>
+      </verification>
+      <resources>
+        <url type="http" preference="35">http://{{SRV_HOST}}:{{SRV_PORT}}/wrong_file</url>
+        <url type="http" preference="40">http://{{SRV_HOST}}:{{SRV_PORT}}/404</url>
+        <url type="http" preference="25">http://{{SRV_HOST}}:{{SRV_PORT}}/File3_lowPref</url>
+        <url type="http" preference="30">http://{{SRV_HOST}}:{{SRV_PORT}}/File3</url>
+      </resources>
+    </file>
+    <file name="dir/File4">
+      <verification>
+        <hash type="sha256">{{FILE4_HASH}}</hash>
+      </verification>
+      <resources>
+        <url type="http" preference="35">http://{{SRV_HOST}}:{{SRV_PORT}}/wrong_file</url>
+        <url type="http" preference="40">http://{{SRV_HOST}}:{{SRV_PORT}}/404</url>
+        <url type="http" preference="25">http://{{SRV_HOST}}:{{SRV_PORT}}/File4_lowPref</url>
+        <url type="http" preference="30">http://{{SRV_HOST}}:{{SRV_PORT}}/File4</url>
+      </resources>
+    </file>
+    <file name="dir/File5">
+      <verification>
+        <hash type="sha256">{{FILE5_HASH}}</hash>
+      </verification>
+      <resources>
+        <url type="http" preference="35">http://{{SRV_HOST}}:{{SRV_PORT}}/wrong_file</url>
+        <url type="http" preference="40">http://{{SRV_HOST}}:{{SRV_PORT}}/404</url>
+        <url type="http" preference="25">http://{{SRV_HOST}}:{{SRV_PORT}}/File5_lowPref</url>
+        <url type="http" preference="30">http://{{SRV_HOST}}:{{SRV_PORT}}/File5</url>
+      </resources>
+    </file>
+  </files>
+</metalink>
+"""
+
+MetaXml2 = \
+"""<?xml version="1.0" encoding="utf-8"?>
+<metalink version="3.0" xmlns="http://www.metalinker.org/">
+  <publisher>
+    <name>GNU Wget</name>
+  </publisher>
+  <license>
+    <name>GNU GPL</name>
+    <url>http://www.gnu.org/licenses/gpl.html</url>
+  </license>
+  <identity>Wget Test Files</identity>
+  <version>1.2.3</version>
+  <description>Wget Test Files description</description>
+  <files>
+    <file name="bad">
+      <verification>
+        <hash type="sha256">{{BAD_HASH}}</hash>
+      </verification>
+      <resources>
+        <url type="http" preference="35">http://{{SRV_HOST}}:{{SRV_PORT}}/wrong_file</url>
+        <url type="http" preference="40">http://{{SRV_HOST}}:{{SRV_PORT}}/404</url>
+        <url type="http" preference="30">http://{{SRV_HOST}}:{{SRV_PORT}}/bad</url>
+      </resources>
+    </file>
+  </files>
+</metalink>
+"""
+
+LinkHeaders = [
+    # This file has the lowest priority, and should go last
+    "<http://{{SRV_HOST}}:{{SRV_PORT}}/test1.metalink>; rel=describedby; pri=2; type=\"application/metalink4+xml\"",
+    # This file has the highest priority, and should go first
+    "<http://{{SRV_HOST}}:{{SRV_PORT}}/test2.metalink>; rel=describedby; pri=1; type=\"application/metalink4+xml\""
+]
+
+# This will be filled as soon as we know server hostname and port
+MetaHTTPRules = {'SendHeader' : {}}
+
+MetaHTTP = WgetFile ("main.metalink", rules=MetaHTTPRules)
+
+wrong_file = WgetFile ("wrong_file", bad)
+
+File1_orig = WgetFile ("File1", File1)
+File1_down = WgetFile ("main.metalink.meta#2.#1", File1)
+File1_nono = WgetFile ("File1_lowPref", File1_lowPref)
+
+File2_orig = WgetFile ("File2", File2)
+File2_down = WgetFile ("main.metalink.meta#2.#2", File2)
+File2_nono = WgetFile ("File2_lowPref", File2_lowPref)
+
+# rejected by libmetalink
+File3_orig = WgetFile ("File3", File3)
+File3_nono = WgetFile ("File3_lowPref", File3_lowPref)
+
+File4_orig = WgetFile ("File4", File4)
+File4_down = WgetFile ("main.metalink.meta#2.#3", File4)
+File4_nono = WgetFile ("File4_lowPref", File4_lowPref)
+
+File5_orig = WgetFile ("File5", File5)
+File5_down = WgetFile ("main.metalink.meta#2.#4", File5)
+File5_nono = WgetFile ("File5_lowPref", File5_lowPref)
+
+MetaFile1 = WgetFile ("test1.metalink", MetaXml1)
+MetaFile1_down = WgetFile ("main.metalink.meta#2", MetaXml1)
+
+MetaFile2 = WgetFile ("test2.metalink", MetaXml2)
+
+WGET_OPTIONS = "--metalink-over-http --metalink-index=2"
+WGET_URLS = [["main.metalink"]]
+
+RequestList = [[
+    "HEAD /main.metalink",
+    "GET /404",
+    "GET /wrong_file",
+    "GET /test1.metalink",
+    "GET /File1",
+    "GET /File2",
+    "GET /File4",
+    "GET /File5"
+]]
+
+Files = [[
+    MetaHTTP,
+    wrong_file,
+    MetaFile1, MetaFile2,
+    File1_orig, File1_nono,
+    File2_orig, File2_nono,
+    File3_orig, File3_nono,
+    File4_orig, File4_nono,
+    File5_orig, File5_nono
+]]
+Existing_Files = []
+
+ExpectedReturnCode = 0
+ExpectedDownloadedFiles = [
+    MetaFile1_down,
+    File1_down,
+    File2_down,
+    File4_down,
+    File5_down
+]
+
+################ Pre and Post Test Hooks #####################################
+pre_test = {
+    "ServerFiles"       : Files,
+    "LocalFiles"        : Existing_Files
+}
+test_options = {
+    "WgetCommands"      : WGET_OPTIONS,
+    "Urls"              : WGET_URLS
+}
+post_test = {
+    "ExpectedFiles"     : ExpectedDownloadedFiles,
+    "ExpectedRetcode"   : ExpectedReturnCode,
+    "FilesCrawled"      : RequestList
+}
+
+http_test = HTTPTest (
+                pre_hook=pre_test,
+                test_params=test_options,
+                post_hook=post_test
+)
+
+http_test.server_setup()
+### Get and use dynamic server sockname
+srv_host, srv_port = http_test.servers[0].server_inst.socket.getsockname ()
+
+MetaXml1 = MetaXml1.replace('{{FILE1_HASH}}', File1_sha256)
+MetaXml1 = MetaXml1.replace('{{FILE2_HASH}}', File2_sha256)
+MetaXml1 = MetaXml1.replace('{{FILE3_HASH}}', File3_sha256)
+MetaXml1 = MetaXml1.replace('{{FILE4_HASH}}', File4_sha256)
+MetaXml1 = MetaXml1.replace('{{FILE5_HASH}}', File5_sha256)
+MetaXml1 = MetaXml1.replace('{{SRV_HOST}}', srv_host)
+MetaXml1 = MetaXml1.replace('{{SRV_PORT}}', str (srv_port))
+MetaFile1.content = MetaXml1
+MetaFile1_down.content = MetaXml1
+
+MetaXml2 = MetaXml2.replace('{{BAD_HASH}}', bad_sha256)
+MetaXml2 = MetaXml2.replace('{{SRV_HOST}}', srv_host)
+MetaXml2 = MetaXml2.replace('{{SRV_PORT}}', str (srv_port))
+MetaFile2.content = MetaXml2
+
+# Helper function for hostname, port and digest substitution
+def SubstituteServerInfo (text, host, port):
+    text = text.replace('{{SRV_HOST}}', host)
+    text = text.replace('{{SRV_PORT}}', str (port))
+    return text
+
+MetaHTTPRules["SendHeader"] = {
+        'Link': [ SubstituteServerInfo (LinkHeader, srv_host, srv_port)
+                    for LinkHeader in LinkHeaders ]
+}
+
+err = http_test.begin ()
+
+exit (err)

testenv/Test-metalink-http.py

@@ -2,7 +2,6 @@
 from sys import exit
 from test.http_test import HTTPTest
 from misc.wget_file import WgetFile
-import re
 import hashlib
 from base64 import b64encode
 
@@ -12,9 +11,9 @@ from base64 import b64encode
 
 # Helper function for hostname, port and digest substitution
 def SubstituteServerInfo (text, host, port, digest):
-    text = re.sub (r'{{FILE1_HASH}}', digest, text)
-    text = re.sub (r'{{SRV_HOST}}', host, text)
-    text = re.sub (r'{{SRV_PORT}}', str (port), text)
+    text = text.replace('{{FILE1_HASH}}', digest)
+    text = text.replace('{{SRV_HOST}}', host)
+    text = text.replace('{{SRV_PORT}}', str (port))
     return text
 
 ############# File Definitions ###############################################

testenv/Test-metalink-xml-abspath-trust.py

@@ -0,0 +1,101 @@
+#!/usr/bin/env python3
+
+from sys import exit
+from misc.metalinkv3_xml import Metalinkv3_XML
+
+"""
+    This is to test if Metalink/XML forbids absolute paths.
+
+    With --trust-server-names, trust the metalink:file names.
+
+    Without --trust-server-names, don't trust the metalink:file names:
+    use the basename of --input-metalink, and add a sequential number
+    (e.g. .#1, .#2, etc.).
+
+    Strip the directory from unsafe paths.
+"""
+
+############# File Definitions ###############################################
+wrong_file = "Ouch!"
+
+File1 = "Would you like some Tea?"
+File1_lowPref = "Do not take this"
+
+File2 = "This is gonna be good"
+File2_lowPref = "Not this one too"
+
+File3 = "A little more, please"
+File3_lowPref = "That's just too much"
+
+File4 = "Maybe a biscuit?"
+File4_lowPref = "No, thanks"
+
+File5 = "More Tea...?"
+File5_lowPref = "I have to go..."
+
+############# Metalink/XML ###################################################
+Meta = Metalinkv3_XML()
+
+# file_name: metalink:file "name" field
+# save_name: metalink:file save name, if None the file is rejected
+# content  : metalink:file content
+#
+# size:
+#   True     auto-compute size
+#   None     no <size></size>
+#    any     use this size
+#
+# hash_sha256:
+#   False    no <verification></verification>
+#   True     auto-compute sha256
+#   None     no <hash></hash>
+#    any     use this hash
+#
+# srv_file   : metalink:url server file
+# srv_content: metalink:url server file content, if None the file doesn't exist
+# utype      : metalink:url type
+# location   : metalink:url location (default 'no location field')
+# preference : metalink:url preference (default 999999)
+
+XmlName = "test.metalink"
+
+Meta.xml (
+    # Metalink/XML file name
+    XmlName,
+    # file_name, save_name, content, size, hash_sha256
+    ["/File1", None, File1, None, True,
+     # srv_file, srv_content, utype, location, preference
+     ["wrong_file", wrong_file, "http", None, 35],
+     ["404", None, "http", None, 40],
+     ["File1_lowPref", File1_lowPref, "http", None, 25],
+     ["File1", File1, "http", None, 30]],
+    ["File2", "File2", File2, None, True,
+     ["wrong_file", wrong_file, "http", None, 35],
+     ["404", None, "http", None, 40],
+     ["File2_lowPref", File2_lowPref, "http", None, 25],
+     ["File2", File2, "http", None, 30]],
+    ["/File3", None, File3, None, True,
+     ["wrong_file", wrong_file, "http", None, 35],
+     ["404", None, "http", None, 40],
+     ["File3_lowPref", File3_lowPref, "http", None, 25],
+     ["File3", File3, "http", None, 30]],
+    ["/File4", None, File4, None, True,
+     ["wrong_file", wrong_file, "http", None, 35],
+     ["404", None, "http", None, 40],
+     ["File4_lowPref", File4_lowPref, "http", None, 25],
+     ["File4", File4, "http", None, 30]],
+    ["File5", "File5", File5, None, True,
+     ["wrong_file", wrong_file, "http", None, 35],
+     ["404", None, "http", None, 40],
+     ["File5_lowPref", File5_lowPref, "http", None, 25],
+     ["File5", File5, "http", None, 30]],
+)
+
+Meta.print_meta ()
+
+err = Meta.http_test (
+    "--trust-server-names " + \
+    "--input-metalink " + XmlName, 0
+)
+
+exit (err)

testenv/Test-metalink-xml-abspath.py

@@ -0,0 +1,100 @@
+#!/usr/bin/env python3
+
+from sys import exit
+from misc.metalinkv3_xml import Metalinkv3_XML
+
+"""
+    This is to test if Metalink/XML forbids absolute paths.
+
+    With --trust-server-names, trust the metalink:file names.
+
+    Without --trust-server-names, don't trust the metalink:file names:
+    use the basename of --input-metalink, and add a sequential number
+    (e.g. .#1, .#2, etc.).
+
+    Strip the directory from unsafe paths.
+"""
+
+############# File Definitions ###############################################
+wrong_file = "Ouch!"
+
+File1 = "Would you like some Tea?"
+File1_lowPref = "Do not take this"
+
+File2 = "This is gonna be good"
+File2_lowPref = "Not this one too"
+
+File3 = "A little more, please"
+File3_lowPref = "That's just too much"
+
+File4 = "Maybe a biscuit?"
+File4_lowPref = "No, thanks"
+
+File5 = "More Tea...?"
+File5_lowPref = "I have to go..."
+
+############# Metalink/XML ###################################################
+Meta = Metalinkv3_XML()
+
+# file_name: metalink:file "name" field
+# save_name: metalink:file save name, if None the file is rejected
+# content  : metalink:file content
+#
+# size:
+#   True     auto-compute size
+#   None     no <size></size>
+#    any     use this size
+#
+# hash_sha256:
+#   False    no <verification></verification>
+#   True     auto-compute sha256
+#   None     no <hash></hash>
+#    any     use this hash
+#
+# srv_file   : metalink:url server file
+# srv_content: metalink:url server file content, if None the file doesn't exist
+# utype      : metalink:url type
+# location   : metalink:url location (default 'no location field')
+# preference : metalink:url preference (default 999999)
+
+XmlName = "test.metalink"
+
+Meta.xml (
+    # Metalink/XML file name
+    XmlName,
+    # file_name, save_name, content, size, hash_sha256
+    ["/File1", None, File1, None, True,
+     # srv_file, srv_content, utype, location, preference
+     ["wrong_file", wrong_file, "http", None, 35],
+     ["404", None, "http", None, 40],
+     ["File1_lowPref", File1_lowPref, "http", None, 25],
+     ["File1", File1, "http", None, 30]],
+    ["File2", XmlName + ".#1", File2, None, True,
+     ["wrong_file", wrong_file, "http", None, 35],
+     ["404", None, "http", None, 40],
+     ["File2_lowPref", File2_lowPref, "http", None, 25],
+     ["File2", File2, "http", None, 30]],
+    ["/File3", None, File3, None, True,
+     ["wrong_file", wrong_file, "http", None, 35],
+     ["404", None, "http", None, 40],
+     ["File3_lowPref", File3_lowPref, "http", None, 25],
+     ["File3", File3, "http", None, 30]],
+    ["/File4", None, File4, None, True,
+     ["wrong_file", wrong_file, "http", None, 35],
+     ["404", None, "http", None, 40],
+     ["File4_lowPref", File4_lowPref, "http", None, 25],
+     ["File4", File4, "http", None, 30]],
+    ["File5", XmlName + ".#2", File5, None, True,
+     ["wrong_file", wrong_file, "http", None, 35],
+     ["404", None, "http", None, 40],
+     ["File5_lowPref", File5_lowPref, "http", None, 25],
+     ["File5", File5, "http", None, 30]],
+)
+
+Meta.print_meta ()
+
+err = Meta.http_test (
+    "--input-metalink " + XmlName, 0
+)
+
+exit (err)

testenv/Test-metalink-xml-absprefix-trust.py

@@ -0,0 +1,102 @@
+#!/usr/bin/env python3
+
+from sys import exit
+from misc.metalinkv3_xml import Metalinkv3_XML
+
+"""
+    This is to test Metalink/XML absolute directory prefix support in Wget.
+
+    With --trust-server-names, trust the metalink:file names.
+
+    Without --trust-server-names, don't trust the metalink:file names:
+    use the basename of --input-metalink, and add a sequential number
+    (e.g. .#1, .#2, etc.).
+
+    Strip the directory from unsafe paths.
+"""
+
+############# File Definitions ###############################################
+wrong_file = "Ouch!"
+
+File1 = "Would you like some Tea?"
+File1_lowPref = "Do not take this"
+
+File2 = "This is gonna be good"
+File2_lowPref = "Not this one too"
+
+File3 = "A little more, please"
+File3_lowPref = "That's just too much"
+
+File4 = "Maybe a biscuit?"
+File4_lowPref = "No, thanks"
+
+File5 = "More Tea...?"
+File5_lowPref = "I have to go..."
+
+############# Metalink/XML ###################################################
+Meta = Metalinkv3_XML()
+
+# file_name: metalink:file "name" field
+# save_name: metalink:file save name, if None the file is rejected
+# content  : metalink:file content
+#
+# size:
+#   True     auto-compute size
+#   None     no <size></size>
+#    any     use this size
+#
+# hash_sha256:
+#   False    no <verification></verification>
+#   True     auto-compute sha256
+#   None     no <hash></hash>
+#    any     use this hash
+#
+# srv_file   : metalink:url server file
+# srv_content: metalink:url server file content, if None the file doesn't exist
+# utype      : metalink:url type
+# location   : metalink:url location (default 'no location field')
+# preference : metalink:url preference (default 999999)
+
+XmlName = "test.metalink"
+
+Meta.xml (
+    # Metalink/XML file name
+    XmlName,
+    # file_name, save_name, content, size, hash_sha256
+    ["subdir/File1", "File1", File1, None, True,
+     # srv_file, srv_content, utype, location, preference
+     ["wrong_file", wrong_file, "http", None, 35],
+     ["404", None, "http", None, 40],
+     ["File1_lowPref", File1_lowPref, "http", None, 25],
+     ["File1", File1, "http", None, 30]],
+    ["/subdir/File2", None, File2, None, True,
+     ["wrong_file", wrong_file, "http", None, 35],
+     ["404", None, "http", None, 40],
+     ["File2_lowPref", File2_lowPref, "http", None, 25],
+     ["File2", File2, "http", None, 30]],
+    ["~/subdir/File3", None, File3, None, True,
+     ["wrong_file", wrong_file, "http", None, 35],
+     ["404", None, "http", None, 40],
+     ["File3_lowPref", File3_lowPref, "http", None, 25],
+     ["File3", File3, "http", None, 30]],
+    ["../subdir/File4", None, File4, None, True,
+     ["wrong_file", wrong_file, "http", None, 35],
+     ["404", None, "http", None, 40],
+     ["File4_lowPref", File4_lowPref, "http", None, 25],
+     ["File4", File4, "http", None, 30]],
+    ["subdir/File5", "File5", File5, None, True,
+     ["wrong_file", wrong_file, "http", None, 35],
+     ["404", None, "http", None, 40],
+     ["File5_lowPref", File5_lowPref, "http", None, 25],
+     ["File5", File5, "http", None, 30]],
+)
+
+Meta.print_meta ()
+
+err = Meta.http_test (
+    "--trust-server-names " + \
+    "--directory-prefix /dir " + \
+    "--input-metalink " + XmlName, 0
+)
+
+exit (err)

testenv/Test-metalink-xml-absprefix.py

@@ -0,0 +1,101 @@
+#!/usr/bin/env python3
+
+from sys import exit
+from misc.metalinkv3_xml import Metalinkv3_XML
+
+"""
+    This is to test Metalink/XML absolute directory prefix support in Wget.
+
+    With --trust-server-names, trust the metalink:file names.
+
+    Without --trust-server-names, don't trust the metalink:file names:
+    use the basename of --input-metalink, and add a sequential number
+    (e.g. .#1, .#2, etc.).
+
+    Strip the directory from unsafe paths.
+"""
+
+############# File Definitions ###############################################
+wrong_file = "Ouch!"
+
+File1 = "Would you like some Tea?"
+File1_lowPref = "Do not take this"
+
+File2 = "This is gonna be good"
+File2_lowPref = "Not this one too"
+
+File3 = "A little more, please"
+File3_lowPref = "That's just too much"
+
+File4 = "Maybe a biscuit?"
+File4_lowPref = "No, thanks"
+
+File5 = "More Tea...?"
+File5_lowPref = "I have to go..."
+
+############# Metalink/XML ###################################################
+Meta = Metalinkv3_XML()
+
+# file_name: metalink:file "name" field
+# save_name: metalink:file save name, if None the file is rejected
+# content  : metalink:file content
+#
+# size:
+#   True     auto-compute size
+#   None     no <size></size>
+#    any     use this size
+#
+# hash_sha256:
+#   False    no <verification></verification>
+#   True     auto-compute sha256
+#   None     no <hash></hash>
+#    any     use this hash
+#
+# srv_file   : metalink:url server file
+# srv_content: metalink:url server file content, if None the file doesn't exist
+# utype      : metalink:url type
+# location   : metalink:url location (default 'no location field')
+# preference : metalink:url preference (default 999999)
+
+XmlName = "test.metalink"
+
+Meta.xml (
+    # Metalink/XML file name
+    XmlName,
+    # file_name, save_name, content, size, hash_sha256
+    ["subdir/File1", XmlName + ".#1", File1, None, True,
+     # srv_file, srv_content, utype, location, preference
+     ["wrong_file", wrong_file, "http", None, 35],
+     ["404", None, "http", None, 40],
+     ["File1_lowPref", File1_lowPref, "http", None, 25],
+     ["File1", File1, "http", None, 30]],
+    ["/subdir/File2", None, File2, None, True,
+     ["wrong_file", wrong_file, "http", None, 35],
+     ["404", None, "http", None, 40],
+     ["File2_lowPref", File2_lowPref, "http", None, 25],
+     ["File2", File2, "http", None, 30]],
+    ["~/subdir/File3", None, File3, None, True,
+     ["wrong_file", wrong_file, "http", None, 35],
+     ["404", None, "http", None, 40],
+     ["File3_lowPref", File3_lowPref, "http", None, 25],
+     ["File3", File3, "http", None, 30]],
+    ["../subdir/File4", None, File4, None, True,
+     ["wrong_file", wrong_file, "http", None, 35],
+     ["404", None, "http", None, 40],
+     ["File4_lowPref", File4_lowPref, "http", None, 25],
+     ["File4", File4, "http", None, 30]],
+    ["subdir/File5", XmlName + ".#2", File5, None, True,
+     ["wrong_file", wrong_file, "http", None, 35],
+     ["404", None, "http", None, 40],
+     ["File5_lowPref", File5_lowPref, "http", None, 25],
+     ["File5", File5, "http", None, 30]],
+)
+
+Meta.print_meta ()
+
+err = Meta.http_test (
+    "--directory-prefix /dir " + \
+    "--input-metalink " + XmlName, 0
+)
+
+exit (err)

testenv/Test-metalink-xml-continue.py

@@ -0,0 +1,107 @@
+#!/usr/bin/env python3
+
+from sys import exit
+from misc.metalinkv3_xml import Metalinkv3_XML
+
+"""
+    This is to test Metalink/XML --continue support in Wget.
+
+    With --trust-server-names, trust the metalink:file names.
+
+    Without --trust-server-names, don't trust the metalink:file names:
+    use the basename of --input-metalink, and add a sequential number
+    (e.g. .#1, .#2, etc.).
+
+    Strip the directory from unsafe paths.
+"""
+
+############# File Definitions ###############################################
+wrong_file = "Ouch!"
+
+File1 = "Would you like some Tea?"
+File1_lowPref = "Do not take this"
+
+File2 = "This is gonna be good"
+File2_lowPref = "Not this one too"
+
+File3 = "A little more, please"
+File3_lowPref = "That's just too much"
+
+File4 = "Maybe a biscuit?"
+File4_lowPref = "No, thanks"
+
+File5 = "More Tea...?"
+File5_lowPref = "I have to go..."
+
+############# Metalink/XML ###################################################
+Meta = Metalinkv3_XML()
+
+# file_name: metalink:file "name" field
+# save_name: metalink:file save name, if None the file is rejected
+# content  : metalink:file content
+#
+# size:
+#   True     auto-compute size
+#   None     no <size></size>
+#    any     use this size
+#
+# hash_sha256:
+#   False    no <verification></verification>
+#   True     auto-compute sha256
+#   None     no <hash></hash>
+#    any     use this hash
+#
+# srv_file   : metalink:url server file
+# srv_content: metalink:url server file content, if None the file doesn't exist
+# utype      : metalink:url type (http, ftp, etc.)
+# location   : metalink:url location (default 'no location field')
+# preference : metalink:url preference (default 999999)
+
+XmlName = "test.metalink"
+
+Meta.add_LocalFiles (
+    [XmlName + ".#1", File1[0:14]], # File1 to continue from partial file
+    [XmlName + ".#2", File2],       # File2 to keep, different from server
+    [XmlName + ".#3", File3],       # File3 to keep, already fully retrieved
+    [XmlName + ".#4", File4],       # File4 to keep, different from server
+)
+
+Meta.add_ExpectedFiles (
+    [XmlName + ".#2", File2],       # File2 to keep, different from server
+    [XmlName + ".#3", File3],       # File3 to keep, already fully retrieved
+    [XmlName + ".#4", File4],       # File4 to keep, different from server
+)
+
+Meta.print_meta ()
+
+Meta.xml (
+    # Metalink/XML file name
+    XmlName,
+    # file_name, save_name, content, size, hash_sha256
+    ["File1", XmlName + ".#1", File1, None, True,
+     # srv_file, srv_content, utype, location, preference
+     ["File1_lowPref", File1_lowPref, "http", None, 25],
+     ["File1", File1, "http", None, 30]],
+    ["File2", XmlName + ".#2", wrong_file, None, True,
+     ["wrong_file", wrong_file, "http", None, 35],
+     ["404", None, "http", None, 40],
+     ["File2_lowPref", File2_lowPref, "http", None, 25],
+     ["File2", File2, "http", None, 30]],
+    ["File3", XmlName + ".#3", File3, None, True,
+     ["File3_lowPref", File3_lowPref, "http", None, 25],
+     ["File3", File3, "http", None, 30]],
+    ["File4", XmlName + ".#4", File4_lowPref, None, True,
+     ["wrong_file", wrong_file, "http", None, 35]],
+    ["File5", XmlName + ".#5", File5, None, True,
+     ["File5_lowPref", File5_lowPref, "http", None, 25],
+     ["File5", File5, "http", None, 30]],
+)
+
+Meta.print_meta ()
+
+err = Meta.http_test (
+    "--continue " + \
+    "--input-metalink " + XmlName, 1
+)
+
+exit (err)

testenv/Test-metalink-xml-emptyprefix-trust.py

@@ -0,0 +1,102 @@
+#!/usr/bin/env python3
+
+from sys import exit
+from misc.metalinkv3_xml import Metalinkv3_XML
+
+"""
+    This is to test Metalink/XML with an empty directory prefix.
+
+    With --trust-server-names, trust the metalink:file names.
+
+    Without --trust-server-names, don't trust the metalink:file names:
+    use the basename of --input-metalink, and add a sequential number
+    (e.g. .#1, .#2, etc.).
+
+    Strip the directory from unsafe paths.
+"""
+
+############# File Definitions ###############################################
+wrong_file = "Ouch!"
+
+File1 = "Would you like some Tea?"
+File1_lowPref = "Do not take this"
+
+File2 = "This is gonna be good"
+File2_lowPref = "Not this one too"
+
+File3 = "A little more, please"
+File3_lowPref = "That's just too much"
+
+File4 = "Maybe a biscuit?"
+File4_lowPref = "No, thanks"
+
+File5 = "More Tea...?"
+File5_lowPref = "I have to go..."
+
+############# Metalink/XML ###################################################
+Meta = Metalinkv3_XML()
+
+# file_name: metalink:file "name" field
+# save_name: metalink:file save name, if None the file is rejected
+# content  : metalink:file content
+#
+# size:
+#   True     auto-compute size
+#   None     no <size></size>
+#    any     use this size
+#
+# hash_sha256:
+#   False    no <verification></verification>
+#   True     auto-compute sha256
+#   None     no <hash></hash>
+#    any     use this hash
+#
+# srv_file   : metalink:url server file
+# srv_content: metalink:url server file content, if None the file doesn't exist
+# utype      : metalink:url type
+# location   : metalink:url location (default 'no location field')
+# preference : metalink:url preference (default 999999)
+
+XmlName = "test.metalink"
+
+Meta.xml (
+    # Metalink/XML file name
+    XmlName,
+    # file_name, save_name, content, size, hash_sha256
+    ["subdir/File1", "subdir/File1", File1, None, True,
+     # srv_file, srv_content, utype, location, preference
+     ["wrong_file", wrong_file, "http", None, 35],
+     ["404", None, "http", None, 40],
+     ["File1_lowPref", File1_lowPref, "http", None, 25],
+     ["File1", File1, "http", None, 30]],
+    ["/subdir/File2", None, File2, None, True,
+     ["wrong_file", wrong_file, "http", None, 35],
+     ["404", None, "http", None, 40],
+     ["File2_lowPref", File2_lowPref, "http", None, 25],
+     ["File2", File2, "http", None, 30]],
+    ["~/subdir/File3", None, File3, None, True,
+     ["wrong_file", wrong_file, "http", None, 35],
+     ["404", None, "http", None, 40],
+     ["File3_lowPref", File3_lowPref, "http", None, 25],
+     ["File3", File3, "http", None, 30]],
+    ["../subdir/File4", None, File4, None, True,
+     ["wrong_file", wrong_file, "http", None, 35],
+     ["404", None, "http", None, 40],
+     ["File4_lowPref", File4_lowPref, "http", None, 25],
+     ["File4", File4, "http", None, 30]],
+    ["subdir/File5", "subdir/File5", File5, None, True,
+     ["wrong_file", wrong_file, "http", None, 35],
+     ["404", None, "http", None, 40],
+     ["File5_lowPref", File5_lowPref, "http", None, 25],
+     ["File5", File5, "http", None, 30]],
+)
+
+Meta.print_meta ()
+
+err = Meta.http_test (
+    "--trust-server-names " + \
+    "--directory-prefix '' " + \
+    "--input-metalink " + XmlName, 0
+)
+
+exit (err)

testenv/Test-metalink-xml-homepath-trust.py

@@ -0,0 +1,102 @@
+#!/usr/bin/env python3
+
+from sys import exit
+from misc.metalinkv3_xml import Metalinkv3_XML
+
+"""
+    This is to test if Metalink/XML forbids the home path and names
+    beginning with the ~ (tilde) character.
+
+    With --trust-server-names, trust the metalink:file names.
+
+    Without --trust-server-names, don't trust the metalink:file names:
+    use the basename of --input-metalink, and add a sequential number
+    (e.g. .#1, .#2, etc.).
+
+    Strip the directory from unsafe paths.
+"""
+
+############# File Definitions ###############################################
+wrong_file = "Ouch!"
+
+File1 = "Would you like some Tea?"
+File1_lowPref = "Do not take this"
+
+File2 = "This is gonna be good"
+File2_lowPref = "Not this one too"
+
+File3 = "A little more, please"
+File3_lowPref = "That's just too much"
+
+File4 = "Maybe a biscuit?"
+File4_lowPref = "No, thanks"
+
+File5 = "More Tea...?"
+File5_lowPref = "I have to go..."
+
+############# Metalink/XML ###################################################
+Meta = Metalinkv3_XML()
+
+# file_name: metalink:file "name" field
+# save_name: metalink:file save name, if None the file is rejected
+# content  : metalink:file content
+#
+# size:
+#   True     auto-compute size
+#   None     no <size></size>
+#    any     use this size
+#
+# hash_sha256:
+#   False    no <verification></verification>
+#   True     auto-compute sha256
+#   None     no <hash></hash>
+#    any     use this hash
+#
+# srv_file   : metalink:url server file
+# srv_content: metalink:url server file content, if None the file doesn't exist
+# utype      : metalink:url type
+# location   : metalink:url location (default 'no location field')
+# preference : metalink:url preference (default 999999)
+
+XmlName = "test.metalink"
+
+Meta.xml (
+    # Metalink/XML file name
+    XmlName,
+    # file_name, save_name, content, size, hash_sha256
+    ["~File1", None, File1, None, True,
+     # srv_file, srv_content, utype, location, preference
+     ["wrong_file", wrong_file, "http", None, 35],
+     ["404", None, "http", None, 40],
+     ["File1_lowPref", File1_lowPref, "http", None, 25],
+     ["File1", File1, "http", None, 30]],
+    ["~/File2", None, File2, None, True,
+     ["wrong_file", wrong_file, "http", None, 35],
+     ["404", None, "http", None, 40],
+     ["File2_lowPref", File2_lowPref, "http", None, 25],
+     ["File2", File2, "http", None, 30]],
+    ["dir/~File3", None, File3, None, True,
+     ["wrong_file", wrong_file, "http", None, 35],
+     ["404", None, "http", None, 40],
+     ["File3_lowPref", File3_lowPref, "http", None, 25],
+     ["File3", File3, "http", None, 30]],
+    ["dir/File4~", "dir/File4~", File4, None, True,
+     ["wrong_file", wrong_file, "http", None, 35],
+     ["404", None, "http", None, 40],
+     ["File4_lowPref", File4_lowPref, "http", None, 25],
+     ["File4", File4, "http", None, 30]],
+    ["dir/~/File5", "dir/~/File5", File5, None, True,
+     ["wrong_file", wrong_file, "http", None, 35],
+     ["404", None, "http", None, 40],
+     ["File5_lowPref", File5_lowPref, "http", None, 25],
+     ["File5", File5, "http", None, 30]],
+)
+
+Meta.print_meta ()
+
+err = Meta.http_test (
+    "--trust-server-names " + \
+    "--input-metalink " + XmlName, 0
+)
+
+exit (err)

testenv/Test-metalink-xml-homepath.py

@@ -0,0 +1,101 @@
+#!/usr/bin/env python3
+
+from sys import exit
+from misc.metalinkv3_xml import Metalinkv3_XML
+
+"""
+    This is to test if Metalink/XML forbids the home path and names
+    beginning with the ~ (tilde) character.
+
+    With --trust-server-names, trust the metalink:file names.
+
+    Without --trust-server-names, don't trust the metalink:file names:
+    use the basename of --input-metalink, and add a sequential number
+    (e.g. .#1, .#2, etc.).
+
+    Strip the directory from unsafe paths.
+"""
+
+############# File Definitions ###############################################
+wrong_file = "Ouch!"
+
+File1 = "Would you like some Tea?"
+File1_lowPref = "Do not take this"
+
+File2 = "This is gonna be good"
+File2_lowPref = "Not this one too"
+
+File3 = "A little more, please"
+File3_lowPref = "That's just too much"
+
+File4 = "Maybe a biscuit?"
+File4_lowPref = "No, thanks"
+
+File5 = "More Tea...?"
+File5_lowPref = "I have to go..."
+
+############# Metalink/XML ###################################################
+Meta = Metalinkv3_XML()
+
+# file_name: metalink:file "name" field
+# save_name: metalink:file save name, if None the file is rejected
+# content  : metalink:file content
+#
+# size:
+#   True     auto-compute size
+#   None     no <size></size>
+#    any     use this size
+#
+# hash_sha256:
+#   False    no <verification></verification>
+#   True     auto-compute sha256
+#   None     no <hash></hash>
+#    any     use this hash
+#
+# srv_file   : metalink:url server file
+# srv_content: metalink:url server file content, if None the file doesn't exist
+# utype      : metalink:url type
+# location   : metalink:url location (default 'no location field')
+# preference : metalink:url preference (default 999999)
+
+XmlName = "test.metalink"
+
+Meta.xml (
+    # Metalink/XML file name
+    XmlName,
+    # file_name, save_name, content, size, hash_sha256
+    ["~File1", None, File1, None, True,
+     # srv_file, srv_content, utype, location, preference
+     ["wrong_file", wrong_file, "http", None, 35],
+     ["404", None, "http", None, 40],
+     ["File1_lowPref", File1_lowPref, "http", None, 25],
+     ["File1", File1, "http", None, 30]],
+    ["~/File2", None, File2, None, True,
+     ["wrong_file", wrong_file, "http", None, 35],
+     ["404", None, "http", None, 40],
+     ["File2_lowPref", File2_lowPref, "http", None, 25],
+     ["File2", File2, "http", None, 30]],
+    ["dir/~File3", None, File3, None, True,
+     ["wrong_file", wrong_file, "http", None, 35],
+     ["404", None, "http", None, 40],
+     ["File3_lowPref", File3_lowPref, "http", None, 25],
+     ["File3", File3, "http", None, 30]],
+    ["dir/File4~", XmlName + ".#1", File4, None, True,
+     ["wrong_file", wrong_file, "http", None, 35],
+     ["404", None, "http", None, 40],
+     ["File4_lowPref", File4_lowPref, "http", None, 25],
+     ["File4", File4, "http", None, 30]],
+    ["dir/~/File5", XmlName + ".#2", File5, None, True,
+     ["wrong_file", wrong_file, "http", None, 35],
+     ["404", None, "http", None, 40],
+     ["File5_lowPref", File5_lowPref, "http", None, 25],
+     ["File5", File5, "http", None, 30]],
+)
+
+Meta.print_meta ()
+
+err = Meta.http_test (
+    "--input-metalink " + XmlName, 0
+)
+
+exit (err)

testenv/Test-metalink-xml-homeprefix-trust.py

@@ -0,0 +1,102 @@
+#!/usr/bin/env python3
+
+from sys import exit
+from misc.metalinkv3_xml import Metalinkv3_XML
+
+"""
+    This is to test Metalink/XML home directory prefix support in Wget.
+
+    With --trust-server-names, trust the metalink:file names.
+
+    Without --trust-server-names, don't trust the metalink:file names:
+    use the basename of --input-metalink, and add a sequential number
+    (e.g. .#1, .#2, etc.).
+
+    Strip the directory from unsafe paths.
+"""
+
+############# File Definitions ###############################################
+wrong_file = "Ouch!"
+
+File1 = "Would you like some Tea?"
+File1_lowPref = "Do not take this"
+
+File2 = "This is gonna be good"
+File2_lowPref = "Not this one too"
+
+File3 = "A little more, please"
+File3_lowPref = "That's just too much"
+
+File4 = "Maybe a biscuit?"
+File4_lowPref = "No, thanks"
+
+File5 = "More Tea...?"
+File5_lowPref = "I have to go..."
+
+############# Metalink/XML ###################################################
+Meta = Metalinkv3_XML()
+
+# file_name: metalink:file "name" field
+# save_name: metalink:file save name, if None the file is rejected
+# content  : metalink:file content
+#
+# size:
+#   True     auto-compute size
+#   None     no <size></size>
+#    any     use this size
+#
+# hash_sha256:
+#   False    no <verification></verification>
+#   True     auto-compute sha256
+#   None     no <hash></hash>
+#    any     use this hash
+#
+# srv_file   : metalink:url server file
+# srv_content: metalink:url server file content, if None the file doesn't exist
+# utype      : metalink:url type
+# location   : metalink:url location (default 'no location field')
+# preference : metalink:url preference (default 999999)
+
+XmlName = "test.metalink"
+
+Meta.xml (
+    # Metalink/XML file name
+    XmlName,
+    # file_name, save_name, content, size, hash_sha256
+    ["subdir/File1", "File1", File1, None, True,
+     # srv_file, srv_content, utype, location, preference
+     ["wrong_file", wrong_file, "http", None, 35],
+     ["404", None, "http", None, 40],
+     ["File1_lowPref", File1_lowPref, "http", None, 25],
+     ["File1", File1, "http", None, 30]],
+    ["/subdir/File2", None, File2, None, True,
+     ["wrong_file", wrong_file, "http", None, 35],
+     ["404", None, "http", None, 40],
+     ["File2_lowPref", File2_lowPref, "http", None, 25],
+     ["File2", File2, "http", None, 30]],
+    ["~/subdir/File3", None, File3, None, True,
+     ["wrong_file", wrong_file, "http", None, 35],
+     ["404", None, "http", None, 40],
+     ["File3_lowPref", File3_lowPref, "http", None, 25],
+     ["File3", File3, "http", None, 30]],
+    ["../subdir/File4", None, File4, None, True,
+     ["wrong_file", wrong_file, "http", None, 35],
+     ["404", None, "http", None, 40],
+     ["File4_lowPref", File4_lowPref, "http", None, 25],
+     ["File4", File4, "http", None, 30]],
+    ["subdir/File5", "File5", File5, None, True,
+     ["wrong_file", wrong_file, "http", None, 35],
+     ["404", None, "http", None, 40],
+     ["File5_lowPref", File5_lowPref, "http", None, 25],
+     ["File5", File5, "http", None, 30]],
+)
+
+Meta.print_meta ()
+
+err = Meta.http_test (
+    "--trust-server-names " + \
+    "--directory-prefix ~/dir " + \
+    "--input-metalink " + XmlName, 0
+)
+
+exit (err)

testenv/Test-metalink-xml-homeprefix.py

@@ -0,0 +1,101 @@
+#!/usr/bin/env python3
+
+from sys import exit
+from misc.metalinkv3_xml import Metalinkv3_XML
+
+"""
+    This is to test Metalink/XML home directory prefix support in Wget.
+
+    With --trust-server-names, trust the metalink:file names.
+
+    Without --trust-server-names, don't trust the metalink:file names:
+    use the basename of --input-metalink, and add a sequential number
+    (e.g. .#1, .#2, etc.).
+
+    Strip the directory from unsafe paths.
+"""
+
+############# File Definitions ###############################################
+wrong_file = "Ouch!"
+
+File1 = "Would you like some Tea?"
+File1_lowPref = "Do not take this"
+
+File2 = "This is gonna be good"
+File2_lowPref = "Not this one too"
+
+File3 = "A little more, please"
+File3_lowPref = "That's just too much"
+
+File4 = "Maybe a biscuit?"
+File4_lowPref = "No, thanks"
+
+File5 = "More Tea...?"
+File5_lowPref = "I have to go..."
+
+############# Metalink/XML ###################################################
+Meta = Metalinkv3_XML()
+
+# file_name: metalink:file "name" field
+# save_name: metalink:file save name, if None the file is rejected
+# content  : metalink:file content
+#
+# size:
+#   True     auto-compute size
+#   None     no <size></size>
+#    any     use this size
+#
+# hash_sha256:
+#   False    no <verification></verification>
+#   True     auto-compute sha256
+#   None     no <hash></hash>
+#    any     use this hash
+#
+# srv_file   : metalink:url server file
+# srv_content: metalink:url server file content, if None the file doesn't exist
+# utype      : metalink:url type
+# location   : metalink:url location (default 'no location field')
+# preference : metalink:url preference (default 999999)
+
+XmlName = "test.metalink"
+
+Meta.xml (
+    # Metalink/XML file name
+    XmlName,
+    # file_name, save_name, content, size, hash_sha256
+    ["subdir/File1", XmlName + ".#1", File1, None, True,
+     # srv_file, srv_content, utype, location, preference
+     ["wrong_file", wrong_file, "http", None, 35],
+     ["404", None, "http", None, 40],
+     ["File1_lowPref", File1_lowPref, "http", None, 25],
+     ["File1", File1, "http", None, 30]],
+    ["/subdir/File2", None, File2, None, True,
+     ["wrong_file", wrong_file, "http", None, 35],
+     ["404", None, "http", None, 40],
+     ["File2_lowPref", File2_lowPref, "http", None, 25],
+     ["File2", File2, "http", None, 30]],
+    ["~/subdir/File3", None, File3, None, True,
+     ["wrong_file", wrong_file, "http", None, 35],
+     ["404", None, "http", None, 40],
+     ["File3_lowPref", File3_lowPref, "http", None, 25],
+     ["File3", File3, "http", None, 30]],
+    ["../subdir/File4", None, File4, None, True,
+     ["wrong_file", wrong_file, "http", None, 35],
+     ["404", None, "http", None, 40],
+     ["File4_lowPref", File4_lowPref, "http", None, 25],
+     ["File4", File4, "http", None, 30]],
+    ["subdir/File5", XmlName + ".#2", File5, None, True,
+     ["wrong_file", wrong_file, "http", None, 35],
+     ["404", None, "http", None, 40],
+     ["File5_lowPref", File5_lowPref, "http", None, 25],
+     ["File5", File5, "http", None, 30]],
+)
+
+Meta.print_meta ()
+
+err = Meta.http_test (
+    "--directory-prefix ~/dir " + \
+    "--input-metalink " + XmlName, 0
+)
+
+exit (err)

testenv/Test-metalink-xml-nohash.py

@@ -0,0 +1,100 @@
+#!/usr/bin/env python3
+
+from sys import exit
+from misc.metalinkv3_xml import Metalinkv3_XML
+
+"""
+    This is to test if Metalink/XML with no hashes generates a SIGSEGV.
+
+    With --trust-server-names, trust the metalink:file names.
+
+    Without --trust-server-names, don't trust the metalink:file names:
+    use the basename of --input-metalink, and add a sequential number
+    (e.g. .#1, .#2, etc.).
+
+    Strip the directory from unsafe paths.
+"""
+
+############# File Definitions ###############################################
+wrong_file = "Ouch!"
+
+File1 = "Would you like some Tea?"
+File1_lowPref = "Do not take this"
+
+File2 = "This is gonna be good"
+File2_lowPref = "Not this one too"
+
+File3 = "A little more, please"
+File3_lowPref = "That's just too much"
+
+File4 = "Maybe a biscuit?"
+File4_lowPref = "No, thanks"
+
+File5 = "More Tea...?"
+File5_lowPref = "I have to go..."
+
+############# Metalink/XML ###################################################
+Meta = Metalinkv3_XML()
+
+# file_name: metalink:file "name" field
+# save_name: metalink:file save name, if None the file is rejected
+# content  : metalink:file content
+#
+# size:
+#   True     auto-compute size
+#   None     no <size></size>
+#    any     use this size
+#
+# hash_sha256:
+#   False    no <verification></verification>
+#   True     auto-compute sha256
+#   None     no <hash></hash>
+#    any     use this hash
+#
+# srv_file   : metalink:url server file
+# srv_content: metalink:url server file content, if None the file doesn't exist
+# utype      : metalink:url type (http, ftp, etc.)
+# location   : metalink:url location (default 'no location field')
+# preference : metalink:url preference (default 999999)
+
+XmlName = "test.metalink"
+
+Meta.xml (
+    # Metalink/XML file name
+    XmlName,
+    # file_name, save_name, content, size, hash_sha256
+    ["File1", None, File1, None, False,
+     # srv_file, srv_content, utype, location, preference
+     ["wrong_file", wrong_file, "http", None, 35],
+     ["404", None, "http", None, 40],
+     ["File1_lowPref", File1_lowPref, "http", None, 25],
+     ["File1", File1, "http", None, 30]],
+    ["File2", XmlName + ".#2", File2, None, True,
+     ["wrong_file", wrong_file, "http", None, 35],
+     ["404", None, "http", None, 40],
+     ["File2_lowPref", File2_lowPref, "http", None, 25],
+     ["File2", File2, "http", None, 30]],
+    ["File3", None, File3, None, None,
+     ["wrong_file", wrong_file, "http", None, 35],
+     ["404", None, "http", None, 40],
+     ["File3_lowPref", File3_lowPref, "http", None, 25],
+     ["File3", File3, "http", None, 30]],
+    ["File4", XmlName + ".#4", File4, None, True,
+     ["wrong_file", wrong_file, "http", None, 35],
+     ["404", None, "http", None, 40],
+     ["File4_lowPref", File4_lowPref, "http", None, 25],
+     ["File4", File4, "http", None, 30]],
+    ["File5", None, File5, None, "",
+     ["wrong_file", wrong_file, "http", None, 35],
+     ["404", None, "http", None, 40],
+     ["File5_lowPref", File5_lowPref, "http", None, 25],
+     ["File5", File5, "http", None, 30]],
+)
+
+Meta.print_meta ()
+
+err = Meta.http_test (
+    "--input-metalink " + XmlName, 1
+)
+
+exit (err)

testenv/Test-metalink-xml-nourls.py

@@ -0,0 +1,100 @@
+#!/usr/bin/env python3
+
+from sys import exit
+from misc.metalinkv3_xml import Metalinkv3_XML
+
+"""
+    This is to test Metalink/XML with unknown url types.
+
+    With --trust-server-names, trust the metalink:file names.
+
+    Without --trust-server-names, don't trust the metalink:file names:
+    use the basename of --input-metalink, and add a sequential number
+    (e.g. .#1, .#2, etc.).
+
+    Strip the directory from unsafe paths.
+"""
+
+############# File Definitions ###############################################
+wrong_file = "Ouch!"
+
+File1 = "Would you like some Tea?"
+File1_lowPref = "Do not take this"
+
+File2 = "This is gonna be good"
+File2_lowPref = "Not this one too"
+
+File3 = "A little more, please"
+File3_lowPref = "That's just too much"
+
+File4 = "Maybe a biscuit?"
+File4_lowPref = "No, thanks"
+
+File5 = "More Tea...?"
+File5_lowPref = "I have to go..."
+
+############# Metalink/XML ###################################################
+Meta = Metalinkv3_XML()
+
+# file_name: metalink:file "name" field
+# save_name: metalink:file save name, if None the file is rejected
+# content  : metalink:file content
+#
+# size:
+#   True     auto-compute size
+#   None     no <size></size>
+#    any     use this size
+#
+# hash_sha256:
+#   False    no <verification></verification>
+#   True     auto-compute sha256
+#   None     no <hash></hash>
+#    any     use this hash
+#
+# srv_file   : metalink:url server file
+# srv_content: metalink:url server file content, if None the file doesn't exist
+# utype      : metalink:url type (http, ftp, etc.)
+# location   : metalink:url location (default 'no location field')
+# preference : metalink:url preference (default 999999)
+
+XmlName = "test.metalink"
+
+Meta.xml (
+    # Metalink/XML file name
+    XmlName,
+    # file_name, save_name, content, size, hash_sha256
+    ["File1", XmlName + ".#1", File1, None, True,
+     # srv_file, srv_content, utype, location, preference
+     ["wrong_file", wrong_file, "http", None, 35],
+     ["404", None, "http", None, 40],
+     ["File1_lowPref", File1_lowPref, "http", None, 25],
+     ["File1", File1, "http", None, 30]],
+    ["File2", None, File2, None, True,
+     ["wrong_file", wrong_file, "unknown", None, 35],
+     ["404", None, "unknown", None, 40],
+     ["File2_lowPref", File2_lowPref, "unknown", None, 25],
+     ["File2", File2, "unknown", None, 30]],
+    ["File3", XmlName + ".#3", File3, None, True,
+     ["wrong_file", wrong_file, "http", None, 35],
+     ["404", None, "http", None, 40],
+     ["File3_lowPref", File3_lowPref, "http", None, 25],
+     ["File3", File3, "http", None, 30]],
+    ["File4", XmlName + ".#4", File4, None, True,
+     ["wrong_file", wrong_file, "unknown", None, 35],
+     ["404", None, "unknown", None, 40],
+     ["File4_lowPref", File4_lowPref, "unknown", None, 25],
+     ["File4", File4, "http", None, 30]],
+    ["File5", XmlName + ".#5", File5, None, True,
+     ["wrong_file", wrong_file, "http", None, 35],
+     ["404", None, "http", None, 40],
+     ["File5_lowPref", File5_lowPref, "http", None, 25],
+     ["File5", File5, "http", None, 30]],
+)
+
+Meta.print_meta ()
+
+err = Meta.http_test (
+    "--input-metalink " + XmlName, 1
+)
+
+exit (err)

testenv/Test-metalink-xml-prefix-trust.py

@@ -0,0 +1,102 @@
+#!/usr/bin/env python3
+
+from sys import exit
+from misc.metalinkv3_xml import Metalinkv3_XML
+
+"""
+    This is to test Metalink/XML directory prefix support in Wget.
+
+    With --trust-server-names, trust the metalink:file names.
+
+    Without --trust-server-names, don't trust the metalink:file names:
+    use the basename of --input-metalink, and add a sequential number
+    (e.g. .#1, .#2, etc.).
+
+    Strip the directory from unsafe paths.
+"""
+
+############# File Definitions ###############################################
+wrong_file = "Ouch!"
+
+File1 = "Would you like some Tea?"
+File1_lowPref = "Do not take this"
+
+File2 = "This is gonna be good"
+File2_lowPref = "Not this one too"
+
+File3 = "A little more, please"
+File3_lowPref = "That's just too much"
+
+File4 = "Maybe a biscuit?"
+File4_lowPref = "No, thanks"
+
+File5 = "More Tea...?"
+File5_lowPref = "I have to go..."
+
+############# Metalink/XML ###################################################
+Meta = Metalinkv3_XML()
+
+# file_name: metalink:file "name" field
+# save_name: metalink:file save name, if None the file is rejected
+# content  : metalink:file content
+#
+# size:
+#   True     auto-compute size
+#   None     no <size></size>
+#    any     use this size
+#
+# hash_sha256:
+#   False    no <verification></verification>
+#   True     auto-compute sha256
+#   None     no <hash></hash>
+#    any     use this hash
+#
+# srv_file   : metalink:url server file
+# srv_content: metalink:url server file content, if None the file doesn't exist
+# utype      : metalink:url type
+# location   : metalink:url location (default 'no location field')
+# preference : metalink:url preference (default 999999)
+
+XmlName = "test.metalink"
+
+Meta.xml (
+    # Metalink/XML file name
+    XmlName,
+    # file_name, save_name, content, size, hash_sha256
+    ["subdir/File1", "dir/" + "subdir/File1", File1, None, True,
+     # srv_file, srv_content, utype, location, preference
+     ["wrong_file", wrong_file, "http", None, 35],
+     ["404", None, "http", None, 40],
+     ["File1_lowPref", File1_lowPref, "http", None, 25],
+     ["File1", File1, "http", None, 30]],
+    ["/subdir/File2", None, File2, None, True,
+     ["wrong_file", wrong_file, "http", None, 35],
+     ["404", None, "http", None, 40],
+     ["File2_lowPref", File2_lowPref, "http", None, 25],
+     ["File2", File2, "http", None, 30]],
+    ["~/subdir/File3", None, File3, None, True,
+     ["wrong_file", wrong_file, "http", None, 35],
+     ["404", None, "http", None, 40],
+     ["File3_lowPref", File3_lowPref, "http", None, 25],
+     ["File3", File3, "http", None, 30]],
+    ["../subdir/File4", None, File4, None, True,
+     ["wrong_file", wrong_file, "http", None, 35],
+     ["404", None, "http", None, 40],
+     ["File4_lowPref", File4_lowPref, "http", None, 25],
+     ["File4", File4, "http", None, 30]],
+    ["subdir/File5", "dir/" + "subdir/File5", File5, None, True,
+     ["wrong_file", wrong_file, "http", None, 35],
+     ["404", None, "http", None, 40],
+     ["File5_lowPref", File5_lowPref, "http", None, 25],
+     ["File5", File5, "http", None, 30]],
+)
+
+Meta.print_meta ()
+
+err = Meta.http_test (
+    "--trust-server-names " + \
+    "--directory-prefix dir " + \
+    "--input-metalink " + XmlName, 0
+)
+
+exit (err)

testenv/Test-metalink-xml-prefix.py

@@ -0,0 +1,101 @@
+#!/usr/bin/env python3
+
+from sys import exit
+from misc.metalinkv3_xml import Metalinkv3_XML
+
+"""
+    This is to test Metalink/XML directory prefix support in Wget.
+
+    With --trust-server-names, trust the metalink:file names.
+
+    Without --trust-server-names, don't trust the metalink:file names:
+    use the basename of --input-metalink, and add a sequential number
+    (e.g. .#1, .#2, etc.).
+
+    Strip the directory from unsafe paths.
+"""
+
+############# File Definitions ###############################################
+wrong_file = "Ouch!"
+
+File1 = "Would you like some Tea?"
+File1_lowPref = "Do not take this"
+
+File2 = "This is gonna be good"
+File2_lowPref = "Not this one too"
+
+File3 = "A little more, please"
+File3_lowPref = "That's just too much"
+
+File4 = "Maybe a biscuit?"
+File4_lowPref = "No, thanks"
+
+File5 = "More Tea...?"
+File5_lowPref = "I have to go..."
+
+############# Metalink/XML ###################################################
+Meta = Metalinkv3_XML()
+
+# file_name: metalink:file "name" field
+# save_name: metalink:file save name, if None the file is rejected
+# content  : metalink:file content
+#
+# size:
+#   True     auto-compute size
+#   None     no <size></size>
+#    any     use this size
+#
+# hash_sha256:
+#   False    no <verification></verification>
+#   True     auto-compute sha256
+#   None     no <hash></hash>
+#    any     use this hash
+#
+# srv_file   : metalink:url server file
+# srv_content: metalink:url server file content, if None the file doesn't exist
+# utype      : metalink:url type
+# location   : metalink:url location (default 'no location field')
+# preference : metalink:url preference (default 999999)
+
+XmlName = "test.metalink"
+
+Meta.xml (
+    # Metalink/XML file name
+    XmlName,
+    # file_name, save_name, content, size, hash_sha256
+    ["subdir/File1", "dir/" + XmlName + ".#1", File1, None, True,
+     # srv_file, srv_content, utype, location, preference
+     ["wrong_file", wrong_file, "http", None, 35],
+     ["404", None, "http", None, 40],
+     ["File1_lowPref", File1_lowPref, "http", None, 25],
+     ["File1", File1, "http", None, 30]],
+    ["/subdir/File2", None, File2, None, True,
+     ["wrong_file", wrong_file, "http", None, 35],
+     ["404", None, "http", None, 40],
+     ["File2_lowPref", File2_lowPref, "http", None, 25],
+     ["File2", File2, "http", None, 30]],
+    ["~/subdir/File3", None, File3, None, True,
+     ["wrong_file", wrong_file, "http", None, 35],
+     ["404", None, "http", None, 40],
+     ["File3_lowPref", File3_lowPref, "http", None, 25],
+     ["File3", File3, "http", None, 30]],
+    ["../subdir/File4", None, File4, None, True,
+     ["wrong_file", wrong_file, "http", None, 35],
+     ["404", None, "http", None, 40],
+     ["File4_lowPref", File4_lowPref, "http", None, 25],
+     ["File4", File4, "http", None, 30]],
+    ["subdir/File5", "dir/" + XmlName + ".#2", File5, None, True,
+     ["wrong_file", wrong_file, "http", None, 35],
+     ["404", None, "http", None, 40],
+     ["File5_lowPref", File5_lowPref, "http", None, 25],
+     ["File5", File5, "http", None, 30]],
+)
+
+Meta.print_meta ()
+
+err = Meta.http_test (
+    "--directory-prefix dir " + \
+    "--input-metalink " + XmlName, 0
+)
+
+exit (err)

testenv/Test-metalink-xml-relpath-trust.py

@@ -0,0 +1,101 @@
+#!/usr/bin/env python3
+
+from sys import exit
+from misc.metalinkv3_xml import Metalinkv3_XML
+
+"""
+    This is to test if Metalink/XML forbids relative paths.
+
+    With --trust-server-names, trust the metalink:file names.
+
+    Without --trust-server-names, don't trust the metalink:file names:
+    use the basename of --input-metalink, and add a sequential number
+    (e.g. .#1, .#2, etc.).
+
+    Strip the directory from unsafe paths.
+"""
+
+############# File Definitions ###############################################
+wrong_file = "Ouch!"
+
+File1 = "Would you like some Tea?"
+File1_lowPref = "Do not take this"
+
+File2 = "This is gonna be good"
+File2_lowPref = "Not this one too"
+
+File3 = "A little more, please"
+File3_lowPref = "That's just too much"
+
+File4 = "Maybe a biscuit?"
+File4_lowPref = "No, thanks"
+
+File5 = "More Tea...?"
+File5_lowPref = "I have to go..."
+
+############# Metalink/XML ###################################################
+Meta = Metalinkv3_XML()
+
+# file_name: metalink:file "name" field
+# save_name: metalink:file save name, if None the file is rejected
+# content  : metalink:file content
+#
+# size:
+#   True     auto-compute size
+#   None     no <size></size>
+#    any     use this size
+#
+# hash_sha256:
+#   False    no <verification></verification>
+#   True     auto-compute sha256
+#   None     no <hash></hash>
+#    any     use this hash
+#
+# srv_file   : metalink:url server file
+# srv_content: metalink:url server file content, if None the file doesn't exist
+# utype      : metalink:url type
+# location   : metalink:url location (default 'no location field')
+# preference : metalink:url preference (default 999999)
+
+XmlName = "test.metalink"
+
+Meta.xml (
+    # Metalink/XML file name
+    XmlName,
+    # file_name, save_name, content, size, hash_sha256
+    ["./File1", None, File1, None, True,
+     # srv_file, srv_content, utype, location, preference
+     ["wrong_file", wrong_file, "http", None, 35],
+     ["404", None, "http", None, 40],
+     ["File1_lowPref", File1_lowPref, "http", None, 25],
+     ["File1", File1, "http", None, 30]],
+    ["../File2", None, File2, None, True,
+     ["wrong_file", wrong_file, "http", None, 35],
+     ["404", None, "http", None, 40],
+     ["File2_lowPref", File2_lowPref, "http", None, 25],
+     ["File2", File2, "http", None, 30]],
+    ["dir/./File3", None, File3, None, True,
+     ["wrong_file", wrong_file, "http", None, 35],
+     ["404", None, "http", None, 40],
+     ["File3_lowPref", File3_lowPref, "http", None, 25],
+     ["File3", File3, "http", None, 30]],
+    ["dir/../File4", None, File4, None, True,
+     ["wrong_file", wrong_file, "http", None, 35],
+     ["404", None, "http", None, 40],
+     ["File4_lowPref", File4_lowPref, "http", None, 25],
+     ["File4", File4, "http", None, 30]],
+    ["File5", "File5", File5, None, True,
+     ["wrong_file", wrong_file, "http", None, 35],
+     ["404", None, "http", None, 40],
+     ["File5_lowPref", File5_lowPref, "http", None, 25],
+     ["File5", File5, "http", None, 30]],
+)
+
+Meta.print_meta ()
+
+err = Meta.http_test (
+    "--trust-server-names " + \
+    "--input-metalink " + XmlName, 0
+)
+
+exit (err)

testenv/Test-metalink-xml-relpath.py

@@ -0,0 +1,100 @@
+#!/usr/bin/env python3
+
+from sys import exit
+from misc.metalinkv3_xml import Metalinkv3_XML
+
+"""
+    This is to test if Metalink/XML forbids relative paths.
+
+    With --trust-server-names, trust the metalink:file names.
+
+    Without --trust-server-names, don't trust the metalink:file names:
+    use the basename of --input-metalink, and add a sequential number
+    (e.g. .#1, .#2, etc.).
+
+    Strip the directory from unsafe paths.
+"""
+
+############# File Definitions ###############################################
+wrong_file = "Ouch!"
+
+File1 = "Would you like some Tea?"
+File1_lowPref = "Do not take this"
+
+File2 = "This is gonna be good"
+File2_lowPref = "Not this one too"
+
+File3 = "A little more, please"
+File3_lowPref = "That's just too much"
+
+File4 = "Maybe a biscuit?"
+File4_lowPref = "No, thanks"
+
+File5 = "More Tea...?"
+File5_lowPref = "I have to go..."
+
+############# Metalink/XML ###################################################
+Meta = Metalinkv3_XML()
+
+# file_name: metalink:file "name" field
+# save_name: metalink:file save name, if None the file is rejected
+# content  : metalink:file content
+#
+# size:
+#   True     auto-compute size
+#   None     no <size></size>
+#    any     use this size
+#
+# hash_sha256:
+#   False    no <verification></verification>
+#   True     auto-compute sha256
+#   None     no <hash></hash>
+#    any     use this hash
+#
+# srv_file   : metalink:url server file
+# srv_content: metalink:url server file content, if None the file doesn't exist
+# utype      : metalink:url type
+# location   : metalink:url location (default 'no location field')
+# preference : metalink:url preference (default 999999)
+
+XmlName = "test.metalink"
+
+Meta.xml (
+    # Metalink/XML file name
+    XmlName,
+    # file_name, save_name, content, size, hash_sha256
+    ["./File1", None, File1, None, True,
+     # srv_file, srv_content, utype, location, preference
+     ["wrong_file", wrong_file, "http", None, 35],
+     ["404", None, "http", None, 40],
+     ["File1_lowPref", File1_lowPref, "http", None, 25],
+     ["File1", File1, "http", None, 30]],
+    ["../File2", None, File2, None, True,
+     ["wrong_file", wrong_file, "http", None, 35],
+     ["404", None, "http", None, 40],
+     ["File2_lowPref", File2_lowPref, "http", None, 25],
+     ["File2", File2, "http", None, 30]],
+    ["dir/./File3", None, File3, None, True,
+     ["wrong_file", wrong_file, "http", None, 35],
+     ["404", None, "http", None, 40],
+     ["File3_lowPref", File3_lowPref, "http", None, 25],
+     ["File3", File3, "http", None, 30]],
+    ["dir/../File4", None, File4, None, True,
+     ["wrong_file", wrong_file, "http", None, 35],
+     ["404", None, "http", None, 40],
+     ["File4_lowPref", File4_lowPref, "http", None, 25],
+     ["File4", File4, "http", None, 30]],
+    ["File5", XmlName + ".#1", File5, None, True,
+     ["wrong_file", wrong_file, "http", None, 35],
+     ["404", None, "http", None, 40],
+     ["File5_lowPref", File5_lowPref, "http", None, 25],
+     ["File5", File5, "http", None, 30]],
+)
+
+Meta.print_meta ()
+
+err = Meta.http_test (
+    "--input-metalink " + XmlName, 0
+)
+
+exit (err)

testenv/Test-metalink-xml-relprefix-trust.py

@@ -0,0 +1,102 @@
+#!/usr/bin/env python3
+
+from sys import exit
+from misc.metalinkv3_xml import Metalinkv3_XML
+
+"""
+    This is to test Metalink/XML relative directory prefix support in Wget.
+
+    With --trust-server-names, trust the metalink:file names.
+
+    Without --trust-server-names, don't trust the metalink:file names:
+    use the basename of --input-metalink, and add a sequential number
+    (e.g. .#1, .#2, etc.).
+
+    Strip the directory from unsafe paths.
+"""
+
+############# File Definitions ###############################################
+wrong_file = "Ouch!"
+
+File1 = "Would you like some Tea?"
+File1_lowPref = "Do not take this"
+
+File2 = "This is gonna be good"
+File2_lowPref = "Not this one too"
+
+File3 = "A little more, please"
+File3_lowPref = "That's just too much"
+
+File4 = "Maybe a biscuit?"
+File4_lowPref = "No, thanks"
+
+File5 = "More Tea...?"
+File5_lowPref = "I have to go..."
+
+############# Metalink/XML ###################################################
+Meta = Metalinkv3_XML()
+
+# file_name: metalink:file "name" field
+# save_name: metalink:file save name, if None the file is rejected
+# content  : metalink:file content
+#
+# size:
+#   True     auto-compute size
+#   None     no <size></size>
+#    any     use this size
+#
+# hash_sha256:
+#   False    no <verification></verification>
+#   True     auto-compute sha256
+#   None     no <hash></hash>
+#    any     use this hash
+#
+# srv_file   : metalink:url server file
+# srv_content: metalink:url server file content, if None the file doesn't exist
+# utype      : metalink:url type
+# location   : metalink:url location (default 'no location field')
+# preference : metalink:url preference (default 999999)
+
+XmlName = "test.metalink"
+
+Meta.xml (
+    # Metalink/XML file name
+    XmlName,
+    # file_name, save_name, content, size, hash_sha256
+    ["subdir/File1", "File1", File1, None, True,
+     # srv_file, srv_content, utype, location, preference
+     ["wrong_file", wrong_file, "http", None, 35],
+     ["404", None, "http", None, 40],
+     ["File1_lowPref", File1_lowPref, "http", None, 25],
+     ["File1", File1, "http", None, 30]],
+    ["/subdir/File2", None, File2, None, True,
+     ["wrong_file", wrong_file, "http", None, 35],
+     ["404", None, "http", None, 40],
+     ["File2_lowPref", File2_lowPref, "http", None, 25],
+     ["File2", File2, "http", None, 30]],
+    ["~/subdir/File3", None, File3, None, True,
+     ["wrong_file", wrong_file, "http", None, 35],
+     ["404", None, "http", None, 40],
+     ["File3_lowPref", File3_lowPref, "http", None, 25],
+     ["File3", File3, "http", None, 30]],
+    ["../subdir/File4", None, File4, None, True,
+     ["wrong_file", wrong_file, "http", None, 35],
+     ["404", None, "http", None, 40],
+     ["File4_lowPref", File4_lowPref, "http", None, 25],
+     ["File4", File4, "http", None, 30]],
+    ["subdir/File5", "File5", File5, None, True,
+     ["wrong_file", wrong_file, "http", None, 35],
+     ["404", None, "http", None, 40],
+     ["File5_lowPref", File5_lowPref, "http", None, 25],
+     ["File5", File5, "http", None, 30]],
+)
+
+Meta.print_meta ()
+
+err = Meta.http_test (
+    "--trust-server-names " + \
+    "--directory-prefix ../dir " + \
+    "--input-metalink " + XmlName, 0
+)
+
+exit (err)

testenv/Test-metalink-xml-relprefix.py

@@ -0,0 +1,101 @@
+#!/usr/bin/env python3
+
+from sys import exit
+from misc.metalinkv3_xml import Metalinkv3_XML
+
+"""
+    This is to test Metalink/XML relative directory prefix support in Wget.
+
+    With --trust-server-names, trust the metalink:file names.
+
+    Without --trust-server-names, don't trust the metalink:file names:
+    use the basename of --input-metalink, and add a sequential number
+    (e.g. .#1, .#2, etc.).
+
+    Strip the directory from unsafe paths.
+"""
+
+############# File Definitions ###############################################
+wrong_file = "Ouch!"
+
+File1 = "Would you like some Tea?"
+File1_lowPref = "Do not take this"
+
+File2 = "This is gonna be good"
+File2_lowPref = "Not this one too"
+
+File3 = "A little more, please"
+File3_lowPref = "That's just too much"
+
+File4 = "Maybe a biscuit?"
+File4_lowPref = "No, thanks"
+
+File5 = "More Tea...?"
+File5_lowPref = "I have to go..."
+
+############# Metalink/XML ###################################################
+Meta = Metalinkv3_XML()
+
+# file_name: metalink:file "name" field
+# save_name: metalink:file save name, if None the file is rejected
+# content  : metalink:file content
+#
+# size:
+#   True     auto-compute size
+#   None     no <size></size>
+#    any     use this size
+#
+# hash_sha256:
+#   False    no <verification></verification>
+#   True     auto-compute sha256
+#   None     no <hash></hash>
+#    any     use this hash
+#
+# srv_file   : metalink:url server file
+# srv_content: metalink:url server file content, if None the file doesn't exist
+# utype      : metalink:url type (http, ftp, etc.)
+# location   : metalink:url location (default 'no location field')
+# preference : metalink:url preference (default 999999)
+
+XmlName = "test.metalink"
+
+Meta.xml (
+    # Metalink/XML file name
+    XmlName,
+    # file_name, save_name, content, size, hash_sha256
+    ["subdir/File1", XmlName + ".#1", File1, None, True,
+     # srv_file, srv_content, utype, location, preference
+     ["wrong_file", wrong_file, "http", None, 35],
+     ["404", None, "http", None, 40],
+     ["File1_lowPref", File1_lowPref, "http", None, 25],
+     ["File1", File1, "http", None, 30]],
+    ["/subdir/File2", None, File2, None, True,
+     ["wrong_file", wrong_file, "http", None, 35],
+     ["404", None, "http", None, 40],
+     ["File2_lowPref", File2_lowPref, "http", None, 25],
+     ["File2", File2, "http", None, 30]],
+    ["~/subdir/File3", None, File3, None, True,
+     ["wrong_file", wrong_file, "http", None, 35],
+     ["404", None, "http", None, 40],
+     ["File3_lowPref", File3_lowPref, "http", None, 25],
+     ["File3", File3, "http", None, 30]],
+    ["../subdir/File4", None, File4, None, True,
+     ["wrong_file", wrong_file, "http", None, 35],
+     ["404", None, "http", None, 40],
+     ["File4_lowPref", File4_lowPref, "http", None, 25],
+     ["File4", File4, "http", None, 30]],
+    ["subdir/File5", XmlName + ".#2", File5, None, True,
+     ["wrong_file", wrong_file, "http", None, 35],
+     ["404", None, "http", None, 40],
+     ["File5_lowPref", File5_lowPref, "http", None, 25],
+     ["File5", File5, "http", None, 30]],
+)
+
+Meta.print_meta ()
+
+err = Meta.http_test (
+    "--directory-prefix ../dir " + \
+    "--input-metalink " + XmlName, 0
+)
+
+exit (err)

testenv/Test-metalink-xml-size.py

@@ -0,0 +1,100 @@
+#!/usr/bin/env python3
+
+from sys import exit
+from misc.metalinkv3_xml import Metalinkv3_XML
+
+"""
+    This is to test Metalink/XML file size check in Wget.
+
+    With --trust-server-names, trust the metalink:file names.
+
+    Without --trust-server-names, don't trust the metalink:file names:
+    use the basename of --input-metalink, and add a sequential number
+    (e.g. .#1, .#2, etc.).
+
+    Strip the directory from unsafe paths.
+"""
+
+############# File Definitions ###############################################
+wrong_file = "Ouch!"
+
+File1 = "Would you like some Tea?"
+File1_lowPref = "Do not take this"
+
+File2 = "This is gonna be good"
+File2_lowPref = "Not this one too"
+
+File3 = "A little more, please"
+File3_lowPref = "That's just too much"
+
+File4 = "Maybe a biscuit?"
+File4_lowPref = "No, thanks"
+
+File5 = "More Tea...?"
+File5_lowPref = "I have to go..."
+
+############# Metalink/XML ###################################################
+Meta = Metalinkv3_XML()
+
+# file_name: metalink:file "name" field
+# save_name: metalink:file save name, if None the file is rejected
+# content  : metalink:file content
+#
+# size:
+#   True     auto-compute size
+#   None     no <size></size>
+#    any     use this size
+#
+# hash_sha256:
+#   False    no <verification></verification>
+#   True     auto-compute sha256
+#   None     no <hash></hash>
+#    any     use this hash
+#
+# srv_file   : metalink:url server file
+# srv_content: metalink:url server file content, if None the file doesn't exist
+# utype      : metalink:url type (http, ftp, etc.)
+# location   : metalink:url location (default 'no location field')
+# preference : metalink:url preference (default 999999)
+
+XmlName = "test.metalink"
+
+Meta.xml (
+    # Metalink/XML file name
+    XmlName,
+    # file_name, save_name, content, size, hash_sha256
+    ["File1", XmlName + ".#1", File1, "", True,
+     # srv_file, srv_content, utype, location, preference
+     ["wrong_file", wrong_file, "http", None, 35],
+     ["404", None, "http", None, 40],
+     ["File1_lowPref", File1_lowPref, "http", None, 25],
+     ["File1", File1, "http", None, 30]],
+    ["File2", None, File2, 2, True,
+     ["wrong_file", wrong_file, "http", None, 35],
+     ["404", None, "http", None, 40],
+     ["File2_lowPref", File2_lowPref, "http", None, 25],
+     ["File2", File2, "http", None, 30]],
+    ["File3", XmlName + ".#3", File3, 0, True,
+     ["wrong_file", wrong_file, "http", None, 35],
+     ["404", None, "http", None, 40],
+     ["File3_lowPref", File3_lowPref, "http", None, 25],
+     ["File3", File3, "http", None, 30]],
+    ["File4", XmlName + ".#4", File4, -5, True,
+     ["wrong_file", wrong_file, "http", None, 35],
+     ["404", None, "http", None, 40],
+     ["File4_lowPref", File4_lowPref, "http", None, 25],
+     ["File4", File4, "http", None, 30]],
+    ["File5", XmlName + ".#5", File5, True, True,
+     ["wrong_file", wrong_file, "http", None, 35],
+     ["404", None, "http", None, 40],
+     ["File5_lowPref", File5_lowPref, "http", None, 25],
+     ["File5", File5, "http", None, 30]],
+)
+
+Meta.print_meta ()
+
+err = Meta.http_test (
+    "--input-metalink " + XmlName, 1
+)
+
+exit (err)

testenv/Test-metalink-xml-trust.py

@@ -0,0 +1,101 @@
+#!/usr/bin/env python3
+
+from sys import exit
+from misc.metalinkv3_xml import Metalinkv3_XML
+
+"""
+    This is to test Metalink/XML file support in Wget.
+
+    With --trust-server-names, trust the metalink:file names.
+
+    Without --trust-server-names, don't trust the metalink:file names:
+    use the basename of --input-metalink, and add a sequential number
+    (e.g. .#1, .#2, etc.).
+
+    Strip the directory from unsafe paths.
+"""
+
+############# File Definitions ###############################################
+wrong_file = "Ouch!"
+
+File1 = "Would you like some Tea?"
+File1_lowPref = "Do not take this"
+
+File2 = "This is gonna be good"
+File2_lowPref = "Not this one too"
+
+File3 = "A little more, please"
+File3_lowPref = "That's just too much"
+
+File4 = "Maybe a biscuit?"
+File4_lowPref = "No, thanks"
+
+File5 = "More Tea...?"
+File5_lowPref = "I have to go..."
+
+############# Metalink/XML ###################################################
+Meta = Metalinkv3_XML()
+
+# file_name: metalink:file "name" field
+# save_name: metalink:file save name, if None the file is rejected
+# content  : metalink:file content
+#
+# size:
+#   True     auto-compute size
+#   None     no <size></size>
+#    any     use this size
+#
+# hash_sha256:
+#   False    no <verification></verification>
+#   True     auto-compute sha256
+#   None     no <hash></hash>
+#    any     use this hash
+#
+# srv_file   : metalink:url server file
+# srv_content: metalink:url server file content, if None the file doesn't exist
+# utype      : metalink:url type (http, ftp, etc.)
+# location   : metalink:url location (default 'no location field')
+# preference : metalink:url preference (default 999999)
+
+XmlName = "test.metalink"
+
+Meta.xml (
+    # Metalink/XML file name
+    XmlName,
+    # file_name, save_name, content, size, hash_sha256
+    ["File1", "File1", File1, None, True,
+     # srv_file, srv_content, utype, location, preference
+     ["wrong_file", wrong_file, "http", None, 35],
+     ["404", None, "http", None, 40],
+     ["File1_lowPref", File1_lowPref, "http", None, 25],
+     ["File1", File1, "http", None, 30]],
+    ["File2", "File2", File2, None, True,
+     ["wrong_file", wrong_file, "http", None, 35],
+     ["404", None, "http", None, 40],
+     ["File2_lowPref", File2_lowPref, "http", None, 25],
+     ["File2", File2, "http", None, 30]],
+    ["File3", "File3", File3, None, True,
+     ["wrong_file", wrong_file, "http", None, 35],
+     ["404", None, "http", None, 40],
+     ["File3_lowPref", File3_lowPref, "http", None, 25],
+     ["File3", File3, "http", None, 30]],
+    ["File4", "File4", File4, None, True,
+     ["wrong_file", wrong_file, "http", None, 35],
+     ["404", None, "http", None, 40],
+     ["File4_lowPref", File4_lowPref, "http", None, 25],
+     ["File4", File4, "http", None, 30]],
+    ["File5", "File5", File5, None, True,
+     ["wrong_file", wrong_file, "http", None, 35],
+     ["404", None, "http", None, 40],
+     ["File5_lowPref", File5_lowPref, "http", None, 25],
+     ["File5", File5, "http", None, 30]],
+)
+
+Meta.print_meta ()
+
+err = Meta.http_test (
+    "--trust-server-names " + \
+    "--input-metalink " + XmlName, 0
+)
+
+exit (err)

testenv/Test-metalink-xml-urlbreak.py

@@ -0,0 +1,236 @@
+#!/usr/bin/env python3
+from sys import exit
+from test.http_test import HTTPTest
+from misc.wget_file import WgetFile
+import hashlib
+
+"""
+    This is to test Metalink/XML white spaces in url resources.
+
+    With --trust-server-names, trust the metalink:file names.
+
+    Without --trust-server-names, don't trust the metalink:file names:
+    use the basename of --input-metalink, and add a sequential number
+    (e.g. .#1, .#2, etc.).
+
+    Strip the directory from unsafe paths.
+"""
+############# File Definitions ###############################################
+bad = "Ouch!"
+
+File1 = "Would you like some Tea?"
+File1_lowPref = "Do not take this"
+File1_sha256 = hashlib.sha256 (File1.encode ('UTF-8')).hexdigest ()
+
+File2 = "This is gonna be good"
+File2_lowPref = "Not this one too"
+File2_sha256 = hashlib.sha256 (File2.encode ('UTF-8')).hexdigest ()
+
+File3 = "A little more, please"
+File3_lowPref = "That's just too much"
+File3_sha256 = hashlib.sha256 (File3.encode ('UTF-8')).hexdigest ()
+
+File4 = "Maybe a biscuit?"
+File4_lowPref = "No, thanks"
+File4_sha256 = hashlib.sha256 (File4.encode ('UTF-8')).hexdigest ()
+
+File5 = "More Tea...?"
+File5_lowPref = "I have to go..."
+File5_sha256 = hashlib.sha256 (File5.encode ('UTF-8')).hexdigest ()
+
+MetaXml = \
+"""<?xml version="1.0" encoding="utf-8"?>
+<metalink version="3.0" xmlns="http://www.metalinker.org/">
+  <publisher>
+    <name>GNU Wget</name>
+  </publisher>
+  <license>
+    <name>GNU GPL</name>
+    <url>http://www.gnu.org/licenses/gpl.html</url>
+  </license>
+  <identity>Wget Test Files</identity>
+  <version>1.2.3</version>
+  <description>Wget Test Files description</description>
+  <files>
+    <file name="File1">
+      <verification>
+        <hash type="sha256">{{FILE1_HASH}}</hash>
+      </verification>
+      <resources>
+        <url type="http" preference="35">
+                   http://{{SRV_HOST}}:{{SRV_PORT}}/wrong_file
+        </url>
+        <url type="http" preference="40">
+                   http://{{SRV_HOST}}:{{SRV_PORT}}/404
+        </url>
+        <url type="http" preference="25">
+                   http://{{SRV_HOST}}:{{SRV_PORT}}/File1_lowPref
+        </url>
+        <url type="http" preference="30">
+                   http://{{SRV_HOST}}:{{SRV_PORT}}/File1
+        </url>
+      </resources>
+    </file>
+    <file name="File2">
+      <verification>
+        <hash type="sha256">{{FILE2_HASH}}</hash>
+      </verification>
+      <resources>
+        <url type="http" preference="35">
+                   http://{{SRV_HOST}}:{{SRV_PORT}}/wrong_file
+        </url>
+        <url type="http" preference="40">
+                   http://{{SRV_HOST}}:{{SRV_PORT}}/404
+        </url>
+        <url type="http" preference="25">
+                   http://{{SRV_HOST}}:{{SRV_PORT}}/File2_lowPref
+        </url>
+        <url type="http" preference="30">
+                   http://{{SRV_HOST}}:{{SRV_PORT}}/File2
+        </url>
+      </resources>
+    </file>
+    <file name="File3">
+      <verification>
+        <hash type="sha256">{{FILE3_HASH}}</hash>
+      </verification>
+      <resources>
+        <url type="http" preference="35">
+                   http://{{SRV_HOST}}:{{SRV_PORT}}/wrong_file
+        </url>
+        <url type="http" preference="40">
+                   http://{{SRV_HOST}}:{{SRV_PORT}}/404
+        </url>
+        <url type="http" preference="25">
+                   http://{{SRV_HOST}}:{{SRV_PORT}}/File3_lowPref
+        </url>
+        <url type="http" preference="30">
+                   http://{{SRV_HOST}}:{{SRV_PORT}}/File3
+        </url>
+      </resources>
+    </file>
+    <file name="File4">
+      <verification>
+        <hash type="sha256">{{FILE4_HASH}}</hash>
+      </verification>
+      <resources>
+        <url type="http" preference="35">
+                   http://{{SRV_HOST}}:{{SRV_PORT}}/wrong_file
+        </url>
+        <url type="http" preference="40">
+                   http://{{SRV_HOST}}:{{SRV_PORT}}/404
+        </url>
+        <url type="http" preference="25">
+                   http://{{SRV_HOST}}:{{SRV_PORT}}/File4_lowPref
+        </url>
+        <url type="http" preference="30">
+                   http://{{SRV_HOST}}:{{SRV_PORT}}/File4
+        </url>
+      </resources>
+    </file>
+    <file name="File5">
+      <verification>
+        <hash type="sha256">{{FILE5_HASH}}</hash>
+      </verification>
+      <resources>
+        <url type="http" preference="35">
+                   http://{{SRV_HOST}}:{{SRV_PORT}}/wrong_file
+        </url>
+        <url type="http" preference="40">
+                   http://{{SRV_HOST}}:{{SRV_PORT}}/404
+        </url>
+        <url type="http" preference="25">
+                   http://{{SRV_HOST}}:{{SRV_PORT}}/File5_lowPref
+        </url>
+        <url type="http" preference="30">
+                   http://{{SRV_HOST}}:{{SRV_PORT}}/File5
+        </url>
+      </resources>
+    </file>
+  </files>
+</metalink>
+"""
+
+wrong_file = WgetFile ("wrong_file", bad)
+
+File1_orig = WgetFile ("File1", File1)
+File1_down = WgetFile ("test.metalink.#1", File1)
+File1_nono = WgetFile ("File1_lowPref", File1_lowPref)
+
+File2_orig = WgetFile ("File2", File2)
+File2_down = WgetFile ("test.metalink.#2", File2)
+File2_nono = WgetFile ("File2_lowPref", File2_lowPref)
+
+File3_orig = WgetFile ("File3", File3)
+File3_down = WgetFile ("test.metalink.#3", File3)
+File3_nono = WgetFile ("File3_lowPref", File3_lowPref)
+
+File4_orig = WgetFile ("File4", File4)
+File4_down = WgetFile ("test.metalink.#4", File4)
+File4_nono = WgetFile ("File4_lowPref", File4_lowPref)
+
+File5_orig = WgetFile ("File5", File5)
+File5_down = WgetFile ("test.metalink.#5", File5)
+File5_nono = WgetFile ("File5_lowPref", File5_lowPref)
+
+MetaFile = WgetFile ("test.metalink", MetaXml)
+
+WGET_OPTIONS = "--input-metalink test.metalink"
+WGET_URLS = [[]]
+
+Files = [[
+    wrong_file,
+    File1_orig, File1_nono,
+    File2_orig, File2_nono,
+    File3_orig, File3_nono,
+    File4_orig, File4_nono,
+    File5_orig, File5_nono
+]]
+Existing_Files = [MetaFile]
+
+ExpectedReturnCode = 0
+ExpectedDownloadedFiles = [
+    File1_down,
+    File2_down,
+    File3_down,
+    File4_down,
+    File5_down,
+    MetaFile
+]
+
+################ Pre and Post Test Hooks #####################################
+pre_test = {
+    "ServerFiles"       : Files,
+    "LocalFiles"        : Existing_Files
+}
+test_options = {
+    "WgetCommands"      : WGET_OPTIONS,
+    "Urls"              : WGET_URLS
+}
+post_test = {
+    "ExpectedFiles"     : ExpectedDownloadedFiles,
+    "ExpectedRetcode"   : ExpectedReturnCode
+}
+
+http_test = HTTPTest (
+                pre_hook=pre_test,
+                test_params=test_options,
+                post_hook=post_test,
+)
+
+http_test.server_setup()
+### Get and use dynamic server sockname
+srv_host, srv_port = http_test.servers[0].server_inst.socket.getsockname ()
+
+MetaXml = MetaXml.replace('{{FILE1_HASH}}', File1_sha256)
+MetaXml = MetaXml.replace('{{FILE2_HASH}}', File2_sha256)
+MetaXml = MetaXml.replace('{{FILE3_HASH}}', File3_sha256)
+MetaXml = MetaXml.replace('{{FILE4_HASH}}', File4_sha256)
+MetaXml = MetaXml.replace('{{FILE5_HASH}}', File5_sha256)
+MetaXml = MetaXml.replace('{{SRV_HOST}}', srv_host)
+MetaXml = MetaXml.replace('{{SRV_PORT}}', str (srv_port))
+MetaFile.content = MetaXml
+
+err = http_test.begin ()
+
+exit (err)

testenv/Test-metalink-xml.py

@@ -1,87 +1,100 @@
 #!/usr/bin/env python3
+
 from sys import exit
-from test.http_test import HTTPTest
-from misc.wget_file import WgetFile
-import re
-import hashlib
+from misc.metalinkv3_xml import Metalinkv3_XML
 
 """
-    This is to test Metalink as XML file support in Wget.
+    This is to test Metalink/XML file support in Wget.
+
+    With --trust-server-names, trust the metalink:file names.
+
+    Without --trust-server-names, don't trust the metalink:file names:
+    use the basename of --input-metalink, and add a sequential number
+    (e.g. .#1, .#2, etc.).
+
+    Strip the directory from unsafe paths.
 """
+
 ############# File Definitions ###############################################
+wrong_file = "Ouch!"
+
 File1 = "Would you like some Tea?"
 File1_lowPref = "Do not take this"
-File1_sha256 = hashlib.sha256 (File1.encode ('UTF-8')).hexdigest ()
-MetaXml = \
-"""<?xml version="1.0" encoding="utf-8"?>
-<metalink version="3.0" xmlns="http://www.metalinker.org/">
-  <publisher>
-    <name>GNU Wget</name>
-  </publisher>
-  <license>
-    <name>GNU GPL</name>
-    <url>http://www.gnu.org/licenses/gpl.html</url>
-  </license>
-  <identity>Wget Test File 1</identity>
-  <version>1.2.3</version>
-  <description>Wget Test File 1 description</description>
-  <files>
-    <file name="File1">
-      <verification>
-        <hash type="sha256">{{FILE1_HASH}}</hash>
-      </verification>
-      <resources>
-        <url type="http" preference="40">http://broken.example/File1</url>
-        <url type="http" preference="25">http://{{SRV_HOST}}:{{SRV_PORT}}/File1_lowPref</url>
-        <url type="http" preference="30">http://{{SRV_HOST}}:{{SRV_PORT}}/File1</url>
-      </resources>
-    </file>
-  </files>
-</metalink>
-"""
 
-A_File = WgetFile ("File1", File1)
-B_File = WgetFile ("File1_lowPref", File1_lowPref)
-MetaFile = WgetFile ("test.meta4", MetaXml)
+File2 = "This is gonna be good"
+File2_lowPref = "Not this one too"
 
-WGET_OPTIONS = "--input-metalink test.meta4"
-WGET_URLS = [[]]
+File3 = "A little more, please"
+File3_lowPref = "That's just too much"
 
-Files = [[A_File, B_File]]
-Existing_Files = [MetaFile]
+File4 = "Maybe a biscuit?"
+File4_lowPref = "No, thanks"
 
-ExpectedReturnCode = 0
-ExpectedDownloadedFiles = [A_File, MetaFile]
+File5 = "More Tea...?"
+File5_lowPref = "I have to go..."
 
-################ Pre and Post Test Hooks #####################################
-pre_test = {
-    "ServerFiles"       : Files,
-    "LocalFiles"        : Existing_Files
-}
-test_options = {
-    "WgetCommands"      : WGET_OPTIONS,
-    "Urls"              : WGET_URLS
-}
-post_test = {
-    "ExpectedFiles"     : ExpectedDownloadedFiles,
-    "ExpectedRetcode"   : ExpectedReturnCode
-}
+############# Metalink/XML ###################################################
+Meta = Metalinkv3_XML()
 
-http_test = HTTPTest (
-                pre_hook=pre_test,
-                test_params=test_options,
-                post_hook=post_test,
+# file_name: metalink:file "name" field
+# save_name: metalink:file save name, if None the file is rejected
+# content  : metalink:file content
+#
+# size:
+#   True     auto-compute size
+#   None     no <size></size>
+#    any     use this size
+#
+# hash_sha256:
+#   False    no <verification></verification>
+#   True     auto-compute sha256
+#   None     no <hash></hash>
+#    any     use this hash
+#
+# srv_file   : metalink:url server file
+# srv_content: metalink:url server file content, if None the file doesn't exist
+# utype      : metalink:url type (http, ftp, etc.)
+# location   : metalink:url location (default 'no location field')
+# preference : metalink:url preference (default 999999)
+
+XmlName = "test.metalink"
+
+Meta.xml (
+    # Metalink/XML file name
+    XmlName,
+    # file_name, save_name, content, size, hash_sha256
+    ["File1", XmlName + ".#1", File1, None, True,
+     # srv_file, srv_content, utype, location, preference
+     ["wrong_file", wrong_file, "http", None, 35],
+     ["404", None, "http", None, 40],
+     ["File1_lowPref", File1_lowPref, "http", None, 25],
+     ["File1", File1, "http", None, 30]],
+    ["File2", XmlName + ".#2", File2, None, True,
+     ["wrong_file", wrong_file, "http", None, 35],
+     ["404", None, "http", None, 40],
+     ["File2_lowPref", File2_lowPref, "http", None, 25],
+     ["File2", File2, "http", None, 30]],
+    ["File3", XmlName + ".#3", File3, None, True,
+     ["wrong_file", wrong_file, "http", None, 35],
+     ["404", None, "http", None, 40],
+     ["File3_lowPref", File3_lowPref, "http", None, 25],
+     ["File3", File3, "http", None, 30]],
+    ["File4", XmlName + ".#4", File4, None, True,
+     ["wrong_file", wrong_file, "http", None, 35],
+     ["404", None, "http", None, 40],
+     ["File4_lowPref", File4_lowPref, "http", None, 25],
+     ["File4", File4, "http", None, 30]],
+    ["File5", XmlName + ".#5", File5, None, True,
+     ["wrong_file", wrong_file, "http", None, 35],
+     ["404", None, "http", None, 40],
+     ["File5_lowPref", File5_lowPref, "http", None, 25],
+     ["File5", File5, "http", None, 30]],
 )
 
-http_test.server_setup()
-### Get and use dynamic server sockname
-srv_host, srv_port = http_test.servers[0].server_inst.socket.getsockname ()
+Meta.print_meta ()
 
-MetaXml = re.sub (r'{{FILE1_HASH}}', File1_sha256, MetaXml)
-MetaXml = re.sub (r'{{SRV_HOST}}', srv_host, MetaXml)
-MetaXml = re.sub (r'{{SRV_PORT}}', str (srv_port), MetaXml)
-MetaFile.content = MetaXml
-
-err = http_test.begin ()
+err = Meta.http_test (
+    "--input-metalink " + XmlName, 0
+)
 
 exit (err)

testenv/Test-pinnedpubkey-hash-https.py

@@ -20,8 +20,12 @@ A_File = WgetFile ("File1", File1)
 B_File = WgetFile ("File2", File2)
 C_File = WgetFile ("File3", File3)
 
-CAFILE = os.path.abspath(os.path.join(os.getenv('srcdir', '.'), 'certs', 'ca-cert.pem'))
-WGET_OPTIONS = "--pinnedpubkey=sha256//mHiEhWHvusnzP7COZk+SzSJ+Gl7nZT+ADx0PUnDD7mM= --ca-certificate=" + CAFILE
+CERTDIR = os.path.abspath(os.path.join(os.getenv('srcdir', '.'), 'certs'))
+CAFILE = CERTDIR + '/ca-cert.pem'
+KEYFILE = CERTDIR + '/server-pubkey-sha256.base64'
+with open(KEYFILE, 'r') as f:
+    KEY = f.read().replace('\n', '')
+WGET_OPTIONS = "--pinnedpubkey=sha256//" + KEY + " --ca-certificate=" + CAFILE
 WGET_URLS = [["File1", "File2"]]
 
 Files = [[A_File, B_File]]

testenv/Test-pinnedpubkey-hash-no-check-fail-https.py

@@ -18,7 +18,7 @@ File2 = "With lemon or cream?"
 A_File = WgetFile ("File1", File1)
 B_File = WgetFile ("File2", File2)
 
-WGET_OPTIONS = "--no-check-certificate --pinnedpubkey=sha256//mHiEhWHvusnzP7COZk+SzSJ+Gl7ZZT+ADx0PUnDD7mM="
+WGET_OPTIONS = "--no-check-certificate --pinnedpubkey=sha256//invalid"
 WGET_URLS = [["File1", "File2"]]
 
 Files = [[A_File, B_File]]

testenv/Test-recursive-basic.py

@@ -0,0 +1,57 @@
+#!/usr/bin/env python3
+from sys import exit
+from test.http_test import HTTPTest
+from test.base_test import HTTP, HTTPS
+from misc.wget_file import WgetFile
+
+"""
+    Basic test of --recursive.
+"""
+############# File Definitions ###############################################
+File1 = """<html><body>
+<a href=\"/a/File2.html\">text</a>
+<a href=\"/b/File3.html\">text</a>
+</body></html>"""
+File2 = "With lemon or cream?"
+File3 = "Surely you're joking Mr. Feynman"
+
+File1_File = WgetFile ("a/File1.html", File1)
+File2_File = WgetFile ("a/File2.html", File2)
+File3_File = WgetFile ("b/File3.html", File3)
+
+WGET_OPTIONS = "--recursive --no-host-directories"
+WGET_URLS = [["a/File1.html"]]
+
+Servers = [HTTP]
+
+Files = [[File1_File, File2_File, File3_File]]
+Existing_Files = []
+
+ExpectedReturnCode = 0
+ExpectedDownloadedFiles = [File1_File, File2_File, File3_File]
+Request_List = [["GET /a/File1.html",
+                 "GET /a/File2.html",
+                 "GET /b/File3.html"]]
+
+################ Pre and Post Test Hooks #####################################
+pre_test = {
+    "ServerFiles"       : Files,
+    "LocalFiles"        : Existing_Files
+}
+test_options = {
+    "WgetCommands"      : WGET_OPTIONS,
+    "Urls"              : WGET_URLS
+}
+post_test = {
+    "ExpectedFiles"     : ExpectedDownloadedFiles,
+    "ExpectedRetcode"   : ExpectedReturnCode
+}
+
+err = HTTPTest (
+                pre_hook=pre_test,
+                test_params=test_options,
+                post_hook=post_test,
+                protocols=Servers
+).begin ()
+
+exit (err)

testenv/Test-recursive-include.py

@@ -0,0 +1,56 @@
+#!/usr/bin/env python3
+from sys import exit
+from test.http_test import HTTPTest
+from test.base_test import HTTP, HTTPS
+from misc.wget_file import WgetFile
+
+"""
+    Basic test of --recursive.
+"""
+############# File Definitions ###############################################
+File1 = """<html><body>
+<a href=\"/a/File2.html\">text</a>
+<a href=\"/b/File3.html\">text</a>
+</body></html>"""
+File2 = "With lemon or cream?"
+File3 = "Surely you're joking Mr. Feynman"
+
+File1_File = WgetFile ("a/File1.html", File1)
+File2_File = WgetFile ("a/File2.html", File2)
+File3_File = WgetFile ("b/File3.html", File3)
+
+WGET_OPTIONS = "--recursive --no-host-directories --include-directories=a"
+WGET_URLS = [["a/File1.html"]]
+
+Servers = [HTTP]
+
+Files = [[File1_File, File2_File, File3_File]]
+Existing_Files = []
+
+ExpectedReturnCode = 0
+ExpectedDownloadedFiles = [File1_File, File2_File]
+Request_List = [["GET /a/File1.html",
+                 "GET /a/File2.html"]]
+
+################ Pre and Post Test Hooks #####################################
+pre_test = {
+    "ServerFiles"       : Files,
+    "LocalFiles"        : Existing_Files
+}
+test_options = {
+    "WgetCommands"      : WGET_OPTIONS,
+    "Urls"              : WGET_URLS
+}
+post_test = {
+    "ExpectedFiles"     : ExpectedDownloadedFiles,
+    "ExpectedRetcode"   : ExpectedReturnCode
+}
+
+err = HTTPTest (
+                pre_hook=pre_test,
+                test_params=test_options,
+                post_hook=post_test,
+                protocols=Servers
+).begin ()
+
+exit (err)

testenv/Test-recursive-redirect.py

@@ -0,0 +1,64 @@
+#!/usr/bin/env python3
+from sys import exit
+from test.http_test import HTTPTest
+from test.base_test import HTTP, HTTPS
+from misc.wget_file import WgetFile
+
+"""
+    Basic test of --recursive.
+"""
+############# File Definitions ###############################################
+File1 = """<html><body>
+<a href=\"/a/File2.html\">text</a>
+<a href=\"/b/File3.html\">text</a>
+</body></html>"""
+File2 = "With lemon or cream?"
+File3 = "Surely you're joking Mr. Feynman"
+
+File1_rules = {
+    "Response"          : 301,
+    "SendHeader"        : {"Location" : "/b/File1.html"}
+}
+
+File1_File = WgetFile ("a/File1.html", "", rules=File1_rules)
+File1_Redirected = WgetFile ("b/File1.html", File1)
+File1_Retrieved = WgetFile ("a/File1.html", File1)
+File2_File = WgetFile ("a/File2.html", File2)
+File3_File = WgetFile ("b/File3.html", File3)
+
+WGET_OPTIONS = "--recursive --no-host-directories --include-directories=a"
+WGET_URLS = [["a/File1.html"]]
+
+Servers = [HTTP]
+
+Files = [[File1_Redirected, File1_File, File2_File, File3_File]]
+Existing_Files = []
+
+ExpectedReturnCode = 0
+ExpectedDownloadedFiles = [File1_Retrieved, File2_File]
+Request_List = [["GET /a/File1.html",
+                 "GET /a/File2.html",
+                 "GET /b/File3.html"]]
+
+################ Pre and Post Test Hooks #####################################
+pre_test = {
+    "ServerFiles"       : Files,
+    "LocalFiles"        : Existing_Files
+}
+test_options = {
+    "WgetCommands"      : WGET_OPTIONS,
+    "Urls"              : WGET_URLS
+}
+post_test = {
+    "ExpectedFiles"     : ExpectedDownloadedFiles,
+    "ExpectedRetcode"   : ExpectedReturnCode
+}
+
+err = HTTPTest (
+                pre_hook=pre_test,
+                test_params=test_options,
+                post_hook=post_test,
+                protocols=Servers
+).begin ()
+
+exit (err)

testenv/Test-redirect.py

@@ -0,0 +1,57 @@
+#!/usr/bin/env python3
+from sys import exit
+from test.http_test import HTTPTest
+from test.base_test import HTTP, HTTPS
+from misc.wget_file import WgetFile
+
+"""
+    This is a Prototype Test File.
+    Ideally this File should be copied and edited to write new tests.
+"""
+############# File Definitions ###############################################
+File2 = "Would you like some Tea?"
+
+File1_rules = {
+    "Response"          : 301,
+    "SendHeader"        : {"Location" : "/File2.txt"}
+}
+
+# /File1.txt is only a redirect, and so has no file content.
+File1_File = WgetFile ("File1.txt", "", rules=File1_rules)
+# File1_Retrieved is what will be retrieved for URL /File1.txt.
+File1_Retrieved = WgetFile ("File1.txt", File2)
+File2_File = WgetFile ("File2.txt", File2)
+
+WGET_OPTIONS = ""
+WGET_URLS = [["File1.txt"]]
+
+Servers = [HTTP]
+
+Files = [[File1_File, File2_File]]
+Existing_Files = []
+
+ExpectedReturnCode = 0
+ExpectedDownloadedFiles = [File1_Retrieved]
+
+################ Pre and Post Test Hooks #####################################
+pre_test = {
+    "ServerFiles"       : Files,
+    "LocalFiles"        : Existing_Files
+}
+test_options = {
+    "WgetCommands"      : WGET_OPTIONS,
+    "Urls"              : WGET_URLS
+}
+post_test = {
+    "ExpectedFiles"     : ExpectedDownloadedFiles,
+    "ExpectedRetcode"   : ExpectedReturnCode
+}
+
+err = HTTPTest (
+                pre_hook=pre_test,
+                test_params=test_options,
+                post_hook=post_test,
+                protocols=Servers
+).begin ()
+
+exit (err)

testenv/certs/ca-template.cfg

@@ -0,0 +1,246 @@
+# X.509 Certificate options
+#
+# DN options
+
+# The organization of the subject.
+organization = "GNU"
+
+# The organizational unit of the subject.
+unit = "Wget"
+
+# The locality of the subject.
+# locality =
+
+# The state of the certificate owner.
+# state = ""
+
+# The country of the subject. Two letter code.
+# country = GR
+
+# The common name of the certificate owner.
+cn = "GNU Wget"
+
+# A user id of the certificate owner.
+#uid = ""
+
+# Set domain components
+#dc = "name"
+#dc = "domain"
+
+# If the supported DN OIDs are not adequate you can set
+# any OID here.
+# For example set the X.520 Title and the X.520 Pseudonym
+# by using OID and string pairs.
+#dn_oid = 2.5.4.12 Dr.
+#dn_oid = 2.5.4.65 jackal
+
+# This is deprecated and should not be used in new
+# certificates.
+# pkcs9_email = "bug-wget@gnu.org"
+
+# An alternative way to set the certificate's distinguished name directly
+# is with the "dn" option. The attribute names allowed are:
+# C (country), street, O (organization), OU (unit), title, CN (common name),
+# L (locality), ST (state), placeOfBirth, gender, countryOfCitizenship,
+# countryOfResidence, serialNumber, telephoneNumber, surName, initials,
+# generationQualifier, givenName, pseudonym, dnQualifier, postalCode, name,
+# businessCategory, DC, UID, jurisdictionOfIncorporationLocalityName,
+# jurisdictionOfIncorporationStateOrProvinceName,
+# jurisdictionOfIncorporationCountryName, XmppAddr, and numeric OIDs.
+
+#dn = "cn = Nikos,st = New Something,C=GR,surName=Mavrogiannopoulos,2.5.4.9=Arkadias"
+
+# The serial number of the certificate
+# Comment the field for a time-based serial number.
+# serial = 007
+
+# In how many days, counting from today, this certificate will expire.
+# Use -1 if there is no expiration date.
+expiration_days = -1
+
+# Alternatively you may set concrete dates and time. The GNU date string
+# formats are accepted. See:
+# http://www.gnu.org/software/tar/manual/html_node/Date-input-formats.html
+
+#activation_date = "2004-02-29 16:21:42"
+#expiration_date = "2025-02-29 16:24:41"
+
+# X.509 v3 extensions
+
+# A dnsname in case of a WWW server.
+#dns_name = "www.none.org"
+#dns_name = "www.morethanone.org"
+
+# A subject alternative name URI
+#uri = "http://www.example.com"
+
+# An IP address in case of a server.
+#ip_address = "192.168.1.1"
+
+# An email in case of a person
+# email = "none@none.org"
+
+# Challenge password used in certificate requests
+challenge_password = 123456
+
+# Password when encrypting a private key
+#password = secret
+
+# An URL that has CRLs (certificate revocation lists)
+# available. Needed in CA certificates.
+#crl_dist_points = "http://www.getcrl.crl/getcrl/"
+
+# Whether this is a CA certificate or not
+ca
+
+# Subject Unique ID (in hex)
+#subject_unique_id = 00153224
+
+# Issuer Unique ID (in hex)
+#issuer_unique_id = 00153225
+
+#### Key usage
+
+# The following key usage flags are used by CAs and end certificates
+
+# Whether this certificate will be used to sign data (needed
+# in TLS DHE ciphersuites). This is the digitalSignature flag
+# in RFC5280 terminology.
+# signing_key
+
+# Whether this certificate will be used to encrypt data (needed
+# in TLS RSA ciphersuites). Note that it is preferred to use different
+# keys for encryption and signing. This is the keyEncipherment flag
+# in RFC5280 terminology.
+# encryption_key
+
+# Whether this key will be used to sign other certificates. The
+# keyCertSign flag in RFC5280 terminology.
+cert_signing_key
+
+# Whether this key will be used to sign CRLs. The
+# cRLSign flag in RFC5280 terminology.
+crl_signing_key
+
+# The keyAgreement flag of RFC5280. It's purpose is loosely
+# defined. Not use it unless required by a protocol.
+#key_agreement
+
+# The dataEncipherment flag of RFC5280. It's purpose is loosely
+# defined. Not use it unless required by a protocol.
+#data_encipherment
+
+# The nonRepudiation flag of RFC5280. It's purpose is loosely
+# defined. Not use it unless required by a protocol.
+#non_repudiation
+
+#### Extended key usage (key purposes)
+
+# The following extensions are used in an end certificate
+# to clarify its purpose. Some CAs also use it to indicate
+# the types of certificates they are purposed to sign.
+
+# Whether this certificate will be used for a TLS client;
+# this sets the id-kp-serverAuth (1.3.6.1.5.5.7.3.1) of
+# extended key usage.
+#tls_www_client
+
+# Whether this certificate will be used for a TLS server;
+# This sets the id-kp-clientAuth (1.3.6.1.5.5.7.3.2) of
+# extended key usage.
+#tls_www_server
+
+# Whether this key will be used to sign code. This sets the
+# id-kp-codeSigning (1.3.6.1.5.5.7.3.3) of extended key usage
+# extension.
+#code_signing_key
+
+# Whether this key will be used to sign OCSP data. This sets the
+# id-kp-OCSPSigning (1.3.6.1.5.5.7.3.9) of extended key usage extension.
+#ocsp_signing_key
+
+# Whether this key will be used for time stamping. This sets the
+# id-kp-timeStamping (1.3.6.1.5.5.7.3.8) of extended key usage extension.
+#time_stamping_key
+
+# Whether this key will be used for email protection. This sets the
+# id-kp-emailProtection (1.3.6.1.5.5.7.3.4) of extended key usage extension.
+#email_protection_key
+
+# Whether this key will be used for IPsec IKE operations (1.3.6.1.5.5.7.3.17).
+#ipsec_ike_key
+
+## adding custom key purpose OIDs
+
+# for microsoft smart card logon
+# key_purpose_oid = 1.3.6.1.4.1.311.20.2.2
+
+# for email protection
+# key_purpose_oid = 1.3.6.1.5.5.7.3.4
+
+# for any purpose (must not be used in intermediate CA certificates)
+# key_purpose_oid = 2.5.29.37.0
+
+### end of key purpose OIDs
+
+# When generating a certificate from a certificate
+# request, then honor the extensions stored in the request
+# and store them in the real certificate.
+honor_crq_extensions
+
+# Path length contraint. Sets the maximum number of
+# certificates that can be used to certify this certificate.
+# (i.e. the certificate chain length)
+#path_len = -1
+#path_len = 2
+
+# OCSP URI
+# ocsp_uri = http://my.ocsp.server/ocsp
+
+# CA issuers URI
+# ca_issuers_uri = http://my.ca.issuer
+
+# Certificate policies
+#policy1 = 1.3.6.1.4.1.5484.1.10.99.1.0
+#policy1_txt = "This is a long policy to summarize"
+#policy1_url = http://www.example.com/a-policy-to-read
+
+#policy2 = 1.3.6.1.4.1.5484.1.10.99.1.1
+#policy2_txt = "This is a short policy"
+#policy2_url = http://www.example.com/another-policy-to-read
+
+# Name constraints
+
+# DNS
+#nc_permit_dns = example.com
+#nc_exclude_dns = test.example.com
+
+# EMAIL
+#nc_permit_email = "nmav@ex.net"
+
+# Exclude subdomains of example.com
+#nc_exclude_email = .example.com
+
+# Exclude all e-mail addresses of example.com
+#nc_exclude_email = example.com
+
+# Options for proxy certificates
+#proxy_policy_language = 1.3.6.1.5.5.7.21.1
+
+# Options for generating a CRL
+
+# The number of days the next CRL update will be due.
+# next CRL update will be in 43 days
+#crl_next_update = 43
+
+# this is the 5th CRL by this CA
+# Comment the field for a time-based number.
+#crl_number = 5
+
+# Specify the update dates more precisely.
+#crl_this_update_date = "2004-02-29 16:21:42"
+#crl_next_update_date = "2025-02-29 16:24:41"
+
+# The date that the certificates will be made seen as
+# being revoked.
+#crl_revocation_date = "2025-02-29 16:24:41"

testenv/certs/make_ca.sh

@@ -0,0 +1,23 @@
+#!/bin/sh -e
+
+# create a self signed CA certificate
+certtool --generate-privkey --outfile ca-key.pem
+certtool --generate-self-signed --load-privkey ca-key.pem --template=ca-template.cfg --outfile ca-cert.pem
+
+# create the server RSA private key
+certtool --generate-privkey --outfile server-key.pem --rsa
+
+# generate a server certificate using the private key only
+certtool --generate-certificate --load-privkey server-key.pem --template=server-template.cfg --outfile server-cert.pem --load-ca-certificate ca-cert.pem --load-ca-privkey ca-key.pem
+
+# create a CRL for the server certificate
+certtool --generate-crl --load-ca-privkey ca-key.pem --load-ca-certificate ca-cert.pem --load-certificate server-cert.pem --outfile server-crl.pem --template=server-template.cfg
+
+# generate a public key in PEM format
+openssl x509 -noout -pubkey < server-cert.pem > server-pubkey.pem
+
+# generate a public key in DER format
+openssl x509 -noout -pubkey < server-cert.pem | openssl asn1parse -noout -inform pem -out server-pubkey.der
+
+# generate a sha256 hash of the public key
+openssl x509 -noout -pubkey < server-cert.pem | openssl asn1parse -noout -inform pem -out /dev/stdout | openssl dgst -sha256 -binary | openssl base64 > server-pubkey-sha256.base64

testenv/certs/server-pubkey-sha256.base64

@@ -0,0 +1 @@
+mHiEhWHvusnzP7COZk+SzSJ+Gl7nZT+ADx0PUnDD7mM=