* NEWS: update

* doc/wget.text: Add documentation
* src/http.c: Add function check_retry_on_http_error ()
* src/init.c: Add opt.retry_on_http_error
* src/main.c: Add struct for retry-on-http-error to option_data[]
* src/options.h: Add retry_on_http_error to struct options

NEWS

@@ -5,6 +5,13 @@ See the end for copying conditions.
 
 Please send GNU Wget bug reports to <bug-wget@gnu.org>.
 
+* Changes in Wget 1.19.1
+
+* Fix bugs, a regression, portability/build issues
+
+* Add new option --retry-on-http-error
+
+
 * Changes in Wget 1.19
 
 * New option --use-askpass=COMMAND. Fetch user/password by calling

bootstrap.conf

@@ -111,11 +111,12 @@ unlink
 unlocked-io
 update-copyright
 libunistring-optional
-unistr/base
-unicase/tolower
+unistr/u8-strlen
+unicase/u8-tolower
 utimens
 vasprintf
 vsnprintf
+wcwidth
 write
 xmemdup0
 xstrndup

cfg.mk

@@ -1,3 +1,6 @@
+# Explicit syntax-check exceptions.
+exclude_file_name_regexp--sc_trailing_blank = ^doc/annou.*$$
+
 export VC_LIST_EXCEPT_DEFAULT=^(lib/.*|m4/.*|md5/.*|build-aux/.*|src/gettext\.h|.*ChangeLog)$$
 
 local-checks-to-skip = \
@@ -21,6 +24,7 @@ local-checks-to-skip = \
     sc_program_name \
     sc_makefile_at_at_check \
     sc_cast_of_argument_to_free \
+    sc_prohibit_undesirable_word_seq
 
 #SHELL=bash -x
 show-vc-list-except:

configure.ac

@@ -663,11 +663,6 @@ if test "X$iri" != "Xno"; then
 #    ])
   ])
 fi
-if test "X$iri" = "Xno"; then
-  # we don't need libunistring - clear settings from gnulib module
-  LIBUNISTRING=""
-  LTLIBUNISTRING=""
-fi
 
 dnl
 dnl Check for UUID

contrib/check-hard

@@ -12,7 +12,8 @@ if [ $CORES -lt 1 ]; then CORES=1; fi
 echo "Running: make distclean"
 make distclean > /dev/null || true
 
-CFLAGS="-std=c89 -pedantic -O0 -g -Wall -Wextra -Wstrict-prototypes -Wold-style-definition -Wwrite-strings -Wshadow -Wformat -Wformat-security -Wunreachable-code -Wstrict-prototypes -Wmissing-prototypes -Wold-style-definition"
+#CFLAGS="-std=c89 -pedantic -O0 -g -Wall -Wextra -Wstrict-prototypes -Wold-style-definition -Wwrite-strings -Wshadow -Wformat -Wformat-security -Wunreachable-code -Wstrict-prototypes -Wmissing-prototypes -Wold-style-definition"
+CFLAGS="-Wall -Wextra -O2 -g"
 
 CACHEFILE=$PWD/config_check.cache
 
@@ -30,7 +31,7 @@ for CC in gcc clang; do
   # the compiler changed, so we have to remove the cache file here
   rm -f $CACHEFILE
 
-  for options in "" "--with-ssl=openssl"; do
+  for options in "" "--with-ssl=openssl" "--disable-iri"; do
     export DISTCHECK_CONFIGURE_FLAGS="-C --cache-file=$CACHEFILE $options"
     echo "  ./configure $DISTCHECK_CONFIGURE_FLAGS"
     ./configure $DISTCHECK_CONFIGURE_FLAGS CFLAGS="$CFLAGS" > /dev/null

contrib/release

@@ -0,0 +1,112 @@
+#!/bin/bash -e
+#
+# Written 2016 by Tim Ruehsen
+
+extensions="gz xz"
+app_name=$(grep "^PACKAGE_NAME='" configure|cut -d "'" -f2)
+app_ver=$(grep "^PACKAGE_VERSION='" configure|cut -d "'" -f2)
+
+echo "Current version of $app_name is $app_ver"
+echo
+echo -n "Enter Y if that is OK, else enter N and edit configure.ac: "
+read x
+[ "$x" != "y" -a "$x" != "Y" ] && exit 0
+
+echo
+grep -i "Changes in $app_name" NEWS|head -1
+echo -n "Did you edit the NEWS file ? [y/N] "
+read x
+[ "$x" != "y" -a "$x" != "Y" ] && exit 0
+
+echo
+
+export CFLAGS="-g -O2 -Wall"
+
+echo "* Bootstrap"
+./bootstrap >/dev/null
+
+echo "* Configure"
+./configure -q
+
+echo "* Make"
+make -s clean
+make -s
+
+echo "* Make distcheck"
+make -s distcheck >/dev/null
+
+echo "* Make dist"
+make -s dist >/dev/null
+make -s dist-xz >/dev/null
+
+while true; do
+  echo "Alpha or Regular release ? [r/A] "
+  read x
+  if test "$x" = "r" || test "$x" = "R"; then
+    directory="/incoming/ftp"
+    break
+  elif test "$x" = "a" || test "$x" = "A"; then
+    directory="/incoming/alpha"
+    break
+  fi
+done
+
+x=$(gpg --list-secret-keys 2>/dev/null|grep ^sec|awk '{ print $2 }')
+echo $x
+if [ "$x" = "" ]; then
+  echo No signing key found - please check with 'gpg --list-secret-keys'
+  exit 1
+fi
+
+# arrays just work in Bash
+keys=($x)
+if [ ${#keys[@]} -eq 1 ]; then
+  IFS='/' arr=($x)
+  signing_key=${arr[1]}
+else
+  for ((i=0; i < ${#keys[@]}; i++)); do
+    echo "$((i+1))) ${keys[i]}"
+  done
+  while true; do
+    echo -n "Enter 1-${#keys[@]}: "
+    read x
+
+    # check for digits only
+    if [ -n "${i//[0-9]/}" ]; then continue; fi
+
+    if test $x -ge 1 || test $x -le ${#keys[@]}; then
+      IFS='/' arr=(${keys[$((x-1))]})
+      signing_key=${arr[1]}
+      break
+    fi
+  done
+fi
+unset IFS
+echo $signing_key
+
+# fetch app_version again, just in case
+app_ver=$(grep "^PACKAGE_VERSION='" configure|cut -d "'" -f2)
+
+for ext in $extensions; do
+  app_tarball="${app_name}-${app_ver}.tar.$ext"
+  gpg --default-key $signing_key --yes -b $app_tarball
+
+  echo -e "version: 1.2\nfilename: ${app_tarball}\ndirectory: ${app_name}\n" >${app_tarball}.directive
+  gpg --default-key $signing_key --yes --clearsign ${app_tarball}.directive
+
+ftp -v -n -i -p ftp-upload.gnu.org <<EOF
+user anonymous
+cd $directory
+put ${app_tarball}
+put ${app_tarball}.sig
+put ${app_tarball}.directive.asc
+EOF
+
+done
+
+prev_tag=`git tag|tail -2|head -1`
+echo "Don't forget to mention the following contributors in the announcement:"
+(
+  git log ${prev_tag}.. --format="%aN" --reverse
+  git log ${prev_tag}.. --reverse|grep -i reported|cut -d':' -f2|cut -d'<' -f1|cut -d' ' -f2-|tr -d '"'
+) | sort -u

doc/announcement_1.19.txt

@@ -0,0 +1,104 @@
+Hello,
+
+we are pleased to announce the new version of GNU wget 1.19.
+
+It comes with major improvements for Metalink, IDNA2008 for
+international domain names, an option to call external tools
+for fetching user/password, several bugfixes and improvements.
+
+Many thanks go to all the contributors and list activists !
+
+Contributors (from the git log):
+Ander Juaristi
+Dale R. Worley 
+Eli Zaretskii 
+Gisle Vanem 
+Giuseppe Scrivano
+Göran Uddeborg 
+Jann Horn 
+Jeffery To
+Jérémie Courrèges-Anglas 
+Liam R. Howlett
+losgrandes
+Matthew White
+Misra, Deapesh 
+Nikos Mavrogiannopoulos
+Noël Köthe
+Piotr Wajda
+Rahul Bedarkar
+Sean Burford
+Tim Rühsen
+Tobias Stoeckmann
+vijeth-aradhya
+
+
+The new version is available for download here:
+
+https://ftp.gnu.org/gnu/wget/wget-1.19.tar.gz
+https://ftp.gnu.org/gnu/wget/wget-1.19.tar.xz
+
+and the GPG detached signatures using the key 0x08302DB6A2670428:
+
+https://ftp.gnu.org/gnu/wget/wget-1.19.tar.gz.sig
+https://ftp.gnu.org/gnu/wget/wget-1.19.tar.xz.sig
+
+To reduce load on the main server, you can use this redirector service
+which automatically redirects you to a mirror:
+
+http://ftpmirror.gnu.org/wget/wget-1.19.tar.gz
+http://ftpmirror.gnu.org/wget/wget-1.19.tar.xz
+
+
+Noteworthy changes:
+
+* New option --use-askpass=COMMAND. Fetch user/password by calling
+  an external program.
+
+* Use IDNA2008 (+ TR46 if available) through libidn2
+
+* When processing a Metalink header, --metalink-index=<number> allows
+  to process the header's application/metalink4+xml files.
+
+* When processing a Metalink file, --trust-server-names enables the
+  use of the destination file names specified in the Metalink file,
+  otherwise a safe destination file name is computed.
+
+* When processing a Metalink file, enforce a safe destination path.
+  Remove any drive letter prefix under w32, i.e. 'C:D:file'.  Call
+  libmetalink's metalink_check_safe_path() to prevent absolute,
+  relative, or home paths:
+  https://tools.ietf.org/html/rfc5854#section-4.1.2.1
+  https://tools.ietf.org/html/rfc5854#section-4.2.8.3
+
+* When processing a Metalink file, --directory-prefix=<prefix> sets
+  the top of the retrieval tree to prefix for Metalink downloads.
+
+* When processing a Metalink file, reject downloaded files which don't
+  agree with their own metalink:size value:
+  https://tools.ietf.org/html/rfc5854#section-4.2.16
+
+* When processing a Metalink file, with --continue resume partially
+  downloaded files and keep fully downloaded files even if they fail
+  the verification.
+
+* When processing a Metalink file, create the parent directories of a
+  "path/file" destination file name:
+  https://tools.ietf.org/html/rfc5854#section-4.1.2.1
+  https://tools.ietf.org/html/rfc5854#section-4.2.8.3
+
+* On a recursive download, append a .tmp suffix to temporary files
+  that will be deleted after being parsed, and create them
+  readable/writable only by the owner.
+
+* New make target 'check-valgrind'
+
+* Fix several bugs
+
+* Fix compatibility issues
+
+
+Please report any problem you may experience to the bug-wget@gnu.org
+mailing list.
+
+For the maintainers of Wget,
+Tim

doc/wget.texi

@@ -1718,6 +1718,21 @@ some few obscure servers, which never send HTTP authentication
 challenges, but accept unsolicited auth info, say, in addition to
 form-based authentication.
 
+@item --retry-on-http-error=@var{code[,code,...]}
+Consider given HTTP response codes as non-fatal, transient errors.
+Supply a comma-separated list of 3-digit HTTP response codes as
+argument. Useful to work around special circumstances where retries
+are required, but the server responds with an error code normally not
+retried by Wget. Such errors might be 503 (Service Unavailable) and
+429 (Too Many Requests). Retries enabled by this option are performed
+subject to the normal retry timing and retry count limitations of
+Wget.
+
+Using this option is intended to support special use cases only and is
+generally not recommended, as it can force retries even in cases where
+the server is actually trying to decrease its load. Please use wisely
+and only if you know what you are doing.
+
 @end table
 
 @node HTTPS (SSL/TLS) Options, FTP Options, HTTP Options, Invoking

src/Makefile.am

@@ -65,8 +65,8 @@ nodist_wget_SOURCES = version.c
 EXTRA_wget_SOURCES = iri.c
 LDADD = $(LIBOBJS) ../lib/libgnu.a $(GETADDRINFO_LIB) $(HOSTENT_LIB)\
  $(INET_NTOP_LIB) $(LIBSOCKET) $(LIB_CLOCK_GETTIME) $(LIB_CRYPTO)\
- $(LIB_NANOSLEEP) $(LIB_POSIX_SPAWN) $(LIB_SELECT) $(LTLIBICONV) $(LTLIBINTL)\
- $(LTLIBTHREAD) $(LTLIBUNISTRING) $(SERVENT_LIB)
+ $(LIB_NANOSLEEP) $(LIB_POSIX_SPAWN) $(LIB_SELECT) $(LIBICONV) $(LIBINTL)\
+ $(LIBTHREAD) $(LIBUNISTRING) $(SERVENT_LIB)
 AM_CPPFLAGS = -I$(top_builddir)/lib -I$(top_srcdir)/lib
 
 

src/ftp.c

@@ -1463,7 +1463,7 @@ Error in server response, closing control connection.\n"));
       else if (opt.noclobber || opt.always_rest || opt.timestamping || opt.dirstruct
                || opt.output_document || count > 0)
         {
-          if (opt.unlink && file_exists_p (con->target))
+          if (opt.unlink_requested && file_exists_p (con->target))
             {
               if (unlink (con->target) < 0)
                 {

src/http.c

@@ -1900,7 +1900,7 @@ initialize_request (const struct url *u, struct http_stat *hs, int *dt, struct u
     *passwd = NULL;
 
   /* Check for ~/.netrc if none of the above match */
-  if (opt.netrc && (!user || !passwd))
+  if (opt.netrc && (!user || (!passwd || !*passwd)))
     search_netrc (u->host, (const char **) user, (const char **) passwd, 0);
 
   /* We only do "site-wide" authentication with "global" user/password
@@ -2486,7 +2486,7 @@ open_output_stream (struct http_stat *hs, int count, FILE **fp)
         }
       else if (ALLOW_CLOBBER || count > 0)
         {
-          if (opt.unlink && file_exists_p (hs->local_file))
+          if (opt.unlink_requested && file_exists_p (hs->local_file))
             {
               if (unlink (hs->local_file) < 0)
                 {
@@ -3982,6 +3982,22 @@ gethttp (const struct url *u, struct url *original_url, struct http_stat *hs,
   return retval;
 }
 
+/* Check whether the supplied HTTP status code is among those
+   listed for the --retry-on-http-error option. */
+static bool
+check_retry_on_http_error (const int statcode)
+{
+  const char *tok = opt.retry_on_http_error;
+  while (tok && *tok)
+    {
+      if (atoi (tok) == statcode)
+        return true;
+      if ((tok = strchr (tok, ',')))
+        ++tok;
+    }
+  return false;
+}
+
 /* The genuine HTTP loop!  This is the part where the retrieval is
    retried, and retried, and retried, and...  */
 uerr_t
@@ -4319,6 +4335,11 @@ http_loop (const struct url *u, struct url *original_url, char **newloc,
               logprintf (LOG_NOTQUIET, _("\
 Remote file does not exist -- broken link!!!\n"));
             }
+          else if (check_retry_on_http_error (hstat.statcode))
+            {
+              printwhat (count, opt.ntry);
+              continue;
+            }
           else
             {
               logprintf (LOG_NOTQUIET, _("%s ERROR %d: %s.\n"),

src/init.c

@@ -304,6 +304,7 @@ static const struct {
   { "restrictfilenames", NULL,                  cmd_spec_restrict_file_names },
   { "retrsymlinks",     &opt.retr_symlinks,     cmd_boolean },
   { "retryconnrefused", &opt.retry_connrefused, cmd_boolean },
+  { "retryonhttperror", &opt.retry_on_http_error, cmd_string },
   { "robots",           &opt.use_robots,        cmd_boolean },
   { "savecookies",      &opt.cookies_output,    cmd_file },
   { "saveheaders",      &opt.save_headers,      cmd_boolean },
@@ -321,7 +322,7 @@ static const struct {
   { "timestamping",     &opt.timestamping,      cmd_boolean },
   { "tries",            &opt.ntry,              cmd_number_inf },
   { "trustservernames", &opt.trustservernames,  cmd_boolean },
-  { "unlink",           &opt.unlink,            cmd_boolean },
+  { "unlink",           &opt.unlink_requested,  cmd_boolean },
   { "useaskpass" ,      &opt.use_askpass,       cmd_use_askpass },
   { "useproxy",         &opt.use_proxy,         cmd_boolean },
   { "user",             &opt.user,              cmd_string },
@@ -1979,6 +1980,7 @@ cleanup (void)
   xfree (opt.body_file);
   xfree (opt.rejected_log);
   xfree (opt.use_askpass);
+  xfree (opt.retry_on_http_error);
 
 #ifdef HAVE_LIBCARES
 #include <ares.h>

src/iri.c

@@ -286,6 +286,8 @@ idn_encode (const struct iri *i, const char *host)
   xfree (lower);
 #endif
 
+  xfree (utf8_encoded);
+
   return ret == IDN2_OK ? ascii_encoded : NULL;
 }
 

src/main.c

@@ -404,6 +404,7 @@ static struct cmdline_option option_data[] =
     { "restrict-file-names", 0, OPT_BOOLEAN, "restrictfilenames", -1 },
     { "retr-symlinks", 0, OPT_BOOLEAN, "retrsymlinks", -1 },
     { "retry-connrefused", 0, OPT_BOOLEAN, "retryconnrefused", -1 },
+    { "retry-on-http-error", 0, OPT_VALUE, "retryonhttperror", -1 },
     { "save-cookies", 0, OPT_VALUE, "savecookies", -1 },
     { "save-headers", 0, OPT_BOOLEAN, "saveheaders", -1 },
     { IF_SSL ("secure-protocol"), 0, OPT_VALUE, "secureprotocol", -1 },

src/options.h

@@ -43,6 +43,7 @@ struct options
   bool quiet;                   /* Are we quiet? */
   int ntry;                     /* Number of tries per URL */
   bool retry_connrefused;       /* Treat CONNREFUSED as non-fatal. */
+  char *retry_on_http_error;    /* Treat given HTTP errors as non-fatal. */
   bool background;              /* Whether we should work in background. */
   bool ignore_length;           /* Do we heed content-length at all?  */
   bool recursive;               /* Are we recursive? */
@@ -61,7 +62,7 @@ struct options
   bool add_hostdir;             /* Do we add hostname directory? */
   bool protocol_directories;    /* Whether to prepend "http"/"ftp" to dirs. */
   bool noclobber;               /* Disables clobbering of existing data. */
-  bool unlink;                  /* remove file before clobbering */
+  bool unlink_requested;        /* remove file before clobbering */
   char *dir_prefix;             /* The top of directory tree */
   char *lfilename;              /* Log filename */
   char *input_filename;         /* Input filename */

src/xattr.h

@@ -30,14 +30,14 @@ int set_file_metadata (const char *origin_url, const char *referrer_url, FILE *f
 #elif defined(__APPLE__)
 /* libc on OS/X has fsetxattr (6 arguments). */
 #  include <sys/xattr.h>
-#  define fsetxattr (file, name, buffer, size, flags) \
+#  define fsetxattr(file, name, buffer, size, flags) \
           fsetxattr ((file), (name), (buffer), (size), 0, (flags))
 #  define USE_XATTR
 #elif defined(__FreeBSD_version) && (__FreeBSD_version > 500000)
 /* FreeBSD */
 #  include <sys/types.h>
 #  include <sys/extattr.h>
-#  define fsetxattr (file, name, buffer, size, flags) \
+#  define fsetxattr(file, name, buffer, size, flags) \
           extattr_set_fd ((file), EXTATTR_NAMESPACE_USER, (name), (buffer), (size))
 #  define USE_XATTR
 #endif

tests/Makefile.am

@@ -138,8 +138,8 @@ check_PROGRAMS = unit-tests
 unit_tests_SOURCES =
 LDADD = ../src/libunittest.a ../lib/libgnu.a $(GETADDRINFO_LIB) $(HOSTENT_LIB)\
  $(INET_NTOP_LIB) $(LIBSOCKET) $(LIB_CLOCK_GETTIME) $(LIB_CRYPTO)\
- $(LIB_NANOSLEEP) $(LIB_POSIX_SPAWN) $(LIB_SELECT) $(LTLIBICONV) $(LTLIBINTL)\
- $(LTLIBTHREAD) $(LTLIBUNISTRING) $(SERVENT_LIB)
+ $(LIB_NANOSLEEP) $(LIB_POSIX_SPAWN) $(LIB_SELECT) $(LIBICONV) $(LIBINTL)\
+ $(LIBTHREAD) $(LIBUNISTRING) $(SERVENT_LIB)
 
 
 CLEANFILES = *~ *.bak core core.[0-9]*

tests/WgetTests.pm

@@ -13,7 +13,7 @@ use IO::Handle;
 use POSIX qw(locale_h);
 use locale;
 
-our $WGETPATH = '../src/wget';
+our $WGETPATH = '../src/wget --no-config';
 our $VALGRIND_SUPP_FILE = Cwd::getcwd();
 if (defined $ENV{'srcdir'}) {
     $VALGRIND_SUPP_FILE = $VALGRIND_SUPP_FILE
@@ -359,8 +359,9 @@ sub _verify_download
               );
     if (@unexpected_downloads)
     {
-        return 'Test failed: unexpected downloaded files [' . join ', ',
-          @unexpected_downloads . "]\n";
+        return 'Test failed: unexpected downloaded files [' .
+          (join ', ', @unexpected_downloads) . "]\n";
+
     }
 
     return q{};