Prepare new release 1.16

When Wget retrieves a file through FTP, it first downloads a .listing
file and parses it for information about the files and other metadata.
Some servers may serve invalid .listing files. This patch checks for one
such known inconsistency wherein multiple lines in a listing file have
the same name. Such a filesystem is clearly not possible and hence we
eliminate duplicate entries here.

Signed-off-by: Darshit Shah <darnir@gmail.com>

.gitignore

@@ -1,7 +1,19 @@
 # Project Root
+# Wildcard Ignores
+# build-aux/
+# doc/
+# m4/
+# others
+# po/
+# src/
+# testenv/
+# tests/
+# util/
+*~
 /.sc-start-sc_bindtextdomain
 /.sc-start-sc_prohibit_HAVE_MBRTOWC
 /.version
+/ABOUT-NLS~
 /GNUmakefile
 /INSTALL
 /Makefile
@@ -12,16 +24,10 @@
 /config.log
 /config.status
 /configure
+/cscope.out
 /lib/
 /maint.mk
-# Wildcard Ignores
-*~
-po/*.gmo*
-po/*.po*
-src/*.o
-testenv/*.log
-testenv/*.trs
-# build-aux/
+/tags
 build-aux/.gitignore
 build-aux/compile
 build-aux/config.guess
@@ -33,7 +39,6 @@ build-aux/missing
 build-aux/snippet/
 build-aux/test-driver
 build-aux/texinfo.tex
-# doc/
 doc/Makefile
 doc/Makefile.in
 doc/sample.wgetrc.munged_for_texi_inclusion
@@ -42,10 +47,10 @@ doc/version.texi
 doc/wget.1
 doc/wget.info
 doc/wget.pod
-# m4/
 m4/.gitignore
 m4/gnulib-cache.m4
-# po/
+po/*.gmo*
+po/*.po*
 po/.gitignore
 po/.reference/
 po/LINGUAS
@@ -54,7 +59,7 @@ po/Makefile.in
 po/Makevars
 po/POTFILES
 po/wget.pot
-# src/
+src/*.o
 src/.deps/
 src/Makefile
 src/Makefile.in
@@ -69,23 +74,20 @@ src/stamp-h1
 src/tags
 src/version.c
 src/wget
-# testenv/
+testenv/*.log
+testenv/*.trs
+testenv/Makefile
+testenv/Makefile.in
 testenv/conf/__pycache__/
 testenv/exc/__pycache__/
 testenv/misc/__pycache__/
 testenv/server/__pycache__/
 testenv/server/http/__pycache__/
 testenv/test/__pycache__/
-testenv/Makefile
-testenv/Makefile.in
-# tests/
+tests/*.log
+tests/*.trs
 tests/Makefile
 tests/Makefile.in
-tests/WgetTest.pm
 tests/unit-tests
-# util/
 util/Makefile
 util/Makefile.in
-# others
-/tags
-/cscope.out

ABOUT-NLS

@@ -1065,3 +1065,4 @@ developed inside the GNU project.  Therefore the information given above
 applies also for every other Free Software Project.  Contact
 `coordinator@translationproject.org' to make the `.pot' files available
 to the translation teams.
+

ChangeLog

@@ -1,3 +1,16 @@
+2013-10-22  Ángel González <keisial@gmail.com>
+
+	* bootstrap.conf (gnulib_modules): Add module xstrndup.
+
+2014-09-25  Tim Ruehsen <tim.ruehsen@gmx.de>
+
+	* configure.ac: removed WgetTest.pm.in
+	* .gitignore: removed WgetTest.pm
+
+2014-09-12  Darshit Shah  <darnir@gmail.com>
+
+	* bootstrap.conf: Add GNULib module mbiter
+
 2014-07-25  Darshit Shah  <darnir@gmail.com>
 
 	* .gitignore: Add a gitignore file for the project.

NEWS

@@ -6,15 +6,17 @@ See the end for copying conditions.
 
 Please send GNU Wget bug reports to <bug-wget@gnu.org>.
 
-* Changes in Wget X.Y.Z
+* Changes in Wget 1.16
 
-** Use libpsl for verifying cookie domains
+** No longer create local symbolic links by default.  Closes CVE-2014-4877.
 
-** Default progress bar output changed
+** Use libpsl for verifying cookie domains.
 
-** Introduce --show-progress to force display the progress bar
+** Default progress bar output changed.
 
-** Introduce --no-config.
+** Introduce --show-progress to force display the progress bar.
+
+** Introduce --no-config.  The wgetrc files will not be read.
 
 ** Introduce --start-pos to allow starting downloads from a specified position.
 

bootstrap.conf

@@ -49,6 +49,7 @@ iconv
 iconv-h
 listen
 maintainer-makefile
+mbiter
 mbtowc
 mkdir
 mkstemp
@@ -77,6 +78,7 @@ update-copyright
 vasprintf
 vsnprintf
 write
+xstrndup
 "
 
 gnulib_extra_files="

configure.ac

@@ -578,7 +578,7 @@ dnl
 dnl Create output
 dnl
 AC_CONFIG_FILES([Makefile src/Makefile doc/Makefile util/Makefile
-                 po/Makefile.in tests/Makefile tests/WgetTest.pm
+                 po/Makefile.in tests/Makefile
                  lib/Makefile testenv/Makefile])
 AC_CONFIG_HEADERS([src/config.h])
 AC_OUTPUT

doc/ChangeLog

@@ -1,3 +1,13 @@
+2014-09-08  Darshit Shah  <darnir@gmail.com>
+
+	* wget.texi (symbolic links): Update documentation of retr-symlinks to
+	reflect the new default. Add warning about potential security issues with
+	--retr-symlinks=yes.
+
+2014-10-16  Tim Ruehsen  <tim.ruehsen@gmx.de>
+
+	* wget.texi (Download Options): update --secure-protocol description
+
 2014-08-03  Giuseppe Scrivano  <gscrivano@gnu.org>
 
 	* wget.texi (Download Options): Fix texinfo warning.

doc/wget.texi

@@ -1641,16 +1641,16 @@ without SSL support, none of these options are available.
 @cindex SSL protocol, choose
 @item --secure-protocol=@var{protocol}
 Choose the secure protocol to be used.  Legal values are @samp{auto},
-@samp{SSLv2}, @samp{SSLv3}, @samp{TLSv1} and @samp{PFS}.  If @samp{auto}
-is used, the SSL library is given the liberty of choosing the appropriate
-protocol automatically, which is achieved by sending an SSLv2 greeting
-and announcing support for SSLv3 and TLSv1.  This is the default.
+@samp{SSLv2}, @samp{SSLv3}, @samp{TLSv1}, @samp{TLSv1_1}, @samp{TLSv1_2}
+and @samp{PFS}.  If @samp{auto} is used, the SSL library is given the
+liberty of choosing the appropriate protocol automatically, which is
+achieved by sending a TLSv1 greeting. This is the default.
 
-Specifying @samp{SSLv2}, @samp{SSLv3}, or @samp{TLSv1} forces the use
-of the corresponding protocol.  This is useful when talking to old and
-buggy SSL server implementations that make it hard for the underlying
-SSL library to choose the correct protocol version.  Fortunately, such
-servers are quite rare.
+Specifying @samp{SSLv2}, @samp{SSLv3}, @samp{TLSv1}, @samp{TLSv1_1} or
+@samp{TLSv1_2} forces the use of the corresponding protocol.  This is
+useful when talking to old and buggy SSL server implementations that
+make it hard for the underlying SSL library to choose the correct
+protocol version.  Fortunately, such servers are quite rare.
 
 Specifying @samp{PFS} enforces the use of the so-called Perfect Forward
 Security cipher suites. In short, PFS adds security by creating a one-time
@@ -1883,17 +1883,18 @@ Preserve remote file permissions instead of permissions set by umask.
 
 @cindex symbolic links, retrieving
 @item --retr-symlinks
-Usually, when retrieving @sc{ftp} directories recursively and a symbolic
-link is encountered, the linked-to file is not downloaded.  Instead, a
-matching symbolic link is created on the local filesystem.  The
-pointed-to file will not be downloaded unless this recursive retrieval
-would have encountered it separately and downloaded it anyway.
+By default, when retrieving @sc{ftp} directories recursively and a symbolic link
+is encountered, the symbolic link is traversed and the pointed-to files are
+retrieved.  Currently, Wget does not traverse symbolic links to directories to
+download them recursively, though this feature may be added in the future.
 
-When @samp{--retr-symlinks} is specified, however, symbolic links are
-traversed and the pointed-to files are retrieved.  At this time, this
-option does not cause Wget to traverse symlinks to directories and
-recurse through them, but in the future it should be enhanced to do
-this.
+When @samp{--retr-symlinks=no} is specified, the linked-to file is not
+downloaded.  Instead, a matching symbolic link is created on the local
+filesystem.  The pointed-to file will not be retrieved unless this recursive
+retrieval would have encountered it separately and downloaded it anyway.  This
+option poses a security risk where a malicious FTP Server may cause Wget to
+write to files outside of the intended directories through a specially crafted
+@sc{.listing} file.
 
 Note that when retrieving a file (not a directory) because it was
 specified on the command-line, rather than because it was recursed to,

src/ChangeLog

@@ -1,3 +1,41 @@
+2014-09-08  Darshit Shah  <darnir@gmail.com>
+
+	* ftp.c (ftp_retrieve_glob): Also check for invalid entries along with
+	harmful filenames
+	(is_valid_entry): New function. Check if the provided node is a valid entry
+	in a listing file.
+
+2014-09-08  Darshit Shah  <darnir@gmail.com>
+
+	* init.c (defaults): Set retr-symlinks to true by default. This changes a
+	default setting of wget. Fixes security bug CVE-2014-4877
+
+2014-10-08 Nikolay Morozov <n.morozov@securitycode.ru>
+	   Sergey Lvov <s.lvov@securitycode.ru>
+
+	* init.c (cmd_spec_secure_protocol): Add support for
+	TLS v1.1 and TLS v1.2 protocols
+	* openssl.c (ssl_init): Add support for OpenSSL engines
+
+2014-10-22  Ángel González <keisial@gmail.com>
+
+	* css-url.c (get_uri_string): Honor the specified length argument.
+
+2014-10-16  Tim Ruehsen  <tim.ruehsen@gmx.de>
+
+	* url.c (url_parse): little code cleanup
+	* html-url.c (get_urls_html): HTTP header Content-Type charset preceeds http-equiv
+	* iri.c (do_conversion): moved iconv code completely into the function
+	* iri.c (do_conversion): call url_unescape to fix charset conversion
+	* iri.c (remote_to_utf8): use strcasecmp to compare encoding
+
+	* gnutls.c (ssl_connect_wget): do not use SSLv3 except explicitely requested
+	* openssl.c (ssl_init): do not use SSLv3 except explicitely requested
+
+2014-05-03  Tim Ruehsen  <tim.ruehsen@gmx.de>
+
+	* retr.c (retrieve_url): fixed memory leak
+
 2014-07-23  Darshit Shah  <darnir@gmail.com>
 
 	* http.c (gethttp): Fix a memory leak when retrying authorization

src/css-url.c

@@ -52,6 +52,7 @@ as that of the covered work.  */
 #include "html-url.h"
 #include "css-tokens.h"
 #include "css-url.h"
+#include "xstrndup.h"
 
 /* from lex.yy.c */
 extern char *yytext;
@@ -97,7 +98,7 @@ get_uri_string (const char *at, int *pos, int *length)
       *length -= 2;
     }
 
-  return xstrdup (at + *pos);
+  return xstrndup (at + *pos, *length);
 }
 
 void

src/ftp.c

@@ -2211,6 +2211,29 @@ has_insecure_name_p (const char *s)
   return false;
 }
 
+/* Test if the file node is invalid. This can occur due to malformed or
+ * maliciously crafted listing files being returned by the server.
+ *
+ * Currently, this function only tests if there are multiple entries in the
+ * listing file by the same name. However this function can be expanded as more
+ * such illegal listing formats are discovered. */
+static bool
+is_invalid_entry (struct fileinfo *f)
+{
+  struct fileinfo *cur;
+  cur = f;
+  char *f_name = f->name;
+  /* If the node we're currently checking has a duplicate later, we eliminate
+   * the current node and leave the next one intact. */
+  while (cur->next)
+    {
+      cur = cur->next;
+      if (strcmp(f_name, cur->name) == 0)
+          return true;
+    }
+  return false;
+}
+
 /* A near-top-level function to retrieve the files in a directory.
    The function calls ftp_get_listing, to get a linked list of files.
    Then it weeds out the file names that do not match the pattern.
@@ -2248,11 +2271,11 @@ ftp_retrieve_glob (struct url *u, ccon *con, int action)
             f = f->next;
         }
     }
-  /* Remove all files with possible harmful names */
+  /* Remove all files with possible harmful names or invalid entries. */
   f = start;
   while (f)
     {
-      if (has_insecure_name_p (f->name))
+      if (has_insecure_name_p (f->name) || is_invalid_entry (f))
         {
           logprintf (LOG_VERBOSE, _("Rejecting %s.\n"),
                      quote (f->name));

src/gnutls.c

@@ -433,6 +433,7 @@ ssl_connect_wget (int fd, const char *hostname)
   switch (opt.secure_protocol)
     {
     case secure_protocol_auto:
+      err = gnutls_priority_set_direct (session, "NORMAL:%COMPAT:-VERS-SSL3.0", NULL);
       break;
     case secure_protocol_sslv2:
     case secure_protocol_sslv3:
@@ -442,10 +443,10 @@ ssl_connect_wget (int fd, const char *hostname)
       err = gnutls_priority_set_direct (session, "NORMAL:-VERS-SSL3.0", NULL);
       break;
     case secure_protocol_pfs:
-      err = gnutls_priority_set_direct (session, "PFS", NULL);
+      err = gnutls_priority_set_direct (session, "PFS:-VERS-SSL3.0", NULL);
       if (err != GNUTLS_E_SUCCESS)
         /* fallback if PFS is not available */
-        err = gnutls_priority_set_direct (session, "NORMAL:-RSA", NULL);
+        err = gnutls_priority_set_direct (session, "NORMAL:-RSA:-VERS-SSL3.0", NULL);
       break;
     default:
       abort ();

src/html-url.c

@@ -748,8 +748,9 @@ get_urls_html (const char *file, const char *url, bool *meta_disallow_follow,
   map_html_tags (fm->content, fm->length, collect_tags_mapper, &ctx, flags,
                  NULL, interesting_attributes);
 
-  /* If meta charset isn't null, override content encoding */
-  if (iri && meta_charset)
+  /* Meta charset is only valid if there was no HTTP header Content-Type charset. */
+  /* This is true for HTTP 1.0 and 1.1. */
+  if (iri && !iri->content_encoding && meta_charset)
     set_content_encoding (iri, meta_charset);
 
   DEBUGP (("no-follow in %s: %d\n", file, ctx.nofollow));

src/init.c

@@ -366,6 +366,22 @@ defaults (void)
 
   opt.dns_cache = true;
   opt.ftp_pasv = true;
+  /* 2014-09-07  Darshit Shah  <darnir@gmail.com>
+   * opt.retr_symlinks is set to true by default. Creating symbolic links on the
+   * local filesystem pose a security threat by malicious FTP Servers that
+   * server a specially crafted .listing file akin to this:
+   *
+   * lrwxrwxrwx   1 root     root           33 Dec 25  2012 JoCxl6d8rFU -> /
+   * drwxrwxr-x  15 1024     106          4096 Aug 28 02:02 JoCxl6d8rFU
+   *
+   * A .listing file in this fashion makes Wget susceptiple to a symlink attack
+   * wherein the attacker is able to create arbitrary files, directories and
+   * symbolic links on the target system and even set permissions.
+   *
+   * Hence, by default Wget attempts to retrieve the pointed-to files and does
+   * not create the symbolic links locally.
+   */
+  opt.retr_symlinks = true;
 
 #ifdef HAVE_SSL
   opt.check_cert = true;
@@ -1498,6 +1514,8 @@ cmd_spec_secure_protocol (const char *com, const char *val, void *place)
     { "sslv2", secure_protocol_sslv2 },
     { "sslv3", secure_protocol_sslv3 },
     { "tlsv1", secure_protocol_tlsv1 },
+    { "tlsv1_1", secure_protocol_tlsv1_1 },
+    { "tlsv1_2", secure_protocol_tlsv1_2 },
     { "pfs", secure_protocol_pfs },
   };
   int ok = decode_string (val, choices, countof (choices), place);

src/iri.c

@@ -38,15 +38,13 @@ as that of the covered work.  */
 #include <errno.h>
 
 #include "utils.h"
+#include "url.h"
 
 /* RFC3987 section 3.1 mandates STD3 ASCII RULES */
 #define IDNA_FLAGS  IDNA_USE_STD3_ASCII_RULES
 
 /* Note: locale encoding is kept in options struct (opt.locale) */
 
-static bool do_conversion (iconv_t cd, char *in, size_t inlen, char **out);
-
-
 /* Given a string containing "charset=XXX", return the encoding if found,
    or NULL otherwise */
 char *
@@ -110,52 +108,34 @@ check_encoding_name (char *encoding)
   return true;
 }
 
-/* Try converting string str from locale to UTF-8. Return a new string
-   on success, or str on error or if conversion isn't needed. */
-const char *
-locale_to_utf8 (const char *str)
-{
-  iconv_t l2u;
-  char *new;
-
-  /* That shouldn't happen, just in case */
-  if (!opt.locale)
-    {
-      logprintf (LOG_VERBOSE, _("locale_to_utf8: locale is unset\n"));
-      opt.locale = find_locale ();
-    }
-
-  if (!opt.locale || !strcasecmp (opt.locale, "utf-8"))
-    return str;
-
-  l2u = iconv_open ("UTF-8", opt.locale);
-  if (l2u == (iconv_t)(-1))
-    {
-      logprintf (LOG_VERBOSE, _("Conversion from %s to %s isn't supported\n"),
-                 quote (opt.locale), quote ("UTF-8"));
-      return str;
-    }
-
-  if (do_conversion (l2u, (char *) str, strlen ((char *) str), &new))
-    return (const char *) new;
-
-  return str;
-}
-
 /* Do the conversion according to the passed conversion descriptor cd. *out
    will contain the transcoded string on success. *out content is
    unspecified otherwise. */
 static bool
-do_conversion (iconv_t cd, char *in, size_t inlen, char **out)
+do_conversion (const char *tocode, const char *fromcode, char *in, size_t inlen, char **out)
 {
+  iconv_t cd;
   /* sXXXav : hummm hard to guess... */
-  size_t len, done, outlen = inlen * 2;
+  size_t len, done, outlen;
   int invalid = 0, tooshort = 0;
-  char *s;
+  char *s, *in_org, *in_save;
 
-  s = xmalloc (outlen + 1);
-  *out = s;
-  len = outlen;
+  cd = iconv_open (tocode, fromcode);
+  if (cd == (iconv_t)(-1))
+    {
+      logprintf (LOG_VERBOSE, _("Conversion from %s to %s isn't supported\n"),
+                 quote (opt.locale), quote ("UTF-8"));
+      return false;
+    }
+
+  /* iconv() has to work on an unescaped string */
+  in_org = in;
+  in_save = in = strndup(in, inlen);
+  url_unescape(in);
+  inlen = strlen(in);
+
+  len = outlen = inlen * 2;
+  *out = s = xmalloc (outlen + 1);
   done = 0;
 
   for (;;)
@@ -164,6 +144,9 @@ do_conversion (iconv_t cd, char *in, size_t inlen, char **out)
         {
           *out = s;
           *(s + len - outlen - done) = '\0';
+          xfree(in_save);
+          iconv_close(cd);
+          logprintf (LOG_VERBOSE, _("converted '%s' (%s) -> '%s' (%s)\n"), in_org, fromcode, *out, tocode);
           return true;
         }
 
@@ -202,9 +185,35 @@ do_conversion (iconv_t cd, char *in, size_t inlen, char **out)
         }
     }
 
+    xfree(in_save);
+    iconv_close(cd);
+    logprintf (LOG_VERBOSE, _("converted '%s' (%s) -> '%s' (%s)\n"), in_org, fromcode, *out, tocode);
     return false;
 }
 
+/* Try converting string str from locale to UTF-8. Return a new string
+   on success, or str on error or if conversion isn't needed. */
+const char *
+locale_to_utf8 (const char *str)
+{
+  char *new;
+
+  /* That shouldn't happen, just in case */
+  if (!opt.locale)
+    {
+      logprintf (LOG_VERBOSE, _("locale_to_utf8: locale is unset\n"));
+      opt.locale = find_locale ();
+    }
+
+  if (!opt.locale || !strcasecmp (opt.locale, "utf-8"))
+    return str;
+
+  if (do_conversion ("UTF-8", opt.locale, (char *) str, strlen ((char *) str), &new))
+    return (const char *) new;
+
+  return str;
+}
+
 /* Try to "ASCII encode" UTF-8 host. Return the new domain on success or NULL
    on error. */
 char *
@@ -258,7 +267,6 @@ idn_decode (char *host)
 bool
 remote_to_utf8 (struct iri *iri, const char *str, const char **new)
 {
-  iconv_t cd;
   bool ret = false;
 
   if (!iri->uri_encoding)
@@ -267,7 +275,7 @@ remote_to_utf8 (struct iri *iri, const char *str, const char **new)
   /* When `i->uri_encoding' == "UTF-8" there is nothing to convert.  But we must
      test for non-ASCII symbols for correct hostname processing in `idn_encode'
      function. */
-  if (!strcmp (iri->uri_encoding, "UTF-8"))
+  if (!strcasecmp (iri->uri_encoding, "UTF-8"))
     {
       const char *p = str;
       for (p = str; *p; p++)
@@ -279,15 +287,9 @@ remote_to_utf8 (struct iri *iri, const char *str, const char **new)
       return false;
     }
 
-  cd = iconv_open ("UTF-8", iri->uri_encoding);
-  if (cd == (iconv_t)(-1))
-    return false;
-
-  if (do_conversion (cd, (char *) str, strlen ((char *) str), (char **) new))
+  if (do_conversion ("UTF-8", iri->uri_encoding, (char *) str, strlen (str), (char **) new))
     ret = true;
 
-  iconv_close (cd);
-
   /* Test if something was converted */
   if (!strcmp (str, *new))
     {

src/openssl.c

@@ -40,6 +40,9 @@ as that of the covered work.  */
 #include <openssl/x509v3.h>
 #include <openssl/err.h>
 #include <openssl/rand.h>
+#if OPENSSL_VERSION_NUMBER >= 0x00907000
+#include <openssl/conf.h>
+#endif
 
 #include "utils.h"
 #include "connect.h"
@@ -187,6 +190,12 @@ ssl_init (void)
       goto error;
     }
 
+#if OPENSSL_VERSION_NUMBER >= 0x00907000
+  OPENSSL_load_builtin_modules();
+  ENGINE_load_builtin_engines();
+  CONF_modules_load_file(NULL, NULL,
+      CONF_MFLAGS_DEFAULT_SECTION|CONF_MFLAGS_IGNORE_MISSING_FILE);
+#endif
   SSL_library_init ();
   SSL_load_error_strings ();
   SSLeay_add_all_algorithms ();
@@ -194,9 +203,6 @@ ssl_init (void)
 
   switch (opt.secure_protocol)
     {
-    case secure_protocol_auto:
-      meth = SSLv23_client_method ();
-      break;
 #ifndef OPENSSL_NO_SSL2
     case secure_protocol_sslv2:
       meth = SSLv2_client_method ();
@@ -205,10 +211,19 @@ ssl_init (void)
     case secure_protocol_sslv3:
       meth = SSLv3_client_method ();
       break;
+    case secure_protocol_auto:
     case secure_protocol_pfs:
     case secure_protocol_tlsv1:
       meth = TLSv1_client_method ();
       break;
+#if OPENSSL_VERSION_NUMBER >= 0x01001000
+    case secure_protocol_tlsv1_1:
+      meth = TLSv1_1_client_method ();
+      break;
+    case secure_protocol_tlsv1_2:
+      meth = TLSv1_2_client_method ();
+      break;
+#endif
     default:
       abort ();
     }

src/options.h

@@ -202,6 +202,8 @@ struct options
     secure_protocol_sslv2,
     secure_protocol_sslv3,
     secure_protocol_tlsv1,
+    secure_protocol_tlsv1_1,
+    secure_protocol_tlsv1_2,
     secure_protocol_pfs
   } secure_protocol;            /* type of secure protocol to use. */
   bool check_cert;              /* whether to validate the server's cert */

src/progress.c

@@ -37,6 +37,7 @@ as that of the covered work.  */
 #include <unistd.h>
 #include <signal.h>
 #include <wchar.h>
+#include <mbiter.h>
 
 #include "progress.h"
 #include "utils.h"
@@ -812,8 +813,37 @@ count_cols (const char *mbs)
     }
   return cols;
 }
+
+static int
+cols_to_bytes (const char *mbs, const int cols, int *ncols)
+{
+  int p_cols = 0, bytes = 0;
+  mbchar_t mbc;
+  mbi_iterator_t iter;
+  mbi_init (iter, mbs, strlen(mbs));
+  while (p_cols < cols && mbi_avail (iter))
+    {
+      mbc = mbi_cur (iter);
+      p_cols += mb_width (mbc);
+      /* The multibyte character has exceeded the total number of columns we
+       * have available. The remaining bytes will be padded with a space. */
+      if (p_cols > cols)
+        {
+          p_cols -= mb_width (mbc);
+          break;
+        }
+      bytes += mb_len (mbc);
+      mbi_advance (iter);
+    }
+  *ncols = p_cols;
+  return bytes;
+}
 #else
 # define count_cols(mbs) ((int)(strlen(mbs)))
+# define cols_to_bytes(mbs, cols, *ncols) do {  \
+    *ncols = cols;                              \
+    bytes = cols;                               \
+}while (0)
 #endif
 
 static const char *
@@ -873,7 +903,7 @@ get_eta (int *bcd)
 static void
 create_image (struct bar_progress *bp, double dl_total_time, bool done)
 {
-  const int MAX_FILENAME_LEN = bp->width / 4;
+  const int MAX_FILENAME_COLS = bp->width / 4;
   char *p = bp->buffer;
   wgint size = bp->initial_length + bp->count;
 
@@ -884,7 +914,7 @@ create_image (struct bar_progress *bp, double dl_total_time, bool done)
   int size_grouped_pad; /* Used to pad the field width for size_grouped. */
 
   struct bar_progress_hist *hist = &bp->hist;
-  int orig_filename_len = strlen (bp->f_download);
+  int orig_filename_cols = count_cols (bp->f_download);
 
   /* The progress bar should look like this:
      file xx% [=======>             ] nnn.nnK 12.34KB/s  eta 36m 51s
@@ -896,7 +926,7 @@ create_image (struct bar_progress *bp, double dl_total_time, bool done)
      It would be especially bad for the progress bar to be resized
      randomly.
 
-     "file "           - Downloaded filename      - MAX_FILENAME_LEN chars + 1
+     "file "           - Downloaded filename      - MAX_FILENAME_COLS chars + 1
      "xx% " or "100%"  - percentage               - 4 chars
      "[]"              - progress bar decorations - 2 chars
      " nnn.nnK"        - downloaded bytes         - 7 chars + 1
@@ -906,7 +936,7 @@ create_image (struct bar_progress *bp, double dl_total_time, bool done)
      "=====>..."       - progress bar             - the rest
   */
 
-#define PROGRESS_FILENAME_LEN  MAX_FILENAME_LEN + 1
+#define PROGRESS_FILENAME_LEN  MAX_FILENAME_COLS + 1
 #define PROGRESS_PERCENT_LEN   4
 #define PROGRESS_DECORAT_LEN   2
 #define PROGRESS_FILESIZE_LEN  7 + 1
@@ -924,24 +954,31 @@ create_image (struct bar_progress *bp, double dl_total_time, bool done)
   if (progress_size < 5)
     progress_size = 0;
 
-  if (orig_filename_len <= MAX_FILENAME_LEN)
+  if (orig_filename_cols <= MAX_FILENAME_COLS)
     {
-      int padding = MAX_FILENAME_LEN - orig_filename_len;
+      int padding = MAX_FILENAME_COLS - orig_filename_cols;
       sprintf (p, "%s ", bp->f_download);
-      p += orig_filename_len + 1;
+      p += orig_filename_cols + 1;
       for (;padding;padding--)
         *p++ = ' ';
     }
   else
     {
-      int offset;
+      int offset_cols;
+      int bytes_in_filename, offset_bytes, col;
+      int *cols_ret = &col;
 
-      if (((orig_filename_len > MAX_FILENAME_LEN) && !opt.noscroll) && !done)
-        offset = ((int) bp->tick) % (orig_filename_len - MAX_FILENAME_LEN);
+      if (((orig_filename_cols > MAX_FILENAME_COLS) && !opt.noscroll) && !done)
+        offset_cols = ((int) bp->tick) % (orig_filename_cols - MAX_FILENAME_COLS);
       else
-        offset = 0;
-      memcpy (p, bp->f_download + offset, MAX_FILENAME_LEN);
-      p += MAX_FILENAME_LEN;
+        offset_cols = 0;
+      offset_bytes = cols_to_bytes (bp->f_download, offset_cols, cols_ret);
+      bytes_in_filename = cols_to_bytes (bp->f_download + offset_bytes, MAX_FILENAME_COLS, cols_ret);
+      memcpy (p, bp->f_download + offset_bytes, bytes_in_filename);
+      p += bytes_in_filename;
+      int padding = MAX_FILENAME_COLS - *cols_ret;
+      for (;padding;padding--)
+          *p++ = ' ';
       *p++ = ' ';
     }
 

src/retr.c

@@ -913,6 +913,10 @@ retrieve_url (struct url * orig_parsed, const char *origurl, char **file,
 
       goto redirected;
     }
+  else
+    {
+      xfree(mynewloc);
+    }
 
   /* Try to not encode in UTF-8 if fetching failed */
   if (!(*dt & RETROKF) && iri->utf8_encode)

src/url.c

@@ -681,7 +681,6 @@ url_parse (const char *url, int *error, struct iri *iri, bool percent_encode)
   char *user = NULL, *passwd = NULL;
 
   const char *url_encoded = NULL;
-  char *new_url = NULL;
 
   int error_code;
 
@@ -695,29 +694,29 @@ url_parse (const char *url, int *error, struct iri *iri, bool percent_encode)
       goto error;
     }
 
+  url_encoded = url;
+
   if (iri && iri->utf8_encode)
     {
+      char *new_url = NULL;
+
       iri->utf8_encode = remote_to_utf8 (iri, iri->orig_url ? iri->orig_url : url, (const char **) &new_url);
       if (!iri->utf8_encode)
         new_url = NULL;
       else
         {
           iri->orig_url = xstrdup (url);
-          percent_encode = true;
+          url_encoded = reencode_escapes (new_url);
+          if (url_encoded != new_url)
+            xfree (new_url);
+          percent_encode = false;
         }
     }
 
-  /* XXX XXX Could that change introduce (security) bugs ???  XXX XXX*/
   if (percent_encode)
-    url_encoded = reencode_escapes (new_url ? new_url : url);
-  else
-    url_encoded = new_url ? new_url : url;
+    url_encoded = reencode_escapes (url);
 
   p = url_encoded;
-
-  if (new_url && url_encoded != new_url)
-    xfree (new_url);
-
   p += strlen (supported_schemes[scheme].leading_string);
   uname_b = p;
   p = url_skip_credentials (p);

testenv/ChangeLog

@@ -1,3 +1,28 @@
+2014-10-01  Darshit Shah  <darnir@gmail.com>
+
+	* Makefile.am: Run the tests in Python's Optimizedmode
+	* conf/__init__.py (gen_hook): Use try..except instead of if..else
+	* misc/color_terminal.py: System and check will not change while a test is
+	run. Do not test for them on every invokation of printer()
+	* server/http/http_server.py: The ssl and re modules are required by
+	specific functions. Load them lazily
+	(HTTPSServer.__init__): Lazy load ssl module here
+	(_handler.parse_range_header): Lazy load re module here
+	(_Handler.get_rule_list): get() can return a default value. Use it
+	(_Handler.guess_type): Same
+	(_Handler.is_authorized): Unused function artefact. Remove
+	(_Handler.reject_headers): Unused function artefact. Remove
+
+2014-10-08  Darshit Shah  <darnir@gmail.com>
+
+	* Makefile.am: Fix EXTRA_DIST variable for make distcheck
+	* server/http/http_server.py (HTTPServer.__init__):  Fix how CERTFILE is
+	found when running make dist / make distcheck.
+
+2014-09-30  Tim Ruehsen  <tim.ruehsen@gmx.de>
+
+	* test/base_test.py: Add --track-origins=yes to valgrind testing
+
 2014-08-08  Darshit Shah  <darnir@gmail.com>
 
 	* conf/__init__.py: Add extra newline according to PEP8

testenv/Makefile.am

@@ -27,7 +27,7 @@
 
 
 AUTOMAKE_OPTIONS = parallel-tests
-AM_TESTS_ENVIRONMENT = MAKE_CHECK=True; export MAKE_CHECK;
+AM_TESTS_ENVIRONMENT = export WGETRC=/dev/null; MAKE_CHECK=True; export MAKE_CHECK; export PYTHONPATH=$$PYTHONPATH:$(srcdir);
 TESTS = Test-auth-basic-fail.py             \
     Test-auth-basic.py                      \
     Test-auth-both.py                       \
@@ -44,39 +44,15 @@ TESTS = Test-auth-basic-fail.py             \
     Test-cookie-expires.py                  \
     Test-cookie.py                          \
     Test-Head.py                            \
-    Test--https.py							\
+    Test--https.py                          \
     Test-O.py                               \
     Test-Post.py                            \
     Test--spider-r.py
 
 XFAIL_TESTS = Test-auth-both.py
 
-LOG_COMPILER = python3
+EXTRA_DIST = certs conf exc misc server test README $(TESTS) $(XFAIL_TESTS)
 
-EXTRA_DIST = ColourTerm.py          \
-    FTPServer.py                \
-    HTTPServer.py               \
-    README                  \
-    Test--spider-r.py           \
-    Test--https.py				\
-    Test-Content-disposition-2.py       \
-    Test-Content-disposition.py     \
-    Test-Head.py                \
-    Test-O.py               \
-    Test-Parallel-Proto.py          \
-    Test-Post.py                \
-    Test-Proto.py               \
-    Test-auth-basic-fail.py         \
-    Test-auth-basic.py          \
-    Test-auth-both.py           \
-    Test-auth-digest.py         \
-    Test-auth-no-challenge-url.py       \
-    Test-auth-no-challenge.py       \
-    Test-auth-retcode.py            \
-    Test-auth-with-content-disposition.py   \
-    Test-c-full.py              \
-    Test-cookie-401.py          \
-    Test-cookie-domain-mismatch.py      \
-    Test-cookie-expires.py          \
-    Test-cookie.py              \
-    WgetTest.py
+TEST_EXTENSIONS = .py
+PY_LOG_COMPILER = python3
+AM_PY_LOG_FLAGS = -O

testenv/conf/__init__.py

@@ -3,7 +3,6 @@ import os
 # this file implements the mechanism of conf class auto-registration,
 # don't modify this file if you have no idea what you're doing
 
-
 def gen_hook():
     hook_table = {}
 
@@ -24,9 +23,9 @@ def gen_hook():
             return cls
 
     def find_hook(name):
-        if name in hook_table:
+        try:
             return hook_table[name]
-        else:
+        except:
             raise AttributeError
 
     return Wrapper, find_hook

testenv/misc/colour_terminal.py

@@ -25,14 +25,12 @@ T_COLORS = {
     'ENDC'   : '\033[0m'
 }
 
+system = True if platform.system() == 'Linux' else False
+check = False if getenv("MAKE_CHECK") == 'True' else True
 
 def printer (color, string):
-    if platform.system () == 'Linux':
-        if getenv ("MAKE_CHECK", "False") == "True":
-            print (string)
-        else:
-            print (T_COLORS.get (color) + string + T_COLORS.get ('ENDC'))
-
+    if system and check:
+        print (T_COLORS.get (color) + string + T_COLORS.get ('ENDC'))
     else:
         print (string)
 

testenv/server/http/http_server.py

@@ -7,8 +7,6 @@ from random import random
 from hashlib import md5
 import threading
 import socket
-import re
-import ssl
 import os
 
 
@@ -37,10 +35,10 @@ class HTTPSServer (StoppableHTTPServer):
     additional support for secure connections through SSL. """
 
     def __init__ (self, address, handler):
+        import ssl
         BaseServer.__init__ (self, address, handler)
-        print (os.getcwd())
-        CERTFILE = os.path.abspath(os.path.join('..', 'certs', 'wget-cert.pem'))
-        print (CERTFILE)
+        # step one up because test suite change directory away from $srcdir (don't do that !!!)
+        CERTFILE = os.path.abspath(os.path.join('..', os.getenv('srcdir', '.'), 'certs', 'wget-cert.pem'))
         fop = open (CERTFILE)
         print (fop.readline())
         self.socket = ssl.wrap_socket (
@@ -62,8 +60,7 @@ class _Handler (BaseHTTPRequestHandler):
     requests. """
 
     def get_rule_list (self, name):
-        r_list = self.rules.get (name) if name in self.rules else None
-        return r_list
+        return self.rules.get(name)
 
     # The defailt protocol version of the server we run is HTTP/1.1 not
     # HTTP/1.0 which is the default with the http.server module.
@@ -136,6 +133,7 @@ class _Handler (BaseHTTPRequestHandler):
     """ Helper functions for the Handlers. """
 
     def parse_range_header (self, header_line, length):
+        import re
         if header_line is None:
             return None
         if not header_line.startswith ("bytes="):
@@ -317,23 +315,6 @@ class _Handler (BaseHTTPRequestHandler):
         if is_auth is False:
             raise ServerError ("Unable to Authenticate")
 
-    def is_authorized (self):
-        is_auth = True
-        auth_rule = self.get_rule_list ('Authentication')
-        if auth_rule:
-            auth_header = self.headers.get ("Authorization")
-            req_auth = auth_rule.auth_type
-            if req_auth == "Both" or req_auth == "Both_inline":
-                auth_type = auth_header.split(' ')[0] if auth_header else req_auth
-            else:
-                auth_type = req_auth
-            assert hasattr (self, "authorize_" + auth_type)
-            is_auth = getattr (self, "authorize_" + auth_type) (auth_header, auth_rule)
-            if is_auth is False:
-                self.send_response (401)
-                self.send_challenge (auth_type)
-                self.finish_headers ()
-        return is_auth
 
     def ExpectHeader (self, header_obj):
         exp_headers = header_obj.headers
@@ -344,6 +325,7 @@ class _Handler (BaseHTTPRequestHandler):
                 self.finish_headers ()
                 raise ServerError ("Header " + header_line + " not found")
 
+
     def RejectHeader (self, header_obj):
         rej_headers = header_obj.headers
         for header_line in rej_headers:
@@ -353,18 +335,6 @@ class _Handler (BaseHTTPRequestHandler):
                 self.finish_headers ()
                 raise ServerError ("Header " + header_line + ' received')
 
-    def reject_headers (self):
-        rej_headers = self.get_rule_list ("RejectHeader")
-        if rej_headers:
-            rej_headers = rej_headers.headers
-            for header_line in rej_headers:
-                header_re = self.headers.get (header_line)
-                if header_re is not None and header_re == rej_headers[header_line]:
-                    self.send_error (400, 'Blacklisted Header was Sent')
-                    self.end_headers ()
-                    return False
-        return True
-
     def __log_request (self, method):
         req = method + " " + self.path
         self.server.request_headers.append (req)
@@ -440,11 +410,7 @@ class _Handler (BaseHTTPRequestHandler):
             ".css"   :   "text/css",
             ".html"  :   "text/html"
         }
-        if ext in extension_map:
-            return extension_map[ext]
-        else:
-            return "text/plain"
-
+        return extension_map.get(ext, "text/plain")
 
 class HTTPd (threading.Thread):
     server_class = StoppableHTTPServer

testenv/test/base_test.py

@@ -101,7 +101,7 @@ class BaseTest:
         wget_options = '--debug --no-config %s' % self.wget_options
 
         if os.getenv("VALGRIND_TESTS"):
-            valgrind_test = "valgrind --error-exitcode=301 --leak-check=full"
+            valgrind_test = "valgrind --error-exitcode=301 --leak-check=full --track-origins=yes"
         else:
             valgrind_test = ""
         cmd_line = '%s %s %s ' % (valgrind_test, wget_path, wget_options)
@@ -116,9 +116,8 @@ class BaseTest:
             # 1 a
             # 5 e
             # 3 c
-            protocol = protocol.lower()
             for url in urls:
-                cmd_line += '%s://%s/%s ' % (protocol, domain, url)
+                cmd_line += '%s://%s/%s ' % (protocol.lower(), domain, url)
 
         print(cmd_line)
 

tests/ChangeLog

@@ -1,3 +1,41 @@
+2014-10-06  Tim Ruehsen <tim.ruehsen@gmx.de>
+
+	* Test-iri.px: fixed encodings
+	* Test-iri-forced-remote.px: fixed encodings
+	* Test-iri-percent.px: fixed encodings
+	* Test-idn-meta.px: fixed encodings
+
+2014-10-02  Tim Ruehsen <tim.ruehsen@gmx.de>
+
+	* tests/WgetTests.pm: use filename as default test name
+	* tests/Test-*.px: removed redundant 'name => ...'
+
+2014-10-01  Tim Ruehsen <tim.ruehsen@gmx.de>
+
+	* tests/Makefile.am: remove run-px, fixing dist-check
+
+2014-10-01  Tim Ruehsen <tim.ruehsen@gmx.de>
+
+	* Test--post-file.px: name => "Test--post-file", fixing race
+	condition.
+	* Test-N-no-info.px: name => "Test-N-no-info", fixing race
+	condition.
+	* Test-N-smaller.px: name => "Test-N-smaller", fixing race
+	condition.
+	* Test-c-shorter.px: name => "Test-c-shorter", fixing race
+	condition.
+	* Test-proxy-auth-basic.px: name => "Test-proxy-auth-basic",
+	fixing race condition.
+	* Test-proxied-https-auth.px: removed debug print line.
+
+2014-09-25  Tim Ruehsen <tim.ruehsen@gmx.de>
+
+	* Makefile.am: Modified to use parallel test harness
+	* Test-proxied-https-auth.px: get $top_srcdir from ENV
+	* run-px: removed
+	* WgetTest.pm.in: removed
+	* WgetTest.pm: get $top_srcdir from ENV
+
 2014-06-11  Giuseppe Scrivano  <gscrivan@redhat.com>
 
 	* Makefile.am: Remove @VAR@ with $FOO.

tests/FTPTest.pm

@@ -4,7 +4,7 @@ use strict;
 use warnings;
 
 use FTPServer;
-use WgetTest;
+use WgetTests;
 
 our @ISA = qw(WgetTest);
 my $VERSION = 0.01;

tests/HTTPTest.pm

@@ -4,7 +4,7 @@ use strict;
 use warnings;
 
 use HTTPServer;
-use WgetTest;
+use WgetTests;
 
 our @ISA = qw(WgetTest);
 my $VERSION = 0.01;

tests/Makefile.am

@@ -27,20 +27,12 @@
 # shall include the source code for the parts of OpenSSL used as well
 # as that of the covered work.
 
+# see http://www.gnu.org/software/automake/manual/html_node/Parallel-Test-Harness.html#Parallel-Test-Harness
+
 #
 # Version: $(VERSION)
 #
 
-PERL     = perl
-PERLRUN = $(PERL) -I$(srcdir)
-
-LIBS     += $(LIBICONV) $(LIBINTL) $(LIB_CLOCK_GETTIME)
-
-.PHONY: test run-unit-tests run-px-tests
-
-check-local: test
-
-test: ../src/wget$(EXEEXT) run-unit-tests run-px-tests
 
 ../src/wget$(EXEEXT):
 	cd ../src && $(MAKE) $(AM_MAKEFLAGS)
@@ -53,14 +45,7 @@ test: ../src/wget$(EXEEXT) run-unit-tests run-px-tests
 ../lib/libgnu.a:
 	cd ../lib && $(MAKE) $(AM_MAKEFLAGS)
 
-run-unit-tests: unit-tests$(EXEEXT) ../src/libunittest.a
-	./unit-tests$(EXEEXT)
-
-run-px-tests: WgetTest.pm ../src/wget$(EXEEXT)
-	$(srcdir)/run-px $(top_srcdir)
-
-EXTRA_DIST = FTPServer.pm FTPTest.pm HTTPServer.pm HTTPTest.pm \
-             WgetFeature.pm WgetFeature.cfg \
+PX_TESTS = \
              Test-auth-basic.px \
              Test-auth-no-challenge.px \
              Test-auth-no-challenge-url.px \
@@ -142,11 +127,20 @@ EXTRA_DIST = FTPServer.pm FTPTest.pm HTTPServer.pm HTTPTest.pm \
              Test--start-pos.px \
              Test--start-pos--continue.px \
              Test--httpsonly-r.px \
-             Test-204.px \
-             run-px certs
+             Test-204.px
+
+EXTRA_DIST = FTPServer.pm FTPTest.pm HTTPServer.pm HTTPTest.pm \
+             WgetTests.pm WgetFeature.pm WgetFeature.cfg $(PX_TESTS) \
+             certs
 
 check_PROGRAMS = unit-tests
 unit_tests_SOURCES =
 LDADD = ../src/libunittest.a ../lib/libgnu.a $(LIBS)
 
 CLEANFILES = *~ *.bak core core.[0-9]*
+
+TESTS = ./unit-tests$(EXEEXT) $(PX_TESTS)
+TEST_EXTENSIONS = .px
+AM_TESTS_ENVIRONMENT = export WGETRC=/dev/null; export SYSTEM_WGETRC=/dev/null;
+PX_LOG_COMPILER = $(PERL)
+AM_PX_LOG_FLAGS = -I$(srcdir)

tests/Test--httpsonly-r.px

@@ -68,12 +68,10 @@ my %expected_downloaded_files = (
 
 ###############################################################################
 
-my $the_test = HTTPTest->new (name => "Test--httpsonly-r",
-                              input => \%urls,
+my $the_test = HTTPTest->new (input => \%urls,
                               cmdline => $cmdline,
                               errcode => $expected_error_code,
                               output => \%expected_downloaded_files);
-print $expected_error_code."\n";
 
 exit $the_test->run();
 

tests/Test--no-content-disposition-trivial.px

@@ -45,8 +45,7 @@ my %expected_downloaded_files = (
 
 ###############################################################################
 
-my $the_test = HTTPTest->new (name => "Test--no-content-disposition-trivial",
-                              input => \%urls,
+my $the_test = HTTPTest->new (input => \%urls,
                               cmdline => $cmdline,
                               errcode => $expected_error_code,
                               output => \%expected_downloaded_files);

tests/Test--no-content-disposition.px

@@ -46,8 +46,7 @@ my %expected_downloaded_files = (
 
 ###############################################################################
 
-my $the_test = HTTPTest->new (name => "Test--no-content-disposition",
-                              input => \%urls,
+my $the_test = HTTPTest->new (input => \%urls,
                               cmdline => $cmdline,
                               errcode => $expected_error_code,
                               output => \%expected_downloaded_files);

tests/Test--post-file.px

@@ -15,8 +15,7 @@ my $expected_error_code = 3;
 
 ###############################################################################
 
-my $the_test = HTTPTest->new (name => "Test-missing-file",
-                              cmdline => $cmdline,
+my $the_test = HTTPTest->new (cmdline => $cmdline,
                               errcode => $expected_error_code);
 exit $the_test->run();
 

tests/Test--spider-fail.px

@@ -42,8 +42,7 @@ my %expected_downloaded_files = (
 
 ###############################################################################
 
-my $the_test = HTTPTest->new (name => "Test--spider-fail",
-                              input => \%urls,
+my $the_test = HTTPTest->new (input => \%urls,
                               cmdline => $cmdline,
                               errcode => $expected_error_code,
                               output => \%expected_downloaded_files);

tests/Test--spider-r--no-content-disposition-trivial.px

@@ -99,8 +99,7 @@ my %expected_downloaded_files = (
 
 ###############################################################################
 
-my $the_test = HTTPTest->new (name => "Test--spider-r--no-content-disposition-trivial",
-                              input => \%urls,
+my $the_test = HTTPTest->new (input => \%urls,
                               cmdline => $cmdline,
                               errcode => $expected_error_code,
                               output => \%expected_downloaded_files);

tests/Test--spider-r--no-content-disposition.px

@@ -100,8 +100,7 @@ my %expected_downloaded_files = (
 
 ###############################################################################
 
-my $the_test = HTTPTest->new (name => "Test--spider-r--no-content-disposition",
-                              input => \%urls,
+my $the_test = HTTPTest->new (input => \%urls,
                               cmdline => $cmdline,
                               errcode => $expected_error_code,
                               output => \%expected_downloaded_files);

tests/Test--spider-r-HTTP-Content-Disposition.px

@@ -100,8 +100,7 @@ my %expected_downloaded_files = (
 
 ###############################################################################
 
-my $the_test = HTTPTest->new (name => "Test--spider-r-HTTP-Content-Disposition",
-                              input => \%urls,
+my $the_test = HTTPTest->new (input => \%urls,
                               cmdline => $cmdline,
                               errcode => $expected_error_code,
                               output => \%expected_downloaded_files);

tests/Test--spider-r.px

@@ -99,8 +99,7 @@ my %expected_downloaded_files = (
 
 ###############################################################################
 
-my $the_test = HTTPTest->new (name => "Test--spider-r",
-                              input => \%urls,
+my $the_test = HTTPTest->new (input => \%urls,
                               cmdline => $cmdline,
                               errcode => $expected_error_code,
                               output => \%expected_downloaded_files);

tests/Test--spider.px

@@ -42,8 +42,7 @@ my %expected_downloaded_files = (
 
 ###############################################################################
 
-my $the_test = HTTPTest->new (name => "Test--spider",
-                              input => \%urls,
+my $the_test = HTTPTest->new (input => \%urls,
                               cmdline => $cmdline,
                               errcode => $expected_error_code,
                               output => \%expected_downloaded_files);

tests/Test--start-pos--continue.px

@@ -44,8 +44,7 @@ my %expected_downloaded_files = (
 
 ###############################################################################
 
-my $the_test = HTTPTest->new (name => "Test--start-pos--continue",
-                              input => \%urls,
+my $the_test = HTTPTest->new (input => \%urls,
                               cmdline => $cmdline,
                               errcode => $expected_error_code,
                               existing => \%existing_files,

tests/Test--start-pos.px

@@ -34,8 +34,7 @@ my %expected_downloaded_files = (
 
 ###############################################################################
 
-my $the_test = HTTPTest->new (name => "Test--start-pos",
-                              input => \%urls,
+my $the_test = HTTPTest->new (input => \%urls,
                               cmdline => $cmdline,
                               errcode => $expected_error_code,
                               output => \%expected_downloaded_files);

tests/Test-204.px

@@ -25,8 +25,7 @@ my %expected_downloaded_files = (
 
 ###############################################################################
 
-my $the_test = HTTPTest->new (name => "Test-204",
-                              input => \%urls,
+my $the_test = HTTPTest->new (input => \%urls,
                               cmdline => $cmdline,
                               errcode => $expected_error_code,
                               output => \%expected_downloaded_files);

tests/Test-E-k-K.px

@@ -79,8 +79,7 @@ my %expected_downloaded_files = (
 
 ###############################################################################
 
-my $the_test = HTTPTest->new (name => "Test-E-k-K",
-                              input => \%urls,
+my $the_test = HTTPTest->new (input => \%urls,
                               cmdline => $cmdline,
                               errcode => $expected_error_code,
                               output => \%expected_downloaded_files);

tests/Test-E-k.px

@@ -76,8 +76,7 @@ my %expected_downloaded_files = (
 
 ###############################################################################
 
-my $the_test = HTTPTest->new (name => "Test-E-k",
-                              input => \%urls,
+my $the_test = HTTPTest->new (input => \%urls,
                               cmdline => $cmdline,
                               errcode => $expected_error_code,
                               output => \%expected_downloaded_files);

tests/Test-HTTP-Content-Disposition-1.px

@@ -65,8 +65,7 @@ my %expected_downloaded_files = (
 
 ###############################################################################
 
-my $the_test = HTTPTest->new (name => "Test-HTTP-Content-Disposition-1",
-                              input => \%urls,
+my $the_test = HTTPTest->new (input => \%urls,
                               cmdline => $cmdline,
                               errcode => $expected_error_code,
                               existing => \%existing_files,

tests/Test-HTTP-Content-Disposition-2.px

@@ -65,8 +65,7 @@ my %expected_downloaded_files = (
 
 ###############################################################################
 
-my $the_test = HTTPTest->new (name => "Test-HTTP-Content-Disposition-2",
-                              input => \%urls,
+my $the_test = HTTPTest->new (input => \%urls,
                               cmdline => $cmdline,
                               errcode => $expected_error_code,
                               existing => \%existing_files,

tests/Test-HTTP-Content-Disposition.px

@@ -46,8 +46,7 @@ my %expected_downloaded_files = (
 
 ###############################################################################
 
-my $the_test = HTTPTest->new (name => "Test-HTTP-Content-Disposition",
-                              input => \%urls,
+my $the_test = HTTPTest->new (input => \%urls,
                               cmdline => $cmdline,
                               errcode => $expected_error_code,
                               output => \%expected_downloaded_files);

tests/Test-N--no-content-disposition-trivial.px

@@ -38,8 +38,7 @@ my %expected_downloaded_files = (
 
 ###############################################################################
 
-my $the_test = HTTPTest->new (name => "Test-N--no-content-disposition-trivial",
-                              input => \%urls,
+my $the_test = HTTPTest->new (input => \%urls,
                               cmdline => $cmdline,
                               errcode => $expected_error_code,
                               output => \%expected_downloaded_files);

tests/Test-N--no-content-disposition.px

@@ -39,8 +39,7 @@ my %expected_downloaded_files = (
 
 ###############################################################################
 
-my $the_test = HTTPTest->new (name => "Test-N--no-content-disposition",
-                              input => \%urls,
+my $the_test = HTTPTest->new (input => \%urls,
                               cmdline => $cmdline,
                               errcode => $expected_error_code,
                               output => \%expected_downloaded_files);

tests/Test-N-HTTP-Content-Disposition.px

@@ -40,8 +40,7 @@ my %expected_downloaded_files = (
 
 ###############################################################################
 
-my $the_test = HTTPTest->new (name => "Test-N-HTTP-Content-Disposition",
-                              input => \%urls,
+my $the_test = HTTPTest->new (input => \%urls,
                               cmdline => $cmdline,
                               errcode => $expected_error_code,
                               output => \%expected_downloaded_files);

tests/Test-N-current.px

@@ -54,8 +54,7 @@ my %expected_downloaded_files = (
 
 ###############################################################################
 
-my $the_test = HTTPTest->new (name => "Test-N-current",
-                              input => \%urls,
+my $the_test = HTTPTest->new (input => \%urls,
                               cmdline => $cmdline,
                               errcode => $expected_error_code,
                               existing => \%existing_files,

tests/Test-N-no-info.px

@@ -52,8 +52,7 @@ my %expected_downloaded_files = (
 
 ###############################################################################
 
-my $the_test = HTTPTest->new (name => "Test-N-current",
-                              input => \%urls,
+my $the_test = HTTPTest->new (input => \%urls,
                               cmdline => $cmdline,
                               errcode => $expected_error_code,
                               existing => \%existing_files,

tests/Test-N-old.px

@@ -52,8 +52,7 @@ my %expected_downloaded_files = (
 
 ###############################################################################
 
-my $the_test = HTTPTest->new (name => "Test-N-old",
-                              input => \%urls,
+my $the_test = HTTPTest->new (input => \%urls,
                               cmdline => $cmdline,
                               errcode => $expected_error_code,
                               existing => \%existing_files,

tests/Test-N-smaller.px

@@ -55,8 +55,7 @@ my %expected_downloaded_files = (
 
 ###############################################################################
 
-my $the_test = HTTPTest->new (name => "Test-N-current",
-                              input => \%urls,
+my $the_test = HTTPTest->new (input => \%urls,
                               cmdline => $cmdline,
                               errcode => $expected_error_code,
                               existing => \%existing_files,

tests/Test-N.px

@@ -38,8 +38,7 @@ my %expected_downloaded_files = (
 
 ###############################################################################
 
-my $the_test = HTTPTest->new (name => "Test-N",
-                              input => \%urls,
+my $the_test = HTTPTest->new (input => \%urls,
                               cmdline => $cmdline,
                               errcode => $expected_error_code,
                               output => \%expected_downloaded_files);

tests/Test-O--no-content-disposition-trivial.px

@@ -36,8 +36,7 @@ my %expected_downloaded_files = (
 
 ###############################################################################
 
-my $the_test = HTTPTest->new (name => "Test-O--no-content-disposition-trivial",
-                              input => \%urls,
+my $the_test = HTTPTest->new (input => \%urls,
                               cmdline => $cmdline,
                               errcode => $expected_error_code,
                               output => \%expected_downloaded_files);

tests/Test-O--no-content-disposition.px

@@ -37,8 +37,7 @@ my %expected_downloaded_files = (
 
 ###############################################################################
 
-my $the_test = HTTPTest->new (name => "Test-O--no-content-disposition",
-                              input => \%urls,
+my $the_test = HTTPTest->new (input => \%urls,
                               cmdline => $cmdline,
                               errcode => $expected_error_code,
                               output => \%expected_downloaded_files);

tests/Test-O-HTTP-Content-Disposition.px

@@ -37,8 +37,7 @@ my %expected_downloaded_files = (
 
 ###############################################################################
 
-my $the_test = HTTPTest->new (name => "Test-O-HTTP-Content-Disposition",
-                              input => \%urls,
+my $the_test = HTTPTest->new (input => \%urls,
                               cmdline => $cmdline,
                               errcode => $expected_error_code,
                               output => \%expected_downloaded_files);

tests/Test-O-nc.px

@@ -36,8 +36,7 @@ my %expected_downloaded_files = (
 
 ###############################################################################
 
-my $the_test = HTTPTest->new (name => "Test-O-nc",
-                              input => \%urls,
+my $the_test = HTTPTest->new (input => \%urls,
                               cmdline => $cmdline,
                               errcode => $expected_error_code,
                               output => \%expected_downloaded_files);

tests/Test-O-nonexisting.px

@@ -36,8 +36,7 @@ my %expected_downloaded_files = (
 
 ###############################################################################
 
-my $the_test = HTTPTest->new (name => "Test-O-nonexisting",
-                              input => \%urls,
+my $the_test = HTTPTest->new (input => \%urls,
                               cmdline => $cmdline,
                               errcode => $expected_error_code,
                               output => \%expected_downloaded_files);

tests/Test-O.px

@@ -36,8 +36,7 @@ my %expected_downloaded_files = (
 
 ###############################################################################
 
-my $the_test = HTTPTest->new (name => "Test-O",
-                              input => \%urls,
+my $the_test = HTTPTest->new (input => \%urls,
                               cmdline => $cmdline,
                               errcode => $expected_error_code,
                               output => \%expected_downloaded_files);

tests/Test-Restrict-Lowercase.px

@@ -45,8 +45,7 @@ my %expected_downloaded_files = (
 
 ###############################################################################
 
-my $the_test = HTTPTest->new (name => "Test-Restrict-Lowercase",
-                              input => \%urls,
+my $the_test = HTTPTest->new (input => \%urls,
                               cmdline => $cmdline,
                               errcode => $expected_error_code,
                               output => \%expected_downloaded_files);

tests/Test-Restrict-Uppercase.px

@@ -45,8 +45,7 @@ my %expected_downloaded_files = (
 
 ###############################################################################
 
-my $the_test = HTTPTest->new (name => "Test-Restrict-Uppercase",
-                              input => \%urls,
+my $the_test = HTTPTest->new (input => \%urls,
                               cmdline => $cmdline,
                               errcode => $expected_error_code,
                               output => \%expected_downloaded_files);

tests/Test-auth-basic.px

@@ -38,8 +38,7 @@ my %expected_downloaded_files = (
 
 ###############################################################################
 
-my $the_test = HTTPTest->new (name => "Test-auth-basic",
-                              input => \%urls,
+my $the_test = HTTPTest->new (input => \%urls,
                               cmdline => $cmdline,
                               errcode => $expected_error_code,
                               output => \%expected_downloaded_files);

tests/Test-auth-no-challenge-url.px

@@ -39,8 +39,7 @@ my %expected_downloaded_files = (
 
 ###############################################################################
 
-my $the_test = HTTPTest->new (name => "Test-auth-no-challenge-url",
-                              input => \%urls,
+my $the_test = HTTPTest->new (input => \%urls,
                               cmdline => $cmdline,
                               errcode => $expected_error_code,
                               output => \%expected_downloaded_files);

tests/Test-auth-no-challenge.px

@@ -40,8 +40,7 @@ my %expected_downloaded_files = (
 
 ###############################################################################
 
-my $the_test = HTTPTest->new (name => "Test-auth-no-challenge",
-                              input => \%urls,
+my $the_test = HTTPTest->new (input => \%urls,
                               cmdline => $cmdline,
                               errcode => $expected_error_code,
                               output => \%expected_downloaded_files);

tests/Test-auth-retcode.px

@@ -28,8 +28,7 @@ my %expected_downloaded_files = ();
 
 ###############################################################################
 
-my $the_test = HTTPTest->new (name => "Test-auth-retcode",
-                              input => \%urls,
+my $the_test = HTTPTest->new (input => \%urls,
                               cmdline => $cmdline,
                               errcode => $expected_error_code,
                               output => \%expected_downloaded_files);

tests/Test-auth-with-content-disposition.px

@@ -39,8 +39,7 @@ my %expected_downloaded_files = (
 
 ###############################################################################
 
-my $the_test = HTTPTest->new (name => "Test-auth-with-content-disposition",
-                              input => \%urls,
+my $the_test = HTTPTest->new (input => \%urls,
                               cmdline => $cmdline,
                               errcode => $expected_error_code,
                               output => \%expected_downloaded_files);

tests/Test-c-full.px

@@ -46,8 +46,7 @@ my %expected_downloaded_files = (
 
 ###############################################################################
 
-my $the_test = HTTPTest->new (name => "Test-c-full",
-                              input => \%urls,
+my $the_test = HTTPTest->new (input => \%urls,
                               cmdline => $cmdline,
                               errcode => $expected_error_code,
                               existing => \%existing_files,

tests/Test-c-partial.px

@@ -56,8 +56,7 @@ my %expected_downloaded_files = (
 
 ###############################################################################
 
-my $the_test = HTTPTest->new (name => "Test-c-partial",
-                              input => \%urls,
+my $the_test = HTTPTest->new (input => \%urls,
                               cmdline => $cmdline,
                               errcode => $expected_error_code,
                               existing => \%existing_files,

tests/Test-c-shorter.px

@@ -53,8 +53,7 @@ my %expected_downloaded_files = (
 
 ###############################################################################
 
-my $the_test = HTTPTest->new (name => "Test-c-partial",
-                              input => \%urls,
+my $the_test = HTTPTest->new (input => \%urls,
                               cmdline => $cmdline,
                               errcode => $expected_error_code,
                               existing => \%existing_files,

tests/Test-c.px

@@ -43,8 +43,7 @@ my %expected_downloaded_files = (
 
 ###############################################################################
 
-my $the_test = HTTPTest->new (name => "Test-c",
-                              input => \%urls,
+my $the_test = HTTPTest->new (input => \%urls,
                               cmdline => $cmdline,
                               errcode => $expected_error_code,
                               existing => \%existing_files,

tests/Test-cookies-401.px

@@ -42,8 +42,7 @@ my %expected_downloaded_files = (
 
 ###############################################################################
 
-my $the_test = HTTPTest->new (name => "Test-cookies-401",
-                              input => \%urls,
+my $the_test = HTTPTest->new (input => \%urls,
                               cmdline => $cmdline,
                               errcode => $expected_error_code,
                               output => \%expected_downloaded_files);

tests/Test-cookies.px

@@ -103,8 +103,7 @@ my %expected_downloaded_files = (
 
 ###############################################################################
 
-my $the_test = HTTPTest->new (name => "Test-cookies",
-                              input => \%urls,
+my $the_test = HTTPTest->new (input => \%urls,
                               cmdline => $cmdline,
                               errcode => $expected_error_code,
                               output => \%expected_downloaded_files);

tests/Test-ftp--start-pos.px

@@ -29,7 +29,7 @@ my %expected_downloaded_files = (
 
 ###############################################################################
 
-my $the_test = FTPTest->new (name => "Test-ftp--start-pos",
+my $the_test = FTPTest->new (
                               input => \%urls,
                               cmdline => $cmdline,
                               errcode => $expected_error_code,

tests/Test-ftp-bad-list.px

@@ -56,7 +56,7 @@ my %preexisting_files = (
 
 ###############################################################################
 
-my $the_test = FTPTest->new (name => "Test-ftp-bad-list",
+my $the_test = FTPTest->new (
                              input => \%urls,
                              cmdline => $cmdline,
                              errcode => $expected_error_code,

tests/Test-ftp-iri-disabled.px

@@ -41,7 +41,7 @@ my %expected_downloaded_files = (
 
 ###############################################################################
 
-my $the_test = FTPTest->new (name => "Test-ftp-iri-disabled",
+my $the_test = FTPTest->new (
                              input => \%urls,
                              cmdline => $cmdline,
                              errcode => $expected_error_code,

tests/Test-ftp-iri-fallback.px

@@ -37,7 +37,7 @@ my %expected_downloaded_files = (
 
 ###############################################################################
 
-my $the_test = FTPTest->new (name => "Test-ftp-iri-fallback",
+my $the_test = FTPTest->new (
                              input => \%urls,
                              cmdline => $cmdline,
                              errcode => $expected_error_code,

tests/Test-ftp-iri-recursive.px

@@ -37,7 +37,7 @@ my %expected_downloaded_files = (
 
 ###############################################################################
 
-my $the_test = FTPTest->new (name => "Test-ftp-iri-recursive",
+my $the_test = FTPTest->new (
                              input => \%urls,
                              cmdline => $cmdline,
                              errcode => $expected_error_code,

tests/Test-ftp-iri.px

@@ -38,7 +38,7 @@ my %expected_downloaded_files = (
 
 ###############################################################################
 
-my $the_test = FTPTest->new (name => "Test-ftp-iri",
+my $the_test = FTPTest->new (
                              input => \%urls,
                              cmdline => $cmdline,
                              errcode => $expected_error_code,

tests/Test-ftp-list-Multinet.px

@@ -56,7 +56,7 @@ my %expected_downloaded_files = (
 
 ###############################################################################
 
-my $the_test = FTPTest->new (name => "Test-ftp-list-Multinet",
+my $the_test = FTPTest->new (
                              input => \%urls,
                              cmdline => $cmdline,
                              errcode => $expected_error_code,

tests/Test-ftp-list-UNIX-hidden.px

@@ -55,7 +55,7 @@ my %expected_downloaded_files = (
 
 ###############################################################################
 
-my $the_test = FTPTest->new (name => "Test-ftp-list-UNIX-hidden",
+my $the_test = FTPTest->new (
                              input => \%urls,
                              cmdline => $cmdline,
                              errcode => $expected_error_code,

tests/Test-ftp-list-Unknown-a.px

@@ -66,7 +66,7 @@ my %expected_downloaded_files = (
 
 ###############################################################################
 
-my $the_test = FTPTest->new (name => "Test-ftp-list-Unknown-a",
+my $the_test = FTPTest->new (
                              input => \%urls,
                              cmdline => $cmdline,
                              errcode => $expected_error_code,

tests/Test-ftp-list-Unknown-hidden.px

@@ -58,7 +58,7 @@ my %expected_downloaded_files = (
 
 ###############################################################################
 
-my $the_test = FTPTest->new (name => "Test-ftp-list-Unknown-hidden",
+my $the_test = FTPTest->new (
                              input => \%urls,
                              cmdline => $cmdline,
                              errcode => $expected_error_code,

tests/Test-ftp-list-Unknown-list-a-fails.px

@@ -51,7 +51,7 @@ my %expected_downloaded_files = (
 
 ###############################################################################
 
-my $the_test = FTPTest->new (name => "Test-ftp-list-Unknown-list-a-fails",
+my $the_test = FTPTest->new (
                              input => \%urls,
                              cmdline => $cmdline,
                              errcode => $expected_error_code,

tests/Test-ftp-list-Unknown.px

@@ -54,7 +54,7 @@ my %expected_downloaded_files = (
 
 ###############################################################################
 
-my $the_test = FTPTest->new (name => "Test-ftp-list-Unknown",
+my $the_test = FTPTest->new (
                              input => \%urls,
                              cmdline => $cmdline,
                              errcode => $expected_error_code,

tests/Test-ftp-pasv-fail.px

@@ -46,7 +46,7 @@ my %expected_downloaded_files = (
 
 ###############################################################################
 
-my $the_test = FTPTest->new (name => "Test-ftp-pasv-fail",
+my $the_test = FTPTest->new (
                              server_behavior => {fail_on_pasv => 1},
                              input => \%urls,
                              cmdline => $cmdline,

tests/Test-ftp-recursive.px

@@ -44,7 +44,7 @@ my %expected_downloaded_files = (
 
 ###############################################################################
 
-my $the_test = FTPTest->new (name => "Test-ftp-recursive",
+my $the_test = FTPTest->new (
                              input => \%urls,
                              cmdline => $cmdline,
                              errcode => $expected_error_code,

tests/Test-ftp.px

@@ -34,7 +34,7 @@ my %expected_downloaded_files = (
 
 ###############################################################################
 
-my $the_test = FTPTest->new (name => "Test-ftp",
+my $the_test = FTPTest->new (
                              input => \%urls,
                              cmdline => $cmdline,
                              errcode => $expected_error_code,

tests/Test-i-ftp.px

@@ -69,7 +69,7 @@ my %expected_downloaded_files = (
 
 ###############################################################################
 
-my $the_test = FTPTest->new (name => "Test-i-ftp",
+my $the_test = FTPTest->new (
                              input => \%urls,
                              cmdline => $cmdline,
                              errcode => $expected_error_code,

tests/Test-i-http.px

@@ -81,8 +81,7 @@ my %expected_downloaded_files = (
 
 ###############################################################################
 
-my $the_test = HTTPTest->new (name => "Test-i-http",
-                              input => \%urls,
+my $the_test = HTTPTest->new (input => \%urls,
                               cmdline => $cmdline,
                               errcode => $expected_error_code,
                               output => \%expected_downloaded_files);

tests/Test-idn-cmd-utf8.px

@@ -41,8 +41,7 @@ my %expected_downloaded_files = (
 
 ###############################################################################
 
-my $the_test = HTTPTest->new (name => "Test-idn-cmd-utf8",
-                              input => \%urls,
+my $the_test = HTTPTest->new (input => \%urls,
                               cmdline => $cmdline,
                               errcode => $expected_error_code,
                               output => \%expected_downloaded_files);

tests/Test-idn-cmd.px

@@ -41,8 +41,7 @@ my %expected_downloaded_files = (
 
 ###############################################################################
 
-my $the_test = HTTPTest->new (name => "Test-idn-cmd",
-                              input => \%urls,
+my $the_test = HTTPTest->new (input => \%urls,
                               cmdline => $cmdline,
                               errcode => $expected_error_code,
                               output => \%expected_downloaded_files);

tests/Test-idn-headers.px

@@ -56,8 +56,7 @@ my %expected_downloaded_files = (
 
 ###############################################################################
 
-my $the_test = HTTPTest->new (name => "Test-idn-headers",
-                              input => \%urls,
+my $the_test = HTTPTest->new (input => \%urls,
                               cmdline => $cmdline,
                               errcode => $expected_error_code,
                               output => \%expected_downloaded_files);

tests/Test-idn-meta.px

@@ -27,7 +27,8 @@ my %urls = (
         code => "200",
         msg => "You want fries with that?",
         headers => {
-            'Content-Type' => 'text/html; charset=UTF-8',
+            # HTTP header preceeds http-equiv, simply just omit it here
+            #'Content-Type' => 'text/html; charset=UTF-8',
         },
         content => $starter_file,
     },
@@ -57,8 +58,7 @@ my %expected_downloaded_files = (
 
 ###############################################################################
 
-my $the_test = HTTPTest->new (name => "Test-idn-meta",
-                              input => \%urls,
+my $the_test = HTTPTest->new (input => \%urls,
                               cmdline => $cmdline,
                               errcode => $expected_error_code,
                               output => \%expected_downloaded_files);

tests/Test-idn-robots-utf8.px

@@ -68,8 +68,7 @@ my %expected_downloaded_files = (
 
 ###############################################################################
 
-my $the_test = HTTPTest->new (name => "Test-idn-robots-utf8",
-                              input => \%urls,
+my $the_test = HTTPTest->new (input => \%urls,
                               cmdline => $cmdline,
                               errcode => $expected_error_code,
                               output => \%expected_downloaded_files);

tests/Test-idn-robots.px

@@ -93,8 +93,7 @@ my %expected_downloaded_files = (
 
 ###############################################################################
 
-my $the_test = HTTPTest->new (name => "Test-idn-robots",
-                              input => \%urls,
+my $the_test = HTTPTest->new (input => \%urls,
                               cmdline => $cmdline,
                               errcode => $expected_error_code,
                               output => \%expected_downloaded_files);

tests/Test-iri-disabled.px

@@ -186,8 +186,7 @@ my %expected_downloaded_files = (
 
 ###############################################################################
 
-my $the_test = HTTPTest->new (name => "Test-iri-disabled",
-                              input => \%urls,
+my $the_test = HTTPTest->new (input => \%urls,
                               cmdline => $cmdline,
                               errcode => $expected_error_code,
                               output => \%expected_downloaded_files);

tests/Test-iri-forced-remote.px

@@ -48,7 +48,7 @@ my $pagefrancais = <<EOF;
 <html>
 <head>
   <title>La seule page en français</title>
-  <meta http-equiv="Content-Type" content="text/html; charset=ISO-8859-1"/>
+  <meta http-equiv="Content-Type" content="text/html; charset=UTF-8"/>
 </head>
 <body>
   <p>
@@ -117,17 +117,10 @@ my %urls = (
         content => "",
     },
     '/p1_fran%C3%A7ais.html' => {	# UTF-8 encoded
-        code => "404",
-        msg => "File not found",
-        headers => {
-            "Content-type" => "text/html; charset=UTF-8",
-        },
-        content => $page404,
-    },
-    '/p1_fran%E7ais.html' => {
         code => "200",
         msg => "Ok",
         headers => {
+            # wrong charset here, overridden by --remote-encoding=iso-8859-1
             "Content-type" => "text/html; charset=UTF-8",
         },
         content => $pagefrancais,
@@ -140,14 +133,6 @@ my %urls = (
         },
         content => $pageeen,
     },
-    '/p2_%E9%E9n.html' => {
-        code => "200",
-        msg => "Ok",
-        headers => {
-            "Content-type" => "text/html; charset=ISO-8859-1",
-        },
-        content => $pageeen,
-    },
     '/p3_%E2%82%AC%E2%82%AC%E2%82%AC.html' => {	# UTF-8 encoded
         code => "200",
         msg => "Ok",
@@ -156,14 +141,6 @@ my %urls = (
         },
         content => $pageeuro,
     },
-    '/p3_%A4%A4%A4.html' => {
-        code => "200",
-        msg => "Ok",
-        headers => {
-            "Content-type" => "text/plain",
-        },
-        content => $pageeuro,
-    },
     '/p3_%C2%A4%C2%A4%C2%A4.html' => {	# UTF-8 encoded
         code => "200",
         msg => "Ok",
@@ -174,7 +151,7 @@ my %urls = (
     },
 );
 
-my $cmdline = $WgetTest::WGETPATH . " --iri --trust-server-names --remote-encoding=iso-8859-1 -nH -r http://localhost:{{port}}/";
+my $cmdline = $WgetTest::WGETPATH . " --iri -e robots=on --trust-server-names --remote-encoding=iso-8859-1 -nH -r http://localhost:{{port}}/";
 
 my $expected_error_code = 0;
 
@@ -185,7 +162,7 @@ my %expected_downloaded_files = (
     'robots.txt' => {
         content => "",
     },
-    "p1_fran${ccedilla_l15}ais.html" => {
+    "p1_fran${ccedilla_u8}ais.html" => {
         content => $pagefrancais,
     },
     "p2_${eacute_u8}${eacute_u8}n.html" => {
@@ -198,8 +175,7 @@ my %expected_downloaded_files = (
 
 ###############################################################################
 
-my $the_test = HTTPTest->new (name => "Test-iri-forced-remote",
-                              input => \%urls,
+my $the_test = HTTPTest->new (input => \%urls,
                               cmdline => $cmdline,
                               errcode => $expected_error_code,
                               output => \%expected_downloaded_files);