Prepare new release 1.16

When Wget retrieves a file through FTP, it first downloads a .listing
file and parses it for information about the files and other metadata.
Some servers may serve invalid .listing files. This patch checks for one
such known inconsistency wherein multiple lines in a listing file have
the same name. Such a filesystem is clearly not possible and hence we
eliminate duplicate entries here.

Signed-off-by: Darshit Shah <darnir@gmail.com>

.gitignore

@@ -0,0 +1,93 @@
+# Project Root
+# Wildcard Ignores
+# build-aux/
+# doc/
+# m4/
+# others
+# po/
+# src/
+# testenv/
+# tests/
+# util/
+*~
+/.sc-start-sc_bindtextdomain
+/.sc-start-sc_prohibit_HAVE_MBRTOWC
+/.version
+/ABOUT-NLS~
+/GNUmakefile
+/INSTALL
+/Makefile
+/Makefile.in
+/aclocal.m4
+/autom4te.cache/
+/config.cache
+/config.log
+/config.status
+/configure
+/cscope.out
+/lib/
+/maint.mk
+/tags
+build-aux/.gitignore
+build-aux/compile
+build-aux/config.guess
+build-aux/config.sub
+build-aux/depcomp
+build-aux/install-sh
+build-aux/mdate-sh
+build-aux/missing
+build-aux/snippet/
+build-aux/test-driver
+build-aux/texinfo.tex
+doc/Makefile
+doc/Makefile.in
+doc/sample.wgetrc.munged_for_texi_inclusion
+doc/stamp-vti
+doc/version.texi
+doc/wget.1
+doc/wget.info
+doc/wget.pod
+m4/.gitignore
+m4/gnulib-cache.m4
+po/*.gmo*
+po/*.po*
+po/.gitignore
+po/.reference/
+po/LINGUAS
+po/Makefile
+po/Makefile.in
+po/Makevars
+po/POTFILES
+po/wget.pot
+src/*.o
+src/.deps/
+src/Makefile
+src/Makefile.in
+src/build_info.c
+src/config.h
+src/config.h.in
+src/cscope.out
+src/css.c
+src/css_.c
+src/libunittest.a
+src/stamp-h1
+src/tags
+src/version.c
+src/wget
+testenv/*.log
+testenv/*.trs
+testenv/Makefile
+testenv/Makefile.in
+testenv/conf/__pycache__/
+testenv/exc/__pycache__/
+testenv/misc/__pycache__/
+testenv/server/__pycache__/
+testenv/server/http/__pycache__/
+testenv/test/__pycache__/
+tests/*.log
+tests/*.trs
+tests/Makefile
+tests/Makefile.in
+tests/unit-tests
+util/Makefile
+util/Makefile.in

ChangeLog

@@ -1,3 +1,87 @@
+2013-10-22  Ángel González <keisial@gmail.com>
+
+	* bootstrap.conf (gnulib_modules): Add module xstrndup.
+
+2014-09-25  Tim Ruehsen <tim.ruehsen@gmx.de>
+
+	* configure.ac: removed WgetTest.pm.in
+	* .gitignore: removed WgetTest.pm
+
+2014-09-12  Darshit Shah  <darnir@gmail.com>
+
+	* bootstrap.conf: Add GNULib module mbiter
+
+2014-07-25  Darshit Shah  <darnir@gmail.com>
+
+	* .gitignore: Add a gitignore file for the project.
+
+2013-08-31  Darshit Shah  <darnir@gmail.com>
+
+	* configure.ac: Add testenv/Makefile to	AC_CONFIG_FILES.
+	* Makefile.am: Add testenv to SUBDIRS
+
+2014-07-22  Darshit Shah  <darnir@gmail.com>
+
+	* configure.ac: Fix broken code for detecting libpsl
+
+2014-07-21  Darshit Shah  <darnir@gmail.com>
+
+	* configure.ac: Fix check for Libpsl
+
+2014-06-28  Giuseppe Scrivano  <gscrivan@redhat.com>
+
+	* cfg.mk (local-checks-to-skip): Remove some checks.
+
+2014-06-27  Giuseppe Scrivano  <gscrivan@redhat.com>
+
+	* cfg.mk (VC_LIST_ALWAYS_EXCLUDE_REGEX): Ignore msdos/* and vms/* from
+	syntax-check.
+
+2014-06-22  Giuseppe Scrivano  <gscrivan@redhat.com>
+
+	* configure.ac: Add check for uuid_create.
+
+2014-06-11  Giuseppe Scrivano  <gscrivan@redhat.com>
+
+	* NEWS: Remove repeated word.
+
+	* po/POTFILES.in: Add lib/regcomp.c.
+
+	* configure.ac: Do not depend on always defined macros.
+	* vms/vms.h: Likewise.
+
+	* Makefile.am: Remove @VAR@ with $FOO.
+	* util/Makefile.am: Likewise.
+
+	* ABOUT-NLS: Remove terminating empty lines.
+	* MAILING-LIST: Likewise.
+	* util/rmold.pl: Likewise.
+
+	* util/trunc.c (main): Ensure bindtextdomain is called.
+
+2014-06-10  Giuseppe Scrivano  <gscrivan@redhat.com>
+
+	* build-aux/build_info.pl: Remove trailing whitespaces.
+	* configure.ac: Likewise.
+
+2014-05-30  Darshit Shah  <darnir@gmail.com>
+
+	* configure.ac: Allow compilation without libpsl.
+	* README.checkout: Add libpsl as a dependency.
+
+2014-05-24  Giuseppe Scrivano  <gscrivan@redhat.com>
+
+	* gnulib: update module.
+
+2014-02-24  Giuseppe Scrivano  <gscrivan@redhat.com>
+
+	* gnulib: update module.
+	Reported by: Darshit Shah  <darnir@gmail.com>.
+
+2014-02-06  Giuseppe Scrivano  <gscrivan@redhat.com>
+
+	* configure.ac: Update copyright years.
+
 2013-12-22  Giuseppe Scrivano  <gscrivan@redhat.com>
 
 	* gnulib: add git submodule.

MAILING-LIST

@@ -47,4 +47,3 @@ for submitting and discussing patches to GNU Wget.
 
    Messages from <wget-patches@sunsite.dk> are archived at
      `http://news.gmane.org/gmane.comp.web.wget.patches'.
-

Makefile.am

@@ -27,7 +27,7 @@
 # as that of the covered work.
 
 #
-# Version: @VERSION@
+# Version: $(VERSION)
 #
 
 
@@ -41,7 +41,7 @@ distuninstallcheck_listfiles = find . -type f | \
 ACLOCAL_AMFLAGS = -I m4
 
 # subdirectories in the distribution
-SUBDIRS = lib src doc po tests util
+SUBDIRS = lib src doc po tests util testenv
 
 EXTRA_DIST = ChangeLog.README MAILING-LIST \
              msdos/ChangeLog msdos/config.h msdos/Makefile.DJ \

NEWS

@@ -6,6 +6,22 @@ See the end for copying conditions.
 
 Please send GNU Wget bug reports to <bug-wget@gnu.org>.
 
+* Changes in Wget 1.16
+
+** No longer create local symbolic links by default.  Closes CVE-2014-4877.
+
+** Use libpsl for verifying cookie domains.
+
+** Default progress bar output changed.
+
+** Introduce --show-progress to force display the progress bar.
+
+** Introduce --no-config.  The wgetrc files will not be read.
+
+** Introduce --start-pos to allow starting downloads from a specified position.
+
+** Fix a problem with ISA Server Proxy and keep-alive connections.
+
 * Changes in Wget 1.15
 
 ** Add support for --method.
@@ -672,7 +688,7 @@ ASCII type transfer:
 
     wget "ftp://ftp.somewhere.com/welcome.msg;type=a"
 
-** `--help' and `--version' options have been redone to to conform to
+** `--help' and `--version' options have been redone to conform to
 standards set by other GNU utilities.
 
 ** Wget should now be compilable under MS Windows environment.  MS

README.checkout

@@ -4,8 +4,8 @@ Compiling From Repository Sources
    does not contain automatically-generated files, even when these are
    normally present in the distribution tarballs. Therefore, to build GNU
    Wget from the sources in the repository, you'll need to have one or
-   more of the following (note that gettext, OpenSSL, libidn and libiconv
-   are not absolutely required):
+   more of the following (note that gettext, OpenSSL, libidn, libiconv
+   and libpsl are not absolutely required):
 
      * [20]autoconf (currently, GNU Wget requires version 2.61). This is
        needed to generate the configure script from configure.in. This is
@@ -68,6 +68,9 @@ Compiling From Repository Sources
 
      * [34]git is used to fetch gnulib files trough the bootstrap.sh script.
 
+     * [35]libpsl is required for using a public suffix list to check for valid
+       cookie domains.
+
    For those who might be confused as to what to do once they check out
    the source code, considering configure and Makefile do not yet exist at
    that point, a shell script called bootstrap.sh has been provided. After
@@ -96,21 +99,21 @@ Compiling From Repository Sources
 
  Originally written by Hrvoje Niksic <hniksic@xemacs.org>.
 
-     * [35]Edit
-     * [36]Comments
-     * [37]Info
-     * [38]Attachments
+     * [36]Edit
+     * [37]Comments
+     * [38]Info
+     * [39]Attachments
      * More Actions:
        [Raw Text................] Do
 
-     * [39]MoinMoin Powered
-     * [40]Python Powered
-     * [41]GPL licensed
-     * [42]Valid HTML 4.01
+     * [40]MoinMoin Powered
+     * [41]Python Powered
+     * [42]GPL licensed
+     * [43]Valid HTML 4.01
      __________________________________________________________________
 
    All content © 2007 Free Software Foundation. For terms of use,
-   redistribution, and modification, please see the [43]WikiLicense page.
+   redistribution, and modification, please see the [44]WikiLicense page.
 
 References
 
@@ -129,12 +132,13 @@ References
   32. http://www.gnu.org/software/libidn/
   33. http://www.gnu.org/software/libiconv/
   34. http://git-scm.com/
-  35. http://wget.addictivecode.org/CompilingRepoSources?action=edit&editor=text
-  36. http://wget.addictivecode.org/CompilingRepoSources
-  37. http://wget.addictivecode.org/CompilingRepoSources?action=info
-  38. http://wget.addictivecode.org/CompilingRepoSources?action=AttachFile
-  39. http://moinmo.in/
-  40. http://moinmo.in/Python
-  41. http://moinmo.in/GPL
-  42. http://validator.w3.org/check?uri=referer
-  43. http://wget.addictivecode.org/WikiLicense
+  35. https://github.com/rockdaboot/libpsl
+  36. http://wget.addictivecode.org/CompilingRepoSources?action=edit&editor=text
+  37. http://wget.addictivecode.org/CompilingRepoSources
+  38. http://wget.addictivecode.org/CompilingRepoSources?action=info
+  39. http://wget.addictivecode.org/CompilingRepoSources?action=AttachFile
+  40. http://moinmo.in/
+  41. http://moinmo.in/Python
+  42. http://moinmo.in/GPL
+  43. http://validator.w3.org/check?uri=referer
+  44. http://wget.addictivecode.org/WikiLicense

bootstrap.conf

@@ -49,6 +49,7 @@ iconv
 iconv-h
 listen
 maintainer-makefile
+mbiter
 mbtowc
 mkdir
 mkstemp
@@ -77,6 +78,7 @@ update-copyright
 vasprintf
 vsnprintf
 write
+xstrndup
 "
 
 gnulib_extra_files="

build-aux/build_info.pl

@@ -100,7 +100,7 @@ EOC
 EOC
             }
                 print $fh <<EOC;
-#else 
+#else
   "-$name",
 #endif
 

cfg.mk

@@ -4,7 +4,6 @@ local-checks-to-skip = \
     sc_const_long_option \
     sc_error_message_uppercase \
     sc_file_system \
-    sc_makefile_TAB_only_indentation \
     sc_immutable_NEWS \
     sc_copyright_check \
     sc_makefile_path_separator_check \
@@ -13,10 +12,8 @@ local-checks-to-skip = \
     sc_two_space_separator_in_usage \
     sc_useless_cpp_parens \
     \
-    sc_space_tab \
     sc_prohibit_atoi_atof \
     sc_prohibit_strcmp \
-    sc_makefile_check \
     sc_cast_of_alloca_return_value \
     sc_cast_of_x_alloc_return_value \
     sc_prohibit_S_IS_definition \
@@ -27,3 +24,5 @@ local-checks-to-skip = \
 #SHELL=bash -x
 show-vc-list-except:
 	@$(VC_LIST_EXCEPT)
+
+VC_LIST_ALWAYS_EXCLUDE_REGEX = ^ABOUT-NLS|((msdos|vms)/.*)$$

configure.ac

@@ -1,6 +1,6 @@
 dnl Template file for GNU Autoconf
-dnl Copyright (C) 1995, 1996, 1997, 2001, 2007, 2008, 2009, 2010, 2011
-dnl Free Software Foundation, Inc.
+dnl Copyright (C) 1995, 1996, 1997, 2001, 2007, 2008, 2009, 2010, 2011, 2012,
+dnl 2013, 2014 Free Software Foundation, Inc.
 
 dnl This program is free software; you can redistribute it and/or modify
 dnl it under the terms of the GNU General Public License as published by
@@ -61,6 +61,16 @@ dnl
 dnl Process features.
 dnl
 
+ENABLE_PSL=no
+AC_ARG_WITH(libpsl,
+    AS_HELP_STRING([--without-libpsl],
+                   [disable support for libpsl cookie checking.]),
+    [],
+    [AC_SEARCH_LIBS(psl_builtin, psl,
+                    [ENABLE_PSL=yes; AC_DEFINE([HAVE_LIBPSL], [1], [PSL Support Enabled])],
+                    [AC_MSG_WARN(*** libpsl not found. Falling back to Wget builtin cookie checking.)])
+    ])
+
 AC_ARG_WITH(ssl,
 [[  --without-ssl           disable SSL autodetection
   --with-ssl={gnutls,openssl} specify the SSL backend.  GNU TLS is the default.]])
@@ -142,8 +152,7 @@ AC_C_VOLATILE
 
 dnl Check for basic headers, even though we expect them to exist and
 dnl #include them unconditionally in the code.  Their detection is
-dnl still needed because test programs used by Autoconf macros check
-dnl for STDC_HEADERS, HAVE_SYS_TYPES_H, etc. before using them.
+dnl still needed because test programs used by Autoconf macros check.
 dnl Without the checks they will fail to be included in test programs,
 dnl which will subsequently fail.
 AC_HEADER_STDC
@@ -184,9 +193,7 @@ AC_CHECK_TYPES([uint32_t, uintptr_t, intptr_t, int64_t])
 AC_CHECK_TYPES(sig_atomic_t, [], [], [
 #include <stdio.h>
 #include <sys/types.h>
-#if HAVE_INTTYPES_H
-# include <inttypes.h>
-#endif
+#include <inttypes.h>
 #include <signal.h>
 ])
 
@@ -255,7 +262,7 @@ AS_IF([test x"$with_ssl" = xopenssl], [
   case $host_os in
     *mingw32* )
       dnl prefer link to openssl dlls if possible. if not then fallback on static libs. if not then error
-        
+
       AC_CHECK_LIB(eay32, EVP_MD_CTX_init)
       if test x"$ac_cv_lib_eay32_EVP_MD_CTX_init" != xno
       then
@@ -267,7 +274,7 @@ AS_IF([test x"$with_ssl" = xopenssl], [
           AC_DEFINE([HAVE_LIBSSL32], [1], [Define to 1 if you have the `ssl32' library (-lssl32).])
         ],
         AC_MSG_ERROR([openssl not found: shared lib eay32 found but ssl32 not found]))
-          
+
       else
         LIBS+=' -lgdi32'
         dnl fallback and test static libs
@@ -358,6 +365,7 @@ else
   fi
 fi
 
+
 dnl **********************************************************************
 dnl Checks for IPv6
 dnl **********************************************************************
@@ -534,6 +542,12 @@ fi
 dnl
 dnl Check for UUID
 dnl
+AC_CHECK_HEADER(uuid.h,
+                AC_CHECK_FUNC(uuid, uuid_create,
+                  [AC_DEFINE([HAVE_UUID_CREATE], 1,
+                             [Define if uuid_create is available.])
+                  ])
+)
 
 AC_CHECK_HEADER(uuid/uuid.h,
                 AC_CHECK_LIB(uuid, uuid_generate,
@@ -555,7 +569,7 @@ AC_CHECK_HEADER(pcre.h,
                   ])
 )
 
- 
+
 dnl Needed by src/Makefile.am
 AM_CONDITIONAL([IRI_IS_ENABLED], [test "X$iri" != "Xno"])
 
@@ -564,8 +578,8 @@ dnl
 dnl Create output
 dnl
 AC_CONFIG_FILES([Makefile src/Makefile doc/Makefile util/Makefile
-                 po/Makefile.in tests/Makefile tests/WgetTest.pm
-                 lib/Makefile])
+                 po/Makefile.in tests/Makefile
+                 lib/Makefile testenv/Makefile])
 AC_CONFIG_HEADERS([src/config.h])
 AC_OUTPUT
 
@@ -580,6 +594,7 @@ AC_MSG_NOTICE([Summary of build options:
   Libs:              $LIBS
   SSL:               $with_ssl
   Zlib:              $with_zlib
+  PSL:               $ENABLE_PSL
   Digest:            $ENABLE_DIGEST
   NTLM:              $ENABLE_NTLM
   OPIE:              $ENABLE_OPIE

doc/ChangeLog

@@ -1,3 +1,41 @@
+2014-09-08  Darshit Shah  <darnir@gmail.com>
+
+	* wget.texi (symbolic links): Update documentation of retr-symlinks to
+	reflect the new default. Add warning about potential security issues with
+	--retr-symlinks=yes.
+
+2014-10-16  Tim Ruehsen  <tim.ruehsen@gmx.de>
+
+	* wget.texi (Download Options): update --secure-protocol description
+
+2014-08-03  Giuseppe Scrivano  <gscrivano@gnu.org>
+
+	* wget.texi (Download Options): Fix texinfo warning.
+
+2014-06-11  Giuseppe Scrivano  <gscrivan@redhat.com>
+
+	* Makefile.am: Remove @VAR@ with $FOO.
+
+	* fdl.texi: Remove terminating empty lines.
+
+2014-06-10  Giuseppe Scrivano  <gscrivan@redhat.com>
+
+	* wget.texi: Remove trailing whitespaces.
+
+	* fdl.texi: Copy from gnulib.
+
+2014-05-24  Darshit Shah  <darnir@gmail.com>
+
+	* wget.texi: Document noscroll parameter to progress=bar
+
+2014-05-01  Darshit Shah  <darnir@gmail.com>
+
+	* wget.texi: Add documentation for --show-progress
+
+2014-02-10  Yousong Zhou  <yszhou4tech@gmail.com>
+
+	* wget.texi: Add documentation for --start-pos.
+
 2013-12-29  Giuseppe Scrivano  <gscrivan@redhat.com>
 
 	* wget.texi: Update to GFDL 1.3.

doc/Makefile.am

@@ -16,7 +16,7 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 #
-# Version: @VERSION@
+# Version: $(VERSION)
 #
 
 # Program to convert DVI files to PostScript
@@ -28,7 +28,6 @@ manext     = 1
 RM = rm -f
 
 TEXI2POD     = $(srcdir)/texi2pod.pl
-POD2MAN      = @POD2MAN@
 MAN          = wget.$(manext)
 WGETRC       = $(sysconfdir)/wgetrc
 SAMPLERCTEXI = sample.wgetrc.munged_for_texi_inclusion
@@ -39,7 +38,7 @@ SAMPLERCTEXI = sample.wgetrc.munged_for_texi_inclusion
 
 man_MANS = $(MAN)
 
-all: wget.info @COMMENT_IF_NO_POD2MAN@$(MAN)
+all: wget.info $(COMMENT_IF_NO_POD2MAN)$(MAN)
 
 everything: all wget_us.ps wget_a4.ps wget_toc.html
 
@@ -57,7 +56,7 @@ wget.pod: $(srcdir)/wget.texi version.texi
 	$(TEXI2POD) -D VERSION="$(VERSION)" $(srcdir)/wget.texi $@
 
 $(MAN): wget.pod
-	$(POD2MAN) --center="GNU Wget" --release="GNU Wget @VERSION@" $? > $@
+	$(POD2MAN) --center="GNU Wget" --release="GNU Wget $(VERSION)" $? > $@
 
 #wget.cat: $(MAN)
 #	nroff -man $? > $@
@@ -76,10 +75,10 @@ wget_toc.html: $(srcdir)/wget.texi
 #
 
 # install all the documentation
-install-data-local: install.wgetrc @COMMENT_IF_NO_POD2MAN@install.man
+install-data-local: install.wgetrc $(COMMENT_IF_NO_POD2MAN)install.man
 
 # uninstall all the documentation
-uninstall-local: @COMMENT_IF_NO_POD2MAN@uninstall.man
+uninstall-local: $(COMMENT_IF_NO_POD2MAN)uninstall.man
 
 
 # install man page, creating install directory if necessary

doc/fdl.texi

@@ -5,8 +5,7 @@
 @c hence no sectioning command or @node.
 
 @display
-Copyright @copyright{} 2000, 2001, 2002, 2007, 2008, 2009, 2010, 2011
-Free Software Foundation, Inc.
+Copyright @copyright{} 2000, 2001, 2002, 2007, 2008 Free Software Foundation, Inc.
 @uref{http://fsf.org/}
 
 Everyone is permitted to copy and distribute verbatim copies
@@ -93,16 +92,16 @@ An image format is not Transparent if used for any substantial amount
 of text.  A copy that is not ``Transparent'' is called ``Opaque''.
 
 Examples of suitable formats for Transparent copies include plain
-@sc{ascii} without markup, Texinfo input format, La@TeX{} input
-format, @acronym{SGML} or @acronym{XML} using a publicly available
-@acronym{DTD}, and standard-conforming simple @acronym{HTML},
-PostScript or @acronym{PDF} designed for human modification.  Examples
-of transparent image formats include @acronym{PNG}, @acronym{XCF} and
-@acronym{JPG}.  Opaque formats include proprietary formats that can be
-read and edited only by proprietary word processors, @acronym{SGML} or
-@acronym{XML} for which the @acronym{DTD} and/or processing tools are
-not generally available, and the machine-generated @acronym{HTML},
-PostScript or @acronym{PDF} produced by some word processors for
+ASCII without markup, Texinfo input format, La@TeX{} input
+format, SGML or XML using a publicly available
+DTD, and standard-conforming simple HTML,
+PostScript or PDF designed for human modification.  Examples
+of transparent image formats include PNG, XCF and
+JPG@.  Opaque formats include proprietary formats that can be
+read and edited only by proprietary word processors, SGML or
+XML for which the DTD and/or processing tools are
+not generally available, and the machine-generated HTML,
+PostScript or PDF produced by some word processors for
 output purposes only.
 
 The ``Title Page'' means, for a printed book, the title page itself,
@@ -482,7 +481,7 @@ license notices just after the title page:
 @end smallexample
 
 If you have Invariant Sections, Front-Cover Texts and Back-Cover Texts,
-replace the ``with@dots{}Texts.'' line with this:
+replace the ``with@dots{}Texts.''@: line with this:
 
 @smallexample
 @group
@@ -504,4 +503,3 @@ to permit their use in free software.
 @c Local Variables:
 @c ispell-local-pdict: "ispell-dict"
 @c End:
-

doc/wget.texi

@@ -701,6 +701,22 @@ Another instance where you'll get a garbled file if you try to use
 Note that @samp{-c} only works with @sc{ftp} servers and with @sc{http}
 servers that support the @code{Range} header.
 
+@cindex offset
+@cindex continue retrieval
+@cindex incomplete downloads
+@cindex resume download
+@cindex start position
+@item --start-pos=@var{OFFSET}
+Start downloading at zero-based position @var{OFFSET}.  Offset may be expressed
+in bytes, kilobytes with the `k' suffix, or megabytes with the `m' suffix, etc.
+
+@samp{--start-pos} has higher precedence over @samp{--continue}.  When
+@samp{--start-pos} and @samp{--continue} are both specified, wget will emit a
+warning then proceed as if @samp{--continue} was absent.
+
+Server support for continued download is required, otherwise @samp{--start-pos}
+cannot help.  See @samp{-c} for details.
+
 @cindex progress indicator
 @cindex dot style
 @item --progress=@var{type}
@@ -716,7 +732,12 @@ Use @samp{--progress=dot} to switch to the ``dot'' display.  It traces
 the retrieval by printing dots on the screen, each dot representing a
 fixed amount of downloaded data.
 
-When using the dotted retrieval, you may also set the @dfn{style} by
+The progress @var{type} can also take one or more parameters.  The parameters
+vary based on the @var{type} selected.  Parameters to @var{type} are passed by
+appending them to the type sperated by a colon (:) like this:
+@samp{--progress=@var{type}:@var{parameter1}:@var{parameter2}}.
+
+When using the dotted retrieval, you may set the @dfn{style} by
 specifying the type as @samp{dot:@var{style}}.  Different styles assign
 different meaning to one dot.  With the @code{default} style each dot
 represents 1K, there are ten dots in a cluster and 50 dots in a line.
@@ -729,11 +750,35 @@ If @code{mega} is not enough then you can use the @code{giga}
 style---each dot represents 1M retrieved, there are eight dots in a
 cluster, and 32 dots on each line (so each line contains 32M).
 
+With @samp{--progress=bar}, there are currently two possible parameters,
+@var{force} and @var{noscroll}.
+
+When the output is not a TTY, the progress bar always falls back to ``dot'',
+even if @samp{--progress=bar} was passed to Wget during invokation. This
+behaviour can be overridden and the ``bar'' output forced by using the ``force''
+parameter as @samp{--progress=bar:force}.
+
+By default, the @samp{bar} style progress bar scroll the name of the file from
+left to right for the file being downloaded if the filename exceeds the maximum
+length allotted for its display.  In certain cases, such as with
+@samp{--progress=bar:force}, one may not want the scrolling filename in the
+progress bar.  By passing the ``noscroll'' parameter, Wget can be forced to
+display as much of the filename as possible without scrolling through it.
+
 Note that you can set the default style using the @code{progress}
 command in @file{.wgetrc}.  That setting may be overridden from the
-command line.  The exception is that, when the output is not a TTY, the
-``dot'' progress will be favored over ``bar''.  To force the bar output,
-use @samp{--progress=bar:force}.
+command line.  For example, to force the bar output without scrolling,
+use @samp{--progress=bar:force:noscroll}.
+
+@item --show-progress
+Force wget to display the progress bar in any verbosity.
+
+By default, wget only displays the progress bar in verbose mode.  One may
+however want wget to display the progress bar on screen in conjunction with
+any other verbosity modes like @samp{--no-verbose} or @samp{--quiet}.  This
+is often a desired a property when invoking wget to download several small/large
+files.  In such a case, wget could simply be invoked with this parameter to get
+a much cleaner output on the screen.
 
 @item -N
 @itemx --timestamping
@@ -890,7 +935,8 @@ Don't use proxies, even if the appropriate @code{*_proxy} environment
 variable is defined.
 
 @c man end
-For more information about the use of proxies with Wget, @xref{Proxies}.
+@xref{Proxies}, for more information about the use of proxies with
+Wget.
 @c man begin OPTIONS
 
 @cindex quota
@@ -1232,8 +1278,8 @@ really important, do not leave them lying in those files either---edit
 the files and delete them after Wget has started the download.
 
 @iftex
-For more information about security issues with Wget, @xref{Security
-Considerations}.
+@xref{Security Considerations}, for more information about security
+issues with Wget.
 @end iftex
 
 @cindex Keep-Alive, turning off
@@ -1595,16 +1641,16 @@ without SSL support, none of these options are available.
 @cindex SSL protocol, choose
 @item --secure-protocol=@var{protocol}
 Choose the secure protocol to be used.  Legal values are @samp{auto},
-@samp{SSLv2}, @samp{SSLv3}, @samp{TLSv1} and @samp{PFS}.  If @samp{auto}
-is used, the SSL library is given the liberty of choosing the appropriate
-protocol automatically, which is achieved by sending an SSLv2 greeting
-and announcing support for SSLv3 and TLSv1.  This is the default.
+@samp{SSLv2}, @samp{SSLv3}, @samp{TLSv1}, @samp{TLSv1_1}, @samp{TLSv1_2}
+and @samp{PFS}.  If @samp{auto} is used, the SSL library is given the
+liberty of choosing the appropriate protocol automatically, which is
+achieved by sending a TLSv1 greeting. This is the default.
 
-Specifying @samp{SSLv2}, @samp{SSLv3}, or @samp{TLSv1} forces the use
-of the corresponding protocol.  This is useful when talking to old and
-buggy SSL server implementations that make it hard for the underlying
-SSL library to choose the correct protocol version.  Fortunately, such
-servers are quite rare.
+Specifying @samp{SSLv2}, @samp{SSLv3}, @samp{TLSv1}, @samp{TLSv1_1} or
+@samp{TLSv1_2} forces the use of the corresponding protocol.  This is
+useful when talking to old and buggy SSL server implementations that
+make it hard for the underlying SSL library to choose the correct
+protocol version.  Fortunately, such servers are quite rare.
 
 Specifying @samp{PFS} enforces the use of the so-called Perfect Forward
 Security cipher suites. In short, PFS adds security by creating a one-time
@@ -1769,8 +1815,8 @@ really important, do not leave them lying in those files either---edit
 the files and delete them after Wget has started the download.
 
 @iftex
-For more information about security issues with Wget, @xref{Security
-Considerations}.
+@xref{Security Considerations}, for more information about security
+issues with Wget.
 @end iftex
 
 @cindex .listing files, removing
@@ -1837,17 +1883,18 @@ Preserve remote file permissions instead of permissions set by umask.
 
 @cindex symbolic links, retrieving
 @item --retr-symlinks
-Usually, when retrieving @sc{ftp} directories recursively and a symbolic
-link is encountered, the linked-to file is not downloaded.  Instead, a
-matching symbolic link is created on the local filesystem.  The
-pointed-to file will not be downloaded unless this recursive retrieval
-would have encountered it separately and downloaded it anyway.
+By default, when retrieving @sc{ftp} directories recursively and a symbolic link
+is encountered, the symbolic link is traversed and the pointed-to files are
+retrieved.  Currently, Wget does not traverse symbolic links to directories to
+download them recursively, though this feature may be added in the future.
 
-When @samp{--retr-symlinks} is specified, however, symbolic links are
-traversed and the pointed-to files are retrieved.  At this time, this
-option does not cause Wget to traverse symlinks to directories and
-recurse through them, but in the future it should be enhanced to do
-this.
+When @samp{--retr-symlinks=no} is specified, the linked-to file is not
+downloaded.  Instead, a matching symbolic link is created on the local
+filesystem.  The pointed-to file will not be retrieved unless this recursive
+retrieval would have encountered it separately and downloaded it anyway.  This
+option poses a security risk where a malicious FTP Server may cause Wget to
+write to files outside of the intended directories through a specially crafted
+@sc{.listing} file.
 
 Note that when retrieving a file (not a directory) because it was
 specified on the command-line, rather than because it was recursed to,

msdos/ChangeLog

@@ -1,3 +1,9 @@
+2014-06-11  Giuseppe Scrivano  <gscrivan@redhat.com>
+
+	* Makefile.DJ: Remove terminating empty lines.
+	* Makefile.WC: Likewise.
+	* msdos.c: Likewise.
+
 2009-09-06  Gisle Vanem  <gvanem@broadpark.no>
 
 	* Makefile.WC: Added compilation of new file msdos.c.

msdos/Makefile.DJ

@@ -87,4 +87,3 @@ depend: version.c
         sed -e 's/^\([a-zA-Z0-9_-]*\.o:\)/$$(OBJ_DIR)\/\1/' > depend.dj
 
 -include depend.dj
-

msdos/Makefile.WC

@@ -65,4 +65,3 @@ version.c: ..\msdos\Makefile.WC
 clean: .SYMBOLIC
 	- rm $(OBJ_DIR)\*.obj wget.exe wget.map version.c css.c
 	- rmdir $(OBJ_DIR)
-

msdos/msdos.c

@@ -105,4 +105,3 @@ getpass (const char *prompt)
 
   return strdup (getpassbuf);
 }
-

po/POTFILES.in

@@ -8,6 +8,7 @@ lib/error.c
 lib/gai_strerror.c
 lib/getopt.c
 lib/quotearg.c
+lib/regcomp.c
 lib/spawn-pipe.c
 lib/w32spawn.h
 lib/wait-process.c

src/ChangeLog

@@ -1,3 +1,419 @@
+2014-09-08  Darshit Shah  <darnir@gmail.com>
+
+	* ftp.c (ftp_retrieve_glob): Also check for invalid entries along with
+	harmful filenames
+	(is_valid_entry): New function. Check if the provided node is a valid entry
+	in a listing file.
+
+2014-09-08  Darshit Shah  <darnir@gmail.com>
+
+	* init.c (defaults): Set retr-symlinks to true by default. This changes a
+	default setting of wget. Fixes security bug CVE-2014-4877
+
+2014-10-08 Nikolay Morozov <n.morozov@securitycode.ru>
+	   Sergey Lvov <s.lvov@securitycode.ru>
+
+	* init.c (cmd_spec_secure_protocol): Add support for
+	TLS v1.1 and TLS v1.2 protocols
+	* openssl.c (ssl_init): Add support for OpenSSL engines
+
+2014-10-22  Ángel González <keisial@gmail.com>
+
+	* css-url.c (get_uri_string): Honor the specified length argument.
+
+2014-10-16  Tim Ruehsen  <tim.ruehsen@gmx.de>
+
+	* url.c (url_parse): little code cleanup
+	* html-url.c (get_urls_html): HTTP header Content-Type charset preceeds http-equiv
+	* iri.c (do_conversion): moved iconv code completely into the function
+	* iri.c (do_conversion): call url_unescape to fix charset conversion
+	* iri.c (remote_to_utf8): use strcasecmp to compare encoding
+
+	* gnutls.c (ssl_connect_wget): do not use SSLv3 except explicitely requested
+	* openssl.c (ssl_init): do not use SSLv3 except explicitely requested
+
+2014-05-03  Tim Ruehsen  <tim.ruehsen@gmx.de>
+
+	* retr.c (retrieve_url): fixed memory leak
+
+2014-07-23  Darshit Shah  <darnir@gmail.com>
+
+	* http.c (gethttp): Fix a memory leak when retrying authorization
+	(gethttp): Fix memory leak when trying to parse content disposition headers
+	(http_loop): Assigning a new value to *local)file without freeing the old
+	one causes a memory leak
+	(http_loop): Free the HTTP message and error strings before continuing loop
+
+2014-07-21  Daniel Stenberg  <daniel@haxx.se>
+
+	* main.c (print_help): HTTP Method is a part of the Request not Header
+
+2014-07-21  Darshit Shah  <darnir@gmail.com>
+
+	* cookies.c (check_domain_match): Fix a potential memory leak when checking
+	cookie domain names
+
+2014-07-07  Tomas Hozza  <thozza@redhat.com>
+
+	* iri.c (locale_to_utf8): Fix checking of iconv_open return code.
+
+2014-07-05  Darshit Shah  <darnir@gmail.com>
+
+	* cookies.c (check_domain_match): Libpsl requires that all domain names
+	passed to it be in utf8 lower case.
+
+2014-07-05  Darshit Shah  <darnir@gmail.com>
+
+	* http.c (gethttp): Fix indentation of conditional block
+	(gethttp): Remove unneeded variable
+
+2014-07-03  Darshit Shah  <darnir@gmail.com>
+
+	* wget.h (uerr_t): Remove unused error codes
+	* http.c: (http_loop): Remove reference to unused error code
+
+2014-06-30  Giuseppe Scrivano  <gscrivan@redhat.com>
+
+	* convert.c (local_quote_string): Initialize newname.
+
+2014-06-29  Giuseppe Scrivano  <gscrivan@redhat.com>
+
+	* warc.c (warc_write_date_header): Avoid out-of-scope variable usage.
+
+2014-06-28  Giuseppe Scrivano  <gscrivan@redhat.com>
+
+	* Makefile.am (wget_SOURCES): Remove space-tab indentation.
+
+2014-06-22  Giuseppe Scrivano  <gscrivan@redhat.com>
+
+	* warc.c [HAVE_UUID_CREATE]: Include <uuid.h>
+	(warc_uuid_str) [HAVE_UUID_CREATE]: Use uuid_create and
+	uuid_to_string to generate the UUID.
+	Reported by: Alex Zimnitsky <aavzz@yandex.ru>
+
+2014-06-22  Darshit Shah  <darnir@gmail.com> (tiny change)
+
+	* progress.c (create_image): Align percentage download output better.
+
+2014-06-21  Giuseppe Scrivano  <gscrivan@redhat.com>
+
+	* mswindows.c (fake_fork_child): Fix build error.
+	Reported by: Gisle Vanem <gvanem@yahoo.no>.
+
+2014-06-21  Nikita Vetrov  <admin@kanaria.ru> (tiny change)
+
+	* openssl.c: Add one-time call OPENSSL_config in ssl_init.
+
+2014-06-19  Giuseppe Scrivano  <gscrivan@redhat.com>
+
+	* mswindows.c (fake_fork_child): Revert dinamic allocation of
+	info->lfilename.
+	Reported by: Gisle Vanem <gvanem@yahoo.no>.
+
+2014-06-16  Darshit Shah  <darnir@gmail.com>
+
+	* test.c: Include locale.h header for gettext.
+
+2014-06-16  Giuseppe Scrivano  <gscrivan@redhat.com>
+
+	* http.c (gethttp): Set "Connection: Keep-Alive" with keep-alive
+	connections when using a proxy too.
+	Reported by: Thorsten Schroeteler <Thorsten.Schroeteler@cassidian.com>.
+
+2014-06-16  Darshit Shah  <darnir@gmail.com>
+
+	* main.c (print_help, print_version): Mark as noreturn.
+	* utils.c (memfatal, abort_run_with_timout): Same
+
+2014-06-11  Giuseppe Scrivano  <gscrivan@redhat.com>
+
+	* ftp.c: Replace main() with main in comments.
+	* http.c: Likewise.
+	* init.c: Likewise.
+
+	* cmpt.c (day_of_the_week): Remove repeated word in a comment.
+	* utils.c: Remove repeated word in a comment.
+
+	* gnutls.c: Do not include <assert.h>.
+
+	* connect.c: Do not depend on always defined macros.
+	* convert.c: Likewise.
+	* css-url.c: Likewise.
+	* hash.c: Likewise.
+	* mswindows.h: Likewise.
+	* netrc.c: Likewise.
+	* progress.c: Likewise.
+	* sysdep.h: Likewise.
+	* utils.c: Likewise.
+	* wget.h: Likewise.
+
+	* Makefile.am: Remove @VAR@ with $FOO.
+
+	* DESCRIP_MODS.MMS: Remove terminating empty lines.
+	* DESCRIP_SRC.MMS: Likewise.
+	* convert.c: Likewise.
+	* exits.c: Likewise.
+	* http.c: Likewise.
+	* init.c: Likewise.
+	* iri.c: Likewise.
+	* res.c: Likewise.
+	* spider.c: Likewise.
+	* test.c: Likewise.
+	* test.h
+	* url.c: Likewise.
+	* utils.c: Likewise.
+	* vms.c: Likewise.
+
+	* decc_ver.c (main): Ensure bindtextdomain is called.
+	* hash.c (main): Likewise.
+	* html-parse.c: Likewise.
+	* netrc.c: Likewise.
+	* test.c: Likewise.
+	* trunc.c: Likewise.
+
+2014-06-10  Giuseppe Scrivano  <gscrivan@redhat.com>
+
+	* wget.h: Remove trailing whitespaces.
+
+	* retr.c (getproxy): Return a dinamically allocated string and...
+	(retrieve_from_file, retrieve_url, url_uses_proxy): ...fix the caller
+	to handle it.
+	* init.c (home_dir): Replace strncpy with strdup.
+	* mswindows.c (struct fake_fork_info): Make lfilename a pointer.
+	(fake_fork_child): Replace strncpy with strdup.
+	* http.c (ensure_extension): Replace strncpy with memcpy, not much
+	better but make "make syntax-check" happy and we know the size.
+	* ftp.c (getftp): Add parameter last_expected_bytes.
+	(ftp_loop_internal): Pass parameter last_expected_bytes to getftp.
+	* ftp-basic.c: Remove declaration of ftp_last_respline.
+	(ftp_response): Do not set ftp_last_respline.
+	* css-url.c (get_uri_string): Replace strncpy with strdup.
+	* vms.c (set_vms_name): Replace strncpy with strdup.
+
+	* exits.c: Move WGET_EXIT_* definitions to...
+	* exits.h: ...here.  Add WGET_EXIT_GENERIC_ERROR, WGET_EXIT_PARSE_ERROR.
+	Remove WGET_EXIT_MINIMUM.
+	* init.c: Fix calls to exit().
+	* log.c: Likewise.
+	* main.c: Likewise.
+	* mswindows.c: Likewise.
+	* netrc.c: Likewise.
+	* utils.c: Likewise.
+	* warc.c: Likewise.
+	* Test-stdouterr.px: Likewise.
+
+2014-06-08  Giuseppe Scrivano  <gscrivan@redhat.com>
+
+	* main.c: Make `program_name' not static.
+
+	* cookies.c [HAVE_PSL]: Include <libpsl.h> only when HAVE_PSL is defined.
+
+2014-05-30  Darshit Shah  <darnir@gmail.com>
+
+	* cookies.c (check_domain_match): Use libpsl to check if the cookie domain
+	is valid. Also remove unneeded test for numeric addresses.
+
+2014-05-30  Darshit Shah  <darnir@gmail.com>
+
+	* connect.{c,h}, convert.{c,h}, cookies.{c,h}, ftp-ls.c, ftp.h, gettext.h,
+	hash.h, host.h, html-parse.h, html-url.h, http.c, init.c, main.c, mswindows.c,
+	netrc.h, openssl.c, options.h, ptimer.h, recur.c, retr.c, sysdep.h, url.h,
+	vms.c, warc.c, wget.h: Whitespace and formatting changes only.
+
+2014-05-20  Darshit Shah  <darnir@gmail.com>
+
+	* utils.c (human_readable): Add new parameters, acc and decimals for the
+	function. acc decides the number under which decimal values are shown for a
+	certain value, while decimals decides the number of decimal digits displayed
+	* utils.h (human_readable): Update declartion of the function
+	* ftp.c (print_length): Update call to human_readable
+	* http.c (gethttp): Same
+	* init.c (defaults): Enable scrolling filenames by default
+	* main.c (main): Update call to human_readable
+	* options.h (options): Add new option noscroll
+	* progress.c (create_image): Update the look of the progress bar. Human
+	readable download size by default and add support for noscroll.
+	(dot_set_params, bar_set_params): Change paramter to char * since strtok
+	needs to modify the string.
+	(bar_set_params): Add support for noscroll parameter to bar.
+
+2014-05-03  Tim Ruehsen  <tim.ruehsen@gmx.de>
+
+	* ftp-ls.c (ftp_parse_vms_ls): Explicitly typecast strlen's output
+	* ftp.c (getftp): Prevent declaration of shadow variable
+	* html-url.c (tag_handle_base, tag_handle_form, tag_handle_link,
+	tag_handle_meta): Compiler directive to ignore unused variable
+	* html-url.h (cleanup_html_url): Add function declaration
+	* http-ntlm.c (ntlm_input, mkhash): Use ssize_t to declare variables that store
+	sizes.
+	(mkhash): Explicitly typecast output of c_toupper
+	(short_pair): Add typecasts to prevent compiler warnings
+	(ntlm-output): Fix datatypes of various variables
+	* http.c (gethttp): Prevent declaration of shadow variable err
+	(gethttp): remove unreachable code
+	(test_parse_content_disposition): Fix variable declarations and use
+	countof() macro
+	* init.c (run_command): Prevent declaration of ghost variable
+	(cmd_string, cmd_string_uppercase, cmd_file, cnd_vector,
+	cmd_directory_vector, cmd_spec_dirstruct, cmd_spec_header,
+	cmd_spec_warc_header, cmd_spec_htmlify, cmd_spec_mirror,
+	cmd_spec_prefer_family, cmd_spec_progress, cmd_spec_recursive,
+	cmd_spec_regex_type, cmd_spec_restrict_file_names, cmd_spec_report_speed,
+	cmd_spec_timeout, cmd_spec_useragent, cmd_spec_verbose): Add compiler
+	directive to ignore unused variable
+	(cleanup_html_url, spider_cleanup): Remove declarations
+	(test_commands_sorted): Cleanup code
+	(test_cmd_spec_restrict_file_names): Use correct data types
+	* iri.c (remote_to_utf8): Use more verbose variable name
+	* main.c (init_switches): Prevent declaration of ghost variable
+	* netrc.c (netrc_cleanup): Create cleanup function on common naming scheme
+	* netrc.h (netrc_cleanup): Declare function
+	* openssl.c (openssl_write, openssl_errstr): Compiler directive to ignore
+	unused parameter
+	(openssl_errstr): Explcicitly typecasr output of ASN1_STRING_length
+	* options.h (struct options): Declare includes and excludes as const char **
+	* progress.c (progress_interactive_p, progress_handle_sigwinch): Compiler
+	directive to ignore unused parameter
+	* res.c (test_is_robots_txt_url): Fix datatypes
+	* retr.c (line_terminator): Add compiler directive for ignoring unused
+	paramter. Remove unused variable.
+	* spider.h (spider_cleanup): Declare function
+	* test,c (main): declare unused paramter
+	* test.h (test_*): Declare functions
+	* url.c (url_string): Explicit typecast of password strings
+	(run_test): Declare *test, struct tests and *expected_result as const
+	(test_path_simplify, test_append_uri_pathl, test_are_urls_equal): Fix datatypes
+	* utls.c (fork_to_background): Be more verbose when errors occur
+	(dir_matches_p, test_dir_matches_p): Declare char * as const as required
+	(base64_encode, base64_decode, get_max_length): Set correct return type
+	(match_pcre_regex, match_posix_regex): Use correct datatypes and typecasts
+	(test_subdir_p): static const struct
+	* utils.h (base64_encode, base64_decode, get_max_length): Fix return type
+	* warc.c (ward_write_cdx_record): unused parameter
+	(struct hash_table *warc_cdx_dedup_table): Declare as static
+
+2014-05-01  Benjamin Goose  <gans+wget@tngtech.com>
+
+	* http.c: Fix small memory leak
+
+2014-05-01  Darshit Shah  <darnir@gmail.com> (tiny change)
+
+	* progress.c (dot_finish): Do not print extra newlines when not in verbose
+	mode. (Purely aesthetic change)
+	(get_eta): Add extra space when eta is printed.
+	(create_image): Remove erroneous space from being added to progress bar when
+	filename > MAX_FILENAME_LEN
+	(create_image): Remove extra space before printed download speeds
+	Make the filename a fixed width column.
+
+2014-04-19  Darshit Shah  <darnir@gmail.com>
+
+	* log.h (log_options): Add new logging options, LOG_PROGRESS. All progress
+	bar related output should use LOG_PROGRESS.
+	* log.c (CHECK_VERBOSE): Implement LOG_PROGRESS output
+	* progress.c (dot_create, print_row_stats, dot_update, dot_finish,
+	bar_finish, display_image): Output progress information through LOG_PROGRESS
+	(progress_implementation, dot_create, bar_create, progress_create): The
+	progress bar create functions accept an extra paramter for the filename of the
+	local file
+	(bar_progress): Add new variable to store filename of currently downloading
+	file
+	(bar_finish): Aesthetic change. Print two newlines if in verbose mode, else
+	only one.
+	(MACRO): Define new macro, MIN
+	(create_image): Implement displaying filename in progress bar output
+	Filename tick implementation copied from Giuseppe's patch on parallel-wget
+	* progress.h (progress_create): Accept another parameter for filename
+	* http.c (gethttp): Remove unnecessary conditional
+	(read_response_body): Send local filename to fd_read_body so that it can be
+	printed on the progress bar
+	* main.c (option_data): Add new switch, --show-progress
+	(main): If in verbose output, show progress bar by default
+	(main): Set progress implemetation when displaying progress bar
+	(no_prefix): Increase buffer size to 2048 to prevent overflows
+	* init.c (commands): Add new command, showprogress
+	(defaults): By default initialize show_progress to false
+	* options.h (options): Add new option, show_progress
+	* retr.c (fd_read_body): Accept new parameter for filename of currently
+	downlaoding file
+	(fd_read_body): Create and update progress bar when opt.show_progress is set
+	(fd_read_body): Display progress information in Windows console titlebars
+	* retr.h (fd_read_body): Update declaration
+	* ftp.c (getftp): Send filename of the local file so that it can be printed
+	with the progress bar.
+
+2013-11-02  Giuseppe Scrivano  <gscrivan@redhat.com>
+
+	* progress.c (struct progress_implementation): New method draw.
+	(bar_update): Split into...
+	(bar_draw): ...this new function.
+	(dot_update): Split into...
+	(dot_draw): New function.
+	(progress_update): Also invoke draw on `current_impl'.
+
+2014-04-22  Tim Ruehsen  <tim.ruehsen@gmx.de>
+
+	* http.c (gethttp): Fix 204 response handling
+
+2014-03-26  Darshit Shah  <darnir@gmail.com>
+
+	* ftp.c (getftp): Rearrange parameters to fix compiler warning
+	* utils.c (get_dir_matches_p): Do not pass a const char** to a function that
+	expects char**
+
+2014-03-25  Daniel Stenberg  <daniel@haxx.se>
+
+	* url.c (shorten_length): Remove unused function.
+
+2014-03-19  Yousong Zhou  <yszhou4tech@gmail.com>
+
+	* init.c, main.c, options.h: Add option --start-pos for specifying
+	start position of a download.
+	* http.c: Utilize opt.start_pos for HTTP download.
+	* ftp.c: Utilize opt.start_pos for FTP retrieval.
+
+2014-03-04  Giuseppe Scrivano  <gscrivan@redhat.com>
+
+	* http.c (modify_param_value, extract_param): Aesthetic change.
+
+2014-02-14  Vladimír Pýcha  <vpycha@gmail.com> (tiny change)
+
+	* http.c (parse_content_disposition, extract_param)
+	(append_value_to_filename, digest_authentication_encode): URL-decode the
+	filename parameter of Content-Disposition HTTP header if it is encoded. This
+	is related to --content-disposition.
+	New parameter of extract_param(), "is_url_encoded".
+	Add argument NULL to the call of extract_param() in
+	digest_authentication_encode().
+	* http.h: Add the new parameter to the declaration of extract_param().
+	* cookies.c (parse_set_cookie, test_cookies): Add argument NULL to the calls
+	of extract_param().
+	* url.c (url_unescape): Remove "static" modifier.
+	* url.h: Add declaration of url_unescape().
+
+2014-02-06  Giuseppe Scrivano  <gscrivan@redhat.com>
+
+	* main.c (print_version): Move copyright year out of the localized
+	string and update it.
+
+2014-01-29  Darshit Shah  <darnir@gmail.com>
+
+	* main.c: Remove pre-processor variable WHEN_DEBUG
+	(option_data[]): Do not fail on --debug even if debug support is not
+	compiled in.
+	(main): Explicitly set opt.debug to false in case debugging support was not
+	compiled.
+	* init.c (commands[]): Support --debug wven when support is not compiled in.
+	* options.h: Same
+
+2014-01-17  Darshit Shah  <darnir@gmail.com>
+
+	* init.c (commands[]): Add --no-config.
+	* options.h: Same.
+	* main.c (option_data[]): Same.
+	(print_help): Same.
+	(main): If --no-config is set, then do not read the wgetrc files.
+
 2014-01-05  Håkon Vågsether <hauk142@gmail.com> (tiny change)
 
 	* http.c (http_loop): Fix checking the URL length when filename is

src/DESCRIP_MODS.MMS

@@ -56,4 +56,3 @@ MODS_OBJS_LIB_SRC_SSL =
 .ENDIF                          # CDEFS_SSL [else]
 
 MODS_OBJS_LIB_SRC = $(MODS_OBJS_LIB_SRC_NOSSL) $(MODS_OBJS_LIB_SRC_SSL)
-

src/DESCRIP_SRC.MMS

@@ -32,4 +32,3 @@ CFLAGS_SPEC = /prefix_library_entries = \
 # Object library modules.
 
 .INCLUDE DESCRIP_MODS.MMS
-

src/Makefile.am

@@ -28,7 +28,7 @@
 # as that of the covered work.
 
 #
-# Version: @VERSION@
+# Version: $(VERSION)
 #
 
 if IRI_IS_ENABLED
@@ -36,24 +36,24 @@ IRI_OBJ = iri.c
 endif
 
 # The following line is losing on some versions of make!
-DEFS     = @DEFS@ -DSYSTEM_WGETRC=\"$(sysconfdir)/wgetrc\" -DLOCALEDIR=\"$(localedir)\"
-LIBS     = @LIBICONV@ @LIBINTL@ @LIBS@ $(LIB_CLOCK_GETTIME)
+DEFS     += -DSYSTEM_WGETRC=\"$(sysconfdir)/wgetrc\" -DLOCALEDIR=\"$(localedir)\"
+LIBS     += $(LIBICONV) $(LIBINTL) $(LIB_CLOCK_GETTIME)
 
 EXTRA_DIST = css.l css.c css_.c build_info.c.in
 
 bin_PROGRAMS = wget
-wget_SOURCES = cmpt.c connect.c convert.c cookies.c ftp.c    		  \
-	       css_.c css-url.c \
-	       ftp-basic.c ftp-ls.c hash.c host.c html-parse.c html-url.c \
-	       http.c init.c log.c main.c netrc.c progress.c ptimer.c     \
-	       recur.c res.c retr.c spider.c url.c warc.c	          	  \
-	       utils.c exits.c build_info.c $(IRI_OBJ)			  \
-	       css-url.h css-tokens.h connect.h convert.h cookies.h       \
-	       ftp.h hash.h host.h html-parse.h html-url.h      \
-	       http.h http-ntlm.h init.h log.h mswindows.h netrc.h        \
-	       options.h progress.h ptimer.h recur.h res.h retr.h         \
-	       spider.h ssl.h sysdep.h url.h warc.h utils.h wget.h iri.h 	  \
-	       exits.h gettext.h
+wget_SOURCES = cmpt.c connect.c convert.c cookies.c ftp.c	\
+		css_.c css-url.c	\
+		ftp-basic.c ftp-ls.c hash.c host.c html-parse.c html-url.c	\
+		http.c init.c log.c main.c netrc.c progress.c ptimer.c	\
+		recur.c res.c retr.c spider.c url.c warc.c	\
+		utils.c exits.c build_info.c $(IRI_OBJ)	\
+		css-url.h css-tokens.h connect.h convert.h cookies.h	\
+		ftp.h hash.h host.h html-parse.h html-url.h	\
+		http.h http-ntlm.h init.h log.h mswindows.h netrc.h	\
+		options.h progress.h ptimer.h recur.h res.h retr.h	\
+		spider.h ssl.h sysdep.h url.h warc.h utils.h wget.h iri.h	\
+		exits.h gettext.h
 nodist_wget_SOURCES = version.c
 EXTRA_wget_SOURCES = iri.c
 LDADD = $(LIBOBJS) ../lib/libgnu.a
@@ -74,7 +74,7 @@ version.c:  $(wget_SOURCES) ../lib/libgnu.a
 	echo '/* version.c */' > $@
 	echo '/* Autogenerated by Makefile - DO NOT EDIT */' >> $@
 	echo '' >> $@
-	echo 'const char *version_string = "@VERSION@";' >> $@
+	echo 'const char *version_string = "$(VERSION)";' >> $@
 	echo 'const char *compilation_string = "'$(COMPILE)'";' \
 	    | $(ESCAPEQUOTE) >> $@
 	echo 'const char *link_string = "'$(CCLD) $(AM_CFLAGS) $(CFLAGS) \

src/build_info.c.in

@@ -7,6 +7,7 @@ large-file      SIZEOF_OFF_T >= 8
 nls             defined ENABLE_NLS
 ntlm            defined ENABLE_NTLM
 opie            defined ENABLE_OPIE
+psl             defined HAVE_LIBPSL
 
 ssl choice:
     openssl     defined HAVE_LIBSSL || defined HAVE_LIBSSL32

src/cmpt.c

@@ -372,7 +372,7 @@ static void
 day_of_the_week (struct tm *tm)
 {
   /* We know that January 1st 1970 was a Thursday (= 4).  Compute the
-     the difference between this data in the one on TM and so determine
+     difference between this data in the one on TM and so determine
      the weekday.  */
   int corr_year = 1900 + tm->tm_year - (tm->tm_mon < 2);
   int wday = (-473

src/connect.c

@@ -59,10 +59,7 @@ as that of the covered work.  */
 #include "connect.h"
 #include "hash.h"
 
-/* Apparently needed for Interix: */
-#ifdef HAVE_STDINT_H
-# include <stdint.h>
-#endif
+#include <stdint.h>
 
 /* Define sockaddr_storage where unavailable (presumably on IPv4-only
    hosts).  */
@@ -273,7 +270,7 @@ connect_to_ip (const ip_address *ip, int port, const char *print)
       const char *txt_addr = print_address (ip);
       if (0 != strcmp (print, txt_addr))
         {
-				  char *str = NULL, *name;
+          char *str = NULL, *name;
 
           if (opt.enable_iri && (name = idn_decode ((char *) print)) != NULL)
             {
@@ -287,18 +284,18 @@ connect_to_ip (const ip_address *ip, int port, const char *print)
           logprintf (LOG_VERBOSE, _("Connecting to %s|%s|:%d... "),
                      str ? str : escnonprint_uri (print), txt_addr, port);
 
-					if (str)
-					  xfree (str);
+          if (str)
+              xfree (str);
         }
       else
-       {
+        {
            if (ip->family == AF_INET)
                logprintf (LOG_VERBOSE, _("Connecting to %s:%d... "), txt_addr, port);
 #ifdef ENABLE_IPV6
            else if (ip->family == AF_INET6)
                logprintf (LOG_VERBOSE, _("Connecting to [%s]:%d... "), txt_addr, port);
 #endif
-       }
+        }
     }
 
   /* Store the sockaddr info to SA.  */

src/connect.h

@@ -32,7 +32,7 @@ as that of the covered work.  */
 #ifndef CONNECT_H
 #define CONNECT_H
 
-#include "host.h"		/* for definition of ip_address */
+#include "host.h"       /* for definition of ip_address */
 
 /* Function declarations */
 

src/convert.c

@@ -1,5 +1,5 @@
 /* Conversion of links to local files.
-   Copyright (C) 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011
+   Copyright (C) 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2014
    Free Software Foundation, Inc.
 
 This file is part of GNU Wget.
@@ -125,7 +125,7 @@ convert_links_in_hashtable (struct hash_table *downloaded_set,
 
           u = url_parse (cur_url->url->url, NULL, pi, true);
           if (!u)
-	    continue;
+              continue;
 
           local_name = hash_table_get (dl_url_file_map, u->url);
 
@@ -465,7 +465,7 @@ write_backup_file (const char *file, downloaded_file_t downloaded_file_return)
     converted_files = make_string_hash_table (0);
 
   /* We can get called twice on the same URL thanks to the
-     convert_all_links() call in main().  If we write the .orig file
+     convert_all_links() call in main.  If we write the .orig file
      each time in such a case, it'll end up containing the first-pass
      conversion, not the original file.  So, see if we've already been
      called on this file. */
@@ -626,6 +626,7 @@ local_quote_string (const char *file, bool no_html_quote)
   /* Allocate space assuming the worst-case scenario, each character
      having to be quoted.  */
   to = newname = (char *)alloca (3 * strlen (file) + 1);
+  newname[0] = '\0';
   for (from = file; *from; from++)
     switch (*from)
       {
@@ -835,7 +836,7 @@ register_download (const char *url, const char *file)
   hash_table_put (dl_url_file_map, xstrdup (url), xstrdup (file));
 }
 
-/* Register that FROM has been redirected to TO.  This assumes that TO
+/* Register that FROM has been redirected to "TO".  This assumes that TO
    is successfully downloaded and already registered using
    register_download() above.  */
 
@@ -1087,4 +1088,3 @@ html_quote_string (const char *s)
 /*
  * vim: et ts=2 sw=2
  */
-

src/convert.h

@@ -31,18 +31,18 @@ as that of the covered work.  */
 #ifndef CONVERT_H
 #define CONVERT_H
 
-struct hash_table;		/* forward decl */
+struct hash_table;              /* forward decl */
 extern struct hash_table *dl_url_file_map;
 extern struct hash_table *downloaded_html_set;
 extern struct hash_table *downloaded_css_set;
 
 enum convert_options {
-  CO_NOCONVERT = 0,		/* don't convert this URL */
-  CO_CONVERT_TO_RELATIVE,	/* convert to relative, e.g. to
+  CO_NOCONVERT = 0,             /* don't convert this URL */
+  CO_CONVERT_TO_RELATIVE,       /* convert to relative, e.g. to
                                    "../../otherdir/foo.gif" */
-  CO_CONVERT_TO_COMPLETE,	/* convert to absolute, e.g. to
-				   "http://orighost/somedir/bar.jpg". */
-  CO_NULLIFY_BASE		/* change to empty string. */
+  CO_CONVERT_TO_COMPLETE,       /* convert to absolute, e.g. to
+                                   "http://orighost/somedir/bar.jpg". */
+  CO_NULLIFY_BASE               /* change to empty string. */
 };
 
 struct url;
@@ -51,36 +51,36 @@ struct url;
    position in an HTML document, etc.  */
 
 struct urlpos {
-  struct url *url;		/* the URL of the link, after it has
-				   been merged with the base */
-  char *local_name;		/* local file to which it was saved
-				   (used by convert_links) */
+  struct url *url;              /* the URL of the link, after it has
+                                   been merged with the base */
+  char *local_name;             /* local file to which it was saved
+                                   (used by convert_links) */
 
   /* reserved for special links such as <base href="..."> which are
      used when converting links, but ignored when downloading.  */
-  unsigned int ignore_when_downloading	:1;
+  unsigned int ignore_when_downloading  :1;
 
   /* Information about the original link: */
 
-  unsigned int link_relative_p	:1; /* the link was relative */
-  unsigned int link_complete_p	:1; /* the link was complete (had host name) */
-  unsigned int link_base_p	:1; /* the url came from <base href=...> */
-  unsigned int link_inline_p	:1; /* needed to render the page */
-  unsigned int link_css_p	:1; /* the url came from CSS */
-  unsigned int link_expect_html	:1; /* expected to contain HTML */
-  unsigned int link_expect_css	:1; /* expected to contain CSS */
+  unsigned int link_relative_p  :1; /* the link was relative */
+  unsigned int link_complete_p  :1; /* the link was complete (had host name) */
+  unsigned int link_base_p  :1;     /* the url came from <base href=...> */
+  unsigned int link_inline_p    :1; /* needed to render the page */
+  unsigned int link_css_p   :1;     /* the url came from CSS */
+  unsigned int link_expect_html :1; /* expected to contain HTML */
+  unsigned int link_expect_css  :1; /* expected to contain CSS */
 
-  unsigned int link_refresh_p	:1; /* link was received from
-				       <meta http-equiv=refresh content=...> */
-  int refresh_timeout;		/* for reconstructing the refresh. */
+  unsigned int link_refresh_p   :1; /* link was received from
+                                       <meta http-equiv=refresh content=...> */
+  int refresh_timeout;              /* for reconstructing the refresh. */
 
   /* Conversion requirements: */
-  enum convert_options convert;	/* is conversion required? */
+  enum convert_options convert;     /* is conversion required? */
 
   /* URL's position in the buffer. */
   int pos, size;
 
-  struct urlpos *next;		/* next list element */
+  struct urlpos *next;              /* next list element */
 };
 
 /* downloaded_file() takes a parameter of this type and returns this type. */

src/cookies.c

@@ -51,6 +51,9 @@ as that of the covered work.  */
 #include <assert.h>
 #include <errno.h>
 #include <time.h>
+#ifdef HAVE_LIBPSL
+# include <libpsl.h>
+#endif
 #include "utils.h"
 #include "hash.h"
 #include "cookies.h"
@@ -95,7 +98,7 @@ struct cookie {
   int port;                     /* port number */
   char *path;                   /* path prefix of the cookie */
 
-  unsigned discard_requested :1; /* whether cookie was created to
+  unsigned discard_requested :1;/* whether cookie was created to
                                    request discarding another
                                    cookie. */
 
@@ -346,7 +349,7 @@ parse_set_cookie (const char *set_cookie, bool silent)
   struct cookie *cookie = cookie_new ();
   param_token name, value;
 
-  if (!extract_param (&ptr, &name, &value, ';'))
+  if (!extract_param (&ptr, &name, &value, ';', NULL))
     goto error;
   if (!value.b)
     goto error;
@@ -360,7 +363,7 @@ parse_set_cookie (const char *set_cookie, bool silent)
   cookie->attr = strdupdelim (name.b, name.e);
   cookie->value = strdupdelim (value.b, value.e);
 
-  while (extract_param (&ptr, &name, &value, ';'))
+  while (extract_param (&ptr, &name, &value, ';', NULL))
     {
       if (TOKEN_IS (name, "domain"))
         {
@@ -393,7 +396,7 @@ parse_set_cookie (const char *set_cookie, bool silent)
 
           /* Check if expiration spec is valid.
              If not, assume default (cookie doesn't expire, but valid only for
-	     this session.) */
+             this session.) */
           expires = http_atotm (value_copy);
           if (expires != (time_t) -1)
             {
@@ -460,9 +463,9 @@ parse_set_cookie (const char *set_cookie, bool silent)
 
 
 #define REQUIRE_DIGITS(p) do {                  \
-  if (!c_isdigit (*p))                            \
+  if (!c_isdigit (*p))                          \
     return false;                               \
-  for (++p; c_isdigit (*p); p++)                  \
+  for (++p; c_isdigit (*p); p++)                \
     ;                                           \
 } while (0)
 
@@ -498,19 +501,61 @@ numeric_address_p (const char *addr)
 /* Check whether COOKIE_DOMAIN is an appropriate domain for HOST.
    Originally I tried to make the check compliant with rfc2109, but
    the sites deviated too often, so I had to fall back to "tail
-   matching", as defined by the original Netscape's cookie spec.  */
+   matching", as defined by the original Netscape's cookie spec.
+
+   Wget now uses libpsl to check domain names against a public suffix
+   list to see if they are valid. However, since we don't provide a
+   psl on our own, if libpsl is compiled without a public suffix list,
+   fall back to using the original "tail matching" heuristic. Also if
+   libpsl is unable to convert the domain to lowercase, which means that
+   it doesnt have any runtime conversion support, we again fall back to
+   "tail matching" since libpsl states the results are unpredictable with
+   upper case strings.
+   */
 
 static bool
 check_domain_match (const char *cookie_domain, const char *host)
 {
+
+#ifdef HAVE_LIBPSL
   DEBUGP (("cdm: 1"));
+  char *cookie_domain_lower = NULL;
+  char *host_lower = NULL;
+  const psl_ctx_t *psl;
+  int is_acceptable;
 
-  /* Numeric address requires exact match.  It also requires HOST to
-     be an IP address.  */
-  if (numeric_address_p (cookie_domain))
-    return 0 == strcmp (cookie_domain, host);
+  if (!(psl = psl_builtin()))
+    {
+      DEBUGP (("\nlibpsl not built with a public suffix list. "
+               "Falling back to simple heuristics.\n"));
+      goto no_psl;
+    }
 
-  DEBUGP ((" 2"));
+  if (psl_str_to_utf8lower (cookie_domain, NULL, NULL, &cookie_domain_lower) == PSL_SUCCESS &&
+      psl_str_to_utf8lower (host, NULL, NULL, &host_lower) == PSL_SUCCESS)
+    {
+      is_acceptable = psl_is_cookie_domain_acceptable (psl, host_lower, cookie_domain_lower);
+    }
+  else
+    {
+        DEBUGP (("libpsl unable to parse domain name. "
+                 "Falling back to simple heuristics.\n"));
+        goto no_psl;
+    }
+
+  xfree (cookie_domain_lower);
+  xfree (host_lower);
+
+  return is_acceptable == 1;
+
+no_psl:
+  /* Cleanup the PSL pointers first */
+  xfree (cookie_domain_lower);
+  xfree (host_lower);
+#endif
+
+  /* For efficiency make some elementary checks first */
+  DEBUGP (("cdm: 2"));
 
   /* For the sake of efficiency, check for exact match first. */
   if (0 == strcasecmp (cookie_domain, host))
@@ -1377,7 +1422,7 @@ test_cookies (void)
         param_token name, value;
         const char *ptr = data;
         int j = 0;
-        while (extract_param (&ptr, &name, &value, ';'))
+        while (extract_param (&ptr, &name, &value, ';', NULL))
           {
             char *n = strdupdelim (name.b, name.e);
             char *v = strdupdelim (value.b, value.e);

src/cookies.h

@@ -37,9 +37,9 @@ struct cookie_jar *cookie_jar_new (void);
 void cookie_jar_delete (struct cookie_jar *);
 
 void cookie_handle_set_cookie (struct cookie_jar *, const char *, int,
-			       const char *, const char *);
+                               const char *, const char *);
 char *cookie_header (struct cookie_jar *, const char *, int,
-		     const char *, bool);
+                     const char *, bool);
 
 void cookie_jar_load (struct cookie_jar *, const char *);
 void cookie_jar_save (struct cookie_jar *, const char *);

src/css-url.c

@@ -1,5 +1,5 @@
 /* Collect URLs from CSS source.
-   Copyright (C) 1998, 2000, 2001, 2002, 2003, 2009, 2010, 2011 Free
+   Copyright (C) 1998, 2000, 2001, 2002, 2003, 2009, 2010, 2011, 2014 Free
    Software Foundation, Inc.
 
 This file is part of GNU Wget.
@@ -41,11 +41,7 @@ as that of the covered work.  */
 #include <wget.h>
 
 #include <stdio.h>
-#ifdef HAVE_STRING_H
-# include <string.h>
-#else
-# include <strings.h>
-#endif
+#include <string.h>
 #include <stdlib.h>
 #include <ctype.h>
 #include <errno.h>
@@ -56,6 +52,7 @@ as that of the covered work.  */
 #include "html-url.h"
 #include "css-tokens.h"
 #include "css-url.h"
+#include "xstrndup.h"
 
 /* from lex.yy.c */
 extern char *yytext;
@@ -64,41 +61,6 @@ typedef struct yy_buffer_state *YY_BUFFER_STATE;
 extern YY_BUFFER_STATE yy_scan_bytes (const char *bytes,int len  );
 extern int yylex (void);
 
-#if 1
-const char *token_names[] = {
-  "CSSEOF",
-  "S",
-  "CDO",
-  "CDC",
-  "INCLUDES",
-  "DASHMATCH",
-  "LBRACE",
-  "PLUS",
-  "GREATER",
-  "COMMA",
-  "STRING",
-  "INVALID",
-  "IDENT",
-  "HASH",
-  "IMPORT_SYM",
-  "PAGE_SYM",
-  "MEDIA_SYM",
-  "CHARSET_SYM",
-  "IMPORTANT_SYM",
-  "EMS",
-  "EXS",
-  "LENGTH",
-  "ANGLE",
-  "TIME",
-  "FREQ",
-  "DIMENSION",
-  "PERCENTAGE",
-  "NUMBER",
-  "URI",
-  "FUNCTION"
-};
-#endif
-
 /*
   Given a detected URI token, get only the URI specified within.
   Also adjust the starting position and length of the string.
@@ -111,12 +73,6 @@ const char *token_names[] = {
 static char *
 get_uri_string (const char *at, int *pos, int *length)
 {
-  char *uri;
-  /*char buf[1024];
-  strncpy(buf,at + *pos, *length);
-  buf[*length] = '\0';
-  DEBUGP (("get_uri_string: \"%s\"\n", buf));*/
-
   if (0 != strncasecmp (at + *pos, "url(", 4))
     return NULL;
 
@@ -142,14 +98,7 @@ get_uri_string (const char *at, int *pos, int *length)
       *length -= 2;
     }
 
-  uri = xmalloc (*length + 1);
-  if (uri)
-    {
-      strncpy (uri, at + *pos, *length);
-      uri[*length] = '\0';
-    }
-
-  return uri;
+  return xstrndup (at + *pos, *length);
 }
 
 void
@@ -161,12 +110,6 @@ get_urls_css (struct map_context *ctx, int offset, int buf_length)
   int pos, length;
   char *uri;
 
-  /*
-  strncpy(tmp,ctx->text + offset, buf_length);
-  tmp[buf_length] = '\0';
-  DEBUGP (("get_urls_css: \"%s\"\n", tmp));
-  */
-
   /* tell flex to scan from this buffer */
   yy_scan_bytes (ctx->text + offset, buf_length);
 
@@ -200,7 +143,7 @@ get_urls_css (struct map_context *ctx, int offset, int buf_length)
                   pos++;
                   length -= 2;
                   uri = xmalloc (length + 1);
-                  strncpy (uri, yytext + 1, length);
+                  memcpy (uri, yytext + 1, length);
                   uri[length] = '\0';
                 }
 

src/decc_ver.c

@@ -2,6 +2,15 @@
 
 int main( void)
 {
+
+#ifdef ENABLE_NLS
+  /* Set the current locale.  */
+  setlocale (LC_ALL, "");
+  /* Set the text message domain.  */
+  bindtextdomain ("wget", LOCALEDIR);
+  textdomain ("wget");
+#endif /* ENABLE_NLS */
+
 #ifdef __VMS_VERSION
 printf( " __VMS_VERSION: >%s<.\n", __VMS_VERSION);
 #else /* def __VMS_VERSION */

src/exits.c

@@ -20,24 +20,6 @@
 #include "wget.h"
 #include "exits.h"
 
-/* Final exit code possibilities. Exit codes 1 and 2 are reserved
- * for situations that lead to direct exits from Wget, not using the
- * value of final_exit_status. */
-enum
-  {
-    WGET_EXIT_SUCCESS = 0,
-
-    WGET_EXIT_MINIMUM = 3,
-    WGET_EXIT_IO_FAIL = WGET_EXIT_MINIMUM,
-    WGET_EXIT_NETWORK_FAIL = 4,
-    WGET_EXIT_SSL_AUTH_FAIL = 5,
-    WGET_EXIT_SERVER_AUTH_FAIL = 6,
-    WGET_EXIT_PROTOCOL_ERROR = 7,
-    WGET_EXIT_SERVER_ERROR = 8,
-
-    WGET_EXIT_UNKNOWN
-  };
-
 static int final_exit_status = WGET_EXIT_SUCCESS;
 
 /* XXX: I don't like that newly-added uerr_t codes will doubtless fall
@@ -108,4 +90,3 @@ get_exit_status (void)
       ? 1
       : final_exit_status;
 }
-

src/exits.h

@@ -21,6 +21,23 @@ along with Wget.  If not, see <http://www.gnu.org/licenses/>.  */
 
 #include "wget.h"
 
+/* Final exit code possibilities. Exit codes 1 and 2 are reserved
+ * for situations that lead to direct exits from Wget, not using the
+ * value of final_exit_status. */
+enum
+  {
+    WGET_EXIT_SUCCESS = 0,
+    WGET_EXIT_GENERIC_ERROR = 1,
+    WGET_EXIT_PARSE_ERROR = 2,
+    WGET_EXIT_IO_FAIL = 3,
+    WGET_EXIT_NETWORK_FAIL = 4,
+    WGET_EXIT_SSL_AUTH_FAIL = 5,
+    WGET_EXIT_SERVER_AUTH_FAIL = 6,
+    WGET_EXIT_PROTOCOL_ERROR = 7,
+    WGET_EXIT_SERVER_ERROR = 8,
+
+    WGET_EXIT_UNKNOWN
+  };
 
 void inform_exit_status (uerr_t err);
 

src/ftp-basic.c

@@ -1,6 +1,6 @@
 /* Basic FTP routines.
    Copyright (C) 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004,
-   2005, 2006, 2007, 2008, 2009, 2010, 2011 Free Software Foundation,
+   2005, 2006, 2007, 2008, 2009, 2010, 2011, 2014 Free Software Foundation,
    Inc.
 
 This file is part of GNU Wget.
@@ -44,8 +44,6 @@ as that of the covered work.  */
 #include "ftp.h"
 #include "retr.h"
 
-char ftp_last_respline[128];
-
 
 /* Get the response of FTP server and allocate enough room to handle
    it.  <CR> and <LF> characters are stripped from the line, and the
@@ -84,8 +82,6 @@ ftp_response (int fd, char **ret_line)
       if (c_isdigit (line[0]) && c_isdigit (line[1]) && c_isdigit (line[2])
           && line[3] == ' ')
         {
-          strncpy (ftp_last_respline, line, sizeof (ftp_last_respline));
-          ftp_last_respline[sizeof (ftp_last_respline) - 1] = '\0';
           *ret_line = line;
           return FTPOK;
         }
@@ -1037,6 +1033,7 @@ ftp_syst (int csock, enum stype *server_type, enum ustype *unix_type)
   char *request, *respline;
   int nwritten;
   uerr_t err;
+  char *ftp_last_respline;
 
   /* Send SYST request.  */
   request = ftp_request ("SYST", NULL);
@@ -1058,6 +1055,8 @@ ftp_syst (int csock, enum stype *server_type, enum ustype *unix_type)
       return FTPSRVERR;
     }
 
+  ftp_last_respline = strdup (respline);
+
   /* Skip the number (215, but 200 (!!!) in case of VMS) */
   strtok (respline, " ");
 
@@ -1092,6 +1091,7 @@ ftp_syst (int csock, enum stype *server_type, enum ustype *unix_type)
   else
     *server_type = ST_OTHER;
 
+  xfree (ftp_last_respline);
   xfree (respline);
   /* All OK.  */
   return FTPOK;

src/ftp-ls.c

@@ -106,7 +106,7 @@ ftp_parse_unix_ls (const char *file, int ignore_perms)
   size_t bufsize = 0;
 
   char *line = NULL, *tok, *ptok;      /* tokenizer */
-  struct fileinfo *dir, *l, cur; /* list creation */
+  struct fileinfo *dir, *l, cur;       /* list creation */
 
   fp = fopen (file, "rb");
   if (!fp)
@@ -895,7 +895,8 @@ ftp_parse_vms_ls (const char *file)
               /* Protections (permissions). */
               perms = 0;
               j = 0;
-              for (i = 0; i < strlen(tok); i++)
+              /*FIXME: Should not be using the variable like this. */
+              for (i = 0; i < (int) strlen(tok); i++)
                 {
                   switch (tok[ i])
                     {

src/ftp.c

@@ -1,6 +1,6 @@
 /* File Transfer Protocol support.
    Copyright (C) 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004,
-   2005, 2006, 2007, 2008, 2009, 2010, 2011 Free Software Foundation,
+   2005, 2006, 2007, 2008, 2009, 2010, 2011, 2014 Free Software Foundation,
    Inc.
 
 This file is part of GNU Wget.
@@ -221,13 +221,13 @@ print_length (wgint size, wgint start, bool authoritative)
 {
   logprintf (LOG_VERBOSE, _("Length: %s"), number_to_static_string (size));
   if (size >= 1024)
-    logprintf (LOG_VERBOSE, " (%s)", human_readable (size));
+    logprintf (LOG_VERBOSE, " (%s)", human_readable (size, 10, 1));
   if (start > 0)
     {
       if (size - start >= 1024)
         logprintf (LOG_VERBOSE, _(", %s (%s) remaining"),
                    number_to_static_string (size - start),
-                   human_readable (size - start));
+                   human_readable (size - start, 10, 1));
       else
         logprintf (LOG_VERBOSE, _(", %s remaining"),
                    number_to_static_string (size - start));
@@ -243,7 +243,8 @@ static uerr_t ftp_get_listing (struct url *, ccon *, struct fileinfo **);
    is non-NULL, the downloaded data will be written there as well.  */
 static uerr_t
 getftp (struct url *u, wgint passed_expected_bytes, wgint *qtyread,
-        wgint restval, ccon *con, int count, FILE *warc_tmp)
+        wgint restval, ccon *con, int count, wgint *last_expected_bytes,
+        FILE *warc_tmp)
 {
   int csock, dtsock, local_sock, res;
   uerr_t err = RETROK;          /* appease the compiler */
@@ -802,8 +803,12 @@ Error in server response, closing control connection.\n"));
           abort ();
         }
         if (!opt.server_response)
-          logprintf (LOG_VERBOSE, expected_bytes ? "%s\n" : _("done.\n"),
-                     number_to_static_string (expected_bytes));
+          {
+            logprintf (LOG_VERBOSE, "%s\n",
+                    expected_bytes ?
+                    number_to_static_string (expected_bytes) :
+                    _("done.\n"));
+          }
     }
 
   if (cmd & DO_RETR && restval > 0 && restval == expected_bytes)
@@ -986,15 +991,14 @@ Error in server response, closing control connection.\n"));
       if (opt.spider)
         {
           bool exists = false;
-          uerr_t res;
           struct fileinfo *f;
-          res = ftp_get_listing (u, con, &f);
+          uerr_t _res = ftp_get_listing (u, con, &f);
           /* Set the DO_RETR command flag again, because it gets unset when
              calling ftp_get_listing() and would otherwise cause an assertion
              failure earlier on when this function gets repeatedly called
              (e.g., when recursing).  */
           con->cmd |= DO_RETR;
-          if (res == RETROK)
+          if (_res == RETROK)
             {
               while (f)
                 {
@@ -1075,7 +1079,7 @@ Error in server response, closing control connection.\n"));
         logputs (LOG_VERBOSE, _("done.\n"));
 
       if (! got_expected_bytes)
-        expected_bytes = ftp_expected_bytes (ftp_last_respline);
+        expected_bytes = *last_expected_bytes;
     } /* do retrieve */
 
   if (cmd & DO_LIST)
@@ -1124,7 +1128,7 @@ Error in server response, closing control connection.\n"));
         logputs (LOG_VERBOSE, _("done.\n"));
 
       if (! got_expected_bytes)
-        expected_bytes = ftp_expected_bytes (ftp_last_respline);
+        expected_bytes = *last_expected_bytes;
     } /* cmd & DO_LIST */
 
   if (!(cmd & (DO_LIST | DO_RETR)) || (opt.spider && !(cmd & DO_LIST)))
@@ -1156,7 +1160,7 @@ Error in server response, closing control connection.\n"));
   /* Open the file -- if output_stream is set, use it instead.  */
 
   /* 2005-04-17 SMS.
-     Note that having the output_stream ("-O") file opened in main()
+     Note that having the output_stream ("-O") file opened in main
      (main.c) rather limits the ability in VMS to open the file
      differently for ASCII versus binary FTP here.  (Of course, doing it
      there allows a open failure to be detected immediately, without first
@@ -1231,8 +1235,7 @@ Error in server response, closing control connection.\n"));
         {
           if (opt.unlink && file_exists_p (con->target))
             {
-              int res = unlink (con->target);
-              if (res < 0)
+              if (unlink (con->target) < 0)
                 {
                   logprintf (LOG_NOTQUIET, "%s: %s\n", con->target,
                     strerror (errno));
@@ -1306,7 +1309,7 @@ Error in server response, closing control connection.\n"));
   if (restval && rest_failed)
     flags |= rb_skip_startpos;
   rd_size = 0;
-  res = fd_read_body (dtsock, fp,
+  res = fd_read_body (con->target, dtsock, fp,
                       expected_bytes ? expected_bytes - restval : 0,
                       restval, &rd_size, qtyread, &con->dltime, flags, warc_tmp);
 
@@ -1343,6 +1346,7 @@ Error in server response, closing control connection.\n"));
 
   /* Get the server to tell us if everything is retrieved.  */
   err = ftp_response (csock, &respline);
+  *last_expected_bytes = ftp_expected_bytes (respline);
   if (err != FTPOK)
     {
       /* The control connection is decidedly closed.  Print the time
@@ -1545,6 +1549,7 @@ ftp_loop_internal (struct url *u, struct fileinfo *f, ccon *con, char **local_fi
   bool warc_enabled = (opt.warc_filename != NULL);
   FILE *warc_tmp = NULL;
   ip_address *warc_ip = NULL;
+  wgint last_expected_bytes = 0;
 
   /* Get the target, and set the name for the message accordingly. */
   if ((f == NULL) && (con->target))
@@ -1632,6 +1637,8 @@ ftp_loop_internal (struct url *u, struct fileinfo *f, ccon *con, char **local_fi
       /* Decide whether or not to restart.  */
       if (con->cmd & DO_LIST)
         restval = 0;
+      else if (opt.start_pos >= 0)
+        restval = opt.start_pos;
       else if (opt.always_rest
           && stat (locf, &st) == 0
           && S_ISREG (st.st_mode))
@@ -1669,7 +1676,8 @@ ftp_loop_internal (struct url *u, struct fileinfo *f, ccon *con, char **local_fi
 
       /* If we are working on a WARC record, getftp should also write
          to the warc_tmp file. */
-      err = getftp (u, len, &qtyread, restval, con, count, warc_tmp);
+      err = getftp (u, len, &qtyread, restval, con, count, &last_expected_bytes,
+                    warc_tmp);
 
       if (con->csock == -1)
         con->st &= ~DONE_CWD;
@@ -2203,6 +2211,29 @@ has_insecure_name_p (const char *s)
   return false;
 }
 
+/* Test if the file node is invalid. This can occur due to malformed or
+ * maliciously crafted listing files being returned by the server.
+ *
+ * Currently, this function only tests if there are multiple entries in the
+ * listing file by the same name. However this function can be expanded as more
+ * such illegal listing formats are discovered. */
+static bool
+is_invalid_entry (struct fileinfo *f)
+{
+  struct fileinfo *cur;
+  cur = f;
+  char *f_name = f->name;
+  /* If the node we're currently checking has a duplicate later, we eliminate
+   * the current node and leave the next one intact. */
+  while (cur->next)
+    {
+      cur = cur->next;
+      if (strcmp(f_name, cur->name) == 0)
+          return true;
+    }
+  return false;
+}
+
 /* A near-top-level function to retrieve the files in a directory.
    The function calls ftp_get_listing, to get a linked list of files.
    Then it weeds out the file names that do not match the pattern.
@@ -2240,11 +2271,11 @@ ftp_retrieve_glob (struct url *u, ccon *con, int action)
             f = f->next;
         }
     }
-  /* Remove all files with possible harmful names */
+  /* Remove all files with possible harmful names or invalid entries. */
   f = start;
   while (f)
     {
-      if (has_insecure_name_p (f->name))
+      if (has_insecure_name_p (f->name) || is_invalid_entry (f))
         {
           logprintf (LOG_VERBOSE, _("Rejecting %s.\n"),
                      quote (f->name));

src/ftp.h

@@ -106,49 +106,49 @@ enum parsetype
 /* Information about one filename in a linked list.  */
 struct fileinfo
 {
-  enum ftype type;		/* file type */
-  char *name;			/* file name */
-  wgint size;			/* file size */
-  long tstamp;			/* time-stamp */
-  enum parsetype ptype; /* time parsing */
-  int perms;			/* file permissions */
-  char *linkto;			/* link to which file points */
-  struct fileinfo *prev;	/* previous... */
-  struct fileinfo *next;	/* ...and next structure. */
+  enum ftype type;          /* file type */
+  char *name;               /* file name */
+  wgint size;               /* file size */
+  long tstamp;              /* time-stamp */
+  enum parsetype ptype;     /* time parsing */
+  int perms;                /* file permissions */
+  char *linkto;             /* link to which file points */
+  struct fileinfo *prev;    /* previous... */
+  struct fileinfo *next;    /* ...and next structure. */
 };
 
 /* Commands for FTP functions.  */
 enum wget_ftp_command
 {
-  DO_LOGIN      = 0x0001,	/* Connect and login to the server.  */
-  DO_CWD        = 0x0002,	/* Change current directory.  */
-  DO_RETR       = 0x0004,	/* Retrieve the file.  */
-  DO_LIST       = 0x0008,	/* Retrieve the directory list.  */
-  LEAVE_PENDING = 0x0010	/* Do not close the socket.  */
+  DO_LOGIN      = 0x0001,   /* Connect and login to the server.  */
+  DO_CWD        = 0x0002,   /* Change current directory.  */
+  DO_RETR       = 0x0004,   /* Retrieve the file.  */
+  DO_LIST       = 0x0008,   /* Retrieve the directory list.  */
+  LEAVE_PENDING = 0x0010    /* Do not close the socket.  */
 };
 
 enum wget_ftp_fstatus
 {
-  NOTHING       = 0x0000,	/* Nothing done yet.  */
-  ON_YOUR_OWN   = 0x0001,	/* The ftp_loop_internal sets the
-				   defaults.  */
-  DONE_CWD      = 0x0002,	/* The current working directory is
-				   correct.  */
+  NOTHING       = 0x0000,   /* Nothing done yet.  */
+  ON_YOUR_OWN   = 0x0001,   /* The ftp_loop_internal sets the
+                               defaults.  */
+  DONE_CWD      = 0x0002,   /* The current working directory is
+                               correct.  */
 
   /* 2013-10-17 Andrea Urbani (matfanjol)
      For more information about the following entries, please,
      look at ftp.c, function getftp, text "__LIST_A_EXPLANATION__". */
-  AVOID_LIST_A  = 0x0004,	/* It tells us if during this
-				 session we have to avoid the use
-				 of "LIST -a".*/
-  AVOID_LIST    = 0x0008,	/* It tells us if during this
-				 session we have to avoid to use
-				 "LIST". */
+  AVOID_LIST_A  = 0x0004,   /* It tells us if during this
+                               session we have to avoid the use
+                               of "LIST -a".*/
+  AVOID_LIST    = 0x0008,   /* It tells us if during this
+                               session we have to avoid to use
+                               "LIST". */
   LIST_AFTER_LIST_A_CHECK_DONE  = 0x0010
-				/* It tells us if we have already
-				 checked "LIST" after the first
-				 "LIST -a" to handle the case of
-				 file/folders named "-a". */
+                            /* It tells us if we have already
+                               checked "LIST" after the first
+                               "LIST -a" to handle the case of
+                               file/folders named "-a". */
 };
 
 struct fileinfo *ftp_parse_ls (const char *, const enum stype);

src/gettext.h

@@ -133,8 +133,8 @@ inline
 #endif
 static const char *
 pgettext_aux (const char *domain,
-	      const char *msg_ctxt_id, const char *msgid,
-	      int category)
+              const char *msg_ctxt_id, const char *msgid,
+              int category)
 {
   const char *translation = dcgettext (domain, msg_ctxt_id, category);
   if (translation == msg_ctxt_id)
@@ -152,9 +152,9 @@ inline
 #endif
 static const char *
 npgettext_aux (const char *domain,
-	       const char *msg_ctxt_id, const char *msgid,
-	       const char *msgid_plural, unsigned long int n,
-	       int category)
+               const char *msg_ctxt_id, const char *msgid,
+               const char *msgid_plural, unsigned long int n,
+               int category)
 {
   const char *translation =
     dcngettext (domain, msg_ctxt_id, msgid_plural, n, category);
@@ -192,8 +192,8 @@ inline
 #endif
 static const char *
 dcpgettext_expr (const char *domain,
-		 const char *msgctxt, const char *msgid,
-		 int category)
+                 const char *msgctxt, const char *msgid,
+                 int category)
 {
   size_t msgctxt_len = strlen (msgctxt) + 1;
   size_t msgid_len = strlen (msgid) + 1;
@@ -215,10 +215,10 @@ dcpgettext_expr (const char *domain,
       translation = dcgettext (domain, msg_ctxt_id, category);
 #if !_LIBGETTEXT_HAVE_VARIABLE_SIZE_ARRAYS
       if (msg_ctxt_id != buf)
-	free (msg_ctxt_id);
+        free (msg_ctxt_id);
 #endif
       if (translation != msg_ctxt_id)
-	return translation;
+        return translation;
     }
   return msgid;
 }
@@ -237,9 +237,9 @@ inline
 #endif
 static const char *
 dcnpgettext_expr (const char *domain,
-		  const char *msgctxt, const char *msgid,
-		  const char *msgid_plural, unsigned long int n,
-		  int category)
+                  const char *msgctxt, const char *msgid,
+                  const char *msgid_plural, unsigned long int n,
+                  int category)
 {
   size_t msgctxt_len = strlen (msgctxt) + 1;
   size_t msgid_len = strlen (msgid) + 1;
@@ -261,10 +261,10 @@ dcnpgettext_expr (const char *domain,
       translation = dcngettext (domain, msg_ctxt_id, msgid_plural, n, category);
 #if !_LIBGETTEXT_HAVE_VARIABLE_SIZE_ARRAYS
       if (msg_ctxt_id != buf)
-	free (msg_ctxt_id);
+        free (msg_ctxt_id);
 #endif
       if (!(translation == msg_ctxt_id || translation == msgid_plural))
-	return translation;
+        return translation;
     }
   return (n == 1 ? msgid : msgid_plural);
 }

src/gnutls.c

@@ -30,7 +30,6 @@ as that of the covered work.  */
 
 #include "wget.h"
 
-#include <assert.h>
 #include <errno.h>
 #include <unistd.h>
 #include <string.h>
@@ -434,6 +433,7 @@ ssl_connect_wget (int fd, const char *hostname)
   switch (opt.secure_protocol)
     {
     case secure_protocol_auto:
+      err = gnutls_priority_set_direct (session, "NORMAL:%COMPAT:-VERS-SSL3.0", NULL);
       break;
     case secure_protocol_sslv2:
     case secure_protocol_sslv3:
@@ -443,10 +443,10 @@ ssl_connect_wget (int fd, const char *hostname)
       err = gnutls_priority_set_direct (session, "NORMAL:-VERS-SSL3.0", NULL);
       break;
     case secure_protocol_pfs:
-      err = gnutls_priority_set_direct (session, "PFS", NULL);
+      err = gnutls_priority_set_direct (session, "PFS:-VERS-SSL3.0", NULL);
       if (err != GNUTLS_E_SUCCESS)
         /* fallback if PFS is not available */
-        err = gnutls_priority_set_direct (session, "NORMAL:-RSA", NULL);
+        err = gnutls_priority_set_direct (session, "NORMAL:-RSA:-VERS-SSL3.0", NULL);
       break;
     default:
       abort ();

src/hash.c

@@ -55,11 +55,7 @@ as that of the covered work.  */
 # endif
 # include <ctype.h>
 # define c_tolower(x) tolower ((unsigned char) (x))
-# ifdef HAVE_STDINT_H
-#  include <stdint.h>
-# else
-   typedef unsigned long uintptr_t;
-# endif
+# include <stdint.h>
 #endif
 
 #include "hash.h"
@@ -767,6 +763,15 @@ main (void)
 {
   struct hash_table *ht = make_string_hash_table (0);
   char line[80];
+
+#ifdef ENABLE_NLS
+  /* Set the current locale.  */
+  setlocale (LC_ALL, "");
+  /* Set the text message domain.  */
+  bindtextdomain ("wget", LOCALEDIR);
+  textdomain ("wget");
+#endif /* ENABLE_NLS */
+
   while ((fgets (line, sizeof (line), stdin)))
     {
       int len = strlen (line);

src/hash.h

@@ -34,7 +34,7 @@ as that of the covered work.  */
 struct hash_table;
 
 struct hash_table *hash_table_new (int, unsigned long (*) (const void *),
-				   int (*) (const void *, const void *));
+                                   int (*) (const void *, const void *));
 void hash_table_destroy (struct hash_table *);
 
 void *hash_table_get (const struct hash_table *, const void *);
@@ -47,11 +47,11 @@ int hash_table_remove (struct hash_table *, const void *);
 void hash_table_clear (struct hash_table *);
 
 void hash_table_for_each (struct hash_table *,
-		          int (*) (void *, void *, void *), void *);
+                          int (*) (void *, void *, void *), void *);
 
 typedef struct {
-  void *key, *value;		/* public members */
-  void *pos, *end;		/* private members */
+  void *key, *value;    /* public members */
+  void *pos, *end;      /* private members */
 } hash_table_iterator;
 void hash_table_iterate (struct hash_table *, hash_table_iterator *);
 int hash_table_iter_next (hash_table_iterator *);

src/host.h

@@ -58,9 +58,9 @@ typedef struct {
 
   /* The actual data, in the form of struct in_addr or in6_addr: */
   union {
-    struct in_addr d4;		/* IPv4 address */
+    struct in_addr d4;      /* IPv4 address */
 #ifdef ENABLE_IPV6
-    struct in6_addr d6;		/* IPv6 address */
+    struct in6_addr d6;     /* IPv6 address */
 #endif
   } data;
 

src/html-parse.c

@@ -1189,6 +1189,14 @@ int main ()
   int read_count;
   int tag_counter = 0;
 
+#ifdef ENABLE_NLS
+  /* Set the current locale.  */
+  setlocale (LC_ALL, "");
+  /* Set the text message domain.  */
+  bindtextdomain ("wget", LOCALEDIR);
+  textdomain ("wget");
+#endif /* ENABLE_NLS */
+
   while ((read_count = fread (x + length, 1, size - length, stdin)))
     {
       length += read_count;

src/html-parse.h

@@ -32,8 +32,8 @@ as that of the covered work.  */
 #define HTML_PARSE_H
 
 struct attr_pair {
-  char *name;			/* attribute name */
-  char *value;			/* attribute value */
+  char *name;           /* attribute name */
+  char *value;          /* attribute value */
 
   /* Needed for URL conversion; the places where the value begins and
      ends, including the quotes and everything. */
@@ -45,19 +45,19 @@ struct attr_pair {
 };
 
 struct taginfo {
-  char *name;			/* tag name */
-  int end_tag_p;		/* whether this is an end-tag */
-  int nattrs;			/* number of attributes */
-  struct attr_pair *attrs;	/* attributes */
+  char *name;                   /* tag name */
+  int end_tag_p;                /* whether this is an end-tag */
+  int nattrs;                   /* number of attributes */
+  struct attr_pair *attrs;      /* attributes */
 
-  const char *start_position;	/* start position of tag */
-  const char *end_position;	/* end position of tag */
+  const char *start_position;   /* start position of tag */
+  const char *end_position;     /* end position of tag */
 
   const char *contents_begin;   /* delimiters of tag contents */
   const char *contents_end;     /* only valid if end_tag_p */
 };
 
-struct hash_table;		/* forward declaration */
+struct hash_table;              /* forward declaration */
 
 /* Flags for map_html_tags: */
 #define MHT_STRICT_COMMENTS  1  /* use strict comment interpretation */
@@ -65,7 +65,7 @@ struct hash_table;		/* forward declaration */
                                    <a href=" foo "> as "foo" */
 
 void map_html_tags (const char *, int,
-		    void (*) (struct taginfo *, void *), void *, int,
-		    const struct hash_table *, const struct hash_table *);
+                    void (*) (struct taginfo *, void *), void *, int,
+                    const struct hash_table *, const struct hash_table *);
 
 #endif /* HTML_PARSE_H */

src/html-url.c

@@ -472,7 +472,7 @@ tag_find_urls (int tagid, struct taginfo *tag, struct map_context *ctx)
 /* Handle the BASE tag, for <base href=...>. */
 
 static void
-tag_handle_base (int tagid, struct taginfo *tag, struct map_context *ctx)
+tag_handle_base (int tagid _GL_UNUSED, struct taginfo *tag, struct map_context *ctx)
 {
   struct urlpos *base_urlpos;
   int attrind;
@@ -498,7 +498,7 @@ tag_handle_base (int tagid, struct taginfo *tag, struct map_context *ctx)
 /* Mark the URL found in <form action=...> for conversion. */
 
 static void
-tag_handle_form (int tagid, struct taginfo *tag, struct map_context *ctx)
+tag_handle_form (int tagid _GL_UNUSED, struct taginfo *tag, struct map_context *ctx)
 {
   int attrind;
   char *action = find_attr (tag, "action", &attrind);
@@ -516,7 +516,7 @@ tag_handle_form (int tagid, struct taginfo *tag, struct map_context *ctx)
    links will be followed in -p mode depends on the REL attribute.  */
 
 static void
-tag_handle_link (int tagid, struct taginfo *tag, struct map_context *ctx)
+tag_handle_link (int tagid _GL_UNUSED, struct taginfo *tag, struct map_context *ctx)
 {
   int attrind;
   char *href = find_attr (tag, "href", &attrind);
@@ -565,7 +565,7 @@ tag_handle_link (int tagid, struct taginfo *tag, struct map_context *ctx)
    refresh feature and because of robot exclusion.  */
 
 static void
-tag_handle_meta (int tagid, struct taginfo *tag, struct map_context *ctx)
+tag_handle_meta (int tagid _GL_UNUSED, struct taginfo *tag, struct map_context *ctx)
 {
   char *name = find_attr (tag, "name", NULL);
   char *http_equiv = find_attr (tag, "http-equiv", NULL);
@@ -748,8 +748,9 @@ get_urls_html (const char *file, const char *url, bool *meta_disallow_follow,
   map_html_tags (fm->content, fm->length, collect_tags_mapper, &ctx, flags,
                  NULL, interesting_attributes);
 
-  /* If meta charset isn't null, override content encoding */
-  if (iri && meta_charset)
+  /* Meta charset is only valid if there was no HTTP header Content-Type charset. */
+  /* This is true for HTTP 1.0 and 1.1. */
+  if (iri && !iri->content_encoding && meta_charset)
     set_content_encoding (iri, meta_charset);
 
   DEBUGP (("no-follow in %s: %d\n", file, ctx.nofollow));

src/html-url.h

@@ -32,20 +32,21 @@ as that of the covered work.  */
 #define HTML_URL_H
 
 struct map_context {
-  char *text;			/* HTML text. */
-  char *base;			/* Base URI of the document, possibly
-				   changed through <base href=...>. */
-  const char *parent_base;	/* Base of the current document. */
-  const char *document_file;	/* File name of this document. */
-  bool nofollow;		/* whether NOFOLLOW was specified in a
+  char *text;                   /* HTML text. */
+  char *base;                   /* Base URI of the document, possibly
+                                   changed through <base href=...>. */
+  const char *parent_base;      /* Base of the current document. */
+  const char *document_file;    /* File name of this document. */
+  bool nofollow;                /* whether NOFOLLOW was specified in a
                                    <meta name=robots> tag. */
 
-  struct urlpos *head;	/* List of URLs that is being built. */
+  struct urlpos *head;          /* List of URLs that is being built. */
 };
 
 struct urlpos *get_urls_file (const char *);
 struct urlpos *get_urls_html (const char *, const char *, bool *, struct iri *);
 struct urlpos *append_url (const char *, int, int, struct map_context *);
 void free_urlpos (struct urlpos *);
+void cleanup_html_url (void);
 
 #endif /* HTML_URL_H */

src/http-ntlm.c

@@ -73,42 +73,13 @@ as that of the covered work.  */
 
 /* Define this to make the type-3 message include the NT response message */
 #define USE_NTRESPONSES 1
+
 
 /* Flag bits definitions available at on
    http://davenport.sourceforge.net/ntlm.html */
 
-#define NTLMFLAG_NEGOTIATE_UNICODE               (1<<0)
 #define NTLMFLAG_NEGOTIATE_OEM                   (1<<1)
-#define NTLMFLAG_REQUEST_TARGET                  (1<<2)
-/* unknown (1<<3) */
-#define NTLMFLAG_NEGOTIATE_SIGN                  (1<<4)
-#define NTLMFLAG_NEGOTIATE_SEAL                  (1<<5)
-#define NTLMFLAG_NEGOTIATE_DATAGRAM_STYLE        (1<<6)
-#define NTLMFLAG_NEGOTIATE_LM_KEY                (1<<7)
-#define NTLMFLAG_NEGOTIATE_NETWARE               (1<<8)
 #define NTLMFLAG_NEGOTIATE_NTLM_KEY              (1<<9)
-/* unknown (1<<10) */
-/* unknown (1<<11) */
-#define NTLMFLAG_NEGOTIATE_DOMAIN_SUPPLIED       (1<<12)
-#define NTLMFLAG_NEGOTIATE_WORKSTATION_SUPPLIED  (1<<13)
-#define NTLMFLAG_NEGOTIATE_LOCAL_CALL            (1<<14)
-#define NTLMFLAG_NEGOTIATE_ALWAYS_SIGN           (1<<15)
-#define NTLMFLAG_TARGET_TYPE_DOMAIN              (1<<16)
-#define NTLMFLAG_TARGET_TYPE_SERVER              (1<<17)
-#define NTLMFLAG_TARGET_TYPE_SHARE               (1<<18)
-#define NTLMFLAG_NEGOTIATE_NTLM2_KEY             (1<<19)
-#define NTLMFLAG_REQUEST_INIT_RESPONSE           (1<<20)
-#define NTLMFLAG_REQUEST_ACCEPT_RESPONSE         (1<<21)
-#define NTLMFLAG_REQUEST_NONNT_SESSION_KEY       (1<<22)
-#define NTLMFLAG_NEGOTIATE_TARGET_INFO           (1<<23)
-/* unknown (1<24) */
-/* unknown (1<25) */
-/* unknown (1<26) */
-/* unknown (1<27) */
-/* unknown (1<28) */
-#define NTLMFLAG_NEGOTIATE_128                   (1<<29)
-#define NTLMFLAG_NEGOTIATE_KEY_EXCHANGE          (1<<30)
-#define NTLMFLAG_NEGOTIATE_56                    (1<<31)
 
 /*
   (*) = A "security buffer" is a triplet consisting of two shorts and one
@@ -146,7 +117,7 @@ ntlm_input (struct ntlmdata *ntlm, const char *header)
          (40)    Target Information  (optional) security buffer(*)
          32 (48) start of data block
       */
-      int size;
+      ssize_t size;
       char *buffer = (char *) alloca (strlen (header));
 
       DEBUGP (("Received a type-2 NTLM message.\n"));
@@ -277,8 +248,7 @@ mkhash(const char *password,
   static const unsigned char magic[] = {
     0x4B, 0x47, 0x53, 0x21, 0x40, 0x23, 0x24, 0x25
   };
-  int i;
-  int len = strlen(password);
+  size_t i, len = strlen(password);
 
   /* make it fit at least 14 bytes */
   pw = (unsigned char *) alloca (len < 7 ? 14 : len * 2);
@@ -287,7 +257,7 @@ mkhash(const char *password,
     len = 14;
 
   for (i=0; i<len; i++)
-    pw[i] = c_toupper (password[i]);
+    pw[i] = (unsigned char) c_toupper (password[i]);
 
   for (; i<14; i++)
     pw[i] = 0;
@@ -330,13 +300,13 @@ mkhash(const char *password,
     len = strlen(password);
 
     for (i=0; i<len; i++) {
-      pw[2*i]   = password[i];
+      pw[2*i]   = (unsigned char) password[i];
       pw[2*i+1] = 0;
     }
 
 #ifdef HAVE_NETTLE
     nettle_md4_init(&MD4);
-    nettle_md4_update(&MD4, 2*len, pw);
+    nettle_md4_update(&MD4, (unsigned) (2 * len), pw);
     nettle_md4_digest(&MD4, MD4_DIGEST_SIZE, ntbuffer);
 #else
     /* create NT hashed password */
@@ -352,7 +322,7 @@ mkhash(const char *password,
 #endif
 }
 
-#define SHORTPAIR(x) ((x) & 0xff), ((x) >> 8)
+#define SHORTPAIR(x) (char) ((x) & 0xff), (char) ((x) >> 8)
 #define LONGQUARTET(x) ((x) & 0xff), (((x) >> 8)&0xff), \
   (((x) >>16)&0xff), ((x)>>24)
 
@@ -361,19 +331,19 @@ char *
 ntlm_output (struct ntlmdata *ntlm, const char *user, const char *passwd,
              bool *ready)
 {
-  const char *domain=""; /* empty */
-  const char *host=""; /* empty */
-  int domlen=strlen(domain);
-  int hostlen = strlen(host);
-  int hostoff; /* host name offset */
-  int domoff;  /* domain name offset */
-  int size;
+  const char *domain = ""; /* empty */
+  const char *host = ""; /* empty */
+  size_t domlen = strlen(domain);
+  size_t hostlen = strlen(host);
+  size_t hostoff; /* host name offset */
+  size_t domoff;  /* domain name offset */
+  size_t size;
   char *base64;
   char ntlmbuf[256]; /* enough, unless the host/domain is very long */
 
   /* point to the address of the pointer that holds the string to sent to the
      server, which is for a plain host or for a HTTP proxy */
-  char *output;
+  char *output = NULL;
 
   *ready = false;
 
@@ -386,7 +356,8 @@ ntlm_output (struct ntlmdata *ntlm, const char *user, const char *passwd,
 
   switch(ntlm->state) {
   case NTLMSTATE_TYPE1:
-  default: /* for the weird cases we (re)start here */
+  case NTLMSTATE_NONE:
+  case NTLMSTATE_LAST:
     hostoff = 32;
     domoff = hostoff + hostlen;
 
@@ -464,15 +435,15 @@ ntlm_output (struct ntlmdata *ntlm, const char *user, const char *passwd,
     */
 
   {
-    int lmrespoff;
-    int ntrespoff;
-    int useroff;
+    size_t lmrespoff;
+    size_t ntrespoff;
+    size_t useroff;
     unsigned char lmresp[0x18]; /* fixed-size */
 #ifdef USE_NTRESPONSES
     unsigned char ntresp[0x18]; /* fixed-size */
 #endif
     const char *usr;
-    int userlen;
+    size_t userlen;
 
     DEBUGP (("Creating a type-3 NTLM message.\n"));
 
@@ -482,7 +453,7 @@ ntlm_output (struct ntlmdata *ntlm, const char *user, const char *passwd,
 
     if (usr) {
       domain = user;
-      domlen = usr - domain;
+      domlen = (size_t) (usr - domain);
       usr++;
     }
     else
@@ -503,7 +474,7 @@ ntlm_output (struct ntlmdata *ntlm, const char *user, const char *passwd,
 
     /* Create the big type-3 message binary blob */
 
-    size = snprintf (ntlmbuf, sizeof(ntlmbuf),
+    size = (size_t) snprintf (ntlmbuf, sizeof(ntlmbuf),
                      "NTLMSSP%c"
                      "\x03%c%c%c" /* type-3, 32 bits */
 
@@ -586,7 +557,7 @@ ntlm_output (struct ntlmdata *ntlm, const char *user, const char *passwd,
 
     /* Make sure that the user and domain strings fit in the target buffer
        before we copy them there. */
-    if(((size_t) size + userlen + domlen) >= sizeof(ntlmbuf))
+    if((size + userlen + domlen) >= sizeof(ntlmbuf))
       return NULL;
 
     memcpy(&ntlmbuf[size], domain, domlen);
@@ -596,20 +567,20 @@ ntlm_output (struct ntlmdata *ntlm, const char *user, const char *passwd,
     size += userlen;
 
     /* we append the binary hashes to the end of the blob */
-    if(size < ((int)sizeof(ntlmbuf) - 0x18)) {
+    if(size < (sizeof(ntlmbuf) - 0x18)) {
       memcpy(&ntlmbuf[size], lmresp, 0x18);
       size += 0x18;
     }
 
 #ifdef USE_NTRESPONSES
-    if(size < ((int)sizeof(ntlmbuf) - 0x18)) {
+    if(size < (sizeof(ntlmbuf) - 0x18)) {
       memcpy(&ntlmbuf[size], ntresp, 0x18);
       size += 0x18;
     }
 #endif
 
-    ntlmbuf[56] = size & 0xff;
-    ntlmbuf[57] = size >> 8;
+    ntlmbuf[56] = (char) (size & 0xff);
+    ntlmbuf[57] = (char) (size >> 8);
 
     /* convert the binary blob into base64 */
     base64 = (char *) alloca (BASE64_LENGTH (size) + 1);

src/http.c

@@ -1,6 +1,6 @@
 /* HTTP support.
    Copyright (C) 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004,
-   2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012 Free Software Foundation,
+   2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012, 2014 Free Software Foundation,
    Inc.
 
 This file is part of GNU Wget.
@@ -1016,13 +1016,13 @@ modify_param_name(param_token *name)
   else if(delim1 == delim2)
     {
       if ((name->e - 1) == delim1)
-	{
-	  result = RFC2231_ENCODING;
-	}
+        {
+          result = RFC2231_ENCODING;
+        }
       else
-	{
-	  result = RFC2231_NOENCODING;
-	}
+        {
+          result = RFC2231_NOENCODING;
+        }
       name->e = delim1;
     }
   else
@@ -1040,13 +1040,13 @@ modify_param_name(param_token *name)
 static void
 modify_param_value (param_token *value, int encoding_type )
 {
-  if (RFC2231_ENCODING == encoding_type)
+  if (encoding_type == RFC2231_ENCODING)
     {
       const char *delim = memrchr (value->b, '\'', value->e - value->b);
-      if ( delim != NULL )
-	{
-	  value->b = (delim+1);
-	}
+      if (delim != NULL)
+        {
+          value->b = (delim+1);
+        }
     }
 }
 
@@ -1060,13 +1060,22 @@ modify_param_value (param_token *value, int encoding_type )
    filename=\"foo bar\"", the first call to this function will return
    the token named "attachment" and no value, and the second call will
    return the token named "filename" and value "foo bar".  The third
-   call will return false, indicating no more valid tokens.  */
+   call will return false, indicating no more valid tokens.
+
+   is_url_encoded is an out parameter. If not NULL, a boolean value will be
+   stored into it, letting the caller know whether or not the extracted value is
+   URL-encoded. The caller can then decode it with url_unescape(), which however
+   performs decoding in-place. URL-encoding is used by RFC 2231 to support
+   non-US-ASCII characters in HTTP header values.  */
 
 bool
 extract_param (const char **source, param_token *name, param_token *value,
-               char separator)
+               char separator, bool *is_url_encoded)
 {
   const char *p = *source;
+  int param_type;
+  if (is_url_encoded)
+    *is_url_encoded = false;   /* initializing the out parameter */
 
   while (c_isspace (*p)) ++p;
   if (!*p)
@@ -1122,9 +1131,11 @@ extract_param (const char **source, param_token *name, param_token *value,
     }
   *source = p;
 
-  int param_type = modify_param_name(name);
-  if (NOT_RFC2231 != param_type)
+  param_type = modify_param_name(name);
+  if (param_type != NOT_RFC2231)
     {
+      if (param_type == RFC2231_ENCODING && is_url_encoded)
+        *is_url_encoded = true;
       modify_param_value(value, param_type);
     }
   return true;
@@ -1137,13 +1148,16 @@ extract_param (const char **source, param_token *name, param_token *value,
 /* Appends the string represented by VALUE to FILENAME */
 
 static void
-append_value_to_filename (char **filename, param_token const * const value)
+append_value_to_filename (char **filename, param_token const * const value,
+                          bool is_url_encoded)
 {
   int original_length = strlen(*filename);
   int new_length = strlen(*filename) + (value->e - value->b);
   *filename = xrealloc (*filename, new_length+1);
-  memcpy (*filename + original_length, value->b, (value->e - value->b)); 
+  memcpy (*filename + original_length, value->b, (value->e - value->b));
   (*filename)[new_length] = '\0';
+  if (is_url_encoded)
+    url_unescape (*filename + original_length);
 }
 
 #undef MAX
@@ -1176,7 +1190,9 @@ parse_content_disposition (const char *hdr, char **filename)
 {
   param_token name, value;
   *filename = NULL;
-  while (extract_param (&hdr, &name, &value, ';'))
+  bool is_url_encoded = false;
+  for ( ; extract_param (&hdr, &name, &value, ';', &is_url_encoded);
+        is_url_encoded = false)
     {
       int isFilename = BOUNDED_EQUAL_NO_CASE ( name.b, name.e, "filename" );
       if ( isFilename && value.b != NULL)
@@ -1192,9 +1208,13 @@ parse_content_disposition (const char *hdr, char **filename)
             continue;
 
           if (*filename)
-            append_value_to_filename (filename, &value);
+            append_value_to_filename (filename, &value, is_url_encoded);
           else
-            *filename = strdupdelim (value.b, value.e);
+            {
+              *filename = strdupdelim (value.b, value.e);
+              if (is_url_encoded)
+                url_unescape (*filename);
+            }
         }
     }
 
@@ -1495,13 +1515,13 @@ File %s already there; not retrieving.\n\n"), quote (filename));
    url, warc_timestamp_str, warc_request_uuid, warc_ip, type
    and statcode will be saved in the headers of the WARC record.
    The head parameter contains the HTTP headers of the response.
- 
+
    If fp is NULL and WARC is enabled, the response body will be
    written only to the WARC file.  If WARC is disabled and fp
    is a file pointer, the data will be written to the file.
    If fp is a file pointer and WARC is enabled, the body will
    be written to both destinations.
-   
+
    Returns the error code.   */
 static int
 read_response_body (struct http_stat *hs, int sock, FILE *fp, wgint contlen,
@@ -1566,7 +1586,7 @@ read_response_body (struct http_stat *hs, int sock, FILE *fp, wgint contlen,
   /* Download the response body and write it to fp.
      If we are working on a WARC file, we simultaneously write the
      response body to warc_tmp.  */
-  hs->res = fd_read_body (sock, fp, contlen != -1 ? contlen : 0,
+  hs->res = fd_read_body (hs->local_file, sock, fp, contlen != -1 ? contlen : 0,
                           hs->restval, &hs->rd_size, &hs->len, &hs->dltime,
                           flags, warc_tmp);
   if (hs->res >= 0)
@@ -1590,7 +1610,7 @@ read_response_body (struct http_stat *hs, int sock, FILE *fp, wgint contlen,
 
       return RETRFINISHED;
     }
-  
+
   if (warc_tmp != NULL)
     fclose (warc_tmp);
 
@@ -1826,13 +1846,9 @@ gethttp (struct url *u, struct http_stat *hs, int *dt, struct url *proxy,
     request_set_header (req, "Connection", "Close", rel_none);
   else
     {
-      if (proxy == NULL)
-        request_set_header (req, "Connection", "Keep-Alive", rel_none);
-      else
-        {
-          request_set_header (req, "Connection", "Close", rel_none);
-          request_set_header (req, "Proxy-Connection", "Keep-Alive", rel_none);
-        }
+      request_set_header (req, "Connection", "Keep-Alive", rel_none);
+      if (proxy)
+        request_set_header (req, "Proxy-Connection", "Keep-Alive", rel_none);
     }
 
   if (opt.method)
@@ -2315,23 +2331,23 @@ read_header:
          But if we are writing a WARC file we are: we like to keep everyting.  */
       if (warc_enabled)
         {
-          int err;
+          int _err;
           type = resp_header_strdup (resp, "Content-Type");
-          err = read_response_body (hs, sock, NULL, contlen, 0,
+          _err = read_response_body (hs, sock, NULL, contlen, 0,
                                     chunked_transfer_encoding,
                                     u->url, warc_timestamp_str,
                                     warc_request_uuid, warc_ip, type,
                                     statcode, head);
           xfree_null (type);
 
-          if (err != RETRFINISHED || hs->res < 0)
+          if (_err != RETRFINISHED || hs->res < 0)
             {
               CLOSE_INVALIDATE (sock);
               request_free (req);
               xfree_null (message);
               resp_free (resp);
               xfree (head);
-              return err;
+              return _err;
             }
           else
             CLOSE_FINISH (sock);
@@ -2407,6 +2423,7 @@ read_header:
                   resp_free (resp);
                   xfree (head);
                   xfree (auth_stat);
+                  xfree (hs->message);
                   goto retry_with_auth;
                 }
               else
@@ -2458,6 +2475,8 @@ read_header:
                   local_file));
           hs->local_file = url_file_name (u, local_file);
         }
+
+      xfree_null (local_file);
     }
 
   /* TODO: perform this check only once. */
@@ -2576,6 +2595,7 @@ read_header:
               tmp = parse_charset (tmp2);
               if (tmp)
                 set_content_encoding (iri, tmp);
+              xfree_null(tmp);
             }
         }
     }
@@ -2598,6 +2618,22 @@ read_header:
   if (H_20X (statcode))
     *dt |= RETROKF;
 
+  if (statcode == HTTP_STATUS_NO_CONTENT)
+    {
+      /* 204 response has no body (RFC 2616, 4.3) */
+
+      /* In case the caller cares to look...  */
+      hs->len = 0;
+      hs->res = 0;
+      hs->restval = 0;
+
+      CLOSE_FINISH (sock);
+      xfree_null (type);
+      xfree (head);
+
+      return RETRFINISHED;
+    }
+
   /* Return if redirected.  */
   if (H_REDIRECTED (statcode) || statcode == HTTP_STATUS_MULTIPLE_CHOICES)
     {
@@ -2614,7 +2650,7 @@ read_header:
                      _("Location: %s%s\n"),
                      hs->newloc ? escnonprint_uri (hs->newloc) : _("unspecified"),
                      hs->newloc ? _(" [following]") : "");
- 
+
           /* In case the caller cares to look...  */
           hs->len = 0;
           hs->res = 0;
@@ -2624,18 +2660,18 @@ read_header:
              But if we are writing a WARC file we are: we like to keep everyting.  */
           if (warc_enabled)
             {
-              int err = read_response_body (hs, sock, NULL, contlen, 0,
+              int _err = read_response_body (hs, sock, NULL, contlen, 0,
                                             chunked_transfer_encoding,
                                             u->url, warc_timestamp_str,
                                             warc_request_uuid, warc_ip, type,
                                             statcode, head);
 
-              if (err != RETRFINISHED || hs->res < 0)
+              if (_err != RETRFINISHED || hs->res < 0)
                 {
                   CLOSE_INVALIDATE (sock);
                   xfree_null (type);
                   xfree (head);
-                  return err;
+                  return _err;
                 }
               else
                 CLOSE_FINISH (sock);
@@ -2671,7 +2707,6 @@ read_header:
             {
             case HTTP_STATUS_TEMPORARY_REDIRECT:
               return NEWLOCATION_KEEP_POST;
-              break;
             case HTTP_STATUS_MOVED_PERMANENTLY:
               if (opt.method && strcasecmp (opt.method, "post") != 0)
                 return NEWLOCATION_KEEP_POST;
@@ -2682,7 +2717,6 @@ read_header:
               break;
             default:
               return NEWLOCATION;
-              break;
             }
           return NEWLOCATION;
         }
@@ -2769,13 +2803,13 @@ read_header:
               logputs (LOG_VERBOSE, number_to_static_string (contlen + contrange));
               if (contlen + contrange >= 1024)
                 logprintf (LOG_VERBOSE, " (%s)",
-                           human_readable (contlen + contrange));
+                           human_readable (contlen + contrange, 10, 1));
               if (contrange)
                 {
                   if (contlen >= 1024)
                     logprintf (LOG_VERBOSE, _(", %s (%s) remaining"),
                                number_to_static_string (contlen),
-                               human_readable (contlen));
+                               human_readable (contlen, 10, 1));
                   else
                     logprintf (LOG_VERBOSE, _(", %s remaining"),
                                number_to_static_string (contlen));
@@ -2803,18 +2837,18 @@ read_header:
          But if we are writing a WARC file we are: we like to keep everyting.  */
       if (warc_enabled)
         {
-          int err = read_response_body (hs, sock, NULL, contlen, 0,
+          int _err = read_response_body (hs, sock, NULL, contlen, 0,
                                         chunked_transfer_encoding,
                                         u->url, warc_timestamp_str,
                                         warc_request_uuid, warc_ip, type,
                                         statcode, head);
 
-          if (err != RETRFINISHED || hs->res < 0)
+          if (_err != RETRFINISHED || hs->res < 0)
             {
               CLOSE_INVALIDATE (sock);
               xfree (head);
               xfree_null (type);
-              return err;
+              return _err;
             }
           else
             CLOSE_FINISH (sock);
@@ -2871,19 +2905,18 @@ read_header:
         }
       else if (ALLOW_CLOBBER || count > 0)
         {
-	  if (opt.unlink && file_exists_p (hs->local_file))
-	    {
-	      int res = unlink (hs->local_file);
-	      if (res < 0)
-		{
-		  logprintf (LOG_NOTQUIET, "%s: %s\n", hs->local_file,
-			     strerror (errno));
-		  CLOSE_INVALIDATE (sock);
-		  xfree (head);
-      xfree_null (type);
-		  return UNLINKERR;
-		}
-	    }
+          if (opt.unlink && file_exists_p (hs->local_file))
+            {
+              if (unlink (hs->local_file) < 0)
+                {
+                  logprintf (LOG_NOTQUIET, "%s: %s\n", hs->local_file,
+                             strerror (errno));
+                  CLOSE_INVALIDATE (sock);
+                  xfree (head);
+                  xfree_null (type);
+                  return UNLINKERR;
+                }
+            }
 
 #ifdef __VMS
           int open_id;
@@ -2925,11 +2958,8 @@ read_header:
     fp = output_stream;
 
   /* Print fetch message, if opt.verbose.  */
-  if (opt.verbose)
-    {
-      logprintf (LOG_NOTQUIET, _("Saving to: %s\n"),
+      logprintf (LOG_VERBOSE, _("Saving to: %s\n"),
                  HYPHENP (hs->local_file) ? quote ("STDOUT") : quote (hs->local_file));
-    }
 
 
   err = read_response_body (hs, sock, fp, contlen, contrange,
@@ -2990,7 +3020,7 @@ http_loop (struct url *u, struct url *original_url, char **newloc,
   /* Reset NEWLOC parameter. */
   *newloc = NULL;
 
-  /* This used to be done in main(), but it's a better idea to do it
+  /* This used to be done in main, but it's a better idea to do it
      here so that we don't go through the hoops if we're just using
      FTP or whatever. */
   if (opt.cookies)
@@ -3101,6 +3131,8 @@ Spider mode enabled. Check if remote file exists.\n"));
       /* Decide whether or not to restart.  */
       if (force_full_retrieve)
         hstat.restval = hstat.len;
+      else if (opt.start_pos >= 0)
+        hstat.restval = opt.start_pos;
       else if (opt.always_rest
           && got_name
           && stat (hstat.local_file, &st) == 0
@@ -3139,7 +3171,7 @@ Spider mode enabled. Check if remote file exists.\n"));
 
       switch (err)
         {
-        case HERR: case HEOF: case CONSOCKERR: case CONCLOSED:
+        case HERR: case HEOF: case CONSOCKERR:
         case CONERROR: case READERR: case WRITEFAILED:
         case RANGEERR: case FOPEN_EXCL_ERR:
           /* Non-fatal errors continue executing the loop, which will
@@ -3382,6 +3414,8 @@ Remote file exists.\n\n"));
               got_name = true;
               *dt &= ~HEAD_ONLY;
               count = 0;          /* the retrieve count for HEAD is reset */
+              xfree_null (hstat.message);
+              xfree_null (hstat.error);
               continue;
             } /* send_head_first */
         } /* !got_head */
@@ -3529,7 +3563,10 @@ Remote file exists.\n\n"));
 
 exit:
   if (ret == RETROK && local_file)
-    *local_file = xstrdup (hstat.local_file);
+    {
+      xfree_null (*local_file);
+      *local_file = xstrdup (hstat.local_file);
+    }
   free_hstat (&hstat);
 
   return ret;
@@ -3730,7 +3767,7 @@ digest_authentication_encode (const char *au, const char *user,
   realm = opaque = nonce = algorithm = qop = NULL;
 
   au += 6;                      /* skip over `Digest' */
-  while (extract_param (&au, &name, &value, ','))
+  while (extract_param (&au, &name, &value, ',', NULL))
     {
       size_t i;
       size_t namelen = name.e - name.b;
@@ -3999,10 +4036,12 @@ ensure_extension (struct http_stat *hs, const char *ext, int *dt)
 {
   char *last_period_in_local_filename = strrchr (hs->local_file, '.');
   char shortext[8];
-  int len = strlen (ext);
+  int len;
+  shortext[0] = '\0';
+  len = strlen (ext);
   if (len == 5)
     {
-      strncpy (shortext, ext, len - 1);
+      memcpy (shortext, ext, len - 1);
       shortext[len - 1] = '\0';
     }
 
@@ -4034,12 +4073,12 @@ ensure_extension (struct http_stat *hs, const char *ext, int *dt)
 #ifdef TESTING
 
 const char *
-test_parse_content_disposition()
+test_parse_content_disposition(void)
 {
-  int i;
-  struct {
-    char *hdrval;
-    char *filename;
+  unsigned i;
+  static const struct {
+    const char *hdrval;
+    const char *filename;
     bool result;
   } test_array[] = {
     { "filename=\"file.ext\"", "file.ext", true },
@@ -4050,7 +4089,7 @@ test_parse_content_disposition()
     { "attachement; filename*0=\"hello\"; filename*1=\"world.txt\"", "helloworld.txt", true },
   };
 
-  for (i = 0; i < sizeof(test_array)/sizeof(test_array[0]); ++i)
+  for (i = 0; i < countof(test_array); ++i)
     {
       char *filename;
       bool res;
@@ -4071,4 +4110,3 @@ test_parse_content_disposition()
 /*
  * vim: et sts=2 sw=2 cino+={s
  */
-

src/http.h

@@ -43,7 +43,7 @@ typedef struct {
   /* A token consists of characters in the [b, e) range. */
   const char *b, *e;
 } param_token;
-bool extract_param (const char **, param_token *, param_token *, char);
+bool extract_param (const char **, param_token *, param_token *, char, bool *);
 
 
 #endif /* HTTP_H */

src/init.c

@@ -1,6 +1,6 @@
 /* Reading/parsing the initialization file.
    Copyright (C) 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004,
-   2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012 Free Software Foundation,
+   2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012, 2014 Free Software Foundation,
    Inc.
 
 This file is part of GNU Wget.
@@ -68,6 +68,7 @@ as that of the covered work.  */
 #include "http.h"               /* for http_cleanup */
 #include "retr.h"               /* for output_stream */
 #include "warc.h"               /* for warc_close */
+#include "spider.h"             /* for spider_cleanup */
 
 #ifdef TESTING
 #include "test.h"
@@ -149,7 +150,7 @@ static const struct {
   { "certificatetype",  &opt.cert_type,         cmd_cert_type },
   { "checkcertificate", &opt.check_cert,        cmd_boolean },
 #endif
-  { "chooseconfig",     &opt.choose_config,	cmd_file },
+  { "chooseconfig",     &opt.choose_config,     cmd_file },
   { "connecttimeout",   &opt.connect_timeout,   cmd_time },
   { "contentdisposition", &opt.content_disposition, cmd_boolean },
   { "contentonerror",   &opt.content_on_error,  cmd_boolean },
@@ -157,10 +158,8 @@ static const struct {
   { "convertlinks",     &opt.convert_links,     cmd_boolean },
   { "cookies",          &opt.cookies,           cmd_boolean },
   { "cutdirs",          &opt.cut_dirs,          cmd_number },
-#ifdef ENABLE_DEBUG
   { "debug",            &opt.debug,             cmd_boolean },
-#endif
-  { "defaultpage", 	&opt.default_page,      cmd_string },
+  { "defaultpage",      &opt.default_page,      cmd_string },
   { "deleteafter",      &opt.delete_after,      cmd_boolean },
   { "dirprefix",        &opt.dir_prefix,        cmd_directory },
   { "dirstruct",        NULL,                   cmd_spec_dirstruct },
@@ -220,6 +219,7 @@ static const struct {
   { "mirror",           NULL,                   cmd_spec_mirror },
   { "netrc",            &opt.netrc,             cmd_boolean },
   { "noclobber",        &opt.noclobber,         cmd_boolean },
+  { "noconfig",         &opt.noconfig,          cmd_boolean },
   { "noparent",         &opt.no_parent,         cmd_boolean },
   { "noproxy",          &opt.no_proxy,          cmd_vector },
   { "numtries",         &opt.ntry,              cmd_number_inf },/* deprecated*/
@@ -269,8 +269,10 @@ static const struct {
 #endif
   { "serverresponse",   &opt.server_response,   cmd_boolean },
   { "showalldnsentries", &opt.show_all_dns_entries, cmd_boolean },
+  { "showprogress",     &opt.show_progress,      cmd_boolean },
   { "spanhosts",        &opt.spanhost,          cmd_boolean },
   { "spider",           &opt.spider,            cmd_boolean },
+  { "startpos",         &opt.start_pos,         cmd_bytes },
   { "strictcomments",   &opt.strict_comments,   cmd_boolean },
   { "timeout",          NULL,                   cmd_spec_timeout },
   { "timestamping",     &opt.timestamping,      cmd_boolean },
@@ -364,6 +366,22 @@ defaults (void)
 
   opt.dns_cache = true;
   opt.ftp_pasv = true;
+  /* 2014-09-07  Darshit Shah  <darnir@gmail.com>
+   * opt.retr_symlinks is set to true by default. Creating symbolic links on the
+   * local filesystem pose a security threat by malicious FTP Servers that
+   * server a specially crafted .listing file akin to this:
+   *
+   * lrwxrwxrwx   1 root     root           33 Dec 25  2012 JoCxl6d8rFU -> /
+   * drwxrwxr-x  15 1024     106          4096 Aug 28 02:02 JoCxl6d8rFU
+   *
+   * A .listing file in this fashion makes Wget susceptiple to a symlink attack
+   * wherein the attacker is able to create arbitrary files, directories and
+   * symbolic links on the target system and even set permissions.
+   *
+   * Hence, by default Wget attempts to retrieve the pointed-to files and does
+   * not create the symbolic links locally.
+   */
+  opt.retr_symlinks = true;
 
 #ifdef HAVE_SSL
   opt.check_cert = true;
@@ -407,6 +425,11 @@ defaults (void)
   opt.warc_cdx_dedup_filename = NULL;
   opt.warc_tempdir = NULL;
   opt.warc_keep_log = true;
+
+  /* Use a negative value to mark the absence of --start-pos option */
+  opt.start_pos = -1;
+  opt.show_progress = false;
+  opt.noscroll = false;
 }
 
 /* Return the user's home directory (strdup-ed), or NULL if none is
@@ -438,12 +461,7 @@ home_dir (void)
           assert (p);
 
           len = p - buff + 1;
-          buff = malloc (len + 1);
-          if (buff == NULL)
-            return NULL;
-
-          strncpy (buff, _w32_get_argv0 (), len);
-          buff[len] = '\0';
+          buff = strdup (_w32_get_argv0 ());
 
           home = buf;
 #elif !defined(WINDOWS)
@@ -481,7 +499,7 @@ wgetrc_env_file_name (void)
         {
           fprintf (stderr, _("%s: WGETRC points to %s, which doesn't exist.\n"),
                    exec_name, env);
-          exit (1);
+          exit (WGET_EXIT_GENERIC_ERROR);
         }
       return xstrdup (env);
     }
@@ -653,7 +671,7 @@ initialize (void)
 Parsing system wgetrc file (env SYSTEM_WGETRC) failed.  Please check\n\
 '%s',\n\
 or specify a different file using --config.\n"), env_sysrc);
-          exit (2);
+          exit (WGET_EXIT_PARSE_ERROR);
         }
     }
   /* Otherwise, if SYSTEM_WGETRC is defined, use it.  */
@@ -668,7 +686,7 @@ or specify a different file using --config.\n"), env_sysrc);
 Parsing system wgetrc file failed.  Please check\n\
 '%s',\n\
 or specify a different file using --config.\n"), SYSTEM_WGETRC);
-      exit (2);
+      exit (WGET_EXIT_PARSE_ERROR);
     }
 #endif
   /* Override it with your own, if one exists.  */
@@ -690,7 +708,7 @@ or specify a different file using --config.\n"), SYSTEM_WGETRC);
 
   /* If there were errors processing either `.wgetrc', abort. */
   if (!ok)
-    exit (2);
+    exit (WGET_EXIT_PARSE_ERROR);
 
   xfree (file);
   return;
@@ -814,16 +832,16 @@ setval_internal_tilde (int comind, const char *com, const char *val)
       pstring = commands[comind].place;
       home = home_dir ();
       if (home)
-	{
-	  homelen = strlen (home);
-	  while (homelen && ISSEP (home[homelen - 1]))
-            home[--homelen] = '\0';
+        {
+          homelen = strlen (home);
+          while (homelen && ISSEP (home[homelen - 1]))
+                 home[--homelen] = '\0';
 
-	  /* Skip the leading "~/". */
-	  for (++val; ISSEP (*val); val++)
-  	    ;
-	  *pstring = concat_strings (home, "/", val, (char *)0);
-	}
+          /* Skip the leading "~/". */
+          for (++val; ISSEP (*val); val++)
+            ;
+          *pstring = concat_strings (home, "/", val, (char *)0);
+        }
     }
   return ret;
 }
@@ -831,8 +849,8 @@ setval_internal_tilde (int comind, const char *com, const char *val)
 /* Run command COM with value VAL.  If running the command produces an
    error, report the error and exit.
 
-   This is intended to be called from main() to modify Wget's behavior
-   through command-line switches.  Since COM is hard-coded in main(),
+   This is intended to be called from main to modify Wget's behavior
+   through command-line switches.  Since COM is hard-coded in main,
    it is not canonicalized, and this aborts when COM is not found.
 
    If COMIND's are exported to init.h, this function will be changed
@@ -849,7 +867,7 @@ setoptval (const char *com, const char *val, const char *optname)
 
   assert (val != NULL);
   if (!setval_internal (command_by_name (com), dd_optname, val))
-    exit (2);
+    exit (WGET_EXIT_PARSE_ERROR);
 }
 
 /* Parse OPT into command and value and run it.  For example,
@@ -857,22 +875,22 @@ setoptval (const char *com, const char *val, const char *optname)
    This is used by the `--execute' flag in main.c.  */
 
 void
-run_command (const char *opt)
+run_command (const char *cmdopt)
 {
   char *com, *val;
   int comind;
-  switch (parse_line (opt, &com, &val, &comind))
+  switch (parse_line (cmdopt, &com, &val, &comind))
     {
     case line_ok:
       if (!setval_internal (comind, com, val))
-        exit (2);
+        exit (WGET_EXIT_PARSE_ERROR);
       xfree (com);
       xfree (val);
       break;
     default:
       fprintf (stderr, _("%s: Invalid --execute command %s\n"),
-               exec_name, quote (opt));
-      exit (2);
+               exec_name, quote (cmdopt));
+      exit (WGET_EXIT_PARSE_ERROR);
     }
 }
 
@@ -954,7 +972,7 @@ cmd_number_inf (const char *com, const char *val, void *place)
 /* Copy (strdup) the string at COM to a new location and place a
    pointer to *PLACE.  */
 static bool
-cmd_string (const char *com, const char *val, void *place)
+cmd_string (const char *com _GL_UNUSED, const char *val, void *place)
 {
   char **pstring = (char **)place;
 
@@ -965,7 +983,7 @@ cmd_string (const char *com, const char *val, void *place)
 
 /* Like cmd_string but ensure the string is upper case.  */
 static bool
-cmd_string_uppercase (const char *com, const char *val, void *place)
+cmd_string_uppercase (const char *com _GL_UNUSED, const char *val, void *place)
 {
   char *q, **pstring;
   pstring = (char **)place;
@@ -985,7 +1003,7 @@ cmd_string_uppercase (const char *com, const char *val, void *place)
    `.wgetrc'.  In that case, and if VAL begins with `~', the tilde
    gets expanded to the user's home directory.  */
 static bool
-cmd_file (const char *com, const char *val, void *place)
+cmd_file (const char *com _GL_UNUSED, const char *val, void *place)
 {
   char **pstring = (char **)place;
 
@@ -1032,7 +1050,7 @@ cmd_directory (const char *com, const char *val, void *place)
    PLACE vector is cleared instead.  */
 
 static bool
-cmd_vector (const char *com, const char *val, void *place)
+cmd_vector (const char *com _GL_UNUSED, const char *val, void *place)
 {
   char ***pvec = (char ***)place;
 
@@ -1047,7 +1065,7 @@ cmd_vector (const char *com, const char *val, void *place)
 }
 
 static bool
-cmd_directory_vector (const char *com, const char *val, void *place)
+cmd_directory_vector (const char *com _GL_UNUSED, const char *val, void *place)
 {
   char ***pvec = (char ***)place;
 
@@ -1263,7 +1281,7 @@ cmd_cert_type (const char *com, const char *val, void *place)
 static bool check_user_specified_header (const char *);
 
 static bool
-cmd_spec_dirstruct (const char *com, const char *val, void *place_ignored)
+cmd_spec_dirstruct (const char *com, const char *val, void *place_ignored _GL_UNUSED)
 {
   if (!cmd_boolean (com, val, &opt.dirstruct))
     return false;
@@ -1277,7 +1295,7 @@ cmd_spec_dirstruct (const char *com, const char *val, void *place_ignored)
 }
 
 static bool
-cmd_spec_header (const char *com, const char *val, void *place_ignored)
+cmd_spec_header (const char *com, const char *val, void *place_ignored _GL_UNUSED)
 {
   /* Empty value means reset the list of headers. */
   if (*val == '\0')
@@ -1298,7 +1316,7 @@ cmd_spec_header (const char *com, const char *val, void *place_ignored)
 }
 
 static bool
-cmd_spec_warc_header (const char *com, const char *val, void *place_ignored)
+cmd_spec_warc_header (const char *com, const char *val, void *place_ignored _GL_UNUSED)
 {
   /* Empty value means reset the list of headers. */
   if (*val == '\0')
@@ -1319,7 +1337,7 @@ cmd_spec_warc_header (const char *com, const char *val, void *place_ignored)
 }
 
 static bool
-cmd_spec_htmlify (const char *com, const char *val, void *place_ignored)
+cmd_spec_htmlify (const char *com, const char *val, void *place_ignored _GL_UNUSED)
 {
   int flag = cmd_boolean (com, val, &opt.htmlify);
   if (flag && !opt.htmlify)
@@ -1331,7 +1349,7 @@ cmd_spec_htmlify (const char *com, const char *val, void *place_ignored)
    no limit on max. recursion depth, and don't remove listings.  */
 
 static bool
-cmd_spec_mirror (const char *com, const char *val, void *place_ignored)
+cmd_spec_mirror (const char *com, const char *val, void *place_ignored _GL_UNUSED)
 {
   int mirror;
 
@@ -1353,7 +1371,7 @@ cmd_spec_mirror (const char *com, const char *val, void *place_ignored)
    "IPv4", "IPv6", and "none".  */
 
 static bool
-cmd_spec_prefer_family (const char *com, const char *val, void *place_ignored)
+cmd_spec_prefer_family (const char *com, const char *val, void *place_ignored _GL_UNUSED)
 {
   static const struct decode_item choices[] = {
     { "IPv4", prefer_ipv4 },
@@ -1372,7 +1390,7 @@ cmd_spec_prefer_family (const char *com, const char *val, void *place_ignored)
    implementation before that.  */
 
 static bool
-cmd_spec_progress (const char *com, const char *val, void *place_ignored)
+cmd_spec_progress (const char *com, const char *val, void *place_ignored _GL_UNUSED)
 {
   if (!valid_progress_implementation_p (val))
     {
@@ -1383,7 +1401,7 @@ cmd_spec_progress (const char *com, const char *val, void *place_ignored)
   xfree_null (opt.progress_type);
 
   /* Don't call set_progress_implementation here.  It will be called
-     in main() when it becomes clear what the log output is.  */
+     in main when it becomes clear what the log output is.  */
   opt.progress_type = xstrdup (val);
   return true;
 }
@@ -1393,7 +1411,7 @@ cmd_spec_progress (const char *com, const char *val, void *place_ignored)
    is specified.  */
 
 static bool
-cmd_spec_recursive (const char *com, const char *val, void *place_ignored)
+cmd_spec_recursive (const char *com, const char *val, void *place_ignored _GL_UNUSED)
 {
   if (!cmd_boolean (com, val, &opt.recursive))
     return false;
@@ -1408,7 +1426,7 @@ cmd_spec_recursive (const char *com, const char *val, void *place_ignored)
 /* Validate --regex-type and set the choice.  */
 
 static bool
-cmd_spec_regex_type (const char *com, const char *val, void *place_ignored)
+cmd_spec_regex_type (const char *com, const char *val, void *place_ignored _GL_UNUSED)
 {
   static const struct decode_item choices[] = {
     { "posix", regex_type_posix },
@@ -1425,7 +1443,7 @@ cmd_spec_regex_type (const char *com, const char *val, void *place_ignored)
 }
 
 static bool
-cmd_spec_restrict_file_names (const char *com, const char *val, void *place_ignored)
+cmd_spec_restrict_file_names (const char *com, const char *val, void *place_ignored _GL_UNUSED)
 {
   int restrict_os = opt.restrict_files_os;
   int restrict_ctrl = opt.restrict_files_ctrl;
@@ -1479,7 +1497,7 @@ cmd_spec_restrict_file_names (const char *com, const char *val, void *place_igno
 }
 
 static bool
-cmd_spec_report_speed (const char *com, const char *val, void *place_ignored)
+cmd_spec_report_speed (const char *com, const char *val, void *place_ignored _GL_UNUSED)
 {
   opt.report_bps = strcasecmp (val, "bits") == 0;
   if (!opt.report_bps)
@@ -1496,6 +1514,8 @@ cmd_spec_secure_protocol (const char *com, const char *val, void *place)
     { "sslv2", secure_protocol_sslv2 },
     { "sslv3", secure_protocol_sslv3 },
     { "tlsv1", secure_protocol_tlsv1 },
+    { "tlsv1_1", secure_protocol_tlsv1_1 },
+    { "tlsv1_2", secure_protocol_tlsv1_2 },
     { "pfs", secure_protocol_pfs },
   };
   int ok = decode_string (val, choices, countof (choices), place);
@@ -1508,7 +1528,7 @@ cmd_spec_secure_protocol (const char *com, const char *val, void *place)
 /* Set all three timeout values. */
 
 static bool
-cmd_spec_timeout (const char *com, const char *val, void *place_ignored)
+cmd_spec_timeout (const char *com, const char *val, void *place_ignored _GL_UNUSED)
 {
   double value;
   if (!cmd_time (com, val, &value))
@@ -1520,7 +1540,7 @@ cmd_spec_timeout (const char *com, const char *val, void *place_ignored)
 }
 
 static bool
-cmd_spec_useragent (const char *com, const char *val, void *place_ignored)
+cmd_spec_useragent (const char *com, const char *val, void *place_ignored _GL_UNUSED)
 {
   /* Disallow embedded newlines.  */
   if (strchr (val, '\n'))
@@ -1539,7 +1559,7 @@ cmd_spec_useragent (const char *com, const char *val, void *place_ignored)
    some random hackery for disallowing -q -v).  */
 
 static bool
-cmd_spec_verbose (const char *com, const char *val, void *place_ignored)
+cmd_spec_verbose (const char *com, const char *val, void *place_ignored _GL_UNUSED)
 {
   bool flag;
   if (cmd_boolean (com, val, &flag))
@@ -1692,11 +1712,6 @@ decode_string (const char *val, const struct decode_item *items, int itemcount,
   return false;
 }
 
-
-void cleanup_html_url (void);
-void spider_cleanup (void);
-
-
 /* Free the memory allocated by global variables.  */
 void
 cleanup (void)
@@ -1732,14 +1747,11 @@ cleanup (void)
   spider_cleanup ();
   host_cleanup ();
   log_cleanup ();
+  netrc_cleanup (netrc_list);
 
   for (i = 0; i < nurl; i++)
     xfree (url[i]);
 
-  {
-    extern acc_t *netrc_list;
-    free_netrc (netrc_list);
-  }
   xfree_null (opt.choose_config);
   xfree_null (opt.lfilename);
   xfree_null (opt.dir_prefix);
@@ -1789,34 +1801,27 @@ cleanup (void)
 #ifdef TESTING
 
 const char *
-test_commands_sorted()
+test_commands_sorted(void)
 {
-  int prev_idx = 0, next_idx = 1;
-  int command_count = countof (commands) - 1;
-  int cmp = 0;
-  while (next_idx <= command_count)
+  unsigned i;
+
+  for (i = 1; i < countof(commands); ++i)
     {
-      cmp = strcasecmp (commands[prev_idx].name, commands[next_idx].name);
-      if (cmp > 0)
+      if (strcasecmp (commands[i - 1].name, commands[i].name) > 0)
         {
           mu_assert ("FAILED", false);
           break;
         }
-      else
-        {
-          prev_idx ++;
-	  next_idx ++;
-        }
     }
   return NULL;
 }
 
 const char *
-test_cmd_spec_restrict_file_names()
+test_cmd_spec_restrict_file_names(void)
 {
-  int i;
-  struct {
-    char *val;
+  unsigned i;
+  static const struct {
+    const char *val;
     int expected_restrict_files_os;
     int expected_restrict_files_ctrl;
     int expected_restrict_files_case;
@@ -1828,7 +1833,7 @@ test_cmd_spec_restrict_file_names()
     { "unix,nocontrol,lowercase,", restrict_unix, false, restrict_lowercase, true },
   };
 
-  for (i = 0; i < sizeof(test_array)/sizeof(test_array[0]); ++i)
+  for (i = 0; i < countof(test_array); ++i)
     {
       bool res;
 
@@ -1852,4 +1857,3 @@ test_cmd_spec_restrict_file_names()
 }
 
 #endif /* TESTING */
-

src/iri.c

@@ -38,15 +38,13 @@ as that of the covered work.  */
 #include <errno.h>
 
 #include "utils.h"
+#include "url.h"
 
 /* RFC3987 section 3.1 mandates STD3 ASCII RULES */
 #define IDNA_FLAGS  IDNA_USE_STD3_ASCII_RULES
 
 /* Note: locale encoding is kept in options struct (opt.locale) */
 
-static bool do_conversion (iconv_t cd, char *in, size_t inlen, char **out);
-
-
 /* Given a string containing "charset=XXX", return the encoding if found,
    or NULL otherwise */
 char *
@@ -110,52 +108,34 @@ check_encoding_name (char *encoding)
   return true;
 }
 
-/* Try converting string str from locale to UTF-8. Return a new string
-   on success, or str on error or if conversion isn't needed. */
-const char *
-locale_to_utf8 (const char *str)
-{
-  iconv_t l2u;
-  char *new;
-
-  /* That shouldn't happen, just in case */
-  if (!opt.locale)
-    {
-      logprintf (LOG_VERBOSE, _("locale_to_utf8: locale is unset\n"));
-      opt.locale = find_locale ();
-    }
-
-  if (!opt.locale || !strcasecmp (opt.locale, "utf-8"))
-    return str;
-
-  l2u = iconv_open ("UTF-8", opt.locale);
-  if (l2u != (iconv_t)(-1))
-    {
-      logprintf (LOG_VERBOSE, _("Conversion from %s to %s isn't supported\n"),
-                 quote (opt.locale), quote ("UTF-8"));
-      return str;
-    }
-
-  if (do_conversion (l2u, (char *) str, strlen ((char *) str), &new))
-    return (const char *) new;
-
-  return str;
-}
-
 /* Do the conversion according to the passed conversion descriptor cd. *out
    will contain the transcoded string on success. *out content is
    unspecified otherwise. */
 static bool
-do_conversion (iconv_t cd, char *in, size_t inlen, char **out)
+do_conversion (const char *tocode, const char *fromcode, char *in, size_t inlen, char **out)
 {
+  iconv_t cd;
   /* sXXXav : hummm hard to guess... */
-  size_t len, done, outlen = inlen * 2;
+  size_t len, done, outlen;
   int invalid = 0, tooshort = 0;
-  char *s;
+  char *s, *in_org, *in_save;
 
-  s = xmalloc (outlen + 1);
-  *out = s;
-  len = outlen;
+  cd = iconv_open (tocode, fromcode);
+  if (cd == (iconv_t)(-1))
+    {
+      logprintf (LOG_VERBOSE, _("Conversion from %s to %s isn't supported\n"),
+                 quote (opt.locale), quote ("UTF-8"));
+      return false;
+    }
+
+  /* iconv() has to work on an unescaped string */
+  in_org = in;
+  in_save = in = strndup(in, inlen);
+  url_unescape(in);
+  inlen = strlen(in);
+
+  len = outlen = inlen * 2;
+  *out = s = xmalloc (outlen + 1);
   done = 0;
 
   for (;;)
@@ -164,6 +144,9 @@ do_conversion (iconv_t cd, char *in, size_t inlen, char **out)
         {
           *out = s;
           *(s + len - outlen - done) = '\0';
+          xfree(in_save);
+          iconv_close(cd);
+          logprintf (LOG_VERBOSE, _("converted '%s' (%s) -> '%s' (%s)\n"), in_org, fromcode, *out, tocode);
           return true;
         }
 
@@ -202,9 +185,35 @@ do_conversion (iconv_t cd, char *in, size_t inlen, char **out)
         }
     }
 
+    xfree(in_save);
+    iconv_close(cd);
+    logprintf (LOG_VERBOSE, _("converted '%s' (%s) -> '%s' (%s)\n"), in_org, fromcode, *out, tocode);
     return false;
 }
 
+/* Try converting string str from locale to UTF-8. Return a new string
+   on success, or str on error or if conversion isn't needed. */
+const char *
+locale_to_utf8 (const char *str)
+{
+  char *new;
+
+  /* That shouldn't happen, just in case */
+  if (!opt.locale)
+    {
+      logprintf (LOG_VERBOSE, _("locale_to_utf8: locale is unset\n"));
+      opt.locale = find_locale ();
+    }
+
+  if (!opt.locale || !strcasecmp (opt.locale, "utf-8"))
+    return str;
+
+  if (do_conversion ("UTF-8", opt.locale, (char *) str, strlen ((char *) str), &new))
+    return (const char *) new;
+
+  return str;
+}
+
 /* Try to "ASCII encode" UTF-8 host. Return the new domain on success or NULL
    on error. */
 char *
@@ -256,22 +265,21 @@ idn_decode (char *host)
 /* Try to transcode string str from remote encoding to UTF-8. On success, *new
    contains the transcoded string. *new content is unspecified otherwise. */
 bool
-remote_to_utf8 (struct iri *i, const char *str, const char **new)
+remote_to_utf8 (struct iri *iri, const char *str, const char **new)
 {
-  iconv_t cd;
   bool ret = false;
 
-  if (!i->uri_encoding)
+  if (!iri->uri_encoding)
     return false;
 
   /* When `i->uri_encoding' == "UTF-8" there is nothing to convert.  But we must
      test for non-ASCII symbols for correct hostname processing in `idn_encode'
      function. */
-  if (!strcmp (i->uri_encoding, "UTF-8"))
+  if (!strcasecmp (iri->uri_encoding, "UTF-8"))
     {
-      int i, len = strlen (str);
-      for (i = 0; i < len; i++)
-        if ((unsigned char) str[i] >= (unsigned char) '\200')
+      const char *p = str;
+      for (p = str; *p; p++)
+        if (*p < 0)
           {
             *new = strdup (str);
             return true;
@@ -279,15 +287,9 @@ remote_to_utf8 (struct iri *i, const char *str, const char **new)
       return false;
     }
 
-  cd = iconv_open ("UTF-8", i->uri_encoding);
-  if (cd == (iconv_t)(-1))
-    return false;
-
-  if (do_conversion (cd, (char *) str, strlen ((char *) str), (char **) new))
+  if (do_conversion ("UTF-8", iri->uri_encoding, (char *) str, strlen (str), (char **) new))
     ret = true;
 
-  iconv_close (cd);
-
   /* Test if something was converted */
   if (!strcmp (str, *new))
     {
@@ -365,4 +367,3 @@ set_content_encoding (struct iri *i, char *charset)
 
   i->content_encoding = charset ? xstrdup (charset) : NULL;
 }
-

src/log.c

@@ -39,6 +39,7 @@ as that of the covered work.  */
 #include <errno.h>
 
 #include "utils.h"
+#include "exits.h"
 #include "log.h"
 
 /* 2005-10-25 SMS.
@@ -278,6 +279,10 @@ saved_append (const char *s)
 #define CHECK_VERBOSE(x)                        \
   switch (x)                                    \
     {                                           \
+    case LOG_PROGRESS:                          \
+      if (!opt.show_progress)                   \
+        return;                                 \
+      break;                                    \
     case LOG_ALWAYS:                            \
       break;                                    \
     case LOG_NOTQUIET:                          \
@@ -543,7 +548,7 @@ logprintf (enum log_options o, const char *fmt, ...)
       va_end (args);
 
       if (done && errno == EPIPE)
-        exit (1);
+        exit (WGET_EXIT_GENERIC_ERROR);
     }
   while (!done);
 }
@@ -587,7 +592,7 @@ log_init (const char *file, bool appendp)
       if (!logfp)
         {
           fprintf (stderr, "%s: %s: %s\n", exec_name, file, strerror (errno));
-          exit (1);
+          exit (WGET_EXIT_GENERIC_ERROR);
         }
     }
   else

src/log.h

@@ -36,7 +36,7 @@ as that of the covered work.  */
 
 #include <stdio.h>
 
-enum log_options { LOG_VERBOSE, LOG_NOTQUIET, LOG_NONVERBOSE, LOG_ALWAYS };
+enum log_options { LOG_VERBOSE, LOG_NOTQUIET, LOG_NONVERBOSE, LOG_ALWAYS, LOG_PROGRESS };
 
 void log_set_warc_log_fp (FILE *);
 

src/mswindows.c

@@ -1,6 +1,6 @@
 /* mswindows.c -- Windows-specific support
    Copyright (C) 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004,
-   2005, 2006, 2007, 2008, 2009, 2010, 2011 Free Software Foundation,
+   2005, 2006, 2007, 2008, 2009, 2010, 2011, 2014 Free Software Foundation,
    Inc.
 
 This file is part of GNU Wget.
@@ -42,6 +42,7 @@ as that of the covered work.  */
 
 #include "utils.h"
 #include "url.h"
+#include "exits.h"
 
 #ifndef ES_SYSTEM_REQUIRED
 #define ES_SYSTEM_REQUIRED  0x00000001
@@ -164,8 +165,8 @@ fake_fork_child (void)
       if (new_log_fp)
         {
           info->logfile_changed = true;
-          strncpy (info->lfilename, opt.lfilename, sizeof (info->lfilename));
-          info->lfilename[sizeof (info->lfilename) - 1] = '\0';
+          snprintf (info->lfilename, sizeof (info->lfilename), "%s",
+                    opt.lfilename);
           fclose (new_log_fp);
         }
     }
@@ -308,7 +309,7 @@ cleanup:
 
   /* We're the parent.  If all is well, terminate.  */
   if (rv)
-    exit (0);
+    exit (WGET_EXIT_SUCCESS);
 
   /* We failed, return.  */
 }
@@ -461,7 +462,7 @@ ws_startup (void)
     {
       fprintf (stderr, _("%s: Couldn't find usable socket driver.\n"),
                exec_name);
-      exit (1);
+      exit (WGET_EXIT_GENERIC_ERROR);
     }
 
   if (data.wVersion < requested)
@@ -469,7 +470,7 @@ ws_startup (void)
       fprintf (stderr, _("%s: Couldn't find usable socket driver.\n"),
                exec_name);
       WSACleanup ();
-      exit (1);
+      exit (WGET_EXIT_GENERIC_ERROR);
     }
 
   atexit (ws_cleanup);
@@ -620,35 +621,35 @@ inet_ntop (int af, const void *src, char *dst, socklen_t cnt)
 void
 set_windows_fd_as_blocking_socket (int fd)
 {
-	/* 04/2011
+    /* 04/2011
      gnulib select() converts blocking sockets to nonblocking in windows
      discussed here:
      http://old.nabble.com/blocking-socket-is-nonblocking-after-calling-gnulib-
      select%28%29-in-windows-td31432857.html
 
      wget uses blocking sockets so we must convert them back to blocking.
-	*/
-	int ret = 0;
-	int wsagle = 0;
-	const int zero = 0;
+    */
+    int ret = 0;
+    int wsagle = 0;
+    const int zero = 0;
 
-	do
-	{
-		if(wsagle == WSAEINPROGRESS)
-		  Sleep(1);  /* use windows sleep */
-		
-		WSASetLastError (0);
-		ret = ioctl (fd, FIONBIO, &zero);
-		wsagle = WSAGetLastError();
-	}
+    do
+    {
+        if(wsagle == WSAEINPROGRESS)
+          Sleep(1);  /* use windows sleep */
+
+        WSASetLastError (0);
+        ret = ioctl (fd, FIONBIO, &zero);
+        wsagle = WSAGetLastError ();
+    }
   while (ret && (wsagle == WSAEINPROGRESS));
 
-	if(ret)
+  if(ret)
     {
       fprintf (stderr,
                _("ioctl() failed.  The socket could not be set as blocking.\n") );
       DEBUGP (("Winsock error: %d\n", WSAGetLastError ()));
       abort ();
     }
-	return;
+  return;
 }

src/mswindows.h

@@ -57,13 +57,6 @@ as that of the covered work.  */
 /* Declares getpid(). */
 #include <process.h>
 
-#ifndef S_ISDIR
-# define S_ISDIR(m) (((m) & (_S_IFMT)) == (_S_IFDIR))
-#endif
-#ifndef S_ISLNK
-# define S_ISLNK(a) 0
-#endif
-
 /* We have strcasecmp and strncasecmp, just under different names.  */
 #ifndef HAVE_STRCASECMP
 # define strcasecmp stricmp
@@ -90,18 +83,6 @@ typedef __int64 wgint;
 # define strtoll_type __int64
 #endif
 
-/* Windows has no symlink, therefore no lstat.  Without symlinks lstat
-   is equivalent to stat anyway.  */
-#define lstat stat
-
-/* Define LFS aliases for stat and fstat. */
-#ifdef stat_alias
-# define stat(f, b) stat_alias (f, b)
-#endif
-#ifdef fstat_alias
-# define fstat(f, b) fstat_alias (f, b)
-#endif
-
 #define PATH_SEPARATOR '\\'
 
 /* Additional declarations needed for IPv6: */

src/netrc.c

@@ -44,10 +44,16 @@ as that of the covered work.  */
 
 #define NETRC_FILE_NAME ".netrc"
 
-acc_t *netrc_list;
+static acc_t *netrc_list;
 
 static acc_t *parse_netrc (const char *);
 
+void
+netrc_cleanup(void)
+{
+  free_netrc (netrc_list);
+}
+
 /* Return the correct user and password, given the host, user (as
    given in the URL), and password (as given in the URL).  May return
    NULL.
@@ -155,8 +161,6 @@ search_netrc (const char *host, const char **acc, const char **passwd,
 
 #ifdef STANDALONE
 
-#include <assert.h>
-
 /* Normally, these functions would be defined by your package.  */
 # define xmalloc malloc
 # define xfree free
@@ -442,18 +446,26 @@ main (int argc, char **argv)
   if (argc < 2 || argc > 3)
     {
       fprintf (stderr, _("Usage: %s NETRC [HOSTNAME]\n"), argv[0]);
-      exit (1);
+      exit (WGET_EXIT_GENERIC_ERROR);
     }
 
   program_name = argv[0];
   file = argv[1];
   target = argv[2];
 
+#ifdef ENABLE_NLS
+  /* Set the current locale.  */
+  setlocale (LC_ALL, "");
+  /* Set the text message domain.  */
+  bindtextdomain ("wget", LOCALEDIR);
+  textdomain ("wget");
+#endif /* ENABLE_NLS */
+
   if (stat (file, &sb))
     {
       fprintf (stderr, _("%s: cannot stat %s: %s\n"), argv[0], file,
                strerror (errno));
-      exit (1);
+      exit (WGET_EXIT_GENERIC_ERROR);
     }
 
   head = parse_netrc (file);
@@ -492,14 +504,14 @@ main (int argc, char **argv)
 
       /* Exit if we found the target.  */
       if (target)
-        exit (0);
+        exit (WGET_EXIT_SUCCESS);
       a = a->next;
     }
 
   /* Exit with failure if we had a target, success otherwise.  */
   if (target)
-    exit (1);
+    exit (WGET_EXIT_GENERIC_ERROR);
 
-  exit (0);
+  exit (WGET_EXIT_SUCCESS);
 }
 #endif /* STANDALONE */

src/netrc.h

@@ -33,14 +33,15 @@ as that of the covered work.  */
 
 typedef struct _acc_t
 {
-  char *host;			/* NULL if this is the default machine
-				   entry.  */
+  char *host;           /* NULL if this is the default machine
+                           entry.  */
   char *acc;
-  char *passwd;			/* NULL if there is no password.  */
+  char *passwd;         /* NULL if there is no password.  */
   struct _acc_t *next;
 } acc_t;
 
 void search_netrc (const char *, const char **, const char **, int);
 void free_netrc (acc_t *l);
+void netrc_cleanup(void);
 
 #endif /* NETRC_H */

src/openssl.c

@@ -40,6 +40,9 @@ as that of the covered work.  */
 #include <openssl/x509v3.h>
 #include <openssl/err.h>
 #include <openssl/rand.h>
+#if OPENSSL_VERSION_NUMBER >= 0x00907000
+#include <openssl/conf.h>
+#endif
 
 #include "utils.h"
 #include "connect.h"
@@ -153,6 +156,9 @@ key_type_to_ssl_type (enum keyfile_type type)
     }
 }
 
+/* SSL has been initialized */
+static int ssl_true_initialized = 0;
+
 /* Create an SSL Context and set default paths etc.  Called the first
    time an HTTP download is attempted.
 
@@ -161,6 +167,14 @@ key_type_to_ssl_type (enum keyfile_type type)
 bool
 ssl_init (void)
 {
+#if OPENSSL_VERSION_NUMBER >= 0x00907000
+  if (ssl_true_initialized == 0)
+    {
+      OPENSSL_config (NULL);
+      ssl_true_initialized = 1;
+    }
+#endif
+
   SSL_METHOD const *meth;
 
   if (ssl_ctx)
@@ -176,6 +190,12 @@ ssl_init (void)
       goto error;
     }
 
+#if OPENSSL_VERSION_NUMBER >= 0x00907000
+  OPENSSL_load_builtin_modules();
+  ENGINE_load_builtin_engines();
+  CONF_modules_load_file(NULL, NULL,
+      CONF_MFLAGS_DEFAULT_SECTION|CONF_MFLAGS_IGNORE_MISSING_FILE);
+#endif
   SSL_library_init ();
   SSL_load_error_strings ();
   SSLeay_add_all_algorithms ();
@@ -183,9 +203,6 @@ ssl_init (void)
 
   switch (opt.secure_protocol)
     {
-    case secure_protocol_auto:
-      meth = SSLv23_client_method ();
-      break;
 #ifndef OPENSSL_NO_SSL2
     case secure_protocol_sslv2:
       meth = SSLv2_client_method ();
@@ -194,10 +211,19 @@ ssl_init (void)
     case secure_protocol_sslv3:
       meth = SSLv3_client_method ();
       break;
+    case secure_protocol_auto:
     case secure_protocol_pfs:
     case secure_protocol_tlsv1:
       meth = TLSv1_client_method ();
       break;
+#if OPENSSL_VERSION_NUMBER >= 0x01001000
+    case secure_protocol_tlsv1_1:
+      meth = TLSv1_1_client_method ();
+      break;
+    case secure_protocol_tlsv1_2:
+      meth = TLSv1_2_client_method ();
+      break;
+#endif
     default:
       abort ();
     }
@@ -305,7 +331,7 @@ openssl_read (int fd, char *buf, int bufsize, void *arg)
 }
 
 static int
-openssl_write (int fd, char *buf, int bufsize, void *arg)
+openssl_write (int fd _GL_UNUSED, char *buf, int bufsize, void *arg)
 {
   int ret = 0;
   struct openssl_transport_context *ctx = arg;
@@ -347,7 +373,7 @@ openssl_peek (int fd, char *buf, int bufsize, void *arg)
 }
 
 static const char *
-openssl_errstr (int fd, void *arg)
+openssl_errstr (int fd _GL_UNUSED, void *arg)
 {
   struct openssl_transport_context *ctx = arg;
   unsigned long errcode;
@@ -459,10 +485,10 @@ ssl_connect_wget (int fd, const char *hostname)
   if (! is_valid_ip_address (hostname))
     {
       if (! SSL_set_tlsext_host_name (conn, hostname))
-	{
-	DEBUGP (("Failed to set TLS server-name indication."));
-	goto error;
-	}
+        {
+          DEBUGP (("Failed to set TLS server-name indication."));
+          goto error;
+        }
     }
 #endif
 
@@ -689,7 +715,7 @@ ssl_check_certificate (int fd, const char *host)
                       /* Compare and check for NULL attack in ASN1_STRING */
                       if (pattern_match ((char *)name_in_utf8, host) &&
                             (strlen ((char *)name_in_utf8) ==
-                                ASN1_STRING_length (name->d.dNSName)))
+                                (size_t) ASN1_STRING_length (name->d.dNSName)))
                         {
                           OPENSSL_free (name_in_utf8);
                           break;
@@ -712,7 +738,7 @@ ssl_check_certificate (int fd, const char *host)
           success = false;
         }
     }
-  
+
   if (alt_name_checked == false)
     {
       /* Test commomName */
@@ -753,7 +779,7 @@ ssl_check_certificate (int fd, const char *host)
 
           xentry = X509_NAME_get_entry(xname,i);
           sdata = X509_NAME_ENTRY_get_data(xentry);
-          if (strlen (common_name) != ASN1_STRING_length (sdata))
+          if (strlen (common_name) != (size_t) ASN1_STRING_length (sdata))
             {
               logprintf (LOG_NOTQUIET, _("\
     %s: certificate common name is invalid (contains a NUL character).\n\

src/options.h

@@ -31,169 +31,170 @@ as that of the covered work.  */
 
 struct options
 {
-  int verbose;			/* Are we verbose?  (First set to -1,
+  int verbose;                  /* Are we verbose?  (First set to -1,
                                    hence not boolean.) */
-  bool quiet;			/* Are we quiet? */
-  int ntry;			/* Number of tries per URL */
-  bool retry_connrefused;	/* Treat CONNREFUSED as non-fatal. */
-  bool background;		/* Whether we should work in background. */
-  bool ignore_length;		/* Do we heed content-length at all?  */
-  bool recursive;		/* Are we recursive? */
-  bool spanhost;		/* Do we span across hosts in
-				   recursion? */
+  bool quiet;                   /* Are we quiet? */
+  int ntry;                     /* Number of tries per URL */
+  bool retry_connrefused;       /* Treat CONNREFUSED as non-fatal. */
+  bool background;              /* Whether we should work in background. */
+  bool ignore_length;           /* Do we heed content-length at all?  */
+  bool recursive;               /* Are we recursive? */
+  bool spanhost;                /* Do we span across hosts in
+                                   recursion? */
   int  max_redirect;            /* Maximum number of times we'll allow
                                    a page to redirect. */
-  bool relative_only;		/* Follow only relative links. */
-  bool no_parent;		/* Restrict access to the parent
-				   directory.  */
-  int reclevel;			/* Maximum level of recursion */
-  bool dirstruct;		/* Do we build the directory structure
-				  as we go along? */
-  bool no_dirstruct;		/* Do we hate dirstruct? */
-  int cut_dirs;			/* Number of directory components to cut. */
-  bool add_hostdir;		/* Do we add hostname directory? */
-  bool protocol_directories;	/* Whether to prepend "http"/"ftp" to dirs. */
-  bool noclobber;		/* Disables clobbering of existing
-				   data. */
-  bool unlink;			/* remove file before clobbering */
-  char *dir_prefix;		/* The top of directory tree */
-  char *lfilename;		/* Log filename */
-  char *input_filename;		/* Input filename */
-  char *choose_config;		/* Specified config file */
-  bool force_html;		/* Is the input file an HTML file? */
+  bool relative_only;           /* Follow only relative links. */
+  bool no_parent;               /* Restrict access to the parent
+                                   directory.  */
+  int reclevel;                 /* Maximum level of recursion */
+  bool dirstruct;               /* Do we build the directory structure
+                                   as we go along? */
+  bool no_dirstruct;            /* Do we hate dirstruct? */
+  int cut_dirs;                 /* Number of directory components to cut. */
+  bool add_hostdir;             /* Do we add hostname directory? */
+  bool protocol_directories;    /* Whether to prepend "http"/"ftp" to dirs. */
+  bool noclobber;               /* Disables clobbering of existing data. */
+  bool unlink;                  /* remove file before clobbering */
+  char *dir_prefix;             /* The top of directory tree */
+  char *lfilename;              /* Log filename */
+  char *input_filename;         /* Input filename */
+  char *choose_config;          /* Specified config file */
+  bool noconfig;                /* Ignore all config files? */
+  bool force_html;              /* Is the input file an HTML file? */
 
   char *default_page;           /* Alternative default page (index file) */
 
-  bool spider;			/* Is Wget in spider mode? */
+  bool spider;                  /* Is Wget in spider mode? */
 
-  char **accepts;		/* List of patterns to accept. */
-  char **rejects;		/* List of patterns to reject. */
-  char **excludes;		/* List of excluded FTP directories. */
-  char **includes;		/* List of FTP directories to
-				   follow. */
-  bool ignore_case;		/* Whether to ignore case when
-				   matching dirs and files */
+  char **accepts;               /* List of patterns to accept. */
+  char **rejects;               /* List of patterns to reject. */
+  const char **excludes;        /* List of excluded FTP directories. */
+  const char **includes;        /* List of FTP directories to
+                                   follow. */
+  bool ignore_case;             /* Whether to ignore case when
+                                   matching dirs and files */
 
-  char *acceptregex_s;		/* Patterns to accept (a regex string). */
-  char *rejectregex_s;		/* Patterns to reject (a regex string). */
-  void *acceptregex;		/* Patterns to accept (a regex struct). */
-  void *rejectregex;		/* Patterns to reject (a regex struct). */
+  char *acceptregex_s;          /* Patterns to accept (a regex string). */
+  char *rejectregex_s;          /* Patterns to reject (a regex string). */
+  void *acceptregex;            /* Patterns to accept (a regex struct). */
+  void *rejectregex;            /* Patterns to reject (a regex struct). */
   enum {
 #ifdef HAVE_LIBPCRE
     regex_type_pcre,
 #endif
     regex_type_posix
-  } regex_type;   /* The regex library. */
-  void *(*regex_compile_fun)(const char *);  /* Function to compile a regex. */
+  } regex_type;                 /* The regex library. */
+  void *(*regex_compile_fun)(const char *);             /* Function to compile a regex. */
   bool (*regex_match_fun)(const void *, const char *);  /* Function to match a string to a regex. */
 
-  char **domains;		/* See host.c */
+  char **domains;               /* See host.c */
   char **exclude_domains;
-  bool dns_cache;		/* whether we cache DNS lookups. */
+  bool dns_cache;               /* whether we cache DNS lookups. */
 
   char **follow_tags;           /* List of HTML tags to recursively follow. */
   char **ignore_tags;           /* List of HTML tags to ignore if recursing. */
 
-  bool follow_ftp;		/* Are FTP URL-s followed in recursive
-				   retrieving? */
-  bool retr_symlinks;		/* Whether we retrieve symlinks in
-				   FTP. */
-  char *output_document;	/* The output file to which the
-				   documents will be printed.  */
-  char *warc_filename;		/* WARC output filename */
-  char *warc_tempdir;	/* WARC temp dir */
-  char *warc_cdx_dedup_filename;	/* CDX file to be used for deduplication. */
+  bool follow_ftp;              /* Are FTP URL-s followed in recursive
+                                   retrieving? */
+  bool retr_symlinks;           /* Whether we retrieve symlinks in
+                                   FTP. */
+  char *output_document;        /* The output file to which the
+                                   documents will be printed.  */
+  char *warc_filename;          /* WARC output filename */
+  char *warc_tempdir;           /* WARC temp dir */
+  char *warc_cdx_dedup_filename;/* CDX file to be used for deduplication. */
   wgint warc_maxsize;           /* WARC max archive size */
-  bool warc_compression_enabled;  /* For GZIP compression. */
-  bool warc_digests_enabled;  /* For SHA1 digests. */
-  bool warc_cdx_enabled;      /* Create CDX files? */
-  bool warc_keep_log;         /* Store the log file in a WARC record. */
-  char **warc_user_headers;		/* User-defined WARC header(s). */
+  bool warc_compression_enabled;/* For GZIP compression. */
+  bool warc_digests_enabled;    /* For SHA1 digests. */
+  bool warc_cdx_enabled;        /* Create CDX files? */
+  bool warc_keep_log;           /* Store the log file in a WARC record. */
+  char **warc_user_headers;     /* User-defined WARC header(s). */
 
-  char *user;			/* Generic username */
-  char *passwd;			/* Generic password */
+  char *user;                   /* Generic username */
+  char *passwd;                 /* Generic password */
   bool ask_passwd;              /* Ask for password? */
 
-  bool always_rest;		/* Always use REST. */
-  char *ftp_user;		/* FTP username */
-  char *ftp_passwd;		/* FTP password */
-  bool netrc;			/* Whether to read .netrc. */
-  bool ftp_glob;		/* FTP globbing */
-  bool ftp_pasv;			/* Passive FTP. */
+  bool always_rest;             /* Always use REST. */
+  wgint start_pos;              /* Start position of a download. */
+  char *ftp_user;               /* FTP username */
+  char *ftp_passwd;             /* FTP password */
+  bool netrc;                   /* Whether to read .netrc. */
+  bool ftp_glob;                /* FTP globbing */
+  bool ftp_pasv;                /* Passive FTP. */
 
-  char *http_user;		/* HTTP username. */
-  char *http_passwd;		/* HTTP password. */
-  char **user_headers;		/* User-defined header(s). */
-  bool http_keep_alive;		/* whether we use keep-alive */
+  char *http_user;              /* HTTP username. */
+  char *http_passwd;            /* HTTP password. */
+  char **user_headers;          /* User-defined header(s). */
+  bool http_keep_alive;         /* whether we use keep-alive */
 
-  bool use_proxy;		/* Do we use proxy? */
-  bool allow_cache;		/* Do we allow server-side caching? */
+  bool use_proxy;               /* Do we use proxy? */
+  bool allow_cache;             /* Do we allow server-side caching? */
   char *http_proxy, *ftp_proxy, *https_proxy;
   char **no_proxy;
   char *base_href;
-  char *progress_type;		/* progress indicator type. */
+  char *progress_type;          /* progress indicator type. */
+  bool show_progress;           /* Show only the progress bar */
+  bool noscroll;                /* Don't scroll the filename in the progressbar */
   char *proxy_user; /*oli*/
   char *proxy_passwd;
 
-  double read_timeout;		/* The read/write timeout. */
-  double dns_timeout;		/* The DNS timeout. */
-  double connect_timeout;	/* The connect timeout. */
+  double read_timeout;          /* The read/write timeout. */
+  double dns_timeout;           /* The DNS timeout. */
+  double connect_timeout;       /* The connect timeout. */
 
-  bool random_wait;		/* vary from 0 .. wait secs by random()? */
-  double wait;			/* The wait period between retrievals. */
-  double waitretry;		/* The wait period between retries. - HEH */
-  bool use_robots;		/* Do we heed robots.txt? */
+  bool random_wait;             /* vary from 0 .. wait secs by random()? */
+  double wait;                  /* The wait period between retrievals. */
+  double waitretry;             /* The wait period between retries. - HEH */
+  bool use_robots;              /* Do we heed robots.txt? */
 
-  wgint limit_rate;		/* Limit the download rate to this
-				   many bps. */
-  SUM_SIZE_INT quota;		/* Maximum file size to download and
-				   store. */
+  wgint limit_rate;             /* Limit the download rate to this
+                                   many bps. */
+  SUM_SIZE_INT quota;           /* Maximum file size to download and
+                                   store. */
 
-  bool server_response;		/* Do we print server response? */
-  bool save_headers;		/* Do we save headers together with
-				   file? */
-  bool content_on_error;	/* Do we output the content when the HTTP
-				   status code indicates a server error */
+  bool server_response;         /* Do we print server response? */
+  bool save_headers;            /* Do we save headers together with
+                                   file? */
+  bool content_on_error;        /* Do we output the content when the HTTP
+                                   status code indicates a server error */
 
-#ifdef ENABLE_DEBUG
-  bool debug;			/* Debugging on/off */
-#endif
+  bool debug;                   /* Debugging on/off */
 
 #ifdef USE_WATT32
   bool wdebug;                  /* Watt-32 tcp/ip debugging on/off */
 #endif
 
-  bool timestamping;		/* Whether to use time-stamping. */
+  bool timestamping;            /* Whether to use time-stamping. */
 
-  bool backup_converted;	/* Do we save pre-converted files as *.orig? */
-  int backups;			/* Are numeric backups made? */
+  bool backup_converted;        /* Do we save pre-converted files as *.orig? */
+  int backups;                  /* Are numeric backups made? */
 
-  char *useragent;		/* User-Agent string, which can be set
-				   to something other than Wget. */
-  char *referer;		/* Naughty Referer, which can be
-				   set to something other than
-				   NULL. */
-  bool convert_links;		/* Will the links be converted
-				   locally? */
-  bool remove_listing;		/* Do we remove .listing files
-				   generated by FTP? */
-  bool htmlify;			/* Do we HTML-ify the OS-dependent
-				   listings? */
+  char *useragent;              /* User-Agent string, which can be set
+                                   to something other than Wget. */
+  char *referer;                /* Naughty Referer, which can be
+                                   set to something other than
+                                   NULL. */
+  bool convert_links;           /* Will the links be converted
+                                   locally? */
+  bool remove_listing;          /* Do we remove .listing files
+                                   generated by FTP? */
+  bool htmlify;                 /* Do we HTML-ify the OS-dependent
+                                   listings? */
 
   char *dot_style;
-  wgint dot_bytes;		/* How many bytes in a printing
-				   dot. */
-  int dots_in_line;		/* How many dots in one line. */
-  int dot_spacing;		/* How many dots between spacings. */
+  wgint dot_bytes;              /* How many bytes in a printing
+                                   dot. */
+  int dots_in_line;             /* How many dots in one line. */
+  int dot_spacing;              /* How many dots between spacings. */
 
-  bool delete_after;		/* Whether the files will be deleted
-				   after download. */
+  bool delete_after;            /* Whether the files will be deleted
+                                   after download. */
 
-  bool adjust_extension;		/* Use ".html" extension on all text/html? */
+  bool adjust_extension;        /* Use ".html" extension on all text/html? */
 
-  bool page_requisites;		/* Whether we need to download all files
-				   necessary to display a page properly. */
-  char *bind_address;		/* What local IP address to bind to. */
+  bool page_requisites;         /* Whether we need to download all files
+                                   necessary to display a page properly. */
+  char *bind_address;           /* What local IP address to bind to. */
 
 #ifdef HAVE_SSL
   enum {
@@ -201,34 +202,36 @@ struct options
     secure_protocol_sslv2,
     secure_protocol_sslv3,
     secure_protocol_tlsv1,
+    secure_protocol_tlsv1_1,
+    secure_protocol_tlsv1_2,
     secure_protocol_pfs
-  } secure_protocol;		/* type of secure protocol to use. */
-  bool check_cert;		/* whether to validate the server's cert */
-  char *cert_file;		/* external client certificate to use. */
-  char *private_key;		/* private key file (if not internal). */
+  } secure_protocol;            /* type of secure protocol to use. */
+  bool check_cert;              /* whether to validate the server's cert */
+  char *cert_file;              /* external client certificate to use. */
+  char *private_key;            /* private key file (if not internal). */
   enum keyfile_type {
     keyfile_pem,
     keyfile_asn1
-  } cert_type;			/* type of client certificate file */
+  } cert_type;                  /* type of client certificate file */
   enum keyfile_type
-    private_key_type;		/* type of private key file */
+    private_key_type;           /* type of private key file */
 
-  char *ca_directory;		/* CA directory (hash files) */
-  char *ca_cert;		/* CA certificate file to use */
+  char *ca_directory;           /* CA directory (hash files) */
+  char *ca_cert;                /* CA certificate file to use */
 
-  char *random_file;		/* file with random data to seed the PRNG */
-  char *egd_file;		/* file name of the egd daemon socket */
-  bool https_only;		/* whether to follow HTTPS only */
+  char *random_file;            /* file with random data to seed the PRNG */
+  char *egd_file;               /* file name of the egd daemon socket */
+  bool https_only;              /* whether to follow HTTPS only */
 #endif /* HAVE_SSL */
 
-  bool cookies;			/* whether cookies are used. */
-  char *cookies_input;		/* file we're loading the cookies from. */
-  char *cookies_output;		/* file we're saving the cookies to. */
-  bool keep_session_cookies;	/* whether session cookies should be
-				   saved and loaded. */
+  bool cookies;                 /* whether cookies are used. */
+  char *cookies_input;          /* file we're loading the cookies from. */
+  char *cookies_output;         /* file we're saving the cookies to. */
+  bool keep_session_cookies;    /* whether session cookies should be
+                                   saved and loaded. */
 
-  char *post_data;		/* POST query string */
-  char *post_file_name;		/* File to post */
+  char *post_data;              /* POST query string */
+  char *post_file_name;         /* File to post */
   char *method;                 /* HTTP Method to use in Header */
   char *body_data;              /* HTTP Method Data String */
   char *body_file;              /* HTTP Method File */
@@ -236,36 +239,36 @@ struct options
   enum {
     restrict_unix,
     restrict_windows
-  } restrict_files_os;		/* file name restriction ruleset. */
-  bool restrict_files_ctrl;	/* non-zero if control chars in URLs
-				   are restricted from appearing in
-				   generated file names. */
+  } restrict_files_os;          /* file name restriction ruleset. */
+  bool restrict_files_ctrl;     /* non-zero if control chars in URLs
+                                   are restricted from appearing in
+                                   generated file names. */
   bool restrict_files_nonascii; /* non-zero if bytes with values greater
                                    than 127 are restricted. */
   enum {
     restrict_no_case_restriction,
     restrict_lowercase,
     restrict_uppercase
-  } restrict_files_case;	/* file name case restriction. */
+  } restrict_files_case;        /* file name case restriction. */
 
-  bool strict_comments;		/* whether strict SGML comments are
-				   enforced.  */
+  bool strict_comments;         /* whether strict SGML comments are
+                                   enforced.  */
 
   bool preserve_perm;           /* whether remote permissions are used
-				  or that what is set by umask. */
+                                   or that what is set by umask. */
 
 #ifdef ENABLE_IPV6
-  bool ipv4_only;		/* IPv4 connections have been requested. */
-  bool ipv6_only;		/* IPv4 connections have been requested. */
+  bool ipv4_only;               /* IPv4 connections have been requested. */
+  bool ipv6_only;               /* IPv4 connections have been requested. */
 #endif
   enum {
     prefer_ipv4,
     prefer_ipv6,
     prefer_none
-  } prefer_family;		/* preferred address family when more
-				   than one type is available */
+  } prefer_family;              /* preferred address family when more
+                                   than one type is available */
 
-  bool content_disposition;	/* Honor HTTP Content-Disposition header. */
+  bool content_disposition;     /* Honor HTTP Content-Disposition header. */
   bool auth_without_challenge;  /* Issue Basic authentication creds without
                                    waiting for a challenge. */
 
@@ -278,11 +281,11 @@ struct options
   int ftp_stmlf;                /* Force Stream_LF format for binary FTP. */
 #endif /* def __VMS */
 
-  bool useservertimestamps;  	/* Update downloaded files' timestamps to
-				   match those on server? */
+  bool useservertimestamps;     /* Update downloaded files' timestamps to
+                                   match those on server? */
 
-  bool show_all_dns_entries; /* Show all the DNS entries when resolving a
-                                name. */
+  bool show_all_dns_entries;    /* Show all the DNS entries when resolving a
+                                   name. */
   bool report_bps;              /*Output bandwidth in bits format*/
 };
 

src/progress.c

@@ -36,9 +36,8 @@ as that of the covered work.  */
 #include <assert.h>
 #include <unistd.h>
 #include <signal.h>
-#ifdef HAVE_WCHAR_H
-# include <wchar.h>
-#endif
+#include <wchar.h>
+#include <mbiter.h>
 
 #include "progress.h"
 #include "utils.h"
@@ -47,27 +46,30 @@ as that of the covered work.  */
 struct progress_implementation {
   const char *name;
   bool interactive;
-  void *(*create) (wgint, wgint);
+  void *(*create) (const char *, wgint, wgint);
   void (*update) (void *, wgint, double);
+  void (*draw) (void *);
   void (*finish) (void *, double);
-  void (*set_params) (const char *);
+  void (*set_params) (char *);
 };
 
 /* Necessary forward declarations. */
 
-static void *dot_create (wgint, wgint);
+static void *dot_create (const char *, wgint, wgint);
 static void dot_update (void *, wgint, double);
 static void dot_finish (void *, double);
-static void dot_set_params (const char *);
+static void dot_draw (void *);
+static void dot_set_params (char *);
 
-static void *bar_create (wgint, wgint);
+static void *bar_create (const char *, wgint, wgint);
 static void bar_update (void *, wgint, double);
+static void bar_draw (void *);
 static void bar_finish (void *, double);
-static void bar_set_params (const char *);
+static void bar_set_params (char *);
 
 static struct progress_implementation implementations[] = {
-  { "dot", 0, dot_create, dot_update, dot_finish, dot_set_params },
-  { "bar", 1, bar_create, bar_update, bar_finish, bar_set_params }
+  { "dot", 0, dot_create, dot_update, dot_draw, dot_finish, dot_set_params },
+  { "bar", 1, bar_create, bar_update, bar_draw, bar_finish, bar_set_params }
 };
 static struct progress_implementation *current_impl;
 static int current_impl_locked;
@@ -109,7 +111,7 @@ set_progress_implementation (const char *name)
 {
   size_t i, namelen;
   struct progress_implementation *pi = implementations;
-  const char *colon;
+  char *colon;
 
   if (!name)
     name = DEFAULT_PROGRESS_IMPLEMENTATION;
@@ -151,7 +153,7 @@ progress_schedule_redirect (void)
    advance.  */
 
 void *
-progress_create (wgint initial, wgint total)
+progress_create (const char *f_download, wgint initial, wgint total)
 {
   /* Check if the log status has changed under our feet. */
   if (output_redirected)
@@ -161,7 +163,7 @@ progress_create (wgint initial, wgint total)
       output_redirected = 0;
     }
 
-  return current_impl->create (initial, total);
+  return current_impl->create (f_download, initial, total);
 }
 
 /* Return true if the progress gauge is "interactive", i.e. if it can
@@ -170,7 +172,7 @@ progress_create (wgint initial, wgint total)
    and current update.  */
 
 bool
-progress_interactive_p (void *progress)
+progress_interactive_p (void *progress _GL_UNUSED)
 {
   return current_impl->interactive;
 }
@@ -182,6 +184,7 @@ void
 progress_update (void *progress, wgint howmuch, double dltime)
 {
   current_impl->update (progress, howmuch, dltime);
+  current_impl->draw (progress);
 }
 
 /* Tell the progress gauge to clean up.  Calling this will free the
@@ -204,6 +207,7 @@ struct dot_progress {
   int accumulated;              /* number of bytes accumulated after
                                    the last printed dot */
 
+  double dltime;                /* download time so far */
   int rows;                     /* number of rows printed so far */
   int dots;                     /* number of dots printed in this row */
 
@@ -213,7 +217,7 @@ struct dot_progress {
 /* Dot-progress backend for progress_create. */
 
 static void *
-dot_create (wgint initial, wgint total)
+dot_create (const char *f_download _GL_UNUSED, wgint initial, wgint total)
 {
   struct dot_progress *dp = xnew0 (struct dot_progress);
   dp->initial_length = initial;
@@ -237,18 +241,18 @@ dot_create (wgint initial, wgint total)
           /* Align the [ skipping ... ] line with the dots.  To do
              that, insert the number of spaces equal to the number of
              digits in the skipped amount in K.  */
-          logprintf (LOG_VERBOSE, _("\n%*s[ skipping %sK ]"),
+          logprintf (LOG_PROGRESS, _("\n%*s[ skipping %sK ]"),
                      2 + skipped_k_len, "",
                      number_to_static_string (skipped_k));
         }
 
-      logprintf (LOG_VERBOSE, "\n%6sK",
+      logprintf (LOG_PROGRESS, "\n%6sK",
                  number_to_static_string (skipped / 1024));
       for (; remainder >= dot_bytes; remainder -= dot_bytes)
         {
           if (dp->dots % opt.dot_spacing == 0)
-            logputs (LOG_VERBOSE, " ");
-          logputs (LOG_VERBOSE, ",");
+            logputs (LOG_PROGRESS, " ");
+          logputs (LOG_PROGRESS, ",");
           ++dp->dots;
         }
       assert (dp->dots < opt.dots_in_line);
@@ -289,7 +293,7 @@ print_row_stats (struct dot_progress *dp, double dltime, bool last)
          been retrieved.  12.8% will round to 12% because the 13% mark
          has not yet been reached.  100% is only shown when done.  */
       int percentage = 100.0 * bytes_displayed / dp->total_length;
-      logprintf (LOG_VERBOSE, "%3d%%", percentage);
+      logprintf (LOG_PROGRESS, "%3d%%", percentage);
     }
 
   {
@@ -306,7 +310,7 @@ print_row_stats (struct dot_progress *dp, double dltime, bool last)
     if (dp->rows == dp->initial_length / ROW_BYTES)
       bytes_this_row -= dp->initial_length % ROW_BYTES;
     rate = calc_rate (bytes_this_row, dltime - dp->last_timer_value, &units);
-    logprintf (LOG_VERBOSE, " %4.*f%c",
+    logprintf (LOG_PROGRESS, " %4.*f%c",
                rate >= 99.95 ? 0 : rate >= 9.995 ? 1 : 2,
                rate, names[units]);
     dp->last_timer_value = dltime;
@@ -323,7 +327,7 @@ print_row_stats (struct dot_progress *dp, double dltime, bool last)
           wgint bytes_sofar = bytes_displayed - dp->initial_length;
           double eta = dltime * bytes_remaining / bytes_sofar;
           if (eta < INT_MAX - 1)
-            logprintf (LOG_VERBOSE, " %s",
+            logprintf (LOG_PROGRESS, " %s",
                        eta_to_human_short ((int) (eta + 0.5), true));
         }
     }
@@ -331,10 +335,10 @@ print_row_stats (struct dot_progress *dp, double dltime, bool last)
     {
       /* When done, print the total download time */
       if (dltime >= 10)
-        logprintf (LOG_VERBOSE, "=%s",
+        logprintf (LOG_PROGRESS, "=%s",
                    eta_to_human_short ((int) (dltime + 0.5), true));
       else
-        logprintf (LOG_VERBOSE, "=%ss", print_decimal (dltime));
+        logprintf (LOG_PROGRESS, "=%ss", print_decimal (dltime));
     }
 }
 
@@ -342,6 +346,14 @@ print_row_stats (struct dot_progress *dp, double dltime, bool last)
 
 static void
 dot_update (void *progress, wgint howmuch, double dltime)
+{
+  struct dot_progress *dp = progress;
+  dp->accumulated += howmuch;
+  dp->dltime = dltime;
+}
+
+static void
+dot_draw (void *progress)
 {
   struct dot_progress *dp = progress;
   int dot_bytes = opt.dot_bytes;
@@ -349,16 +361,15 @@ dot_update (void *progress, wgint howmuch, double dltime)
 
   log_set_flush (false);
 
-  dp->accumulated += howmuch;
   for (; dp->accumulated >= dot_bytes; dp->accumulated -= dot_bytes)
     {
       if (dp->dots == 0)
-        logprintf (LOG_VERBOSE, "\n%6sK",
+        logprintf (LOG_PROGRESS, "\n%6sK",
                    number_to_static_string (dp->rows * ROW_BYTES / 1024));
 
       if (dp->dots % opt.dot_spacing == 0)
-        logputs (LOG_VERBOSE, " ");
-      logputs (LOG_VERBOSE, ".");
+        logputs (LOG_PROGRESS, " ");
+      logputs (LOG_PROGRESS, ".");
 
       ++dp->dots;
       if (dp->dots >= opt.dots_in_line)
@@ -366,7 +377,7 @@ dot_update (void *progress, wgint howmuch, double dltime)
           ++dp->rows;
           dp->dots = 0;
 
-          print_row_stats (dp, dltime, false);
+          print_row_stats (dp, dp->dltime, false);
         }
     }
 
@@ -385,13 +396,13 @@ dot_finish (void *progress, double dltime)
   log_set_flush (false);
 
   if (dp->dots == 0)
-    logprintf (LOG_VERBOSE, "\n%6sK",
+    logprintf (LOG_PROGRESS, "\n%6sK",
                number_to_static_string (dp->rows * ROW_BYTES / 1024));
   for (i = dp->dots; i < opt.dots_in_line; i++)
     {
       if (i % opt.dot_spacing == 0)
-        logputs (LOG_VERBOSE, " ");
-      logputs (LOG_VERBOSE, " ");
+        logputs (LOG_PROGRESS, " ");
+      logputs (LOG_PROGRESS, " ");
     }
 
   print_row_stats (dp, dltime, true);
@@ -407,7 +418,7 @@ dot_finish (void *progress, double dltime)
    giga.  */
 
 static void
-dot_set_params (const char *params)
+dot_set_params (char *params)
 {
   if (!params || !*params)
     params = opt.dot_style;
@@ -495,6 +506,7 @@ static volatile sig_atomic_t received_sigwinch;
 #define ETA_REFRESH_INTERVAL 0.99
 
 struct bar_progress {
+  const char *f_download;       /* Filename of the downloaded file */
   wgint initial_length;         /* how many bytes have been downloaded
                                    previously. */
   wgint total_length;           /* expected total byte count when the
@@ -505,6 +517,7 @@ struct bar_progress {
                                    measured since the beginning of
                                    download. */
 
+  double dltime;                /* download time so far */
   int width;                    /* screen width we're using at the
                                    time the progress gauge was
                                    created.  this is different from
@@ -551,7 +564,7 @@ static void create_image (struct bar_progress *, double, bool);
 static void display_image (char *);
 
 static void *
-bar_create (wgint initial, wgint total)
+bar_create (const char *f_download, wgint initial, wgint total)
 {
   struct bar_progress *bp = xnew0 (struct bar_progress);
 
@@ -562,6 +575,7 @@ bar_create (wgint initial, wgint total)
 
   bp->initial_length = initial;
   bp->total_length   = total;
+  bp->f_download     = f_download;
 
   /* Initialize screen_width if this hasn't been done or if it might
      have changed, as indicated by receiving SIGWINCH.  */
@@ -595,8 +609,8 @@ static void
 bar_update (void *progress, wgint howmuch, double dltime)
 {
   struct bar_progress *bp = progress;
-  bool force_screen_update = false;
 
+  bp->dltime = dltime;
   bp->count += howmuch;
   if (bp->total_length > 0
       && bp->count + bp->initial_length > bp->total_length)
@@ -608,6 +622,13 @@ bar_update (void *progress, wgint howmuch, double dltime)
     bp->total_length = bp->initial_length + bp->count;
 
   update_speed_ring (bp, howmuch, dltime);
+}
+
+static void
+bar_draw (void *progress)
+{
+  bool force_screen_update = false;
+  struct bar_progress *bp = progress;
 
   /* If SIGWINCH (the window size change signal) been received,
      determine the new screen size and update the screen.  */
@@ -628,13 +649,13 @@ bar_update (void *progress, wgint howmuch, double dltime)
       received_sigwinch = 0;
     }
 
-  if (dltime - bp->last_screen_update < REFRESH_INTERVAL && !force_screen_update)
+  if (bp->dltime - bp->last_screen_update < REFRESH_INTERVAL && !force_screen_update)
     /* Don't update more often than five times per second. */
     return;
 
-  create_image (bp, dltime, false);
+  create_image (bp, bp->dltime, false);
   display_image (bp->buffer);
-  bp->last_screen_update = dltime;
+  bp->last_screen_update = bp->dltime;
 }
 
 static void
@@ -650,7 +671,8 @@ bar_finish (void *progress, double dltime)
   create_image (bp, dltime, true);
   display_image (bp->buffer);
 
-  logputs (LOG_VERBOSE, "\n\n");
+  logputs (LOG_VERBOSE, "\n");
+  logputs (LOG_PROGRESS, "\n");
 
   xfree (bp->buffer);
   xfree (bp);
@@ -791,8 +813,37 @@ count_cols (const char *mbs)
     }
   return cols;
 }
+
+static int
+cols_to_bytes (const char *mbs, const int cols, int *ncols)
+{
+  int p_cols = 0, bytes = 0;
+  mbchar_t mbc;
+  mbi_iterator_t iter;
+  mbi_init (iter, mbs, strlen(mbs));
+  while (p_cols < cols && mbi_avail (iter))
+    {
+      mbc = mbi_cur (iter);
+      p_cols += mb_width (mbc);
+      /* The multibyte character has exceeded the total number of columns we
+       * have available. The remaining bytes will be padded with a space. */
+      if (p_cols > cols)
+        {
+          p_cols -= mb_width (mbc);
+          break;
+        }
+      bytes += mb_len (mbc);
+      mbi_advance (iter);
+    }
+  *ncols = p_cols;
+  return bytes;
+}
 #else
 # define count_cols(mbs) ((int)(strlen(mbs)))
+# define cols_to_bytes(mbs, cols, *ncols) do {  \
+    *ncols = cols;                              \
+    bytes = cols;                               \
+}while (0)
 #endif
 
 static const char *
@@ -800,7 +851,7 @@ get_eta (int *bcd)
 {
   /* TRANSLATORS: "ETA" is English-centric, but this must
      be short, ideally 3 chars.  Abbreviate if necessary.  */
-  static const char eta_str[] = N_("  eta %s");
+  static const char eta_str[] = N_("   eta %s");
   static const char *eta_trans;
   static int bytes_cols_diff;
   if (eta_trans == NULL)
@@ -845,10 +896,14 @@ get_eta (int *bcd)
 #ifndef MAX
 # define MAX(a, b) ((a) >= (b) ? (a) : (b))
 #endif
+#ifndef MIN
+# define MIN(a, b) ((a) <= (b) ? (a) : (b))
+#endif
 
 static void
 create_image (struct bar_progress *bp, double dl_total_time, bool done)
 {
+  const int MAX_FILENAME_COLS = bp->width / 4;
   char *p = bp->buffer;
   wgint size = bp->initial_length + bp->count;
 
@@ -859,9 +914,10 @@ create_image (struct bar_progress *bp, double dl_total_time, bool done)
   int size_grouped_pad; /* Used to pad the field width for size_grouped. */
 
   struct bar_progress_hist *hist = &bp->hist;
+  int orig_filename_cols = count_cols (bp->f_download);
 
   /* The progress bar should look like this:
-     xx% [=======>             ] nn,nnn 12.34KB/s  eta 36m 51s
+     file xx% [=======>             ] nnn.nnK 12.34KB/s  eta 36m 51s
 
      Calculate the geometry.  The idea is to assign as much room as
      possible to the progress bar.  The other idea is to never let
@@ -870,16 +926,26 @@ create_image (struct bar_progress *bp, double dl_total_time, bool done)
      It would be especially bad for the progress bar to be resized
      randomly.
 
+     "file "           - Downloaded filename      - MAX_FILENAME_COLS chars + 1
      "xx% " or "100%"  - percentage               - 4 chars
      "[]"              - progress bar decorations - 2 chars
-     " nnn,nnn,nnn"    - downloaded bytes         - 12 chars or very rarely more
-     " 12.5KB/s"        - download rate           - 9 chars
+     " nnn.nnK"        - downloaded bytes         - 7 chars + 1
+     " 12.5KB/s"       - download rate            - 8 chars + 1
      "  eta 36m 51s"   - ETA                      - 14 chars
 
      "=====>..."       - progress bar             - the rest
   */
-  int dlbytes_size = 1 + MAX (size_grouped_len, 11);
-  int progress_size = bp->width - (4 + 2 + dlbytes_size + 8 + 14);
+
+#define PROGRESS_FILENAME_LEN  MAX_FILENAME_COLS + 1
+#define PROGRESS_PERCENT_LEN   4
+#define PROGRESS_DECORAT_LEN   2
+#define PROGRESS_FILESIZE_LEN  7 + 1
+#define PROGRESS_DWNLOAD_RATE  8 + 1
+#define PROGRESS_ETA_LEN       14
+
+  int progress_size = bp->width - (PROGRESS_FILENAME_LEN + PROGRESS_PERCENT_LEN +
+                                   PROGRESS_DECORAT_LEN + PROGRESS_FILESIZE_LEN +
+                                   PROGRESS_DWNLOAD_RATE + PROGRESS_ETA_LEN);
 
   /* The difference between the number of bytes used,
      and the number of columns used. */
@@ -888,6 +954,34 @@ create_image (struct bar_progress *bp, double dl_total_time, bool done)
   if (progress_size < 5)
     progress_size = 0;
 
+  if (orig_filename_cols <= MAX_FILENAME_COLS)
+    {
+      int padding = MAX_FILENAME_COLS - orig_filename_cols;
+      sprintf (p, "%s ", bp->f_download);
+      p += orig_filename_cols + 1;
+      for (;padding;padding--)
+        *p++ = ' ';
+    }
+  else
+    {
+      int offset_cols;
+      int bytes_in_filename, offset_bytes, col;
+      int *cols_ret = &col;
+
+      if (((orig_filename_cols > MAX_FILENAME_COLS) && !opt.noscroll) && !done)
+        offset_cols = ((int) bp->tick) % (orig_filename_cols - MAX_FILENAME_COLS);
+      else
+        offset_cols = 0;
+      offset_bytes = cols_to_bytes (bp->f_download, offset_cols, cols_ret);
+      bytes_in_filename = cols_to_bytes (bp->f_download + offset_bytes, MAX_FILENAME_COLS, cols_ret);
+      memcpy (p, bp->f_download + offset_bytes, bytes_in_filename);
+      p += bytes_in_filename;
+      int padding = MAX_FILENAME_COLS - *cols_ret;
+      for (;padding;padding--)
+          *p++ = ' ';
+      *p++ = ' ';
+    }
+
   /* "xx% " */
   if (bp->total_length > 0)
     {
@@ -895,7 +989,7 @@ create_image (struct bar_progress *bp, double dl_total_time, bool done)
       assert (percentage <= 100);
 
       if (percentage < 100)
-        sprintf (p, "%2d%% ", percentage);
+        sprintf (p, "%3d%%", percentage);
       else
         strcpy (p, "100%");
       p += 4;
@@ -961,16 +1055,23 @@ create_image (struct bar_progress *bp, double dl_total_time, bool done)
         }
       *p++ = ']';
 
-      ++bp->tick;
     }
+ ++bp->tick;
 
-  /* " 234,567,890" */
-  sprintf (p, " %s", size_grouped);
+  /* " 234.56M" */
+  const char * down_size = human_readable (size, 1000, 2);
+  int cols_diff = 7 - count_cols (down_size);
+  while (cols_diff > 0)
+  {
+    *p++=' ';
+    cols_diff--;
+  }
+  sprintf (p, " %s", down_size);
   move_to_end (p);
-  /* Pad with spaces to 11 chars for the size_grouped field;
+  /* Pad with spaces to 7 chars for the size_grouped field;
    * couldn't use the field width specifier in sprintf, because
    * it counts in bytes, not characters. */
-  for (size_grouped_pad = 11 - size_grouped_len;
+  for (size_grouped_pad = PROGRESS_FILESIZE_LEN - 7;
        size_grouped_pad > 0;
        --size_grouped_pad)
     {
@@ -980,8 +1081,8 @@ create_image (struct bar_progress *bp, double dl_total_time, bool done)
   /* " 12.52Kb/s or 12.52KB/s" */
   if (hist->total_time > 0 && hist->total_bytes)
     {
-      static const char *short_units[] = { "B/s", "KB/s", "MB/s", "GB/s" };
-      static const char *short_units_bits[] = { "b/s", "Kb/s", "Mb/s", "Gb/s" };
+      static const char *short_units[] = { " B/s", "KB/s", "MB/s", "GB/s" };
+      static const char *short_units_bits[] = { " b/s", "Kb/s", "Mb/s", "Gb/s" };
       int units = 0;
       /* Calculate the download speed using the history ring and
          recent data that hasn't made it to the ring yet.  */
@@ -993,7 +1094,7 @@ create_image (struct bar_progress *bp, double dl_total_time, bool done)
       move_to_end (p);
     }
   else
-    APPEND_LITERAL (" --.-K/s");
+    APPEND_LITERAL (" --.-KB/s");
 
   if (!done)
     {
@@ -1070,19 +1171,27 @@ static void
 display_image (char *buf)
 {
   bool old = log_set_save_context (false);
-  logputs (LOG_VERBOSE, "\r");
-  logputs (LOG_VERBOSE, buf);
+  logputs (LOG_PROGRESS, "\r");
+  logputs (LOG_PROGRESS, buf);
   log_set_save_context (old);
 }
 
 static void
-bar_set_params (const char *params)
+bar_set_params (char *params)
 {
   char *term = getenv ("TERM");
 
-  if (params
-      && 0 == strcmp (params, "force"))
-    current_impl_locked = 1;
+  if (params)
+    {
+      char *param = strtok (params, ":");
+      do
+        {
+          if (0 == strcmp (param, "force"))
+            current_impl_locked = 1;
+          else if (0 == strcmp (param, "noscroll"))
+            opt.noscroll = true;
+        } while ((param = strtok (NULL, ":")) != NULL);
+    }
 
   if ((opt.lfilename
 #ifdef HAVE_ISATTY
@@ -1112,7 +1221,7 @@ bar_set_params (const char *params)
 
 #ifdef SIGWINCH
 void
-progress_handle_sigwinch (int sig)
+progress_handle_sigwinch (int sig _GL_UNUSED)
 {
   received_sigwinch = 1;
   signal (SIGWINCH, progress_handle_sigwinch);

src/progress.h

@@ -35,7 +35,7 @@ bool valid_progress_implementation_p (const char *);
 void set_progress_implementation (const char *);
 void progress_schedule_redirect (void);
 
-void *progress_create (wgint, wgint);
+void *progress_create (const char *, wgint, wgint);
 bool progress_interactive_p (void *);
 void progress_update (void *, wgint, double);
 void progress_finish (void *, double);

src/ptimer.h

@@ -31,7 +31,7 @@ as that of the covered work.  */
 #ifndef PTIMER_H
 #define PTIMER_H
 
-struct ptimer;			/* forward declaration; all struct
+struct ptimer;                  /* forward declaration; all struct
                                    members are private */
 
 struct ptimer *ptimer_new (void);

src/recur.c

@@ -253,23 +253,23 @@ retrieve_tree (struct url *start_url_parsed, struct iri *pi)
          the second time.  */
       if (dl_url_file_map && hash_table_contains (dl_url_file_map, url))
         {
-	  bool is_css_bool;
+          bool is_css_bool;
 
           file = xstrdup (hash_table_get (dl_url_file_map, url));
 
           DEBUGP (("Already downloaded \"%s\", reusing it from \"%s\".\n",
                    url, file));
 
-	  if ((is_css_bool = (css_allowed
-			      && downloaded_css_set
-			      && string_set_contains (downloaded_css_set, file)))
-	      || (html_allowed
-		  && downloaded_html_set
-		  && string_set_contains (downloaded_html_set, file)))
-	    {
-	      descend = true;
-	      is_css = is_css_bool;
-	    }
+          if ((is_css_bool = (css_allowed
+                  && downloaded_css_set
+                  && string_set_contains (downloaded_css_set, file)))
+              || (html_allowed
+                && downloaded_html_set
+                && string_set_contains (downloaded_html_set, file)))
+            {
+              descend = true;
+              is_css = is_css_bool;
+            }
         }
       else
         {

src/res.c

@@ -615,11 +615,11 @@ res_cleanup (void)
 #ifdef TESTING
 
 const char *
-test_is_robots_txt_url()
+test_is_robots_txt_url(void)
 {
-  int i;
-  struct {
-    char *url;
+  unsigned i;
+  static const struct {
+    const char *url;
     bool expected_result;
   } test_array[] = {
     { "http://www.yoyodyne.com/robots.txt", true },
@@ -627,7 +627,7 @@ test_is_robots_txt_url()
     { "http://www.yoyodyne.com/somepath/robots.txt", false },
   };
 
-  for (i = 0; i < sizeof(test_array)/sizeof(test_array[0]); ++i)
+  for (i = 0; i < countof(test_array); ++i)
     {
       mu_assert ("test_is_robots_txt_url: wrong result",
                  is_robots_txt_url (test_array[i].url) == test_array[i].expected_result);
@@ -641,4 +641,3 @@ test_is_robots_txt_url()
 /*
  * vim: et ts=2 sw=2
  */
-

src/retr.c

@@ -1,6 +1,6 @@
 /* File retrieval.
    Copyright (C) 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004,
-   2005, 2006, 2007, 2008, 2009, 2010, 2011 Free Software Foundation,
+   2005, 2006, 2007, 2008, 2009, 2010, 2011, 2014 Free Software Foundation,
    Inc.
 
 This file is part of GNU Wget.
@@ -226,7 +226,8 @@ write_data (FILE *out, FILE *out2, const char *buf, int bufsize,
    data to OUT2, -3 is returned.  */
 
 int
-fd_read_body (int fd, FILE *out, wgint toread, wgint startpos,
+fd_read_body (const char *downloaded_filename, int fd, FILE *out, wgint toread, wgint startpos,
+
               wgint *qtyread, wgint *qtywritten, double *elapsed, int flags,
               FILE *out2)
 {
@@ -262,13 +263,13 @@ fd_read_body (int fd, FILE *out, wgint toread, wgint startpos,
   if (flags & rb_skip_startpos)
     skip = startpos;
 
-  if (opt.verbose)
+  if (opt.show_progress)
     {
       /* If we're skipping STARTPOS bytes, pass 0 as the INITIAL
          argument to progress_create because the indicator doesn't
          (yet) know about "skipping" data.  */
       wgint start = skip ? 0 : startpos;
-      progress = progress_create (start, start + toread);
+      progress = progress_create (downloaded_filename, start, start + toread);
       progress_interactive = progress_interactive_p (progress);
     }
 
@@ -411,7 +412,7 @@ fd_read_body (int fd, FILE *out, wgint toread, wgint startpos,
       if (progress)
         progress_update (progress, ret, ptimer_read (timer));
 #ifdef WINDOWS
-      if (toread > 0 && !opt.quiet)
+      if (toread > 0 && opt.show_progress)
         ws_percenttitle (100.0 *
                          (startpos + sum_read) / (startpos + toread));
 #endif
@@ -587,7 +588,7 @@ fd_read_hunk (int fd, hunk_terminator_t terminator, long sizehint, long maxsize)
 }
 
 static const char *
-line_terminator (const char *start, const char *peeked, int peeklen)
+line_terminator (const char *start _GL_UNUSED, const char *peeked, int peeklen)
 {
   const char *p = memchr (peeked, '\n', peeklen);
   if (p)
@@ -649,7 +650,7 @@ calc_rate (wgint bytes, double secs, int *units)
 {
   double dlrate;
   double bibyte = 1000.0;
- 
+
   if (!opt.report_bps)
     bibyte = 1024.0;
 
@@ -781,6 +782,7 @@ retrieve_url (struct url * orig_parsed, const char *origurl, char **file,
           result = PROXERR;
           goto bail;
         }
+      free (proxy);
     }
 
   if (u->scheme == SCHEME_HTTP
@@ -902,15 +904,19 @@ retrieve_url (struct url * orig_parsed, const char *origurl, char **file,
          index page; that redirection is clearly a GET.  We "suspend"
          POST data for the duration of the redirections, and restore
          it when we're done.
-	 
-	 RFC2616 HTTP/1.1 introduces code 307 Temporary Redirect
-	 specifically to preserve the method of the request.
-	 */
+
+         RFC2616 HTTP/1.1 introduces code 307 Temporary Redirect
+         specifically to preserve the method of the request.
+     */
       if (result != NEWLOCATION_KEEP_POST && !method_suspended)
         SUSPEND_METHOD;
 
       goto redirected;
     }
+  else
+    {
+      xfree(mynewloc);
+    }
 
   /* Try to not encode in UTF-8 if fetching failed */
   if (!(*dt & RETROKF) && iri->utf8_encode)
@@ -1004,7 +1010,6 @@ retrieve_from_file (const char *file, bool html, int *count)
   if (url_valid_scheme (url))
     {
       int dt,url_err;
-      uerr_t status;
       struct url *url_parsed = url_parse (url, &url_err, iri, true);
       if (!url_parsed)
         {
@@ -1030,7 +1035,7 @@ retrieve_from_file (const char *file, bool html, int *count)
       /* If we have a found a content encoding, use it.
        * ( == is okay, because we're checking for identical object) */
       if (iri->content_encoding != opt.locale)
-	  set_uri_encoding (iri, iri->content_encoding, false);
+          set_uri_encoding (iri, iri->content_encoding, false);
 
       /* Reset UTF-8 encode status */
       iri->utf8_encode = opt.enable_iri;
@@ -1065,8 +1070,9 @@ retrieve_from_file (const char *file, bool html, int *count)
 
       parsed_url = url_parse (cur_url->url->url, NULL, tmpiri, true);
 
+      char *proxy = getproxy (cur_url->url);
       if ((opt.recursive || opt.page_requisites)
-          && (cur_url->url->scheme != SCHEME_FTP || getproxy (cur_url->url)))
+          && (cur_url->url->scheme != SCHEME_FTP || proxy))
         {
           int old_follow_ftp = opt.follow_ftp;
 
@@ -1084,6 +1090,7 @@ retrieve_from_file (const char *file, bool html, int *count)
                                cur_url->url->url, &filename,
                                &new_file, NULL, &dt, opt.recursive, tmpiri,
                                true);
+      free(proxy);
 
       if (parsed_url)
           url_free (parsed_url);
@@ -1236,7 +1243,6 @@ getproxy (struct url *u)
 {
   char *proxy = NULL;
   char *rewritten_url;
-  static char rewritten_storage[1024];
 
   if (!opt.use_proxy)
     return NULL;
@@ -1266,13 +1272,9 @@ getproxy (struct url *u)
      getproxy() to return static storage. */
   rewritten_url = rewrite_shorthand_url (proxy);
   if (rewritten_url)
-    {
-      strncpy (rewritten_storage, rewritten_url, sizeof (rewritten_storage));
-      rewritten_storage[sizeof (rewritten_storage) - 1] = '\0';
-      proxy = rewritten_storage;
-    }
+    return rewritten_url;
 
-  return proxy;
+  return strdup(proxy);
 }
 
 /* Returns true if URL would be downloaded through a proxy. */
@@ -1283,7 +1285,9 @@ url_uses_proxy (struct url * u)
   bool ret;
   if (!u)
     return false;
-  ret = getproxy (u) != NULL;
+  char *proxy = getproxy (u);
+  ret = proxy != NULL;
+  free(proxy);
   return ret;
 }
 

src/retr.h

@@ -50,7 +50,7 @@ enum {
   rb_chunked_transfer_encoding = 4
 };
 
-int fd_read_body (int, FILE *, wgint, wgint, wgint *, wgint *, double *, int, FILE *);
+int fd_read_body (const char *, int, FILE *, wgint, wgint, wgint *, wgint *, double *, int, FILE *);
 
 typedef const char *(*hunk_terminator_t) (const char *, const char *, int);
 

src/spider.c

@@ -97,4 +97,3 @@ print_broken_links (void)
 /*
  * vim: et ts=2 sw=2
  */
-

src/spider.h

@@ -34,5 +34,6 @@ as that of the covered work.  */
 #define visited_url(a,b)
 void nonexisting_url (const char *);
 void print_broken_links (void);
+void spider_cleanup (void);
 
 #endif /* SPIDER_H */

src/sysdep.h

@@ -78,35 +78,15 @@ as that of the covered work.  */
    These have to be after the above namespace tweaks, but before any
    non-preprocessor code.  */
 
-#if HAVE_ALLOCA_H
-# include <alloca.h>
-#elif defined WINDOWS
-# include <malloc.h>
-# ifndef alloca
-#  define alloca _alloca
-# endif
-#elif defined __GNUC__
-# define alloca __builtin_alloca
-#elif defined _AIX
-# define alloca __alloca
-#else
-# include <stddef.h>
-# ifdef  __cplusplus
-extern "C"
-# endif
-void *alloca (size_t);
-#endif
+#include <alloca.h>
 
 /* Must include these, so we can test for the missing stat macros and
    define them as necessary.  */
 #include <sys/types.h>
 #include <sys/stat.h>
 
-#ifdef HAVE_INTTYPES_H
-  /* Compaq C V6.5-303 (dtk) on HP Tru64 UNIX V5.1B (Rev. 2650) needs: */
-# include <stdint.h>
-# include <inttypes.h>
-#endif
+#include <stdint.h>
+#include <inttypes.h>
 
 #ifdef WINDOWS
 /* Windows doesn't have some functions normally found on Unix-like
@@ -121,34 +101,6 @@ void *alloca (size_t);
 /* Provided by gnulib on systems that don't have it: */
 # include <stdbool.h>
 
-/* Needed for compilation under OS/2 and MSDOS */
-#if defined(__EMX__) || defined(MSDOS)
-# ifndef S_ISLNK
-#  define S_ISLNK(m) 0
-# endif
-# ifndef lstat
-#  define lstat stat
-# endif
-#endif /* __EMX__ || MSDOS */
-
-/* Reportedly, stat() macros are broken on some old systems.  Those
-   systems will have to fend for themselves, as I will not introduce
-   new code to handle it.
-
-   However, I will add code for *missing* macros, and the following
-   are reportedly missing from many systems.  */
-#ifndef S_ISLNK
-# define S_ISLNK(m) (((m) & S_IFMT) == S_IFLNK)
-#endif
-#ifndef S_ISDIR
-# define S_ISDIR(m) (((m) & (_S_IFMT)) == (_S_IFDIR))
-#endif
-#ifndef S_ISREG
-# define S_ISREG(m) (((m) & _S_IFMT) == _S_IFREG)
-#endif
-
-/* These are needed so we can #define struct_stat to struct _stati64
-   under Windows. */
 #ifndef struct_stat
 # define struct_stat struct stat
 #endif
@@ -156,6 +108,8 @@ void *alloca (size_t);
 # define struct_fstat struct stat
 #endif
 
+#include <intprops.h>
+
 /* For CHAR_BIT, LONG_MAX, etc. */
 #include <limits.h>
 
@@ -163,9 +117,6 @@ void *alloca (size_t);
 # define CHAR_BIT 8
 #endif
 
-/* From gnulib, simplified to assume a signed type. */
-#define TYPE_MAXIMUM(t) ((t) (~ (~ (t) 0 << (sizeof (t) * CHAR_BIT - 1))))
-
 /* These are defined in cmpt.c if missing, so we must declare
    them.  */
 #ifndef HAVE_STRCASECMP
@@ -212,23 +163,7 @@ int vsnprintf (char *str, size_t count, const char *fmt, va_list arg);
 # define SYSTEM_FNMATCH
 #endif
 
-#ifdef SYSTEM_FNMATCH
-# include <fnmatch.h>
-#else  /* not SYSTEM_FNMATCH */
-/* Define fnmatch flags.  Undef them first to avoid warnings in case
-   an evil library include chose to include system fnmatch.h.  */
-# undef FNM_PATHNAME
-# undef FNM_NOESCAPE
-# undef FNM_PERIOD
-# undef FNM_NOMATCH
-
-# define FNM_PATHNAME	(1 << 0) /* No wildcard can ever match `/'.  */
-# define FNM_NOESCAPE	(1 << 1) /* Backslashes don't quote special chars.  */
-# define FNM_PERIOD	(1 << 2) /* Leading `.' is matched only explicitly.  */
-# define FNM_NOMATCH	1
-
-int fnmatch (const char *, const char *, int);
-#endif
+#include <fnmatch.h>
 
 /* Provide sig_atomic_t if the system doesn't.  */
 #ifndef HAVE_SIG_ATOMIC_T

src/test.c

@@ -31,6 +31,9 @@ as that of the covered work.  */
 #include "wget.h"
 
 #include <stdio.h>
+#ifdef ENABLE_NLS
+# include <locale.h>
+#endif
 
 #include "test.h"
 
@@ -71,10 +74,18 @@ all_tests()
 char *program_name; /* Needed by lib/error.c. */
 
 int
-main (int argc, char *argv[])
+main (int argc _GL_UNUSED, char *argv[])
 {
   const char *result;
 
+#ifdef ENABLE_NLS
+  /* Set the current locale.  */
+  setlocale (LC_ALL, "");
+  /* Set the text message domain.  */
+  bindtextdomain ("wget", LOCALEDIR);
+  textdomain ("wget");
+#endif /* ENABLE_NLS */
+
   program_name = argv[0];
 
   result = all_tests();
@@ -96,4 +107,3 @@ main (int argc, char *argv[])
 /*
  * vim: et ts=2 sw=2
  */
-

src/test.h

@@ -45,9 +45,18 @@ do { \
 
 extern int tests_run;
 
+const char *test_parse_content_disposition(void);
+const char *test_commands_sorted(void);
+const char *test_cmd_spec_restrict_file_names(void);
+const char *test_is_robots_txt_url(void);
+const char *test_path_simplify (void);
+const char *test_append_uri_pathel(void);
+const char *test_are_urls_equal(void);
+const char *test_subdir_p(void);
+const char *test_dir_matches_p(void);
+
 #endif /* TEST_H */
 
 /*
  * vim: et ts=2 sw=2
  */
-

src/url.c

@@ -169,7 +169,7 @@ static const unsigned char urlchr_table[256] =
    The transformation is done in place.  If you need the original
    string intact, make a copy before calling this function.  */
 
-static void
+void
 url_unescape (char *s)
 {
   char *t = s;                  /* t - tortoise */
@@ -681,7 +681,6 @@ url_parse (const char *url, int *error, struct iri *iri, bool percent_encode)
   char *user = NULL, *passwd = NULL;
 
   const char *url_encoded = NULL;
-  char *new_url = NULL;
 
   int error_code;
 
@@ -695,29 +694,29 @@ url_parse (const char *url, int *error, struct iri *iri, bool percent_encode)
       goto error;
     }
 
+  url_encoded = url;
+
   if (iri && iri->utf8_encode)
     {
+      char *new_url = NULL;
+
       iri->utf8_encode = remote_to_utf8 (iri, iri->orig_url ? iri->orig_url : url, (const char **) &new_url);
       if (!iri->utf8_encode)
         new_url = NULL;
       else
         {
           iri->orig_url = xstrdup (url);
-          percent_encode = true;
+          url_encoded = reencode_escapes (new_url);
+          if (url_encoded != new_url)
+            xfree (new_url);
+          percent_encode = false;
         }
     }
 
-  /* XXX XXX Could that change introduce (security) bugs ???  XXX XXX*/
   if (percent_encode)
-    url_encoded = reencode_escapes (new_url ? new_url : url);
-  else
-    url_encoded = new_url ? new_url : url;
+    url_encoded = reencode_escapes (url);
 
   p = url_encoded;
-
-  if (new_url && url_encoded != new_url)
-    xfree (new_url);
-
   p += strlen (supported_schemes[scheme].leading_string);
   uname_b = p;
   p = url_skip_credentials (p);
@@ -1284,16 +1283,6 @@ append_null (struct growable *dest)
   *TAIL (dest) = 0;
 }
 
-/* Shorten DEST to LENGTH. */
-static void
-shorten_length (size_t length, struct growable *dest)
-{
-  if (length < dest->tail)
-    dest->tail = length;
-
-  append_null (dest);
-}
-
 /* Append CH to DEST. */
 static void
 append_char (char ch, struct growable *dest)
@@ -2046,7 +2035,7 @@ url_string (const struct url *url, enum url_auth_mode auth_mode)
           if (url->passwd)
             {
               if (auth_mode == URL_AUTH_HIDE_PASSWD)
-                quoted_passwd = HIDDEN_PASSWORD;
+                quoted_passwd = (char *) HIDDEN_PASSWORD;
               else
                 quoted_passwd = url_escape_allow_passthrough (url->passwd);
             }
@@ -2215,7 +2204,7 @@ ps (char *path)
 #endif
 
 static const char *
-run_test (char *test, char *expected_result, enum url_scheme scheme,
+run_test (const char *test, const char *expected_result, enum url_scheme scheme,
           bool expected_change)
 {
   char *test_copy = xstrdup (test);
@@ -2244,8 +2233,8 @@ run_test (char *test, char *expected_result, enum url_scheme scheme,
 const char *
 test_path_simplify (void)
 {
-  static struct {
-    char *test, *result;
+  static const struct {
+    const char *test, *result;
     enum url_scheme scheme;
     bool should_modify;
   } tests[] = {
@@ -2277,15 +2266,16 @@ test_path_simplify (void)
     { "a/b/../../c",            "c",            SCHEME_HTTP, true },
     { "./a/../b",               "b",            SCHEME_HTTP, true }
   };
-  int i;
+  unsigned i;
 
   for (i = 0; i < countof (tests); i++)
     {
       const char *message;
-      char *test = tests[i].test;
-      char *expected_result = tests[i].result;
+      const char *test = tests[i].test;
+      const char *expected_result = tests[i].result;
       enum url_scheme scheme = tests[i].scheme;
       bool  expected_change = tests[i].should_modify;
+
       message = run_test (test, expected_result, scheme, expected_change);
       if (message) return message;
     }
@@ -2293,19 +2283,19 @@ test_path_simplify (void)
 }
 
 const char *
-test_append_uri_pathel()
+test_append_uri_pathel(void)
 {
-  int i;
-  struct {
-    char *original_url;
-    char *input;
+  unsigned i;
+  static const struct {
+    const char *original_url;
+    const char *input;
     bool escaped;
-    char *expected_result;
+    const char *expected_result;
   } test_array[] = {
     { "http://www.yoyodyne.com/path/", "somepage.html", false, "http://www.yoyodyne.com/path/somepage.html" },
   };
 
-  for (i = 0; i < sizeof(test_array)/sizeof(test_array[0]); ++i)
+  for (i = 0; i < countof(test_array); ++i)
     {
       struct growable dest;
       const char *p = test_array[i].input;
@@ -2322,13 +2312,13 @@ test_append_uri_pathel()
   return NULL;
 }
 
-const char*
-test_are_urls_equal()
+const char *
+test_are_urls_equal(void)
 {
-  int i;
-  struct {
-    char *url1;
-    char *url2;
+  unsigned i;
+  static const struct {
+    const char *url1;
+    const char *url2;
     bool expected_result;
   } test_array[] = {
     { "http://www.adomain.com/apath/", "http://www.adomain.com/apath/",       true },
@@ -2339,7 +2329,7 @@ test_are_urls_equal()
     { "http://www.adomain.com/path%2f", "http://www.adomain.com/path/",       false },
   };
 
-  for (i = 0; i < sizeof(test_array)/sizeof(test_array[0]); ++i)
+  for (i = 0; i < countof(test_array); ++i)
     {
       mu_assert ("test_are_urls_equal: wrong result",
                  are_urls_equal (test_array[i].url1, test_array[i].url2) == test_array[i].expected_result);
@@ -2353,4 +2343,3 @@ test_are_urls_equal()
 /*
  * vim: et ts=2 sw=2
  */
-

src/url.h

@@ -76,11 +76,11 @@ enum url_scheme {
 /* Structure containing info on a URL.  */
 struct url
 {
-  char *url;			/* Original URL */
-  enum url_scheme scheme;	/* URL scheme */
+  char *url;                /* Original URL */
+  enum url_scheme scheme;   /* URL scheme */
 
-  char *host;			/* Extracted hostname */
-  int port;			/* Port number */
+  char *host;               /* Extracted hostname */
+  int port;                 /* Port number */
 
   /* URL components (URL-quoted). */
   char *path;
@@ -101,6 +101,7 @@ struct url
 
 char *url_escape (const char *);
 char *url_escape_unsafe_and_reserved (const char *);
+void url_unescape (char *);
 
 struct url *url_parse (const char *, int *, struct iri *iri, bool percent_encode);
 char *url_error (const char *, int);

src/utils.c

@@ -65,9 +65,7 @@ as that of the covered work.  */
 
 /* For TIOCGWINSZ and friends: */
 #include <sys/ioctl.h>
-#ifdef HAVE_TERMIOS_H
-# include <termios.h>
-#endif
+#include <termios.h>
 
 /* Needed for Unix version of run_with_timeout. */
 #include <signal.h>
@@ -100,7 +98,9 @@ as that of the covered work.  */
 #include "test.h"
 #endif
 
-static void
+#include "exits.h"
+
+static void _Noreturn
 memfatal (const char *context, long attempted_size)
 {
   /* Make sure we don't try to store part of the log line, and thus
@@ -123,7 +123,7 @@ memfatal (const char *context, long attempted_size)
                  exec_name, context, attempted_size);
     }
 
-  exit (1);
+  exit (WGET_EXIT_GENERIC_ERROR);
 }
 
 /* Character property table for (re-)escaping VMS ODS5 extended file
@@ -471,7 +471,7 @@ fork_to_background (void)
     {
       /* parent, error */
       perror ("fork");
-      exit (1);
+      exit (WGET_EXIT_GENERIC_ERROR);
     }
   else if (pid != 0)
     {
@@ -479,14 +479,17 @@ fork_to_background (void)
       printf (_("Continuing in background, pid %d.\n"), (int) pid);
       if (logfile_changed)
         printf (_("Output will be written to %s.\n"), quote (opt.lfilename));
-      exit (0);                 /* #### should we use _exit()? */
+      exit (WGET_EXIT_SUCCESS);                 /* #### should we use _exit()? */
     }
 
   /* child: give up the privileges and keep running. */
   setsid ();
-  freopen ("/dev/null", "r", stdin);
-  freopen ("/dev/null", "w", stdout);
-  freopen ("/dev/null", "w", stderr);
+  if (freopen ("/dev/null", "r", stdin) == NULL)
+    DEBUGP (("Failed to redirect stdin to /dev/null.\n"));
+  if (freopen ("/dev/null", "w", stdout) == NULL)
+    DEBUGP (("Failed to redirect stdout to /dev/null.\n"));
+  if (freopen ("/dev/null", "w", stderr) == NULL)
+    DEBUGP (("Failed to redirect stderr to /dev/null.\n"));
 }
 #endif /* !WINDOWS && !MSDOS */
 
@@ -957,16 +960,16 @@ subdir_p (const char *d1, const char *d2)
    first element that matches DIR, through wildcards or front comparison (as
    appropriate).  */
 static bool
-dir_matches_p (char **dirlist, const char *dir)
+dir_matches_p (const char **dirlist, const char *dir)
 {
-  char **x;
+  const char **x;
   int (*matcher) (const char *, const char *, int)
     = opt.ignore_case ? fnmatch_nocase : fnmatch;
 
   for (x = dirlist; *x; x++)
     {
       /* Remove leading '/' */
-      char *p = *x + (**x == '/');
+      const char *p = *x + (**x == '/');
       if (has_wildcards_p (p))
         {
           if (matcher (p, dir, FNM_PATHNAME) == 0)
@@ -1520,7 +1523,7 @@ with_thousand_seps (wgint n)
    some detail.  */
 
 char *
-human_readable (HR_NUMTYPE n)
+human_readable (HR_NUMTYPE n, const int acc, const int decimals)
 {
   /* These suffixes are compatible with those of GNU `ls -lh'. */
   static char powers[] =
@@ -1553,10 +1556,10 @@ human_readable (HR_NUMTYPE n)
       if ((n / 1024) < 1024 || i == countof (powers) - 1)
         {
           double val = n / 1024.0;
-          /* Print values smaller than 10 with one decimal digits, and
-             others without any decimals.  */
+          /* Print values smaller than the accuracy level (acc) with (decimal)
+           * decimal digits, and others without any decimals.  */
           snprintf (buf, sizeof (buf), "%.*f%c",
-                    val < 10 ? 1 : 0, val, powers[i]);
+                    val < acc ? decimals : 0, val, powers[i]);
           return buf;
         }
       n /= 1024;
@@ -1893,7 +1896,7 @@ random_float (void)
 
 static sigjmp_buf run_with_timeout_env;
 
-static void
+static void _Noreturn
 abort_run_with_timeout (int sig)
 {
   assert (sig == SIGALRM);
@@ -2102,8 +2105,8 @@ xsleep (double seconds)
    This implementation does not emit newlines after 76 characters of
    base64 data.  */
 
-int
-base64_encode (const void *data, int length, char *dest)
+size_t
+base64_encode (const void *data, size_t length, char *dest)
 {
   /* Conversion table.  */
   static const char tbl[64] = {
@@ -2165,12 +2168,12 @@ base64_encode (const void *data, int length, char *dest)
 
    Since DEST is assumed to contain binary data, it is not
    NUL-terminated.  The function returns the length of the data
-   written to TO.  -1 is returned in case of error caused by malformed
+   written to "TO".  -1 is returned in case of error caused by malformed
    base64 input.
 
    This function originates from Free Recode.  */
 
-int
+ssize_t
 base64_decode (const char *base64, void *dest)
 {
   /* Table of base64 values for first 128 characters.  Note that this
@@ -2283,7 +2286,7 @@ compile_posix_regex (const char *str)
   int errcode = regcomp ((regex_t *) regex, str, REG_EXTENDED | REG_NOSUB);
   if (errcode != 0)
     {
-      int errbuf_size = regerror (errcode, (regex_t *) regex, NULL, 0);
+      size_t errbuf_size = regerror (errcode, (regex_t *) regex, NULL, 0);
       char *errbuf = xmalloc (errbuf_size);
       regerror (errcode, (regex_t *) regex, errbuf, errbuf_size);
       fprintf (stderr, _("Invalid regular expression %s, %s\n"),
@@ -2301,10 +2304,10 @@ compile_posix_regex (const char *str)
 bool
 match_pcre_regex (const void *regex, const char *str)
 {
-  int l = strlen (str);
+  size_t l = strlen (str);
   int ovector[OVECCOUNT];
 
-  int rc = pcre_exec ((pcre *) regex, 0, str, l, 0, 0, ovector, OVECCOUNT);
+  int rc = pcre_exec ((pcre *) regex, 0, str, (int) l, 0, 0, ovector, OVECCOUNT);
   if (rc == PCRE_ERROR_NOMATCH)
     return false;
   else if (rc < 0)
@@ -2330,7 +2333,7 @@ match_posix_regex (const void *regex, const char *str)
     return true;
   else
     {
-      int errbuf_size = regerror (rc, opt.acceptregex, NULL, 0);
+      size_t errbuf_size = regerror (rc, opt.acceptregex, NULL, 0);
       char *errbuf = xmalloc (errbuf_size);
       regerror (rc, opt.acceptregex, errbuf, errbuf_size);
       logprintf (LOG_VERBOSE, _("Error while matching %s: %d\n"),
@@ -2426,7 +2429,7 @@ print_decimal (double number)
 
 /* Get the maximum name length for the given path. */
 /* Return 0 if length is unknown. */
-size_t
+long
 get_max_length (const char *path, int length, int name)
 {
   long ret;
@@ -2481,9 +2484,9 @@ get_max_length (const char *path, int length, int name)
 #ifdef TESTING
 
 const char *
-test_subdir_p()
+test_subdir_p(void)
 {
-  static struct {
+  static const struct {
     const char *d1;
     const char *d2;
     bool result;
@@ -2506,7 +2509,7 @@ test_subdir_p()
 }
 
 const char *
-test_dir_matches_p()
+test_dir_matches_p(void)
 {
   static struct {
     const char *dirlist[3];
@@ -2543,4 +2546,3 @@ test_dir_matches_p()
 }
 
 #endif /* TESTING */
-

src/utils.h

@@ -121,7 +121,7 @@ const char *with_thousand_seps (wgint);
 #else
 # define HR_NUMTYPE double
 #endif
-char *human_readable (HR_NUMTYPE);
+char *human_readable (HR_NUMTYPE, const int, const int);
 
 
 int numdigit (wgint);
@@ -139,8 +139,8 @@ void xsleep (double);
 /* How many bytes it will take to store LEN bytes in base64.  */
 #define BASE64_LENGTH(len) (4 * (((len) + 2) / 3))
 
-int base64_encode (const void *, int, char *);
-int base64_decode (const char *, void *);
+size_t base64_encode (const void *, size_t, char *);
+ssize_t base64_decode (const char *, void *);
 
 #ifdef HAVE_LIBPCRE
 void *compile_pcre_regex (const char *);
@@ -154,7 +154,7 @@ void stable_sort (void *, size_t, size_t, int (*) (const void *, const void *));
 
 const char *print_decimal (double);
 
-size_t get_max_length (const char *path, int length, int name);
+long get_max_length (const char *path, int length, int name);
 
 extern unsigned char char_prop[];
 

src/vms.c

@@ -45,7 +45,7 @@
  *
  *   Simplifies a fancy URL-derived file name into an ODS2- or
  *   ODS5-compatible file name.
- *    
+ *
  *----------------------------------------------------------------------
  *
  *       utime()
@@ -299,7 +299,7 @@ int dmy_lib$initialize = (int) lib$initialize;
 /*--------------------------------------------------------------------*/
 
 /* vms_arch()
-  
+
       Returns (run-time) VMS architecture string.
 */
 
@@ -556,12 +556,12 @@ char *vms_getpass( const char *prompt)
         fclose( sdc);
     }
     return ret;
-}    
+}
 
 /*--------------------------------------------------------------------*/
 
 /* vms_vers()
-  
+
       Returns (run-time) VMS version string.
 */
 
@@ -841,12 +841,11 @@ else
 
 /* Action routine for decc$to_vms(), in utime(). */
 
-char vms_path[ NAMX$C_MAXRSS+ 1];
+char *vms_path;
 
 int set_vms_name( char *name, int type)
 {
-   strncpy( vms_path, name, NAMX$C_MAXRSS);
-   vms_path[ NAMX$C_MAXRSS] = '\0';
+   vms_path = strdup(name);
    return 1;
 }
 
@@ -1337,7 +1336,7 @@ localtime_r (t, tp)
   return tp;
 }
 
-#endif	/* __CRTL_VER < 70000000 */
+#endif  /* __CRTL_VER < 70000000 */
 
 /*--------------------------------------------------------------------*/
 
@@ -1350,4 +1349,3 @@ const char *link_string = NULL;
 const char *version_string = VERSION;
 
 /*--------------------------------------------------------------------*/
-

src/warc.c

@@ -36,7 +36,6 @@ as that of the covered work.  */
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
-#include <strings.h>
 #include <time.h>
 #include <tmpdir.h>
 #include <sha1.h>
@@ -45,8 +44,11 @@ as that of the covered work.  */
 #ifdef HAVE_LIBZ
 #include <zlib.h>
 #endif
+
 #ifdef HAVE_LIBUUID
 #include <uuid/uuid.h>
+#elif HAVE_UUID_CREATE
+#include <uuid.h>
 #endif
 
 #ifndef WINDOWS
@@ -56,6 +58,7 @@ as that of the covered work.  */
 #endif
 
 #include "warc.h"
+#include "exits.h"
 
 #ifndef O_TEMPORARY
 #define O_TEMPORARY 0
@@ -108,7 +111,7 @@ static char *warc_current_filename;
 static int warc_current_file_number;
 
 /* The table of CDX records, if deduplication is enabled. */
-struct hash_table * warc_cdx_dedup_table;
+static struct hash_table * warc_cdx_dedup_table;
 
 static bool warc_start_new_file (bool meta);
 
@@ -383,9 +386,9 @@ warc_write_end_record (void)
 static bool
 warc_write_date_header (const char *timestamp)
 {
+  char current_timestamp[21];
   if (timestamp == NULL)
     {
-      char current_timestamp[21];
       warc_timestamp (current_timestamp);
       timestamp = current_timestamp;
     }
@@ -594,7 +597,7 @@ warc_timestamp (char *timestamp)
   strftime (timestamp, 21, "%Y-%m-%dT%H:%M:%SZ", timeinfo);
 }
 
-#ifdef HAVE_LIBUUID
+#if HAVE_LIBUUID || HAVE_UUID_CREATE
 /* Fills urn_str with a UUID in the format required
    for the WARC-Record-Id header.
    The string will be 47 characters long. */
@@ -604,8 +607,13 @@ warc_uuid_str (char *urn_str)
   char uuid_str[37];
 
   uuid_t record_id;
+#if HAVE_UUID_CREATE
+  uuid_create (&record_id, NULL);
+  uuid_to_string (&record_id, &uuid_str, NULL);
+#else
   uuid_generate (record_id);
   uuid_unparse (record_id, uuid_str);
+#endif
 
   sprintf (urn_str, "<urn:uuid:%s>", uuid_str);
 }
@@ -726,7 +734,7 @@ warc_start_new_file (bool meta)
 
   if (warc_current_file != NULL)
     fclose (warc_current_file);
-  
+
   free (warc_current_warcinfo_uuid_str);
   free (warc_current_filename);
 
@@ -1038,7 +1046,7 @@ warc_init (void)
               logprintf (LOG_NOTQUIET,
                          _("Could not read CDX file %s for deduplication.\n"),
                          quote (opt.warc_cdx_dedup_filename));
-              exit(1);
+              exit (WGET_EXIT_GENERIC_ERROR);
             }
         }
 
@@ -1047,7 +1055,7 @@ warc_init (void)
         {
           logprintf (LOG_NOTQUIET,
                      _("Could not open temporary WARC manifest file.\n"));
-          exit(1);
+          exit (WGET_EXIT_GENERIC_ERROR);
         }
 
       if (opt.warc_keep_log)
@@ -1057,7 +1065,7 @@ warc_init (void)
             {
               logprintf (LOG_NOTQUIET,
                          _("Could not open temporary WARC log file.\n"));
-              exit(1);
+              exit (WGET_EXIT_GENERIC_ERROR);
             }
           log_set_warc_log_fp (warc_log_fp);
         }
@@ -1066,7 +1074,7 @@ warc_init (void)
       if (! warc_start_new_file (false))
         {
           logprintf (LOG_NOTQUIET, _("Could not open WARC file.\n"));
-          exit(1);
+          exit (WGET_EXIT_GENERIC_ERROR);
         }
 
       if (opt.warc_cdx_enabled)
@@ -1075,7 +1083,7 @@ warc_init (void)
             {
               logprintf (LOG_NOTQUIET,
                          _("Could not open CDX file for output.\n"));
-              exit(1);
+              exit (WGET_EXIT_GENERIC_ERROR);
             }
         }
     }
@@ -1103,7 +1111,7 @@ warc_write_metadata (void)
   if (warc_tmp_fp == NULL)
     {
       logprintf (LOG_NOTQUIET, _("Could not open temporary WARC file.\n"));
-      exit(1);
+      exit (WGET_EXIT_GENERIC_ERROR);
     }
   fflush (warc_tmp_fp);
   fprintf (warc_tmp_fp, "%s\n", program_argstring);
@@ -1231,7 +1239,7 @@ static bool
 warc_write_cdx_record (const char *url, const char *timestamp_str,
                        const char *mime_type, int response_code,
                        const char *payload_digest, const char *redirect_location,
-                       off_t offset, const char *warc_filename,
+                       off_t offset, const char *warc_filename _GL_UNUSED,
                        const char *response_uuid)
 {
   /* Transform the timestamp. */

src/wget.h

@@ -72,12 +72,6 @@ as that of the covered work.  */
    variables.  -- explanation partly taken from GNU make.  */
 #define N_(string) string
 
-#if ! ENABLE_NLS
-# undef HAVE_WCHAR_H
-# undef HAVE_WCWIDTH
-# undef HAVE_MBTOWC
-#endif /* not ENABLE_NLS */
-
 #if HAVE_WCWIDTH && HAVE_MBTOWC
 # define USE_NLS_PROGRESS_BAR 1
 #else
@@ -254,24 +248,24 @@ typedef double SUM_SIZE_INT;
 /* Copy the data delimited with BEG and END to alloca-allocated
    storage, and zero-terminate it.  Arguments are evaluated only once,
    in the order BEG, END, PLACE.  */
-#define BOUNDED_TO_ALLOCA(beg, end, place) do {	\
-  const char *BTA_beg = (beg);			\
-  int BTA_len = (end) - BTA_beg;		\
-  char **BTA_dest = &(place);			\
-  *BTA_dest = alloca (BTA_len + 1);		\
-  memcpy (*BTA_dest, BTA_beg, BTA_len);		\
-  (*BTA_dest)[BTA_len] = '\0';			\
+#define BOUNDED_TO_ALLOCA(beg, end, place) do { \
+  const char *BTA_beg = (beg);                  \
+  int BTA_len = (end) - BTA_beg;                \
+  char **BTA_dest = &(place);                   \
+  *BTA_dest = alloca (BTA_len + 1);             \
+  memcpy (*BTA_dest, BTA_beg, BTA_len);         \
+  (*BTA_dest)[BTA_len] = '\0';                  \
 } while (0)
 
 /* Return non-zero if string bounded between BEG and END is equal to
    STRING_LITERAL.  The comparison is case-sensitive.  */
-#define BOUNDED_EQUAL(beg, end, string_literal)				\
-  ((end) - (beg) == sizeof (string_literal) - 1				\
+#define BOUNDED_EQUAL(beg, end, string_literal)             \
+  ((end) - (beg) == sizeof (string_literal) - 1             \
    && !memcmp (beg, string_literal, sizeof (string_literal) - 1))
 
 /* The same as above, except the comparison is case-insensitive. */
-#define BOUNDED_EQUAL_NO_CASE(beg, end, string_literal)			\
-  ((end) - (beg) == sizeof (string_literal) - 1				\
+#define BOUNDED_EQUAL_NO_CASE(beg, end, string_literal)         \
+  ((end) - (beg) == sizeof (string_literal) - 1                 \
    && !strncasecmp (beg, string_literal, sizeof (string_literal) - 1))
 
 /* Like ptr=strdup(str), but allocates the space for PTR on the stack.
@@ -280,11 +274,11 @@ typedef double SUM_SIZE_INT;
    The problem is that some compilers can't handle alloca() being an
    argument to a function.  */
 
-#define STRDUP_ALLOCA(ptr, str) do {			\
-  char **SA_dest = &(ptr);				\
-  const char *SA_src = (str);				\
-  *SA_dest = (char *)alloca (strlen (SA_src) + 1);	\
-  strcpy (*SA_dest, SA_src);				\
+#define STRDUP_ALLOCA(ptr, str) do {                \
+  char **SA_dest = &(ptr);                          \
+  const char *SA_src = (str);                       \
+  *SA_dest = (char *)alloca (strlen (SA_src) + 1);  \
+  strcpy (*SA_dest, SA_src);                        \
 } while (0)
 
 /* Generally useful if you want to avoid arbitrary size limits but
@@ -295,17 +289,17 @@ typedef double SUM_SIZE_INT;
    NEEDED_SIZE objects.  The reallocing is done by doubling, which
    ensures constant amortized time per element.  */
 
-#define DO_REALLOC(basevar, sizevar, needed_size, type)	do {		\
-  long DR_needed_size = (needed_size);					\
-  long DR_newsize = 0;							\
-  while ((sizevar) < (DR_needed_size)) {				\
-    DR_newsize = sizevar << 1;						\
-    if (DR_newsize < 16)						\
-      DR_newsize = 16;							\
-    (sizevar) = DR_newsize;						\
-  }									\
-  if (DR_newsize)							\
-    basevar = xrealloc (basevar, DR_newsize * sizeof (type));		\
+#define DO_REALLOC(basevar, sizevar, needed_size, type) do {    \
+  long DR_needed_size = (needed_size);                          \
+  long DR_newsize = 0;                                          \
+  while ((sizevar) < (DR_needed_size)) {                        \
+    DR_newsize = sizevar << 1;                                  \
+    if (DR_newsize < 16)                                        \
+      DR_newsize = 16;                                          \
+    (sizevar) = DR_newsize;                                     \
+  }                                                             \
+  if (DR_newsize)                                               \
+    basevar = xrealloc (basevar, DR_newsize * sizeof (type));   \
 } while (0)
 
 /* Used to print pointers (usually for debugging).  Print pointers
@@ -323,14 +317,14 @@ extern const char *exec_name;
 /* Document type ("dt") flags */
 enum
 {
-  TEXTHTML             = 0x0001,	/* document is of type text/html
+  TEXTHTML             = 0x0001,        /* document is of type text/html
                                            or application/xhtml+xml */
-  RETROKF              = 0x0002,	/* retrieval was OK */
-  HEAD_ONLY            = 0x0004,	/* only send the HEAD request */
-  SEND_NOCACHE         = 0x0008,	/* send Pragma: no-cache directive */
-  ACCEPTRANGES         = 0x0010,	/* Accept-ranges header was found */
+  RETROKF              = 0x0002,        /* retrieval was OK */
+  HEAD_ONLY            = 0x0004,        /* only send the HEAD request */
+  SEND_NOCACHE         = 0x0008,        /* send Pragma: no-cache directive */
+  ACCEPTRANGES         = 0x0010,        /* Accept-ranges header was found */
   ADDED_HTML_EXTENSION = 0x0020,        /* added ".html" extension due to -E */
-  TEXTCSS              = 0x0040	        /* document is of type text/css */
+  TEXTCSS              = 0x0040         /* document is of type text/css */
 };
 
 /* Universal error type -- used almost everywhere.  Error reporting of
@@ -338,27 +332,18 @@ enum
    simplified.  */
 typedef enum
 {
-  /*  0  */
   NOCONERROR, HOSTERR, CONSOCKERR, CONERROR, CONSSLERR,
-  CONIMPOSSIBLE, NEWLOCATION, NOTENOUGHMEM /* ! */,
-  CONPORTERR /* ! */, CONCLOSED /* ! */,
-  /* 10  */
+  CONIMPOSSIBLE, NEWLOCATION,
   FTPOK, FTPLOGINC, FTPLOGREFUSED, FTPPORTERR, FTPSYSERR,
-  FTPNSFOD, FTPRETROK /* ! */, FTPUNKNOWNTYPE, FTPRERR, FTPREXC /* ! */,
-  /* 20  */
+  FTPNSFOD, FTPUNKNOWNTYPE, FTPRERR,
   FTPSRVERR, FTPRETRINT, FTPRESTFAIL, URLERROR, FOPENERR,
-  FOPEN_EXCL_ERR, FWRITEERR, HOK /* ! */, HLEXC /* ! */, HEOF,
-  /* 30  */
-  HERR, RETROK, RECLEVELEXC, FTPACCDENIED /* ! */, WRONGCODE,
+  FOPEN_EXCL_ERR, FWRITEERR, HEOF,
+  HERR, RETROK, RECLEVELEXC, WRONGCODE,
   FTPINVPASV, FTPNOPASV, CONTNOTSUPPORTED, RETRUNNEEDED, RETRFINISHED,
-  /* 40  */
-  READERR, TRYLIMEXC, URLBADPATTERN /* ! */, FILEBADFILE /* ! */, RANGEERR,
-  RETRBADPATTERN, RETNOTSUP /* ! */, ROBOTSOK /* ! */, NOROBOTS /* ! */,
-  PROXERR,
-  /* 50  */
+  READERR, TRYLIMEXC, FILEBADFILE, RANGEERR,
+  RETRBADPATTERN, PROXERR,
   AUTHFAILED, QUOTEXC, WRITEFAILED, SSLINITFAILED, VERIFCERTERR,
   UNLINKERR, NEWLOCATION_KEEP_POST, CLOSEFAILED, ATTRMISSING, UNKNOWNATTR,
-
   WARC_ERR, WARC_TMP_FOPENERR, WARC_TMP_FWRITEERR
 } uerr_t;
 
@@ -366,7 +351,7 @@ typedef enum
    Select an appropriate "orig" suffix and a separator character for
    adding a unique suffix to a file name.
 
-   A VMS ODS2 file system can not tolerate multiple dots.  An ODS5 file
+   A VMS ODS2 file system can't tolerate multiple dots.  An ODS5 file
    system can, but even there not all dots are equal, and heroic effort
    would be needed to get ".html^.orig" rather than (the less desirable)
    "^.html.orig".  It's more satisfactory always to use "_orig" on VMS

testenv/ChangeLog

@@ -0,0 +1,527 @@
+2014-10-01  Darshit Shah  <darnir@gmail.com>
+
+	* Makefile.am: Run the tests in Python's Optimizedmode
+	* conf/__init__.py (gen_hook): Use try..except instead of if..else
+	* misc/color_terminal.py: System and check will not change while a test is
+	run. Do not test for them on every invokation of printer()
+	* server/http/http_server.py: The ssl and re modules are required by
+	specific functions. Load them lazily
+	(HTTPSServer.__init__): Lazy load ssl module here
+	(_handler.parse_range_header): Lazy load re module here
+	(_Handler.get_rule_list): get() can return a default value. Use it
+	(_Handler.guess_type): Same
+	(_Handler.is_authorized): Unused function artefact. Remove
+	(_Handler.reject_headers): Unused function artefact. Remove
+
+2014-10-08  Darshit Shah  <darnir@gmail.com>
+
+	* Makefile.am: Fix EXTRA_DIST variable for make distcheck
+	* server/http/http_server.py (HTTPServer.__init__):  Fix how CERTFILE is
+	found when running make dist / make distcheck.
+
+2014-09-30  Tim Ruehsen  <tim.ruehsen@gmx.de>
+
+	* test/base_test.py: Add --track-origins=yes to valgrind testing
+
+2014-08-08  Darshit Shah  <darnir@gmail.com>
+
+	* conf/__init__.py: Add extra newline according to PEP8
+	* conf/{authentication,expect_header,expected_files,expected_ret_code,
+	files_crawled,hook_sample,local_files,reject_header,response,send_header,
+	server_files,urls,wget_commands}.py: Add docstrings explaining the conf file
+	and how it should be used
+	* server/http/http_server (InvalidRangeHeader): Clear TODO and eliminate
+	this exception. Use ServerError for all such purposes.
+	(_Handler): Remove reference to InvalidRangeHeader
+	(_handler.parse_range_header): User ServerError instead of InvalidRangeHeader
+	(_Handler.do_GET):  Add docstring
+	(_Handler.do_POST): Add docstring. Also create an empty dict for rules if
+	no rules are supplied. Send the Location header as suggested in RFC 7231
+	(_Handler.do_PUT): Don't pop the server file already. Push it to later in ..
+	(_Handler.send_put): .. Here. If the file exists respond with a 204 No
+	Content message and pop the file for replacement. Do not send the
+	Content-Length, Content-Type headers since PUT requests should not respond
+	with data.
+	(_Handler.parse_auth_header): Fit line within 80 chars
+	(_Handler.check_response): Better visual indent
+	(_Handler.authorize_digest): Better visual indent.
+	(_Handler.expect_headers): Remove unused function
+	(_Handler.guess_type): Fix indentation
+	(HTTPd): Add newline according to PEP8 guidelines
+	(HTTPSd): Fix indentation
+	(StoppableHTTPServer): Add docstring
+	(HTTPSServer): Fix indentation
+	(WgetHTTPRequestHandler): Merge class into _handler.
+	(_Handler): Add docstring
+	(_Handler.parse_range_header): Fix indentation
+	(ServerError): Split exception into separate file ...
+	* exc/server_error.py: ... Here
+	* misc/colour_terminal.py: Add docstring, fix indentation
+	* test/base_test.py: Fix visual indent
+	* test/http_test.py: Fit within 80 char lines
+
+2014-08-04  Darshit Shah  <darnir@gmail.com>
+
+	* conf/server_conf.py: Delete file. Server configuration is now done via the
+	server_conf() method.
+	* server/http/http_server.py (StppableHTTPServer.server_sett): Delete
+	method required by the above hook
+	(HTTPd.server_sett): Same
+
+2014-07-26  Darshit Shah  <darnir@gmail.com>
+
+	* Test-*.py: Remove the '-d' switch from WGET_OPTIONS.
+	* test/base_test (BaseTest.gen_cmd_line): Add --debug and --no-config to the
+	list of switches passed to wget unconditionally.
+
+2014-07-23  Darshit Shah  <darnir@gmail.com>
+
+	* test/base_test.py (BaseTest.gen_cmd_line): Add support for running all
+	tests through valgrind if the relevant environment variable is set
+	* conf/expected_ret_code (ExpectedRetCode.__call__): Valgrind returns error
+	code 45 when it detects a memory leak.
+	* Readme: Update with details about valgrind tests
+
+2014-07-22  Darshit Shah  <darnir@gmail.com>
+
+	* (README): Remove old TODO and document SERVER_WAIT variable
+
+2014-06-22  Darshit Shah  <darnir@gmail.com>
+
+	* (conf.files_crawled): diff is a set object and needs explicit str
+	conversion.
+
+2014-03-13  Zihang Chen  <chsc4698@gmail.com>
+
+	* base_test.py:
+	(CommonMethods): Rename to BaseTest.
+	(BaseTest): Implement __init__ method where the class-wide variables are
+	initialized. Also variable names like `xxx_list` is renamed to its plural
+	form, e.g. `server_list` => `servers`.
+	(BaseTest.init_test_env): Remove name argument due to its unnecessarity.
+	(BaseTest.get_test_dir): Because the path of the test directory is needed
+	in multiple methods, this method is implemented.
+	(BaseTest.get_domain_addr): Rewrite the return statement utilizing str
+	formatting (which is more Pythonic).
+	(BaseTest.get_cmd_line): Rename to gen_cmd_line. Change the variables with
+	capitcal characters to lower ones. Also, the nested for loop is rewritten
+	to a plain loop using the zip function.
+	(BaseTest.__gen_local_filesys): Rename to gen_local_fs_snapshot. Move to
+	ExpectedFiles in conf/expected_files.py and is marked as a static
+	method. Refactor to a less verbose implementation.
+	(BaseTest._check_downloaded_files): Rename to __call__ to agree with the
+	invocation in test case classes. Move to ExpectedFiles in
+	conf/expected_files.py.
+	(BaseTest.get_server_rules): Refactor to a more Pythonic form utilizing
+	dict.items() and is marked static.
+	(BaseTest.stop_server): (new method) an abstract method which should stop
+	the currently using servers.
+	(BaseTest.instantiate_server_by): (new method) an abstract method which
+	should instantiate a server instance according to the given argument.
+	(BaseTest.__enter__): (new method) method which initialize the context
+	manager
+	(BaseTest.__exit__): (new method) method that finilize the context manager
+	and deal with the exceptions during the execution of the with statement,
+	subclasses can override this method for extensibility
+	* http_test.py:
+	(HTTPTest.__init__): Add call to super.__init__. Default values of
+	pre_hook, test_params, post_hook are set to None to avoid a subtle bug of
+	Python. Argument servers is renamed to protocols.
+	(HTTPTest.Server_setup): Move to BaseTest and rename to server_setup.
+	Calls to pre_hook_call, call_test, post_hook_call are removed.
+	(HTTPTest.hook_call, pre_hook_call, call_test, post_hook_call): Move to
+	BaseTest for that both HTTP test cases and FTP test cases may use these
+	methods.
+	(HTTPTest.init_HTTP_Server, init_HTTPS_Server): Merge and rename to
+	instantiate_server_by to implement the abstract method in BaseTest.
+	(HTTPTest.stop_HTTP_Server): Rename to stop_server to implement the
+	abstract method in BaseTest. Also, pull out the part where remaining
+	requests are gathered into a new method request_remaining.
+	(BaseTest.act_retcode): Rename to ret_code because ExpectedRetCode is
+	moved out from BaseTest, so the name act_retcode is actually a bit
+	verbose.
+	* conf/expected_ret_code.py:
+	(ExpectedRetCode.__call__): Rewrite the str into a more readable form.
+	* conf/files_crawled.py:
+	(FilesCrawled.__call__): Refactor this method into a more Pythonic form
+	utilizing the zip function.
+	* conf/local_files.py:
+	(LocalFiles__call__): Rewrite this method with the recommended with
+	statement.
+	* conf/server_conf.py:
+	(ServerConf.__call__): Rewrite this method due to BaseTest.server_list is
+	renamed to BaseTest.servers.
+	* conf/server_files.py:
+	(ServerFiles.__call__): Refactor the nested for loop into a plain one
+	utilizing the zip function.
+	* conf/urls.py:
+	(URLs): Rename url_list to urls.
+	* conf/wget_commands.py:
+	(WgetCommands): Rename command_list to commands, rename test_obj.options
+	to test_obj.wget_options.
+	* Test--https.py, Test-Proto.py, Test-Parallel-Proto.py: Argument servers
+	is changed to protocols due to change in the signature of
+	HTTPTest.__init__.
+
+2014-03-13  Zihang Chen  <chsc4698@gmail.com>
+
+	* test: (new package) package for test case classes
+	* WgetTest.py: Split into test/base_test.py and test/http_test.py.
+	* Test-*.py: Optimize the imports according to changes of WgetTest.py
+
+2014-03-13  Zihang Chen  <chsc4698@gmail.com>
+
+	* server: (new package) package for the server classes
+	* server.http: (new package) package for HTTP server
+	* server.ftp: (new package) package for FTP server
+	* HTTPServer.py: Move to server/http/http_server.py. Also change the
+	CERTFILE to '../certs/wget-cert.pem'.
+	* FTPServer.py: Move to server/ftp/ftp_server.py.
+	* WgetTest.py: Optimize import respect to the server classes.
+
+2014-03-13  Zihang Chen  <chsc4698@gmail.com>
+
+	* conf: (new package) package for rule classes and hook methods
+	* WgetTest.py:
+	(CommonMethods.Authentication): Move to conf/authentication.py.
+	(CommonMethods.ExpectHeader): Move to conf/expect_header.py.
+	(CommonMethods.RejectHeader): Move to conf/reject_header.py.
+	(CommonMethods.Response): Move to conf/response.py.
+	(CommonMethods.SendHeader): Move to conf/send_header.py.
+	(CommonMethods.ServerFiles): Move to conf/server_files.py.
+	(CommonMethods.LocalFiles): Move to conf/local_files.py.
+	(CommonMethods.ServerConf): Move to conf/server_conf.py.
+	(CommonMethods.WgetCommands): Move to conf/wget_commands.py.
+	(CommonMethods.Urls): Move to conf/urls.py.
+	(CommonMethods.ExpectedRetcode): Move to conf/expected_retcode.py.
+	(CommonMethods.ExpectedFiles): Move to conf/expected_files.py.
+	(CommonMethods.FilesCrawled): Move to conf/files_crawled.py.
+	(CommonMethods.__check_downloaded_files): Rename to
+	_check_downloaded_files, so that the method is callable from outside the
+	class.
+	(CommomMethods.get_server_rules): Modify so that it utilizes the conf
+	package.
+	(HTTPTest): Add a method hook_call(configs, name) to reduce duplications
+	in pre_hook_call, call_test and post_hook_call utilizing the conf package.
+	* conf/hook_sample.py: (new file) sample for hooks
+	* conf/rule_sample.py: (new file) sample for rules
+	* REAMDE: Update sections about customizing rules and hooks.
+
+2014-03-13  Zihang Chen  <chsc4698@gmail.com>
+
+	* exc: (new package) package for miscellaneous exceptions
+	* WgetTest.py: Move TestFailed to exc/test_failed.py.
+
+2014-03-13  Zihang Chen  <chsc4698@gmail.com>
+
+	* Test-Proto.py: Fix a typo (line 71: server to servers).
+
+2014-03-13  Zihang Chen  <chsc4698@gmail.com>
+
+	* WgetTest.py: Move WgetFile to package misc.
+	* README: Modify documentation respect to WgetFile.
+	* Test-*.py: Optimize imports about WgetFile.
+
+2014-03-13  Zihang Chen  <chsc4698@gmail.com>
+
+	* misc: (new package) package for miscellaneous modules
+	* ColourTerm.py: Move to package misc and rename to colour_terminal.py,
+	add print_color functions to reduce the use of string literals like
+	"BLUE", "RED" etc.
+	* WgetTest.py:
+	(CommonMethods.Server_setup): Change invocation to printer to print_blue.
+	(CommonMethods.FilesCrawled): Change invocation to printer to print_red.
+	(HTTPTest.__init__): Change invocations to printer to print_red and
+	print_green respectively.
+
+2014-01-02  Darshit Shah  <darnir@gmail.com>
+	* Makefile.am: Add new Test--https.py to list of tests and EXTRA_DIST.
+	Also replace all tabs with spaces in file for conformity.
+	* Test--https.py: New test to check if Wget works correctly with HTTPS
+	servers
+	* HTTPServer.py: Import new modules for use in HTTPS Servers
+	(HTTPSServer): New class that generates a SSL-wrapped socket for use in a
+	HTTPS Server.
+	(HTTPSd): HTTPS daemon class. Analogous to the HTTPd class
+	* WgetTest.py: Define global variables HTTP and HTTPS to reflect Server
+	types
+	(CommonMethods.exec_wget): Add the protocol information to the URL before
+	passing it to wget
+	(HTTPTest.__init__): Edit syntax. The servers variable now accepts a list of
+	servers defined by their type. E.g. HTTP, HTTPS.
+	(HTTPTest.Server_setup): Reflect change in type of variable servers.
+	However, we maintin the value of self.servers to allow most of the code to
+	remain unchanged.
+	(HTTPTest.init_HTTPS_Server): Initialize a HTTPS Server
+	* Test-Parallel-Proto.py: Edit to reflect slight change in Test Fiel Syntax.
+	* Test-Proto.py: Same
+
+2014-01-02  Darshit Shah  <darnir@gmail.com>
+
+	* WgetTest.py (CommonMentods.exec_wget): Wait for n seconds before calling
+	the Wget executable.
+
+2013-12-27  Darshit Shah  <darnir@gmail.com>
+
+	* WgetTest.py: Add modeline
+	(CommonMethods.ServerConf): New pre-test hook that sets
+	BaseHTTPRequestHandler class variables in all available servers
+	* HTTPServer.py (HTTPd.ServerConf): Call the respective method in the Server
+	to set the class variables
+	(StoppableHTTPServer.server_sett): Set the handler class variables
+
+2013-12-26  Darshit Shah  <darnir@gmail.com>
+
+	* WgetTest.py (HTTPTest.call_test): Correct the call to stop_HTTP_Server.
+
+2013-12-25  Darshit Shah  <darnir@gmail.com>
+
+	* WgetTest.py (CommonMehtods.exec_wget): Catch and handle exception if the
+	Wget executable is not found at src/wget
+	(HTTPTest.call_test): In case of error during execution, remove all existing
+	servers before quitting
+
+2013-12-15  Darshit Shah  <darnir@gmail.com>
+
+	* WgetTest.py (HTTPTest.HTTP_setup): Rename to Server_setup so it can be
+	easily reused for other non-HTTP servers.
+	(HTTPTest.__init__): Call Server_setup instead of HTTP_setup
+	(HTTPTest.Server_setup): Split into three more functions, that handle
+	pre-hooks, test execution and post-hooks respectively.
+	(HTTPTest.pre_hook_call): Set up and execute the pre-test hooks. Code split
+	from HTTPTest.Server_setup
+	(HTTPTest.call_test): Execute wget and log exit code. Code split from
+	HTTPTest.Server_setup
+	(HTTPTest.post_hook_call): Set up and execute post-test hooks. Code split
+	from HTTPTest.Server_setup
+
+2013-10-14  Giuseppe Scrivano  <gscrivan@redhat.com>
+
+	* Makefile.am (XFAIL_TESTS): Remove Test--spider-r.py.
+
+2013-10-06  Giuseppe Scrivano  <gscrivan@redhat.com>
+
+	* Makefile.am (EXTRA_DIST): Distribute test files.
+
+2013-09-16  Darshit Shah  <darnir@gmail.com>
+
+	* README: Update documentation
+
+2013-09-14  Darshit Shah  <darnir@gmail.com>
+
+	* HTTPServer.py (StoppableHTTPServer): Define object variable
+	request_headers which stores a list of requests received by the server
+	(StoppableHTTPServer.get_req_headers): Return the list of Request
+	Headers stored by the server
+	(_Handler.do_HEAD): Send the Request MEthod string for identification
+	(_Handler.do_GET): Same
+	(_Handler.__log_request): Log the request in Request_Headers list
+	(_Handler.send_head): Make a call to __log_request
+	* Test--spider-r.py: Add new list, Request_List, which contains all
+	the requests that Wget is expected to send. This will allow for
+	fine-grained tests on recursive downloading.
+	* WgetTest.py (CommonMethods.FilesCrawled): New Post-Test Hook, that
+	ensures that all the expected Files on the server were accessed as
+	expected.
+	(HTTPTest.stop_HTTP_server): On stopping server, asks it to respond
+	with list of all requests it received.
+
+2013-09-13  Darshit Shah  <darnir@gmail.com>
+
+	* Test--spider-r.py: Test retrieval in recursive spider mode.
+	* Makefile.am: add new file
+
+2013-09-13  Darshit Shah  <darnir@gmail.com>
+
+	* HTTPServer.py (_Handler.do_HEAD): If requested path is /, respond
+	with /index.html
+	(_Handler.do_HEAD): Smartly guess value of Content-Type Header from
+	file extension
+	(_Handler.guess_type): Use a preset list of extensions and
+	Content-Type strings. If the extension matches one in the list, use
+	that string, else default to "text/plain"
+
+2013-09-13  Darshit Shah  <darnir@gmail.com>
+
+	* WgetTest.py (CommonMethods._replace_substring): New method that will
+	replace a substring delimited by {{ }} characters by the value of
+	self.<substring> variable
+	(CommonMethods.WgetCommands): Use the _replace_substring () call to
+	replace the substrings in the the command line.
+	(CommonMethods.ServerFiles): Run the _replace_substring () method on
+	the File contents too.
+
+2013-09-11  Darshit Shah  <darnir@gmail.com>
+
+	* WgetTest.py (CommonMethods.exec_wget): Expect domain_list instead of
+	domain.
+	(CommonMethods.get_cmd_line): Same. Generate command line by
+	prepending to each file it's respective domain string
+	(CommonMethods.ServerFiles): Generate file_list and server_rules for
+	each Server and set the config details
+	(HTTPTest): New named parameter, servers which signifies number of
+	servers to spawn
+	(HTTPTest.HTTP_setup): This method now takes servers as a new
+	parameter. Instead of storing server and domain, we now store
+	server_list and domain_list. Each server must be initialized through a
+	loop.
+	(HTTPTest.stop_HTTP_server): Stop all servers in a loop.
+	* Test-Parallel-Proto.py: Prototype test file for multiple servers.
+
+2013-09-10  Darshit Shah  <darnir@gmail.com>
+
+	* WgetTest.py (HTTPTest.stop_HTTP_server): With the threaded servers,
+	we can simply use the socketserver.shutdown() method to close the
+	server instead of sending a QUIT command
+	* HTTPServer.py (StoppabelHTTPServer.serve_forever): Delete method. No
+	need to override this method anymore.
+	(WgetHTTPRequestHandler.do_QUIT): No longer required
+	(HTTPd): Rename self.server to self.server_inst to reduce ambiguity
+	when referenced from WgetTest
+
+2013-09-08  Darshit Shah  <darnir@gmail.com>
+
+	* README (File Structure): Add explanation about various variables
+	used consistently across all tests.
+
+2013-09-07  Darshit Shah  <darnir@gmail.com>
+
+	* HTTPServer.py: Remove bunch of old code artefacts
+	* WgetTest.py: Same
+
+2013-09-07  Darshit Shah  <darnir@gmail.com>
+
+	* HTTPServer.py (StoppableHTTPServer.server_conf): Change global
+	variable fileSys to an object variable. This is good programming
+	practice and required for parallel-wget support.
+	(StoppableHTTPServer.server_forever): Edit overridden method to remove
+	the global queue variable. No longer required under the new working
+	(WgetHTTPRequestHandler.do_QUIT): Don't push fileSys through the queue
+	(_Handler): Rename class __Handler to _Handler to match Python's
+	encapsulation rules
+	(_Handler.do_POST): fileSys is now an object variable of the server
+	(_Handler.do_PUT): Same
+	(_Handler.send_put): Same
+	(_Handler.send_head): Same
+	(HTTPd): New class that wraps around the server for Threading
+	(create_server): Make new object of HTTPd.
+	(spawn_server): Start the thread created through create_server
+	(ret_fileSys): Removed method. No longer required.
+	* WgetTest.py (HTTPTest.__init__): Don't explicitly set
+	self.act_retcode. Instead toggle tests_passed boolean to set the
+	correct return code.
+	(HTTPTest.HTTP_setup): We no longer call HTTPServer.spawn_server to
+	start a new instance of the server.
+	(HTTPTest.init_HTTP_server): We no longer call the old
+	create_server(), spawn_server() methods. Instead use the new HTTPd
+	class interface to create new instances of the server
+	(HTTPTest.stop_HTTP_server): Don't ask server to return fileSys.
+
+2013-09-07  Darshit Shah  <darnir@gmail.com>
+
+	* Test-Post.py: Test basic functionality for sending HTTP POST
+	requests using the --method command
+	* Makefile.am: Add new test
+
+2013-09-06  Darshit Shah  <darnir@gmail.com>
+
+	* WgetTest.py (CommonMethods.__check_downloaded_files): Print a
+	unified diff in case there is a mismatch in the file contents
+
+2013-09-06  Darshit Shah  <darnir@gmail.com>
+
+	* HTTPServer.py (WgetHTTPRequestHandler.test_cookies): Comment out the
+	old test_cookies code. This is no longer used and was causing problems
+	with expected cookies. The code will soon be removed anyways
+	* Test-cookie.py: Add new test for basic cookie functionality
+	* Test-cookie-401.py: Ensure cookies are saved during a 401 response
+	* Test-cookie-expires.py: Ensure that the Expires field is correctly
+	handled
+	* Test-cookies-domain-mismatch.py: Ensure that mismatched domains are
+	handled by Wget
+	* Makefile.am: Add the new tests
+
+2013-09-06  Darshit Shah  <darnir@gmail.com>
+
+	* README: New section on pending work. Will keep updating this to keep
+	track of work that remains to be done on this implementation
+
+2013-09-05  Darshit Shah  <darnir@gmail.com>
+
+	* Test-auth-with-content-disposition.py: Add test that ensures Content
+	Disposition works alongwith authentication
+	* Makefile.am: Add new test
+
+2013-09-04  Darshit Shah  <darnir@gmail.com>
+
+	* Test-c-full.py: Test Continue options
+	* Makefile.am: Add Test-c-full.py and Test-O
+
+2013-09-02  Darshit Shah  <darnir@gmail.com>
+
+	* Makefile.am:	Add new Test
+	* Test-Head.py: New Test to ensure HEAD requests are handled correctly
+
+2013-08-31  Darshit Shah  <darnir@gmail.com>
+
+	* README: Explain that TEST_NAME needs to be unique
+	* Test-auth-no-challenge.py: Edit non-unique TEST_NAME
+
+2013-08-31  Darshit Shah  <darnir@gmail.com>
+
+	* HTTPTest.py (ServerError): Define new Exception for handling
+	internal control flow.
+	(StoppableHTTPServer.SendHeader): Simply pass. Do nothing. Adding
+	functionality here seems to crash for no apparent reason.
+	(stoppableHTTPServer.send_cust_headers): Minor optimization. No need
+	for extra variable.
+	(__Handler.Response): Handle explicit Response Code Rules
+	(__Handler.Authentication): Handle Authentication rules
+	(__Handler.handle_auth): Actual worker method for authentication
+	(__Handler.ExpectHeader): Ensure Expected Headers are received
+	(__Handler.RejectHeader): Ensure Blacklisted Headers are NOT received
+	(__Handler.send_HEAD): Dynamically call server rule functions based on
+	the self.rules list. This feature will later be added to POST/PUT, etc
+
+2013-08-31  Darshit Shah  <darnir@gmail.com>
+
+	* WgetTest.py: Remove import module defaultdict.
+	(CommonMethods.get_server_rules): server_rules should be a dict, not a
+	defaultdict (list).
+	* HTTPServer.py (WgetHTTPRequestHandler.get_rule_list): If rule does
+	not exist, return None.	Not an emppty list.
+	(WgetHTTPRequestHandler.test_cookies): Rule variable is not a list
+	(__Handler.send_cust_headers): Same
+	(__Handler.custom_response): Same
+	(__Handler.is_authorized): Same
+	(__Handler.expect_headers): Same
+	(__Handler.reject_headers): Same
+
+2013-08-31  Darshit Shah  <darnir@gmail.com>
+
+	* README: (newfile) Simple help / instructions about using the Test
+	Environment.
+	* Makefile.am: (newfile) Makefile for the Test Environment. Uses the
+	Automake Parallel Test Harness
+	* WgetTest.py: (newfile) Base module that executes the Test.
+	* HTTPServer.py: (newfile) Contains the custom HTTP Server for the
+	Test Environment. Creates an instance of http.server in Python3.
+	* FTPServer.py: (newfile) Overrides methods from pyftpdlib for use in
+	the Test Environment. ** Work under progress **.
+	* ColourTerm.py: (newfile) A custom module to output coloured text to
+	the terminal. Known to work on POSIX shells.
+	* Test-Proto.py: (newfile) A prototype Test File. This should be
+	copied when writing a new Test Case.
+	* Test-Content-disposition-2.py: Test Content Disposition clobbering
+	* Test-Content-disposition.py: Test Content Disposition Headers
+	* Test-O.py: Test Output filename command
+	* Test-auth-basic-fail.py: Test returncode on auth failure
+	* Test-auth-basic.py: Test Basic Auth negotiation
+	* Test-auth-both.py: Test handling of Multiple auth providers. This
+	test currently fails.
+	* Test-auth-digest.py: Test Digest Auth Negotiation
+	* Test-auth-no-challenge-url.py: Ensure --auth-no-challenge is handled
+	when auth details are in-URL.
+	* Test-auth-no-challenge.py: Ensure --auth-no-challenge is honoured
+	* Test-auth-retcode.py: Ensure correct return code after 403 Forbidden
+	response.

testenv/Makefile.am

@@ -0,0 +1,58 @@
+# Makefile for `wget' utility
+# Copyright (C) 2013 Free Software Foundation, Inc.
+
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+
+# You should have received a copy of the GNU General Public License
+# along with Wget.  If not, see <http://www.gnu.org/licenses/>.
+
+# Additional permission under GNU GPL version 3 section 7
+
+# If you modify this program, or any covered work, by linking or
+# combining it with the OpenSSL project's OpenSSL library (or a
+# modified version of that library), containing parts covered by the
+# terms of the OpenSSL or SSLeay licenses, the Free Software Foundation
+# grants you additional permission to convey the resulting work.
+# Corresponding Source for a non-source form of such a combination
+# shall include the source code for the parts of OpenSSL used as well
+# as that of the covered work.
+
+
+AUTOMAKE_OPTIONS = parallel-tests
+AM_TESTS_ENVIRONMENT = export WGETRC=/dev/null; MAKE_CHECK=True; export MAKE_CHECK; export PYTHONPATH=$$PYTHONPATH:$(srcdir);
+TESTS = Test-auth-basic-fail.py             \
+    Test-auth-basic.py                      \
+    Test-auth-both.py                       \
+    Test-auth-digest.py                     \
+    Test-auth-no-challenge.py               \
+    Test-auth-no-challenge-url.py           \
+    Test-auth-retcode.py                    \
+    Test-auth-with-content-disposition.py   \
+    Test-c-full.py                          \
+    Test-Content-disposition-2.py           \
+    Test-Content-disposition.py             \
+    Test-cookie-401.py                      \
+    Test-cookie-domain-mismatch.py          \
+    Test-cookie-expires.py                  \
+    Test-cookie.py                          \
+    Test-Head.py                            \
+    Test--https.py                          \
+    Test-O.py                               \
+    Test-Post.py                            \
+    Test--spider-r.py
+
+XFAIL_TESTS = Test-auth-both.py
+
+EXTRA_DIST = certs conf exc misc server test README $(TESTS) $(XFAIL_TESTS)
+
+TEST_EXTENSIONS = .py
+PY_LOG_COMPILER = python3
+AM_PY_LOG_FLAGS = -O

testenv/README

@@ -0,0 +1,303 @@
+This document describes the working of the GNU Wget Test Suite.
+
+Install Instructions:
+================================================================================
+
+This Test Suite exploits the Parallel Test Harness available in GNU Autotools.
+Since it uses features from a relatively recent verion of Autotools, the minimum
+required version as been bumped up to 1.11.
+Run the './configure' command to generate the Makefile and then run 'make check'
+to execute the Test Suite. Use the '-j n' option with 'make check' to execute
+n tests simultaneously.
+
+Structure:
+================================================================================
+
+    * server: This package contains custom programmatically configurable servers
+    (both HTTP and FTP) for testing Wget. The HTTP server runs an instance of
+    Python's http.server module. The FTP server is to be implemented.
+
+    * test: This package contains the test case classes for HTTP and FTP. The
+    test case classes includes methods for initializing and cleaning up of the
+    test environment.
+
+    * Test-Proto.py: This is a prototype Test Case file. The file defines all
+    the acceptable elements and their uses. Typically, one must copy this file
+    and edit it for writing Test Cases.
+
+    * exc: This package contains custom exception classes used in this test
+    suite.
+
+    * conf: This package contains the configuration classes for servers to be
+    configured with.
+
+    * misc: This package contains several helper modules used in this test
+    suite.
+        - colour_terminal.py: A custom module for printing coloured output to
+        the terminal. Currently it only supports 4 colours in a *nix
+        environment.
+        - wget_file.py: Module which contains WgetFile, which is a file data
+        container object.
+
+Working:
+================================================================================
+
+The Test Files are valid Python scripts and the default mask for them is 755.
+A singular Test must be invoked in the following manner:
+$ ./python3 <Name of Test File>      OR
+$ ./<Name of Test File>
+The script will then initialize the various elements and pass them to an object
+of the respective Test Class. A directory with the name <Test name>-test will be
+created and the PWD will be changed to this directory. The server is then
+spawned with the required configuration elements. A blocking call to Wget is
+made with the command line arguments specified in the Test Case along with the
+list of URLs that it must download. The server is killed once Wget returns and
+the following checks are used to determine the pass/fail status of the test:
+    * Return Code: The Exit code of Wget is matched against the expected Exit
+    Code as mentioned in the Test Case File.
+    * Downloaded Files: Check whether the expected downloaded files exist on
+    disk.
+    * File Content: Test whether the file contents were correctly downloaded by
+    Wget and not corrupted mid-way.
+    * Excess Files: Check to see whether any unexpected files were downloaded
+    by Wget.
+
+Exit Codes:
+===============================================================================
+
+Following is a list of Exit Status Codes for the tests:
+*   0  Test Successful
+*  66  Errors/Warnings Reported by Thread Sanitizer (If built with -fsanitize)
+*  77  Test Skipped
+*  99  Hard Error
+* 100  Test Failed
+
+Tests are skipped when they are either not supported by the platform, or Wget
+is not compiled with support for that feature. This feature has not yet been
+implemented.
+
+Hard Errors occur when there are problems with the Environment code. Hard
+Error reporting is currently not enabled and all errors are reported as
+failures.
+
+All exceptions should ideally be handled gracefully. If you see any unhandled
+exceptions, please file a bug report at <bug-wget@gnu.org>
+
+Environment Variables:
+================================================================================
+
+* SERVER_WAIT: Set this environment variable with a value for the number of
+  seconds the test should sleep between invoking the server and calling the Wget
+  executable. This is used when one would like to test a different version of
+  the executable or for running the test through external utilities like gdb and
+  valgrind.
+* NO_CLEANUP: Do not remove the temporary files created by the test.
+  This will prevent the ${testname}-test directory from being deleted
+* VALGRIND_TESTS: If this variable is set, the test suite will execute all the
+  tests through valgrind's memcheck tool.
+
+
+File Structure:
+================================================================================
+
+The test case files are Python scripts. It is believed that Python is a simple
+yet elegant language and should be easy for everyone to comprehend. This test
+suite is written with the objective of making it easy to write new tests. The
+structure has been kept as intuitive as possible and should not require much
+effort to get accustomed to.
+
+All Test Files MUST begin with the following Three Lines:
+#!/usr/bin/python3
+from sys import exit
+from WgetTest import {HTTPTest|FTPTest}
+from misc.wget_file import WgetFile
+
+It is recommended that a small description of the Test Case is provided next.
+This would be very helpful to future contributors.
+Next, is the const variable, TEST_NAME that defines the name of the Test.
+
+Each File in the Test must be represented as a WgetFile object. The WgetFile
+Class has the following prototype:
+WgetFile (str name, str contents, str timestamp, dict rules)
+None except name is a mandatory paramter, one may pass only those parameters
+that are required by the File object.
+
+The timestamp string should be a valid Unix Timestamp as defined in RFC xxxx.
+The rules object is a dictionary element, with the key as the Rule Name and
+value as the Rule Data. In most cases, the Rule Data is another dictionary.
+
+Various variables used consistently across all tests are:
+    * WGET_OPTIONS: The command line string passed to Wget upon invokation. This
+    string may contain URLs, like in the case where in-URL authentication is
+    used. Variable names passed like {{var_name}} will be replaced by the
+    contents of the variable self.var_name before being passed to Wget
+    * WGET_URLS: This is a list of filenames which will be appended as the URLs
+    to Wget during invokation. This is a list of lists, where WGET_URLS[0]
+    represents the list of Filenames called from Server[0], WGET_URLS[1] is a
+    list of files downloaded from Server[2], etc.
+    * Files: This variable defines the files that exist in the Server's
+    filesystem. The Files variable is a list of lists of WgetFile objects.
+    This means that File[0] is a list of WgetFile objects that lie on Server[0],
+    File[1] a list of files on Server[1] and so on.
+    * Existing_Files: This is a list of files that already exist in the
+    directory from which Wget is invoked.
+    * ExpectedReturnCode: The Exit Code expected to be returned by Wget after
+    the test.
+    * ExpectedDownloadedFiles: A list of files that are expected in the local
+    directory after Wget has finished executing. This does not include the files
+    already existing before Wget was launched and must be mentioned again.
+    * Request_List: An unordered list of Requests that each server must receive.
+    This too is a list of lists and follows the same convention as others above.
+
+Both, the HTTPTest and FTPTest modules have the same prototype:
+{
+    name,
+    pre_hook,
+    test_options,
+    post_hook,
+    protocols
+}
+name should be a string, and is usually passed to the TEST_NAME variable,
+the three hooks should be Python dict objects and protocols should be a list of
+protocols, like [HTTP, HTTPS].
+
+Valid File Rules:
+================================================================================
+
+This section lists the currently supported File Rules and their structure.
+
+    * Authentication: Used when a File must require Authorization for access.
+    The value for this key is the following dictionary:
+    |-->Type        :   Basic|Digest|Both|Both_inline
+    |-->User        :   <Username>
+    --->Pass        :   <Password>
+
+    * ExpectHeader  : The following Headers MUST exist in every Request for the
+    File. The value for this key is a dictionary object where each header is
+    represented as:
+    |-->Header Name :   <Header Data>
+
+    * RejectHeader  : This list of Headers must NEVER occur in a request. It
+    uses the same value format as ExpectHeader.
+
+    * SendHeader    : This list of Headers will be sent in EVERY response to a
+    request for the respective file. It follows the same value format as
+    ExpectHeader.
+
+    * Response      : The HTTP Response Code to send to a request for this File.
+    The value is an Integer that represents a valid HTTP Response Code.
+
+Pre Test Hooks:
+================================================================================
+
+The Pre-Test Hooks are executed just after starting the server and just before
+spawning an instance of the server. These are usually used for setting up the
+Test Environment and Server Rules. The currently supported Pre-Test Hooks are:
+
+    * ServerFiles   : A list of WgetFile objects that must exist on the Server
+    * LocalFiles    : A list of WgetFile objects that exist locally on disk
+    before Wget is executed.
+
+Since pre_test is a dictionary, one may not assume that the hooks will be
+executed in the same order as they are defined.
+
+Test Options:
+================================================================================
+
+The test_options dictionary defines the commands to be used when the Test is
+executed. The currently supported options are:
+
+    * Urls          : A list of the filenames that Wget must attempt to
+    download. The complete URL will be created and passed to Wget
+    automatically. (alias URLs)
+    * WgetCommands  : A string consisting of the various commandline switches
+    sent to Wget upon invokation. Any data placed between {{ }} in this string
+    will be replaced with the contents of self.<data> before being passed to
+    Wget. This is particularly useful for getting the hostname and port for a
+    file. While all Download URL's are passed to Urls, a notable exception is
+    when in-url authentication is used. In such a case, the URL is specified in
+    the WgetCommands string.
+
+Post-Test Hooks:
+================================================================================
+
+These hooks are executed as soon as the call to Wget returns. The post-test
+hooks are usually used to run checks on the data, files downloaded, return code,
+etc. The following hooks are currently supported:
+
+    * ExpectedRetcode : This is an integer value of the ReturnCode with which
+    Wget is expected to exit. (alias ExpectedRetCode)
+    * ExpectedFiles   : This is a list of WgetFile objects of the files that
+    must exist locally on disk in the Test directory.
+    * FilesCrawled    : This requires a list of the Requests that the server is
+    expected to receive. The order is un-important since it will vary on the
+    parallel-wget branch. This hook is used in tests for Recursive mode to
+    ensure that the website is traversed correctly.
+
+Writing New Tests:
+================================================================================
+
+See Test-Proto.py for an example of how to write Test Case files. The
+recommended method for writing new Test Case files is to copy Test-Proto.py and
+modify it to ones needs.
+
+In case you require any functionality that is not currently defined in List of
+Rules defined above, you should implement a new class in the conf package. The
+file name doesn't matter (though it's better to give it an appropriate name).
+The new rule or hook class should be like this:
+============================================
+from conf import rule
+
+
+@rule()
+class MyNewRule:
+    def __init__(self, rule_arg):
+        self.rule_arg = rule_arg
+        # your rule initialization code goes here
+============================================
+from conf import hook
+
+
+@hook()
+class MyNewHook:
+    def __init__(self, hook_arg):
+        self.hook_arg = hook_arg
+        # your hook initialization code goes here
+
+    def __call__(self, test_obj):
+        # your hook code goes here
+============================================
+
+Once a new Test File is created, it must be added to the TESTS variable in
+Makefile.am. This way the Test will be executed on running a 'make check'.
+If a Test is expected to fail on the current master branch, then the Test should
+also be added to the XFAIL_TESTS variable. This will allow expected failures to
+pass through. If a test mentioned in the XFAIL_TESTS variable passes, it gets
+red-flagged as a XPASS. Currently, tests expected to fail under valgrind are not
+explicitly marked as XFAIL. Tests failing under valgrind must always be
+considered a blocking error.
+
+Remember to always name the Test correctly using the TEST_NAME variable. This
+is essential since a directory with the Test Name is created and this can
+cause synchronization problems when the Parallel Test Harness is used.
+One can use the following command on Unix systems to check for TEST_NAME
+clashes:
+$ grep -r -h "TEST_NAME =" | cut -c13- | uniq -c -d
+
+Work Remaining:
+================================================================================
+
+Some amount of work still remains to be done.
+    * Errors in server-side checks need to be handled more explicitly
+    * Support parallel-wget branch
+        * Support to spawn multiple servers is already in place. Need to handle
+        multiple requests to a server simultaneously. Use THreading MixIn.
+    * SSL Tests. Use xyne's HTTPS server implemention
+    * Complete support for FTP Tests
+    * IRI Support. This shouldn't require much effort
+
+Requirements:
+================================================================================
+
+1. Python   >= 3.0
+2. Automake >= 1.11

testenv/Test--https.py

@@ -0,0 +1,53 @@
+#!/usr/bin/env python3
+from sys import exit
+from test.http_test import HTTPTest
+from test.base_test import HTTP, HTTPS
+from misc.wget_file import WgetFile
+
+"""
+    This test ensures that Wget can download files from HTTPS Servers
+"""
+TEST_NAME = "HTTPS Downloads"
+############# File Definitions ###############################################
+File1 = "Would you like some Tea?"
+File2 = "With lemon or cream?"
+File3 = "Sure you're joking Mr. Feynman"
+
+A_File = WgetFile ("File1", File1)
+B_File = WgetFile ("File2", File2)
+C_File = WgetFile ("File3", File3)
+
+WGET_OPTIONS = "--no-check-certificate"
+WGET_URLS = [["File1", "File2"]]
+
+Files = [[A_File, B_File]]
+Existing_Files = [C_File]
+
+Servers = [HTTPS]
+
+ExpectedReturnCode = 0
+ExpectedDownloadedFiles = [A_File, B_File, C_File]
+
+################ Pre and Post Test Hooks #####################################
+pre_test = {
+    "ServerFiles"       : Files,
+    "LocalFiles"        : Existing_Files
+}
+test_options = {
+    "WgetCommands"      : WGET_OPTIONS,
+    "Urls"              : WGET_URLS
+}
+post_test = {
+    "ExpectedFiles"     : ExpectedDownloadedFiles,
+    "ExpectedRetcode"   : ExpectedReturnCode
+}
+
+err = HTTPTest (
+                name=TEST_NAME,
+                pre_hook=pre_test,
+                test_params=test_options,
+                post_hook=post_test,
+                protocols=Servers
+).begin ()
+
+exit (err)

testenv/Test--spider-r.py

@@ -0,0 +1,106 @@
+#!/usr/bin/env python3
+from sys import exit
+from test.http_test import HTTPTest
+from misc.wget_file import WgetFile
+
+"""
+    This test executed Wget in Spider mode with recursive retrieval.
+"""
+TEST_NAME = "Recursive Spider"
+############# File Definitions ###############################################
+mainpage = """
+<html>
+<head>
+  <title>Main Page</title>
+</head>
+<body>
+  <p>
+    Some text and a link to a <a href="http://127.0.0.1:{{port}}/secondpage.html">second page</a>.
+    Also, a <a href="http://127.0.0.1:{{port}}/nonexistent">broken link</a>.
+  </p>
+</body>
+</html>
+"""
+
+
+secondpage = """
+<html>
+<head>
+  <title>Second Page</title>
+</head>
+<body>
+  <p>
+    Some text and a link to a <a href="http://127.0.0.1:{{port}}/thirdpage.html">third page</a>.
+    Also, a <a href="http://127.0.0.1:{{port}}/nonexistent">broken link</a>.
+  </p>
+</body>
+</html>
+"""
+
+thirdpage = """
+<html>
+<head>
+  <title>Third Page</title>
+</head>
+<body>
+  <p>
+    Some text and a link to a <a href="http://127.0.0.1:{{port}}/dummy.txt">text file</a>.
+    Also, another <a href="http://127.0.0.1:{{port}}/againnonexistent">broken link</a>.
+  </p>
+</body>
+</html>
+"""
+
+dummyfile = "Don't care."
+
+
+index_html = WgetFile ("index.html", mainpage)
+secondpage_html = WgetFile ("secondpage.html", secondpage)
+thirdpage_html = WgetFile ("thirdpage.html", thirdpage)
+dummy_txt = WgetFile ("dummy.txt", dummyfile)
+
+Request_List = [
+    [
+        "HEAD /",
+        "GET /",
+        "GET /robots.txt",
+        "HEAD /secondpage.html",
+        "GET /secondpage.html",
+        "HEAD /nonexistent",
+        "HEAD /thirdpage.html",
+        "GET /thirdpage.html",
+        "HEAD /dummy.txt",
+        "HEAD /againnonexistent"
+    ]
+]
+
+WGET_OPTIONS = "--spider -r"
+WGET_URLS = [[""]]
+
+Files = [[index_html, secondpage_html, thirdpage_html, dummy_txt]]
+
+ExpectedReturnCode = 8
+ExpectedDownloadedFiles = []
+
+################ Pre and Post Test Hooks #####################################
+pre_test = {
+    "ServerFiles"       : Files
+}
+test_options = {
+    "WgetCommands"      : WGET_OPTIONS,
+    "Urls"              : WGET_URLS
+}
+post_test = {
+    "ExpectedFiles"     : ExpectedDownloadedFiles,
+    "ExpectedRetcode"   : ExpectedReturnCode,
+    "FilesCrawled"      : Request_List
+}
+
+err = HTTPTest (
+                name=TEST_NAME,
+                pre_hook=pre_test,
+                test_params=test_options,
+                post_hook=post_test
+).begin ()
+
+exit (err)

testenv/Test-Content-disposition-2.py

@@ -0,0 +1,53 @@
+#!/usr/bin/env python3
+from sys import exit
+from test.http_test import HTTPTest
+from misc.wget_file import WgetFile
+
+"""
+    This test ensures that Wget parses the Content-Disposition header
+    correctly and creates the appropriate file when the said filename exists.
+"""
+TEST_NAME = "Content Disposition Clobber"
+############# File Definitions ###############################################
+File1 = "Teapot"
+File2 = "The Teapot Protocol"
+
+File2_rules = {
+    "SendHeader"        : {
+        "Content-Disposition" : "Attachment; filename=HTTP.Teapot"
+    }
+}
+A_File = WgetFile ("HTTP.Teapot", File1)
+B_File = WgetFile ("File2", File2, rules=File2_rules)
+
+WGET_OPTIONS = "--content-disposition"
+WGET_URLS = [["File2"]]
+
+Files = [[B_File]]
+Existing_Files = [A_File]
+
+ExpectedReturnCode = 0
+ExpectedDownloadedFiles = [WgetFile ("HTTP.Teapot.1", File2), A_File]
+
+################ Pre and Post Test Hooks #####################################
+pre_test = {
+    "ServerFiles"       : Files,
+    "LocalFiles"        : Existing_Files
+}
+test_options = {
+    "WgetCommands"      : WGET_OPTIONS,
+    "Urls"              : WGET_URLS
+}
+post_test = {
+    "ExpectedFiles"     : ExpectedDownloadedFiles,
+    "ExpectedRetcode"   : ExpectedReturnCode
+}
+
+err = HTTPTest (
+                name=TEST_NAME,
+                pre_hook=pre_test,
+                test_params=test_options,
+                post_hook=post_test
+).begin ()
+
+exit (err)

testenv/Test-Content-disposition.py

@@ -0,0 +1,56 @@
+#!/usr/bin/env python3
+from sys import exit
+from test.http_test import HTTPTest
+from misc.wget_file import WgetFile
+
+"""
+    This test ensures that Wget parses the Content-Disposition header
+    correctly and creates a local file accordingly.
+"""
+TEST_NAME = "Content Disposition Header"
+############# File Definitions ###############################################
+File1 = """All that is gold does not glitter,
+        Not all those who wander are lost;
+        The old that is strong does not wither,
+        Deep roots are not reached by the frost.
+        From the ashes a fire shall be woken,
+        A light from the shadows shall spring;
+        Renewed shall be blade that was broken,
+        The crownless again shall be king."""
+
+File1_rules = {
+    "SendHeader"        : {
+        "Content-Disposition" : "Attachment; filename=JRR.Tolkein"
+    }
+}
+A_File = WgetFile ("LOTR", File1, rules=File1_rules)
+
+WGET_OPTIONS = "--content-disposition"
+WGET_URLS = [["LOTR"]]
+
+Files = [[A_File]]
+
+ExpectedReturnCode = 0
+ExpectedDownloadedFiles = [WgetFile ("JRR.Tolkein", File1)]
+
+################ Pre and Post Test Hooks #####################################
+pre_test = {
+    "ServerFiles"       : Files
+}
+test_options = {
+    "WgetCommands"      : WGET_OPTIONS,
+    "Urls"              : WGET_URLS
+}
+post_test = {
+    "ExpectedFiles"     : ExpectedDownloadedFiles,
+    "ExpectedRetcode"   : ExpectedReturnCode
+}
+
+err = HTTPTest (
+                name=TEST_NAME,
+                pre_hook=pre_test,
+                test_params=test_options,
+                post_hook=post_test
+).begin ()
+
+exit (err)

testenv/Test-Head.py

@@ -0,0 +1,44 @@
+#!/usr/bin/env python3
+from sys import exit
+from test.http_test import HTTPTest
+from misc.wget_file import WgetFile
+
+"""
+    This test ensures that Wget correctly handles responses to HEAD requests
+    and does not actually download any data
+"""
+TEST_NAME = "HEAD Requests"
+############# File Definitions ###############################################
+File1 = "You shall not pass!"
+
+A_File = WgetFile ("File1", File1)
+
+WGET_OPTIONS = "--method=HEAD"
+WGET_URLS = [["File1"]]
+
+Files = [[A_File]]
+
+ExpectedReturnCode = 0
+ExpectedDownloadedFiles = []
+
+################ Pre and Post Test Hooks #####################################
+pre_test = {
+    "ServerFiles"       : Files,
+}
+test_options = {
+    "WgetCommands"      : WGET_OPTIONS,
+    "Urls"              : WGET_URLS
+}
+post_test = {
+    "ExpectedFiles"     : ExpectedDownloadedFiles,
+    "ExpectedRetcode"   : ExpectedReturnCode
+}
+
+err = HTTPTest (
+                name=TEST_NAME,
+                pre_hook=pre_test,
+                test_params=test_options,
+                post_hook=post_test
+).begin ()
+
+exit (err)

testenv/Test-O.py

@@ -0,0 +1,45 @@
+#!/usr/bin/env python3
+from sys import exit
+from test.http_test import HTTPTest
+from misc.wget_file import WgetFile
+
+"""
+    This test ensures that Wget correctly handles the -O command for output
+    filenames.
+"""
+TEST_NAME = "Output Filename Command"
+############# File Definitions ###############################################
+File1 = "Test Contents."
+
+A_File = WgetFile ("File1", File1)
+
+WGET_OPTIONS = "-O NewFile.txt"
+WGET_URLS = [["File1"]]
+
+Files = [[A_File]]
+ExistingFiles = [A_File]
+
+ExpectedReturnCode = 0
+ExpectedDownloadedFiles = [WgetFile ("NewFile.txt", File1)]
+
+################ Pre and Post Test Hooks #####################################
+pre_test = {
+    "ServerFiles"       : Files
+}
+test_options = {
+    "WgetCommands"      : WGET_OPTIONS,
+    "Urls"              : WGET_URLS
+}
+post_test = {
+    "ExpectedFiles"     : ExpectedDownloadedFiles,
+    "ExpectedRetcode"   : ExpectedReturnCode
+}
+
+err = HTTPTest (
+                name=TEST_NAME,
+                pre_hook=pre_test,
+                test_params=test_options,
+                post_hook=post_test
+).begin ()
+
+exit (err)

testenv/Test-Parallel-Proto.py

@@ -0,0 +1,54 @@
+#!/usr/bin/env python3
+from sys import exit
+from test.http_test import HTTPTest
+from misc.constants import HTTP, HTTPS
+from misc.wget_file import WgetFile
+
+"""
+    This is a Prototype Test File for multiple servers.
+    Ideally this File should be copied and edited to write new tests.
+"""
+TEST_NAME = "Parallel Prototype"
+############# File Definitions ###############################################
+File1 = "Would you like some Tea?"
+File2 = "With lemon or cream?"
+File3 = "Sure you're joking Mr. Feynman"
+
+A_File = WgetFile ("File1", File1)
+B_File = WgetFile ("File2", File2)
+C_File = WgetFile ("File3", File3)
+
+WGET_OPTIONS = ""
+WGET_URLS = [["File1"], ["File2"]]
+
+Files = [[A_File], [B_File]]
+Existing_Files = [C_File]
+
+Servers = [HTTP, HTTP]
+
+ExpectedReturnCode = 0
+ExpectedDownloadedFiles = [A_File, B_File, C_File]
+
+################ Pre and Post Test Hooks #####################################
+pre_test = {
+    "ServerFiles"       : Files,
+    "LocalFiles"        : Existing_Files
+}
+test_options = {
+    "WgetCommands"      : WGET_OPTIONS,
+    "Urls"              : WGET_URLS
+}
+post_test = {
+    "ExpectedFiles"     : ExpectedDownloadedFiles,
+    "ExpectedRetcode"   : ExpectedReturnCode
+}
+
+err = HTTPTest (
+                name=TEST_NAME,
+                pre_hook=pre_test,
+                test_params=test_options,
+                post_hook=post_test,
+                protocols=Servers
+).begin ()
+
+exit (err)

testenv/Test-Post.py

@@ -0,0 +1,48 @@
+#!/usr/bin/env python3
+from sys import exit
+from test.http_test import HTTPTest
+from misc.wget_file import WgetFile
+
+"""
+    Simple test for HTTP POST Requests usiong the --method command
+"""
+TEST_NAME = "HTTP POST Requests"
+############# File Definitions ###############################################
+File1 = """A reader lives a thousand lives before he dies, said Jojen.
+The man who never reads lives only one"""
+
+File1_response = """A reader lives a thousand lives before he dies, said Jojen.
+The man who never reads lives only one
+TestMessage"""
+
+A_File = WgetFile ("File1", File1)
+
+WGET_OPTIONS = "--method=post --body-data=TestMessage"
+WGET_URLS = [["File1"]]
+
+Files = [[A_File]]
+
+ExpectedReturnCode = 0
+ExpectedDownloadedFiles = [WgetFile ("File1", File1_response)]
+
+################ Pre and Post Test Hooks #####################################
+pre_test = {
+    "ServerFiles"       : Files
+}
+test_options = {
+    "WgetCommands"      : WGET_OPTIONS,
+    "Urls"              : WGET_URLS
+}
+post_test = {
+    "ExpectedFiles"     : ExpectedDownloadedFiles,
+    "ExpectedRetcode"   : ExpectedReturnCode
+}
+
+err = HTTPTest (
+                name=TEST_NAME,
+                pre_hook=pre_test,
+                test_params=test_options,
+                post_hook=post_test
+).begin ()
+
+exit (err)

testenv/Test-Proto.py

@@ -0,0 +1,75 @@
+#!/usr/bin/env python3
+from sys import exit
+from test.http_test import HTTPTest
+from misc.constants import HTTP, HTTPS
+from misc.wget_file import WgetFile
+
+"""
+    This is a Prototype Test File.
+    Ideally this File should be copied and edited to write new tests.
+"""
+TEST_NAME = "Prototype"
+############# File Definitions ###############################################
+File1 = "Would you like some Tea?"
+File2 = "With lemon or cream?"
+File3 = "Sure you're joking Mr. Feynman"
+
+File1_rules = {
+    "Authentication"    : {
+        "Type"          : "Both",
+        "User"          : "Sauron",
+        "Pass"          : "TheEye"
+    },
+    "RejectHeader"      : {
+        "Authorization" : "Basic U2F1cm9uOlRoZUV5ZQ=="
+    }
+}
+File2_rules = {
+    "Authentication"    : {
+        "Type"          : "Both_inline",
+        "User"          : "Sauron",
+        "Pass"          : "TheEye"
+    },
+    "SendHeader"        : {
+        "Content-Disposition"  : "Attachment; filename=newfile"
+    }
+}
+
+A_File = WgetFile ("File1", File1, rules=File1_rules)
+B_File = WgetFile ("File2", File2, rules=File2_rules)
+C_File = WgetFile ("File3", File3)
+
+WGET_OPTIONS = "--content-disposition --user=Sauron --password=TheEye"
+WGET_URLS = [["File1", "File2"]]
+
+Servers = [HTTP]
+
+Files = [[A_File, B_File]]
+Existing_Files = [C_File]
+
+ExpectedReturnCode = 0
+ExpectedDownloadedFiles = [A_File, WgetFile ("newfile", File2), C_File]
+
+################ Pre and Post Test Hooks #####################################
+pre_test = {
+    "ServerFiles"       : Files,
+    "LocalFiles"        : Existing_Files
+}
+test_options = {
+    "WgetCommands"      : WGET_OPTIONS,
+    "Urls"              : WGET_URLS
+}
+post_test = {
+    "ExpectedFiles"     : ExpectedDownloadedFiles,
+    "ExpectedRetcode"   : ExpectedReturnCode
+}
+
+err = HTTPTest (
+                name=TEST_NAME,
+                pre_hook=pre_test,
+                test_params=test_options,
+                post_hook=post_test,
+                protocols=Servers
+).begin ()
+
+exit (err)

testenv/Test-auth-basic-fail.py

@@ -0,0 +1,51 @@
+#!/usr/bin/env python3
+from sys import exit
+from test.http_test import HTTPTest
+from misc.wget_file import WgetFile
+
+"""
+    This test ensures that Wget returns the correct exit code when Basic
+    authentcation failes due to a username/password error.
+"""
+TEST_NAME = "Basic Authentication Failure"
+############# File Definitions ###############################################
+File1 = "I am an invisble man."
+
+File1_rules = {
+    "Authentication"    : {
+        "Type"          : "Basic",
+        "User"          : "Sauron",
+        "Pass"          : "TheEye"
+    }
+}
+A_File = WgetFile ("File1", File1, rules=File1_rules)
+
+WGET_OPTIONS = "--user=Sauron --password=Eye"
+WGET_URLS = [["File1"]]
+
+Files = [[A_File]]
+
+ExpectedReturnCode = 6
+ExpectedDownloadedFiles = []
+
+################ Pre and Post Test Hooks #####################################
+pre_test = {
+    "ServerFiles"       : Files
+}
+test_options = {
+    "WgetCommands"      : WGET_OPTIONS,
+    "Urls"              : WGET_URLS
+}
+post_test = {
+    "ExpectedFiles"     : ExpectedDownloadedFiles,
+    "ExpectedRetcode"   : ExpectedReturnCode
+}
+
+err = HTTPTest (
+                name=TEST_NAME,
+                pre_hook=pre_test,
+                test_params=test_options,
+                post_hook=post_test
+).begin ()
+
+exit (err)

testenv/Test-auth-basic.py

@@ -0,0 +1,59 @@
+#!/usr/bin/env python3
+from sys import exit
+from test.http_test import HTTPTest
+from misc.wget_file import WgetFile
+
+"""
+    This test ensures Wget's Basic Authorization Negotiation.
+    Also, we ensure that Wget saves the host after a successfull auth and
+    doesn't wait for a challenge the second time.
+"""
+TEST_NAME = "Basic Authorization"
+############# File Definitions ###############################################
+File1 = "I am an invisble man."
+File2 = "I too am an invisible man."
+
+File1_rules = {
+    "Authentication"    : {
+        "Type"          : "Basic",
+        "User"          : "Sauron",
+        "Pass"          : "TheEye"
+    }
+}
+File2_rules = {
+    "ExpectHeader"      : {
+        "Authorization" : "Basic U2F1cm9uOlRoZUV5ZQ=="
+    }
+}
+A_File = WgetFile ("File1", File1, rules=File1_rules)
+B_File = WgetFile ("File2", File2, rules=File2_rules)
+
+WGET_OPTIONS = "--user=Sauron --password=TheEye"
+WGET_URLS = [["File1", "File2"]]
+
+Files = [[A_File, B_File]]
+
+ExpectedReturnCode = 0
+ExpectedDownloadedFiles = [A_File, B_File]
+
+################ Pre and Post Test Hooks #####################################
+pre_test = {
+    "ServerFiles"       : Files
+}
+test_options = {
+    "WgetCommands"      : WGET_OPTIONS,
+    "Urls"              : WGET_URLS
+}
+post_test = {
+    "ExpectedFiles"     : ExpectedDownloadedFiles,
+    "ExpectedRetcode"   : ExpectedReturnCode
+}
+
+err = HTTPTest (
+                name=TEST_NAME,
+                pre_hook=pre_test,
+                test_params=test_options,
+                post_hook=post_test
+).begin ()
+
+exit (err)

testenv/Test-auth-both.py

@@ -0,0 +1,77 @@
+#!/usr/bin/env python3
+from sys import exit
+from test.http_test import HTTPTest
+from misc.wget_file import WgetFile
+
+"""
+    This test ensures Wget's Basic Authorization Negotiation.
+    Also, we ensure that Wget saves the host after a successfull auth and
+    doesn't wait for a challenge the second time.
+"""
+TEST_NAME = "Multiple authentication support"
+############# File Definitions ###############################################
+File1 = "Would you like some Tea?"
+File2 = "With lemon or cream?"
+File3 = "Sure you're joking Mr. Feynman"
+
+File1_rules = {
+    "Authentication"    : {
+        "Type"          : "Both",
+        "User"          : "Sauron",
+        "Pass"          : "TheEye"
+    },
+    "RejectHeader"      : {
+        "Authorization" : "Basic U2F1cm9uOlRoZUV5ZQ=="
+    }
+}
+File2_rules = {
+    "Authentication"    : {
+        "Type"          : "Both_inline",
+        "User"          : "Sauron",
+        "Pass"          : "TheEye"
+    },
+    "RejectHeader"      : {
+        "Authorization" : "Basic U2F1cm9uOlRoZUV5ZQ=="
+    }
+}
+File3_rules = {
+    "Authentication"    : {
+        "Type"          : "Digest",
+        "User"          : "Sauron",
+        "Pass"          : "TheEye"
+    }
+}
+
+A_File = WgetFile ("File1", File1, rules=File1_rules)
+B_File = WgetFile ("File2", File2, rules=File2_rules)
+C_File = WgetFile ("File3", File3, rules=File3_rules)
+
+WGET_OPTIONS = "--user=Sauron --password=TheEye"
+WGET_URLS = [["File1", "File2", "File3"]]
+
+Files = [[A_File, B_File, C_File]]
+
+ExpectedReturnCode = 0
+ExpectedDownloadedFiles = [A_File, B_File, C_File]
+
+################ Pre and Post Test Hooks #####################################
+pre_test = {
+    "ServerFiles"       : Files
+}
+test_options = {
+    "WgetCommands"      : WGET_OPTIONS,
+    "Urls"              : WGET_URLS
+}
+post_test = {
+    "ExpectedFiles"     : ExpectedDownloadedFiles,
+    "ExpectedRetcode"   : ExpectedReturnCode
+}
+
+err = HTTPTest (
+                name=TEST_NAME,
+                pre_hook=pre_test,
+                test_params=test_options,
+                post_hook=post_test
+).begin ()
+
+exit (err)

testenv/Test-auth-digest.py

@@ -0,0 +1,50 @@
+#!/usr/bin/env python3
+from sys import exit
+from test.http_test import HTTPTest
+from misc.wget_file import WgetFile
+
+"""
+    This test ensures Wget's Digest Authorization Negotiation.
+"""
+TEST_NAME = "Digest Authorization"
+############# File Definitions ###############################################
+File1 = "Need a cookie?"
+
+File1_rules = {
+    "Authentication"    : {
+        "Type"          : "Digest",
+        "User"          : "Pacman",
+        "Pass"          : "Omnomnom"
+    }
+}
+A_File = WgetFile ("File1", File1, rules=File1_rules)
+
+WGET_OPTIONS = "--user=Pacman --password=Omnomnom"
+WGET_URLS = [["File1"]]
+
+Files = [[A_File]]
+
+ExpectedReturnCode = 0
+ExpectedDownloadedFiles = [A_File]
+
+################ Pre and Post Test Hooks #####################################
+pre_test = {
+    "ServerFiles"       : Files
+}
+test_options = {
+    "WgetCommands"      : WGET_OPTIONS,
+    "Urls"              : WGET_URLS
+}
+post_test = {
+    "ExpectedFiles"     : ExpectedDownloadedFiles,
+    "ExpectedRetcode"   : ExpectedReturnCode
+}
+
+err = HTTPTest (
+                name=TEST_NAME,
+                pre_hook=pre_test,
+                test_params=test_options,
+                post_hook=post_test
+).begin ()
+
+exit (err)

testenv/Test-auth-no-challenge-url.py

@@ -0,0 +1,54 @@
+#!/usr/bin/env python3
+from sys import exit
+from test.http_test import HTTPTest
+from misc.wget_file import WgetFile
+
+"""
+    This test ensures Wget's Basic Authorization Negotiation, when credentials
+    are provided in-URL
+"""
+TEST_NAME = "Auth no challenge in URL"
+############# File Definitions ###############################################
+File1 = "Need a cookie?"
+
+File1_rules = {
+    "Authentication"    : {
+        "Type"          : "Basic",
+        "User"          : "Pacman",
+        "Pass"          : "Omnomnom"
+    },
+    "ExpectHeader"      : {
+        "Authorization" : "Basic UGFjbWFuOk9tbm9tbm9t"
+    }
+}
+A_File = WgetFile ("File1", File1, rules=File1_rules)
+
+WGET_OPTIONS = "--auth-no-challenge http://Pacman:Omnomnom@localhost:{{port}}/File1"
+WGET_URLS = [[]]
+
+Files = [[A_File]]
+
+ExpectedReturnCode = 0
+ExpectedDownloadedFiles = [A_File]
+
+################ Pre and Post Test Hooks #####################################
+pre_test = {
+    "ServerFiles"       : Files
+}
+test_options = {
+    "WgetCommands"      : WGET_OPTIONS,
+    "Urls"              : WGET_URLS
+}
+post_test = {
+    "ExpectedFiles"     : ExpectedDownloadedFiles,
+    "ExpectedRetcode"   : ExpectedReturnCode
+}
+
+err = HTTPTest (
+                name=TEST_NAME,
+                pre_hook=pre_test,
+                test_params=test_options,
+                post_hook=post_test
+).begin ()
+
+exit (err)

testenv/Test-auth-no-challenge.py

@@ -0,0 +1,54 @@
+#!/usr/bin/env python3
+from sys import exit
+from test.http_test import HTTPTest
+from misc.wget_file import WgetFile
+
+"""
+    This test ensures Wget's Basic Authorization Negotiation, when the
+    --auth-no-challenge command is used.
+"""
+TEST_NAME = "Auth No Challenge Command"
+############# File Definitions ###############################################
+File1 = "Need a cookie?"
+
+File1_rules = {
+    "Authentication"    : {
+        "Type"          : "Basic",
+        "User"          : "Pacman",
+        "Pass"          : "Omnomnom"
+    },
+    "ExpectHeader"      : {
+        "Authorization" : "Basic UGFjbWFuOk9tbm9tbm9t"
+    }
+}
+A_File = WgetFile ("File1", File1, rules=File1_rules)
+
+WGET_OPTIONS = "--auth-no-challenge --user=Pacman --password=Omnomnom"
+WGET_URLS = [["File1"]]
+
+Files = [[A_File]]
+
+ExpectedReturnCode = 0
+ExpectedDownloadedFiles = [A_File]
+
+################ Pre and Post Test Hooks #####################################
+pre_test = {
+    "ServerFiles"       : Files
+}
+test_options = {
+    "WgetCommands"      : WGET_OPTIONS,
+    "Urls"              : WGET_URLS
+}
+post_test = {
+    "ExpectedFiles"     : ExpectedDownloadedFiles,
+    "ExpectedRetcode"   : ExpectedReturnCode
+}
+
+err = HTTPTest (
+                name=TEST_NAME,
+                pre_hook=pre_test,
+                test_params=test_options,
+                post_hook=post_test
+).begin ()
+
+exit (err)